Skip to content

Commit d3a39bb

Browse files
helsonxiaohelsonxiao
authored
fix: ftp permission & tests (#22)
1 parent 585ee92 commit d3a39bb

15 files changed

Lines changed: 220 additions & 185 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office
77
**Main features**:
88

99
* Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones.
10-
* Remote file system over SSH/FTP/SMB(WIP) crawling.
10+
* Remote file system over SSH/FTP crawling.
1111
* REST interface to let you "upload" your binary documents to elasticsearch.
1212

1313
You need to install a version matching your Elasticsearch version:

‎core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ private void addFilesRecursively(FileAbstractor<?> path, String filepath, LocalD
257257
logger.trace("FileAbstractModel = {}", child);
258258
String filename = child.getName();
259259

260-
String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, filename));
260+
String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, filename));
261261

262262
// https://github.com/dadoonet/fscrawler/issues/1 : Filter documents
263263
boolean isIndexable = isIndexable(child.isDirectory(), virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes());
@@ -274,9 +274,9 @@ private void addFilesRecursively(FileAbstractor<?> path, String filepath, LocalD
274274
indexFile(child, stats, filepath,
275275
fsSettings.getFs().isIndexContent() || fsSettings.getFs().isStoreSource() ? path.getInputStream(child) : null, child.getSize());
276276
stats.addFile();
277-
} catch (java.io.FileNotFoundException e) {
277+
} catch (Exception e) {
278278
if (fsSettings.getFs().isContinueOnError()) {
279-
logger.warn("Unable to open Input Stream for {}, skipping...: {}", filename, e.getMessage());
279+
logger.warn("Unable to index {}, skipping...: {}", filename, e.getMessage());
280280
} else {
281281
throw e;
282282
}
@@ -319,7 +319,7 @@ private void addFilesRecursively(FileAbstractor<?> path, String filepath, LocalD
319319
for (String esfile : esFiles) {
320320
logger.trace("Checking file [{}]", esfile);
321321

322-
String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfile));
322+
String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, esfile));
323323
if (isIndexable(false, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())
324324
&& !fsFiles.contains(esfile)) {
325325
logger.trace("Removing file [{}] in elasticsearch/workplace", esfile);
@@ -334,7 +334,7 @@ private void addFilesRecursively(FileAbstractor<?> path, String filepath, LocalD
334334

335335
// for the delete folder
336336
for (String esfolder : esFolders) {
337-
String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfolder));
337+
String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, esfolder));
338338
if (isIndexable(true, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())) {
339339
logger.trace("Checking directory [{}]", esfolder);
340340
if (!fsFolders.contains(esfolder)) {
@@ -377,7 +377,7 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats,
377377
final long size = fileAbstractModel.getSize();
378378

379379
logger.debug("fetching content from [{}],[{}]", dirname, filename);
380-
String fullFilename = FsCrawlerUtil.computeRealPathName(dirname, filename);
380+
String fullFilename = computeRealPathName(dirname, filename);
381381

382382
try {
383383
// Create the Doc object (only needed when we have add_as_inner_object: true (default) or when we don't index json or xml)
@@ -496,11 +496,11 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats,
496496
}
497497
}
498498

499-
private String generateIdFromFilename(String _filename, String _filepath) throws NoSuchAlgorithmException {
500-
String filepathForId = _filepath.replace("\\", "/");
501-
String filename = _filename.replace("\\", "").replace("/", "");
502-
String fullFilename = filepathForId.endsWith("/") ? filepathForId.concat(filename) : filepathForId.concat("/").concat(filename);
503-
return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign(fullFilename);
499+
private String generateIdFromFilename(String filename, String filepath) throws NoSuchAlgorithmException {
500+
String filepathForId = filepath.replace("\\", "/");
501+
String filenameForId = filename.replace("\\", "").replace("/", "");
502+
String idSource = filepathForId.endsWith("/") ? filepathForId.concat(filenameForId) : filepathForId.concat("/").concat(filenameForId);
503+
return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign(idSource);
504504
}
505505

506506
private String read(InputStream input) throws IOException {
@@ -525,7 +525,7 @@ private void indexDirectory(String id, Folder folder) throws IOException {
525525

526526
/**
527527
* Index a directory
528-
* @param path complete path like "/", "/path/to/subdir", "/C:/dir", "//SOMEONE/dir"
528+
* @param path complete path like "/", "/path/to/subdir", "C:\\dir", "C:/dir", "/C:/dir", "//SOMEONE/dir"
529529
*/
530530
private void indexDirectory(String path) throws Exception {
531531
String name = path.substring(path.lastIndexOf(pathSeparator) + 1);

‎crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FTPUtils.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package fr.pilato.elasticsearch.crawler.fs.crawler.ftp;
2+
3+
import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.toOctalPermission;
4+
5+
import org.apache.commons.net.ftp.FTPFile;
6+
import org.apache.logging.log4j.LogManager;
7+
import org.apache.logging.log4j.Logger;
8+
9+
public class FTPUtils {
10+
private static final Logger logger = LogManager.getLogger(FTPUtils.class);
11+
12+
/**
13+
* Determines FTPFile permissions.
14+
*/
15+
public static int getFilePermissions(final FTPFile file) {
16+
try {
17+
int user = toOctalPermission(
18+
file.hasPermission(FTPFile.USER_ACCESS, FTPFile.READ_PERMISSION),
19+
file.hasPermission(FTPFile.USER_ACCESS, FTPFile.WRITE_PERMISSION),
20+
file.hasPermission(FTPFile.USER_ACCESS, FTPFile.EXECUTE_PERMISSION));
21+
int group = toOctalPermission(
22+
file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.READ_PERMISSION),
23+
file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.WRITE_PERMISSION),
24+
file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.EXECUTE_PERMISSION));
25+
int others = toOctalPermission(
26+
file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.READ_PERMISSION),
27+
file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.WRITE_PERMISSION),
28+
file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.EXECUTE_PERMISSION));
29+
30+
return user * 100 + group * 10 + others;
31+
} catch (Exception e) {
32+
logger.warn("Failed to determine 'permissions' of {}: {}", file, e.getMessage());
33+
return -1;
34+
}
35+
}
36+
37+
}

‎crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121

2222
import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel;
2323
import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor;
24-
import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil;
2524
import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
2625
import fr.pilato.elasticsearch.crawler.fs.settings.Server;
2726
import java.io.IOException;
@@ -97,25 +96,30 @@ public FileAbstractModel toFileAbstractModel(String path, FTPFile file) {
9796
file.getSize(),
9897
file.getUser(),
9998
file.getGroup(),
100-
FsCrawlerUtil.getFilePermissions(file));
99+
FTPUtils.getFilePermissions(file));
101100
}
102101

103102
@Override
104-
public InputStream getInputStream(FileAbstractModel file) throws Exception {
103+
public InputStream getInputStream(FileAbstractModel file) throws IOException {
105104
String fullPath = file.getFullpath();
106105
if (isUtf8) {
107106
fullPath = new String(fullPath.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING);
108107
} else {
109108
fullPath = new String(fullPath.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING);
110109
}
110+
111111
InputStream inputStream = ftp.retrieveFileStream(fullPath);
112-
ftp.completePendingCommand();
113-
return inputStream;
112+
if (inputStream != null) {
113+
ftp.completePendingCommand();
114+
return inputStream;
115+
} else {
116+
throw new IOException(String.format("FTP client can not retrieve stream for [%s]", file.getFullpath()));
117+
}
114118
}
115119

116120
@Override
117121
public Collection<FileAbstractModel> getFiles(String dir) throws IOException {
118-
logger.debug("Listing local files from {}", dir);
122+
logger.debug("Listing files from {}", dir);
119123
if (isUtf8) {
120124
dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING);
121125
} else {

‎crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java‎

Lines changed: 75 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,13 @@
2626
import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
2727
import fr.pilato.elasticsearch.crawler.fs.settings.Server;
2828
import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase;
29+
import java.io.IOException;
2930
import java.io.InputStream;
3031
import java.nio.charset.StandardCharsets;
3132
import java.util.Collection;
3233

34+
import java.util.List;
35+
import java.util.stream.Collectors;
3336
import org.apache.commons.io.IOUtils;
3437
import org.junit.After;
3538
import org.junit.Before;
@@ -40,36 +43,45 @@
4043
import org.mockftpserver.fake.filesystem.DirectoryEntry;
4144
import org.mockftpserver.fake.filesystem.FileEntry;
4245
import org.mockftpserver.fake.filesystem.FileSystem;
46+
import org.mockftpserver.fake.filesystem.Permissions;
4347
import org.mockftpserver.fake.filesystem.UnixFakeFileSystem;
4448

4549
public class FileAbstractorFTPTest extends AbstractFSCrawlerTestCase {
4650
private FakeFtpServer fakeFtpServer;
47-
private final String home = "/home";
51+
private final String nestedDir = "/nested";
52+
private final String permissionDir = "/permission";
4853
private final String user = "user";
49-
private final String pass = "password";
54+
private final String pass = "pass";
5055

5156
@Before
5257
public void setup() {
53-
// it doesn't support utf-8
5458
fakeFtpServer = new FakeFtpServer();
5559
fakeFtpServer.setServerControlPort(5968);
56-
fakeFtpServer.addUserAccount(new UserAccount(user, pass, home));
60+
fakeFtpServer.addUserAccount(new UserAccount(user, pass, "/"));
5761
FileSystem fileSystem = new UnixFakeFileSystem();
5862

59-
fileSystem.add(new DirectoryEntry(home));
60-
fileSystem.add(new FileEntry(home + "/foo.txt", "文件名不支持中文"));
61-
fileSystem.add(new FileEntry(home + "/bar.txt", "bar"));
63+
fileSystem.add(new DirectoryEntry(nestedDir));
64+
fileSystem.add(new FileEntry(nestedDir + "/foo.txt", "文件名不支持中文"));
65+
fileSystem.add(new FileEntry(nestedDir + "/bar.txt", "filename doesn't support utf-8"));
6266

63-
fileSystem.add(new DirectoryEntry(home + "/buzz"));
64-
fileSystem.add(new FileEntry(home + "/buzz/hello.txt", "hello"));
65-
fileSystem.add(new FileEntry(home + "/buzz/world.txt", "world"));
67+
fileSystem.add(new DirectoryEntry(nestedDir + "/buzz"));
68+
fileSystem.add(new FileEntry(nestedDir + "/buzz/hello.txt", "hello"));
69+
fileSystem.add(new FileEntry(nestedDir + "/buzz/world.txt", "world"));
70+
71+
fileSystem.add(new DirectoryEntry(permissionDir));
72+
FileEntry fileAllPermissions = new FileEntry(permissionDir + "/all.txt", "123");
73+
fileAllPermissions.setPermissions(Permissions.ALL);
74+
fileSystem.add(fileAllPermissions);
75+
FileEntry fileNonePermissions = new FileEntry(permissionDir + "/none.txt", "456");
76+
fileNonePermissions.setPermissions(Permissions.NONE);
77+
fileSystem.add(fileNonePermissions);
6678

6779
fakeFtpServer.setFileSystem(fileSystem);
6880
fakeFtpServer.start();
6981
}
7082

7183
@After
72-
public void teardown() {
84+
public void shutDown() {
7385
fakeFtpServer.stop();
7486
}
7587

@@ -89,9 +101,9 @@ public void testConnectToFakeFTPServer() throws Exception {
89101

90102
FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings);
91103
ftp.open();
92-
boolean exists = ftp.exists(home);
104+
boolean exists = ftp.exists(nestedDir);
93105
assertThat(exists, is(true));
94-
Collection<FileAbstractModel> files = ftp.getFiles(home);
106+
Collection<FileAbstractModel> files = ftp.getFiles(nestedDir);
95107
assertThat(files.size(), is(3));
96108

97109
for (FileAbstractModel file : files) {
@@ -117,6 +129,11 @@ public void testConnectToFakeFTPServer() throws Exception {
117129
ftp.close();
118130
}
119131

132+
/**
133+
* FakeFtpServer doesn't support utf-8
134+
* You have to adapt this test to your own system
135+
* So this test is disabled by default
136+
*/
120137
@Test @Ignore
121138
public void testConnectToFTPServer() throws Exception {
122139
String path = "/中文目录";
@@ -160,4 +177,49 @@ public void testConnectToFTPServer() throws Exception {
160177

161178
ftp.close();
162179
}
180+
181+
@Test
182+
public void testFTPFilePermissions() throws IOException {
183+
int port = fakeFtpServer.getServerControlPort();
184+
FsSettings fsSettings = FsSettings.builder("fake")
185+
.setServer(
186+
Server.builder()
187+
.setHostname("localhost")
188+
.setUsername(user)
189+
.setPassword(pass)
190+
.setPort(port)
191+
.build()
192+
)
193+
.build();
194+
195+
FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings);
196+
ftp.open();
197+
198+
Collection<FileAbstractModel> files = ftp.getFiles(permissionDir);
199+
assertThat(files.size(), is(2));
200+
List<String> filenames = files.stream().map(FileAbstractModel::getName).collect(Collectors.toList());
201+
assertThat(filenames.contains("all.txt"), is(true));
202+
assertThat(filenames.contains("none.txt"), is(true));
203+
for (FileAbstractModel file : files) {
204+
if (file.getName().equals("all.txt")) {
205+
assertThat(file.getPermissions(), is(777));
206+
try (InputStream inputStream = ftp.getInputStream(file)) {
207+
String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
208+
logger.debug(" - {}: {}", file.getName(), content);
209+
}
210+
} else if (file.getName().equals("none.txt")) {
211+
assertThat(file.getPermissions(), is(0));
212+
boolean errorOccurred = false;
213+
try (InputStream ignored = ftp.getInputStream(file)) {
214+
logger.error(ignored);
215+
} catch (IOException e) {
216+
errorOccurred = true;
217+
logger.error(e.getMessage());
218+
}
219+
assertThat(errorOccurred, is(true));
220+
}
221+
}
222+
223+
ftp.close();
224+
}
163225
}

‎distribution/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
<properties>
2222
<!-- Global configuration parameters for docker -->
2323
<!-- By default, build docker images on each modules. -->
24+
<!--suppress UnresolvedMavenProperty -->
2425
<docker.skip>${env.DOCKER_SKIP}</docker.skip>
2526
<docker.verbose>build</docker.verbose>
2627
<docker.username>dadoonet</docker.username>

‎docs/source/admin/fs/ssh.rst‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,10 @@ To specify the drive, you need to use the following format:
9494
password: "password"
9595
protocol: "ssh"
9696
97-
Windows shared folders
98-
~~~~~~~~~~~~~~
97+
Windows shared folder
98+
~~~~~~~~~~~~~~~~~~~~~
99+
100+
When using Windows shared folder, you need to use the following format:
99101

100102
.. code:: yaml
101103

‎docs/source/dev/build.rst‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,12 @@ But you need first to specify the Maven profile to use and rebuild the project.
5151
* ``es-7x`` for Elasticsearch 7.x
5252
* ``es-6x`` for Elasticsearch 6.x
5353

54-
Run specific module tests from your Terminal
55-
""""""""""""""""""""""""""""""
54+
Run a specific test from your Terminal
55+
""""""""""""""""""""""""""""""""""""""
5656

57-
To run integration tests for a specific module, just run::
57+
To run a specific integration test, just run::
5858

59-
mvn test -am -DfailIfNoTests=false -pl [module_name_or_folder_path]
59+
mvn verify -am -Dtests.class=fr.pilato.elasticsearch.crawler.fs.test.integration.CLASS_NAME -Dtests.method="METHOD_NAME"
6060

6161
Run tests with an external cluster
6262
""""""""""""""""""""""""""""""""""

‎docs/source/index.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office
1515
**Main features**:
1616

1717
* Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones.
18-
* Remote file system over SSH/FTP/SMB(WIP) crawling.
18+
* Remote file system over SSH/FTP crawling.
1919
* REST interface to let you "upload" your binary documents to elasticsearch.
2020

2121
.. note::

‎framework/pom.xml‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,17 +39,6 @@
3939
<artifactId>commons-io</artifactId>
4040
</dependency>
4141

42-
<!-- FTP -->
43-
<dependency>
44-
<groupId>commons-net</groupId>
45-
<artifactId>commons-net</artifactId>
46-
</dependency>
47-
<dependency>
48-
<groupId>org.mockftpserver</groupId>
49-
<artifactId>MockFtpServer</artifactId>
50-
<scope>test</scope>
51-
</dependency>
52-
5342
<!-- Jackson -->
5443
<dependency>
5544
<groupId>com.fasterxml.jackson.core</groupId>

0 commit comments

Comments
 (0)