Skip to content

Commit 585ee92

Browse files
helsonxiaohelsonxiao
authored
Merge branch 'dadoonet:master' into master
2 parents f496920 + d8400a4 commit 585ee92

47 files changed

Lines changed: 681 additions & 462 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/maven.yml‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,24 +11,34 @@ jobs:
1111
- uses: actions/checkout@v2
1212
- name: 📣 Deprecation notice
1313
run: echo "We will most likely run integration tests only with security so this build will be removed."
14-
- name: Set up JDK 15
15-
uses: actions/setup-java@v2
16-
with:
17-
java-version: '15'
18-
distribution: 'adopt'
1914
- name: Cache Maven packages
2015
uses: actions/cache@v2
2116
with:
2217
path: ~/.m2
2318
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
2419
restore-keys: ${{ runner.os }}-m2
20+
- name: Set up JDK 15 and Maven Central Repository
21+
uses: actions/setup-java@v2
22+
with:
23+
java-version: '15'
24+
distribution: 'adopt'
25+
server-id: sonatype-nexus-snapshots
26+
server-username: MAVEN_USERNAME
27+
server-password: MAVEN_CENTRAL_TOKEN
28+
- name: Docker Layer Caching
29+
uses: satackey/action-docker-layer-caching@v0.0.11
30+
# Ignore the failure of a step and avoid terminating the job.
31+
continue-on-error: true
32+
with:
33+
key: fscrawler-docker-cache-{hash}
34+
restore-keys: |
35+
fscrawler-docker-cache-
2536
- name: Build the project
26-
run: mvn --batch-mode verify -DskipTests
37+
run: mvn --batch-mode install -DskipTests -Ddocker.skip
2738

2839
deploy:
2940
needs: build
3041
runs-on: ubuntu-latest
31-
3242
steps:
3343
- uses: actions/checkout@v2
3444
- name: Cache Maven packages
@@ -37,19 +47,27 @@ jobs:
3747
path: ~/.m2
3848
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
3949
restore-keys: ${{ runner.os }}-m2
40-
- name: Set up JDK 14 and Maven Central Repository
50+
- name: Set up JDK 15 and Maven Central Repository
4151
uses: actions/setup-java@v2
4252
with:
43-
java-version: '14'
53+
java-version: '15'
4454
distribution: 'adopt'
4555
server-id: sonatype-nexus-snapshots
4656
server-username: MAVEN_USERNAME
47-
server-password: MAVEN_PASSWORD
57+
server-password: MAVEN_CENTRAL_TOKEN
58+
- name: Docker Layer Caching
59+
uses: satackey/action-docker-layer-caching@v0.0.11
60+
# Ignore the failure of a step and avoid terminating the job.
61+
continue-on-error: true
62+
with:
63+
key: fscrawler-docker-cache-{hash}
64+
restore-keys: |
65+
fscrawler-docker-cache-
4866
- name: Deploy with Maven
4967
run: mvn deploy -DskipTests
5068
env:
5169
MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
52-
MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}
70+
MAVEN_CENTRAL_TOKEN: ${{ secrets.OSSRH_TOKEN }}
5371
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
5472
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
5573

‎.github/workflows/pr.yml‎

Lines changed: 43 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,56 @@ name: Build Pull Requests
66
on: [pull_request]
77

88
jobs:
9+
# We build the project but the docker images and run all the tests
910
build:
10-
1111
runs-on: ubuntu-latest
12-
1312
steps:
1413
- uses: actions/checkout@v2
15-
- name: Set up JDK 14 and Maven Central Repository
16-
uses: actions/setup-java@v2
17-
with:
18-
java-version: '14'
19-
distribution: 'adopt'
2014
- name: Cache Maven packages
2115
uses: actions/cache@v2
2216
with:
2317
path: ~/.m2
2418
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
2519
restore-keys: ${{ runner.os }}-m2
26-
- name: Build with Maven
27-
run: mvn --batch-mode --update-snapshots verify
20+
- name: Set up JDK 15
21+
uses: actions/setup-java@v2
22+
with:
23+
java-version: '15'
24+
distribution: 'adopt'
25+
- name: Docker Layer Caching
26+
uses: satackey/action-docker-layer-caching@v0.0.11
27+
# Ignore the failure of a step and avoid terminating the job.
28+
continue-on-error: true
29+
with:
30+
key: fscrawler-docker-cache-{hash}
31+
restore-keys: |
32+
fscrawler-docker-cache-
33+
- name: Build but the docker images and run all the tests
34+
run: mvn --batch-mode --update-snapshots install -Ddocker.skip
35+
36+
# We run this job in parallel
37+
build-docker:
38+
runs-on: ubuntu-latest
39+
steps:
40+
- uses: actions/checkout@v2
41+
- name: Cache Maven packages
42+
uses: actions/cache@v2
43+
with:
44+
path: ~/.m2
45+
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
46+
restore-keys: ${{ runner.os }}-m2
47+
- name: Set up JDK 15
48+
uses: actions/setup-java@v2
49+
with:
50+
java-version: '15'
51+
distribution: 'adopt'
52+
- name: Docker Layer Caching
53+
uses: satackey/action-docker-layer-caching@v0.0.11
54+
# Ignore the failure of a step and avoid terminating the job.
55+
continue-on-error: true
56+
with:
57+
key: fscrawler-docker-cache-{hash}
58+
restore-keys: |
59+
fscrawler-docker-cache-
60+
- name: Build the docker images
61+
run: mvn --batch-mode --update-snapshots install -DskipTests

‎3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchClient.java‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import java.util.Properties;
5151

5252
import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.*;
53+
import static fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil.parseJson;
5354

5455
/**
5556
* Workplace Search Java client
@@ -321,7 +322,7 @@ public List<String> getCustomSourcesByName(String name) {
321322
String json = listAllCustomSources(currentPage);
322323

323324
// We parse the json
324-
Object document = Configuration.defaultConfiguration().jsonProvider().parse(json);
325+
Object document = parseJson(json);
325326
totalPages = JsonPath.read(document, "$.meta.page.total_pages");
326327

327328
// We compare every source
@@ -370,7 +371,7 @@ public String createCustomSource(String sourceName) throws IOException {
370371
logger.trace("Source [{}] created. Response: {}", sourceName, response);
371372

372373
// We parse the json
373-
Object document = Configuration.defaultConfiguration().jsonProvider().parse(response);
374+
Object document = parseJson(response);
374375
String id = JsonPath.read(document, "$.id");
375376

376377
logger.debug("Source [{}/{}] created.", id, sourceName);
@@ -445,11 +446,9 @@ private Invocation.Builder prepareHttpCall(String path) {
445446

446447
@Override
447448
public String toString() {
448-
final StringBuffer sb = new StringBuffer("WPSearchClient{");
449-
sb.append("endpoint='").append(endpoint).append('\'');
450-
sb.append(", host='").append(host).append('\'');
451-
sb.append(", urlForApi='").append(urlForApi).append('\'');
452-
sb.append('}');
453-
return sb.toString();
449+
return "WPSearchClient{" + "endpoint='" + endpoint + '\'' +
450+
", host='" + host + '\'' +
451+
", urlForApi='" + urlForApi + '\'' +
452+
'}';
454453
}
455454
}

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ You need to install a version matching your Elasticsearch version:
1717
| 6.x, 7.x | 2.7-SNAPSHOT| |[2.7-SNAPSHOT](https://fscrawler.readthedocs.io/en/latest/) |
1818
| 2.x, 5.x, 6.x | 2.6 |2019-01-09|[2.6](https://fscrawler.readthedocs.io/en/fscrawler-2.6) |
1919
| 2.x, 5.x, 6.x | 2.5 |2018-08-04|[2.5](https://fscrawler.readthedocs.io/en/fscrawler-2.5) |
20-
| 2.x, 5.x, 6.x | **2.4** |2017-08-11|[2.4](https://github.com/dadoonet/fscrawler/blob/fscrawler-2.4/README.md) |
20+
| 2.x, 5.x, 6.x | 2.4 |2017-08-11|[2.4](https://github.com/dadoonet/fscrawler/blob/fscrawler-2.4/README.md) |
2121
| 2.x, 5.x, 6.x | 2.3 |2017-07-10|[2.3](https://github.com/dadoonet/fscrawler/blob/fscrawler-2.3/README.md) |
2222
| 1.x, 2.x, 5.x | 2.2 |2017-02-03|[2.2](https://github.com/dadoonet/fscrawler/blob/fscrawler-2.2/README.md) |
2323
| 1.x, 2.x, 5.x | 2.1 |2016-07-26|[2.1](https://github.com/dadoonet/fscrawler/blob/fscrawler-2.1/README.md) |

‎beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/Attributes.java‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,6 @@
2424
*/
2525
public class Attributes {
2626

27-
/**
28-
* Generated json field names
29-
*/
30-
static public final class FIELD_NAMES {
31-
public static final String OWNER = "owner";
32-
public static final String GROUP = "group";
33-
public static final String PERMISSIONS = "permissions";
34-
}
35-
3627
private String owner;
3728
private String group;
3829
private int permissions;

‎beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/Doc.java‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,6 @@
2626
*/
2727
public class Doc {
2828

29-
/**
30-
* Generated json field names
31-
*/
32-
static public final class FIELD_NAMES {
33-
public static final String CONTENT = "content";
34-
public static final String ATTACHMENT = "attachment";
35-
public static final String META = "meta";
36-
public static final String FILE = "file";
37-
public static final String PATH = "path";
38-
public static final String ATTRIBUTES = "attributes";
39-
}
40-
4129
private String content;
4230
private String attachment;
4331
private Meta meta;

‎beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/File.java‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -26,23 +26,6 @@
2626
*/
2727
public class File {
2828

29-
/**
30-
* Generated json field names
31-
*/
32-
static public final class FIELD_NAMES {
33-
public static final String EXTENSION = "extension";
34-
public static final String CONTENT_TYPE = "content_type";
35-
public static final String CREATED = "created";
36-
public static final String LAST_MODIFIED = "last_modified";
37-
public static final String LAST_ACCESSED = "last_accessed";
38-
public static final String INDEXING_DATE = "indexing_date";
39-
public static final String FILESIZE = "filesize";
40-
public static final String FILENAME = "filename";
41-
public static final String URL = "url";
42-
public static final String INDEXED_CHARS = "indexed_chars";
43-
public static final String CHECKSUM = "checksum";
44-
}
45-
4629
private String extension;
4730
private String contentType;
4831
private Date created;

‎beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/Folder.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to David Pilato (the "Author") under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. Author licenses this
6+
* file to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package fr.pilato.elasticsearch.crawler.fs.beans;
21+
22+
/**
23+
* Represents a folder we have visited
24+
*/
25+
public class Folder {
26+
27+
public static final String CONTENT_TYPE = "text/directory";
28+
29+
private Path path;
30+
private File file;
31+
32+
public Folder() {
33+
path = new Path();
34+
file = new File();
35+
file.setContentType(CONTENT_TYPE);
36+
}
37+
38+
/**
39+
* Build a folder with a single ctor call
40+
* @param name The folder name
41+
* @param root Root of the folder
42+
* @param real The full path to the folder
43+
* @param virtual The virtual path from the root
44+
*/
45+
public Folder(String name, String root, String real, String virtual) {
46+
path = new Path();
47+
path.setRoot(root);
48+
path.setReal(real);
49+
path.setVirtual(virtual);
50+
file = new File();
51+
file.setFilename(name);
52+
file.setContentType(CONTENT_TYPE);
53+
}
54+
55+
public Path getPath() {
56+
return path;
57+
}
58+
59+
public void setPath(Path path) {
60+
this.path = path;
61+
}
62+
63+
public File getFile() {
64+
return file;
65+
}
66+
67+
public void setFile(File file) {
68+
this.file = file;
69+
}
70+
}

beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/PathParser.java renamed to beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/FolderParser.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,18 @@
2323

2424
import static fr.pilato.elasticsearch.crawler.fs.framework.MetaParser.prettyMapper;
2525

26-
public class PathParser {
26+
public class FolderParser {
2727

28-
public static String toJson(Path path) {
28+
public static String toJson(Folder folder) {
2929
try {
30-
return prettyMapper.writeValueAsString(path);
30+
return prettyMapper.writeValueAsString(folder);
3131
} catch (IOException e) {
3232
// TODO Fix that code. We should log here and return null.
3333
throw new RuntimeException(e);
3434
}
3535
}
3636

37-
public static Path fromJson(String json) throws IOException {
38-
return prettyMapper.readValue(json, Path.class);
37+
public static Folder fromJson(String json) throws IOException {
38+
return prettyMapper.readValue(json, Folder.class);
3939
}
4040
}

‎beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/Meta.java‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -29,37 +29,6 @@
2929
*/
3030
public class Meta {
3131

32-
/**
33-
* Generated json field names
34-
*/
35-
static public final class FIELD_NAMES {
36-
public static final String AUTHOR = "author";
37-
public static final String TITLE = "title";
38-
public static final String DATE = "date";
39-
public static final String KEYWORDS = "keywords";
40-
public static final String LANGUAGE = "language";
41-
public static final String FORMAT = "format";
42-
public static final String IDENTIFIER = "identifier";
43-
public static final String CONTRIBUTOR = "contributor";
44-
public static final String COVERAGE = "coverage";
45-
public static final String MODIFIER = "modifier";
46-
public static final String CREATOR_TOOL = "creator_tool";
47-
public static final String PUBLISHER = "publisher";
48-
public static final String RELATION = "relation";
49-
public static final String RIGHTS = "rights";
50-
public static final String SOURCE = "source";
51-
public static final String TYPE = "type";
52-
public static final String DESCRIPTION = "description";
53-
public static final String CREATED = "created";
54-
public static final String PRINT_DATE = "print_date";
55-
public static final String METADATA_DATE = "metadata_date";
56-
public static final String LATITUDE = "latitude";
57-
public static final String LONGITUDE = "longitude";
58-
public static final String ALTITUDE = "altitude";
59-
public static final String RATING = "rating";
60-
public static final String COMMENTS = "comments";
61-
}
62-
6332
private String author;
6433
private String title;
6534
private Date date;

0 commit comments

Comments
 (0)