Skip to content

Commit d218b02

Browse files
pegasasrickchengx
pegasas
and
rickchengx
authored
[DSIP-25][Remote Logging] Split remote logging configuration (apache#15826)
Co-authored-by: Rick Cheng <[email protected]>
1 parent fa6ea8b commit d218b02

File tree

12 files changed

+249
-14
lines changed

12 files changed

+249
-14
lines changed

Diff for: ‎dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
<directory>${basedir}/../../dolphinscheduler-common/src/main/resources</directory>
5353
<includes>
5454
<include>**/*.properties</include>
55+
<include>**/*.yaml</include>
5556
</includes>
5657
<outputDirectory>conf</outputDirectory>
5758
</fileSet>

Diff for: ‎dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml

+1
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
5454
<includes>
5555
<include>**/*.properties</include>
56+
<include>**/*.yaml</include>
5657
</includes>
5758
<outputDirectory>conf</outputDirectory>
5859
</fileSet>

Diff for: ‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java

+27-6
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,18 @@
3131
* This class will get the property by the priority of the following: env > jvm > properties.
3232
*/
3333
@Slf4j
34-
public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate {
34+
public class ImmutablePriorityPropertyDelegate implements IPropertyDelegate {
3535

3636
private static final Map<String, Optional<ConfigValue<String>>> configValueMap = new ConcurrentHashMap<>();
3737

38-
public ImmutablePriorityPropertyDelegate(String propertyAbsolutePath) {
39-
super(propertyAbsolutePath);
38+
private ImmutablePropertyDelegate immutablePropertyDelegate;
39+
40+
private ImmutableYamlDelegate immutableYamlDelegate;
41+
42+
public ImmutablePriorityPropertyDelegate(ImmutablePropertyDelegate immutablePropertyDelegate,
43+
ImmutableYamlDelegate immutableYamlDelegate) {
44+
this.immutablePropertyDelegate = immutablePropertyDelegate;
45+
this.immutableYamlDelegate = immutableYamlDelegate;
4046
}
4147

4248
@Override
@@ -56,8 +62,14 @@ public String get(String key) {
5662
return value;
5763
}
5864
value = getConfigValueFromProperties(key);
65+
if (value.isPresent()) {
66+
log.debug("Get config value from properties, key: {} actualKey: {}, value: {}",
67+
k, value.get().getActualKey(), value.get().getValue());
68+
return value;
69+
}
70+
value = getConfigValueFromYaml(key);
5971
value.ifPresent(
60-
stringConfigValue -> log.debug("Get config value from properties, key: {} actualKey: {}, value: {}",
72+
stringConfigValue -> log.debug("Get config value from yaml, key: {} actualKey: {}, value: {}",
6173
k, stringConfigValue.getActualKey(), stringConfigValue.getValue()));
6274
return value;
6375
});
@@ -76,7 +88,8 @@ public String get(String key, String defaultValue) {
7688
@Override
7789
public Set<String> getPropertyKeys() {
7890
Set<String> propertyKeys = new HashSet<>();
79-
propertyKeys.addAll(super.getPropertyKeys());
91+
propertyKeys.addAll(this.immutablePropertyDelegate.getPropertyKeys());
92+
propertyKeys.addAll(this.immutableYamlDelegate.getPropertyKeys());
8093
propertyKeys.addAll(System.getProperties().stringPropertyNames());
8194
propertyKeys.addAll(System.getenv().keySet());
8295
return propertyKeys;
@@ -104,7 +117,15 @@ private Optional<ConfigValue<String>> getConfigValueFromJvm(String key) {
104117
}
105118

106119
private Optional<ConfigValue<String>> getConfigValueFromProperties(String key) {
107-
String value = super.get(key);
120+
String value = this.immutablePropertyDelegate.get(key);
121+
if (value != null) {
122+
return Optional.of(ConfigValue.fromProperties(key, value));
123+
}
124+
return Optional.empty();
125+
}
126+
127+
private Optional<ConfigValue<String>> getConfigValueFromYaml(String key) {
128+
String value = this.immutableYamlDelegate.get(key);
108129
if (value != null) {
109130
return Optional.of(ConfigValue.fromProperties(key, value));
110131
}

Diff for: ‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public ImmutablePropertyDelegate(String... propertyAbsolutePath) {
4949
} catch (IOException e) {
5050
log.error("Load property: {} error, please check if the file exist under classpath",
5151
propertyAbsolutePath, e);
52-
System.exit(1);
52+
throw new RuntimeException(e);
5353
}
5454
}
5555
printProperties();

Diff for: ‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.common.config;
19+
20+
import java.io.IOException;
21+
import java.io.InputStream;
22+
import java.util.Properties;
23+
import java.util.Set;
24+
25+
import lombok.extern.slf4j.Slf4j;
26+
27+
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
28+
import org.springframework.core.io.InputStreamResource;
29+
30+
@Slf4j
31+
public class ImmutableYamlDelegate implements IPropertyDelegate {
32+
33+
private static final String REMOTE_LOGGING_YAML_NAME = "/remote-logging.yaml";
34+
35+
private final Properties properties;
36+
37+
public ImmutableYamlDelegate() {
38+
this(REMOTE_LOGGING_YAML_NAME);
39+
}
40+
41+
public ImmutableYamlDelegate(String... yamlAbsolutePath) {
42+
properties = new Properties();
43+
// read from classpath
44+
for (String fileName : yamlAbsolutePath) {
45+
try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) {
46+
YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean();
47+
factory.setResources(new InputStreamResource(fis));
48+
factory.afterPropertiesSet();
49+
Properties subProperties = factory.getObject();
50+
properties.putAll(subProperties);
51+
} catch (IOException e) {
52+
log.error("Load property: {} error, please check if the file exist under classpath",
53+
yamlAbsolutePath, e);
54+
throw new RuntimeException(e);
55+
}
56+
}
57+
printProperties();
58+
}
59+
60+
public ImmutableYamlDelegate(Properties properties) {
61+
this.properties = properties;
62+
}
63+
64+
@Override
65+
public String get(String key) {
66+
return properties.getProperty(key);
67+
}
68+
69+
@Override
70+
public String get(String key, String defaultValue) {
71+
return properties.getProperty(key, defaultValue);
72+
}
73+
74+
@Override
75+
public Set<String> getPropertyKeys() {
76+
return properties.stringPropertyNames();
77+
}
78+
79+
private void printProperties() {
80+
properties.forEach((k, v) -> log.debug("Get property {} -> {}", k, v));
81+
}
82+
}

Diff for: ‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ private Constants() {
3535
*/
3636
public static final String COMMON_PROPERTIES_PATH = "/common.properties";
3737

38+
public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml";
39+
3840
public static final String FORMAT_SS = "%s%s";
3941
public static final String FORMAT_S_S = "%s/%s";
4042
public static final String FORMAT_S_S_COLON = "%s:%s";
@@ -683,9 +685,6 @@ private Constants() {
683685
public static final Integer QUERY_ALL_ON_WORKFLOW = 2;
684686
public static final Integer QUERY_ALL_ON_TASK = 3;
685687

686-
/**
687-
* remote logging
688-
*/
689688
public static final String REMOTE_LOGGING_ENABLE = "remote.logging.enable";
690689

691690
public static final String REMOTE_LOGGING_TARGET = "remote.logging.target";

Diff for: ‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

+7-3
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@
1818
package org.apache.dolphinscheduler.common.utils;
1919

2020
import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
21+
import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
2122

22-
import org.apache.dolphinscheduler.common.config.IPropertyDelegate;
2323
import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate;
24+
import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate;
25+
import org.apache.dolphinscheduler.common.config.ImmutableYamlDelegate;
2426

2527
import java.util.HashMap;
2628
import java.util.Map;
@@ -37,8 +39,10 @@
3739
public class PropertyUtils {
3840

3941
// todo: add another implementation for zookeeper/etcd/consul/xx
40-
private static final IPropertyDelegate propertyDelegate =
41-
new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH);
42+
private final ImmutablePriorityPropertyDelegate propertyDelegate =
43+
new ImmutablePriorityPropertyDelegate(
44+
new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
45+
new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH));
4246

4347
public static String getString(String key) {
4448
return propertyDelegate.get(key.trim());

Diff for: ‎dolphinscheduler-common/src/main/resources/remote-logging.yaml

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
remote-logging:
19+
# Whether to enable remote logging
20+
enable: false
21+
# if remote-logging.enable = true, set the target of remote logging
22+
target: OSS
23+
# if remote-logging.enable = true, set the log base directory
24+
base.dir: logs
25+
# if remote-logging.enable = true, set the number of threads to send logs to remote storage
26+
thread.pool.size: 10
27+
# required if you set remote-logging.target=OSS
28+
oss:
29+
# oss access key id, required if you set remote-logging.target=OSS
30+
access.key.id: <access.key.id>
31+
# oss access key secret, required if you set remote-logging.target=OSS
32+
access.key.secret: <access.key.secret>
33+
# oss bucket name, required if you set remote-logging.target=OSS
34+
bucket.name: <bucket.name>
35+
# oss endpoint, required if you set remote-logging.target=OSS
36+
endpoint: <endpoint>
37+
# required if you set remote-logging.target=S3
38+
s3:
39+
# s3 access key id, required if you set remote-logging.target=S3
40+
access.key.id: <access.key.id>
41+
# s3 access key secret, required if you set remote-logging.target=S3
42+
access.key.secret: <access.key.secret>
43+
# s3 bucket name, required if you set remote-logging.target=S3
44+
bucket.name: <bucket.name>
45+
# s3 endpoint, required if you set remote-logging.target=S3
46+
endpoint: <endpoint>
47+
# s3 region, required if you set remote-logging.target=S3
48+
region: <region>
49+
google.cloud.storage:
50+
# the location of the google cloud credential, required if you set remote-logging.target=GCS
51+
credential: /path/to/credential
52+
# gcs bucket name, required if you set remote-logging.target=GCS
53+
bucket.name: <your-bucket>
54+
abs:
55+
# abs account name, required if you set resource.storage.type=ABS
56+
account.name: <your-account-name>
57+
# abs account key, required if you set resource.storage.type=ABS
58+
account.key: <your-account-key>
59+
# abs container name, required if you set resource.storage.type=ABS
60+
container.name: <your-container-name>
61+

Diff for: ‎dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,17 @@
1919

2020
import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable;
2121
import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
22+
import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
2223

2324
import org.junit.jupiter.api.Assertions;
2425
import org.junit.jupiter.api.Test;
2526

2627
class ImmutablePriorityPropertyDelegateTest {
2728

2829
private final ImmutablePriorityPropertyDelegate immutablePriorityPropertyDelegate =
29-
new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH);
30+
new ImmutablePriorityPropertyDelegate(
31+
new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
32+
new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH));
3033

3134
@Test
3235
void getOverrideFromEnv() throws Exception {

Diff for: ‎dolphinscheduler-common/src/test/resources/remote-logging.yaml

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
remote-logging:
19+
# Whether to enable remote logging
20+
enable: false
21+
# if remote-logging.enable = true, set the target of remote logging
22+
target: OSS
23+
# if remote-logging.enable = true, set the log base directory
24+
base.dir: logs
25+
# if remote-logging.enable = true, set the number of threads to send logs to remote storage
26+
thread.pool.size: 10
27+
# required if you set remote-logging.target=OSS
28+
oss:
29+
# oss access key id, required if you set remote-logging.target=OSS
30+
access.key.id: <access.key.id>
31+
# oss access key secret, required if you set remote-logging.target=OSS
32+
access.key.secret: <access.key.secret>
33+
# oss bucket name, required if you set remote-logging.target=OSS
34+
bucket.name: <bucket.name>
35+
# oss endpoint, required if you set remote-logging.target=OSS
36+
endpoint: <endpoint>
37+
# required if you set remote-logging.target=S3
38+
s3:
39+
# s3 access key id, required if you set remote-logging.target=S3
40+
access.key.id: <access.key.id>
41+
# s3 access key secret, required if you set remote-logging.target=S3
42+
access.key.secret: <access.key.secret>
43+
# s3 bucket name, required if you set remote-logging.target=S3
44+
bucket.name: <bucket.name>
45+
# s3 endpoint, required if you set remote-logging.target=S3
46+
endpoint: <endpoint>
47+
# s3 region, required if you set remote-logging.target=S3
48+
region: <region>
49+
google.cloud.storage:
50+
# the location of the google cloud credential, required if you set remote-logging.target=GCS
51+
credential: /path/to/credential
52+
# gcs bucket name, required if you set remote-logging.target=GCS
53+
bucket.name: <your-bucket>
54+
abs:
55+
# abs account name, required if you set resource.storage.type=ABS
56+
account.name: <your-account-name>
57+
# abs account key, required if you set resource.storage.type=ABS
58+
account.key: <your-account-key>
59+
# abs container name, required if you set resource.storage.type=ABS
60+
container.name: <your-container-name>
61+

Diff for: ‎dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
5353
<includes>
5454
<include>**/*.properties</include>
55+
<include>**/*.yaml</include>
5556
</includes>
5657
<outputDirectory>conf</outputDirectory>
5758
</fileSet>

Diff for: ‎dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml

+1
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
5454
<includes>
5555
<include>**/*.properties</include>
56+
<include>**/*.yaml</include>
5657
</includes>
5758
<outputDirectory>conf</outputDirectory>
5859
</fileSet>

0 commit comments

Comments
 (0)