Skip to content

Commit 2a9317e

Browse files
ruanwenjunruanwenjun
committed
HiveCli use -f to submit job
1 parent 83ac81c commit 2a9317e

File tree

3 files changed

+89
-48
lines changed

3 files changed

+89
-48
lines changed

Diff for: ‎dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java

+32-17
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919

2020
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
2121

22+
import java.io.File;
23+
import java.nio.file.Files;
24+
import java.nio.file.Path;
25+
import java.nio.file.Paths;
26+
import java.nio.file.StandardOpenOption;
2227
import org.apache.dolphinscheduler.common.utils.JSONUtils;
2328
import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask;
2429
import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor;
@@ -38,6 +43,7 @@
3843
import java.util.Collections;
3944
import java.util.List;
4045
import java.util.Map;
46+
import org.apache.dolphinscheduler.plugin.task.api.utils.FileUtils;
4147

4248
public class HiveCliTask extends AbstractRemoteTask {
4349

@@ -107,37 +113,46 @@ protected String buildCommand() {
107113

108114
final List<String> args = new ArrayList<>();
109115

110-
final String type = hiveCliParameters.getHiveCliTaskExecutionType();
116+
String fileContent = HiveSqlScriptReader.readHiveSqlContent(taskExecutionContext.getExecutePath(), hiveCliParameters);
117+
fileContent = ParameterUtils.convertParameterPlaceholders(fileContent, ParamUtils.convert(taskExecutionContext.getPrepareParamsMap()));
118+
String sqlFilePath = generateSqlScriptFile(fileContent);
111119

112-
// TODO: make sure type is not unknown
113-
if (HiveCliConstants.TYPE_FILE.equals(type)) {
114-
args.add(HiveCliConstants.HIVE_CLI_EXECUTE_FILE);
115-
final List<ResourceInfo> resourceInfos = hiveCliParameters.getResourceList();
116-
if (resourceInfos.size() > 1) {
117-
logger.warn("more than 1 files detected, use the first one by default");
118-
}
119-
120-
args.add(StringUtils.stripStart(resourceInfos.get(0).getResourceName(), "/"));
121-
} else {
122-
final String script = hiveCliParameters.getHiveSqlScript();
123-
args.add(String.format(HiveCliConstants.HIVE_CLI_EXECUTE_SCRIPT, script));
124-
}
120+
args.add(HiveCliConstants.HIVE_CLI_EXECUTE_FILE);
121+
args.add(sqlFilePath);
125122

126123
final String hiveCliOptions = hiveCliParameters.getHiveCliOptions();
127124
if (StringUtils.isNotEmpty(hiveCliOptions)) {
128125
args.add(hiveCliOptions);
129126
}
130127

131-
final Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
132-
final String command =
133-
ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap));
128+
String command = String.join(" ", args);
134129

135130
logger.info("hiveCli task command: {}", command);
136131

137132
return command;
138133

139134
}
140135

136+
protected String generateSqlScriptFile(String rawScript) {
137+
String scriptFileName = Paths.get(taskExecutionContext.getExecutePath(), "hive_cli.sql").toString();
138+
139+
try {
140+
File file = new File(scriptFileName);
141+
Path path = file.toPath();
142+
if (Files.exists(path)) {
143+
logger.warn("The HiveCli sql file: {} is already exist, will delete it", scriptFileName);
144+
Files.deleteIfExists(path);
145+
}
146+
if (!Files.exists(path)) {
147+
org.apache.dolphinscheduler.plugin.task.api.utils.FileUtils.createFileWith755(path);
148+
Files.write(path, rawScript.getBytes(), StandardOpenOption.APPEND);
149+
}
150+
return scriptFileName;
151+
} catch (Exception ex) {
152+
throw new TaskException("Generate sql script file: " + scriptFileName + " failed", ex);
153+
}
154+
}
155+
141156
@Override
142157
public AbstractParameters getParameters() {
143158
return hiveCliParameters;

Diff for: ‎dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveSqlScriptReader.java

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.task.hivecli;
19+
20+
import java.io.File;
21+
import java.nio.charset.StandardCharsets;
22+
import java.nio.file.Paths;
23+
import java.util.List;
24+
import lombok.experimental.UtilityClass;
25+
import lombok.extern.slf4j.Slf4j;
26+
import org.apache.commons.collections4.CollectionUtils;
27+
import org.apache.commons.io.FileUtils;
28+
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
29+
30+
@Slf4j
31+
@UtilityClass
32+
public class HiveSqlScriptReader {
33+
34+
public static String readHiveSqlContent(String taskWorkingDirectory, HiveCliParameters hiveCliParameters) {
35+
if (HiveCliConstants.TYPE_FILE.equals(hiveCliParameters.getHiveCliTaskExecutionType())) {
36+
List<ResourceInfo> resources = hiveCliParameters.getResourceList();
37+
if (CollectionUtils.isEmpty(resources)) {
38+
throw new IllegalArgumentException("HiveCliTaskExecutionType is FILE, but resourceList is empty");
39+
}
40+
if (resources.size() > 1) {
41+
log.warn("HiveCliTaskExecutionType is FILE, but resources: {} size > 1, use the first one by default.", resources);
42+
}
43+
try {
44+
return FileUtils.readFileToString(new File(Paths.get(taskWorkingDirectory, resources.get(0).getResourceName()).toString()),
45+
StandardCharsets.UTF_8);
46+
} catch (Exception ex) {
47+
throw new IllegalArgumentException("Read HiveSql from " + resources.get(0) + " failed", ex);
48+
}
49+
} else {
50+
return hiveCliParameters.getHiveSqlScript();
51+
}
52+
}
53+
}

Diff for: ‎dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java

+4-31
Original file line numberDiff line numberDiff line change
@@ -36,42 +36,26 @@
3636
@RunWith(MockitoJUnitRunner.class)
3737
public class HiveCliTaskTest {
3838

39-
public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_SCRIPT_COMMAND =
40-
"hive -e \"SHOW DATABASES;\"";
41-
42-
public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_FILE_COMMAND =
43-
"hive -f sql_tasks/hive_task.sql";
44-
45-
public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_WITH_OPTIONS =
46-
"hive -e \"SHOW DATABASES;\" --verbose";
47-
4839
@Test
4940
public void hiveCliTaskExecuteSqlFromScript() throws Exception {
5041
String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromScriptParameters();
5142
HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters);
5243
hiveCliTask.init();
53-
Assert.assertEquals(hiveCliTask.buildCommand(), EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_SCRIPT_COMMAND);
54-
}
55-
56-
@Test
57-
public void hiveCliTaskExecuteSqlFromFile() throws Exception {
58-
String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromFileParameters();
59-
HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters);
60-
hiveCliTask.init();
61-
Assert.assertEquals(hiveCliTask.buildCommand(), EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_FILE_COMMAND);
44+
Assert.assertEquals("hive -f /tmp/hive_cli.sql", hiveCliTask.buildCommand());
6245
}
6346

6447
@Test
65-
public void hiveCliTaskExecuteWithOptions() throws Exception {
48+
public void hiveCliTaskExecuteWithOptions() {
6649
String hiveCliTaskParameters = buildHiveCliTaskExecuteWithOptionsParameters();
6750
HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters);
6851
hiveCliTask.init();
69-
Assert.assertEquals(hiveCliTask.buildCommand(), EXPECTED_HIVE_CLI_TASK_EXECUTE_WITH_OPTIONS);
52+
Assert.assertEquals("hive -f /tmp/hive_cli.sql --verbose", hiveCliTask.buildCommand());
7053
}
7154

7255
private HiveCliTask prepareHiveCliTaskForTest(final String hiveCliTaskParameters) {
7356
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
7457
when(taskExecutionContext.getTaskParams()).thenReturn(hiveCliTaskParameters);
58+
when(taskExecutionContext.getExecutePath()).thenReturn("/tmp");
7559
HiveCliTask hiveCliTask = spy(new HiveCliTask(taskExecutionContext));
7660
return hiveCliTask;
7761
}
@@ -83,17 +67,6 @@ private String buildHiveCliTaskExecuteSqlFromScriptParameters() {
8367
return JSONUtils.toJsonString(hiveCliParameters);
8468
}
8569

86-
private String buildHiveCliTaskExecuteSqlFromFileParameters() {
87-
final HiveCliParameters hiveCliParameters = new HiveCliParameters();
88-
hiveCliParameters.setHiveCliTaskExecutionType("FILE");
89-
List<ResourceInfo> resources = new ArrayList<>();
90-
ResourceInfo sqlResource = new ResourceInfo();
91-
sqlResource.setResourceName("/sql_tasks/hive_task.sql");
92-
resources.add(sqlResource);
93-
hiveCliParameters.setResourceList(resources);
94-
return JSONUtils.toJsonString(hiveCliParameters);
95-
}
96-
9770
private String buildHiveCliTaskExecuteWithOptionsParameters() {
9871
final HiveCliParameters hiveCliParameters = new HiveCliParameters();
9972
hiveCliParameters.setHiveCliTaskExecutionType("SCRIPT");

0 commit comments

Comments
 (0)