Add CarbonJob job type

CONTRIBUTORS.md

@@ -190,3 +190,6 @@ The following were contributed by Sayali Kadam.
 
 The following were contributed by Frank Gu.
 * `Add ability for Hadoop DSL to understand the KubernetesJob job type`
+
+The following were contributed by Arjun Singh Bora.
+* `Add ability for Hadoop DSL to understand the CarbonJob job type`

VERSIONS.md

@@ -17,6 +17,9 @@ the License.
 Note that the LinkedIn build system occasionally requires that we skip a
 version bump, so you will see a few skipped version numbers in the list below.
 
+0.15.30
+* Adding support for model auto-publish for CarbonJob
+
 0.15.22
 * Adding support for model auto-publish for KabootarJob
 

gradle.properties

@@ -1,2 +1,2 @@
 org.gradle.daemon=true
-version=0.15.22
+version=0.15.30

hadoop-plugin/src/integTest/resources/expectedOutput/negative/missingFields.out

@@ -46,4 +46,5 @@ RequiredFieldsChecker ERROR: WormholePushJob job26 must set inputPath, namespace
 RequiredFieldsChecker ERROR: KabootarJob job27 must set trainingModelLocation, trainingName, aiProjectGroup, wormholeNamespace, initialImport. Please see the job documentation for more details.
 RequiredFieldsChecker ERROR: WormholePushJob2 job28 must set inputPath, namespace and dataset
 RequiredFieldsChecker ERROR: KubernetesJob job29 has the following required fields: kind
+RequiredFieldsChecker ERROR: CarbonJob job30 must set task.type
 RequiredFieldsChecker WARNING: Properties properties1 does not set any confProperties, jobProperties, jvmProperties or basePropertySetName. Nothing will be built for this properties object.

hadoop-plugin/src/integTest/resources/gradle/negative/missingFields.gradle

@@ -138,6 +138,11 @@ hadoop {
     kubernetesJob('job29') {
       depends 'job28'
     }
-    targets 'job29'
+
+    carbonJob('job30') {
+        depends 'job29'
+    }
+
+    targets 'job30'
   }
 }

hadoop-plugin/src/integTest/resources/gradle/positive/jobs1.gradle

@@ -485,6 +485,12 @@ hadoop {
       depends 'job28'
     }
 
+    carbonJob('job30') {
+      taskType 'retention'   // Required
+
+      depends 'job29'
+    }
+
     targets 'job29'
   }
 }

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/BaseNamedScopeContainer.groovy

@@ -13,8 +13,10 @@
  * License for the specific language governing permissions and limitations under
  * the License.
  */
-package com.linkedin.gradle.hadoopdsl;
+package com.linkedin.gradle.hadoopdsl
 
+
+import com.linkedin.gradle.hadoopdsl.job.CarbonJob;
 import com.linkedin.gradle.hadoopdsl.job.CommandJob;
 import com.linkedin.gradle.hadoopdsl.job.GobblinJob;
 import com.linkedin.gradle.hadoopdsl.job.HadoopJavaJob;
@@ -1180,6 +1182,18 @@ abstract class BaseNamedScopeContainer implements NamedScopeContainer {
     return ((KabootarJob)configureJob(factory.makeKabootarJob(name), configure));
   }
 
+  /**
+   * DSL CarbonJob method. Creates a carbonJob in scope with the given name
+   * and configuration.
+   *
+   * @param name The job name
+   * @param configure The configuration closure
+   * @return The new job
+   */
+  @HadoopDslMethod
+  CarbonJob carbonJob(String name, @DelegatesTo(CarbonJob) Closure configure) {
+    return ((CarbonJob)configureJob(factory.makeCarbonJob(name), configure));
+  }
   /**
    * DSL trigger method. Creates a Trigger in scope with the given name and configuration.
    *

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/BaseVisitor.groovy

@@ -13,8 +13,10 @@
  * License for the specific language governing permissions and limitations under
  * the License.
  */
-package com.linkedin.gradle.hadoopdsl;
+package com.linkedin.gradle.hadoopdsl
 
+
+import com.linkedin.gradle.hadoopdsl.job.CarbonJob;
 import com.linkedin.gradle.hadoopdsl.job.CommandJob;
 import com.linkedin.gradle.hadoopdsl.job.GobblinJob;
 import com.linkedin.gradle.hadoopdsl.job.HadoopJavaJob;
@@ -259,4 +261,9 @@ abstract class BaseVisitor implements Visitor {
   void visitJob(KubernetesJob job) {
     visitJob((Job)job);
   }
+
+  @Override
+  void visitJob(CarbonJob job) {
+    visitJob((Job)job);
+  }
 }

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/HadoopDslFactory.groovy

@@ -13,8 +13,10 @@
  * License for the specific language governing permissions and limitations under
  * the License.
  */
-package com.linkedin.gradle.hadoopdsl;
+package com.linkedin.gradle.hadoopdsl
 
+
+import com.linkedin.gradle.hadoopdsl.job.CarbonJob;
 import com.linkedin.gradle.hadoopdsl.job.CommandJob;
 import com.linkedin.gradle.hadoopdsl.job.GobblinJob;
 import com.linkedin.gradle.hadoopdsl.job.HadoopJavaJob;
@@ -435,6 +437,16 @@ class HadoopDslFactory {
     return new KabootarJob(name);
   }
 
+  /**
+   * Factory method to build a CarbonJob.
+   *
+   * @param name The job name
+   * @return The job
+   */
+  CarbonJob makeCarbonJob(String name) {
+    return new CarbonJob(name);
+  }
+
   /**
    * Factory method to build a Trigger.
    *

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/HadoopDslPlugin.groovy

@@ -106,6 +106,7 @@ class HadoopDslPlugin extends BaseNamedScopeContainer implements Plugin<Project>
     project.extensions.add("addPropertySet", this.&addPropertySet);
     project.extensions.add("addWorkflow", this.&addWorkflow);
     project.extensions.add("addTrigger", this.&addTrigger);
+    project.extensions.add("carbonJob", this.&carbonJob);
     project.extensions.add("lookup", this.&lookup);
     project.extensions.add("lookupRef", this.&lookupRef);
     project.extensions.add("namespace", this.&namespace);

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/NamedScopeContainer.groovy

@@ -15,6 +15,7 @@
  */
 package com.linkedin.gradle.hadoopdsl;
 
+import com.linkedin.gradle.hadoopdsl.job.CarbonJob;
 import com.linkedin.gradle.hadoopdsl.job.CommandJob;
 import com.linkedin.gradle.hadoopdsl.job.GobblinJob;
 import com.linkedin.gradle.hadoopdsl.job.HadoopJavaJob;
@@ -559,4 +560,14 @@ interface NamedScopeContainer {
    */
   @HadoopDslMethod
   KubernetesJob kubernetesJob(String name, @DelegatesTo(KubernetesJob) Closure configure);
+
+  /**
+   * DSL carbonJob method. Creates a CarbonJob in scope with the given name and configuration.
+   *
+   * @param name The job name
+   * @param configure The configuration closure
+   * @return The new job
+   */
+  @HadoopDslMethod
+  CarbonJob carbonJob(String name, @DelegatesTo(CarbonJob) Closure configure);
 }

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/Visitor.groovy

@@ -15,6 +15,7 @@
  */
 package com.linkedin.gradle.hadoopdsl;
 
+import com.linkedin.gradle.hadoopdsl.job.CarbonJob;
 import com.linkedin.gradle.hadoopdsl.job.CommandJob;
 import com.linkedin.gradle.hadoopdsl.job.GobblinJob;
 import com.linkedin.gradle.hadoopdsl.job.HadoopJavaJob;
@@ -82,4 +83,5 @@ interface Visitor {
   void visitJob(KabootarJob job);
   void visitJob(WormholePushJob2 job);
   void visitJob(KubernetesJob job);
+  void visitJob(CarbonJob job);
 }

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/checker/RequiredFieldsChecker.groovy

@@ -17,6 +17,7 @@ package com.linkedin.gradle.hadoopdsl.checker;
 
 import com.linkedin.gradle.hadoopdsl.Properties;
 import com.linkedin.gradle.hadoopdsl.BaseStaticChecker;
+import com.linkedin.gradle.hadoopdsl.job.CarbonJob;
 import com.linkedin.gradle.hadoopdsl.job.CommandJob;
 import com.linkedin.gradle.hadoopdsl.job.HadoopJavaJob;
 import com.linkedin.gradle.hadoopdsl.job.HadoopShellJob;
@@ -290,6 +291,13 @@ class RequiredFieldsChecker extends BaseStaticChecker {
     }
   }
 
+  @Override
+  void visitJob(CarbonJob job) {
+    if (job.taskType == null) {
+      project.logger.lifecycle("RequiredFieldsChecker ERROR: CarbonJob ${job.name} must set task.type");
+      foundError = true;
+    }
+  }
 
   @Override
   void visitJob(HdfsToTeradataJob job) {

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/job/CarbonJob.groovy

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2018 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.linkedin.gradle.hadoopdsl.job;
+
+import com.linkedin.gradle.hadoopdsl.HadoopDslMethod;
+
+/**
+ * Job class for type=CarbonJob jobs. This job class is aimed at doing data life cycle management tasks,
+ * e.g. 1) file based distributed copy between HDFS/Azure clusters, 2) hive based dist-cp, 3) hdfs retention.
+ */
+class CarbonJob extends HadoopJavaJob {
+  // Required
+  String taskType;
+
+  /**
+   * Constructor for CarbonJob.
+   *
+   * @param jobName - The job name
+   */
+  CarbonJob(String jobName) {
+    super(jobName);
+    setJobProperty("type", "CarbonJob");
+  }
+
+  /**
+   * Clones the job.
+   *
+   * @return The cloned job
+   */
+  @Override
+  CarbonJob clone() {
+    return clone(new CarbonJob(name));
+  }
+
+  /**
+   * Helper method to set the properties on a cloned job.
+   *
+   * @param cloneJob - The job being cloned
+   * @return The cloned job
+   */
+  CarbonJob clone(CarbonJob cloneJob) {
+    cloneJob.taskType = taskType;
+    return ((CarbonJob) super.clone(cloneJob));
+  }
+
+  /**
+   * DSL taskType method causes task.type=value to be set in the job file.
+   *
+   * @param taskType - type of the task to perform. required parameters of the CarbonJob will depend on the task type.
+   */
+  @HadoopDslMethod
+  void taskType(String taskType) {
+    this.taskType = taskType;
+    setJobProperty("task.type", taskType);
+  }
+}