Merge branch 'master' of github.com:ovh/data-processing-spark-submit

Author: David Morin

README.md

@@ -77,6 +77,7 @@ Options:
    --repositories REPOSITORIES
                           Comma-delimited list of additional repositories (or resolvers in SBT)
    --properties-file      Read properties from the given file
+   --ttl                  Maximum "Time To Live" (in RFC3339 (duration) eg. "P1DT30H4S") of this job, after which it will be automatically terminated
    --help, -h             display this help and exit
                  
 

data_processing.go

@@ -61,6 +61,7 @@ type (
 		EngineVersion    string                `json:"engineVersion"`
 		EngineParameters []*JobEngineParameter `json:"engineParameters"`
 		Status           string                `json:"status"`
+		Ttl              string                `json:"ttl"`
 	}
 
 	// JobEngineParameter representation of JobEngineParameter in OVH API
@@ -89,6 +90,7 @@ type (
 		Engine           string                `json:"engine"`
 		Name             string                `json:"name"`
 		Region           string                `json:"region"`
+		Ttl              string                `json:"ttl,omitempty"`
 		EngineVersion    string                `json:"engineVersion"`
 		EngineParameters []*JobEngineParameter `json:"engineParameters"`
 	}

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/gabriel-vasile/mimetype v1.1.0
 	github.com/ncw/swift v1.0.52
 	github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014
+	github.com/peterhellberg/duration v0.0.0-20191119133758-ec6baeebcd10
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	gopkg.in/ini.v1 v1.55.0
 )

go.sum

@@ -18,6 +18,8 @@ github.com/ncw/swift v1.0.52 h1:ACF3JufDGgeKp/9mrDgQlEgS8kRYC4XKcuzj/8EJjQU=
 github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014 h1:37VE5TYj2m/FLA9SNr4z0+A0JefvTmR60Zwf8XSEV7c=
 github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014/go.mod h1:joRatxRJaZBsY3JAOEMcoOp05CnZzsx4scTxi95DHyQ=
+github.com/peterhellberg/duration v0.0.0-20191119133758-ec6baeebcd10 h1:Jf08dx6hxr6aNpHzUmYitsKGm6BmCFbwDGPb27/Boyc=
+github.com/peterhellberg/duration v0.0.0-20191119133758-ec6baeebcd10/go.mod h1:x5xjkH61fUOJVgCCDgqNzlJvdLXiYpmMzSuum2FBOaw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=

submit.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/Pallinder/go-randomdata"
 	"github.com/alexflint/go-arg"
+	"github.com/peterhellberg/duration"
 	"github.com/ovh/go-ovh/ovh"
 	"gopkg.in/ini.v1"
 )
@@ -42,6 +43,7 @@ var (
 		Packages               string   `arg:"--packages" help:"Comma-delimited list of Maven coordinates"`
 		Repositories           string   `arg:"--repositories" help:"Comma-delimited list of additional repositories (or resolvers in SBT)"`
 		PropertiesFile         string   `arg:"--properties-file" help:"Read properties from the given file"`
+		Ttl                    string   `arg:"--ttl" help:"Maximum \"Time To Live\" (in RFC3339 (duration) eg. "P1DT30H4S") of this job, after which it will be automatically terminated"`
 		File                   string   `arg:"positional,required"`
 		Parameters             []string `arg:"positional"`
 	}
@@ -157,6 +159,7 @@ func ParsArgs() *JobSubmit {
 		Region:           args.Region,
 		EngineVersion:    args.SparkVersion,
 		EngineParameters: []*JobEngineParameter{},
+		Ttl:              args.Ttl,
 	}
 
 	if args.JobName == "" {
@@ -255,6 +258,13 @@ func ParsArgs() *JobSubmit {
 		})
 	}
 
+	if args.Ttl != "" {
+        _, err := duration.Parse(args.Ttl);
+        if err != nil {
+            p.Fail("Invalid value for --ttl. It must be in RFC3339 (duration) format (i.e. PT30H for 30 hours)")
+        }
+    }
+
 	jobSubmit.EngineParameters = append(jobSubmit.EngineParameters, &JobEngineParameter{
 		Name:  ParameterExecutorNumber,
 		Value: args.ExecutorNum,

submit_test.go

@@ -28,6 +28,10 @@ func TestParsArgsJava(t *testing.T) {
 		t.Fail()
 	}
 
+	if job.Ttl != "" {
+		t.Fail()
+	}
+
 	if args.ProjectID != "1377b21260f05b410e4652445ac7c95b" {
 		t.Fail()
 	}
@@ -86,6 +90,10 @@ func TestParsArgsPython(t *testing.T) {
 		t.Fail()
 	}
 
+	if job.Ttl != "" {
+		t.Fail()
+	}
+
 	if args.ProjectID != "1377b21260f05b410e4652445ac7c95b" {
 		t.Fail()
 	}
@@ -125,7 +133,7 @@ func TestParsArgsPython(t *testing.T) {
 func TestParsArgs3(t *testing.T) {
 	// These are the args you would pass in on the command line
 	os.Setenv("OS_PROJECT_ID", "1377b21260f05b410e4652445ac7c95b")
-	os.Args = strings.Split("./ovh-spark-submit --class org.apache.spark.examples.SparkPi --driver-cores 1 --driver-memory 4G --driver-memoryOverhead 385M --executor-cores 1 --executor-memory 1G --executor-memoryOverhead 385M --num-executors 1 s3://odp/test/spark-examples.jar 1000", " ")
+	os.Args = strings.Split("./ovh-spark-submit --class org.apache.spark.examples.SparkPi --driver-cores 1 --driver-memory 4G --driver-memoryOverhead 385M --executor-cores 1 --executor-memory 1G --executor-memoryOverhead 385M --num-executors 1 --ttl P1DT30H4S s3://odp/test/spark-examples.jar 1000", " ")
 
 	job := ParsArgs()
 
@@ -144,6 +152,10 @@ func TestParsArgs3(t *testing.T) {
 		t.Fail()
 	}
 
+	if job.Ttl != "P1DT30H4S" {
+		t.Fail()
+	}
+
 	if args.ProjectID != "1377b21260f05b410e4652445ac7c95b" {
 		t.Fail()
 	}