diff --git a/docker_run.sh b/docker_run.sh index ae8d18a..71ce32e 100755 --- a/docker_run.sh +++ b/docker_run.sh @@ -39,6 +39,7 @@ fi; if [ "${BUILD_ENV_ENV}" = "PRODUCTION" ]; then sleep 5 + go mod tidy go build -buildvcs=false ./batchiepatchie else diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 379d6a8..ac349a7 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -29,6 +29,7 @@ func ReadAllNoSessions(location string) ([]byte, error) { return nil, err } session := session.Must(session.NewSession(&aws.Config{Region: aws.String(region_loc)})) + s3s := s3.New(session) result, err := s3s.GetObject(&s3.GetObjectInput{ diff --git a/frontend/Dockerfile b/frontend/Dockerfile index e421d05..039640f 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,6 +1,7 @@ FROM node:10.15.3-stretch # AWS cli tools +RUN echo "deb http://archive.debian.org/debian stretch main" > /etc/apt/sources.list RUN apt-get update && apt-get install -y \ build-essential \ python \ diff --git a/go.mod b/go.mod index 0422905..0c9f497 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/BurntSushi/toml v1.2.0 - github.com/aws/aws-sdk-go v1.44.62 + github.com/aws/aws-sdk-go v1.45.4 github.com/bakatz/echo-logrusmiddleware v1.1.1 github.com/gorilla/websocket v1.5.0 github.com/jcftang/logentriesrus v0.0.0-20220725204439-b4dedce84d23 @@ -39,9 +39,9 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.27.1 // indirect diff --git a/go.sum b/go.sum index e3a8ce5..ec0627c 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,8 @@ github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/aws/aws-sdk-go v1.44.62 h1:N8qOPnBhl2ZCIFiqyB640Xt5CeX9D8CEVhG/Vj7jGJU= -github.com/aws/aws-sdk-go v1.44.62/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.45.4 h1:6B8oTYNEncxga8EV1C6Q4iJNnpDIqLEigy0v0oh2qYw= +github.com/aws/aws-sdk-go v1.45.4/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.0.0/go.mod h1:smfAbmpW+tcRVuNUjo3MOArSZmW72t62rkCzc2i0TWM= github.com/aws/aws-sdk-go-v2/config v1.0.0/go.mod h1:WysE/OpUgE37tjtmtJd8GXgT8s1euilE5XtUkRNUQ1w= github.com/aws/aws-sdk-go-v2/credentials v1.0.0/go.mod h1:/SvsiqBf509hG4Bddigr3NB12MIpfHhZapyBurJe8aY= @@ -616,6 +616,7 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/zenazn/goji v1.0.1/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= @@ -667,6 +668,7 @@ golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -702,6 +704,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -742,9 +745,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -758,6 +762,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -824,11 +829,15 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -838,8 +847,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -898,6 +908,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/handlers/handlers.go b/handlers/handlers.go index 23b7c1f..245ce9d 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -212,7 +212,7 @@ func (s *Server) FetchLogs(c echo.Context) error { continue } logStreams, err = svc.DescribeLogStreams(&cloudwatchlogs.DescribeLogStreamsInput{ - LogGroupName: aws.String(LOG_GROUP_NAME), + LogGroupName: job.LogGroupName, LogStreamNamePrefix: aws.String(*name), }) if err != nil || len(logStreams.LogStreams) <= 0 { @@ -247,14 +247,14 @@ func (s *Server) FetchLogs(c echo.Context) error { if nextToken != nil { logEvents, err2 = svc.GetLogEvents(&cloudwatchlogs.GetLogEventsInput{ - LogGroupName: aws.String(LOG_GROUP_NAME), + LogGroupName: job.LogGroupName, LogStreamName: logStreams.LogStreams[0].LogStreamName, StartFromHead: &startFromHead, NextToken: nextToken, }) } else { logEvents, err2 = svc.GetLogEvents(&cloudwatchlogs.GetLogEventsInput{ - LogGroupName: aws.String(LOG_GROUP_NAME), + LogGroupName: job.LogGroupName, LogStreamName: logStreams.LogStreams[0].LogStreamName, StartFromHead: &startFromHead, }) diff --git a/jobs/jobs.go b/jobs/jobs.go index 3075bf7..0cd590e 100644 --- a/jobs/jobs.go +++ b/jobs/jobs.go @@ -54,6 +54,7 @@ type Job struct { StatusReason *string `json:"status_reason"` RunStartTime *time.Time `json:"run_start_time"` ExitCode *int64 `json:"exitcode"` + LogGroupName *string `json:"log_group_name"` LogStreamName *string `json:"log_stream_name"` TerminationRequested bool `json:"termination_requested"` TaskARN *string `json:"task_arn"` diff --git a/jobs/postgres_store.go b/jobs/postgres_store.go index f755376..8b23c66 100644 --- a/jobs/postgres_store.go +++ b/jobs/postgres_store.go @@ -90,7 +90,8 @@ func (pq *postgreSQLStore) Find(opts *Options) ([]*Job, error) { log_stream_name, termination_requested, task_arn, - array_properties + array_properties, + log_group_name FROM jobs ` @@ -180,11 +181,10 @@ func (pq *postgreSQLStore) Find(opts *Options) ([]*Job, error) { allJobs := make([]*Job, 0) for rows.Next() { var job Job - if err := rows.Scan(&job.Id, &job.Name, &job.Status, &job.Description, &job.LastUpdated, &job.JobQueue, &job.Image, &job.CreatedAt, &job.StoppedAt, &job.VCpus, &job.Memory, &job.Timeout, &job.CommandLine, &job.StatusReason, &job.RunStartTime, &job.ExitCode, &job.LogStreamName, &job.TerminationRequested, &job.TaskARN, &job.ArrayProperties); err != nil { + if err := rows.Scan(&job.Id, &job.Name, &job.Status, &job.Description, &job.LastUpdated, &job.JobQueue, &job.Image, &job.CreatedAt, &job.StoppedAt, &job.VCpus, &job.Memory, &job.Timeout, &job.CommandLine, &job.StatusReason, &job.RunStartTime, &job.ExitCode, &job.LogStreamName, &job.TerminationRequested, &job.TaskARN, &job.ArrayProperties, &job.LogGroupName); err != nil { log.Warning(err) return nil, err } - allJobs = append(allJobs, &job) } return allJobs, nil @@ -249,7 +249,8 @@ func (pq *postgreSQLStore) FindOne(index string) (*Job, error) { ta.instance_id, ta.public_ip, ta.private_ip, - jobs.array_properties + jobs.array_properties, + log_group_name FROM jobs LEFT OUTER JOIN task_arns_to_instance_info ta ON ta.task_arn = jobs.task_arn @@ -270,7 +271,7 @@ func (pq *postgreSQLStore) FindOne(index string) (*Job, error) { job.StatusReason = &sr } - if err := rows.Scan(&job.Id, &job.Name, &job.Status, &job.Description, &job.LastUpdated, &job.JobQueue, &job.Image, &job.CreatedAt, &job.StoppedAt, &job.VCpus, &job.Memory, &job.Timeout, &job.CommandLine, &job.StatusReason, &job.RunStartTime, &job.ExitCode, &job.LogStreamName, &job.TerminationRequested, &job.TaskARN, &job.InstanceID, &job.PublicIP, &job.PrivateIP, &job.ArrayProperties); err != nil { + if err := rows.Scan(&job.Id, &job.Name, &job.Status, &job.Description, &job.LastUpdated, &job.JobQueue, &job.Image, &job.CreatedAt, &job.StoppedAt, &job.VCpus, &job.Memory, &job.Timeout, &job.CommandLine, &job.StatusReason, &job.RunStartTime, &job.ExitCode, &job.LogStreamName, &job.TerminationRequested, &job.TaskARN, &job.InstanceID, &job.PublicIP, &job.PrivateIP, &job.ArrayProperties, &job.LogGroupName); err != nil { log.Warning(err) return nil, err } @@ -425,15 +426,16 @@ func (pq *postgreSQLStore) Store(jobs []*Job) error { timeout, command_line, last_updated, - status_reason, - run_started_at, - exitcode, - log_stream_name, - task_arn, - array_properties) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) - on conflict (job_id) do update set status = $6, last_updated = $12, status_reason = $13, run_started_at = $14, exitcode = $15, log_stream_name = $16, task_arn = $17, array_properties = $18 - where jobs.status <> $6 or jobs.status_reason <> $13 or jobs.exitcode <> $15 or jobs.log_stream_name <> $16 or jobs.task_arn <> $17 or (jobs.task_arn is null and $17 is not null) or (jobs.log_stream_name is null and $16 is not null) or (jobs.status_reason is null and $13 is not null) or (jobs.exitcode is null and $15 is not null) + status_reason, + run_started_at, + exitcode, + log_stream_name, + task_arn, + array_properties, + log_group_name) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) + on conflict (job_id) do update set status = $6, last_updated = $12, status_reason = $13, run_started_at = $14, exitcode = $15, log_stream_name = $16, task_arn = $17, array_properties = $18, log_group_name = $19 + where jobs.status <> $6 or jobs.status_reason <> $13 or jobs.exitcode <> $15 or jobs.log_stream_name <> $16 or jobs.task_arn <> $17 or (jobs.task_arn is null and $17 is not null) or (jobs.log_stream_name is null and $16 is not null) or (jobs.status_reason is null and $13 is not null) or (jobs.exitcode is null and $15 is not null) or (jobs.log_group_name is null and $19 is not null) ` + extra_where_check result, err := transaction.Exec( query, @@ -454,7 +456,8 @@ func (pq *postgreSQLStore) Store(jobs []*Job) error { job.ExitCode, job.LogStreamName, job.TaskARN, - job.ArrayProperties) + job.ArrayProperties, + job.LogGroupName) if err != nil { log.Warning(err, ": ", job) return err @@ -480,15 +483,16 @@ func (pq *postgreSQLStore) Store(jobs []*Job) error { timeout, command_line, last_updated, - status_reason, - run_started_at, - exitcode, - log_stream_name, - task_arn, - array_properties) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) - on conflict (job_id) do update set status = $6, last_updated = $13, stopped_at = $8, status_reason = $14, run_started_at = $15, exitcode = $16, log_stream_name = $17, task_arn = $18, array_properties = $19 - where jobs.status <> $6 or jobs.status_reason <> $14 or jobs.exitcode <> $16 or jobs.log_stream_name <> $17 or jobs.task_arn <> $18 or (jobs.task_arn is null and $18 is not null) or (jobs.log_stream_name is null and $17 is not null) or (jobs.status_reason is null and $14 is not null) or (jobs.exitcode is null and $16 is not null) + status_reason, + run_started_at, + exitcode, + log_stream_name, + task_arn, + array_properties, + log_group_name) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) + on conflict (job_id) do update set status = $6, last_updated = $13, stopped_at = $8, status_reason = $14, run_started_at = $15, exitcode = $16, log_stream_name = $17, task_arn = $18, array_properties = $19, log_group_name = $20 + where jobs.status <> $6 or jobs.status_reason <> $14 or jobs.exitcode <> $16 or jobs.log_stream_name <> $17 or jobs.task_arn <> $18 or (jobs.task_arn is null and $18 is not null) or (jobs.log_stream_name is null and $17 is not null) or (jobs.status_reason is null and $14 is not null) or (jobs.exitcode is null and $16 is not null) or (jobs.log_group_name is null and $20 is not null) ` + extra_where_check result, err := transaction.Exec( query, @@ -510,7 +514,8 @@ func (pq *postgreSQLStore) Store(jobs []*Job) error { job.ExitCode, job.LogStreamName, job.TaskARN, - job.ArrayProperties) + job.ArrayProperties, + job.LogGroupName) if err != nil { log.Warning(err, ": ", job) return err @@ -906,6 +911,11 @@ func (pq *postgreSQLStore) UpdateTaskArnsInstanceIDs(ec2info map[string]Ec2Info, if !ok { continue } + public_ip := "" + if ec2_info.PublicIP != nil { + log.Info("PUBLIC IP NOT NILL", ec2_info.PublicIP) + public_ip = *ec2_info.PublicIP + } query := ` INSERT INTO task_arns_to_instance_info @@ -913,7 +923,7 @@ func (pq *postgreSQLStore) UpdateTaskArnsInstanceIDs(ec2info map[string]Ec2Info, VALUES ( $1, $2, $3, $4 ) ON CONFLICT DO NOTHING ` - _, err = transaction.Exec(query, task_arn, ec2instance, ec2_info.PublicIP, ec2_info.PrivateIP) + _, err = transaction.Exec(query, task_arn, ec2instance, public_ip, ec2_info.PrivateIP) if err != nil { log.Error("Cannot insert into task_arns_to_instance_info: ", err) return err diff --git a/migrations/00024_add_log_group_name.sql b/migrations/00024_add_log_group_name.sql new file mode 100644 index 0000000..6b50e5b --- /dev/null +++ b/migrations/00024_add_log_group_name.sql @@ -0,0 +1,8 @@ +-- +goose Up +-- SQL in this section is executed when the migration is applied. +ALTER TABLE jobs ADD COLUMN log_group_name TEXT default '/aws/batch/job'; + +-- +goose Down +-- SQL in this section is executed when the migration is rolled back. +ALTER TABLE jobs DROP COLUMN log_group_name; + diff --git a/syncer/batchsync.go b/syncer/batchsync.go index ce5e24a..34ebd90 100644 --- a/syncer/batchsync.go +++ b/syncer/batchsync.go @@ -156,6 +156,8 @@ func syncJobsStatus(storer jobs.Storer, status string, queues []string, job_summ var run_started_time *time.Time var log_stream_name *string var task_arn *string + var log_group_name *string + log_group_name = desc.Container.LogConfiguration.Options["awslogs-group"] if len(desc.Attempts) > 0 { last_attempt := desc.Attempts[len(desc.Attempts)-1] @@ -186,6 +188,7 @@ func syncJobsStatus(storer jobs.Storer, status string, queues []string, job_summ if log_stream_name == nil && desc.Container.LogStreamName != nil { var lsn = *desc.Container.LogStreamName log_stream_name = &lsn + } if (task_arn == nil || *task_arn == "") && desc.Container.TaskArn != nil { task_arn_c := *desc.Container.TaskArn @@ -239,6 +242,7 @@ func syncJobsStatus(storer jobs.Storer, status string, queues []string, job_summ StatusReason: &status_reason, RunStartTime: run_started_time, ExitCode: exit_code, + LogGroupName: log_group_name, LogStreamName: log_stream_name, TaskARN: task_arn, ArrayProperties: array_properties,