From 86c99f99859deb89fbaf9cbc64c355cebba98bec Mon Sep 17 00:00:00 2001 From: mtfoley Date: Thu, 31 Aug 2023 20:08:28 -0400 Subject: [PATCH 01/17] feat: github orgs for /bake --- pkg/server/server.go | 81 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 9021314..04b3e83 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -6,8 +6,11 @@ import ( "database/sql" "encoding/json" "fmt" + "io" "log" "net/http" + "net/url" + "strings" "time" "github.com/go-git/go-git/v5" @@ -54,8 +57,46 @@ func (p PizzaOvenServer) Run(serverPort string) { } type reqData struct { - URL string `json:"url"` - Wait bool `json:"wait,omitempty"` + URL string `json:"url,omitempty"` + Wait bool `json:"wait,omitempty"` + Org string `json:"org,omitempty"` + Archives bool `json:"archives,omitempty"` +} + +type orgRepo struct { + URL string `json:"html_url"` + Archived bool `json:"archived"` +} +type orgRepoList []orgRepo + +func (p PizzaOvenServer) processOrg(orgUrlString string, processArchived bool) (orgRepoList, error) { + repoList := make(orgRepoList, 0) + orgUrl, err := url.Parse(orgUrlString) + if err != nil { + return repoList, err + } + if orgUrl.Hostname() == "github.com" { + orgApiUrlString := fmt.Sprintf("https://api.github.com/orgs%s/repos", orgUrl.Path) + res, err := http.Get(orgApiUrlString) + if err != nil { + return repoList, fmt.Errorf("Unable to request repo list from %s", orgApiUrlString) + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + json.Unmarshal(body, &repoList) + if processArchived { + return repoList, nil + } + filteredRepoList := make(orgRepoList, 0) + for _, repo := range repoList { + if !repo.Archived { + filteredRepoList = append(filteredRepoList, repo) + } + } + return filteredRepoList, nil + } else { + return repoList, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) + } } func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { @@ -75,6 +116,18 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) if data.Wait { + if data.Org != "" { + repoList, err := p.processOrg(data.Org, data.Archives) + if err != nil { + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + for _, repo := range repoList { + err = p.processRepository(repo.URL) + } + return + } err = p.processRepository(data.URL) if err != nil { p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) @@ -82,6 +135,30 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { return } } else { + if data.Org != "" { + repoList, err := p.processOrg(data.Org, data.Archives) + if err != nil { + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + errors := make([]string, 0) + for _, repo := range repoList { + go func(repo orgRepo) { + err = p.processRepository(repo.URL) + if err != nil { + errors = append(errors, fmt.Sprintf("Could not process repo: %v with error: %v", repo, err)) + } + + }(repo) + } + if len(errors) > 0 { + errorString := strings.Join(errors, "\n") + p.Logger.Error(errorString) + http.Error(w, "Could not process input", http.StatusInternalServerError) + } + return + } go func() { err = p.processRepository(data.URL) if err != nil { From a8c2b2a1f1d58a498e2d34353c7060d46ded1a73 Mon Sep 17 00:00:00 2001 From: mtfoley Date: Fri, 1 Sep 2023 09:24:32 -0400 Subject: [PATCH 02/17] use go-github client --- go.mod | 15 +++++--- go.sum | 39 ++++++++++++++++---- pkg/github/github_client.go | 71 +++++++++++++++++++++++++++++++++++++ pkg/server/server.go | 53 +++++++++++++-------------- 4 files changed, 139 insertions(+), 39 deletions(-) create mode 100644 pkg/github/github_client.go diff --git a/go.mod b/go.mod index 2ac5293..6b84ba5 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ go 1.20 require ( github.com/go-git/go-git/v5 v5.6.1 + github.com/google/go-github v17.0.0+incompatible + github.com/google/go-github/v54 v54.0.0 github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 go.uber.org/zap v1.24.0 - golang.org/x/sys v0.5.0 + golang.org/x/sys v0.11.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -15,10 +17,12 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect - github.com/cloudflare/circl v1.1.0 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect @@ -28,7 +32,10 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.6.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index c6820d7..ae3af30 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,9 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -29,8 +30,20 @@ github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlK github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-github/v54 v54.0.0 h1:OZdXwow4EAD5jEo5qg+dGFH2DpkyZvVsAehjvJuUL/c= +github.com/google/go-github/v54 v54.0.0/go.mod h1:Sw1LXWHhXRZtzJ9LI5fyJg9wbQzYvFhW8W5P2yaAQ7s= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -86,10 +99,12 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -97,8 +112,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -117,26 +135,35 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/github/github_client.go b/pkg/github/github_client.go new file mode 100644 index 0000000..419a44e --- /dev/null +++ b/pkg/github/github_client.go @@ -0,0 +1,71 @@ +package github + +import ( + "context" + "net/http" + + "github.com/google/go-github/v54/github" +) + +type GithubClient struct { + client *github.Client +} + +func NewTokenClient(token string) *GithubClient { + ctx := context.Background() + s := &GithubClient{ + client: github.NewTokenClient(ctx, token), + } + return s +} + +func NewClient(httpClient *http.Client) *GithubClient { + s := &GithubClient{ + client: github.NewClient(httpClient), + } + return s +} + +func (s *GithubClient) ListReposByOrg(org string) ([]*github.Repository, error) { + ctx := context.Background() + opt := &github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + // get all pages of results + var allRepos []*github.Repository + for { + repos, resp, err := s.client.Repositories.ListByOrg(ctx, org, opt) + if err != nil { + return allRepos, err + } + allRepos = append(allRepos, repos...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return allRepos, nil +} + +func FilterArchivedRepos(repos []*github.Repository) []*github.Repository { + var filteredRepos []*github.Repository + var falseVal bool = false + for _, repo := range repos { + if repo.Archived != &falseVal { + filteredRepos = append(filteredRepos, repo) + } + } + return filteredRepos +} + +func GetRepoHTMLUrls(repos []*github.Repository) []string { + var urls []string + for _, repo := range repos { + + htmlUrl := repo.GetHTMLURL() + if htmlUrl != "" { + urls = append(urls, htmlUrl) + } + } + return urls +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 04b3e83..f60cef3 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -6,7 +6,6 @@ import ( "database/sql" "encoding/json" "fmt" - "io" "log" "net/http" "net/url" @@ -18,6 +17,7 @@ import ( "go.uber.org/zap" "github.com/open-sauced/pizza/oven/pkg/database" + "github.com/open-sauced/pizza/oven/pkg/github" "github.com/open-sauced/pizza/oven/pkg/insights" "github.com/open-sauced/pizza/oven/pkg/providers" ) @@ -69,33 +69,28 @@ type orgRepo struct { } type orgRepoList []orgRepo -func (p PizzaOvenServer) processOrg(orgUrlString string, processArchived bool) (orgRepoList, error) { - repoList := make(orgRepoList, 0) +func (p PizzaOvenServer) processOrg(orgUrlString string, processArchived bool) ([]string, error) { + + var htmlUrls []string orgUrl, err := url.Parse(orgUrlString) if err != nil { - return repoList, err + return htmlUrls, err } if orgUrl.Hostname() == "github.com" { - orgApiUrlString := fmt.Sprintf("https://api.github.com/orgs%s/repos", orgUrl.Path) - res, err := http.Get(orgApiUrlString) + client := github.NewClient(nil) + orgName := strings.Trim(orgUrl.Path, "/") + repos, err := client.ListReposByOrg(orgName) if err != nil { - return repoList, fmt.Errorf("Unable to request repo list from %s", orgApiUrlString) - } - body, err := io.ReadAll(res.Body) - res.Body.Close() - json.Unmarshal(body, &repoList) - if processArchived { - return repoList, nil + return htmlUrls, fmt.Errorf("Error encountered fetching repositories, org: %s with error: %s ", orgName, err) } - filteredRepoList := make(orgRepoList, 0) - for _, repo := range repoList { - if !repo.Archived { - filteredRepoList = append(filteredRepoList, repo) - } + if !processArchived { + repos = github.FilterArchivedRepos(repos) } - return filteredRepoList, nil + htmlUrls = github.GetRepoHTMLUrls(repos) + + return htmlUrls, nil } else { - return repoList, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) + return htmlUrls, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) } } @@ -117,14 +112,14 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) if data.Wait { if data.Org != "" { - repoList, err := p.processOrg(data.Org, data.Archives) + cloneUrls, err := p.processOrg(data.Org, data.Archives) if err != nil { p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } - for _, repo := range repoList { - err = p.processRepository(repo.URL) + for _, cloneUrl := range cloneUrls { + err = p.processRepository(cloneUrl) } return } @@ -136,21 +131,21 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { } } else { if data.Org != "" { - repoList, err := p.processOrg(data.Org, data.Archives) + cloneUrls, err := p.processOrg(data.Org, data.Archives) if err != nil { p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } errors := make([]string, 0) - for _, repo := range repoList { - go func(repo orgRepo) { - err = p.processRepository(repo.URL) + for _, cloneUrl := range cloneUrls { + go func(cloneUrl string) { + err = p.processRepository(cloneUrl) if err != nil { - errors = append(errors, fmt.Sprintf("Could not process repo: %v with error: %v", repo, err)) + errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", cloneUrl, err)) } - }(repo) + }(cloneUrl) } if len(errors) > 0 { errorString := strings.Join(errors, "\n") From 3efcb6ea079b765bc59c4b1a5b9afb1cc584c9f5 Mon Sep 17 00:00:00 2001 From: mtfoley Date: Fri, 1 Sep 2023 12:27:34 -0400 Subject: [PATCH 03/17] rename to htmlUrls --- pkg/server/server.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index f60cef3..35d1653 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -131,21 +131,21 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { } } else { if data.Org != "" { - cloneUrls, err := p.processOrg(data.Org, data.Archives) + htmlUrls, err := p.processOrg(data.Org, data.Archives) if err != nil { p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } errors := make([]string, 0) - for _, cloneUrl := range cloneUrls { - go func(cloneUrl string) { - err = p.processRepository(cloneUrl) + for _, htmlUrl := range htmlUrls { + go func(htmlUrl string) { + err = p.processRepository(htmlUrl) if err != nil { - errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", cloneUrl, err)) + errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", htmlUrl, err)) } - }(cloneUrl) + }(htmlUrl) } if len(errors) > 0 { errorString := strings.Join(errors, "\n") From 4a5618fe154ee61c45f2097a6377b8c9e76ee4b6 Mon Sep 17 00:00:00 2001 From: mtfoley Date: Mon, 4 Sep 2023 22:49:22 -0400 Subject: [PATCH 04/17] added unit tests --- pkg/github/github_client.go | 3 +- pkg/github/github_client_test.go | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 pkg/github/github_client_test.go diff --git a/pkg/github/github_client.go b/pkg/github/github_client.go index 419a44e..f7ae269 100644 --- a/pkg/github/github_client.go +++ b/pkg/github/github_client.go @@ -49,9 +49,8 @@ func (s *GithubClient) ListReposByOrg(org string) ([]*github.Repository, error) func FilterArchivedRepos(repos []*github.Repository) []*github.Repository { var filteredRepos []*github.Repository - var falseVal bool = false for _, repo := range repos { - if repo.Archived != &falseVal { + if !*repo.Archived { filteredRepos = append(filteredRepos, repo) } } diff --git a/pkg/github/github_client_test.go b/pkg/github/github_client_test.go new file mode 100644 index 0000000..6437f4e --- /dev/null +++ b/pkg/github/github_client_test.go @@ -0,0 +1,50 @@ +package github + +import ( + "fmt" + "testing" + + "github.com/google/go-github/v54/github" +) + +func createRepoList(org string, totalCount int, archiveCount int) []*github.Repository { + var repoList []*github.Repository + for i := 0; i < totalCount; i++ { + var archiveVal = (i < archiveCount) + var htmlUrl = fmt.Sprintf("https://github.com/%s/repo-%d", org, i) + repoList = append(repoList, &github.Repository{ + Archived: &archiveVal, + HTMLURL: &htmlUrl, + }) + } + return repoList +} + +func TestFilterArchivedRepos(t *testing.T) { + totalCount := 10 + archiveCount := 2 + filteredCountExpected := (totalCount - archiveCount) + originalRepoList := createRepoList("open-sauced", totalCount, archiveCount) + filteredRepoList := FilterArchivedRepos(originalRepoList) + if len(filteredRepoList) != filteredCountExpected { + t.Errorf("FilteredArchivedRepos() should yield %d items; got %d", filteredCountExpected, len(filteredRepoList)) + } +} + +func TestGetRepoHTMLUrls(t *testing.T) { + expected := []string{ + "https://github.com/open-sauced/repo-0", + "https://github.com/open-sauced/repo-1", + "https://github.com/open-sauced/repo-2", + } + repos := createRepoList("open-sauced", 3, 0) + got := GetRepoHTMLUrls(repos) + if len(expected) != len(got) { + t.Errorf("GetRepoHTMLUrls() should yield count matching input") + } + for i := 0; i < len(got); i++ { + if got[i] != expected[i] { + t.Errorf(`Expected GetRepoHTMLUrls()[%d] to yield "%s"; got "%s"`, i, expected[i], got[i]) + } + } +} From bfccd906fdbe9285d0d00531cee58ba56198164b Mon Sep 17 00:00:00 2001 From: mtfoley Date: Mon, 4 Sep 2023 22:59:25 -0400 Subject: [PATCH 05/17] update README --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 9114ad4..c97eceb 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,26 @@ curl -d '{"url":"https://github.com/open-sauced/insights"}' \ -X POST http://localhost:8080/bake ``` +Alternatively, the bake route can accept a `POST` request with a body that specifies +a GitHub organization, and the server will query the GitHub REST API to get a list of +repositories for the organization and process these repositories. + +Example: + +```bash +curl -d '{"org":"https://github.com/open-sauced"}' \ + -H "Content-Type: application/json" \ + -X POST http://localhost:8080/bake +``` + +By default, the server will filter out repositories that are archived before processing +them. However, this can be overridden in the `POST` request in this way: + +```bash +curl -d '{"org":"https://github.com/open-sauced", "archives": true}' \ + -H "Content-Type: application/json" \ + -X POST http://localhost:8080/bake +``` ## 🖥️ Local development From d7ef35a9139f81b9d4992a605fd4923e9c9c70eb Mon Sep 17 00:00:00 2001 From: mtfoley Date: Thu, 31 Aug 2023 20:08:28 -0400 Subject: [PATCH 06/17] feat: github orgs for /bake --- pkg/server/server.go | 82 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 551ec52..9f7addf 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -6,8 +6,10 @@ import ( "database/sql" "encoding/json" "fmt" + "io" "log" "net/http" + "net/url" "strings" "sync/atomic" "time" @@ -63,8 +65,46 @@ func (p PizzaOvenServer) Run(serverPort string) { } type reqData struct { - URL string `json:"url"` - Wait bool `json:"wait,omitempty"` + URL string `json:"url,omitempty"` + Wait bool `json:"wait,omitempty"` + Org string `json:"org,omitempty"` + Archives bool `json:"archives,omitempty"` +} + +type orgRepo struct { + URL string `json:"html_url"` + Archived bool `json:"archived"` +} +type orgRepoList []orgRepo + +func (p PizzaOvenServer) processOrg(orgUrlString string, processArchived bool) (orgRepoList, error) { + repoList := make(orgRepoList, 0) + orgUrl, err := url.Parse(orgUrlString) + if err != nil { + return repoList, err + } + if orgUrl.Hostname() == "github.com" { + orgApiUrlString := fmt.Sprintf("https://api.github.com/orgs%s/repos", orgUrl.Path) + res, err := http.Get(orgApiUrlString) + if err != nil { + return repoList, fmt.Errorf("Unable to request repo list from %s", orgApiUrlString) + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + json.Unmarshal(body, &repoList) + if processArchived { + return repoList, nil + } + filteredRepoList := make(orgRepoList, 0) + for _, repo := range repoList { + if !repo.Archived { + filteredRepoList = append(filteredRepoList, repo) + } + } + return filteredRepoList, nil + } else { + return repoList, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) + } } func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { @@ -112,13 +152,49 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) if data.Wait { - err = p.processRepository(repoURLendpoint.String()) + if data.Org != "" { + repoList, err := p.processOrg(data.Org, data.Archives) + if err != nil { + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + for _, repo := range repoList { + err = p.processRepository(repo.URL) + } + return + } + err = p.processRepository(data.URL) if err != nil { p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } } else { + if data.Org != "" { + repoList, err := p.processOrg(data.Org, data.Archives) + if err != nil { + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + errors := make([]string, 0) + for _, repo := range repoList { + go func(repo orgRepo) { + err = p.processRepository(repo.URL) + if err != nil { + errors = append(errors, fmt.Sprintf("Could not process repo: %v with error: %v", repo, err)) + } + + }(repo) + } + if len(errors) > 0 { + errorString := strings.Join(errors, "\n") + p.Logger.Error(errorString) + http.Error(w, "Could not process input", http.StatusInternalServerError) + } + return + } go func() { err = p.processRepository(repoURLendpoint.String()) if err != nil { From 69f639f20373c3bec0816a48383ed7f9390d056f Mon Sep 17 00:00:00 2001 From: mtfoley Date: Fri, 1 Sep 2023 09:24:32 -0400 Subject: [PATCH 07/17] use go-github client --- go.mod | 14 +++++--- go.sum | 37 +++++++++++++++---- pkg/github/github_client.go | 71 +++++++++++++++++++++++++++++++++++++ pkg/server/server.go | 53 +++++++++++++-------------- 4 files changed, 136 insertions(+), 39 deletions(-) create mode 100644 pkg/github/github_client.go diff --git a/go.mod b/go.mod index f7a7d99..cb336a4 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,12 @@ go 1.20 require ( github.com/go-git/go-git/v5 v5.6.1 + github.com/google/go-github/v54 v54.0.0 github.com/google/uuid v1.3.1 github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 go.uber.org/zap v1.24.0 - golang.org/x/sys v0.5.0 + golang.org/x/sys v0.11.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -16,10 +17,12 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect - github.com/cloudflare/circl v1.1.0 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect @@ -29,7 +32,10 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.6.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index cdb5eef..2a08d9a 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,9 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -29,8 +30,18 @@ github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlK github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v54 v54.0.0 h1:OZdXwow4EAD5jEo5qg+dGFH2DpkyZvVsAehjvJuUL/c= +github.com/google/go-github/v54 v54.0.0/go.mod h1:Sw1LXWHhXRZtzJ9LI5fyJg9wbQzYvFhW8W5P2yaAQ7s= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= @@ -88,10 +99,12 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -99,8 +112,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -119,26 +135,35 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/github/github_client.go b/pkg/github/github_client.go new file mode 100644 index 0000000..419a44e --- /dev/null +++ b/pkg/github/github_client.go @@ -0,0 +1,71 @@ +package github + +import ( + "context" + "net/http" + + "github.com/google/go-github/v54/github" +) + +type GithubClient struct { + client *github.Client +} + +func NewTokenClient(token string) *GithubClient { + ctx := context.Background() + s := &GithubClient{ + client: github.NewTokenClient(ctx, token), + } + return s +} + +func NewClient(httpClient *http.Client) *GithubClient { + s := &GithubClient{ + client: github.NewClient(httpClient), + } + return s +} + +func (s *GithubClient) ListReposByOrg(org string) ([]*github.Repository, error) { + ctx := context.Background() + opt := &github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + // get all pages of results + var allRepos []*github.Repository + for { + repos, resp, err := s.client.Repositories.ListByOrg(ctx, org, opt) + if err != nil { + return allRepos, err + } + allRepos = append(allRepos, repos...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return allRepos, nil +} + +func FilterArchivedRepos(repos []*github.Repository) []*github.Repository { + var filteredRepos []*github.Repository + var falseVal bool = false + for _, repo := range repos { + if repo.Archived != &falseVal { + filteredRepos = append(filteredRepos, repo) + } + } + return filteredRepos +} + +func GetRepoHTMLUrls(repos []*github.Repository) []string { + var urls []string + for _, repo := range repos { + + htmlUrl := repo.GetHTMLURL() + if htmlUrl != "" { + urls = append(urls, htmlUrl) + } + } + return urls +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 9f7addf..d8649af 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -6,7 +6,6 @@ import ( "database/sql" "encoding/json" "fmt" - "io" "log" "net/http" "net/url" @@ -22,6 +21,7 @@ import ( "github.com/open-sauced/pizza/oven/pkg/common" "github.com/open-sauced/pizza/oven/pkg/database" + "github.com/open-sauced/pizza/oven/pkg/github" "github.com/open-sauced/pizza/oven/pkg/insights" "github.com/open-sauced/pizza/oven/pkg/providers" ) @@ -77,33 +77,28 @@ type orgRepo struct { } type orgRepoList []orgRepo -func (p PizzaOvenServer) processOrg(orgUrlString string, processArchived bool) (orgRepoList, error) { - repoList := make(orgRepoList, 0) +func (p PizzaOvenServer) processOrg(orgUrlString string, processArchived bool) ([]string, error) { + + var htmlUrls []string orgUrl, err := url.Parse(orgUrlString) if err != nil { - return repoList, err + return htmlUrls, err } if orgUrl.Hostname() == "github.com" { - orgApiUrlString := fmt.Sprintf("https://api.github.com/orgs%s/repos", orgUrl.Path) - res, err := http.Get(orgApiUrlString) + client := github.NewClient(nil) + orgName := strings.Trim(orgUrl.Path, "/") + repos, err := client.ListReposByOrg(orgName) if err != nil { - return repoList, fmt.Errorf("Unable to request repo list from %s", orgApiUrlString) - } - body, err := io.ReadAll(res.Body) - res.Body.Close() - json.Unmarshal(body, &repoList) - if processArchived { - return repoList, nil + return htmlUrls, fmt.Errorf("Error encountered fetching repositories, org: %s with error: %s ", orgName, err) } - filteredRepoList := make(orgRepoList, 0) - for _, repo := range repoList { - if !repo.Archived { - filteredRepoList = append(filteredRepoList, repo) - } + if !processArchived { + repos = github.FilterArchivedRepos(repos) } - return filteredRepoList, nil + htmlUrls = github.GetRepoHTMLUrls(repos) + + return htmlUrls, nil } else { - return repoList, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) + return htmlUrls, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) } } @@ -153,14 +148,14 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) if data.Wait { if data.Org != "" { - repoList, err := p.processOrg(data.Org, data.Archives) + cloneUrls, err := p.processOrg(data.Org, data.Archives) if err != nil { p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } - for _, repo := range repoList { - err = p.processRepository(repo.URL) + for _, cloneUrl := range cloneUrls { + err = p.processRepository(cloneUrl) } return } @@ -172,21 +167,21 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { } } else { if data.Org != "" { - repoList, err := p.processOrg(data.Org, data.Archives) + cloneUrls, err := p.processOrg(data.Org, data.Archives) if err != nil { p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } errors := make([]string, 0) - for _, repo := range repoList { - go func(repo orgRepo) { - err = p.processRepository(repo.URL) + for _, cloneUrl := range cloneUrls { + go func(cloneUrl string) { + err = p.processRepository(cloneUrl) if err != nil { - errors = append(errors, fmt.Sprintf("Could not process repo: %v with error: %v", repo, err)) + errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", cloneUrl, err)) } - }(repo) + }(cloneUrl) } if len(errors) > 0 { errorString := strings.Join(errors, "\n") From 51ce8d105efdb5b910143a7a83fb9e3977142f70 Mon Sep 17 00:00:00 2001 From: mtfoley Date: Fri, 1 Sep 2023 12:27:34 -0400 Subject: [PATCH 08/17] rename to htmlUrls --- pkg/server/server.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index d8649af..21ce3be 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -167,21 +167,21 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { } } else { if data.Org != "" { - cloneUrls, err := p.processOrg(data.Org, data.Archives) + htmlUrls, err := p.processOrg(data.Org, data.Archives) if err != nil { p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } errors := make([]string, 0) - for _, cloneUrl := range cloneUrls { - go func(cloneUrl string) { - err = p.processRepository(cloneUrl) + for _, htmlUrl := range htmlUrls { + go func(htmlUrl string) { + err = p.processRepository(htmlUrl) if err != nil { - errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", cloneUrl, err)) + errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", htmlUrl, err)) } - }(cloneUrl) + }(htmlUrl) } if len(errors) > 0 { errorString := strings.Join(errors, "\n") From 4b83e21624b21604caa3c49b3d6366273d658140 Mon Sep 17 00:00:00 2001 From: mtfoley Date: Mon, 4 Sep 2023 22:49:22 -0400 Subject: [PATCH 09/17] added unit tests --- pkg/github/github_client.go | 3 +- pkg/github/github_client_test.go | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 pkg/github/github_client_test.go diff --git a/pkg/github/github_client.go b/pkg/github/github_client.go index 419a44e..f7ae269 100644 --- a/pkg/github/github_client.go +++ b/pkg/github/github_client.go @@ -49,9 +49,8 @@ func (s *GithubClient) ListReposByOrg(org string) ([]*github.Repository, error) func FilterArchivedRepos(repos []*github.Repository) []*github.Repository { var filteredRepos []*github.Repository - var falseVal bool = false for _, repo := range repos { - if repo.Archived != &falseVal { + if !*repo.Archived { filteredRepos = append(filteredRepos, repo) } } diff --git a/pkg/github/github_client_test.go b/pkg/github/github_client_test.go new file mode 100644 index 0000000..6437f4e --- /dev/null +++ b/pkg/github/github_client_test.go @@ -0,0 +1,50 @@ +package github + +import ( + "fmt" + "testing" + + "github.com/google/go-github/v54/github" +) + +func createRepoList(org string, totalCount int, archiveCount int) []*github.Repository { + var repoList []*github.Repository + for i := 0; i < totalCount; i++ { + var archiveVal = (i < archiveCount) + var htmlUrl = fmt.Sprintf("https://github.com/%s/repo-%d", org, i) + repoList = append(repoList, &github.Repository{ + Archived: &archiveVal, + HTMLURL: &htmlUrl, + }) + } + return repoList +} + +func TestFilterArchivedRepos(t *testing.T) { + totalCount := 10 + archiveCount := 2 + filteredCountExpected := (totalCount - archiveCount) + originalRepoList := createRepoList("open-sauced", totalCount, archiveCount) + filteredRepoList := FilterArchivedRepos(originalRepoList) + if len(filteredRepoList) != filteredCountExpected { + t.Errorf("FilteredArchivedRepos() should yield %d items; got %d", filteredCountExpected, len(filteredRepoList)) + } +} + +func TestGetRepoHTMLUrls(t *testing.T) { + expected := []string{ + "https://github.com/open-sauced/repo-0", + "https://github.com/open-sauced/repo-1", + "https://github.com/open-sauced/repo-2", + } + repos := createRepoList("open-sauced", 3, 0) + got := GetRepoHTMLUrls(repos) + if len(expected) != len(got) { + t.Errorf("GetRepoHTMLUrls() should yield count matching input") + } + for i := 0; i < len(got); i++ { + if got[i] != expected[i] { + t.Errorf(`Expected GetRepoHTMLUrls()[%d] to yield "%s"; got "%s"`, i, expected[i], got[i]) + } + } +} From f00484e83719ef04672c8b3dfb15e649bac50679 Mon Sep 17 00:00:00 2001 From: mtfoley Date: Mon, 4 Sep 2023 22:59:25 -0400 Subject: [PATCH 10/17] update README --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 9114ad4..c97eceb 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,26 @@ curl -d '{"url":"https://github.com/open-sauced/insights"}' \ -X POST http://localhost:8080/bake ``` +Alternatively, the bake route can accept a `POST` request with a body that specifies +a GitHub organization, and the server will query the GitHub REST API to get a list of +repositories for the organization and process these repositories. + +Example: + +```bash +curl -d '{"org":"https://github.com/open-sauced"}' \ + -H "Content-Type: application/json" \ + -X POST http://localhost:8080/bake +``` + +By default, the server will filter out repositories that are archived before processing +them. However, this can be overridden in the `POST` request in this way: + +```bash +curl -d '{"org":"https://github.com/open-sauced", "archives": true}' \ + -H "Content-Type: application/json" \ + -X POST http://localhost:8080/bake +``` ## 🖥️ Local development From 5cf6630dd780eb480a5f6f6af7e8445a2a2f870d Mon Sep 17 00:00:00 2001 From: mtfoley Date: Tue, 5 Sep 2023 20:54:38 -0400 Subject: [PATCH 11/17] restore uuid module import --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 4759160..cb336a4 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/go-git/go-git/v5 v5.6.1 github.com/google/go-github/v54 v54.0.0 + github.com/google/uuid v1.3.1 github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 go.uber.org/zap v1.24.0 diff --git a/go.sum b/go.sum index 998f7ae..2a08d9a 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/google/go-github/v54 v54.0.0 h1:OZdXwow4EAD5jEo5qg+dGFH2DpkyZvVsAehjv github.com/google/go-github/v54 v54.0.0/go.mod h1:Sw1LXWHhXRZtzJ9LI5fyJg9wbQzYvFhW8W5P2yaAQ7s= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= From 532f6e1796b42e483ebd79c343b8049a14ac694f Mon Sep 17 00:00:00 2001 From: mtfoley Date: Tue, 5 Sep 2023 21:11:15 -0400 Subject: [PATCH 12/17] wip: refactor to separate endpoint --- pkg/server/server.go | 60 ++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 21ce3be..49951cf 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -59,7 +59,8 @@ func (p PizzaOvenServer) Run(serverPort string) { //nolint:errcheck defer p.Logger.Sync() p.Logger.Infof("Starting server on port %s", serverPort) - http.HandleFunc("/bake", p.handleRequest) + http.HandleFunc("/bake", p.handleBakeRequest) + http.HandleFunc("/bake-org", p.handleBakeOrgRequest) http.HandleFunc("/ping", p.pingHandler) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", serverPort), nil)) } @@ -102,7 +103,7 @@ func (p PizzaOvenServer) processOrg(orgUrlString string, processArchived bool) ( } } -func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { +func (p PizzaOvenServer) handleBakeRequest(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { p.Logger.Errorf("Received request with invalid method: %v", r.Body) http.Error(w, "Invalid request method, expected post", http.StatusMethodNotAllowed) @@ -145,6 +146,40 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { return } + w.WriteHeader(http.StatusAccepted) + if data.Wait { + err = p.processRepository(data.URL) + if err != nil { + p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + } else { + go func() { + err = p.processRepository(repoURLendpoint.String()) + if err != nil { + p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + }() + } +} + +func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + p.Logger.Errorf("Received request with invalid method: %v", r.Body) + http.Error(w, "Invalid request method, expected post", http.StatusMethodNotAllowed) + return + } + + var data reqData + err := json.NewDecoder(r.Body).Decode(&data) + if err != nil { + p.Logger.Errorf("Could not decode request json body: %v with error: %v", r.Body, err) + http.Error(w, "Could not decode request body", http.StatusBadRequest) + return + } w.WriteHeader(http.StatusAccepted) if data.Wait { if data.Org != "" { @@ -154,15 +189,16 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { http.Error(w, "Could not process input", http.StatusInternalServerError) return } + errors := make([]string, 0) for _, cloneUrl := range cloneUrls { err = p.processRepository(cloneUrl) } - return - } - err = p.processRepository(data.URL) - if err != nil { - p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) - http.Error(w, "Could not process input", http.StatusInternalServerError) + if len(errors) > 0 { + errorString := strings.Join(errors, "\n") + p.Logger.Error(errorString) + http.Error(w, "Could not process input", http.StatusInternalServerError) + } + return } } else { @@ -190,14 +226,6 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { } return } - go func() { - err = p.processRepository(repoURLendpoint.String()) - if err != nil { - p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) - http.Error(w, "Could not process input", http.StatusInternalServerError) - return - } - }() } } From 479a7b2eb3c56fa231b17421f469b7e820e98c3a Mon Sep 17 00:00:00 2001 From: mtfoley Date: Sat, 9 Sep 2023 09:22:11 -0400 Subject: [PATCH 13/17] address linter comments --- README.md | 15 ++-- pkg/{github => clients}/github_client.go | 24 +++---- pkg/{github => clients}/github_client_test.go | 10 +-- pkg/server/server.go | 68 ++++++++++--------- 4 files changed, 61 insertions(+), 56 deletions(-) rename pkg/{github => clients}/github_client.go (63%) rename pkg/{github => clients}/github_client_test.go (86%) diff --git a/README.md b/README.md index c97eceb..3d5ff97 100644 --- a/README.md +++ b/README.md @@ -45,25 +45,26 @@ curl -d '{"url":"https://github.com/open-sauced/insights"}' \ -X POST http://localhost:8080/bake ``` -Alternatively, the bake route can accept a `POST` request with a body that specifies -a GitHub organization, and the server will query the GitHub REST API to get a list of -repositories for the organization and process these repositories. +Alternatively, the /bake-org route can accept a `POST` request with a body that +specifies a GitHub organization, and the server will query the GitHub REST API to +get a list of repositories for the organization and process these repositories. Example: ```bash curl -d '{"org":"https://github.com/open-sauced"}' \ -H "Content-Type: application/json" \ - -X POST http://localhost:8080/bake + -X POST http://localhost:8080/bake-org ``` -By default, the server will filter out repositories that are archived before processing -them. However, this can be overridden in the `POST` request in this way: +By default, the server will filter out repositories that are archived before +processing them. However, this can be overridden in the `POST` request in this +way: ```bash curl -d '{"org":"https://github.com/open-sauced", "archives": true}' \ -H "Content-Type: application/json" \ - -X POST http://localhost:8080/bake + -X POST http://localhost:8080/bake-org ``` ## 🖥️ Local development diff --git a/pkg/github/github_client.go b/pkg/clients/github_client.go similarity index 63% rename from pkg/github/github_client.go rename to pkg/clients/github_client.go index f7ae269..04964e2 100644 --- a/pkg/github/github_client.go +++ b/pkg/clients/github_client.go @@ -1,4 +1,4 @@ -package github +package clients import ( "context" @@ -7,26 +7,26 @@ import ( "github.com/google/go-github/v54/github" ) -type GithubClient struct { +type GithubApiClient struct { client *github.Client } -func NewTokenClient(token string) *GithubClient { +func NewGithubTokenClient(token string) *GithubApiClient { ctx := context.Background() - s := &GithubClient{ + s := &GithubApiClient{ client: github.NewTokenClient(ctx, token), } return s } -func NewClient(httpClient *http.Client) *GithubClient { - s := &GithubClient{ +func NewGithubClient(httpClient *http.Client) *GithubApiClient { + s := &GithubApiClient{ client: github.NewClient(httpClient), } return s } -func (s *GithubClient) ListReposByOrg(org string) ([]*github.Repository, error) { +func (s *GithubApiClient) ListReposByOrg(org string) ([]*github.Repository, error) { ctx := context.Background() opt := &github.RepositoryListByOrgOptions{ ListOptions: github.ListOptions{PerPage: 100}, @@ -47,7 +47,7 @@ func (s *GithubClient) ListReposByOrg(org string) ([]*github.Repository, error) return allRepos, nil } -func FilterArchivedRepos(repos []*github.Repository) []*github.Repository { +func FilterGithubArchivedRepos(repos []*github.Repository) []*github.Repository { var filteredRepos []*github.Repository for _, repo := range repos { if !*repo.Archived { @@ -57,13 +57,13 @@ func FilterArchivedRepos(repos []*github.Repository) []*github.Repository { return filteredRepos } -func GetRepoHTMLUrls(repos []*github.Repository) []string { +func GetGithubRepoHTMLUrls(repos []*github.Repository) []string { var urls []string for _, repo := range repos { - htmlUrl := repo.GetHTMLURL() - if htmlUrl != "" { - urls = append(urls, htmlUrl) + htmlURL := repo.GetHTMLURL() + if htmlURL != "" { + urls = append(urls, htmlURL) } } return urls diff --git a/pkg/github/github_client_test.go b/pkg/clients/github_client_test.go similarity index 86% rename from pkg/github/github_client_test.go rename to pkg/clients/github_client_test.go index 6437f4e..e7f488f 100644 --- a/pkg/github/github_client_test.go +++ b/pkg/clients/github_client_test.go @@ -1,4 +1,4 @@ -package github +package clients import ( "fmt" @@ -11,10 +11,10 @@ func createRepoList(org string, totalCount int, archiveCount int) []*github.Repo var repoList []*github.Repository for i := 0; i < totalCount; i++ { var archiveVal = (i < archiveCount) - var htmlUrl = fmt.Sprintf("https://github.com/%s/repo-%d", org, i) + var htmlURL = fmt.Sprintf("https://github.com/%s/repo-%d", org, i) repoList = append(repoList, &github.Repository{ Archived: &archiveVal, - HTMLURL: &htmlUrl, + HTMLURL: &htmlURL, }) } return repoList @@ -25,7 +25,7 @@ func TestFilterArchivedRepos(t *testing.T) { archiveCount := 2 filteredCountExpected := (totalCount - archiveCount) originalRepoList := createRepoList("open-sauced", totalCount, archiveCount) - filteredRepoList := FilterArchivedRepos(originalRepoList) + filteredRepoList := FilterGithubArchivedRepos(originalRepoList) if len(filteredRepoList) != filteredCountExpected { t.Errorf("FilteredArchivedRepos() should yield %d items; got %d", filteredCountExpected, len(filteredRepoList)) } @@ -38,7 +38,7 @@ func TestGetRepoHTMLUrls(t *testing.T) { "https://github.com/open-sauced/repo-2", } repos := createRepoList("open-sauced", 3, 0) - got := GetRepoHTMLUrls(repos) + got := GetGithubRepoHTMLUrls(repos) if len(expected) != len(got) { t.Errorf("GetRepoHTMLUrls() should yield count matching input") } diff --git a/pkg/server/server.go b/pkg/server/server.go index 49951cf..80ae1e5 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -19,9 +19,9 @@ import ( "github.com/google/uuid" "go.uber.org/zap" + "github.com/open-sauced/pizza/oven/pkg/clients" "github.com/open-sauced/pizza/oven/pkg/common" "github.com/open-sauced/pizza/oven/pkg/database" - "github.com/open-sauced/pizza/oven/pkg/github" "github.com/open-sauced/pizza/oven/pkg/insights" "github.com/open-sauced/pizza/oven/pkg/providers" ) @@ -66,9 +66,13 @@ func (p PizzaOvenServer) Run(serverPort string) { } type reqData struct { - URL string `json:"url,omitempty"` + URL string `json:"url"` + Wait bool `json:"wait,omitempty"` +} + +type orgReqData struct { + Org string `json:"org"` Wait bool `json:"wait,omitempty"` - Org string `json:"org,omitempty"` Archives bool `json:"archives,omitempty"` } @@ -78,29 +82,28 @@ type orgRepo struct { } type orgRepoList []orgRepo -func (p PizzaOvenServer) processOrg(orgUrlString string, processArchived bool) ([]string, error) { +func (p PizzaOvenServer) processOrg(orgURLString string, processArchived bool) ([]string, error) { - var htmlUrls []string - orgUrl, err := url.Parse(orgUrlString) + var htmlURLs []string + orgUrl, err := url.Parse(orgURLString) if err != nil { - return htmlUrls, err + return htmlURLs, err } - if orgUrl.Hostname() == "github.com" { - client := github.NewClient(nil) - orgName := strings.Trim(orgUrl.Path, "/") - repos, err := client.ListReposByOrg(orgName) - if err != nil { - return htmlUrls, fmt.Errorf("Error encountered fetching repositories, org: %s with error: %s ", orgName, err) - } - if !processArchived { - repos = github.FilterArchivedRepos(repos) - } - htmlUrls = github.GetRepoHTMLUrls(repos) - - return htmlUrls, nil - } else { - return htmlUrls, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) + if orgUrl.Hostname() != "github.com" { + return htmlURLs, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) + } + client := clients.NewGithubClient(nil) + orgName := strings.Trim(orgUrl.Path, "/") + repos, err := client.ListReposByOrg(orgName) + if err != nil { + return htmlURLs, fmt.Errorf("Error encountered fetching repositories, org: %s with error: %s ", orgName, err) } + if !processArchived { + repos = clients.FilterGithubArchivedRepos(repos) + } + htmlURLs = clients.GetGithubRepoHTMLUrls(repos) + return htmlURLs, nil + } func (p PizzaOvenServer) handleBakeRequest(w http.ResponseWriter, r *http.Request) { @@ -173,7 +176,7 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req return } - var data reqData + var data orgReqData err := json.NewDecoder(r.Body).Decode(&data) if err != nil { p.Logger.Errorf("Could not decode request json body: %v with error: %v", r.Body, err) @@ -183,15 +186,16 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req w.WriteHeader(http.StatusAccepted) if data.Wait { if data.Org != "" { - cloneUrls, err := p.processOrg(data.Org, data.Archives) + cloneURLs, err := p.processOrg(data.Org, data.Archives) if err != nil { p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } errors := make([]string, 0) - for _, cloneUrl := range cloneUrls { - err = p.processRepository(cloneUrl) + for _, cloneURL := range cloneURLs { + err = p.processRepository(cloneURL) + errors = append(errors, err.Error()) } if len(errors) > 0 { errorString := strings.Join(errors, "\n") @@ -203,21 +207,21 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req } } else { if data.Org != "" { - htmlUrls, err := p.processOrg(data.Org, data.Archives) + htmlURLs, err := p.processOrg(data.Org, data.Archives) if err != nil { p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } errors := make([]string, 0) - for _, htmlUrl := range htmlUrls { - go func(htmlUrl string) { - err = p.processRepository(htmlUrl) + for _, htmlURL := range htmlURLs { + go func(htmlURL string) { + err = p.processRepository(htmlURL) if err != nil { - errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", htmlUrl, err)) + errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", htmlURL, err)) } - }(htmlUrl) + }(htmlURL) } if len(errors) > 0 { errorString := strings.Join(errors, "\n") From 81707e4182661600e39dafcf14fdcbcadec2ca5e Mon Sep 17 00:00:00 2001 From: mtfoley Date: Sun, 10 Sep 2023 08:12:39 -0400 Subject: [PATCH 14/17] error channels, wait groups, utility method for getting a valid URL endpoint --- pkg/server/server.go | 168 ++++++++++++++++++++++++++----------------- 1 file changed, 102 insertions(+), 66 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 80ae1e5..306a47f 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "strings" + "sync" "sync/atomic" "time" @@ -106,6 +107,47 @@ func (p PizzaOvenServer) processOrg(orgURLString string, processArchived bool) ( } +/* +Accepts URL value and calls: +normalized = common.NormalizeGitURL() +endpoint = transport.NewEndpoint(normalized) +common.IsValidGitRepo(endpoint) +return endpoint + +returns first encountered error, appropriate HTTP error message, and endpoint string value +*/ +func (p *PizzaOvenServer) getValidRepoEndpoint(repoURL string) (error, string, string) { + p.Logger.Debugf("Validating and normalizing repository URL: %s", repoURL) + normalizedRepoURL, err := common.NormalizeGitURL(repoURL) + if err != nil { + return fmt.Errorf("Could not normalize repo URL %s: %s", repoURL, err.Error()), + fmt.Sprintf("Could not normalize provided repo URL: %s", err.Error()), + "" + } + + repoURLendpoint, err := transport.NewEndpoint(normalizedRepoURL) + if err != nil { + return fmt.Errorf("Could not create git transport endpoint with repo URL %s: %s", repoURL, err.Error()), + fmt.Sprintf("Could not create git transport endpoint from provided repo URL: %s", err.Error()), + "" + } + + ok, err := common.IsValidGitRepo(repoURLendpoint.String()) + if !ok { + if err != nil { + return fmt.Errorf("Error validating repo URL %s: %s", repoURL, err.Error()), + fmt.Sprintf("Error validating remote git repo URL: %s", err.Error()), + "" + } + + return fmt.Errorf("Could not validate repo URL %s: %s", repoURL, err.Error()), + fmt.Sprintf("not valid git repo URL. Expected format protocol://address but got: %s", err.Error()), + "" + } + // Happy Path + return nil, "", repoURLendpoint.String() +} + func (p PizzaOvenServer) handleBakeRequest(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { p.Logger.Errorf("Received request with invalid method: %v", r.Body) @@ -120,38 +162,15 @@ func (p PizzaOvenServer) handleBakeRequest(w http.ResponseWriter, r *http.Reques http.Error(w, "Could not decode request body", http.StatusBadRequest) return } - - p.Logger.Debugf("Validating and normalizing repository URL: %s", data.URL) - normalizedRepoURL, err := common.NormalizeGitURL(data.URL) - if err != nil { - p.Logger.Debugf("Could not normalize repo URL %s: %s", data.URL, err.Error()) - http.Error(w, fmt.Sprintf("Could not normalize provided repo URL: %s", err.Error()), http.StatusBadRequest) - return - } - - repoURLendpoint, err := transport.NewEndpoint(normalizedRepoURL) + err, httpErrorString, repoURLendpoint := p.getValidRepoEndpoint(data.URL) if err != nil { - p.Logger.Errorf("Could not create git transport endpoint with repo URL %s: %s", data.URL, err.Error()) - http.Error(w, fmt.Sprintf("Could not create git transport endpoint from provided repo URL: %s", err.Error()), http.StatusBadRequest) - return - } - - ok, err := common.IsValidGitRepo(repoURLendpoint.String()) - if !ok { - if err != nil { - p.Logger.Errorf("Error validating repo URL %s: %s", data.URL, err.Error()) - http.Error(w, fmt.Sprintf("Error validating remote git repo URL: %s", err.Error()), http.StatusBadRequest) - return - } - - p.Logger.Debug("Could not validate repo URL %s: %s", data.URL, err.Error()) - http.Error(w, fmt.Sprintf("not valid git repo URL. Expected format protocol://address but got: %s", err.Error()), http.StatusBadRequest) + p.Logger.Debug(err.Error()) + http.Error(w, httpErrorString, http.StatusBadRequest) return } - w.WriteHeader(http.StatusAccepted) if data.Wait { - err = p.processRepository(data.URL) + err = p.processRepository(repoURLendpoint) if err != nil { p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) @@ -159,7 +178,7 @@ func (p PizzaOvenServer) handleBakeRequest(w http.ResponseWriter, r *http.Reques } } else { go func() { - err = p.processRepository(repoURLendpoint.String()) + err = p.processRepository(repoURLendpoint) if err != nil { p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) @@ -184,52 +203,69 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req return } w.WriteHeader(http.StatusAccepted) + if data.Org == "" { + p.Logger.Error("Could not process blank org input") + http.Error(w, "Could not process input", http.StatusBadRequest) + return + } + cloneURLs, err := p.processOrg(data.Org, data.Archives) + if err != nil { + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } if data.Wait { - if data.Org != "" { - cloneURLs, err := p.processOrg(data.Org, data.Archives) + errorChannel := make(chan error) + wg := new(sync.WaitGroup) + wg.Add(len(cloneURLs)) + for _, htmlURL := range cloneURLs { + go func(htmlURL string, wg *sync.WaitGroup, c chan error) { + err, _, repoURLendpoint := p.getValidRepoEndpoint(htmlURL) + if err != nil { + c <- err + } else { + c <- p.processRepository(repoURLendpoint) + } + defer wg.Done() + }(htmlURL, wg, errorChannel) + } + defer wg.Wait() + var errorFound bool = false + for err := range errorChannel { if err != nil { - p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) - http.Error(w, "Could not process input", http.StatusInternalServerError) - return - } - errors := make([]string, 0) - for _, cloneURL := range cloneURLs { - err = p.processRepository(cloneURL) - errors = append(errors, err.Error()) - } - if len(errors) > 0 { - errorString := strings.Join(errors, "\n") - p.Logger.Error(errorString) - http.Error(w, "Could not process input", http.StatusInternalServerError) + p.Logger.Error(err.Error()) + errorFound = true } - - return } + if errorFound { + http.Error(w, "Could not process input", http.StatusInternalServerError) + } + return } else { - if data.Org != "" { - htmlURLs, err := p.processOrg(data.Org, data.Archives) + htmlURLs, err := p.processOrg(data.Org, data.Archives) + if err != nil { + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + errorChannel := make(chan error) + + for _, htmlURL := range htmlURLs { + go func(htmlURL string, c chan error) { + err, _, repoURLendpoint := p.getValidRepoEndpoint(htmlURL) + if err != nil { + c <- err + } else { + c <- p.processRepository(repoURLendpoint) + } + }(htmlURL, errorChannel) + } + for err := range errorChannel { if err != nil { - p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) - http.Error(w, "Could not process input", http.StatusInternalServerError) - return + p.Logger.Error(err.Error()) } - errors := make([]string, 0) - for _, htmlURL := range htmlURLs { - go func(htmlURL string) { - err = p.processRepository(htmlURL) - if err != nil { - errors = append(errors, fmt.Sprintf("Could not process repo clone URL: %v with error: %v", htmlURL, err)) - } - - }(htmlURL) - } - if len(errors) > 0 { - errorString := strings.Join(errors, "\n") - p.Logger.Error(errorString) - http.Error(w, "Could not process input", http.StatusInternalServerError) - } - return } + return } } From 90ea5dcc846b16307ab37320ac9dea4657ccebc8 Mon Sep 17 00:00:00 2001 From: mtfoley Date: Sun, 10 Sep 2023 08:21:21 -0400 Subject: [PATCH 15/17] rename test functions --- pkg/clients/github_client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/clients/github_client_test.go b/pkg/clients/github_client_test.go index e7f488f..eb3a891 100644 --- a/pkg/clients/github_client_test.go +++ b/pkg/clients/github_client_test.go @@ -20,7 +20,7 @@ func createRepoList(org string, totalCount int, archiveCount int) []*github.Repo return repoList } -func TestFilterArchivedRepos(t *testing.T) { +func TestFilterGithubArchivedRepos(t *testing.T) { totalCount := 10 archiveCount := 2 filteredCountExpected := (totalCount - archiveCount) @@ -31,7 +31,7 @@ func TestFilterArchivedRepos(t *testing.T) { } } -func TestGetRepoHTMLUrls(t *testing.T) { +func TestGetGithubRepoHTMLUrls(t *testing.T) { expected := []string{ "https://github.com/open-sauced/repo-0", "https://github.com/open-sauced/repo-1", From 721bd12e470c3c7f82a46584945204982479e716 Mon Sep 17 00:00:00 2001 From: mtfoley Date: Thu, 14 Sep 2023 23:48:24 -0400 Subject: [PATCH 16/17] use goroutines when !Wait --- pkg/server/server.go | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 306a47f..b7f1c54 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -242,30 +242,32 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req } return } else { - htmlURLs, err := p.processOrg(data.Org, data.Archives) - if err != nil { - p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) - http.Error(w, "Could not process input", http.StatusInternalServerError) - return - } - errorChannel := make(chan error) - - for _, htmlURL := range htmlURLs { - go func(htmlURL string, c chan error) { - err, _, repoURLendpoint := p.getValidRepoEndpoint(htmlURL) + go func() { + htmlURLs, err := p.processOrg(data.Org, data.Archives) + if err != nil { + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + errorChannel := make(chan error) + + for _, htmlURL := range htmlURLs { + go func(htmlURL string, c chan error) { + err, _, repoURLendpoint := p.getValidRepoEndpoint(htmlURL) + if err != nil { + c <- err + } else { + c <- p.processRepository(repoURLendpoint) + } + }(htmlURL, errorChannel) + } + for err := range errorChannel { if err != nil { - c <- err - } else { - c <- p.processRepository(repoURLendpoint) + p.Logger.Error(err.Error()) } - }(htmlURL, errorChannel) - } - for err := range errorChannel { - if err != nil { - p.Logger.Error(err.Error()) } - } - return + return + }() } } From 6c7a2863900f7daaf3559ea1345c83bc095a796e Mon Sep 17 00:00:00 2001 From: mtfoley Date: Sat, 16 Sep 2023 19:42:12 -0400 Subject: [PATCH 17/17] address golangci-lint issues --- pkg/clients/github_client.go | 12 +++++----- pkg/server/server.go | 45 +++++++++++++++--------------------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/pkg/clients/github_client.go b/pkg/clients/github_client.go index 04964e2..14f9b45 100644 --- a/pkg/clients/github_client.go +++ b/pkg/clients/github_client.go @@ -7,26 +7,26 @@ import ( "github.com/google/go-github/v54/github" ) -type GithubApiClient struct { +type GithubAPIClient struct { client *github.Client } -func NewGithubTokenClient(token string) *GithubApiClient { +func NewGithubTokenClient(token string) *GithubAPIClient { ctx := context.Background() - s := &GithubApiClient{ + s := &GithubAPIClient{ client: github.NewTokenClient(ctx, token), } return s } -func NewGithubClient(httpClient *http.Client) *GithubApiClient { - s := &GithubApiClient{ +func NewGithubClient(httpClient *http.Client) *GithubAPIClient { + s := &GithubAPIClient{ client: github.NewClient(httpClient), } return s } -func (s *GithubApiClient) ListReposByOrg(org string) ([]*github.Repository, error) { +func (s *GithubAPIClient) ListReposByOrg(org string) ([]*github.Repository, error) { ctx := context.Background() opt := &github.RepositoryListByOrgOptions{ ListOptions: github.ListOptions{PerPage: 100}, diff --git a/pkg/server/server.go b/pkg/server/server.go index b7f1c54..97e2690 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -77,24 +77,18 @@ type orgReqData struct { Archives bool `json:"archives,omitempty"` } -type orgRepo struct { - URL string `json:"html_url"` - Archived bool `json:"archived"` -} -type orgRepoList []orgRepo - func (p PizzaOvenServer) processOrg(orgURLString string, processArchived bool) ([]string, error) { var htmlURLs []string - orgUrl, err := url.Parse(orgURLString) + orgURL, err := url.Parse(orgURLString) if err != nil { return htmlURLs, err } - if orgUrl.Hostname() != "github.com" { - return htmlURLs, fmt.Errorf("Cannot parse organizations from %s", orgUrl.Hostname()) + if orgURL.Hostname() != "github.com" { + return htmlURLs, fmt.Errorf("Cannot parse organizations from %s", orgURL.Hostname()) } client := clients.NewGithubClient(nil) - orgName := strings.Trim(orgUrl.Path, "/") + orgName := strings.Trim(orgURL.Path, "/") repos, err := client.ListReposByOrg(orgName) if err != nil { return htmlURLs, fmt.Errorf("Error encountered fetching repositories, org: %s with error: %s ", orgName, err) @@ -116,36 +110,36 @@ return endpoint returns first encountered error, appropriate HTTP error message, and endpoint string value */ -func (p *PizzaOvenServer) getValidRepoEndpoint(repoURL string) (error, string, string) { +func (p *PizzaOvenServer) getValidRepoEndpoint(repoURL string) (string, string, error) { p.Logger.Debugf("Validating and normalizing repository URL: %s", repoURL) normalizedRepoURL, err := common.NormalizeGitURL(repoURL) if err != nil { - return fmt.Errorf("Could not normalize repo URL %s: %s", repoURL, err.Error()), + return "", fmt.Sprintf("Could not normalize provided repo URL: %s", err.Error()), - "" + fmt.Errorf("Could not normalize repo URL %s: %s", repoURL, err.Error()) } repoURLendpoint, err := transport.NewEndpoint(normalizedRepoURL) if err != nil { - return fmt.Errorf("Could not create git transport endpoint with repo URL %s: %s", repoURL, err.Error()), + return "", fmt.Sprintf("Could not create git transport endpoint from provided repo URL: %s", err.Error()), - "" + fmt.Errorf("Could not create git transport endpoint with repo URL %s: %s", repoURL, err.Error()) } ok, err := common.IsValidGitRepo(repoURLendpoint.String()) if !ok { if err != nil { - return fmt.Errorf("Error validating repo URL %s: %s", repoURL, err.Error()), + return "", fmt.Sprintf("Error validating remote git repo URL: %s", err.Error()), - "" + fmt.Errorf("Error validating repo URL %s: %s", repoURL, err.Error()) } - return fmt.Errorf("Could not validate repo URL %s: %s", repoURL, err.Error()), + return "", fmt.Sprintf("not valid git repo URL. Expected format protocol://address but got: %s", err.Error()), - "" + fmt.Errorf("Could not validate repo URL %s: %s", repoURL, err.Error()) } // Happy Path - return nil, "", repoURLendpoint.String() + return repoURLendpoint.String(), "", nil } func (p PizzaOvenServer) handleBakeRequest(w http.ResponseWriter, r *http.Request) { @@ -162,7 +156,7 @@ func (p PizzaOvenServer) handleBakeRequest(w http.ResponseWriter, r *http.Reques http.Error(w, "Could not decode request body", http.StatusBadRequest) return } - err, httpErrorString, repoURLendpoint := p.getValidRepoEndpoint(data.URL) + httpErrorString, repoURLendpoint, err := p.getValidRepoEndpoint(data.URL) if err != nil { p.Logger.Debug(err.Error()) http.Error(w, httpErrorString, http.StatusBadRequest) @@ -220,7 +214,7 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req wg.Add(len(cloneURLs)) for _, htmlURL := range cloneURLs { go func(htmlURL string, wg *sync.WaitGroup, c chan error) { - err, _, repoURLendpoint := p.getValidRepoEndpoint(htmlURL) + _, repoURLendpoint, err := p.getValidRepoEndpoint(htmlURL) if err != nil { c <- err } else { @@ -230,7 +224,7 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req }(htmlURL, wg, errorChannel) } defer wg.Wait() - var errorFound bool = false + var errorFound = false for err := range errorChannel { if err != nil { p.Logger.Error(err.Error()) @@ -239,8 +233,8 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req } if errorFound { http.Error(w, "Could not process input", http.StatusInternalServerError) + return } - return } else { go func() { htmlURLs, err := p.processOrg(data.Org, data.Archives) @@ -253,7 +247,7 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req for _, htmlURL := range htmlURLs { go func(htmlURL string, c chan error) { - err, _, repoURLendpoint := p.getValidRepoEndpoint(htmlURL) + _, repoURLendpoint, err := p.getValidRepoEndpoint(htmlURL) if err != nil { c <- err } else { @@ -266,7 +260,6 @@ func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Req p.Logger.Error(err.Error()) } } - return }() } }