debug db: adapt to use with the buildkite webhook (#190)

Author: thatportugueseguy

db/sql/failed_builds_webhook.sql

@@ -0,0 +1,47 @@
+-- @ensure_failed_builds_webhook_table
+CREATE TABLE IF NOT EXISTS failed_builds_webhook (
+    id                          TEXT PRIMARY KEY,
+    sha                         VARCHAR(40) NOT NULL,
+    build_payload               TEXT NOT NULL,
+    pipeline_payload            TEXT NOT NULL,
+    jobs                        TEXT NOT NULL,
+    commit_author               VARCHAR(255) NOT NULL,
+    commit_url                  VARCHAR(255) NOT NULL,
+    build_state                 VARCHAR(10) NOT NULL,
+    build_url                   VARCHAR(255) NOT NULL,
+    build_number                INTEGER NOT NULL,
+    is_canceled                 BOOLEAN NOT NULL,
+    pipeline                    VARCHAR(255) NOT NULL,
+    repository                  VARCHAR(255) NOT NULL,
+    branch                      VARCHAR(255) NOT NULL,
+    state_before_notification   TEXT NOT NULL,
+    state_after_notification    TEXT NOT NULL,
+    last_handled_in             TEXT NOT NULL,
+    has_state_update            BOOLEAN NOT NULL,
+    notification_created_at     TIMESTAMP NOT NULL,
+    created_at                  TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
+
+-- @create
+INSERT INTO failed_builds_webhook VALUES;
+
+-- @update_state_after_notification
+UPDATE failed_builds_webhook
+SET has_state_update = @has_state_update, state_after_notification = @state_after_notification, last_handled_in = @last_handled_in
+WHERE id = @id;
+
+-- @get_by_sha
+SELECT id, sha, build_state, build_number, is_canceled, has_state_update, last_handled_in, state_before_notification, state_after_notification
+FROM failed_builds_webhook WHERE sha = ? and pipeline = @pipeline and branch = @branch ORDER BY id DESC;
+
+-- @get_by_build_number
+SELECT id, sha, build_state, build_number, is_canceled, has_state_update, last_handled_in, state_before_notification, state_after_notification
+FROM failed_builds_webhook WHERE build_number = ? and pipeline = @pipeline and branch = @branch ORDER BY id DESC;
+
+-- @get_after
+SELECT id, sha, build_state, build_number, is_canceled, has_state_update, last_handled_in, state_before_notification, state_after_notification
+FROM failed_builds_webhook WHERE build_number >= ? and pipeline = @pipeline and branch = @branch ORDER BY build_number desc, id desc;
+
+-- @get_from_to
+SELECT id, sha, build_state, build_number, is_canceled, has_state_update, last_handled_in, state_before_notification, state_after_notification
+FROM failed_builds_webhook WHERE build_number >= ? and build_number <= @to_ and pipeline = @pipeline and branch = @branch ORDER BY build_number desc, id desc;

db/sql/status_notifications.sql

@@ -238,7 +238,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) (Buildkite_api :
         | None -> n.branches
         | Some build_url ->
           let status_switched (_ : branch) =
-            match Util.Build.get_org_pipeline_build' build_url with
+            match Util.Build.get_org_pipeline_build n with
             | Error e ->
               log#error "failed to get org/pipeline/build_nr from build url %s: %s" build_url e;
               false
@@ -592,20 +592,21 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) (Buildkite_api :
   let process_buildkite_webhook (ctx : Context.t) headers body =
     let open Util.Webhook in
     try_process_notification body
-      (let n = Buildkite_webhook_j.webhook_build_payload_of_string body in
+      (let n = Buildkite_j.webhook_build_payload_of_string body in
        log#info "[buildkite_webhook] [%s] event %s: state=%s, branch=%s, commit=%s, pipeline=%s, build_url=%s"
          n.pipeline.provider.settings.repository
-         (Buildkite_webhook_j.string_of_webhook_event n.event)
+         (Buildkite_j.string_of_webhook_event n.event)
          (Buildkite_j.string_of_build_state n.build.state)
          n.build.branch (String.sub n.build.sha 0 8) (pipeline_name n) n.build.web_url;
        let secrets = Context.get_secrets_exn ctx in
        match validate_signature ?signing_key:secrets.buildkite_signing_secret ~headers body with
        | Error e -> action_error e
        | Ok () ->
+         let%lwt (_ : int64 Database.db_use_result) = Database.Failed_builds.create ~ctx n in
          let repo_url = validate_repo_url secrets n in
          let%lwt cfg =
            match Context.find_repo_config ctx repo_url with
-           | Some config -> Lwt.return config
+           | Some cfg -> Lwt.return cfg
            | None ->
              (* Fetch the config from github.
                 Emulate a repository record with the necessary fields to fetch the config *)
@@ -626,27 +627,48 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) (Buildkite_api :
              | Error e -> action_error @@ Printf.sprintf "failed to fetch config for %s: %s" repo_url e)
          in
          let repo_state = State.find_or_add_repo ctx.state repo_url in
-         let org, pipeline, _build_nr = Result.get_ok @@ Util.Build.get_org_pipeline_build' n.build.web_url in
+         let org, pipeline, _build_nr = Util.Build.get_org_pipeline_build' n.build.web_url in
          let repo_key = repo_key org pipeline in
          let is_main_branch = cfg.main_branch_name |> Option.map_default (String.equal n.build.branch) false in
          let should_notify = is_main_branch && (notify_fail cfg n || notify_success repo_state repo_key n) in
          (match should_notify with
-         | false -> Lwt.return_unit
+         | false ->
+           let%lwt (_ : int64 Database.db_use_result) =
+             Database.Failed_builds.update_state_after_notification ~repo_state ~has_state_update:false n
+               "should notify > false"
+           in
+           Lwt.return_unit
          | true ->
            let channel = Common.Status_notification.inject_channel (failed_builds_channel_exn cfg n) in
            let%lwt notifications =
              match n.build.state with
              | Passed ->
+               (* If the build is successful, we can remove the failed steps from the repo state. *)
                Stringtbl.replace repo_state.failed_steps repo_key
                  { steps = Common.FailedStepSet.empty; last_build = n.build.number };
+
+               let%lwt (_ : int64 Database.db_use_result) =
+                 Database.Failed_builds.update_state_after_notification ~repo_state ~has_state_update:true n
+                   "should notify > true > build state = passed"
+               in
                Lwt.return
                  [
                    Slack.generate_failed_build_notification ~is_fix_build_notification:true
                      ~failed_steps:Common.FailedStepSet.empty n channel;
                  ]
              | Failed ->
                (* repo state is updated upon fetching new failed steps *)
-               (match%lwt new_failed_steps ~repo_state ~get_build:(Buildkite_api.get_build ~cache:`Refresh ~ctx) n with
+               (match%lwt
+                  new_failed_steps ~repo_state
+                    ~get_build:(Buildkite_api.get_build ~cache:`Refresh ~ctx)
+                    ~db_update:(fun ~repo_state ~has_state_update n msg ->
+                      let%lwt (_ : int64 Database.db_use_result) =
+                        Database.Failed_builds.update_state_after_notification ~repo_state ~has_state_update n
+                          (Printf.sprintf "should notify > true > build state = failed > %s" msg)
+                      in
+                      Lwt.return_unit)
+                    n
+                with
                | Error e -> action_error e
                | Ok failed_steps ->
                match FailedStepSet.is_empty failed_steps with

lib/action.ml

@@ -238,7 +238,7 @@ module Buildkite : Api.Buildkite = struct
 
   [@@@warning "-27"]
   let get_build ?(cache : [ `Use | `Refresh ] option) ~ctx build_url =
-    let* org, pipeline, build_nr = Lwt.return @@ Util.Build.get_org_pipeline_build' build_url in
+    let org, pipeline, build_nr = Util.Build.get_org_pipeline_build' build_url in
     let file = clean_forward_slashes (sprintf "organizations/%s/pipelines/%s/builds/%s" org pipeline build_nr) in
     let url = Filename.concat buildkite_cache_dir file in
     with_cache_file url Buildkite_j.get_build_res_of_string

lib/api_local.ml

@@ -331,7 +331,7 @@ module Buildkite : Api.Buildkite = struct
   let cache_key org pipeline build_nr = sprintf "%s/%s/%s" org pipeline build_nr
 
   let get_build ?(cache : [ `Use | `Refresh ] = `Use) ~(ctx : Context.t) build_url =
-    let* org, pipeline, build_nr = Lwt.return @@ Util.Build.get_org_pipeline_build' build_url in
+    let org, pipeline, build_nr = Util.Build.get_org_pipeline_build' build_url in
     let path = sprintf "organizations/%s/pipelines/%s/builds/%s" org pipeline build_nr in
     let build_key = cache_key org pipeline build_nr in
     let get () =

lib/api_remote.ml

@@ -74,12 +74,12 @@ type job_type = [
   | Waiter <json name="waiter"> of non_job
 ] <json adapter.ocaml="Atdgen_runtime.Json_adapter.Type_field"> <ocaml repr="classic">
 
+type jobs = job_type list
+
 type get_build_res = {
-  state: build_state;
+  inherit build_base;
   created_at: string;
-  finished_at: string nullable;
-  jobs: job_type list;
-  branch: string;
+  jobs: jobs;
 }
 
 (* Custom type for the steps state and not a Buildkite type. We have them here
@@ -88,3 +88,57 @@ type failed_step = {
   name: string;
   build_url: string;
 }
+
+(* for now we only subscribe to build.finished *)
+type webhook_event = [
+  | Build_finished <json name="build.finished">
+  | Other of string
+] <json open_enum> <ocaml repr="classic">
+
+type build_creator = {
+  email: string;
+}
+type build_metadata = {
+  ?commit <json name="buildkite:git:commit"> : string nullable;
+}
+
+type build_base = {
+  id: string;
+  api_url <json name="url"> : string;
+  web_url: string;
+  number: int;
+  state: build_state;
+  created_at: string;
+  cancel_reason: string nullable;
+  blocked: bool;
+  blocked_state: string nullable;
+  message: string;
+  sha <json name="commit"> : string;
+  branch: string;
+  ?meta_data: build_metadata nullable;
+  creator: build_creator;
+}
+
+type pipeline_provider_settings = {
+  repository: string;
+}
+
+type pipeline_provider = {
+  settings: pipeline_provider_settings;
+}
+
+type pipeline = {
+  id: string;
+  web_url: string;
+  name: string;
+  description: string;
+  repository: string;
+  slug: string nullable;
+  provider: pipeline_provider;
+}
+
+type webhook_build_payload =  {
+  event: webhook_event;
+  build: build_base;
+  pipeline: pipeline;
+}