|
| 1 | +# Health-check every SIRI feed of a transport.data.gouv.fr dataset: |
| 2 | +# send SIRI CheckStatus to each `original_url`, report HTTP code + <Status>. |
| 3 | +# Sequential, honors 429 Retry-After. Defaults to the Aix-Marseille-Provence dataset. |
| 4 | +# |
| 5 | +# Usage: elixir scripts/siri_check.exs [--dataset <slug-or-id>] [--api-base <url>] |
| 6 | + |
| 7 | +Mix.install([{:req, "~> 0.5"}, {:sweet_xml, "~> 0.7"}]) |
| 8 | + |
| 9 | +defmodule SiriCheck do |
| 10 | + import SweetXml |
| 11 | + @max_attempts 8 |
| 12 | + |
| 13 | + # `local-name()` ignores the various ns prefixes SIRI producers use. |
| 14 | + def status(body), do: xpath(body, ~x"//*[local-name()='Status']/text()"s) |
| 15 | + def error_text(body), do: xpath(body, ~x"//*[local-name()='ErrorText']/text()"s) |
| 16 | + |
| 17 | + def envelope(ref) do |
| 18 | + ts = DateTime.utc_now() |> DateTime.to_iso8601() |
| 19 | + mid = "Test::Message::" <> (:crypto.strong_rand_bytes(8) |> Base.encode16(case: :lower)) |
| 20 | + |
| 21 | + ~s|<?xml version="1.0" encoding="UTF-8"?> |
| 22 | + <S:Envelope xmlns:S="http://schemas.xmlsoap.org/soap/envelope/"><S:Body> |
| 23 | + <sw:CheckStatus xmlns:sw="http://wsdl.siri.org.uk" xmlns:siri="http://www.siri.org.uk/siri"> |
| 24 | + <Request><siri:RequestTimestamp>#{ts}</siri:RequestTimestamp><siri:RequestorRef>#{ref}</siri:RequestorRef><siri:MessageIdentifier>#{mid}</siri:MessageIdentifier></Request> |
| 25 | + <RequestExtension/></sw:CheckStatus></S:Body></S:Envelope>| |
| 26 | + end |
| 27 | + |
| 28 | + def post(url, ref, attempt \\ 0, counts \\ %{rate_limit: 0, error: 0}) do |
| 29 | + case attempt_post(url, ref) do |
| 30 | + {:ok, %{status: 429} = resp, _} when attempt < @max_attempts -> |
| 31 | + retry(url, ref, attempt, retry_after_ms(resp) || backoff(attempt), bump(counts, :rate_limit)) |
| 32 | + |
| 33 | + {:ok, %{status: 429} = resp, elapsed} -> |
| 34 | + {:ok, resp, bump(counts, :rate_limit), elapsed} |
| 35 | + |
| 36 | + {:ok, resp, elapsed} -> |
| 37 | + {:ok, resp, counts, elapsed} |
| 38 | + |
| 39 | + {:error, _msg} when attempt < @max_attempts -> |
| 40 | + retry(url, ref, attempt, backoff(attempt), bump(counts, :error)) |
| 41 | + |
| 42 | + {:error, msg} -> |
| 43 | + {:error, msg, bump(counts, :error)} |
| 44 | + end |
| 45 | + end |
| 46 | + |
| 47 | + defp retry(url, ref, attempt, wait, counts) do |
| 48 | + Process.sleep(wait) |
| 49 | + post(url, ref, attempt + 1, counts) |
| 50 | + end |
| 51 | + |
| 52 | + defp bump(counts, key), do: Map.update!(counts, key, &(&1 + 1)) |
| 53 | + |
| 54 | + def counts_suffix(%{rate_limit: r, error: e}) do |
| 55 | + parts = |
| 56 | + [{r, "429"}, {e, "err"}] |
| 57 | + |> Enum.filter(fn {n, _} -> n > 0 end) |
| 58 | + |> Enum.map_join(" ", fn {n, l} -> "#{n}×#{l}" end) |
| 59 | + |
| 60 | + if parts == "", do: "", else: " (#{parts})" |
| 61 | + end |
| 62 | + |
| 63 | + defp attempt_post(url, ref) do |
| 64 | + t0 = System.monotonic_time(:millisecond) |
| 65 | + resp = Req.post!(url, body: envelope(ref), headers: [{"content-type", "text/xml"}], receive_timeout: 30_000) |
| 66 | + {:ok, resp, System.monotonic_time(:millisecond) - t0} |
| 67 | + rescue |
| 68 | + e -> {:error, Exception.message(e)} |
| 69 | + end |
| 70 | + |
| 71 | + defp backoff(attempt), do: min(trunc(:math.pow(2, attempt) * 1_000), 60_000) |
| 72 | + |
| 73 | + defp retry_after_ms(resp) do |
| 74 | + with [v | _] <- resp.headers["retry-after"], {n, ""} <- Integer.parse(v), do: min(n * 1_000, 60_000) |
| 75 | + end |
| 76 | +end |
| 77 | + |
| 78 | +{opts, _} = OptionParser.parse!(System.argv(), strict: [dataset: :string, api_base: :string]) |
| 79 | +api_base = opts[:api_base] || "https://transport.data.gouv.fr" |
| 80 | +dataset_id = opts[:dataset] || "5b3b3de988ee38708ada1789" |
| 81 | + |
| 82 | +dataset = Req.get!("#{api_base}/api/datasets/#{dataset_id}").body |
| 83 | + |
| 84 | +dataset["resources"] |
| 85 | +|> Enum.filter(&(&1["format"] == "SIRI")) |
| 86 | +|> Enum.with_index(1) |
| 87 | +|> Enum.each(fn {r, i} -> |
| 88 | + ref = r["requestor_ref"] |
| 89 | + url = r["original_url"] |
| 90 | + |
| 91 | + {color, line} = |
| 92 | + if is_nil(ref) do |
| 93 | + {:red, "missing requestor_ref"} |
| 94 | + else |
| 95 | + case SiriCheck.post(url, ref) do |
| 96 | + {:ok, resp, counts, elapsed} -> |
| 97 | + siri = SiriCheck.status(resp.body) |
| 98 | + err = SiriCheck.error_text(resp.body) |
| 99 | + color = if resp.status == 200 and siri == "true", do: :green, else: :red |
| 100 | + err_part = if err != "", do: " — #{err}", else: "" |
| 101 | + time_part = " #{Float.round(elapsed / 1000, 1)}s" |
| 102 | + {color, "HTTP #{resp.status} Status=#{siri}#{time_part}#{SiriCheck.counts_suffix(counts)}#{err_part}"} |
| 103 | + |
| 104 | + {:error, msg, counts} -> |
| 105 | + {:red, "error: #{msg}#{SiriCheck.counts_suffix(counts)}"} |
| 106 | + end |
| 107 | + end |
| 108 | + |
| 109 | + IO.puts(IO.ANSI.format([color, "[#{i}] #{line}", :reset, " — #{r["title"]}\n #{url}"])) |
| 110 | +end) |
0 commit comments