DGUK-290 Fix flaky tests

Run markdown render in spec/rails_helper. This is because
feature tests require the app/views/generated views to exist.

Refactor markdown to be more resilliant to failures

Add tests

Author: Obsiye

app/services/dgu/collections_service.rb

@@ -1,8 +1,5 @@
 module Dgu
   class CollectionsService
-    COLLECTIONS_LOCATION = Rails.configuration.x.generated_collections_location
-    COLLECTION_PAGES = Rails.configuration.x.collection_pages.deep_dup
-
     attr_reader :collection, :page_name, :collection_pages
 
     def initialize(collection, page_name = nil)
@@ -11,7 +8,7 @@ def initialize(collection, page_name = nil)
       end
 
       @collection = collection
-      @collection_pages = COLLECTION_PAGES[collection].map do |collection_page|
+      @collection_pages = collections_config[collection].map do |collection_page|
         {
           url: "/collections/#{@collection}/#{collection_page[:slug]}",
           slug: collection_page[:slug],
@@ -34,18 +31,6 @@ def image_path
       "v2/collections/badge-#{collection}.png"
     end
 
-    def collections_slugs
-      Dir.entries(Rails.root.join(COLLECTIONS_LOCATION)).sort
-      .reject { |files|
-        [".", ".."].include?(files)
-      }.map do |collection|
-        Struct.new(:slug, :title).new(
-          collection,
-          collection.gsub("-", " ").humanize,
-        )
-      end
-    end
-
     def view_template_path
       "generated/collections/#{collection}/#{page}"
     end
@@ -72,8 +57,22 @@ def title
 
   private
 
+    def collections_config
+      Rails.configuration.x.collection_pages
+    end
+
+    def collections_slugs
+      Dir.children(Rails.root.join(collections_location)).sort
+      .map do |collection|
+        Struct.new(:slug, :title).new(
+          collection,
+          collection.gsub("-", " ").humanize,
+        )
+      end
+    end
+
     def pages
-      Dir.entries(Rails.root.join(COLLECTIONS_LOCATION, @collection)).sort
+      Dir.entries(Rails.root.join(collections_location, @collection)).sort
       .reject { |entry|
         [".", ".."].include?(entry)
       }
@@ -85,15 +84,19 @@ def pages
     def page?(page = nil)
       return true if page.blank?
 
-      Rails.root.join(COLLECTIONS_LOCATION, @collection, "#{page}.html.erb").exist?
+      Rails.root.join(collections_location, @collection, "#{page}.html.erb").exist?
     end
 
     def collection?(collection)
-      Rails.root.join(COLLECTIONS_LOCATION, collection).exist?
+      Rails.root.join(collections_location, collection).exist?
     end
 
     def current_page_index
       @collection_pages.index { |page| page[:slug] == @page_name }
     end
+
+    def collections_location
+      Rails.configuration.x.generated_collections_location
+    end
   end
 end

lib/tasks/markdown_to_static_html.rake

@@ -1,53 +1,74 @@
 require Rails.root.join("app/services/dgu/markdown")
 
-def format_date(date_str)
-  Date.iso8601(date_str).strftime("%-d/%-m/%Y")
-rescue ArgumentError, TypeError
-  nil
+def format_date(date)
+  return nil if date.blank?
+
+  Date.iso8601(date).strftime("%-d/%-m/%Y")
+end
+
+def process_visualisation_data(chart_json)
+  return chart_json unless chart_json["visualisation_type"] == "line"
+
+  point_shapes = %w[circle triangle rect rectRot]
+
+  chart_json["series"].each do |series|
+    data_size = series["data"].keys.size
+    next if data_size.zero?
+
+    series["dataset"] = {
+      pointRadius: Array.new(data_size - 1, 0) << 4,
+      pointStyle: Array.new(data_size, point_shapes.pop || "circle"),
+    }
+  end
+
+  chart_json
 end
 
 namespace :markdown do
   desc "Convert markdown to static html pages"
   task render: :environment do
-    markdown_input = Rails.configuration.x.markdown_collections_location_glob
+    markdown_input_glob = Rails.configuration.x.markdown_collections_location_glob
     markdown_output_dir = Rails.configuration.x.markdown_collections_output_location
     visualisations_data_location = Rails.configuration.x.visualisations_data_location
 
-    input_root = Pathname.new(Rails.root.join(markdown_input.split("*").first)).cleanpath
-    markdowns = Dir.glob(Rails.root.join(markdown_input))
-
+    input_root = Pathname.new(Rails.root.join(markdown_input_glob.split("*").first)).cleanpath
     output_directory = Rails.root.join(markdown_output_dir)
+
     FileUtils.mkdir_p(output_directory)
 
+    markdowns = Dir.glob(Rails.root.join(markdown_input_glob))
+
     if markdowns.empty?
-      puts("No markdown files were found")
+      puts("No markdown files were found at #{markdown_input_glob}")
+      next
     end
 
     markdowns.each do |markdown_file|
       parsed_markdown = FrontMatterParser::Parser.parse_file(markdown_file)
       front_matter = parsed_markdown.front_matter
+
+      if front_matter["status"] != "for-publication"
+        puts("Skipping markdown file #{markdown_file} as 'status' is missing or not 'for-publication'")
+        next
+      end
+
       content = parsed_markdown.content
       html_body = Dgu::Markdown.render(content)
 
       if html_body.strip.empty?
-        puts("Skipping markdown file #{markdown_file} no body content present")
+        puts("Skipping markdown file #{markdown_file} - no body content present")
         next
       end
 
-      if front_matter["visualisation-data"]
-        chart_data = File.read(Rails.root.join("#{visualisations_data_location}/#{front_matter['visualisation-data']}")) if front_matter["visualisation-data"]
-        chart_json = JSON.parse(chart_data)
-
-        if chart_json["visualisation_type"] == "line"
-          point_shapes = %w[circle triangle rect rectRot]
+      chart_json = nil
+      if front_matter["visualisation-data"].present?
+        chart_path = Rails.root.join(visualisations_data_location, front_matter["visualisation-data"])
 
-          chart_json["series"].map! do |series|
-            series["dataset"] = {
-              pointRadius: Array.new(series["data"].keys.size - 1, 0) << 4,
-              pointStyle: Array.new(series["data"].keys.size, point_shapes.pop || "circle"),
-            }
-            series
-          end
+        if File.exist?(chart_path)
+          chart_data = File.read(chart_path)
+          chart_json = process_visualisation_data(JSON.parse(chart_data))
+        else
+          puts("Warning: Visualisation data not found for #{markdown_file} - #{chart_path}")
         end
       end
 
@@ -67,20 +88,18 @@ namespace :markdown do
         contact: front_matter["contact"],
         body: html_body.html_safe,
         collection_topic_path: collection_topic_path,
-        chart_data: chart_json || nil,
+        chart_data: chart_json,
       }
 
-      if front_matter["status"].nil? || front_matter["status"] != "for-publication"
-        puts("Skipping markdown file #{markdown_file} as 'status' is missing or not 'for-publication' ")
-        next
-      end
-
       path = Pathname.new(markdown_file)
       relative_path = path.relative_path_from(input_root).sub_ext(".html.erb")
-      output_path = output_directory / relative_path
+      output_path = output_directory.join(relative_path)
+
       FileUtils.mkdir_p(output_path.dirname)
 
-      puts("Render #{path.basename} to => #{output_path}")
+      unless Rails.env.test?
+        puts("Rendering #{path.basename} to => #{output_path}")
+      end
 
       html = ApplicationController.renderer.render(
         partial: "v2/collection/content",
@@ -90,9 +109,10 @@ namespace :markdown do
 
       File.write(output_path, html)
     rescue StandardError => e
-      puts("error processing #{markdown_file}")
-      puts e
+      puts("Error processing #{markdown_file}: #{e.message}")
+      puts(e.backtrace.first(3).join("\n"))
     end
+
     puts("Completed rendering markdown to html files")
   end
 end

spec/features/v2/charts_spec.rb

@@ -1,6 +1,8 @@
 require "rails_helper"
 
 RSpec.describe "Charts", type: :system, js: true do
+  # Setup and teardown for generated views is handled globally in rails_helper.rb
+
   scenario "I can see a single series line chart on a collection page" do
     given_i_visit_a_collection_page_with_a_single_series_line_chart
     then_i_should_see_a_line_chart_with_expected_data

spec/features/v2/collections_spec.rb

@@ -1,16 +1,7 @@
 require "rails_helper"
 
 RSpec.feature "collections", type: :feature do
-  before(:all) do
-    Rake.application.rake_require("tasks/markdown_to_static_html")
-    Rake::Task.define_task(:environment)
-    Rake::Task["markdown:render"].invoke
-  end
-
-  after(:all) do
-    output_directory = Rails.configuration.x.markdown_collections_output_location
-    FileUtils.rm_rf(output_directory)
-  end
+  # Setup and teardown for generated views is handled globally in rails_helper.rb
 
   scenario "I visit a collection page" do
     given_i_am_on_a_collection_page

spec/fixtures/collections/another-collection/page-1.html.erb

@@ -0,0 +1 @@
+<h1>Page One</h1>

spec/fixtures/collections/another-collection/page-2.html.erb

@@ -0,0 +1 @@
+<h1>Page Two</h1>

spec/fixtures/collections/business-and-economy/inflation.html.erb

@@ -0,0 +1,55 @@
+
+<h1 class="govuk-heading-l datagovuk-heading-l">Inflation</h1>
+
+<section class="govuk-section datagovuk-section">
+  <p class="govuk-body-m datagovuk-body">Explore various datasets about inflation and price indices published by the Office for National Statistics (ONS). You can download the data as a PNG, CSV or XLS file.</p>
+<p class="govuk-body-m datagovuk-body">Inflation refers to the rate at which the general level of prices for goods and services rises, while a price index measures the average change in prices over time. Measures of inflation and prices include consumer price inflation, producer price inflation and the House Price Index. </p>
+</section>
+
+  <section class="datagovuk-headline govuk-section datagovuk-section govuk-!-margin-top-8">
+      
+<h2 class="govuk-heading-m datagovuk-heading-m">Consumer Prices Index including owner occupiers&#39; housing costs - annual rate</h2>
+
+<div class="govuk-grid-row line-chart">
+  <div id="chart-1" style="height: 300px; width: 100%; text-align: center; color: #999; line-height: 300px; font-size: 14px; font-family: 'Lucida Grande', 'Lucida Sans Unicode', Verdana, Arial, Helvetica, sans-serif;">Loading...</div>
+<script nonce="/AwItmjN1ASeMELmxl4cAaPb9pr+EwWXcijfVmLjsmU=">
+  (function() {
+    if (document.documentElement.hasAttribute("data-turbolinks-preview")) return;
+    if (document.documentElement.hasAttribute("data-turbo-preview")) return;
+
+    var createChart = function() { new Chartkick["LineChart"]("chart-1", [{"name":"Rate","color":"#000000","data":{"1989":5.7,"1990":8.0,"1991":7.5,"1992":4.6,"1993":2.6,"1994":2.2,"1995":2.7,"1996":2.9,"1997":2.2,"1998":1.8,"1999":1.7,"2000":1.2,"2001":1.6,"2002":1.5,"2003":1.4,"2004":1.4,"2005":2.1,"2006":2.5,"2007":2.4,"2008":3.5,"2009":2.0,"2010":2.5,"2011":3.8,"2012":2.6,"2013":2.3,"2014":1.5,"2015":0.4,"2016":1.0,"2017":2.6,"2018":2.3,"2019":1.7,"2020":1.0,"2021":2.5,"2022":7.9,"2023":6.8,"2024":3.3,"2025":3.9},"dataset":{"pointRadius":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4],"pointStyle":["rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot"]}}], {"suffix":"%","round":2,"points":true,"dataset":{"empty":"No data available","tension":0.5,"pointRadius":0,"pointHoverRadius":10},"library":{"plugins":{"datalabels":{"display":false},"legend":{"display":false,"labels":{"color":"#5D5D5D","font":{"size":20,"family":"Inter, sans-serif","weight":"bold"},"padding":30,"usePointStyle":true}},"tooltip":{"bodyFont":{"size":20,"family":"Inter, sans-serif","weight":"bold"},"titleFont":{"size":20,"family":"Inter, sans-serif","weight":"bold"}}},"responsive":true,"layout":{"padding":{"top":20,"bottom":20,"left":20,"right":20}},"scales":{"y":{"min":0,"grid":{"display":true},"border":{"dash":[5,5]},"ticks":{"stepSize":2,"maxTicksLimit":10,"autoSkip":false,"font":{"size":20}}},"x":{"border":{"display":true},"ticks":{"font":{"size":20}}}}},"legend":"bottom"}); };
+    if ("Chartkick" in window) {
+      createChart();
+    } else {
+      window.addEventListener("chartkick:load", createChart, true);
+    }
+  })();
+</script>
+
+</div>
+
+
+      <p class="govuk-body datagovuk-body govuk-!-margin-top-4">
+        <a href="/collections/business-and-economy/inflation/charts/inflation-1989-2025.csv" class="govuk-link datagovuk-link">Download the chart data</a>
+      </p>
+  </section>
+
+<div class="govuk-inset-text datagovuk-feedback-inset-text">
+  <p class="govuk-body datagovuk-body">
+      Is this data useful?
+    <a href="https://forms.office.com/e/9V26PNFQaR" class="govuk-link datagovuk-link">Give us feedback</a>
+  </p>
+</div>
+
+
+
+<section class="datagovuk-link datagovuk-section" aria-label="Get the data links">
+    <section class="datagovuk-section">
+        <h3 class="govuk-heading-m datagovuk-heading-m">Get the data</h3>
+          <a role="button" class="govuk-button datagovuk-button" rel="nofollow" href="https://www.ons.gov.uk/economy/inflationandpriceindices">Inflation and price indices <span class="datagovuk-right-arrow">&#8594;</span></a>
+    </section>
+    <section class="datagovuk-section">
+            <p class="govuk-body datagovuk-body datagovuk-!-colour-grey">The page was last updated 25/3/2026</p>
+    </section>
+</section>
+

spec/fixtures/collections/business-and-economy/uk-trade.html.erb

@@ -0,0 +1,55 @@
+
+<h1 class="govuk-heading-l datagovuk-heading-l">UK trade</h1>
+
+<section class="govuk-section datagovuk-section">
+  <p class="govuk-body-m datagovuk-body">Explore statistics that set out the UK’s economic relationship with other countries and illustrate the UK’s bilateral trade on top exports and imports for goods and services. In addition, the UK trade summary statistics depict foreign direct investment (FDI), regional trade statistics, and the UK’s position in global trade and investment rankings. </p>
+<p class="govuk-body-m datagovuk-body">You can download more detailed data in ODS format. The data is provided by the Department for Business and Trade.</p>
+</section>
+
+  <section class="datagovuk-headline govuk-section datagovuk-section govuk-!-margin-top-8">
+      
+<h2 class="govuk-heading-m datagovuk-heading-m">Total UK exports and imports in goods and services 2010 to 2025</h2>
+
+<div class="govuk-grid-row line-chart">
+  <div id="chart-1" style="height: 300px; width: 100%; text-align: center; color: #999; line-height: 300px; font-size: 14px; font-family: 'Lucida Grande', 'Lucida Sans Unicode', Verdana, Arial, Helvetica, sans-serif;">Loading...</div>
+<script nonce="4VBZSs68OtXOwcNU1lUED7Y0uhycACWjlb+rG3D91Qg=">
+  (function() {
+    if (document.documentElement.hasAttribute("data-turbolinks-preview")) return;
+    if (document.documentElement.hasAttribute("data-turbo-preview")) return;
+
+    var createChart = function() { new Chartkick["LineChart"]("chart-1", [{"name":"Value of exports in £ billions","color":"#C27A9A","data":{"2010":464.495,"2011":522.107,"2012":525.113,"2013":540.436,"2014":534.994,"2015":537.513,"2016":582.613,"2017":649.59,"2018":691.74,"2019":718.759,"2020":636.325,"2021":680.291,"2022":879.121,"2023":878.278,"2024":897.916,"2025":925.541},"dataset":{"pointRadius":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4],"pointStyle":["rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot","rectRot"]}},{"name":"Value of imports in £ billions","color":"#00890B","data":{"2010":489.677,"2011":535.37,"2012":541.81,"2013":563.771,"2014":567.939,"2015":563.867,"2016":617.023,"2017":675.643,"2018":717.814,"2019":745.746,"2020":620.657,"2021":679.352,"2022":906.868,"2023":910.419,"2024":919.505,"2025":971.472},"dataset":{"pointRadius":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4],"pointStyle":["rect","rect","rect","rect","rect","rect","rect","rect","rect","rect","rect","rect","rect","rect","rect","rect"]}}], {"suffix":"bn","round":2,"points":true,"dataset":{"empty":"No data available","tension":0.5,"pointRadius":0,"pointHoverRadius":10},"library":{"plugins":{"datalabels":{"display":false},"legend":{"display":true,"labels":{"color":"#5D5D5D","font":{"size":20,"family":"Inter, sans-serif","weight":"bold"},"padding":30,"usePointStyle":true}},"tooltip":{"bodyFont":{"size":20,"family":"Inter, sans-serif","weight":"bold"},"titleFont":{"size":20,"family":"Inter, sans-serif","weight":"bold"}}},"responsive":true,"layout":{"padding":{"top":20,"bottom":20,"left":20,"right":20}},"scales":{"y":{"min":100,"grid":{"display":true},"border":{"dash":[5,5]},"ticks":{"stepSize":0,"maxTicksLimit":10,"autoSkip":false,"font":{"size":20}}},"x":{"border":{"display":true},"ticks":{"font":{"size":20}}}}},"legend":"bottom"}); };
+    if ("Chartkick" in window) {
+      createChart();
+    } else {
+      window.addEventListener("chartkick:load", createChart, true);
+    }
+  })();
+</script>
+
+</div>
+
+
+      <p class="govuk-body datagovuk-body govuk-!-margin-top-4">
+        <a href="/collections/business-and-economy/uk-trade/charts/total-uk-imports-exports" class="govuk-link datagovuk-link">Download the chart data</a>
+      </p>
+  </section>
+
+<div class="govuk-inset-text datagovuk-feedback-inset-text">
+  <p class="govuk-body datagovuk-body">
+      Is this data useful?
+    <a href="https://forms.office.com/e/9V26PNFQaR" class="govuk-link datagovuk-link">Give us feedback</a>
+  </p>
+</div>
+
+
+
+<section class="datagovuk-link datagovuk-section" aria-label="Get the data links">
+    <section class="datagovuk-section">
+        <h3 class="govuk-heading-m datagovuk-heading-m">Get the data</h3>
+          <a role="button" class="govuk-button datagovuk-button" rel="nofollow" href="https://www.gov.uk/government/statistics/uk-trade-in-numbers/uk-trade-in-numbers-web-version">UK trade statistics <span class="datagovuk-right-arrow">&#8594;</span></a>
+    </section>
+    <section class="datagovuk-section">
+            <p class="govuk-body datagovuk-body datagovuk-!-colour-grey">The page was last updated 24/3/2026</p>
+    </section>
+</section>
+

spec/fixtures/collections/test-collection/page-1.html.erb

@@ -0,0 +1 @@
+<h1>Page One</h1>

spec/fixtures/collections/test-collection/page-2.html.erb

@@ -0,0 +1 @@
+<h1>Page Two</h1>