diff --git a/.buildkite/commands/build-and-test.sh b/.buildkite/commands/build-and-test.sh index 0e33ad3c9..b12dc8fa8 100755 --- a/.buildkite/commands/build-and-test.sh +++ b/.buildkite/commands/build-and-test.sh @@ -21,6 +21,8 @@ fi echo "--- 📦 Zipping test results" cd build/results/ && zip -rq Simplenote.xcresult.zip Simplenote.xcresult && cd - +.buildkite/commands/track-apple-metrics.sh "build/results/Simplenote.xcresult" "./DerivedData" + echo "--- 🚦 Report Tests Status" if [[ $TESTS_EXIT_STATUS -eq 0 ]]; then echo "Unit Tests seems to have passed (exit code 0). All good 👍" diff --git a/.buildkite/commands/track-apple-metrics.sh b/.buildkite/commands/track-apple-metrics.sh new file mode 100755 index 000000000..630c9cbbf --- /dev/null +++ b/.buildkite/commands/track-apple-metrics.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -euo pipefail + +XCRESULT_PATH="${1:?Error: xcresult path required as first argument}" +DERIVED_DATA_PATH="${2:?Error: DerivedData path required as second argument}" + +echo "--- :xcode: Store raw xcresulttool JSONs" + +mkdir -p build/xcresulttool + +xcrun xcresulttool get build-results \ + --path "$XCRESULT_PATH" \ + --format json > build/xcresulttool/xcresulttool-build-results.json + +buildkite-agent artifact upload "build/xcresulttool/xcresulttool-build-results.json" + +xcrun xcresulttool get test-results tests \ + --path "$XCRESULT_PATH" \ + --format json > build/xcresulttool/xcresulttool-tests-results.json + +buildkite-agent artifact upload "build/xcresulttool/xcresulttool-tests-results.json" + +echo "+++ :json: Extract build info from xcresulttool" +jq '{ + timestamp: .startTime | strftime("%Y-%m-%d %H:%M:%S"), + duration_ms: ((.endTime - .startTime) * 1000 | round), + status: .status, + errors: .errorCount, + warnings: .warningCount, + analyzer_warnings: .analyzerWarningCount, + warning_breakdown: (.warnings | group_by(.issueType) | map({type: .[0].issueType, count: length}) | sort_by(-.count)) +}' build/xcresulttool/xcresulttool-build-results.json + +echo "+++ :json: Extract success/fail test count from xcresulttool" +jq '[.. | objects | select(has("result")) | .result] | {passed: map(select(. == "Passed")) | length, failed: map(select(. == "Failed")) | length}' \ + build/xcresulttool/xcresulttool-tests-results.json + +echo "--- :xcode: Track XCLogParser report" + +echo "~~~ Install XCLogParser" + +brew install xclogparser + +mkdir -p build/xclogparser-reports + +echo "~~~ Dump xcactivitylog to JSON" + +xclogparser dump \ + --xcodeproj Simplenote.xcodeproj \ + --derived_data "$DERIVED_DATA_PATH" \ + --output build/xclogparser-reports/xcactivitylog-raw.json + +buildkite-agent artifact upload "build/xclogparser-reports/xcactivitylog-raw.json" + +echo "~~~ Generate JSON report" + +xclogparser parse \ + --xcodeproj Simplenote.xcodeproj \ + --derived_data "$DERIVED_DATA_PATH" \ + --reporter json > build/xclogparser-reports/report.json + +buildkite-agent artifact upload "build/xclogparser-reports/report.json" + +echo "~~~ Generate Chrome tracer report" + +xclogparser parse \ + --xcodeproj Simplenote.xcodeproj \ + --derived_data "$DERIVED_DATA_PATH" \ + --reporter chromeTracer > build/xclogparser-reports/build-trace.json + +buildkite-agent artifact upload "build/xclogparser-reports/build-trace.json" + +echo "--- :arrow_up::ruby: Upload build metrics to Apps Metrics" +# FIXME: Ignoring errors for the time being. Seems like the token has expired... +set +e +ruby .buildkite/commands/upload_metrics.rb +set -e diff --git a/.buildkite/commands/upload_metrics.rb b/.buildkite/commands/upload_metrics.rb new file mode 100644 index 000000000..5b72293b6 --- /dev/null +++ b/.buildkite/commands/upload_metrics.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'json' +require 'time' +require 'open3' +require 'net/http' +require 'uri' +require 'shellwords' + +PREFIX = 'simplenote-ios' + +XCRESULT_PATH = ARGV[0] || 'build/results/Simplenote.xcresult' + +# Hardcoded auth config (or set via ENV) +METRICS_URL = ENV['METRICS_URL'] || 'https://metrics.a8c-ci.services/api/grouped-metrics' +TOKEN = ENV['APPS_METRICS_UPLOAD_TOKEN'] || nil + +raise 'No APPS_METRICS_UPLOAD_TOKEN found in environment.' if TOKEN.nil? || TOKEN.empty? + +META = [ + { name: 'simplenote-ios-user', value: ENV['USER'] || ENV['USERNAME'] || 'unknown' }, + { name: 'simplenote-ios-project', value: 'simplenote-ios' }, + { name: 'simplenote-ios-environment', value: ENV['CI'] ? 'CI' : 'LOCAL' }, + { name: 'simplenote-ios-architecture', value: `uname -m`.strip }, + { name: 'simplenote-ios-operating-system', value: `uname -s`.strip.downcase }, + { name: 'simplenote-ios-metrics-source', value: 'grouped-metrics' }, + { name: 'simplenote-ios-branch', value: ENV['BUILDKITE_BRANCH'] || `git rev-parse --abbrev-ref HEAD`.strip } +].freeze + +# ---------- HELPERS ---------- +def run_cmd!(cmd) + out, err, status = Open3.capture3(cmd) + raise "Command failed (#{status.exitstatus}): #{cmd}\n#{err}" unless status.success? + + out +end + +def to_epoch_ms(str_time) + return nil if str_time.nil? || str_time.empty? + + (Time.parse(str_time).to_f * 1000).to_i +rescue ArgumentError => e + puts "`Time.parse` failed with '#{e}'." + nil +end + +def dig_count(obj, *path) + v = obj.dig(*path) + return v.to_i if v.is_a?(Numeric) || v.is_a?(String) + return v.length if v.is_a?(Array) + + 0 +end + +raw_json = run_cmd!("xcrun xcresulttool get build-results --path #{Shellwords.escape(XCRESULT_PATH)} --format json") +data = JSON.parse(raw_json) + +end_time = (data['endTime'] * 1_000).round(0) +start_time = (data['startTime'] * 1_000).round(0) + +explicit_fields = { + 'action-title' => data['actionTitle'], + 'analyzer-warning-count' => data['analyzerWarningCount'], + 'end-time-unix-ms' => end_time, + 'error-count' => data['errorCount'], + 'start-time-unix-ms' => start_time, + 'status' => data['status'], + 'warning-count' => data['warningCount'], + 'build-time' => end_time - start_time +} + +metrics_payload = explicit_fields.map do |k, v| + { name: "#{PREFIX}-#{k}", value: v.to_s } +end + +payload = { + meta: META, + metrics: metrics_payload +} + +puts 'Will attempt to upload the following JSON:' +puts JSON.pretty_generate(payload) + +uri = URI(METRICS_URL) +http = Net::HTTP.new(uri.host, uri.port) +http.use_ssl = (uri.scheme == 'https') + +req = Net::HTTP::Post.new(uri.request_uri) +req['Accept'] = 'application/json' +req['Accept-Charset'] = 'UTF-8' +req['Authorization'] = "Bearer #{TOKEN}" +req['User-Agent'] = 'Xcode/xcresulttool' +req['Content-Type'] = 'application/json' +req.body = JSON.dump(payload) + +res = http.request(req) + +puts "POST #{METRICS_URL} -> #{res.code}" +puts res.body +exit(res.code.to_i.between?(200, 299) ? 0 : 1) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 12f53eac6..345854395 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -12,48 +12,11 @@ env: # This is the default pipeline – it will build and test the app steps: - ################# - # Run Unit Tests - ################# - label: "🔬 Build and Test" + key: build_and_test command: ".buildkite/commands/build-and-test.sh" plugins: [$CI_TOOLKIT_PLUGIN] artifact_paths: - "build/results/*" - - ################# - # Linters - ################# - - label: "☢️ Danger - PR Check" - command: danger - key: danger - if: "build.pull_request.id != null" - retry: - manual: - permit_on_passed: true - agents: - queue: "linter" - - - label: ":swift: SwiftLint" - command: swiftlint - agents: - queue: "linter" - - ################# - # Prototype Build - ################# - - label: "🛠 Prototype Build" - command: ".buildkite/commands/build-prototype.sh" - plugins: [$CI_TOOLKIT_PLUGIN] - if: "build.pull_request.id != null || build.pull_request.draft" - artifact_paths: - - "build/results/*" - - ################# - # UI Tests - ################# - - label: "🔬 UI Test (Full)" - command: ".buildkite/commands/build-and-ui-test.sh SimplenoteUITests 'iPhone SE (3rd generation)'" - plugins: [$CI_TOOLKIT_PLUGIN] - artifact_paths: - - "build/results/*" + - DerivedData/**/*.xcactivitylog + - build/xclogparser-reports/index.html diff --git a/Gemfile b/Gemfile index bda4d2272..780dbafd4 100644 --- a/Gemfile +++ b/Gemfile @@ -7,6 +7,7 @@ gem 'fastlane', '~> 2' gem 'fastlane-plugin-firebase_app_distribution', '~> 0.10' gem 'fastlane-plugin-sentry', '~> 1.6' gem 'fastlane-plugin-wpmreleasetoolkit', '~> 13.0' +gem 'openssl' group :screenshots, optional: true do gem 'rmagick', '~> 3.2.0' diff --git a/Gemfile.lock b/Gemfile.lock index 7803ba7bf..fdab63395 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -264,6 +264,7 @@ GEM faraday (>= 1, < 3) sawyer (~> 0.9) open4 (1.3.4) + openssl (3.3.2) options (2.3.2) optparse (0.6.0) os (1.1.4) @@ -358,6 +359,7 @@ DEPENDENCIES fastlane-plugin-firebase_app_distribution (~> 0.10) fastlane-plugin-sentry (~> 1.6) fastlane-plugin-wpmreleasetoolkit (~> 13.0) + openssl rmagick (~> 3.2.0) BUNDLED WITH diff --git a/fastlane/Fastfile b/fastlane/Fastfile index b1c64b51e..5e692de6b 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -200,7 +200,7 @@ platform :ios do project: project_path, scheme: screenshots_scheme, build_for_testing: true, - derived_data_path: derived_data_directory + derived_data_path: DERIVED_DATA_DIRECTORY ) end @@ -232,7 +232,7 @@ platform :ios do # folder. This is so that we can parallelize test runs with multiple # device and locale combinations. test_without_building: true, - derived_data_path: derived_data_directory, + derived_data_path: DERIVED_DATA_DIRECTORY, output_directory: screenshots_directory, clear_previous_screenshots: options.fetch(:clear_previous_screenshots, true), @@ -337,6 +337,7 @@ platform :ios do scheme: 'Simplenote', device: options[:device] || 'iPhone 14', output_directory: OUTPUT_DIRECTORY_PATH, + derived_data_path: DERIVED_DATA_DIRECTORY, reset_simulator: true, result_bundle: true ) @@ -534,9 +535,7 @@ def fastlane_directory __dir__ end -def derived_data_directory - File.join(fastlane_directory, 'DerivedData') -end +DERIVED_DATA_DIRECTORY = File.join(fastlane_directory, 'DerivedData') def project_name 'Simplenote.xcodeproj'