Integrate TigerData into the Nomad infrastructure (#1669)

* Testing pulibrary resource class

* give permission to install

* add sudo

* moving apt install -y command

* adding circleci ruby orb
installing ruby from gemfile

* add node js orb and install node version from gemfile

* updating gem and yarn keys

* brew installing coreutiles and updating datacite version

* swapping docker commands for podman

* adding docker io to podman pull command

* removing mac address assignment

* swapping docker commands for podman in the postgres commands

* adding tzdata to dependencies

* add docker cleanup command

* adding browser tools orbs and steps

* swapping selenium headless for chrome
adding capybara configurations

* running mediaflux as root

* re-adding mac address to mediaflux

* adding sudo to other mediaflux commands

* disabling cgroup

* stop mediaflux as root and circleci

* setting rails env variable

* only publish ports

* adding sudo to podman prune

* adding host to mediaflux

* splitting stop postgres and mediaflux commands

* running postgres as root

* disabling cgroup for postgres

* Try this.

* Try this.

* removing privilege from mediaflux

* adding -v command to the rootless pod

* testing integration tests

* revert change

* Testing integration tests again

* testing with princeton domain

* hard coding test password

* reverting password and excluding integration tests

* changing mediaflux password to mediaflux ci password

* revert password name change

* testing integration tests in a privileged container

* testing integration tests in a privileged container with hard coded password

* removing privilege from mediaflux and disabling apparmor

* disable seccomp

* excluding integration tests and removing extra security opt tags

* attempting to allow SHA1 on the Top Level Pod-container

* testing integration tests AGAIN....

* delay the crypto policy update

* increase delay

* run crypto policy update faster

* exclude integration tests

* install crypto-policies and run the policy

* update the crypto policy

* test integration

* exclude integration tests

* test crypto policy update

* disable sha1 after the test suite is done

* disable sha1 after the test suite is done, for real this time

* testing default sha1

* running the entire test suite

---------

Co-authored-by: Trey Pendragon <[email protected]>

Author: JaymeeH

.circleci/config.yml

@@ -14,7 +14,10 @@ parameters:
     default: ""
 
 orbs:
+  browser-tools: circleci/[email protected]
   coveralls: coveralls/[email protected]
+  ruby: circleci/[email protected]
+  node: circleci/[email protected]
 executors:
   basic-executor:
     docker:
@@ -27,11 +30,11 @@ executors:
 commands:
   install_dependencies:
     steps:
-      - run: sudo apt update && sudo apt install postgresql-client libmsgpack-dev libpq-dev
+      - run: sudo apt update && sudo apt install -y postgresql-client libmsgpack-dev libpq-dev tzdata crypto-policies
       - run: gem install bundler -v '2.5.6'
       - run: cp Gemfile.lock Gemfile.lock.bak
       - restore_cache:
-          key: &gem_key tiger_data-cimg-{{ checksum "Gemfile.lock.bak" }}
+          key: &gem_key tiger_data-cimg-{{ checksum "Gemfile.lock.bak" }}-2
       - run: bundle config set path './vendor/bundle'
       - run: bundle config set --local without production
       - run: bundle install --jobs=4 --retry=3
@@ -41,7 +44,7 @@ commands:
             - ./vendor/bundle
       - restore_cache:
           name: Restore Yarn Package Cache
-          key: &yarn_key tiger_data-yarn-cimg-{{ checksum "yarn.lock" }}
+          key: &yarn_key tiger_data-yarn-cimg-{{ checksum "yarn.lock" }}-2
       - run:
           name: Install NPM Dependencies via Yarn
           command: yarn install --frozen-lockfile
@@ -53,29 +56,64 @@ commands:
 
   run_mediaflux:
     steps:
-      - run: echo "$DOCKERHUB_PASSWORD" | docker login --username $DOCKERHUB_USERNAME --password-stdin
-      - run: docker pull pulibraryrdss/mediaflux_dev:v0.8.0
-      - run: docker run -d --privileged --name mediaflux --publish 0.0.0.0:8888:80  --mac-address 02:42:ac:11:00:02 pulibraryrdss/mediaflux_dev:v0.8.0
+      - run:
+          name: Run Mediaflux in podman
+          command: podman run -t --name mediaflux --rm --security-opt label=disable --security-opt unmask=ALL -p 8888:8888 --user podman --device /dev/net/tun --device /dev/fuse quay.io/podman/stable:latest bin/bash -c "echo $DOCKERHUB_PASSWORD | podman login --username $DOCKERHUB_USERNAME --password-stdin docker.io && podman run -t --rm -v /sys:/sys -p 8888:80 --device /dev/fuse --network bridge --mac-address=02:42:ac:11:00:02 docker.io/pulibraryrdss/mediaflux_dev:v0.8.0"
+          background: true
+
+  # Enable SHA1 for Mediaflux. Neccessary to run integration tests
+  enable_sha1_for_mediaflux:
+    steps:
+      - run:
+          name: Update the Crypto Policies of the Top Level Pod-Container
+          command: sudo update-crypto-policies --set LEGACY
+          background: true
+
+  # Disable SHA1 for security purposes
+  disable_sha1:
+    steps:
+      - run:
+          name: Update the Crypto Policies of the Top Level Pod-Container
+          command: sudo update-crypto-policies --set DEFAULT
+          background: true
 
   run_postgres:
     steps:
-      - run: docker create --name postgres --publish 0.0.0.0:5432:5432 --env POSTGRES_HOST_AUTH_METHOD=trust --env POSTGRES_DB=test_db --env POSTGRES_USER=tiger_data_user postgres:15.2-alpine
-      - run: docker start postgres
+      - run:
+          name: Run Postgres in podman
+          command: podman run -t --name postgres --rm --publish 0.0.0.0:5432:5432 --env POSTGRES_HOST_AUTH_METHOD=trust --env POSTGRES_DB=test_db --env POSTGRES_USER=tiger_data_user docker.io/postgres:15.2-alpine
+          background: true
+
+  cleanup_docker:
+    steps:
+      - run: podman stop postgres || true
+      - run: podman stop mediaflux || true
+      - run: podman system prune -a --volumes -f || true
 
 jobs:
   test:
     working_directory: ~/tiger_data
-    machine:
-      image: ubuntu-2404:2024.05.1
-      docker_layer_caching: true
+    machine: true
+    resource_class: pulibrary/ruby-deploy
     environment:
       POSTGRES_USER: tiger_data_user
       POSTGRES_DB: test_db
       POSTGRES_HOST_AUTH_METHOD: trust
       ARCH: linux
+      RAILS_ENV: test
     steps:
       - checkout
+      - cleanup_docker
+      - run_mediaflux
+      - ruby/install:
+          version: "3.3.0"
+      - node/install:
+          install-yarn: true
+          node-version: "22.9"
+      - browser-tools/install-chrome
+      - browser-tools/install-chromedriver
       - install_dependencies
+      - run_postgres
       - run:
           name: Run rubocop
           command: bundle exec rubocop
@@ -86,17 +124,17 @@ jobs:
       - run:
           name: Run eslint
           command: yarn run eslint 'app/javascript/**'
+      # wait for postgres and mediaflux
+      - run: sleep 10
+      - enable_sha1_for_mediaflux
       - persist_to_workspace:
           root: &root "~/tiger_data"
           paths: "*"
-      - run_mediaflux
-      - run_postgres
-      # wait for postgres and mediaflux
-      - run: sleep 10
       - run: bundle exec rake db:migrate RAILS_ENV=test
       - run:
           name: Run Rspec
-          command: bundle exec rspec --tag \~integration --format progress --format RspecJunitFormatter -o /tmp/rspec/rspec.xml
+          command: bundle exec rspec --format progress --format RspecJunitFormatter -o /tmp/rspec/rspec.xml
+      - disable_sha1
       - store_test_results:
           path: /tmp/rspec
       - store_artifacts:

Gemfile

@@ -57,7 +57,7 @@ gem "devise"
 gem "omniauth-cas", "~> 3.0"
 
 gem "csv"
-gem "datacite", github: "sul-dlss/datacite-ruby", branch: "main"
+gem "datacite", "~> 0.4.0"
 gem "dogstatsd-ruby"
 gem "dry-operation"
 gem "flipflop"

Gemfile.lock

@@ -1,14 +1,3 @@
-GIT
-  remote: https://github.com/sul-dlss/datacite-ruby.git
-  revision: 9da880e450881f33700071b26ff28170b6b4f5bc
-  branch: main
-  specs:
-    datacite (0.3.0)
-      dry-monads (~> 1.3)
-      faraday (~> 2.0)
-      json_schema (~> 0.21.0)
-      zeitwerk (~> 2.4)
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -156,6 +145,11 @@ GEM
     crass (1.0.6)
     csv (3.3.0)
     daemons (1.4.1)
+    datacite (0.4.0)
+      dry-monads (~> 1.3)
+      faraday (~> 2.0)
+      json_schema (~> 0.21.0)
+      zeitwerk (~> 2.4)
     date (3.4.1)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
@@ -169,12 +163,13 @@ GEM
     docile (1.4.0)
     dogstatsd-ruby (5.6.1)
     dry-cli (1.0.0)
-    dry-core (1.0.1)
+    dry-core (1.1.0)
       concurrent-ruby (~> 1.0)
+      logger
       zeitwerk (~> 2.6)
-    dry-monads (1.6.0)
+    dry-monads (1.9.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 1.0, < 2)
+      dry-core (~> 1.1)
       zeitwerk (~> 2.6)
     dry-operation (1.0.0)
       dry-monads (~> 1.6)
@@ -189,10 +184,12 @@ GEM
     factory_bot_rails (6.4.3)
       factory_bot (~> 6.4)
       railties (>= 5.0.0)
-    faraday (2.9.0)
-      faraday-net_http (>= 2.0, < 3.2)
-    faraday-net_http (3.1.0)
-      net-http
+    faraday (2.13.4)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.1)
+      net-http (>= 0.5.0)
     faye-websocket (0.11.3)
       eventmachine (>= 0.12.0)
       websocket-driver (>= 0.5.1)
@@ -224,6 +221,7 @@ GEM
     jbuilder (2.12.0)
       actionview (>= 5.0.0)
       activesupport (>= 5.0.0)
+    json (2.13.2)
     json_schema (0.21.0)
     kaminari (1.2.2)
       activesupport (>= 4.1.0)
@@ -265,7 +263,7 @@ GEM
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     mutex_m (0.2.0)
-    net-http (0.4.1)
+    net-http (0.6.0)
       uri
     net-http-persistent (4.0.2)
       connection_pool (~> 2.2)
@@ -495,7 +493,7 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
-    uri (0.13.2)
+    uri (1.0.3)
     virtus (2.0.0)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
@@ -551,7 +549,7 @@ DEPENDENCIES
   capybara
   coveralls_reborn
   csv
-  datacite!
+  datacite (~> 0.4.0)
   devise
   dogstatsd-ruby
   dry-operation

spec/rails_helper.rb

@@ -84,7 +84,7 @@
     if ENV["RUN_IN_BROWSER"]
       driven_by(:selenium)
     else
-      driven_by(:selenium_headless)
+      driven_by(:chrome_headless)
     end
   end
 end

spec/support/system_specs.rb

@@ -1,4 +1,22 @@
 # frozen_string_literal: true
+Capybara.configure do |config|
+  config.default_driver = :chrome_headless
+end
+
+Capybara.register_driver :chrome_headless do |app|
+  client = Selenium::WebDriver::Remote::Http::Default.new
+  client.read_timeout = 120
+  options = Selenium::WebDriver::Chrome::Options.new(args: %w[disable-gpu no-sandbox headless whitelisted-ips window-size=1400,1400])
+  options.add_argument(
+    "--enable-features=NetworkService,NetworkServiceInProcess"
+  )
+  options.add_argument("--profile-directory=Default")
+  options.add_argument("--disable-dev-shm-usage")
+
+  Capybara::Selenium::Driver.new(app, browser: :chrome, options: options, http_client: client)
+end
+
+Capybara.javascript_driver = :chrome_headless
 
 RSpec.configure do |config|
   config.before(:each, type: :system) do
@@ -9,7 +27,7 @@
     if ENV["RUN_IN_BROWSER"]
       driven_by(:selenium)
     else
-      driven_by(:selenium_headless)
+      driven_by(:chrome_headless)
     end
   end
   config.before(:each, type: :system, js: true, in_browser: true) do