Added connector metadata to DownloadFile

Author: abujeda

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,15 @@
+**What this PR does / why we need it**:
+
+**Which issue(s) this PR closes**:
+
+Closes #
+
+**Special notes for your reviewer**:
+
+**Suggestions on how to test this**:
+
+**Does this PR introduce a user interface change? If mockups are available, please link/include them here**:
+
+**Is there a release notes update needed for this change?**:
+
+**Additional documentation**:

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: test
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches: [ main ]
+
+permissions:
+  contents: write
+
+jobs:
+  build-and-test-app:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ${{ github.sha }}
+        uses: actions/checkout@v4
+      - name: test app
+        working-directory: .
+        run: make tests
+

app/lib/logging_common.rb

@@ -4,8 +4,16 @@ def log_info(message, data = {})
     Rails.logger.info(format_log("INFO", message, data))
   end
 
-  def log_error(message, data = {})
-    Rails.logger.error(format_log("ERROR", message, data))
+  def log_error(message, data = {}, exception = nil)
+    log_message = format_log("ERROR", message, data)
+
+    if exception
+      # First 5 lines as a stack trace
+      log_message += "\n[STACK] " + exception.message
+      log_message += "\n[STACK] " + exception.backtrace&.first(5)&.join("\n[STACK] ")
+    end
+
+    Rails.logger.error(log_message)
   end
 
   private

app/models/dataverse/connector_download_processor.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Dataverse
+  class ConnectorMetadata
+    def initialize(download_file)
+      @metadata = download_file.metadata.to_h.deep_symbolize_keys
+      @metadata.each_key do |key|
+        define_singleton_method("#{key.to_s}="){ |value| @metadata[key] = value }
+        define_singleton_method(key){ @metadata[key] }
+      end
+    end
+
+    # To avoid errors when expected fields are removed from the list of configured attributes
+    def method_missing(method_name, *arguments, &block)
+      nil
+    end
+
+    def to_h
+      @metadata.deep_stringify_keys
+    end
+  end
+end

app/models/dataverse/connector_file_metadata.rb

@@ -7,12 +7,12 @@ class ConnectorStatus
 
     def initialize(file)
       @file = file
-      @connector_metadata = Dataverse::FileMetadata.new(file.connector_metadata)
+      @connector_metadata = file.connector_metadata
     end
 
     def download_progress
-      download_location = connector_metadata[:temp_location]
-      file_size = connector_metadata[:size]
+      download_location = connector_metadata.temp_location
+      file_size = file.size
       return 0 unless File.exist?(download_location) && file_size.to_i.positive?
 
       downloaded_size = File.size(download_location)

app/models/dataverse/connector_metadata.rb

@@ -3,13 +3,11 @@
 class DownloadCollection < ApplicationDiskRecord
   include ActiveModel::Model
 
-  ATTRIBUTES = %w[id type metadata_id name download_dir].freeze
-  TYPES = %w[dataverse]
+  ATTRIBUTES = %w[id name download_dir].freeze
 
   attr_accessor *ATTRIBUTES
 
   validates_presence_of *ATTRIBUTES
-  validates :type, inclusion: { in: TYPES }
 
   def self.all
     Dir.glob(File.join(metadata_directory, '*'))
@@ -27,8 +25,10 @@ def self.find(collection_id)
     load_from_file(filename)
   end
 
-  def initialize(id: nil, type: nil, name: nil, download_dir: nil, connector_metadata: {})
-    self.download_dir = download_dir || File.join(Configuration.download_root, id.to_s)
+  def initialize(id: nil, name: nil, download_dir: nil)
+    self.id = id || DownloadCollection.generate_id
+    self.name = name || self.id
+    self.download_dir = download_dir || File.join(Configuration.download_root, self.id.to_s)
   end
 
   def files

app/models/dataverse/connector_status.rb

@@ -4,7 +4,7 @@ class DownloadFile < ApplicationDiskRecord
   include ActiveModel::Model
   include LoggingCommon
 
-  ATTRIBUTES = %w[id collection_id type metadata_id external_id filename status size checksum content_type connector_metadata].freeze
+  ATTRIBUTES = %w[id collection_id type filename status size metadata].freeze
   TYPES = %w[dataverse].freeze
   STATUS = %w[ready downloading success error].freeze
 
@@ -55,6 +55,10 @@ def connector_status
     ConnectorClassDispatcher.file_connector_status(self)
   end
 
+  def connector_metadata
+    ConnectorClassDispatcher.connector_metadata(self)
+  end
+
   private
 
   #TODO: This needs to be taken from the DownloadCollection object

app/models/download_collection.rb

@@ -8,6 +8,10 @@ def self.file_connector_status(download_file)
     self.load(download_file.type, 'ConnectorStatus', download_file)
   end
 
+  def self.connector_metadata(download_file)
+    self.load(download_file.type, 'ConnectorMetadata', download_file)
+  end
+
   def self.download_processor(download_file)
     self.load(download_file.type, 'ConnectorDownloadProcessor', download_file)
   end