Merge pull request #24637 from opf/bug/jim-150-custom-fields-of-type-select-list-single-choice-are-not-migrated-correctly

[JIM-150] Custom fields of type Select List (single choice) are not migrated correctly

Author: as-op

app/workers/import/jira_import_projects_job.rb

@@ -151,15 +151,7 @@ def import_issue(jira_issue, project, custom_field_registry)
 
     def new_custom_fields_in_type(jira_issue, type, custom_field_registry)
       existing_cf_ids = type.custom_field_ids
-      cfs = custom_field_registry.filter_map do |entry|
-        field_key = entry[:jira_field].jira_field_id
-        raw_value = jira_issue.payload["fields"][field_key]
-        next if raw_value.blank?
-
-        context = find_context_for_issue(entry, jira_issue)
-        context&.dig(:custom_field)
-      end
-      cfs.uniq.reject { |cf| existing_cf_ids.include?(cf.id) }
+      custom_fields_for_issue(custom_field_registry, jira_issue).reject { |cf| existing_cf_ids.include?(cf.id) }
     end
 
     def update_custom_fields_in_type(type, new_custom_fields)
@@ -193,12 +185,9 @@ def add_or_update_jira_import_group(groups, cf_keys)
     end
 
     def update_custom_fields_in_project(project, jira_project, custom_field_registry)
-      project_key = jira_project.payload["key"]
-      applicable_cfs = custom_field_registry.flat_map do |entry|
-        entry[:contexts]
-          .select { |ctx| context_applies_to_project?(ctx, project_key) }
-          .map { |ctx| ctx[:custom_field] }
-      end
+      applicable_cfs = Import::JiraIssue
+                         .where(jira_import_id: @jira_import.id, jira_project_id: jira_project.id)
+                         .flat_map { |jira_issue| custom_fields_for_issue(custom_field_registry, jira_issue) }
       existing_cf_ids = project.work_package_custom_fields.pluck(:id).to_set
       new_cfs = applicable_cfs.uniq.reject { |cf| existing_cf_ids.include?(cf.id) }
       project.work_package_custom_fields << new_cfs if new_cfs.any?

app/workers/import/jira_import_projects_job/jira_import_custom_field_builder.rb

@@ -250,10 +250,14 @@ def list_field_option_values
         if cascading_select_as_list?
           flatten_cascading_allowed_values(allowed)
         else
-          allowed.pluck("value").compact.uniq
+          allowed.pluck("value").compact.map { |value| option_label(value) }.compact_blank.uniq
         end
       end
 
+      def option_label(value)
+        value.to_s.strip
+      end
+
       def cascading_select_as_list?
         schema = jira_field.payload["schema"] || {}
         schema["custom"].to_s.end_with?(":cascadingselect") &&
@@ -264,7 +268,7 @@ def cascading_select_as_list?
       # E.g. Animals -> ["Animals"], Animals / Cat -> ["Animals", "Animals / Cat"]
       def flatten_cascading_allowed_values(allowed_values, parent_path: nil)
         allowed_values.flat_map do |av|
-          label = av["value"]
+          label = option_label(av["value"])
           next [] if label.blank?
 
           full_path = parent_path ? "#{parent_path} / #{label}" : label
@@ -303,7 +307,7 @@ def jira_to_op_field_format(jira_field)
       def convert_multicheckbox_bool_value(raw_value)
         return false unless raw_value.is_a?(Array)
 
-        raw_value.any? { |v| v["value"] == @option_value }
+        raw_value.any? { |v| option_label(v["value"]) == @option_value }
       end
 
       def convert_user_value(raw_value)
@@ -341,7 +345,7 @@ def convert_single_list_value(raw_value, custom_field)
       end
 
       def extract_list_label(value)
-        value.is_a?(Hash) ? value["value"] : value.to_s
+        option_label(value.is_a?(Hash) ? value["value"] : value)
       end
 
       # Walks the parent -> child chain of a cascading select value, returning
@@ -350,7 +354,7 @@ def extract_list_label(value)
       def extract_cascading_chain(value, parent_path: nil)
         return [] unless value.is_a?(Hash) && value["value"].present?
 
-        label = value["value"]
+        label = option_label(value["value"])
         full_path = parent_path ? "#{parent_path} / #{label}" : label
         [full_path] + extract_cascading_chain(value["child"], parent_path: full_path)
       end
@@ -369,7 +373,7 @@ def populate_hierarchy_items(custom_field)
       end
 
       def insert_hierarchy_option(service, contract, parent, option)
-        label = option["value"]
+        label = option_label(option["value"])
         return if label.blank?
 
         result = service.insert_item(contract_class: contract, parent:, label:)
@@ -398,7 +402,7 @@ def convert_hierarchy_value(raw_value, custom_field)
         root = custom_field.hierarchy_root
         return unless root
 
-        parent_item = root.children.find_by(label: raw_value["value"])
+        parent_item = root.children.find_by(label: option_label(raw_value["value"]))
         return unless parent_item
 
         find_hierarchy_child(parent_item, raw_value["child"])&.id || parent_item.id
@@ -407,7 +411,7 @@ def convert_hierarchy_value(raw_value, custom_field)
       def find_hierarchy_child(parent_item, child_data)
         return unless child_data.is_a?(Hash) && child_data["value"].present?
 
-        parent_item.children.find_by(label: child_data["value"])
+        parent_item.children.find_by(label: option_label(child_data["value"]))
       end
 
       def find_field_user(jira_user_key)

app/workers/import/jira_import_projects_job/jira_import_custom_fields.rb

@@ -47,6 +47,15 @@ def collect_custom_field_attributes(custom_field_registry, jira_issue)
         end
       end
 
+      def custom_fields_for_issue(custom_field_registry, jira_issue)
+        custom_field_registry.filter_map do |entry|
+          raw_value = jira_issue.payload["fields"][entry[:jira_field].jira_field_id]
+          next if raw_value.blank?
+
+          find_context_for_issue(entry, jira_issue)&.dig(:custom_field)
+        end.uniq
+      end
+
       # Builds one OP custom field per (Jira field, context group) combination, before any
       # per-project import begins. Context groups describe which (project_key, issuetype_id)
       # tuples share an allowedValues set.
@@ -98,59 +107,30 @@ def string_array_field?(jira_field)
         schema["type"] == "array" && schema["items"] == "string"
       end
 
-      def build_string_array_registry_entries(jira_field)
-        string_values = collect_string_values_from_issues(jira_field)
-        allowed_values = string_values.map { |v| { "value" => v } }
-        groups = jira_field.payload["contextGroups"]
-
-        contexts = if groups.present?
-                     groups.map { |g| build_context_entry(jira_field, g.merge("allowedValues" => allowed_values)) }
-                   else
-                     [
-                       build_context_entry(
-                         jira_field,
-                         {
-                           "projects" => [],
-                           "issuetypes" => [],
-                           "allowedValues" => allowed_values
-                         }
-                       )
-                     ]
-                   end
-        [{ jira_field:, contexts: }]
+      def option_field?(jira_field)
+        schema = jira_field.payload["schema"] || {}
+        %w[option option-with-child].include?(schema["type"]) ||
+          (schema["type"] == "array" && schema["items"] == "option")
       end
 
-      def collect_string_values_from_issues(jira_field)
-        field_key = jira_field.jira_field_id
-        values = Set.new
-        Import::JiraIssue.where(jira_id: @jira_id, jira_project_id: all_jira_import_project_ids).find_each do |issue|
-          raw = issue.payload["fields"][field_key]
-          next unless raw.is_a?(Array)
-
-          raw.each { |v| values << v.to_s if v.present? }
+      def build_string_array_registry_entries(jira_field)
+        allowed_values = Array(jira_field.payload["contextGroups"]).flat_map { |group| Array(group["allowedValues"]) }
+        allowed_values = issue_string_values(jira_field).reduce(allowed_values) do |values, string_value|
+          merge_allowed_value(values, { "value" => string_value })
         end
-        values.to_a.sort
+        contexts = [
+          build_context_entry(jira_field, { "projects" => [], "issuetypes" => [], "allowedValues" => allowed_values })
+        ]
+        [{ jira_field:, contexts: }]
       end
 
       def build_multicheckbox_registry_entries(jira_field)
-        groups = jira_field.payload["contextGroups"]
-
-        return build_multicheckbox_entries_without_context_groups(jira_field) if groups.blank?
+        groups = augmented_context_groups(jira_field)
+        return [] if groups.blank?
 
         build_multicheckbox_entries_with_context_groups(jira_field, groups)
       end
 
-      def build_multicheckbox_entries_without_context_groups(jira_field)
-        option_values = collect_option_values_from_issues(jira_field)
-        return [] if option_values.empty?
-
-        if option_values.size == 1
-          [{ jira_field:, contexts: [build_context_entry(jira_field, nil, option_value: option_values.first)] }]
-        else
-          [{ jira_field:, contexts: [build_context_entry(jira_field, nil)] }]
-        end
-      end
-
       def build_multicheckbox_entries_with_context_groups(jira_field, groups)
         boolean_groups, list_groups = partition_multicheckbox_groups_by_value_count(groups)
 
@@ -162,7 +142,7 @@ def partition_multicheckbox_groups_by_value_count(groups)
         list_groups = []
 
         groups.each do |group|
-          option_values = Array(group["allowedValues"]).pluck("value").compact.uniq
+          option_values = option_labels(group["allowedValues"])
           if option_values.size == 1
             boolean_groups << { group:, option_value: option_values.first }
           elsif option_values.size > 1
@@ -197,19 +177,8 @@ def all_jira_import_project_ids
           .pluck(:id)
       end
 
-      def collect_option_values_from_issues(jira_field)
-        values = Set.new
-        Import::JiraIssue.where(jira_id: @jira_id, jira_project_id: all_jira_import_project_ids).find_each do |issue|
-          raw = issue.payload["fields"][jira_field.jira_field_id]
-          next unless raw.is_a?(Array)
-
-          raw.each { |v| values << v["value"] if v["value"].present? }
-        end
-        values.to_a.sort
-      end
-
       def build_contexts_for_field(jira_field)
-        groups = jira_field.payload["contextGroups"]
+        groups = augmented_context_groups(jira_field)
         if groups.present?
           needs_disambiguation = groups.size > 1
           groups.map { |group| build_context_entry(jira_field, group, needs_disambiguation:) }
@@ -218,6 +187,121 @@ def build_contexts_for_field(jira_field)
         end
       end
 
+      def augmented_context_groups(jira_field)
+        groups = dup_context_groups(jira_field)
+        return groups unless option_field?(jira_field)
+
+        issue_options = issue_option_values(jira_field)
+        return groups if issue_options.empty?
+
+        groups << global_context_group if groups.empty?
+        issue_options.each { |option, *scope| add_option_to_context_group(groups, option, scope) }
+        groups
+      end
+
+      def dup_context_groups(jira_field)
+        Array(jira_field.payload["contextGroups"]).map do |group|
+          group.merge("allowedValues" => Array(group["allowedValues"]))
+        end
+      end
+
+      def global_context_group
+        { "projects" => [], "issuetypes" => [], "allowedValues" => [] }
+      end
+
+      def add_option_to_context_group(groups, option, scope)
+        group = groups[matching_context_group_index(groups, *scope)]
+        group["allowedValues"] = merge_allowed_value(group["allowedValues"], option)
+      end
+
+      def merge_allowed_value(allowed_values, option)
+        label = option["value"].to_s.strip
+        return allowed_values if label.blank?
+
+        existing = allowed_values.find { |av| av["value"].to_s.strip == label }
+        return allowed_values + [build_allowed_value(label, option["child"])] if existing.nil?
+
+        merge_allowed_child(allowed_values, existing, option["child"])
+      end
+
+      def merge_allowed_child(allowed_values, existing, child)
+        return allowed_values if child.blank?
+
+        merged = existing.merge("children" => merge_allowed_value(Array(existing["children"]), child))
+        allowed_values.map { |av| av.equal?(existing) ? merged : av }
+      end
+
+      def build_allowed_value(label, child)
+        entry = { "value" => label }
+        entry["children"] = merge_allowed_value([], child) if child.present?
+        entry
+      end
+
+      def matching_context_group_index(groups, project_key, issuetype_id)
+        groups.index do |group|
+          scope_applies?(group["projects"], project_key) && scope_applies?(group["issuetypes"], issuetype_id)
+        end || 0
+      end
+
+      def option_labels(allowed_values)
+        Array(allowed_values).pluck("value").compact.map { |value| value.to_s.strip }.compact_blank.uniq
+      end
+
+      def issue_option_values(jira_field)
+        issue_field_values_index[:options][jira_field.jira_field_id].values
+      end
+
+      def issue_string_values(jira_field)
+        issue_field_values_index[:strings][jira_field.jira_field_id].to_a.sort
+      end
+
+      def issue_field_values_index
+        @issue_field_values_index ||= build_issue_field_values_index
+      end
+
+      def build_issue_field_values_index
+        index = { options: Hash.new { |h, k| h[k] = {} }, strings: Hash.new { |h, k| h[k] = Set.new } }
+        Import::JiraIssue.where(jira_id: @jira_id, jira_project_id: all_jira_import_project_ids).find_each do |issue|
+          scope = issue_context_scope(issue)
+          used_custom_field_values(issue).each { |field_key, raw| record_issue_field_values(index, field_key, raw, scope) }
+        end
+        index
+      end
+
+      def issue_context_scope(issue)
+        [issue.payload.dig("fields", "project", "key"), issue.payload.dig("fields", "issuetype", "id")]
+      end
+
+      def used_custom_field_values(issue)
+        issue.payload["fields"].select { |field_key, raw| field_key.start_with?("customfield_") && raw.present? }
+      end
+
+      def record_issue_field_values(index, field_key, raw, scope)
+        Array.wrap(raw).each do |value|
+          if value.is_a?(Hash)
+            record_issue_option_value(index[:options][field_key], value, scope)
+          elsif raw.is_a?(Array) && value.is_a?(String)
+            index[:strings][field_key] << value.strip if value.strip.present?
+          end
+        end
+      end
+
+      def record_issue_option_value(field_options, option, scope)
+        return if option["value"].blank?
+
+        field_options[scope + [option_chain_signature(option)]] ||= [option, *scope]
+      end
+
+      def option_chain_signature(option)
+        labels = []
+        node = option
+        while node.is_a?(Hash) && node["value"].present?
+          labels << node["value"].to_s.strip
+          node = node["child"]
+        end
+        labels.join(" / ")
+      end
+
       def build_context_entry(jira_field, context_group, option_value: nil, needs_disambiguation: false)
         builder = JiraImportCustomFieldBuilder.new(
           jira_field,
@@ -287,11 +371,17 @@ def find_context_for_issue(entry, jira_issue)
       end
 
       def context_applies_to_project?(context, project_key)
-        context[:projects].empty? || context[:projects].include?(project_key)
+        scope_applies?(context[:projects], project_key)
       end
 
       def context_applies_to_issuetype?(context, issuetype_id)
-        context[:issuetypes].empty? || context[:issuetypes].include?(issuetype_id)
+        scope_applies?(context[:issuetypes], issuetype_id)
+      end
+
+      # An empty scope means "applies to all projects" / "applies to all issue types".
+      def scope_applies?(scope, value)
+        scope = Array(scope)
+        scope.empty? || scope.include?(value)
       end
     end
   end