01 - Initial.R

# Jihoon Lim
# 01 - Initial Screening
# Health Canada Report
# February 19, 2025

library(xml2)
library(stringdist) # For string distance calculations (fuzzy matching)
library(stringr) # For regular expressions

#### Part A: De-Duplication ####
# 1. Read XML file
xml_file <- "C:/Users/limji/Desktop/Research Associate/Health Canada/0 - Raw.xml"
xml_data <- read_xml(xml_file)

# 2. Find all records
records <- xml_find_all(xml_data, ".//record")
seen_titles <- character(0)

# 3. Create lists for included and excluded records
inclusion_A <- list()
exclusion_A <- list()

# 4. Iterate through records and classify them based on duplicate detection
for (record in records) {
  title_element <- xml_find_first(record, ".//title")
  title <- xml_text(title_element)
  
  if (!is.na(title) && title != "") {
    if (is_duplicate(title, seen_titles)) {
      exclusion_A <- c(exclusion_A, list(record))
    } else {
      seen_titles <- c(seen_titles, tolower(title))
      inclusion_A <- c(inclusion_A, list(record))
    }
  } else {
    # If there is no title, you might choose to include or exclude. Here we include.
    inclusion_A <- c(inclusion_A, list(record))
  }
}

# 5. Build new XML trees for included and excluded articles (de-duplication stage)
# Create a new root element for each XML document.
included_xml_A <- xml_new_root("records")
for (rec in inclusion_A) {
  xml_add_child(included_xml_A, rec)
}
excluded_xml_A <- xml_new_root("records")
for (rec in exclusion_A) {
  xml_add_child(excluded_xml_A, rec)
}

# 6. Write out the XML files for Part A
output_included_A <- "C:/Users/limji/Desktop/Research Associate/Health Canada/1 - Deduplicate - I.xml"
output_excluded_A <- "C:/Users/limji/Desktop/Research Associate/Health Canada/1 - Deduplicate - E.xml"
write_xml(included_xml_A, output_included_A)
write_xml(excluded_xml_A, output_excluded_A)

cat("Included articles:", length(inclusion_A), "\n")
cat("Duplicate articles:", length(exclusion_A), "\n")

#### Part B: Screening ####
# 1. Read the de-duplicated included file from Part A
xml_data_B <- read_xml(output_included_A)

# 2. Find all records
records_B <- xml_find_all(xml_data_B, ".//record")

# 3. Create lists for included and excluded records for screening stage
inclusion_B <- list()
exclusion_B <- list()

# 4. Iterate through records and classify based on screening conditions
for (record in records_B) {
  title_element <- xml_find_first(record, ".//title")
  title <- xml_text(title_element)
  
  if (is_irrelevant(title) || is_digest_brief(title)) {
    exclusion_B <- c(exclusion_B, list(record))
  } else {
    inclusion_B <- c(inclusion_B, list(record))
  }
}

# 5. Build new XML trees for Part B included and excluded articles
included_xml_B <- xml_new_root("records")
for (rec in inclusion_B) {
  xml_add_child(included_xml_B, rec)
}

excluded_xml_B <- xml_new_root("records")
for (rec in exclusion_B) {
  xml_add_child(excluded_xml_B, rec)
}

# 6. Write out the XML files for Part B
output_included_B <- "C:/Users/limji/Desktop/Research Associate/Health Canada/2 - Screen - I.xml"
output_excluded_B <- "C:/Users/limji/Desktop/Research Associate/Health Canada/2 - Screen - E.xml"
write_xml(included_xml_B, output_included_B)
write_xml(excluded_xml_B, output_excluded_B)

cat("Included articles:", length(inclusion_B), "\n")
cat("Excluded articles (screened out):", length(exclusion_B), "\n")