save_rds_from_prototype.R

# Auto-generated script to extract and save .rds objects from Prototype.qmd
library(tidyverse)
library(igraph)
library(ggraph)
library(tidygraph)

# Set output directory
dir.create("rds_objects", showWarnings = FALSE)

# ------------------
library(shiny)
library(wordcloud)
library(RColorBrewer)

# Data and color palette
word <- c("vessels", "mining", "funding", "music", "reef", "conservation", "turtles",
          "fuel", "permit", "?", "suspicious", "tourism", "lighting", "meetings",
          "council", "harbor", "relationships", "communications", "operation", "underwater")
freq <- c(5, 8, 7, 8, 9, 5, 6, 10, 8, 4, 3, 3, 5, 2, 3, 2, 5, 8, 3, 4)
word_data_wc <- data.frame(word = word, freq = freq)

ocean_colors_lp <- c(
  "#1B1B3A", "#0072B2", "#009E73", "#D55E00", "#CC79A7",
  "#882255", "#AA4499", "#004D40", "#333333"
)

# Define UI
ui <- fluidPage(
  titlePanel(" "),
  plotOutput("wordcloudPlot", height = "500px")  # Increased height
)

# Define Server
server <- function(input, output, session) {
  output$wordcloudPlot <- renderPlot({
    par(bg = "#f0f0fb")  # Set background inside renderPlot

    wordcloud(
      words = word_data_wc$word,
      freq = word_data_wc$freq,
      min.freq = 1,
      max.words = 200,
      random.order = FALSE,
      colors = ocean_colors_lp,
      rot.per = 0.20,
      scale = c(4, 0.8)
    )
  })
}

# Launch App
shinyApp(ui, server)


saveRDS(server, file = file.path("rds_objects", "server.rds"))
saveRDS(word, file = file.path("rds_objects", "word.rds"))
saveRDS(freq, file = file.path("rds_objects", "freq.rds"))
saveRDS(ui, file = file.path("rds_objects", "ui.rds"))
saveRDS(ocean_colors_lp, file = file.path("rds_objects", "ocean_colors_lp.rds"))
saveRDS(word_data_wc, file = file.path("rds_objects", "word_data_wc.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
pacman::p_load(
  jsonlite, tidyverse, ggtext,
  knitr, lubridate, hms, scales,
  tidytext, dplyr, tm, SnowballC, 
  patchwork, ggraph, 
  tidygraph, igraph, ggiraph, 
  SmartEDA, plotly, wordcloud, 
  ggh4x, visNetwork, ggplot2,
  RColorBrewer, circlize, 
  stringr, wordcloud2, ggalluvial,
  reactable, readr)


# ------------------
#| code-fold: true
#| code-summary: "Show the code"
mc3_data <- fromJSON("data/mc3_graph.json")
mc3_schema <- fromJSON("data/MC3_schema.json")

saveRDS(mc3_data, file = file.path("rds_objects", "mc3_data.rds"))
saveRDS(mc3_schema, file = file.path("rds_objects", "mc3_schema.rds"))

# ------------------
node_legend_colors_plot <- c(
  "Person" = "#88CCEE",
  "Vessel" = "#D55E00",
  "Organization" = "#117733",
  "Location" = "#AA4499",
  "Group"= "#CC79A7",
  "Event" = "#DDCC77",
  "Relationship" = "#AF8DC3",
  "Nadia Conti" = "red"
)

node_legend_shapes_plot <- c(
  "Person" = "dot",
  "Vessel" = "triangle",
  "Organization" = "square",
  "Location" = "diamond",
  "Group" = "circle plus",
  "Event" = "star",
  "Relationship" = "square x",
  "Nadia Conti" = "star"
)

STYLES <- list(
  node_label_dark = "black",
  font_family = "Roboto Condensed"
)

saveRDS(node_legend_shapes_plot, file = file.path("rds_objects", "node_legend_shapes_plot.rds"))
saveRDS(node_legend_colors_plot, file = file.path("rds_objects", "node_legend_colors_plot.rds"))
saveRDS(STYLES, file = file.path("rds_objects", "STYLES.rds"))

# ------------------
glimpse(mc3_data)


# ------------------
mc3_nodes_raw <- as_tibble(mc3_data$nodes)
mc3_edges_raw <- as_tibble(mc3_data$edges)

saveRDS(mc3_edges_raw, file = file.path("rds_objects", "mc3_edges_raw.rds"))
saveRDS(mc3_nodes_raw, file = file.path("rds_objects", "mc3_nodes_raw.rds"))

# ------------------
ExpData(data=mc3_nodes_raw,type=2)


# ------------------
ExpData(data=mc3_edges_raw,type=2)


# ------------------
library(shiny)
library(ggplot2)

# Run ExpCatViz once at the top to avoid recomputing
ExpCatViz(data=mc3_nodes_raw,
          col="navyblue")



# ------------------
# Step 1: Count and reorder
mc3_nodes_ordered <- mc3_nodes_raw %>%
  count(sub_type) %>%
  arrange((n)) %>%
  mutate(sub_type = factor(sub_type, levels = sub_type))

# Step 2: Plot with navy bars, sorted, and horizontal
ggplot(mc3_nodes_ordered, aes(x = sub_type, y = n)) +
  geom_col(fill = "navy") +
  coord_flip() +
  labs(x = "Sub_type", y = "Count",
    title = "Distribution of Subtypes") +
  theme_minimal()

saveRDS(mc3_nodes_ordered, file = file.path("rds_objects", "mc3_nodes_ordered.rds"))

# ------------------
# Step 1: Filter for type == "Entity", count sub_type, sort 
relationship_subtypes <- mc3_nodes_raw %>%
  filter(type == "Entity") %>%
  count(sub_type) %>%
  arrange(n) %>%
  mutate(sub_type = factor(sub_type, levels = sub_type)) 

# Step 2: Plot
ggplot(relationship_subtypes, aes(x = sub_type, y = n)) +
  geom_col(fill = "navy") +
  coord_flip() +
  labs(
    x = "Entity Subtype",
    y = "Count",
    title = "Distribution of Entity Subtypes"
  ) +
  theme_minimal()

saveRDS(relationship_subtypes, file = file.path("rds_objects", "relationship_subtypes.rds"))

# ------------------
# Step 1: Filter for type == "Event", count sub_type, sort 
relationship_subtypes <- mc3_nodes_raw %>%
  filter(type == "Event") %>%
  count(sub_type) %>%
  arrange(n) %>%
  mutate(sub_type = factor(sub_type, levels = sub_type)) 

# Step 2: Plot
ggplot(relationship_subtypes, aes(x = sub_type, y = n)) +
  geom_col(fill = "navy") +
  coord_flip() +
  labs(
    x = "Event Subtype",
    y = "Count",
    title = "Distribution of Event Subtypes"
  ) +
  theme_minimal()

saveRDS(relationship_subtypes, file = file.path("rds_objects", "relationship_subtypes.rds"))

# ------------------
# Step 1: Filter for type == "Relationship", count sub_type, sort 
relationship_subtypes <- mc3_nodes_raw %>%
  filter(type == "Relationship") %>%
  count(sub_type) %>%
  arrange(n) %>%
  mutate(sub_type = factor(sub_type, levels = sub_type))

# Step 2: Plot
ggplot(relationship_subtypes, aes(x = sub_type, y = n)) +
  geom_col(fill = "navy") +
  coord_flip() +
  labs(
    x = "Relationship Subtype",
    y = "Count",
    title = "Distribution of Relationship Subtypes"
  ) +
  theme_minimal()

saveRDS(relationship_subtypes, file = file.path("rds_objects", "relationship_subtypes.rds"))

# ------------------
ExpCatViz(data=mc3_edges_raw,
          col="navyblue")


# ------------------
# Step 1: Filter for type == "sent"
filtered_edges <- mc3_edges_raw %>%
  filter(type == "sent") %>%
  count(source) %>%
  arrange(desc(n)) %>%
  mutate(source = factor(source, levels = rev(unique(source))))  # descending 

# Step 2: Plot
ggplot(filtered_edges, aes(x = source, y = n)) +
  geom_col(fill = "navy") +
  coord_flip() +
  labs(
    title = "Distribution of 'sent' Edges type by Source",
    x = "Source",
    y = "Count"
  ) +
  theme_minimal()

saveRDS(filtered_edges, file = file.path("rds_objects", "filtered_edges.rds"))

# ------------------
mc3_nodes_cleaned <- mc3_nodes_raw %>%
  mutate(id = as.character(id)) %>%
  filter(!is.na(id)) %>%
  distinct(id, .keep_all = TRUE) %>%
  select(-thing_collected, -time, -date, -friendship_type)

saveRDS(mc3_nodes_cleaned, file = file.path("rds_objects", "mc3_nodes_cleaned.rds"))

# ------------------
# Find the number of unique types in each column and sort descending
unique_counts <- mc3_nodes_cleaned %>%
  summarise_all(n_distinct) %>%
  pivot_longer(cols = everything(), names_to = "column", values_to = "unique_count") %>%
  arrange(desc(unique_count))  # sort by unique_count in descending order

# Print the result
print(unique_counts)

saveRDS(unique_counts, file = file.path("rds_objects", "unique_counts.rds"))

# ------------------
mc3_edges_cleaned <- mc3_edges_raw %>%
  rename(from_id = source,
         to_id = target) %>%
  mutate(across(c(from_id, to_id), as.character)) %>%
  # Parse to_id to get supertype and sub_type for target nodes (e.g., Event_Communication)
  separate(to_id, into = c("to_id_supertype", "to_id_sub_type", "to_id_num"),
           sep = "_", remove = FALSE, fill = "right", extra = "merge") %>%
  # Filter to ensure from_id and to_id exist in mc3_nodes_cleaned (prevent orphaned edges)
  filter(from_id %in% mc3_nodes_cleaned$id,
         to_id %in% mc3_nodes_cleaned$id) %>%
  filter(!is.na(from_id), !is.na(to_id))

print("Columns in mc3_edges_cleaned after initial cleaning:")
print(colnames(mc3_edges_cleaned))
print("Head of mc3_edges_cleaned after initial cleaning:")
print(head(mc3_edges_cleaned))


saveRDS(mc3_edges_cleaned, file = file.path("rds_objects", "mc3_edges_cleaned.rds"))

# ------------------
# Find the number of unique types in each column
unique_counts <- mc3_edges_cleaned %>%
  summarise_all(n_distinct) %>%
  pivot_longer(cols = everything(), names_to = "column", values_to = "unique_count")

# Print the unique counts for each column
print(unique_counts)

saveRDS(unique_counts, file = file.path("rds_objects", "unique_counts.rds"))

# ------------------
node_index_lookup <- mc3_nodes_cleaned %>%
  mutate(.row_id = row_number()) %>%
  select(id, .row_id)

saveRDS(node_index_lookup, file = file.path("rds_objects", "node_index_lookup.rds"))

# ------------------
mc3_edges_indexed <- mc3_edges_cleaned %>%
  left_join(node_index_lookup, by = c("from_id" = "id")) %>%
  rename(from = .row_id) %>%
  left_join(node_index_lookup, by = c("to_id" = "id")) %>%
  rename(to = .row_id) %>%
  # Filter out edges where either source or target node was not found
  filter(!is.na(from) & !is.na(to)) %>%
  # Select all columns to carry forward to mc3_edges_final
  select(from, to, id, is_inferred, type, # Original edge attributes
         from_id, to_id, to_id_supertype, to_id_sub_type, to_id_num # Original IDs and parsed target type
         )

saveRDS(mc3_edges_indexed, file = file.path("rds_objects", "mc3_edges_indexed.rds"))

# ------------------
used_node_indices <- sort(unique(c(mc3_edges_indexed$from, mc3_edges_indexed$to)))
mc3_nodes_final <- mc3_nodes_cleaned %>%
  slice(used_node_indices) %>%
  mutate(new_index = row_number())

saveRDS(mc3_nodes_final, file = file.path("rds_objects", "mc3_nodes_final.rds"))
saveRDS(used_node_indices, file = file.path("rds_objects", "used_node_indices.rds"))

# ------------------
old_to_new_index <- tibble(
  old_index = used_node_indices,
  new_index = seq_along(used_node_indices)
)

saveRDS(old_to_new_index, file = file.path("rds_objects", "old_to_new_index.rds"))

# ------------------
mc3_edges_final <- mc3_edges_indexed %>%
  left_join(old_to_new_index, by = c("from" = "old_index")) %>%
  rename(from_new = new_index) %>%
  left_join(old_to_new_index, by = c("to" = "old_index")) %>%
  rename(to_new = new_index) %>%
  # Explicitly select all columns that are needed downstream
  select(from = from_new, to = to_new,
         id, is_inferred, type,
         from_id, to_id, to_id_supertype, to_id_sub_type, to_id_num)

saveRDS(mc3_edges_final, file = file.path("rds_objects", "mc3_edges_final.rds"))

# ------------------
mc3_graph <- tbl_graph(
  nodes = mc3_nodes_final,
  edges = mc3_edges_final,
  directed = TRUE
)

saveRDS(mc3_graph, file = file.path("rds_objects", "mc3_graph.rds"))

# ------------------
str(mc3_graph)


# ------------------
# ---- 1. Define styles and legends ----

event_subtypes <- c(
  "Communication", "Monitoring", "VesselMovement", "Assessment",
  "Collaborate", "Endorsement", "TourActivity", "TransponderPing",
  "Harbor Report", "Fishing", "Criticize"
)

relationship_subtypes <- c(
  "Coordinates", "AccessPermission", "Operates", "Colleagues",
  "Suspicious", "Reports", "Jurisdiction", "Unfriendly", "Friends"
)

node_legend_colors_plot <- c(
  "Person" = "#88CCEE",
  "Vessel" = "#D55E00",
  "Organization" = "#117733",
  "Location" = "#AA4499",
  "Group"= "#CC79A7",
  "Event" = "#DDCC77",         # type level
  "Relationship" = "#AF8DC3"   # type level
)

node_legend_shapes_plot <- c(
  "Person" = "dot",
  "Vessel" = "triangle",
  "Organization" = "square",
  "Location" = "diamond",
  "Group" = "circle plus",
  "Event" = "star",              # type level
  "Relationship" = "square x"    # type level
)

STYLES <- list(
  node_label_dark = "black",
  font_family = "Roboto Condensed"
)

# ---- 2. Prepare nodes ----
nodes <- mc3_nodes_final %>%
  mutate(
    label = ifelse(is.na(name), id, name),
    
    # These parts are for pulling the related data from other fields
    tooltip_extra = case_when(
      type == "Event" & sub_type == "Communication" ~ content,
      type == "Event" & sub_type == "Monitoring" ~ findings,
      type == "Event" & sub_type == "VesselMovement" ~ destination,
      type == "Event" & sub_type == "Assessment" ~ results,
      type == "Relationship" & sub_type == "Coordinates" ~ coordination_type,
      type == "Relationship" & sub_type == "Operates" ~ operational_role,
      type == "Relationship" & sub_type == "Jurisdiction" ~ jurisdiction_type,
      TRUE ~ NA_character_
    ),
    
    title = paste0(
      "<b>", label, "</b><br>",
      "Type: ", type, "<br>",
      "Sub-type: ", sub_type, "<br>",
      ifelse(!is.na(tooltip_extra), paste0("<br><b>Details:</b> ", tooltip_extra), "")
    ),
    
    # Fallback logic: if sub_type is NA or not in styling list, use type instead
    group = ifelse(sub_type %in% names(node_legend_colors_plot), sub_type, type)
  ) %>%
  select(id, label, group, title) %>%
  distinct()

# ---- 3. Prepare directed edges (type == "sent") ----

edges <- mc3_edges_final %>%
  filter(from_id %in% nodes$id & to_id %in% nodes$id) %>%
  select(from = from_id, to = to_id)

# ---- 4. Build visNetwork ----

net <- visNetwork(nodes, edges, width = "100%", height = "600px") %>%
  visEdges(arrows = list(to = list(enabled = TRUE, scaleFactor = 1.5))) %>%
  visOptions(highlightNearest = TRUE, nodesIdSelection = TRUE) %>%
  visIgraphLayout(layout = "layout_with_fr") %>%
  visNodes(font = list(
    size = 14,
    color = STYLES$node_label_dark,
    face = STYLES$font_family,
    vadjust = -15
  ))

# ---- 5. Apply shape and color per group ----

for (group_name in names(node_legend_colors_plot)) {
  net <- net %>% visGroups(
    groupname = group_name,
    color = node_legend_colors_plot[[group_name]],
    shape = node_legend_shapes_plot[[group_name]]
  )
}
# ---- 6. Add legend ----

used_groups <- unique(nodes$group)

legend_df <- tibble::tibble(
  label = used_groups,
  shape = node_legend_shapes_plot[used_groups],
  color = node_legend_colors_plot[used_groups]
) %>%
  distinct(label, .keep_all = TRUE)  # remove duplicates just in case

net <- net %>% visLegend(
  addNodes = legend_df,
  ncol = 2,                         # number of columns
  position = "left",              
  main = "Entity (Sub)Types",      # title
  useGroups = FALSE                # show custom legend entries
)
# ---- 7. Render ----
net


saveRDS(nodes, file = file.path("rds_objects", "nodes.rds"))
saveRDS(relationship_subtypes, file = file.path("rds_objects", "relationship_subtypes.rds"))
saveRDS(node_legend_shapes_plot, file = file.path("rds_objects", "node_legend_shapes_plot.rds"))
saveRDS(legend_df, file = file.path("rds_objects", "legend_df.rds"))
saveRDS(used_groups, file = file.path("rds_objects", "used_groups.rds"))
saveRDS(node_legend_colors_plot, file = file.path("rds_objects", "node_legend_colors_plot.rds"))
saveRDS(net, file = file.path("rds_objects", "net.rds"))
saveRDS(event_subtypes, file = file.path("rds_objects", "event_subtypes.rds"))
saveRDS(STYLES, file = file.path("rds_objects", "STYLES.rds"))
saveRDS(edges, file = file.path("rds_objects", "edges.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
# ---- 1. Define styles and legends ----

event_subtypes <- c(
  "Communication", "Monitoring", "VesselMovement", "Assessment",
  "Collaborate", "Endorsement", "TourActivity", "TransponderPing",
  "Harbor Report", "Fishing", "Criticize"
)

relationship_subtypes <- c(
  "Coordinates", "AccessPermission", "Operates", "Colleagues",
  "Suspicious", "Reports", "Jurisdiction", "Unfriendly", "Friends"
)

node_legend_colors_plot <- c(
  "Person" = "#88CCEE",
  "Vessel" = "#D55E00",
  "Organization" = "#117733",
  "Location" = "#AA4499",
  "Group"= "#CC79A7",
  "Event" = "#DDCC77",         # type level
  "Relationship" = "#AF8DC3"   # type level
)

node_legend_shapes_plot <- c(
  "Person" = "dot",
  "Vessel" = "triangle",
  "Organization" = "square",
  "Location" = "diamond",
  "Group" = "circle plus",
  "Event" = "star",              # type level
  "Relationship" = "square x"    # type level
)

STYLES <- list(
  node_label_dark = "black",
  font_family = "Roboto Condensed"
)

# ---- 2. Prepare nodes ----
nodes <- mc3_nodes_final %>%
  mutate(
    label = ifelse(is.na(name), id, name),
    
    # These parts are for pulling the related data from other fields
    tooltip_extra = case_when(
      type == "Event" & sub_type == "Communication" ~ content,
      type == "Event" & sub_type == "Monitoring" ~ findings,
      type == "Event" & sub_type == "VesselMovement" ~ destination,
      type == "Event" & sub_type == "Assessment" ~ results,
      type == "Relationship" & sub_type == "Coordinates" ~ coordination_type,
      type == "Relationship" & sub_type == "Operates" ~ operational_role,
      type == "Relationship" & sub_type == "Jurisdiction" ~ jurisdiction_type,
      TRUE ~ NA_character_
    ),
    
    title = paste0(
      "<b>", label, "</b><br>",
      "Type: ", type, "<br>",
      "Sub-type: ", sub_type, "<br>",
      ifelse(!is.na(tooltip_extra), paste0("<br><b>Details:</b> ", tooltip_extra), "")
    ),
    
    # Fallback logic: if sub_type is NA or not in styling list, use type instead
    group = ifelse(sub_type %in% names(node_legend_colors_plot), sub_type, type)
  ) %>%
  select(id, label, group, title) %>%
  distinct()

# ---- 3. Prepare directed edges (type == "sent") ----

edges <- mc3_edges_final %>%
  filter(from_id %in% nodes$id & to_id %in% nodes$id) %>%
  select(from = from_id, to = to_id)

# ---- 4. Build visNetwork ----

net <- visNetwork(nodes, edges, width = "100%", height = "600px") %>%
  visEdges(arrows = list(to = list(enabled = TRUE, scaleFactor = 1.5))) %>%
  visOptions(highlightNearest = TRUE, nodesIdSelection = TRUE) %>%
  visIgraphLayout(layout = "layout_with_fr") %>%
  visNodes(font = list(
    size = 14,
    color = STYLES$node_label_dark,
    face = STYLES$font_family,
    vadjust = -15
  ))

# ---- 5. Apply shape and color per group ----

for (group_name in names(node_legend_colors_plot)) {
  net <- net %>% visGroups(
    groupname = group_name,
    color = node_legend_colors_plot[[group_name]],
    shape = node_legend_shapes_plot[[group_name]]
  )
}
# ---- 6. Add legend ----

used_groups <- unique(nodes$group)

legend_df <- tibble::tibble(
  label = used_groups,
  shape = node_legend_shapes_plot[used_groups],
  color = node_legend_colors_plot[used_groups]
) %>%
  distinct(label, .keep_all = TRUE)  # remove duplicates just in case

net <- net %>% visLegend(
  addNodes = legend_df,
  ncol = 2,                         # number of columns
  position = "left",              
  main = "Entity (Sub)Types",      # title
  useGroups = FALSE                # show custom legend entries
)
# ---- 7. Render ----
net


saveRDS(nodes, file = file.path("rds_objects", "nodes.rds"))
saveRDS(relationship_subtypes, file = file.path("rds_objects", "relationship_subtypes.rds"))
saveRDS(node_legend_shapes_plot, file = file.path("rds_objects", "node_legend_shapes_plot.rds"))
saveRDS(legend_df, file = file.path("rds_objects", "legend_df.rds"))
saveRDS(used_groups, file = file.path("rds_objects", "used_groups.rds"))
saveRDS(node_legend_colors_plot, file = file.path("rds_objects", "node_legend_colors_plot.rds"))
saveRDS(net, file = file.path("rds_objects", "net.rds"))
saveRDS(event_subtypes, file = file.path("rds_objects", "event_subtypes.rds"))
saveRDS(STYLES, file = file.path("rds_objects", "STYLES.rds"))
saveRDS(edges, file = file.path("rds_objects", "edges.rds"))

# ------------------
mc3_nodes_cleaned %>%
  group_by(type, sub_type) %>%
  summarize(count = n()) %>%
  arrange(-count) %>%
  kable()


# ------------------
# Split the 'from_id' column
mc3_edges_cleaned <- mc3_edges_cleaned %>%
  separate(from_id, into = c("from_id_supertype", "from_id_sub_type", "from_id_id"), sep = "_", remove = FALSE, extra = "drop")

# Split the 'target' column into 
mc3_edges_cleaned <- mc3_edges_cleaned %>%
  separate(to_id, into = c("to_id_supertype", "to_id_sub_type","to_id_id"), sep = "_", remove = FALSE, extra = "drop")

# Find the number of unique types in each column
unique_counts <- mc3_edges_cleaned %>%
  summarise_all(n_distinct) %>%
  pivot_longer(cols = everything(), names_to = "column", values_to = "unique_count")

# Print the unique counts for each column
print(unique_counts)

saveRDS(unique_counts, file = file.path("rds_objects", "unique_counts.rds"))
saveRDS(mc3_edges_cleaned, file = file.path("rds_objects", "mc3_edges_cleaned.rds"))

# ------------------
# Check the mapping
mc3_edges_cleaned %>%
  group_by(from_id_supertype, from_id_sub_type) %>%
  summarize(count = n()) %>%
  arrange(-count) %>%
  kable()

# Check the mapping
mc3_edges_cleaned %>%
  group_by(to_id_supertype, to_id_sub_type) %>%
  summarize(count = n()) %>%
  arrange(-count) %>%
  kable()


# ------------------
# checking for duplicates
duplicate_values1 <- mc3_nodes_cleaned %>%
  count(content) %>%
  filter(n > 1)

# View duplicates
print(duplicate_values1)

saveRDS(duplicate_values1, file = file.path("rds_objects", "duplicate_values1.rds"))

# ------------------
# --- 1. Extract All Communications ---
# Logic: Sender (source) --sent--> Event_Communication (target) --received--> Recipient (target)
# This extracts all communication events

# --- 2. Clean and Prepare Nodes ---
mc3_nodes_cleaned <- mc3_nodes_raw %>%
  mutate(id = as.character(id)) %>%
  filter(!is.na(id)) %>%
  distinct(id, .keep_all = TRUE) %>%
  # Rename 'type' to 'supertype' to reduce confusion with communication type
  rename(supertype = type) %>%
  # Select only columns that are needed and are consistently present
  select(id, name, sub_type, content, timestamp) 

# --- 3. Clean and Prepare Edges ---
# Rename 'type' in edges to 'edge_type' to avoid conflict with node 'supertype'
mc3_edges_cleaned <- mc3_edges_raw %>%
  rename(from_id = source,
         to_id = target,
         edge_type = type) %>% # Renamed 'type' to 'edge_type'
  mutate(across(c(from_id, to_id), as.character)) %>%
  # Filter out any edges where from_id or to_id are not in cleaned nodes
  filter(from_id %in% mc3_nodes_cleaned$id,
         to_id %in% mc3_nodes_cleaned$id)

other_communications_df <- mc3_edges_cleaned %>%
  filter(edge_type == "sent") %>% # Start with 'sent' edges
  # Join with nodes to get content and timestamp of the Event_Communication node
  left_join(mc3_nodes_cleaned %>% select(id, content, timestamp),
            by = c("to_id" = "id")) %>%
  rename(event_id = to_id, event_content = content, event_timestamp = timestamp) %>%
  # Now, find the recipient of this communication event
  left_join(mc3_edges_cleaned %>%
              filter(edge_type == "received") %>%
              select(event_id_match = from_id, recipient_id = to_id),
            by = c("event_id" = "event_id_match")) %>%
  # Join with nodes to get the sender's name and sub_type
  left_join(mc3_nodes_cleaned %>% select(id, name, sub_type),
            by = c("from_id" = "id")) %>%
  rename(sender_id_actual = from_id, sender_name = name, sender_sub_type = sub_type) %>%
  # Join with nodes to get the recipient's name and sub_type
  left_join(mc3_nodes_cleaned %>% select(id, name, sub_type),
            by = c("recipient_id" = "id")) %>%
  rename(recipient_name = name, recipient_sub_type = sub_type) %>%
  # Select and rename final columns for all communications
  select(
    communication_type = edge_type, # This will be "sent" from original filter
    sender_id = sender_id_actual,
    sender_name,
    sender_sub_type,
    recipient_id,
    recipient_name,
    recipient_sub_type,
    event_id,
    content = event_content,
    timestamp = event_timestamp
  ) 

# create a timeline visualization or inspect content.
print(knitr::kable(head(other_communications_df %>%
                          select(timestamp, sender_name, 
                                 recipient_name, content), 10),
                   format = "markdown", align = "l"))

saveRDS(mc3_edges_cleaned, file = file.path("rds_objects", "mc3_edges_cleaned.rds"))
saveRDS(other_communications_df, file = file.path("rds_objects", "other_communications_df.rds"))
saveRDS(mc3_nodes_cleaned, file = file.path("rds_objects", "mc3_nodes_cleaned.rds"))

# ------------------
# --- Step 1: Build communication matrix ---
sent_df <- other_communications_df %>%
  filter(communication_type == "sent") %>%
  count(sender_name, recipient_name, name = "sent")

received_df <- other_communications_df %>%
  filter(communication_type == "received") %>%
  count(sender_name = recipient_name, recipient_name = sender_name, name = "received")

combined_df <- full_join(sent_df, received_df, by = c("sender_name", "recipient_name")) %>%
  mutate(across(c(sent, received), ~replace_na(., 0)),
         total = sent + received)

comm_matrix <- xtabs(total ~ sender_name + recipient_name, data = combined_df)

# --- Step 2: Assign color per entity sub-type ---
type_lookup <- other_communications_df %>%
  select(name = sender_name, type = sender_sub_type) %>%
  bind_rows(other_communications_df %>% select(name = recipient_name, type = recipient_sub_type)) %>%
  distinct(name, .keep_all = TRUE)

# Define pastel Set2 colors for each type
type_colors_palette <- brewer.pal(n = 4, name = "Set2")
names(type_colors_palette) <- c("Person", "Organization", "Vessel", "Location")

# Map to nodes in the matrix
grid_colors <- type_colors_palette[type_lookup$type]
names(grid_colors) <- type_lookup$name
grid_colors <- grid_colors[rownames(comm_matrix)]

# --- Step 3: Plot chord diagram ---
circos.clear()
par(mar = c(4, 2, 8, 10))  # bottom, left, top, right

chordDiagram(
  comm_matrix,
  grid.col = grid_colors,
  transparency = 0.25,
  annotationTrack = "grid",
  preAllocateTracks = list(track.height = 0.1)
)

# Add readable sector names
circos.trackPlotRegion(
  track.index = 1,
  panel.fun = function(x, y) {
    name <- get.cell.meta.data("sector.index")
    circos.text(
      x = mean(get.cell.meta.data("xlim")),
      y = 0,
      labels = str_wrap(name, 10),
      facing = "clockwise",
      niceFacing = TRUE,
      adj = c(0, 0.5),
      cex = 0.6
    )
  },
  bg.border = NA
)

# --- Step 4: Title, subtitle ---
title(
  main = "Chord Diagram of Communication Flows",
  cex.main = 1.6,
  font.main = 2,
  line = 5
)
mtext("Each ribbon shows volume of sent + received messages", side = 3, line = 3, cex = 1, col = "gray30")
mtext("Note. Group subtype is excluded from this diagram", side = 1, line = 3, cex = 0.8, col = "gray40")

# --- Step 5: Custom Legend ---
legend_items <- names(type_colors_palette)
legend(
  x = 1.1, y = 0.85, legend = legend_items,
  fill = type_colors_palette,
  border = "gray30",
  bty = "n",
  cex = 0.7,
  pt.cex = 0.7,
  title = "Entity Sub-Type" 

)


saveRDS(combined_df, file = file.path("rds_objects", "combined_df.rds"))
saveRDS(sent_df, file = file.path("rds_objects", "sent_df.rds"))
saveRDS(received_df, file = file.path("rds_objects", "received_df.rds"))
saveRDS(grid_colors, file = file.path("rds_objects", "grid_colors.rds"))
saveRDS(comm_matrix, file = file.path("rds_objects", "comm_matrix.rds"))
saveRDS(type_colors_palette, file = file.path("rds_objects", "type_colors_palette.rds"))
#saveRDS(name, file = file.path("rds_objects", "name.rds"))
saveRDS(legend_items, file = file.path("rds_objects", "legend_items.rds"))
saveRDS(type_lookup, file = file.path("rds_objects", "type_lookup.rds"))

# ------------------
# Step 1: Count interactions
adj_df <- other_communications_df %>%
  count(sender_name, recipient_name, name = "count")

# Step 2: Compute total sent and received counts
sender_order <- adj_df %>%
  group_by(sender_name) %>%
  summarise(total_sent = sum(count)) %>%
  arrange(desc(total_sent)) %>%
  pull(sender_name)

recipient_order <- adj_df %>%
  group_by(recipient_name) %>%
  summarise(total_received = sum(count)) %>%
  arrange(desc(total_received)) %>%
  pull(recipient_name)

# Step 3: Reorder factor levels
adj_df <- adj_df %>%
  mutate(
    sender_name = factor(sender_name, levels = sender_order),
    recipient_name = factor(recipient_name, levels = recipient_order)
  )

# Step 4: Plot heatmap
ggplot(adj_df, aes(x = recipient_name, y = sender_name, fill = count)) +
  geom_tile(color = "white") +
  scale_fill_gradient(low = "white", high = "navyblue") +
  labs(
    title = "Sender-Recipient Communication Heatmap",
    subtitle = "Top communicators sorted to bottom-left",
    x = "Recipient",
    y = "Sender",
    fill = "Messages"
  ) +
  theme_minimal(base_size = 10) +
  theme(
    axis.text.x = element_text(angle = 45, hjust = 1),
    plot.title = element_text(size = 12, face = "bold"),
    plot.subtitle = element_text(size = 10),
    panel.grid = element_blank()
  )

saveRDS(recipient_order, file = file.path("rds_objects", "recipient_order.rds"))
saveRDS(adj_df, file = file.path("rds_objects", "adj_df.rds"))
saveRDS(sender_order, file = file.path("rds_objects", "sender_order.rds"))

# ------------------
sender_names_by_type <- other_communications_df %>%
  group_by(sender_sub_type) %>%
  summarise(
    unique_senders = n_distinct(sender_name),
    sender_names = paste(sort(unique(sender_name)), collapse = ", ")
  ) %>%
  arrange(desc(unique_senders))  # sort from largest to smallest

# View the table
print(sender_names_by_type)

saveRDS(sender_names_by_type, file = file.path("rds_objects", "sender_names_by_type.rds"))

# ------------------
person_vessel_df <- other_communications_df %>%
  filter(
    (sender_sub_type == "Person" & recipient_sub_type == "Vessel") |
    (sender_sub_type == "Vessel" & recipient_sub_type == "Person") |
    (sender_sub_type == "Person" & recipient_sub_type == "Person") |
    (sender_sub_type == "Vessel" & recipient_sub_type == "Vessel")
  )

saveRDS(person_vessel_df, file = file.path("rds_objects", "person_vessel_df.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
# --- FACTORING and DATETIME CLEANING ---
person_vessel_df_for_plot <- person_vessel_df %>%
  mutate(
    timestamp = as.POSIXct(timestamp),
    comm_date = as.Date(timestamp),
    comm_time_of_day = hms::as_hms(format(timestamp, "%H:%M:%S")),
    sender_sub_type = factor(sender_sub_type, levels = c("Person", "Vessel")),
    communicating_pair_sorted = paste(pmin(sender_name, recipient_name), pmax(sender_name, recipient_name), sep = " & ")
  )

# --- WRAPPING CONTENT AND TOOLTIP ---
plot_data1 <- person_vessel_df_for_plot %>%
  mutate(
    timestamp = as.POSIXct(timestamp),
    date = as.Date(timestamp),
    time = format(timestamp, "%H:%M:%S"),
    wrapped_content = str_wrap(content, width = 50),
    tooltip_text = paste0(
      "<b>Date:</b> ", date, "<br>",
      "<b>Time:</b> ", time, "<br>",
      "<b>From:</b> ", sender_name, "<br>",
      "<b>To:</b> ", recipient_name, "<br>",
      "<b>Event_id:</b> ", event_id, "<br><br>",
      "<b>Content:</b><br>", wrapped_content
    )
  )

# Plot
p <-ggplot(plot_data1, aes(x = comm_date, y = comm_time_of_day)) +
  geom_point(aes(
    color = sender_id,
    shape = sender_sub_type,
    text = tooltip_text
  ),show.legend = c(color = TRUE, shape = FALSE), 
  size = 2, alpha = 0.7) +
  scale_shape_manual(values = c("Person" = 16, "Vessel" = 17)) +
  facet_wrap(~ sender_sub_type, ncol = 1, scales = "fixed") +
  scale_y_time(
    limits = hms::as_hms(c("08:00:00", "14:00:00")),  # reversed to show time top-to-bottom
    breaks = hms::as_hms(c("08:00:00", "09:00:00", "10:00:00", "11:00:00", "12:00:00", "13:00:00", "14:00:00")),
    labels = c("08:00", "09:00", "10:00", "11:00", "12:00", "13:00", "14:00")
)+
  scale_x_date(
  date_breaks = "1 day",
  date_labels = "%d %b"
)+
  labs(
    title = "Communication Events Over Time (Sender's Perspective)",
    x = "Date",
    y = "Time of Day",
    color = "Sender (subtype, name)"
  ) +
  theme_grey() +
  theme(
    axis.text.y = element_text(size = 6),
    axis.title.y = element_text(size = 7),
    axis.ticks.y = element_line(),
    axis.text.x = element_text(size = 6, angle = 45, hjust = 1),
    axis.title.x = element_text(margin = margin(t = 8), size = 7),
    panel.spacing = unit(0.5, "lines"),  # Applies to both x and y spacing
    strip.text = element_text(size = 8, face = "bold"),
    legend.position = "bottom",
    legend.text = element_text(size = 6),
    legend.title = element_blank()
  )

# --- Convert to interactive plot ---
ggplotly(p, tooltip = "tooltip_text")

saveRDS(plot_data1, file = file.path("rds_objects", "plot_data1.rds"))
saveRDS(p, file = file.path("rds_objects", "p.rds"))
saveRDS(person_vessel_df_for_plot, file = file.path("rds_objects", "person_vessel_df_for_plot.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
library(reactable)
library(dplyr)
library(stringr)
library(tidyr) # For separate_rows, if needed for 'Event Id' later

# Create the data frame using tribble, consolidating multi-line entries
df_new_data <- tribble(
  ~Sender, ~Recipient, ~Commonalities, ~Rationale, ~Suspicious_Status, ~Date, ~Event_Id,
  "The Lookout (Person)", "The Intern (Person)", "Mako", "Jensen from City Council signed a permit to set a corridor for Mako vessel.", "Jensen", "5/10", "371",
  "Glitters Team aka. Sailor Shift Team", "Boss (Person)", "Permit, Commissioner Torres", "Glitters thanked their Boss who got Commissioner Torres to sign off the permit", "Nadia aka The Boss", "6/10", "389",
  "Remora (Vessel)", "Sailor Shift Team aka. Glitters Team", "Permit, Commissioner Torres", "Davis thanked Nadia who got Commissioner Torres to sign off the permit", "Nadia aka The Boss", "6/10", "388",
  "Mrs. Money (Person)", "Mako (Vessel); Neptune (Vessel)", "Mako, Remora, Neptune", "Mrs. Money is the Financial Controller who provides the payment protocols for Mako and Remora's captions. She also provides the encryption protocols and transfer devices. Mrs. Money coordinates payment protocols for Remora and Mako captains", "Elise aka. Mrs. Money", "7/10", "458 and 464",
  "Neptune/ Neptune Captain", "Elise aka. Mrs. Money (Person)", "Remora, Mako", "Neptune told her to coordinate payment protocols for Remora and Mako captains", "Elise aka. Mrs. Money", "7/10", "456 and 518",
  "Remona (Vessel)", "Rodriquez (Person)", "Sailor Shift", "Nemo Reef selected for Sailor Shift's music shoot.; Nadia personally handled permits for Nemo Reefs after meeting Davis.", "Both", "2/10", "153 and 328",
  "Remona (Vessel)", "Nadia (Person)", "Delta 3, Sailor Shift", "Tourism facade created by Remona and awaiting installation of underwater lighting placements at Nemo Reef.", "Both", "14/10", "943",
  "Davis (Person)", "Mako (Vessel)", "Rodriquez, Nadia, Neptune", "Rodriquez will help with logistics.", "Both", "2/10", "115",
  "Davis (Person)", "Mako (Vessel)", "Rodriquez, Nadia, Neptune", "V. Miesel's Marine Research Permit is CR-7844. There is a 5 day deadline from 8 Oct.", "Both", "8/10", "548",
  "Davis (Person)", "Mako (Vessel)", "Rodriquez, Nadia, Neptune", "V. Miesel approved their operational authority.", "Both", "12/10", "802",
  "Davis (Person)", "Remona (Vessel)", "Client", "Davis wants to maintain client's privacy with Paackland Harbour.", "Both, and also Paackland Harbour", "4/10", "282",
  "Mako (Vessel)", "Davis (Person)", "Permit CR-7844, Boss", "Davis is a captain.", "Both", "5/10", "349",
  "Mako (Vessel)", "Boss (Person)", "Boss, Samantha", "Oceanus Council approved Mako's departure to Nemo Reef with minimal documentation.", "Oceanus Council", "5/10", "365",
  "Boss (Person)", "Marko (Vessel)", "Mission, Boss", "Informed Marko to abort mission as conservation patrols are tracking it", "Both", "4/10", "316",
  "Small Fry aka Rodriquez (Person)", "Remora (Vessel, Mako (Vessel)", "Slip #14, Glitters Team, Sailor Team", "Mako asked Rodriquez when he could come by slip #14.; Small Fry replied he would come by slip #14 in 30 minutes time.; Small Fry surprised that Glitters Team filing for paperwork downtown for protected area of Nemo Reef.", "Both", "2/10", "142, 143 and 156",
  "Neptune", "Nadia (Person); Neptune (Vessel)", "Equipment specs", "Neptune asked to meet Nadia at 0600 and confirm the equipment specs approved by Elise.; Nadia confirmed the time with Neptune and agreed to show her reviewed equipment specs from The Accountant.", "Both; Elise aka The Accountant", "8/10", "537 and 738",
  "Knowles (Vessel)", "Davis (Person)", "Boss", "Instructed by Mako to provide equipment transfer.", "Both", "14/10", "1001 and 1003",
  "Sentinel(Vessel)", "Mako (Vessel)", "Permit", "Mako told Sentinel it was operating under NR-1045.", "Mako", "7/10", "467",
  "Mako (Vessel)", "Sentinel (Vessel)", "Permit", "Sentinel asked the reasons for Mako operating under permit NR-1045", "Mako", "8/10", "574",
  "Mako (Vessel)", "Sentinel (Vessel)", "Permit", "Mako told Sentinel it was operating under permit CR-7844 (marine equipment transport) not NR-1045", "Mako", "8/10", "575",
  "Miranda (Person)", "Clepper Jesen (Person)", "Rodriquez", "Rodriquez filed for permits under Sailor Shifts Team for 2 commercial and 1 private yatch to perform environmental sampling.", "Rodriquez is suspicious. The communicators are not suspicious.", "2/10", "130",
  "Miranda (Person)", "Clepper Jesen (Person)", "Rodriquez", "Rodriquez is connected to a mining consortium with previous violations in protected areas.", "Rodriquez is suspicious. The communicators are not suspicious", "2/10", "134",
  "Miranda (Person)", "Clepper Jesen (Person)", "Rodriquez", "Rodriquez is connected to Council Knowles and they are related to V. Miesel Shipping which is possibly a shell company which has vessels.", "Rodriquez is suspicious. The communicators are not suspicious.", "3/10", "201",
  "Miranda (Person)", "Clepper Jesen (Person)", "Rodriquez is suspicious", "Council Knowles' brother-in-law owns the offshore bank account to V. Miesel Shipping. The company is involved in rare earth extraction.", "Rodriquez is suspicious. The communicators are not suspicious.", "3/10", "204",
  "Miranda (Person)", "Clepper Jesen (Person)", "Rodriquez is suspicious", "Knowles accepted cash from Rodriquez.", "Rodriquez is suspicious. The communicators are not suspicious.", "3/10", "206"
)

# Process the data (primarily for consistent whitespace and Event_Id splitting if necessary)
df_processed <- df_new_data %>%
  # Standardize delimiters and remove excess whitespace across all character columns.
  mutate(
    across(where(is.character), ~ str_replace_all(.x, "[,\n]+", ", ") %>% str_trim())
  ) %>%
  # For Event_Id, replace "and" with comma and then separate rows if needed
  mutate(Event_Id = str_replace_all(Event_Id, " and ", ", ")) %>%
  mutate(across(where(is.character), str_trim)) %>%
  mutate(Event_Id = as.character(Event_Id)) # Ensure Event_Id is character

# Create the reactable table
reactable(
  df_processed,
  filterable = TRUE,
  searchable = TRUE,
  paginationType = "numbers",
  defaultPageSize = 5,
  showPageSizeOptions = TRUE,
  pageSizeOptions = c(5, 10, 20, 50, 100),
  striped = TRUE,
  highlight = TRUE,
  columns = list(
    Sender = colDef(name = "Sender", minWidth = 120),
    Recipient = colDef(name = "Recipient", minWidth = 120),
    Commonalities = colDef(name = "Commonalities", minWidth = 150),
    Rationale = colDef(name = "Rationale", minWidth = 300),
    # Rename Suspicious_Status for display
    Suspicious_Status = colDef(name = "Suspicious?", minWidth = 150),
    # Date column as text, no special date sorting
    Date = colDef(name = "Date", minWidth = 80, align = "center"),
    Event_Id = colDef(name = "Event Id", minWidth = 80, align = "center")
  ),
  theme = reactableTheme(
    borderColor = "#dfe2e5",
    stripedColor = "#f6f8fa",
    highlightColor = "#f0f5f9",
    cellPadding = "8px 12px",
    style = list(fontFamily = "Verdana, Geneva, sans-serif", fontSize = "14px"),
    headerStyle = list(
      "&.rt-th:hover" = list(backgroundColor = "#e0e6eb"),
      fontSize = "15px",
      fontWeight = 600,
      color = "#333",
      background = "#f7f7f7"
    ),
    rowSelectedStyle = list(backgroundColor = "#e6f2ff", "&:hover" = list(backgroundColor = "#e6f2ff")),
    searchInputStyle = list(width = "100%", margin = "5px 0", padding = "5px"),
    filterInputStyle = list(width = "100%", margin = "2px 0", padding = "4px")
  )
)

saveRDS(df_processed, file = file.path("rds_objects", "df_processed.rds"))
saveRDS(df_new_data, file = file.path("rds_objects", "df_new_data.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
# Load necessary packages
library(reactable)
library(dplyr)
library(stringr)
library(tidyr) # For separate_rows
library(lubridate) # For date parsing

# 1. Manually parse the data into a tribble
# Each row in the tribble corresponds to a logical entry from provided text.
# Multi-line strings within a cell are captured using '\n'.
# The 'Suspicious?' column is named 'Suspicious_Status' in R to avoid special characters,
# and will be renamed in the reactable column definition for display.
df_raw_input <- tribble(
  ~Sender, ~Recipient, ~Commonalities, ~Rationale, ~Suspicious_Status, ~Date, ~Event_Id,
  "Samantha (Person)", "Sailor Shift Team aka Glitters Team (Organization)", "Permit, Nemo Reef, Music Video", "Samantha was assisting Sailor Shift Team to coordinate the permit and equipment rental for their music video and wanted confirmation on Nemo Reef as their intended location.", "Unsure", "1/10", "64",
  "Sailor Shift Team aka Glitters Team (Organization)", "Samantha (Person)", "Permit, Nemo Reef,", "Glitters Team confirmed with Samantha that they would like to use Nemo Reef as their primary location", "Unsure", "1/10", "66",
  "Mako (Vessel)", "Samantha (Person)", "Nemo Reef Permit", "Mako claims that Samantha has a coordination team. Preparing for an event in a week's time from 6 Oct.", "Samantha is not", "6/10", "380",
  "Mako (Vessel)", "Samantha (Person)", "Nemo Reef Permit", "Samantha advised them to stop activities which might be illegal without permit", "Samantha is not", "10/10", "687",
  "The Lookout (Person)", "Sentinel (Vessel)", "Marina's Dream, Music Video", "Spotted 3 suspicious vessels around Nemo Reef on 3/10.", "Both are not", "3/10", "171",
  "The Lookout (Person)", "Sentinel (Vessel)", "Marina's Dream, Music Video", "Spotted music video production on 13/10", "Both are not", "13/10", "912",
  "The Lookout (Person)", "Horizon (Vessel)", "Music Video", "Found it suspicious to spot lighting rigs and cameras at 12pm. Wanted Horizon to investigate", "Both are not", "13/10", "926",
  "EcoVigil (Vessel)", "Liam (Person)", "ROVs", "Collection of water samples using ROVs", "EcoVigil is not", "11/10", "737 and 751" # "and" will be handled by separate_rows
)

# 2. Process the data for display and sorting
df_processed <- df_raw_input %>%
  # Standardize delimiters and remove excess whitespace across all character columns.
  # This flattens multi-line strings within cells and makes 'separate_rows' more reliable.
  mutate(
    across(where(is.character), ~ str_replace_all(.x, "[,\n]+", ", ") %>% str_trim())
  ) %>%
  # Handle the "and" in Event_Id, converting it to a comma for consistent splitting.
  mutate(Event_Id = str_replace_all(Event_Id, " and ", ", ")) %>%
  # Now, use separate_rows for Event_Id if it contains multiple comma-separated values.
  # 'Date' column appears to be single-valued in this dataset, so it's not included here.
  # If Date could also have multiple values (e.g., "1/10, 2/10"), you would add it to separate_rows.
  separate_rows(Event_Id, sep = ", ", convert = FALSE) %>%
  # Trim whitespace again after separation in case of leading/trailing spaces
  mutate(across(where(is.character), str_trim)) %>%
  # Convert Date to actual date objects for proper sorting.
  # Using `dmy` for day/month/year format and assuming the current year (2025).
  mutate(
    FullDate = paste0(Date, "/2025"), # Add the current year for correct parsing
    ParsedDate = dmy(FullDate, quiet = TRUE) # Convert to Date object (Day Month Year)
  ) %>%
  # Order the data by parsed date and then event ID
  arrange(ParsedDate, Event_Id) %>%
  select(-FullDate) %>% # Remove the helper column
  # Ensure Event_Id is character type for display purposes
  mutate(Event_Id = as.character(Event_Id))

# 3. Create the reactable table
reactable(
  df_processed,
  filterable = TRUE, # Enable column filters at the top of each column
  searchable = TRUE, # Add a global search box above the table
  paginationType = "numbers", # Display page numbers (e.g., 1, 2, 3 instead of Prev/Next)
  defaultPageSize = 5, # Show 5 rows per page by default
  showPageSizeOptions = TRUE, # Allow users to select different page sizes
  pageSizeOptions = c(5, 10, 20, 50, 100), # Available page size options
  striped = TRUE, # Add alternating row colors for better readability
  highlight = TRUE, # Highlight the row when the user hovers over it
  # Define the default sorting for the table when it first loads
  defaultSorted = list(ParsedDate = "asc"), # Sort by the hidden ParsedDate column in ascending order
  columns = list(
    # The 'ParsedDate' column is used internally for correct date sorting but is not displayed
    ParsedDate = colDef(
      show = FALSE
    ),
    # Define how each visible column should be displayed and behave
    Date = colDef(
      name = "Date", # Display name for the column header
      defaultSortOrder = "asc", # Ensure initial sort is ascending for the displayed date
      minWidth = 80, # Minimum width for the column to ensure content fits
      align = "center" # Align text to the center for better presentation of dates
    ),
    Sender = colDef(name = "Sender", minWidth = 120),
    Recipient = colDef(name = "Recipient", minWidth = 120),
    Commonalities = colDef(name = "Commonalities", minWidth = 150),
    Rationale = colDef(name = "Rationale", minWidth = 300),
    # Custom definition for the 'Suspicious_Status' column to display its original name
    Suspicious_Status = colDef(name = "Suspicious?", minWidth = 150),
    Event_Id = colDef(
      name = "Event ID", # Display name for the column header
      minWidth = 80, # Minimum width
      align = "center" # Align text to the center
    )
  ),
  # Customize the visual theme of the reactable table for a consistent look and feel
  theme = reactableTheme(
    borderColor = "#dfe2e5", # Defines the color of table borders
    stripedColor = "#f6f8fa", # Defines the background color of striped rows
    highlightColor = "#f0f5f9", # Defines the background color when a row is hovered over
    cellPadding = "8px 12px", # Sets the padding inside table cells
    style = list(fontFamily = "Verdana, Geneva, sans-serif", fontSize = "14px"), # General font family and size for table content
    headerStyle = list(
      "&.rt-th:hover" = list(backgroundColor = "#e0e6eb"), # Hover effect for table headers
      fontSize = "15px", # Font size for headers
      fontWeight = 600, # Font weight for headers (bold)
      color = "#333", # Text color for headers
      background = "#f7f7f7" # Background color for headers
    ),
    rowSelectedStyle = list(backgroundColor = "#e6f2ff", "&:hover" = list(backgroundColor = "#e6f2ff")), # Style for selected rows
    searchInputStyle = list(width = "100%", margin = "5px 0", padding = "5px"), # Style for the global search input box
    filterInputStyle = list(width = "100%", margin = "2px 0", padding = "4px") # Style for individual column filter input boxes
  )
)

saveRDS(df_raw_input, file = file.path("rds_objects", "df_raw_input.rds"))
saveRDS(df_processed, file = file.path("rds_objects", "df_processed.rds"))

# ------------------
set.seed(1234)  

# --- STEP 1: Prepare the edge list ---
edge_df <- other_communications_df %>%
  select(sender_id, recipient_id) %>%
  filter(!is.na(sender_id) & !is.na(recipient_id)) %>%
  rename(from = sender_id, to = recipient_id) %>%
  distinct()

# Remove self-loops
edge_df <- edge_df %>% filter(from != to)

# --- STEP 2: Create the graph object (undirected) ---
g <- tbl_graph(edges = edge_df, directed = FALSE)

# --- STEP 3: Run Louvain community detection ---
g <- g %>%
  mutate(community = group_louvain())

# --- STEP 4: Visualize the graph ---
ggraph(g, layout = "fr") +
  geom_edge_link(alpha = 0.3) +
  geom_node_point(aes(color = as.factor(community)), size = 5) +
  geom_node_text(aes(label = name), repel = TRUE, size = 3) +
  theme_void() +
  labs(title = "Community Detection in Communication Network",
       color = "Community")

saveRDS(g, file = file.path("rds_objects", "g.rds"))
saveRDS(edge_df, file = file.path("rds_objects", "edge_df.rds"))

# ------------------
set.seed(1234) 

# --- Compute Centrality Measures ---
g <- g %>%
  mutate(
    pagerank = centrality_pagerank(),
    degree = centrality_degree(),
    betweenness = centrality_betweenness(),
    closeness = centrality_closeness()
  )

# Show top 10 nodes by PageRank
g %>%
  as_tibble() %>%
  select(name, pagerank, degree, betweenness, closeness) %>%
  arrange(desc(pagerank)) %>%
  head(10)

saveRDS(g, file = file.path("rds_objects", "g.rds"))

# ------------------
set.seed(1234) 

# Visualize by Centrality
ggraph(g, layout = "fr") +
  geom_edge_link(alpha = 0.3) +
  geom_node_point(aes(size = pagerank, color = as.factor(community)), alpha = 0.8) +
  geom_node_text(aes(label = name),
                 repel = TRUE,
                 size = 3,
                 max.iter = 5000) + # <--- Increased this value 
  theme_void() +
  labs(title = "Network with PageRank Centrality",
       size = "PageRank", color = "Community")


# ------------------
# 1. Extract node community assignments
g_node_communities <- g %>%
  as_tibble() %>%
  select(sender_name = name, community)

# 2. Join communication content with community assignments
content_with_community <- other_communications_df %>%
  left_join(g_node_communities, by = c("sender_name")) %>%
  filter(!is.na(community), !is.na(content))

# 3. Unnest tokens for unigrams
unigrams <- content_with_community %>%
  unnest_tokens(word, content, token = "words") %>%
  anti_join(stop_words, by = "word") %>%
  count(community, word, sort = TRUE)

# 4. Unnest tokens for bigrams
bigrams <- content_with_community %>%
  unnest_tokens(bigram, content, token = "ngrams", n = 2) %>%
  separate(bigram, into = c("word1", "word2"), sep = " ") %>%
  filter(!word1 %in% stop_words$word,
         !word2 %in% stop_words$word) %>%
  unite(bigram, word1, word2, sep = " ") %>%
  count(community, bigram, sort = TRUE)

# 5a. Plot word clouds per community
par(mfrow = c(2, 3))  # 2 rows, 3 columns layout for 6 communities
for (i in sort(unique(bigrams$community))) {
  words <- bigrams %>% filter(community == i)
  
  if (nrow(words) < 1) next  # Skip if no words
  
  suppressWarnings({
    set.seed(432)  # Set seed for reproducibility
    wordcloud(words = words$bigram,  # <-- FIXED here
              freq = words$n,
              max.words = min(20, nrow(words)),
              scale = c(3, 0.5),
              colors = brewer.pal(8, "Dark2"),
              random.order = FALSE)
  })
  mtext(paste("Community", i), side = 3, line = 1, adj = 0.5, cex = 1.5, col = "black")
}

saveRDS(words, file = file.path("rds_objects", "words.rds"))
saveRDS(content_with_community, file = file.path("rds_objects", "content_with_community.rds"))
saveRDS(unigrams, file = file.path("rds_objects", "unigrams.rds"))
saveRDS(bigrams, file = file.path("rds_objects", "bigrams.rds"))
saveRDS(g_node_communities, file = file.path("rds_objects", "g_node_communities.rds"))

# ------------------
# 5b. Plot word clouds per community
par(mfrow = c(2, 3))  # 2 rows, 3 columns layout for 6 communities
for (i in sort(unique(bigrams$community))) {
  words <- bigrams %>% filter(community == i)
  
  if (nrow(words) < 1) next  # Skip if no words
  
  suppressWarnings({
    set.seed(432)  # Set seed for reproducibility
    wordcloud(words = words$bigram,  # <-- FIXED here
              freq = words$n,
              max.words = min(20, nrow(words)),
              scale = c(3, 0.5),
              colors = brewer.pal(8, "Dark2"),
              random.order = FALSE)
  })
  mtext(paste("Community", i), side = 3, line = 1, adj = 0.5, cex = 1.5, col = "black")
}

saveRDS(words, file = file.path("rds_objects", "words.rds"))

# ------------------
set.seed(1234)
# --- Configuration ---
num_top_bigrams_per_community <- 8
empty_bar_count <- 2 # gaps btw comm.
#excluded_community <- 5 # too little in community 5

# --- 1. Prepare the Combined Dataset ---
all_communities_data <- bigrams %>%
#  filter(community != excluded_community) %>%
  group_by(community) %>%
  arrange(desc(n)) %>%
  slice_head(n = num_top_bigrams_per_community) %>%
  ungroup()

all_communities_data$community <- as.factor(all_communities_data$community)

to_add <- data.frame(
  bigram = NA,
  n = NA,
  community = rep(levels(all_communities_data$community), each = empty_bar_count)
)

plot_data <- rbind(all_communities_data, to_add) %>%
  arrange(community)

plot_data$id <- seq_len(nrow(plot_data)) # Keep ID as numeric here

# --- 2. Prepare Label Data ---
label_data <- plot_data
number_of_bar <- nrow(label_data)
label_data$angle <- 90 - 360 * (label_data$id - 0.5) / number_of_bar
label_data$hjust <- ifelse(label_data$angle < -90, 1, 0)
label_data$angle <- ifelse(label_data$angle < -90, label_data$angle + 180, label_data$angle)

# --- 3. Prepare Data for Baselines (Community Dividers) ---
base_data <- plot_data %>%
  group_by(community) %>%
  summarize(
    start = min(id, na.rm = TRUE), # Keep as numeric
    end = max(id, na.rm = TRUE) - empty_bar_count # Keep as numeric
  ) %>%
  rowwise() %>%
  mutate(
    title_position = mean(c(start, end))
  ) %>%
  ungroup()

# --- 4. Prepare Data for Grid Lines (Optional: Value Scales) ---
max_n_value <- max(plot_data$n, na.rm = TRUE)
grid_lines_values <- c(20, 40, 60, 80, 100)
grid_lines_values <- grid_lines_values[grid_lines_values <= max_n_value]

grid_segments_data <- plot_data %>%
  group_by(community) %>%
  summarize(
    start_id = min(id, na.rm = TRUE), # Keep as numeric
    end_id = max(id, na.rm = TRUE) - empty_bar_count # Keep as numeric
  )

grid_data_final <- tibble()
for(val in grid_lines_values) {
  temp_data <- grid_segments_data %>%
    mutate(y_value = val)
  grid_data_final <- bind_rows(grid_data_final, temp_data)
}

# --- Data for grid line LABELS ---
grid_label_data <- data.frame(
  x_pos = max(plot_data$id, na.rm = TRUE) + 2, # Fixed x position outside the plot
  y_pos = grid_lines_values,
  label_text = as.character(grid_lines_values)
)

# --- 5. Make the Unified Plot ---
p <- ggplot(plot_data, aes(x = id, y = n, fill = community)) + # <--- x = id (numeric)
  # Add background grid lines for value (e.g., 20, 40, 60, 80)
  geom_segment(data = grid_data_final,
               aes(x = start_id - 0.5, y = y_value, xend = end_id + 0.5, yend = y_value),
               inherit.aes = FALSE,
               color = "grey", alpha = 0.8, linewidth = 0.3) +

  # Add text showing the value of each grid line at a fixed position
  geom_text(data = grid_label_data,
            aes(x = x_pos, y = y_pos, label = label_text),
            inherit.aes = FALSE,
            color = "grey", size = 3, angle = 0, fontface = "bold", hjust = 0) +

  # Bars for the bigrams (main plot elements)
  geom_bar(stat = "identity", alpha = 0.8, color = "white", linewidth = 0.1,
           width = 1.4) + # <--- Add width=1 to remove space between bars if id is numeric

  # Set limits for the y-axis, providing space for labels
  ylim(-max_n_value * 0.7, max_n_value * 1.2) +

  theme_minimal() +
  theme(
    legend.position = "none",
    axis.text = element_blank(),
    axis.title = element_blank(),
    panel.grid = element_blank(),
    plot.margin = unit(c(1.5, 1.5, 1.5, 1.5), "cm") # Top, Right, Bottom, Left margins
  ) +
  coord_polar(start = 0) +

  # Add bigram labels
  geom_text(
    data = label_data,
    aes(x = id, y = n + 10, label = bigram, hjust = hjust), # <--- x = id (numeric)
    color = "black", fontface = "bold", alpha = 0.8, size = 2.8,
    angle = label_data$angle, inherit.aes = FALSE
  ) +

  # Add base lines for each community segment
  geom_segment(
    data = base_data,
    aes(x = start - 0.5, y = -10, xend = end + 0.5, yend = -10),
    colour = "black", alpha = 0.8, linewidth = 0.6, inherit.aes = FALSE
  ) +

  # Add community group labels
  geom_text(
    data = base_data,
    aes(x = title_position, y = -40, label = paste("Comm.", community)),
    colour = "black", alpha = 0.9, size = 2, fontface = "bold", inherit.aes = FALSE
  )+
  # --- Add the Title ---
  labs(
    title = "Circular Bar Chart by Community",
    subtitle = "Frequencies of key bigrams within each community", # Updated subtitle
    caption = paste0("AT | Generated: ", Sys.Date())
  ) +
  # Apply the Set2 Brewer palette
  scale_fill_brewer(palette = "Set2") +
  # --- Customize title appearance ---
  theme(
    plot.title = element_text(hjust = 0.5, size = 16, face = "bold", margin = margin(b = 10)),
    plot.subtitle = element_text(hjust = 0.5, size = 12, margin = margin(b = 10)),
    plot.caption = element_text(hjust = 1, size = 7, color = "grey50")
  )

print(p)

saveRDS(plot_data, file = file.path("rds_objects", "plot_data.rds"))
saveRDS(num_top_bigrams_per_community, file = file.path("rds_objects", "num_top_bigrams_per_community.rds"))
saveRDS(empty_bar_count, file = file.path("rds_objects", "empty_bar_count.rds"))
saveRDS(grid_segments_data, file = file.path("rds_objects", "grid_segments_data.rds"))
saveRDS(label_data, file = file.path("rds_objects", "label_data.rds"))
saveRDS(number_of_bar, file = file.path("rds_objects", "number_of_bar.rds"))
saveRDS(to_add, file = file.path("rds_objects", "to_add.rds"))
saveRDS(grid_lines_values, file = file.path("rds_objects", "grid_lines_values.rds"))
saveRDS(temp_data, file = file.path("rds_objects", "temp_data.rds"))
saveRDS(grid_data_final, file = file.path("rds_objects", "grid_data_final.rds"))
saveRDS(base_data, file = file.path("rds_objects", "base_data.rds"))
saveRDS(community, file = file.path("rds_objects", "community.rds"))
saveRDS(all_communities_data, file = file.path("rds_objects", "all_communities_data.rds"))
saveRDS(angle, file = file.path("rds_objects", "angle.rds"))
saveRDS(grid_label_data, file = file.path("rds_objects", "grid_label_data.rds"))
saveRDS(excluded_community, file = file.path("rds_objects", "excluded_community.rds"))
saveRDS(hjust, file = file.path("rds_objects", "hjust.rds"))
saveRDS(max_n_value, file = file.path("rds_objects", "max_n_value.rds"))
saveRDS(id, file = file.path("rds_objects", "id.rds"))
saveRDS(p, file = file.path("rds_objects", "p.rds"))

# ------------------
set.seed(1234)

# 6. Create a tidy summary table of members per community
grouped_members <- g %>%
  as_tibble() %>%
  select(name, community) %>%
  group_by(community) %>%
  summarise(
    Members = paste(sort(name), collapse = ", "),
    .groups = "drop"
  ) %>%
  mutate(
    `Group Number` = community,
    `Topic Area` = case_when(
      community == 1 ~ "green guardians, city council, paackland harbor, green guardians, nemo reef, water quality",
      community == 2 ~ "nemo reef, miesel shipping, permit, cr 7844, delta 3",
      community == 3 ~ "nemo reef, himark harbor, samantha blake, radio silence",
      community == 4 ~ "money, 0500 meeting, nemo reef, intern reporting, conservation activity, financial projections",
      community == 5 ~ "sam, kelly, loading equipment",
      community == 6 ~ "miranda reporting, jensen, project poseidon, conservation vessels, nemo reef",
      TRUE ~ "Other"
    ),
    `Group Name` = case_when(
      community == 1 ~ "Conservationist Group",
      community == 2 ~ "Sailor Shift",
      community == 3 ~ "Maritime",
      community == 4 ~ "Suspicious Characters",
      community == 5 ~ "Sam & Kelly",
      community == 6 ~ "Hacklee Herald",
      TRUE ~ "Miscellaneous"
    )
  ) %>%
  select(`Group Number`, `Topic Area`, `Members`, `Group Name`)

# Show the summary table in a clean format
kable(grouped_members, caption = "Community Group Membership Summary", align = "l")

saveRDS(grouped_members, file = file.path("rds_objects", "grouped_members.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
# Get community membership from graph object
community_df <- g %>%
  as_tibble() %>%
  select(name, community)

saveRDS(community_df, file = file.path("rds_objects", "community_df.rds"))

# ------------------
library(circlize)
library(dplyr)
library(tidyr)
library(RColorBrewer)
library(stringr)
library(lubridate)
library(htmltools) # Essential for building the HTML structure
library(jsonlite) # For passing R data to JavaScript safely

# --- 1. Data Preprocessing and Setup ---

# Bin by 2-hour interval
other_communications_df <- other_communications_df %>%
  mutate(timestamp = as.POSIXct(timestamp)) %>% # Ensure timestamp is POSIXct
  mutate(timestamp_2hr = floor_date(timestamp, unit = "2 hours"))

# Get all unique 2-hour time bins for the slider
all_times <- sort(unique(other_communications_df$timestamp_2hr))

# Define output directory for image frames
output_dir <- "chord_frames"
dir.create(output_dir, showWarnings = FALSE) # Create the directory if it doesn't exist

# Community name mapping (ensure this matches  'community_df' structure)
community_name_map <- c(
  "1" = "Conservationist Group",
  "2" = "Sailor Shift",
  "3" = "Maritime",
  "4" = "Suspicious Characters",
  "5" = "Sam & Kelly",
  "6" = "Hacklee Herald"
)


# Assuming community_df has a 'community' column that's numeric/factor
if (exists("community_df") && "community" %in% names(community_df)) {
  num_unique_communities <- length(unique(community_df$community))
  base_colors <- brewer.pal(max(3, num_unique_communities), "Set2") # Use Set2 for pastel
  community_colors <- setNames(
    base_colors[1:num_unique_communities], # Slice to exactly the number needed
    as.character(sort(unique(community_df$community)))
  )
} else {
  # Fallback if community_df is not defined or missing 'community' column
  message("Warning: 'community_df' or 'community' column not found. Using default colors.")
  community_colors <- c(
    "1" = "#66C2A5", "2" = "#FC8D62", "3" = "#8DA0CB", "4" = "#E78AC3",
    "5" = "#A6D854", "6" = "#FFD92F", "7" = "#E5C494", "8" = "#B3B3B3"
  )
}


# --- 2. Generate and Save Chord Diagrams as PNGs ---

# Loop through each time bin, create a plot, and save it
for (i in seq_along(all_times)) {
  selected_time <- all_times[i]
  end_time <- selected_time + hours(2)

  filtered_df <- other_communications_df %>%
    filter(timestamp_2hr == selected_time)

  # Prepare data for the chord diagram matrix
  sent_df <- filtered_df %>%
    filter(communication_type == "sent") %>%
    count(sender_name, recipient_name, name = "sent")

  received_df <- filtered_df %>%
    filter(communication_type == "received") %>%
    count(sender_name = recipient_name, recipient_name = sender_name, name = "received")

  combined_df <- full_join(sent_df, received_df, by = c("sender_name", "recipient_name")) %>%
    mutate(across(c(sent, received), ~replace_na(., 0)),
           total = sent + received)

  # Skip this iteration if no data for the current time slice
  if (nrow(combined_df) == 0 || sum(combined_df$total) == 0) {
    message(paste("No communications for time:", selected_time, ". Skipping frame."))

    next
  }

  comm_matrix <- xtabs(total ~ sender_name + recipient_name, data = combined_df)

  # Ensure sector_names from comm_matrix exist in community_df for color mapping
  sector_names <- union(rownames(comm_matrix), colnames(comm_matrix))
  
  # Filter community_df to only relevant sectors and ensure distinct entries
  sector_community_df <- community_df %>%
    filter(name %in% sector_names) %>%
    distinct(name, .keep_all = TRUE) %>%
    arrange(match(name, sector_names))

  # Map community colors to sector names based on 'community' column
  grid_colors_current_frame <- setNames(
    community_colors[as.character(sector_community_df$community)],
    sector_community_df$name
  )
  # Ensure only colors for actual sectors in the matrix are used
  grid_colors_current_frame <- grid_colors_current_frame[names(grid_colors_current_frame) %in% sector_names]


  # Open PNG device for saving the plot
  png(sprintf("%s/frame_%03d.png", output_dir, i), width = 800, height = 800)
  
  # Clear existing circlize plot before drawing new one
  circos.clear()
  par(mar = c(6, 2, 10, 6)) # Adjust margins as needed for title and labels

  # Draw the chord diagram
  chordDiagram(
    comm_matrix,
    grid.col = grid_colors_current_frame,
    transparency = 0.25,
    annotationTrack = "grid",
    preAllocateTracks = list(track.height = 0.1)
  )

  # Add labels to the sectors
  circos.trackPlotRegion(
    track.index = 1,
    panel.fun = function(x, y) {
      name <- get.cell.meta.data("sector.index")
      wrapped_name <- str_wrap(name, width = 12)
      circos.text(
        x = mean(get.cell.meta.data("xlim")),
        y = 0,
        labels = wrapped_name,
        facing = "clockwise",
        niceFacing = TRUE,
        adj = c(0, 0.5),
        cex = 0.8
      )
    },
    bg.border = NA
  )

  # Add a main title to the plot
  title(
    main = paste("Communication Flows\n", format(selected_time, "%d %b %Y (%H:%M"), "to", format(end_time, "%H:%M)")),
    cex.main = 1.2,
    font.main = 1,
    line = 6
  )
  dev.off() # CRITICAL: Close the PNG device to save the file
}

# --- 3. Build the HTML Viewer with Embedded Images and JavaScript ---

# Generate HTML <img> tags for each saved frame
# Filter out any frames that might have been skipped if 'next' was used
# Check which frame files actually exist
existing_frames <- list.files(output_dir, pattern = "^frame_\\d{3}\\.png$", full.names = TRUE)
# Extract the numeric index from the filename to match with all_times
frame_indices <- as.numeric(gsub("frame_(\\d{3})\\.png", "\\1", basename(existing_frames)))

# Only create image tags for the frames that were successfully generated
image_tags <- lapply(seq_along(existing_frames), function(idx) {
  # The original 'i' (loop index) corresponds to the 'frame_indices'
  original_time_idx <- frame_indices[idx]
  tags$img(src = existing_frames[idx], # Use the full path here
           style = if (idx == 1) "display:block;" else "display:none;", # Show first frame by default
           class = "chord-frame",
           alt = paste("Chord diagram for time slice", format(all_times[original_time_idx], "%d %b %Y %H:%M")))
})


# JavaScript function to update which image frame is visible
# We need to map the slider value (0 to num_frames-1) to the correct time index
# because some frames might be skipped, causing gaps in numerical sequence.
js_script <- HTML(sprintf("
<script>
  // Ensure the allTimes array correctly maps to the generated frames
  const originalAllTimes = %s; // This is the full list of all_times
  const generatedFrameIndices = %s; // This indicates which original_time_idx corresponds to a generated frame

  function updateFrame(sliderIndex) {
    const frames = document.querySelectorAll('.chord-frame');
    frames.forEach((el, i) => {
      // frames[i] corresponds to existing_frames[i] from R
      // sliderIndex is 0-based for the slider
      el.style.display = (i === sliderIndex) ? 'block' : 'none';
    });

    // Update the time display text based on the current frame's original time
    const timeDisplay = document.getElementById('current-time-display');
    if (timeDisplay && sliderIndex < generatedFrameIndices.length) {
        // Get the original time index for the currently displayed frame
        const actualTimeIndex = generatedFrameIndices[sliderIndex] - 1; // Convert 1-based to 0-based for originalAllTimes
        
        if (actualTimeIndex >= 0 && actualTimeIndex < originalAllTimes.length) {
            const selectedTime = new Date(originalAllTimes[actualTimeIndex]);
            const endTime = new Date(selectedTime.getTime() + 2 * 60 * 60 * 1000); // Add 2 hours in milliseconds

            // Format dates and times for display
            const formatDate = (date) => date.toLocaleDateString('en-US', { day: '2-digit', month: 'short', year: 'numeric' });
            const formatTime = (date) => date.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit', hour12: false });
            
            timeDisplay.innerHTML = `Day: ${formatDate(selectedTime)} | Time: ${formatTime(selectedTime)} to ${formatTime(endTime)}`;
        } else {
            timeDisplay.innerHTML = 'No data for this time slice.';
        }
    }
  }

  // Initialize the display on page load
  document.addEventListener('DOMContentLoaded', () => {
    const slider = document.getElementById('frameSlider');
    if (slider) {
        updateFrame(parseInt(slider.value)); // Set initial frame based on slider's default value
    }
  });
</script>
", jsonlite::toJSON(as.character(all_times)), jsonlite::toJSON(frame_indices))) # Pass all_times and frame_indices to JS


# --- 4. Display the HTML content directly in the Quarto document ---
# This is the key line to make Quarto embed the interactive viewer.
browsable(
  tagList(
    js_script, # The JavaScript for interactivity
    tags$head(
      tags$style(HTML("
        /* Basic styling for the image frames and slider container */
        .chord-frame { width: 100%; max-width: 800px; height: auto; margin: auto; display: block; }
        #slider-container { text-align: center; margin: 20px auto; max-width: 800px; }
        .chord-title { text-align: center; font-size: 1.5em; margin-bottom: 15px; font-weight: bold; }
        #frameSlider { width: 80%; max-width: 700px; margin: 10px auto; display: block; }
        #current-time-display { font-weight: bold; margin-top: 10px; }
      "))
    ), # Close tags$head

    tags$body( 
      tags$div(class = "chord-title", "Interactive Communication Flows Over Time"),
      tags$div(id = "slider-container",
          tags$input(type = "range", min = "0", max = length(existing_frames) - 1, value = 0,
                     id = "frameSlider", oninput = "updateFrame(parseInt(this.value))"),
          tags$p(id = "current-time-display", style = "font-weight:bold; margin-top: 10px;"),
          tags$p("Use the slider to view communication over time")
      ),
      tags$div(id = "chord-images-container", image_tags) # Container for all image frames
    )
  )
)

saveRDS(original_time_idx, file = file.path("rds_objects", "original_time_idx.rds"))
saveRDS(image_tags, file = file.path("rds_objects", "image_tags.rds"))
saveRDS(selected_time, file = file.path("rds_objects", "selected_time.rds"))
saveRDS(other_communications_df, file = file.path("rds_objects", "other_communications_df.rds"))
saveRDS(end_time, file = file.path("rds_objects", "end_time.rds"))
saveRDS(base_colors, file = file.path("rds_objects", "base_colors.rds"))
saveRDS(community_name_map, file = file.path("rds_objects", "community_name_map.rds"))
saveRDS(community_colors, file = file.path("rds_objects", "community_colors.rds"))
saveRDS(wrapped_name, file = file.path("rds_objects", "wrapped_name.rds"))
saveRDS(sent_df, file = file.path("rds_objects", "sent_df.rds"))
saveRDS(js_script, file = file.path("rds_objects", "js_script.rds"))
saveRDS(num_unique_communities, file = file.path("rds_objects", "num_unique_communities.rds"))
saveRDS(sector_community_df, file = file.path("rds_objects", "sector_community_df.rds"))
saveRDS(frame_indices, file = file.path("rds_objects", "frame_indices.rds"))
saveRDS(existing_frames, file = file.path("rds_objects", "existing_frames.rds"))
saveRDS(all_times, file = file.path("rds_objects", "all_times.rds"))
saveRDS(grid_colors_current_frame, file = file.path("rds_objects", "grid_colors_current_frame.rds"))
saveRDS(name, file = file.path("rds_objects", "name.rds"))
saveRDS(received_df, file = file.path("rds_objects", "received_df.rds"))
saveRDS(sector_names, file = file.path("rds_objects", "sector_names.rds"))
saveRDS(comm_matrix, file = file.path("rds_objects", "comm_matrix.rds"))
saveRDS(filtered_df, file = file.path("rds_objects", "filtered_df.rds"))
saveRDS(output_dir, file = file.path("rds_objects", "output_dir.rds"))
saveRDS(combined_df, file = file.path("rds_objects", "combined_df.rds"))

# ------------------
set.seed(1234)

# --- STEP 1: Prepare the edge list ---
edge_df_pv <- person_vessel_df %>%
  select(sender_id, recipient_id) %>%
  filter(!is.na(sender_id) & !is.na(recipient_id)) %>%
  rename(from = sender_id, to = recipient_id) %>%
  distinct()

# Optional: remove self-loops
edge_df_pv <- edge_df_pv %>% filter(from != to)

# --- STEP 2: Create the graph object (undirected) ---
g_pv <- tbl_graph(edges = edge_df_pv, directed = FALSE)

# --- STEP 3: Run Louvain community detection ---
g_pv <- g_pv %>%
  mutate(community = group_louvain())

# --- STEP 4: Visualize the graph ---
ggraph(g_pv, layout = "fr") +
  geom_edge_link(alpha = 0.3) +
  geom_node_point(aes(color = as.factor(community)), size = 5) +
  geom_node_text(aes(label = name), repel = TRUE, size = 3) +
  theme_void() +
  labs(title = "Community Detection in Communication Network",
       color = "Community")

saveRDS(edge_df_pv, file = file.path("rds_objects", "edge_df_pv.rds"))
saveRDS(g_pv, file = file.path("rds_objects", "g_pv.rds"))

# ------------------
# --- Compute Centrality Measures ---
set.seed(1234)
g_pv <- g_pv %>%
  mutate(
    pagerank = centrality_pagerank(),
    degree = centrality_degree(),
    betweenness = centrality_betweenness(),
    closeness = centrality_closeness()
  )

# Show top 10 nodes by PageRank
g_pv %>%
  as_tibble() %>%
  select(name, pagerank, degree, betweenness, closeness) %>%
  arrange(desc(pagerank)) %>%
  head(10)

saveRDS(g_pv, file = file.path("rds_objects", "g_pv.rds"))

# ------------------
set.seed(1234)

# Visualize by Centrality
ggraph(g_pv, layout = "fr") +
  geom_edge_link(alpha = 0.3) +
  geom_node_point(aes(size = pagerank, color = as.factor(community)), alpha = 0.8) +
  geom_node_text(aes(label = name), repel = TRUE, size = 3) +
  theme_void() +
  labs(title = "Network with PageRank Centrality",
       size = "PageRank", color = "Community")


# ------------------
# 5b. Plot word clouds per community
par(mfrow = c(2, 3))  # 2 rows, 3 columns layout for 5 communities
for (i in sort(unique(bigrams$community))) {
  words <- bigrams %>% filter(community == i)
  
  if (nrow(words) < 1) next  # Skip if no words
  
  suppressWarnings({
    set.seed(432)  # Set seed for reproducibility
    wordcloud(words = words$bigram,  # <-- FIXED here
              freq = words$n,
              max.words = min(30, nrow(words)),
              scale = c(3, 0.5),
              colors = brewer.pal(8, "Dark2"),
              random.order = FALSE)
  })
  mtext(paste("Community", i), side = 3, line = 1, adj = 0.5, cex = 1.5, col = "black")
}

saveRDS(words, file = file.path("rds_objects", "words.rds"))

# ------------------
set.seed(1234)

# --- Configuration ---
num_top_bigrams_per_community <- 8
empty_bar_count <- 2 # gaps btw comm.
#excluded_community <- 5 # too little in community 5

# --- 1. Prepare the Combined Dataset ---
all_communities_data <- bigrams %>%
#  filter(community != excluded_community) %>%
  group_by(community) %>%
  arrange(desc(n)) %>%
  slice_head(n = num_top_bigrams_per_community) %>%
  ungroup()

all_communities_data$community <- as.factor(all_communities_data$community)

to_add <- data.frame(
  bigram = NA,
  n = NA,
  community = rep(levels(all_communities_data$community), each = empty_bar_count)
)

plot_data <- rbind(all_communities_data, to_add) %>%
  arrange(community)

plot_data$id <- seq_len(nrow(plot_data)) # Keep ID as numeric here

# --- 2. Prepare Label Data ---
label_data <- plot_data
number_of_bar <- nrow(label_data)
label_data$angle <- 90 - 360 * (label_data$id - 0.5) / number_of_bar
label_data$hjust <- ifelse(label_data$angle < -90, 1, 0)
label_data$angle <- ifelse(label_data$angle < -90, label_data$angle + 180, label_data$angle)

# --- 3. Prepare Data for Baselines (Community Dividers) ---
base_data <- plot_data %>%
  group_by(community) %>%
  summarize(
    start = min(id, na.rm = TRUE), # Keep as numeric
    end = max(id, na.rm = TRUE) - empty_bar_count # Keep as numeric
  ) %>%
  rowwise() %>%
  mutate(
    title_position = mean(c(start, end))
  ) %>%
  ungroup()

# --- 4. Prepare Data for Grid Lines (Optional: Value Scales) ---
max_n_value <- max(plot_data$n, na.rm = TRUE)
grid_lines_values <- c(20, 40, 60, 80, 100)
grid_lines_values <- grid_lines_values[grid_lines_values <= max_n_value]

grid_segments_data <- plot_data %>%
  group_by(community) %>%
  summarize(
    start_id = min(id, na.rm = TRUE), # Keep as numeric
    end_id = max(id, na.rm = TRUE) - empty_bar_count # Keep as numeric
  )

grid_data_final <- tibble()
for(val in grid_lines_values) {
  temp_data <- grid_segments_data %>%
    mutate(y_value = val)
  grid_data_final <- bind_rows(grid_data_final, temp_data)
}

# --- Data for grid line LABELS ---
grid_label_data <- data.frame(
  x_pos = max(plot_data$id, na.rm = TRUE) + 2, # Fixed x position outside the plot
  y_pos = grid_lines_values,
  label_text = as.character(grid_lines_values)
)

# --- 5. Make the Unified Plot ---
p <- ggplot(plot_data, aes(x = id, y = n, fill = community)) + # <--- x = id (numeric)
  # Add background grid lines for value (e.g., 20, 40, 60, 80)
  geom_segment(data = grid_data_final,
               aes(x = start_id - 0.5, y = y_value, xend = end_id + 0.5, yend = y_value),
               inherit.aes = FALSE,
               color = "grey", alpha = 0.8, linewidth = 0.3) +

  # Add text showing the value of each grid line at a fixed position
  geom_text(data = grid_label_data,
            aes(x = x_pos, y = y_pos, label = label_text),
            inherit.aes = FALSE,
            color = "grey", size = 3, angle = 0, fontface = "bold", hjust = 0) +

  # Bars for the bigrams (main plot elements)
  geom_bar(stat = "identity", alpha = 0.8, color = "white", linewidth = 0.1,
           width = 1) + # <--- Add width=1 to remove space between bars if id is numeric

  # Set limits for the y-axis, providing space for labels
  ylim(-max_n_value * 0.7, max_n_value * 1.2) +

  theme_minimal() +
  theme(
    legend.position = "none",
    axis.text = element_blank(),
    axis.title = element_blank(),
    panel.grid = element_blank(),
    plot.margin = unit(c(1.5, 1.5, 1.5, 1.5), "cm") # Top, Right, Bottom, Left margins
  ) +
  coord_polar(start = 0) +

  # Add bigram labels
  geom_text(
    data = label_data,
    aes(x = id, y = n + 10, label = bigram, hjust = hjust), # <--- x = id (numeric)
    color = "black", fontface = "bold", alpha = 0.8, size = 2.8,
    angle = label_data$angle, inherit.aes = FALSE
  ) +

  # Add base lines for each community segment
  geom_segment(
    data = base_data,
    aes(x = start - 0.5, y = -10, xend = end + 0.5, yend = -10),
    colour = "black", alpha = 0.8, linewidth = 0.6, inherit.aes = FALSE
  ) +

  # Add community group labels
  geom_text(
    data = base_data,
    aes(x = title_position, y = -40, label = paste("Comm.", community)),
    colour = "black", alpha = 0.9, size = 2, fontface = "bold", inherit.aes = FALSE
  )+
  # --- Add the Title ---
  labs(
    title = "Circular Bar Chart by Community",
    subtitle = "Frequencies of key bigrams within each community", # Updated subtitle
    caption = paste0("AT | Generated: ", Sys.Date())
  ) +
  # Apply the Set2 Brewer palette
  scale_fill_brewer(palette = "Set2") +
  # --- Customize title appearance ---
  theme(
    plot.title = element_text(hjust = 0.5, size = 16, face = "bold", margin = margin(b = 10)),
    plot.subtitle = element_text(hjust = 0.5, size = 12, margin = margin(b = 10)),
    plot.caption = element_text(hjust = 1, size = 7, color = "grey50")
  )

print(p)

saveRDS(plot_data, file = file.path("rds_objects", "plot_data.rds"))
saveRDS(num_top_bigrams_per_community, file = file.path("rds_objects", "num_top_bigrams_per_community.rds"))
saveRDS(empty_bar_count, file = file.path("rds_objects", "empty_bar_count.rds"))
saveRDS(grid_segments_data, file = file.path("rds_objects", "grid_segments_data.rds"))
saveRDS(label_data, file = file.path("rds_objects", "label_data.rds"))
saveRDS(number_of_bar, file = file.path("rds_objects", "number_of_bar.rds"))
saveRDS(to_add, file = file.path("rds_objects", "to_add.rds"))
saveRDS(grid_lines_values, file = file.path("rds_objects", "grid_lines_values.rds"))
saveRDS(temp_data, file = file.path("rds_objects", "temp_data.rds"))
saveRDS(grid_data_final, file = file.path("rds_objects", "grid_data_final.rds"))
saveRDS(base_data, file = file.path("rds_objects", "base_data.rds"))
saveRDS(community, file = file.path("rds_objects", "community.rds"))
saveRDS(all_communities_data, file = file.path("rds_objects", "all_communities_data.rds"))
saveRDS(angle, file = file.path("rds_objects", "angle.rds"))
saveRDS(grid_label_data, file = file.path("rds_objects", "grid_label_data.rds"))
saveRDS(excluded_community, file = file.path("rds_objects", "excluded_community.rds"))
saveRDS(hjust, file = file.path("rds_objects", "hjust.rds"))
saveRDS(max_n_value, file = file.path("rds_objects", "max_n_value.rds"))
saveRDS(id, file = file.path("rds_objects", "id.rds"))
saveRDS(p, file = file.path("rds_objects", "p.rds"))

# ------------------
set.seed(1234)
# 6. Create a tidy summary table of members per community
grouped_members <- g_pv %>%
  as_tibble() %>%
  select(name, community) %>%
  group_by(community) %>%
  summarise(
    Members = paste(sort(name), collapse = ", "),
    .groups = "drop"
  ) %>%
  mutate(
    `Group Number` = community,
    `Topic Area` = case_when(
      community == 1 ~ "nemo reef, himark harbor, samantha blake, city council",
      community == 2 ~ "south dock, equipment transfer, security team, nemo reef, delta3, cr 7844",
      community == 3 ~ "intern reporting, conservation vessels, nemo reef",
      community == 4 ~ "10am tomorrow, 0500 tomorrow, funding channels, alternative funding",
      community == 5 ~ "classification markings, project poseidon, clearance documents, harbor security",
      TRUE ~ "Other"
    ),
    `Group Name` = case_when(
      community == 1 ~ "Conservationist Group",
      community == 2 ~ "Permit",
      community == 3 ~ "Pseudonym",
      community == 4 ~ "Suspicious",
      community == 5 ~ "Hacklee Herald",
      TRUE ~ "Miscellaneous"
    )
  ) %>%
  select(`Group Number`, `Topic Area`, `Members`, `Group Name`)

# Show the summary table in a clean format
kable(grouped_members, caption = "Community Group Membership Summary", align = "l")

saveRDS(grouped_members, file = file.path("rds_objects", "grouped_members.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
# --- FACTORING and DATETIME CLEANING ---
person_vessel_df_for_plot <- person_vessel_df %>%
  mutate(
    timestamp = as.POSIXct(timestamp),
    comm_date = as.Date(timestamp),
    comm_time_of_day = hms::as_hms(format(timestamp, "%H:%M:%S")),
    sender_sub_type = factor(sender_sub_type, levels = c("Person", "Vessel")),
    communicating_pair_sorted = paste(pmin(sender_name, recipient_name), pmax(sender_name, recipient_name), sep = " & ")
  )

# Get community membership from graph object
community_df <- g_pv %>%
  as_tibble() %>%
  select(name, community)

# --- WRAPPING CONTENT AND TOOLTIP ---
plot_data1 <- person_vessel_df_for_plot %>%
  mutate(
    timestamp = as.POSIXct(timestamp),
    date = as.Date(timestamp),
    time = format(timestamp, "%H:%M:%S"),
    wrapped_content = str_wrap(content, width = 50),
    tooltip_text = paste0(
      "<b>Date:</b> ", date, "<br>",
      "<b>Time:</b> ", time, "<br>",
      "<b>From:</b> ", sender_name, "<br>",
      "<b>To:</b> ", recipient_name, "<br>",
      "<b>Event_id:</b> ", event_id, "<br><br>",
      "<b>Content:</b><br>", wrapped_content
    )
  )

# Merge with plot_data1 using sender_id == name
plot_data1 <- plot_data1 %>%
  left_join(community_df, by = c("sender_id" = "name"))

# Plot
p <- ggplot(plot_data1, aes(x = comm_date, y = comm_time_of_day)) +
  geom_point(aes(
    color = as.factor(community),
    shape = sender_sub_type,
    text = tooltip_text
  ), show.legend = c(color = TRUE, shape = FALSE),
  size = 2, alpha = 0.7) +
  scale_shape_manual(values = c("Person" = 16, "Vessel" = 17)) +
  facet_wrap(~ sender_sub_type, ncol = 1, scales = "fixed") +
    scale_y_time(
    limits = hms::as_hms(c("08:00:00", "14:00:00")),  # reversed to show time top-to-bottom
    breaks = hms::as_hms(c("08:00:00", "09:00:00", "10:00:00", "11:00:00", "12:00:00", "13:00:00", "14:00:00")),
    labels = c("08:00", "09:00", "10:00", "11:00", "12:00", "13:00", "14:00")
)+
  scale_x_date(
  date_breaks = "1 day",
  date_labels = "%d %b"
)+
  labs(
    title = "Communication Events Over Time (Sender's Perspective)",
    x = "Date",
    y = "Time of Day",
    color = "Community"
  ) +
  theme_grey() +
  theme(
    axis.text.y = element_text(size = 6),
    axis.title.y = element_text(size = 7),
    axis.ticks.y = element_line(),
    axis.text.x = element_text(size = 6, angle = 45, hjust = 1),
    axis.title.x = element_text(margin = margin(t = 10), size = 7),
    panel.spacing = unit(0.5, "lines"),  # Applies to both x and y spacing
    strip.text = element_text(size = 8, face = "bold"),
    legend.position = "bottom",
    legend.text = element_text(size = 6),
    legend.title = element_blank()
  )

# --- Convert to interactive plot ---
ggplotly(p, tooltip = "tooltip_text")

saveRDS(p, file = file.path("rds_objects", "p.rds"))
saveRDS(plot_data1, file = file.path("rds_objects", "plot_data1.rds"))
saveRDS(community_df, file = file.path("rds_objects", "community_df.rds"))
saveRDS(person_vessel_df_for_plot, file = file.path("rds_objects", "person_vessel_df_for_plot.rds"))

# ------------------
unique_names <- sort(unique(c(other_communications_df$sender_name, 
                              other_communications_df$recipient_name)))
unique_names


saveRDS(unique_names, file = file.path("rds_objects", "unique_names.rds"))

# ------------------
name_mapping <- tibble::tibble(
  observed_name = c(
    "Boss", "City Officials", "Clepper Jensen", "Davis", "Defender", "EcoVigil",
    "Elise", "Glitters Team", "Green Guardians", "Haacklee Harbor", "Himark Harbor", "Horizon",
    "Kelly", "Knowles", "Liam Thorne", "Mako", "Marlin", "Miranda Jordan",
    "Mrs. Money", "Nadia Conti", "Neptune", "Northern Light", "Oceanus City Council", "Osprey",
    "Paackland Harbor", "Port Security", "Reef Guardian", "Remora", "Rodriguez", "Sailor Shift Team",
    "Sam", "Samantha Blake", "Seawatch", "Sentinel", "Serenity", "Small Fry",
    "The Accountant", "The Intern", "The Lookout", "The Middleman", "V. Miesel Shipping"
  ),
  real_identity = c(
    "Nadia Conti", "Oceanus City Council", "Clepper Jensen", "Captain Davis", "Sentinel", "EcoVigil",
    "Elise", "Sailor Shift Team", "Green Guardians", "Harbor Authority","Harbor Authority", "Horizon",
    "Kelly", "Knowles", "Liam Thorne", "Mako", "Small Fishing Vessel", "Miranda Jordan",
    "Elise", "Nadia Conti", "Neptune", "Commercial Vessel", "Oceanus City Council", "Tourism Vessel",
    "Harbor Authority", "Oceanus City Council", "Reef Guardian", "Remora", "Rodriguez", "Sailor Shift Team",
    "Sam", "Samantha Blake", "Kelly", "Sentinel", "Private Luxury Yacht", "Rodriguez",
    "Elise", "Sam", "Kelly", "Liam Thorne", "V. Miesel Shipping"
  ),
  community = c(
    "V. Miesel Shipping", "City Council", "Haacklee Herald", "V. Miesel Shipping", "Local Conservationist Group", "Local Conservationist Group",
    "V. Miesel Shipping", "Sailor Shift Team", "Local Conservationist Group", "Harbor Authority", "Harbor Authority", "Local Conservationist Group",
    "Local Conservationist Group", "V. Miesel Shipping", "City Council", "V. Miesel Shipping", "Maritime", "Haacklee Herald",
    "V. Miesel Shipping", "V. Miesel Shipping", "V. Miesel Shipping", "Maritime", "City Council", "Maritime",
    "Harbor Authority", "City Council", "Local Conservationist Group", "V. Miesel Shipping", "V. Miesel Shipping", "Sailor Shift Team",
    "V. Miesel Shipping", "Sailor Shift Team", "Local Conservationist Group", "Local Conservationist Group", "Maritime", "V. Miesel Shipping",
    "V. Miesel Shipping", "V. Miesel Shipping", "Local Conservationist Group", "City Council", "V. Miesel Shipping"
  )
)

multi_members <- tibble::tibble(
  observed_name = c(
    #  City Council members (additional to existing)
    "Commissioner Blake", "Commissioner Torres", "Council Knowles", "The Middleman", "Jensen from City Council", "Liam Thorne",

    # Sailor Shift Team (may already exist, but we ensure all)
    "Boss", "Council Knowles", "Davis", "Glitters Team", "Liam Thorne", "Mako", "Mrs. Money", "Nadia Conti", "Neptune",
    "Remora", "Rodriguez", "Sam", "Samantha Blake", "Small Fry", "The Accountant", "The Intern", "The Middleman", "Elise",

    #  Influential Families
    "Council Knowles", "V. Miesel Shipping",

    #  Conservationist Group
    "Defender", "EcoVigil", "Green Guardians", "Horizon", "Kelly", "Reef Guardian", "Seawatch", "Sentinel", "The Lookout"
  ),
  real_identity = c(
    "Commissioner Blake", "Commissioner Torres", "Council Knowles", "Liam Thorne", "Clepper Jensen", "Liam Thorne",

    "Nadia Conti", "Council Knowles", "Captain Davis", "Sailor Shift Team", "Liam Thorne", "Mako", "Elise", "Nadia Conti", "Neptune",
    "Remora", "Rodriguez", "Sam", "Samantha Blake", "Rodriguez", "Elise", "Sam", "Liam Thorne", "Elise",

    "Council Knowles", "V. Miesel Shipping",

    "Sentinel", "EcoVigil", "Green Guardians", "Horizon", "Kelly", "Reef Guardian", "The Lookout", "Sentinel", "Kelly"
  ),
  community = c(
    rep("City Council", 6),
    rep("Sailor Shift Team", 18),
    rep("Influential Families", 2),
    rep("Local Conservationist Group", 9)
  )
)


saveRDS(name_mapping, file = file.path("rds_objects", "name_mapping.rds"))
saveRDS(multi_members, file = file.path("rds_objects", "multi_members.rds"))

# ------------------
library(shiny)
library(ggplot2)
library(ggalluvial)
library(dplyr)

# Example name_mapping
name_mapping <- tibble::tibble(
  observed_name = c("Boss", "Mrs. Money", "The Middleman", "Remora", "Small Fry", "Nadia Conti"),
  real_identity = c("Nadia", "Nadia", "Liam Thorne", "Remora", "Neptune", "Nadia"),
  community = c("V. Miesel Shipping", "V. Miesel Shipping", "Intermediaries", "Conservationists", "Research Vessels", "V. Miesel Shipping")
)

# Build alluvial dataset
alluvial_data <- name_mapping %>%
  filter(!is.na(real_identity) & !is.na(community)) %>%
  count(real_identity, observed_name, community, name = "value")

# UI
ui <- fluidPage(
  titlePanel("Identity Flow Viewer"),
  sidebarLayout(
    sidebarPanel(
      selectInput("person", "Select a Real Identity:", choices = c("All", unique(alluvial_data$real_identity)))
    ),
    mainPanel(
      plotOutput("alluvialPlot", height = "600px")
    )
  )
)

# Server
server <- function(input, output, session) {

  filtered_data <- reactive({
    if (input$person == "All") {
      alluvial_data
    } else {
      filter(alluvial_data, real_identity == input$person)
    }
  })

  output$alluvialPlot <- renderPlot({
    ggplot(filtered_data(),
           aes(axis1 = real_identity, axis2 = observed_name, axis3 = community, y = value)) +
      geom_alluvium(aes(fill = real_identity), width = 1/12, alpha = 0.8) +
      geom_stratum(width = 1/12, fill = "grey90", color = "black") +
      geom_text(stat = "stratum", aes(label = after_stat(stratum)), size = 3.5, hjust = 0) +
      scale_x_discrete(
        limits = c("Real Identity", "Observed Name", "Community"),
        expand = c(.05, .05)
      ) +
      labs(
        title = if (input$person == "All") "All Identity Flows" else paste(input$person, "'s Identity Flow"),
        x = NULL, y = "Link Count"
      ) +
      theme_minimal() +
      theme(
        panel.grid = element_blank(),
        axis.text.y = element_blank(),
        axis.ticks = element_blank()
      )
  })
}

# Run the app
shinyApp(ui, server)


saveRDS(server, file = file.path("rds_objects", "server.rds"))
saveRDS(filtered_data, file = file.path("rds_objects", "filtered_data.rds"))
saveRDS(ui, file = file.path("rds_objects", "ui.rds"))
saveRDS(alluvialPlot, file = file.path("rds_objects", "alluvialPlot.rds"))
saveRDS(alluvial_data, file = file.path("rds_objects", "alluvial_data.rds"))
saveRDS(name_mapping, file = file.path("rds_objects", "name_mapping.rds"))

# ------------------
set.seed(1234)

# Assume g_full includes Nadia — not g from other_communications_df
g_igraph <- as.igraph(g)

# Confirm Nadia exists
if (!"Nadia Conti" %in% V(g_igraph)$name) stop("Nadia Conti not found in the graph.")

# Get ego subgraph
nadia_ego_igraph <- make_ego_graph(g_igraph, order = 1, nodes = which(V(g_igraph)$name == "Nadia Conti"), mode = "all")[[1]]

# Convert to tidygraph
nadia_ego_graph <- as_tbl_graph(nadia_ego_igraph)

# Convert to undirected for Louvain
nadia_ego_graph <- nadia_ego_graph %>% 
  to_undirected() %>% 
  activate(nodes) %>%
  mutate(
    community = group_louvain(),
    pagerank = centrality_pagerank()
  )

# Plot Nadia's ego network
ggraph(nadia_ego_graph, layout = "fr") +
  geom_edge_link(alpha = 0.4) +
  geom_node_point(aes(size = pagerank, color = as.factor(community)), alpha = 0.9) +
  geom_node_text(aes(label = name), repel = TRUE, size = 3) +
  scale_color_brewer(palette = "Set2") +
  theme_void() +
  labs(
    title = "Nadia Conti’s Ego Network",
    subtitle = "Nodes sized by PageRank, colored by Louvain community",
    color = "Community",
    size = "PageRank"
  )

saveRDS(nadia_ego_igraph, file = file.path("rds_objects", "nadia_ego_igraph.rds"))
saveRDS(nadia_ego_graph, file = file.path("rds_objects", "nadia_ego_graph.rds"))
saveRDS(g_igraph, file = file.path("rds_objects", "g_igraph.rds"))

# ------------------
# --- 2. Clean and Prepare Nodes ---
mc3_nodes_cleaned <- mc3_nodes_raw %>%
  mutate(id = as.character(id)) %>%
  filter(!is.na(id)) %>%
  distinct(id, .keep_all = TRUE) %>%
  # Rename 'type' to 'supertype' to reduce confusion with communication type
  rename(supertype = type) %>%
  # Select only columns that are needed and are consistently present
  select(id, name, sub_type, content, timestamp) # Keep timestamp as character for now

# --- 3. Clean and Prepare Edges ---
# Rename 'type' in edges to 'edge_type' to avoid conflict with node 'supertype'
mc3_edges_cleaned <- mc3_edges_raw %>%
  rename(from_id = source,
         to_id = target,
         edge_type = type) %>% # Renamed 'type' to 'edge_type'
  mutate(across(c(from_id, to_id), as.character)) %>%
  # Filter out any edges where from_id or to_id are not in cleaned nodes
  filter(from_id %in% mc3_nodes_cleaned$id,
         to_id %in% mc3_nodes_cleaned$id)

# --- 4. Identify Nadia Conti's ID and Sub_type ---
nadia_info <- mc3_nodes_cleaned %>%
  filter(name == "Nadia Conti") %>%
  select(id, sub_type) # Corrected: Select sub_type here, not supertype

nadia_id <- nadia_info %>% pull(id)
nadia_sub_type <- nadia_info %>% pull(sub_type) # New variable for Nadia's sub_type


if (length(nadia_id) == 0) {
  stop("Nadia Conti not found in the nodes data. Please check the 'name' column or the ID.")
} else if (length(nadia_id) > 1) {
  warning("Multiple entries found for Nadia Conti. Using the first one.")
  nadia_id <- nadia_id[1]
  nadia_sub_type <- nadia_sub_type[1] # Ensure sub_type is also taken for the first one
}

print(paste("Nadia Conti's ID:", nadia_id))
print(paste("Nadia Conti's Sub_type:", nadia_sub_type))

# --- 5. Extract Nadia's Sent Communications ---
# Logic: Nadia (source) --sent--> Event_Communication (target) --received--> Recipient (target)
nadia_sent_communications <- mc3_edges_cleaned %>%
  filter(from_id == nadia_id, edge_type == "sent") %>%
  # Join with nodes to get content and timestamp of the Event_Communication node
  left_join(mc3_nodes_cleaned %>% select(id, content, timestamp),
            by = c("to_id" = "id")) %>%
  rename(event_id = to_id, event_content = content, event_timestamp = timestamp) %>%
  # Now, find the recipient of this communication event
  left_join(mc3_edges_cleaned %>%
              filter(edge_type == "received") %>%
              select(event_id_match = from_id, recipient_id = to_id),
            by = c("event_id" = "event_id_match")) %>%
  # Join with nodes to get the recipient's name AND sub_type
  left_join(mc3_nodes_cleaned %>% select(id, name, sub_type), # Select sub_type here
            by = c("recipient_id" = "id")) %>%
  rename(recipient_name = name, recipient_sub_type = sub_type) %>% # Rename sub_type
  # Select and rename final columns for sent communications
  select(
    communication_type = edge_type, # This will be "sent"
    sender_id = from_id,
    recipient_id,
    recipient_name,
    recipient_sub_type, # Include in final select
    event_id,
    content = event_content,
    timestamp = event_timestamp # Timestamp is still character here
  ) %>%
  mutate(
    sender_name = !!nadia_id, # Explicitly set sender_name to Nadia's ID/name using !!
    sender_sub_type = !!nadia_sub_type # Assign Nadia's sub_type as a constant value using !!
  )

print("--- Nadia's Sent Communications ---")
print(kable(head(nadia_sent_communications, 10), format = "markdown", align = "l"))


# --- 6. Extract Nadia's Received Communications ---
# Logic: Sender (source) --sent--> Event_Communication (target) --received--> Nadia (target)
nadia_received_communications <- mc3_edges_cleaned %>%
  filter(to_id == nadia_id, edge_type == "received") %>%
  # The source of this edge is the Event_Communication node
  rename(event_id = from_id) %>%
  # Join with nodes to get content and timestamp of the Event_Communication node
  left_join(mc3_nodes_cleaned %>% select(id, content, timestamp),
            by = c("event_id" = "id")) %>%
  rename(event_content = content, event_timestamp = timestamp) %>%
  # Now, find the original sender of this communication event
  left_join(mc3_edges_cleaned %>%
              filter(edge_type == "sent") %>%
              select(event_id_match = to_id, sender_id = from_id),
            by = c("event_id" = "event_id_match")) %>%
  # Join with nodes to get the sender's name AND sub_type
  left_join(mc3_nodes_cleaned %>% select(id, name, sub_type), # Select sub_type here
            by = c("sender_id" = "id")) %>%
  rename(sender_name = name, sender_sub_type = sub_type) %>% # Rename sub_type
  # Select and rename final columns for received communications
  select(
    communication_type = edge_type, # This will be "received"
    sender_id,
    sender_name,
    sender_sub_type, # Include in final select
    recipient_id = to_id,
    event_id,
    content = event_content,
    timestamp = event_timestamp
  ) %>%
  mutate(
    recipient_name = !!nadia_id, # Explicitly set recipient_name to Nadia's ID/name using !!
    recipient_sub_type = !!nadia_sub_type # Assign Nadia's sub_type as a constant value using !!
  )

print("--- Nadia's Received Communications ---")
print(kable(head(nadia_received_communications, 10), format = "markdown", align = "l"))

# --- 7. Combine Sent and Received Communications for Full Timeline ---
nadia_full_communications_timeline <- bind_rows(
  nadia_sent_communications,
  nadia_received_communications
) %>%
  arrange(timestamp) %>%
  # CRITICAL FIX: Explicitly convert timestamp to POSIXct after bind_rows
  # Use as.POSIXct with the observed format string
  mutate(timestamp = as.POSIXct(timestamp, format = "%Y-%m-%d %H:%M:%S")) %>%
  # NEW: Create a sorted communicating pair for consistent coloring
  rowwise() %>% # Process row by row
  mutate(communicating_pair_sorted = paste(sort(c(sender_name, recipient_name)), collapse = "_")) %>%
  ungroup() # Return to normal data frame operations

print("--- Nadia's Full Communication Timeline (Combined) ---")
print(kable(head(nadia_full_communications_timeline, 10), format = "markdown", align = "l"))

# --- 8. Prepare Data for Ego Network Visualization (Direct Person-to-Person/Entity) ---
# Build nodes and edges directly from nadia_full_communications_timeline,
# focusing on direct sender-recipient connections.
# Nodes for the ego network graph: Collect all unique sender and recipient IDs
all_ego_person_entity_ids <- nadia_full_communications_timeline %>%
  select(id = sender_id) %>%
  bind_rows(nadia_full_communications_timeline %>% select(id = recipient_id)) %>%
  distinct(id) %>%
  filter(!is.na(id)) # Ensure no NA IDs

# Filter mc3_nodes_cleaned to get attributes for these person/entity nodes
ego_nodes_for_graph <- mc3_nodes_cleaned %>%
  filter(id %in% all_ego_person_entity_ids$id) %>%
  filter(!is.na(id)) %>% # Ensure no NA IDs in nodes for graph
  # Add attributes for visualization
  mutate(
    is_nadia = ifelse(id == nadia_id, TRUE, FALSE),
    # Create a new column for legend grouping that includes Nadia as a distinct category
    node_legend_group = ifelse(is_nadia, "Nadia Conti", sub_type),
    node_size = ifelse(is_nadia, 20, 18), # Further increased node sizes: Nadia 20, others 10
    display_name = ifelse(is_nadia, "Nadia Conti", name) # Use actual name for others
  ) %>%
  filter(!is.na(sub_type)) # Ensure nodes have a sub_type for consistent plotting

# Calculate communication counts for each node for hover text
node_comm_summary <- nadia_full_communications_timeline %>%
  group_by(id = sender_id) %>%
  summarise(sent_count = n(), .groups = 'drop') %>%
  full_join(nadia_full_communications_timeline %>%
              group_by(id = recipient_id) %>%
              summarise(received_count = n(), .groups = 'drop'),
            by = "id") %>%
  mutate(
    sent_count = replace_na(sent_count, 0),
    received_count = replace_na(received_count, 0)
  )

# Join communication summary and create hover text for nodes
ego_nodes_for_graph <- ego_nodes_for_graph %>%
  left_join(node_comm_summary, by = "id") %>%
  mutate(
    hover_text = paste(
      "Name:", display_name, "<br>",
      "Type:", node_legend_group, "<br>",
      "Sent Communications:", sent_count, "<br>",
      "Received Communications:", received_count
    )
  )

# Edges for the ego network graph: Direct Sender -> Recipient edges
# Aggregate to count occurrences for edge thickness
ego_edges_for_graph <- nadia_full_communications_timeline %>%
  # Group by sender, recipient, their names, and communication type to count interactions
  group_by(from = sender_id, to = recipient_id, sender_name, recipient_name, communication_type) %>%
  summarise(
    count_of_comm = n(), # Number of times this specific communication happened
    content_sample = paste(head(content, 1), collapse = "; "), # Sample content
    timestamp_min = min(timestamp, na.rm = TRUE),
    timestamp_max = max(timestamp, na.rm = TRUE),
    .groups = 'drop'
  ) %>%
  # Add the 'nadia_role_in_comm' column for coloring based on Nadia's perspective
  mutate(
    nadia_role_in_comm = case_when(
      from == nadia_id & communication_type == "sent" ~ "Nadia Sent",
      to == nadia_id & communication_type == "received" ~ "Nadia Received",
      TRUE ~ "Other Communication" # For communications not directly involving Nadia as sender/recipient
    )
  ) %>%
  # Ensure 'from' and 'to' are character and non-NA
  mutate(
    from = as.character(from),
    to = as.character(to)
  ) %>%
  filter(!is.na(from) & !is.na(to)) %>%
  # Filter out edges where 'from' or 'to' IDs are NOT in the final ego_nodes_for_graph
  filter(from %in% ego_nodes_for_graph$id, to %in% ego_nodes_for_graph$id) %>%
  # Create hover text for edges
  mutate(
    hover_text = paste(
      "From:", sender_name, "<br>",
      "To:", recipient_name, "<br>",
      "Type:", communication_type, "<br>",
      "Count:", count_of_comm, "<br>",
      "First:", format(timestamp_min, "%Y-%m-%d %H:%M:%S"), "<br>",
      "Last:", format(timestamp_max, "%Y-%m-%d %H:%M:%S"), "<br>",
      "Content Sample:", content_sample
    )
  )

# Create the tbl_graph object for the ego network
nadia_ego_network_graph <- tbl_graph(nodes = ego_nodes_for_graph, edges = ego_edges_for_graph, directed = TRUE)

print("--- Checking: Number of nodes and edges in Nadia's Ego Network Graph ---")
print(paste("Nodes:", gorder(nadia_ego_network_graph), "Edges:", gsize(nadia_ego_network_graph)))
print("---------------------------------------------------------------------")

saveRDS(mc3_edges_cleaned, file = file.path("rds_objects", "mc3_edges_cleaned.rds"))
saveRDS(mc3_nodes_cleaned, file = file.path("rds_objects", "mc3_nodes_cleaned.rds"))
saveRDS(nadia_sub_type, file = file.path("rds_objects", "nadia_sub_type.rds"))
saveRDS(nadia_sent_communications, file = file.path("rds_objects", "nadia_sent_communications.rds"))
saveRDS(nadia_full_communications_timeline, file = file.path("rds_objects", "nadia_full_communications_timeline.rds"))
saveRDS(nadia_info, file = file.path("rds_objects", "nadia_info.rds"))
saveRDS(node_comm_summary, file = file.path("rds_objects", "node_comm_summary.rds"))
saveRDS(all_ego_person_entity_ids, file = file.path("rds_objects", "all_ego_person_entity_ids.rds"))
saveRDS(nadia_received_communications, file = file.path("rds_objects", "nadia_received_communications.rds"))
saveRDS(nadia_id, file = file.path("rds_objects", "nadia_id.rds"))
saveRDS(ego_edges_for_graph, file = file.path("rds_objects", "ego_edges_for_graph.rds"))
saveRDS(nadia_ego_network_graph, file = file.path("rds_objects", "nadia_ego_network_graph.rds"))
saveRDS(ego_nodes_for_graph, file = file.path("rds_objects", "ego_nodes_for_graph.rds"))

# ------------------
# Define custom colors for node types (reusing from previous code)
node_legend_colors_plot <- c(
  "Person" = "#88CCEE",      # Blue
  "Vessel" = "#D55E00",      # Orange
  "Organization" = "#117733", # Green
  "Location" = "#AA4499",   # Purple
  "Nadia Conti" = "red"      # Special color for Nadia
)

# Define custom shapes for node subtypes
node_legend_shapes_plot <- c(
  "Person" = "dot",        # Circle for visNetwork
  "Vessel" = "triangle",
  "Organization" = "square",
  "Location" = "diamond",
  "Nadia Conti" = "star"   # Star shape for Nadia in visNetwork
)

# Separate the edges into two sets: Nadia Sent and Nadia Received
edges_sent <- ego_edges_for_graph %>%
  filter(nadia_role_in_comm == "Nadia Sent")

edges_received <- ego_edges_for_graph %>%
  filter(nadia_role_in_comm == "Nadia Received")

# Function to build visNetwork plot for a given edge set
build_visnetwork_plot <- function(edge_df, title_label) {
  # Prepare nodes from edge list
  node_ids <- unique(c(edge_df$from, edge_df$to))

  nodes <- ego_nodes_for_graph %>%
    filter(id %in% node_ids) %>%
    mutate(
      label = ifelse(display_name == "Nadia Conti", "", display_name),
      title = paste0(
        "<b>", display_name, "</b><br>",
        "Type: ", node_legend_group, "<br>",
        "Sent: ", sent_count, "<br>",
        "Received: ", received_count
      ),
      group = ifelse(display_name == "Nadia Conti", "Nadia Conti", node_legend_group)
    ) %>%
    select(id, label, title, group, value = node_size)

  # Prepare edges with arrow and tooltip
  edges <- edge_df %>%
    filter(!is.na(from) & !is.na(to)) %>%
    mutate(
      arrows = "to",
      title = paste0(
        "<b>From:</b> ", sender_name, "<br>",
        "<b>To:</b> ", recipient_name, "<br>",
        "<b>Type:</b> ", communication_type, "<br>",
        "<b>Count:</b> ", count_of_comm, "<br>",
        "<b>First:</b> ", format(timestamp_min, "%Y-%m-%d %H:%M:%S"), "<br>",
        "<b>Last:</b> ", format(timestamp_max, "%Y-%m-%d %H:%M:%S"), "<br>",
      "<b>Content Sample:</b><br><div style='max-width:300px;white-space:normal;'>", content_sample, "</div>"
      )
    ) %>%
    select(from, to, arrows, title, width = count_of_comm)

  # Create visNetwork
  visNetwork(nodes, edges, width = "100%", height = "600px") %>%
    visEdges(smooth = FALSE, arrows = list(to = list(enabled = TRUE, scaleFactor = 1.5))) %>%
    visOptions(highlightNearest = TRUE, nodesIdSelection = TRUE) %>%
    visIgraphLayout(layout = "layout_with_fr") %>%
    visNodes(font = list(size = 14)) %>%
    visGroups(groupname = "Person", color = node_legend_colors_plot[["Person"]], shape = node_legend_shapes_plot[["Person"]]) %>%
    visGroups(groupname = "Vessel", color = node_legend_colors_plot[["Vessel"]], shape = node_legend_shapes_plot[["Vessel"]]) %>%
    visGroups(groupname = "Organization", color = node_legend_colors_plot[["Organization"]], shape = node_legend_shapes_plot[["Organization"]]) %>%
    visGroups(groupname = "Location", color = node_legend_colors_plot[["Location"]], shape = node_legend_shapes_plot[["Location"]]) %>%
    visGroups(groupname = "Nadia Conti", color = node_legend_colors_plot[["Nadia Conti"]], shape = node_legend_shapes_plot[["Nadia Conti"]]) %>%
      visLegend(
    addNodes = legend_df,
    ncol = 2,
    position = "left",
    main = "Entity (Sub)Types",
    useGroups = FALSE) %>%
    visLayout(randomSeed = 123) %>%
    visNetwork::visExport() %>%
    visNetwork::visPhysics(enabled = TRUE) %>%
    visNetwork::visInteraction(navigationButtons = TRUE) %>%
    visNetwork::visEvents(stabilizationIterationsDone = "function () {this.setOptions({physics:false});}")
}

# Build and show visNetwork plots
build_visnetwork_plot(edges_sent, "Nadia Sent Ego Network")

saveRDS(nodes, file = file.path("rds_objects", "nodes.rds"))
saveRDS(edges_sent, file = file.path("rds_objects", "edges_sent.rds"))
saveRDS(edges_received, file = file.path("rds_objects", "edges_received.rds"))
saveRDS(build_visnetwork_plot, file = file.path("rds_objects", "build_visnetwork_plot.rds"))
saveRDS(node_legend_shapes_plot, file = file.path("rds_objects", "node_legend_shapes_plot.rds"))
saveRDS(node_ids, file = file.path("rds_objects", "node_ids.rds"))
saveRDS(node_legend_colors_plot, file = file.path("rds_objects", "node_legend_colors_plot.rds"))
saveRDS(edges, file = file.path("rds_objects", "edges.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
# Define custom colors for node types (reusing from previous code)
node_legend_colors_plot <- c(
  "Person" = "#88CCEE",      # Blue
  "Vessel" = "#D55E00",      # Orange
  "Organization" = "#117733", # Green
  "Location" = "#AA4499",   # Purple
  "Nadia Conti" = "red"      # Special color for Nadia
)

# Define custom shapes for node subtypes
node_legend_shapes_plot <- c(
  "Person" = "dot",        # Circle for visNetwork
  "Vessel" = "triangle",
  "Organization" = "square",
  "Location" = "diamond",
  "Nadia Conti" = "star"   # Star shape for Nadia in visNetwork
)

# Separate the edges into two sets: Nadia Sent and Nadia Received
edges_sent <- ego_edges_for_graph %>%
  filter(nadia_role_in_comm == "Nadia Sent")

edges_received <- ego_edges_for_graph %>%
  filter(nadia_role_in_comm == "Nadia Received")

# Function to build visNetwork plot for a given edge set
build_visnetwork_plot <- function(edge_df, title_label) {
  # Prepare nodes from edge list
  node_ids <- unique(c(edge_df$from, edge_df$to))

  nodes <- ego_nodes_for_graph %>%
    filter(id %in% node_ids) %>%
    mutate(
      label = ifelse(display_name == "Nadia Conti", "", display_name),
      title = paste0(
        "<b>", display_name, "</b><br>",
        "Type: ", node_legend_group, "<br>",
        "Sent: ", sent_count, "<br>",
        "Received: ", received_count
      ),
      group = ifelse(display_name == "Nadia Conti", "Nadia Conti", node_legend_group)
    ) %>%
    select(id, label, title, group, value = node_size)

  # Prepare edges with arrow and tooltip
  edges <- edge_df %>%
    filter(!is.na(from) & !is.na(to)) %>%
    mutate(
      arrows = "to",
      title = paste0(
        "<b>From:</b> ", sender_name, "<br>",
        "<b>To:</b> ", recipient_name, "<br>",
        "<b>Type:</b> ", communication_type, "<br>",
        "<b>Count:</b> ", count_of_comm, "<br>",
        "<b>First:</b> ", format(timestamp_min, "%Y-%m-%d %H:%M:%S"), "<br>",
        "<b>Last:</b> ", format(timestamp_max, "%Y-%m-%d %H:%M:%S"), "<br>",
      "<b>Content Sample:</b><br><div style='max-width:300px;white-space:normal;'>", content_sample, "</div>"
      )
    ) %>%
    select(from, to, arrows, title, width = count_of_comm)

  # Create visNetwork
  visNetwork(nodes, edges, width = "100%", height = "600px") %>%
    visEdges(smooth = FALSE, arrows = list(to = list(enabled = TRUE, scaleFactor = 1.5))) %>%
    visOptions(highlightNearest = TRUE, nodesIdSelection = TRUE) %>%
    visIgraphLayout(layout = "layout_with_fr") %>%
    visNodes(font = list(size = 14)) %>%
    visGroups(groupname = "Person", color = node_legend_colors_plot[["Person"]], shape = node_legend_shapes_plot[["Person"]]) %>%
    visGroups(groupname = "Vessel", color = node_legend_colors_plot[["Vessel"]], shape = node_legend_shapes_plot[["Vessel"]]) %>%
    visGroups(groupname = "Organization", color = node_legend_colors_plot[["Organization"]], shape = node_legend_shapes_plot[["Organization"]]) %>%
    visGroups(groupname = "Location", color = node_legend_colors_plot[["Location"]], shape = node_legend_shapes_plot[["Location"]]) %>%
    visGroups(groupname = "Nadia Conti", color = node_legend_colors_plot[["Nadia Conti"]], shape = node_legend_shapes_plot[["Nadia Conti"]]) %>%
      visLegend(
    addNodes = legend_df,
    ncol = 2,
    position = "left",
    main = "Entity (Sub)Types",
    useGroups = FALSE) %>%
    visLayout(randomSeed = 123) %>%
    visNetwork::visExport() %>%
    visNetwork::visPhysics(enabled = TRUE) %>%
    visNetwork::visInteraction(navigationButtons = TRUE) %>%
    visNetwork::visEvents(stabilizationIterationsDone = "function () {this.setOptions({physics:false});}")
}

# Build and show visNetwork plots
build_visnetwork_plot(edges_sent, "Nadia Sent Ego Network")

saveRDS(nodes, file = file.path("rds_objects", "nodes.rds"))
saveRDS(edges_sent, file = file.path("rds_objects", "edges_sent.rds"))
saveRDS(edges_received, file = file.path("rds_objects", "edges_received.rds"))
saveRDS(build_visnetwork_plot, file = file.path("rds_objects", "build_visnetwork_plot.rds"))
saveRDS(node_legend_shapes_plot, file = file.path("rds_objects", "node_legend_shapes_plot.rds"))
saveRDS(node_ids, file = file.path("rds_objects", "node_ids.rds"))
saveRDS(node_legend_colors_plot, file = file.path("rds_objects", "node_legend_colors_plot.rds"))
saveRDS(edges, file = file.path("rds_objects", "edges.rds"))

# ------------------
build_visnetwork_plot(edges_received, "Nadia Received Ego Network")


# ------------------
#| code-fold: true
#| code-summary: "Show the code"
build_visnetwork_plot(edges_received, "Nadia Received Ego Network")


# ------------------
# ---- 1. Define styles and legends ----

library(igraph)

event_subtypes <- c(
  "Communication", "Monitoring", "VesselMovement", "Assessment",
  "Collaborate", "Endorsement", "TourActivity", "TransponderPing",
  "Harbor Report", "Fishing", "Criticize"
)

relationship_subtypes <- c(
  "Coordinates", "AccessPermission", "Operates", "Colleagues",
  "Suspicious", "Reports", "Jurisdiction", "Unfriendly", "Friends"
)

node_legend_colors_plot <- c(
  "Person" = "#88CCEE",
  "Vessel" = "#D55E00",
  "Organization" = "#117733",
  "Location" = "#AA4499",
  "Group"= "#CC79A7",
  "Event" = "#DDCC77",
  "Relationship" = "#AF8DC3"
)

node_legend_shapes_plot <- c(
  "Person" = "dot",
  "Vessel" = "triangle",
  "Organization" = "square",
  "Location" = "diamond",
  "Group" = "circle plus",
  "Event" = "star",
  "Relationship" = "square x"
)

STYLES <- list(
  node_label_dark = "black",
  font_family = "Roboto Condensed"
)

# ---- Define ego extraction function ----
extract_ego_subgraph <- function(center_node = "Nadia Conti", hops = 2) {
  g <- graph_from_data_frame(
    d = mc3_edges_final %>% select(from = from_id, to = to_id),
    vertices = mc3_nodes_final %>% select(id) %>% distinct() %>% rename(name = id),
    directed = TRUE
  )
  ego_graph <- make_ego_graph(g, order = hops, nodes = center_node, mode = "all")[[1]]
  node_ids <- V(ego_graph)$name
  edge_df <- as_data_frame(ego_graph, what = "edges")

  ego_nodes <- mc3_nodes_final %>%
    filter(id %in% node_ids) %>%
    mutate(
      label = ifelse(is.na(name), id, name),
      tooltip_extra = case_when(
        type == "Event" & sub_type == "Communication" ~ content,
        type == "Event" & sub_type == "Monitoring" ~ findings,
        type == "Event" & sub_type == "VesselMovement" ~ destination,
        type == "Event" & sub_type == "Assessment" ~ results,
        type == "Relationship" & sub_type == "Coordinates" ~ coordination_type,
        type == "Relationship" & sub_type == "Operates" ~ operational_role,
        type == "Relationship" & sub_type == "Jurisdiction" ~ jurisdiction_type,
        TRUE ~ NA_character_
      ),
      title = paste0(
        "<b>", label, "</b><br>",
        "Type: ", type, "<br>",
        "Sub-type: ", sub_type, "<br>",
        ifelse(!is.na(tooltip_extra), paste0("<br><b>Details:</b> ", tooltip_extra), "")
      ),
      group = ifelse(sub_type %in% names(node_legend_colors_plot), sub_type, type)
    ) %>%
    select(id, label, group, title) %>%
    distinct()

  list(nodes = ego_nodes, edges = edge_df)
}

# ---- Extract and build Nadia 2-hop ego network ----
ego_data <- extract_ego_subgraph("Nadia Conti", hops = 2)

nodes <- ego_data$nodes
edges <- ego_data$edges

# Save for inspection or export
nadia_ego_nodes_2hop <- nodes
nadia_ego_edges_2hop <- edges

# ---- Build visNetwork ----
net <- visNetwork(nodes, edges, width = "100%", height = "600px") %>%
  visEdges(arrows = list(to = list(enabled = TRUE, scaleFactor = 1.5))) %>%
  visOptions(highlightNearest = TRUE, nodesIdSelection = TRUE) %>%
  visIgraphLayout(layout = "layout_with_fr") %>%
  visNodes(font = list(
    size = 14,
    color = STYLES$node_label_dark,
    face = STYLES$font_family,
    vadjust = -15
  ))

# ---- Apply shape and color per group ----
for (group_name in names(node_legend_colors_plot)) {
  net <- net %>% visGroups(
    groupname = group_name,
    color = node_legend_colors_plot[[group_name]],
    shape = node_legend_shapes_plot[[group_name]]
  )
}

# ---- Add legend ----
used_groups <- unique(nodes$group)

legend_df <- tibble::tibble(
  label = used_groups,
  shape = node_legend_shapes_plot[used_groups],
  color = node_legend_colors_plot[used_groups]
) %>%
  distinct(label, .keep_all = TRUE)

net <- net %>% visLegend(
  addNodes = legend_df,
  ncol = 2,
  position = "left",
  main = "Entity (Sub)Types",
  useGroups = FALSE
)

# ---- Render ----
net


saveRDS(g, file = file.path("rds_objects", "g.rds"))
saveRDS(edge_df, file = file.path("rds_objects", "edge_df.rds"))
saveRDS(node_legend_shapes_plot, file = file.path("rds_objects", "node_legend_shapes_plot.rds"))
saveRDS(extract_ego_subgraph, file = file.path("rds_objects", "extract_ego_subgraph.rds"))
saveRDS(node_legend_colors_plot, file = file.path("rds_objects", "node_legend_colors_plot.rds"))
saveRDS(edges, file = file.path("rds_objects", "edges.rds"))
saveRDS(ego_data, file = file.path("rds_objects", "ego_data.rds"))
saveRDS(STYLES, file = file.path("rds_objects", "STYLES.rds"))
saveRDS(nodes, file = file.path("rds_objects", "nodes.rds"))
saveRDS(ego_nodes, file = file.path("rds_objects", "ego_nodes.rds"))
saveRDS(ego_graph, file = file.path("rds_objects", "ego_graph.rds"))
saveRDS(nadia_ego_edges_2hop, file = file.path("rds_objects", "nadia_ego_edges_2hop.rds"))
saveRDS(legend_df, file = file.path("rds_objects", "legend_df.rds"))
saveRDS(net, file = file.path("rds_objects", "net.rds"))
saveRDS(event_subtypes, file = file.path("rds_objects", "event_subtypes.rds"))
saveRDS(relationship_subtypes, file = file.path("rds_objects", "relationship_subtypes.rds"))
saveRDS(node_ids, file = file.path("rds_objects", "node_ids.rds"))
saveRDS(used_groups, file = file.path("rds_objects", "used_groups.rds"))
saveRDS(nadia_ego_nodes_2hop, file = file.path("rds_objects", "nadia_ego_nodes_2hop.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
# ---- 1. Define styles and legends ----

library(igraph)

event_subtypes <- c(
  "Communication", "Monitoring", "VesselMovement", "Assessment",
  "Collaborate", "Endorsement", "TourActivity", "TransponderPing",
  "Harbor Report", "Fishing", "Criticize"
)

relationship_subtypes <- c(
  "Coordinates", "AccessPermission", "Operates", "Colleagues",
  "Suspicious", "Reports", "Jurisdiction", "Unfriendly", "Friends"
)

node_legend_colors_plot <- c(
  "Person" = "#88CCEE",
  "Vessel" = "#D55E00",
  "Organization" = "#117733",
  "Location" = "#AA4499",
  "Group"= "#CC79A7",
  "Event" = "#DDCC77",
  "Relationship" = "#AF8DC3"
)

node_legend_shapes_plot <- c(
  "Person" = "dot",
  "Vessel" = "triangle",
  "Organization" = "square",
  "Location" = "diamond",
  "Group" = "circle plus",
  "Event" = "star",
  "Relationship" = "square x"
)

STYLES <- list(
  node_label_dark = "black",
  font_family = "Roboto Condensed"
)

# ---- Define ego extraction function ----
extract_ego_subgraph <- function(center_node = "Nadia Conti", hops = 2) {
  g <- graph_from_data_frame(
    d = mc3_edges_final %>% select(from = from_id, to = to_id),
    vertices = mc3_nodes_final %>% select(id) %>% distinct() %>% rename(name = id),
    directed = TRUE
  )
  ego_graph <- make_ego_graph(g, order = hops, nodes = center_node, mode = "all")[[1]]
  node_ids <- V(ego_graph)$name
  edge_df <- as_data_frame(ego_graph, what = "edges")

  ego_nodes <- mc3_nodes_final %>%
    filter(id %in% node_ids) %>%
    mutate(
      label = ifelse(is.na(name), id, name),
      tooltip_extra = case_when(
        type == "Event" & sub_type == "Communication" ~ content,
        type == "Event" & sub_type == "Monitoring" ~ findings,
        type == "Event" & sub_type == "VesselMovement" ~ destination,
        type == "Event" & sub_type == "Assessment" ~ results,
        type == "Relationship" & sub_type == "Coordinates" ~ coordination_type,
        type == "Relationship" & sub_type == "Operates" ~ operational_role,
        type == "Relationship" & sub_type == "Jurisdiction" ~ jurisdiction_type,
        TRUE ~ NA_character_
      ),
      title = paste0(
        "<b>", label, "</b><br>",
        "Type: ", type, "<br>",
        "Sub-type: ", sub_type, "<br>",
        ifelse(!is.na(tooltip_extra), paste0("<br><b>Details:</b> ", tooltip_extra), "")
      ),
      group = ifelse(sub_type %in% names(node_legend_colors_plot), sub_type, type)
    ) %>%
    select(id, label, group, title) %>%
    distinct()

  list(nodes = ego_nodes, edges = edge_df)
}

# ---- Extract and build Nadia 2-hop ego network ----
ego_data <- extract_ego_subgraph("Nadia Conti", hops = 2)

nodes <- ego_data$nodes
edges <- ego_data$edges

# Save for inspection or export
nadia_ego_nodes_2hop <- nodes
nadia_ego_edges_2hop <- edges

# ---- Build visNetwork ----
net <- visNetwork(nodes, edges, width = "100%", height = "600px") %>%
  visEdges(arrows = list(to = list(enabled = TRUE, scaleFactor = 1.5))) %>%
  visOptions(highlightNearest = TRUE, nodesIdSelection = TRUE) %>%
  visIgraphLayout(layout = "layout_with_fr") %>%
  visNodes(font = list(
    size = 14,
    color = STYLES$node_label_dark,
    face = STYLES$font_family,
    vadjust = -15
  ))

# ---- Apply shape and color per group ----
for (group_name in names(node_legend_colors_plot)) {
  net <- net %>% visGroups(
    groupname = group_name,
    color = node_legend_colors_plot[[group_name]],
    shape = node_legend_shapes_plot[[group_name]]
  )
}

# ---- Add legend ----
used_groups <- unique(nodes$group)

legend_df <- tibble::tibble(
  label = used_groups,
  shape = node_legend_shapes_plot[used_groups],
  color = node_legend_colors_plot[used_groups]
) %>%
  distinct(label, .keep_all = TRUE)

net <- net %>% visLegend(
  addNodes = legend_df,
  ncol = 2,
  position = "left",
  main = "Entity (Sub)Types",
  useGroups = FALSE
)

# ---- Render ----
net


saveRDS(g, file = file.path("rds_objects", "g.rds"))
saveRDS(edge_df, file = file.path("rds_objects", "edge_df.rds"))
saveRDS(node_legend_shapes_plot, file = file.path("rds_objects", "node_legend_shapes_plot.rds"))
saveRDS(extract_ego_subgraph, file = file.path("rds_objects", "extract_ego_subgraph.rds"))
saveRDS(node_legend_colors_plot, file = file.path("rds_objects", "node_legend_colors_plot.rds"))
saveRDS(edges, file = file.path("rds_objects", "edges.rds"))
saveRDS(ego_data, file = file.path("rds_objects", "ego_data.rds"))
saveRDS(STYLES, file = file.path("rds_objects", "STYLES.rds"))
saveRDS(nodes, file = file.path("rds_objects", "nodes.rds"))
saveRDS(ego_nodes, file = file.path("rds_objects", "ego_nodes.rds"))
saveRDS(ego_graph, file = file.path("rds_objects", "ego_graph.rds"))
saveRDS(nadia_ego_edges_2hop, file = file.path("rds_objects", "nadia_ego_edges_2hop.rds"))
saveRDS(legend_df, file = file.path("rds_objects", "legend_df.rds"))
saveRDS(net, file = file.path("rds_objects", "net.rds"))
saveRDS(event_subtypes, file = file.path("rds_objects", "event_subtypes.rds"))
saveRDS(relationship_subtypes, file = file.path("rds_objects", "relationship_subtypes.rds"))
saveRDS(node_ids, file = file.path("rds_objects", "node_ids.rds"))
saveRDS(used_groups, file = file.path("rds_objects", "used_groups.rds"))
saveRDS(nadia_ego_nodes_2hop, file = file.path("rds_objects", "nadia_ego_nodes_2hop.rds"))

# ------------------
# Extract Nadia Conti's 2-hop ego network
g <- graph_from_data_frame(
  d = mc3_edges_final %>% select(from = from_id, to = to_id),
  vertices = mc3_nodes_final %>% select(id) %>% distinct() %>% rename(name = id),
  directed = TRUE
)

ego_graph <- make_ego_graph(g, order = 2, nodes = "Nadia Conti", mode = "all")[[1]]
node_ids <- V(ego_graph)$name
edge_df <- as_data_frame(ego_graph, what = "edges")

nadia_ego_nodes_2hop <- mc3_nodes_final %>%
  filter(id %in% node_ids) %>%
  mutate(
    label = ifelse(is.na(name), id, name),
    tooltip_extra = case_when(
      type == "Event" & sub_type == "Communication" ~ content,
      type == "Event" & sub_type == "Monitoring" ~ findings,
      type == "Event" & sub_type == "VesselMovement" ~ destination,
      type == "Event" & sub_type == "Assessment" ~ results,
      type == "Relationship" & sub_type == "Coordinates" ~ coordination_type,
      type == "Relationship" & sub_type == "Operates" ~ operational_role,
      type == "Relationship" & sub_type == "Jurisdiction" ~ jurisdiction_type,
      TRUE ~ NA_character_
    )
  )

nadia_ego_edges_2hop <- edge_df %>%
  left_join(nadia_ego_nodes_2hop %>% select(id, sender_name = label, sender_sub_type = sub_type), by = c("from" = "id")) %>%
  left_join(nadia_ego_nodes_2hop %>% select(id, recipient_name = label, recipient_sub_type = sub_type), by = c("to" = "id"))

nadia_ego_comm_df <- edge_df %>%
  left_join(nadia_ego_nodes_2hop %>% select(id, name, sub_type), by = c("from" = "id")) %>%
  rename(
    sender_id = from,
    sender_name = name,
    sender_type = sub_type
  ) %>%
  left_join(nadia_ego_nodes_2hop %>% select(id, name, sub_type, content, findings, destination, results, coordination_type, operational_role, jurisdiction_type, timestamp), 
            by = c("to" = "id")) %>%
  rename(
    recipient_id = to,
    recipient_name = name,
    recipient_type = sub_type
  ) %>%
  mutate(
    details = coalesce(content, findings, destination, results, coordination_type, operational_role, jurisdiction_type),
    timestamp = ymd_hms(timestamp)
  ) %>%
  select(
    sender_id, sender_name, sender_type,
    recipient_id, recipient_name, recipient_type,
    timestamp, details
  )

# Print table of suspicious activities/ characters
suspicious_links <- nadia_ego_comm_df %>%
  filter(
    grepl("Suspicious", sender_type, ignore.case = TRUE) |
    grepl("Suspicious", recipient_type, ignore.case = TRUE)
  ) %>%
  select(
    sender_id, sender_type,
    recipient_id, recipient_type
  )

print(suspicious_links)


saveRDS(g, file = file.path("rds_objects", "g.rds"))
saveRDS(edge_df, file = file.path("rds_objects", "edge_df.rds"))
saveRDS(suspicious_links, file = file.path("rds_objects", "suspicious_links.rds"))
saveRDS(nadia_ego_comm_df, file = file.path("rds_objects", "nadia_ego_comm_df.rds"))
saveRDS(ego_graph, file = file.path("rds_objects", "ego_graph.rds"))
saveRDS(nadia_ego_edges_2hop, file = file.path("rds_objects", "nadia_ego_edges_2hop.rds"))
saveRDS(node_ids, file = file.path("rds_objects", "node_ids.rds"))
saveRDS(nadia_ego_nodes_2hop, file = file.path("rds_objects", "nadia_ego_nodes_2hop.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
#Filtering communications that matches Nadia's 2 hop data
event_ids_to_extract <- nadia_ego_comm_df %>%
  filter(str_starts(sender_id, "Event_Communication_")) %>%
  pull(sender_id) %>%
  unique()

matched_comms_df<- other_communications_df %>%
  filter(event_id %in% event_ids_to_extract)

# --- FACTORING and DATETIME CLEANING ---
matched_comms_df_for_plot <- matched_comms_df %>%
  mutate(
    timestamp = as.POSIXct(timestamp),
    comm_date = as.Date(timestamp),
    comm_time_of_day = hms::as_hms(format(timestamp, "%H:%M:%S")),
    sender_sub_type = factor(sender_sub_type, levels = c("Person", "Vessel", "Organization", "Location")),
    communicating_pair_sorted = paste(pmin(sender_name, recipient_name), pmax(sender_name, recipient_name), sep = " & ")
  )

# 7. Add wrapped tooltip content for timeline plotting
plot_data <- matched_comms_df_for_plot %>%
  mutate(
    timestamp = as.POSIXct(timestamp),
    date = as.Date(timestamp),
    time = format(timestamp, "%H:%M:%S"),
    wrapped_content = str_wrap(content, width = 50),
    tooltip_text = paste0(
      "<b>Date:</b> ", date, "<br>",
      "<b>Time:</b> ", time, "<br>",
      "<b>Event_id:</b> ", event_id, "<br><br>",
      "<b>From:</b> ", sender_name, "<br>",
      "<b>To:</b> ", recipient_name, "<br><br>",
      "<b>Content:</b><br>", wrapped_content
    )
  )

# 8. Create plot with consistent 6-hour y-axis time scale
p <-ggplot(plot_data, aes(x = comm_date, y = comm_time_of_day)) +
  geom_point(aes(
    color = sender_id,
    shape = sender_sub_type,
    text = tooltip_text
  ),show.legend = c(color = TRUE, shape = FALSE), 
  size = 2, alpha = 0.7) +
  scale_shape_manual(values = c("Person" = 16, 
                                "Vessel" = 17,
                                "Organization" = 15,
                                "Location" = 18
                                  )) +
  facet_wrap(~ sender_sub_type, ncol = 1, scales = "fixed") +
    scale_y_time(
    limits = hms::as_hms(c("08:00:00", "14:00:00")),  # reversed to show time top-to-bottom
    breaks = hms::as_hms(c("08:00:00", "09:00:00", "10:00:00", "11:00:00", "12:00:00", "13:00:00", "14:00:00")),
    labels = c("08:00", "09:00", "10:00", "11:00", "12:00", "13:00", "14:00")
)+
  scale_x_date(
  date_breaks = "1 day",
  date_labels = "%d %b"
)+

  facet_wrap(~ sender_sub_type, ncol = 1) +
  labs(
    title = "Nadia's 2-hop Communications (Sender's perspective)",
    x = "Date", y = "Time of Day", color = "Sender (subtype, name)"
  ) +
  theme_grey() +
  theme(
    axis.text.y = element_text(size = 6),
    axis.title.y = element_text(size = 7),
    axis.ticks.y = element_line(),
    axis.text.x = element_text(size = 6, angle = 45, hjust = 1),
    axis.title.x = element_text(margin = margin(t = 10), size = 7),
    panel.spacing = unit(0.5, "lines"),  # Applies to both x and y spacing
    strip.text = element_text(size = 8, face = "bold"),
    legend.position = "bottom",
    legend.text = element_text(size = 6),
    legend.title = element_blank()
  )

# 9. Make interactive with Plotly
ggplotly(p, tooltip = "text")

saveRDS(matched_comms_df, file = file.path("rds_objects", "matched_comms_df.rds"))
saveRDS(plot_data, file = file.path("rds_objects", "plot_data.rds"))
saveRDS(event_ids_to_extract, file = file.path("rds_objects", "event_ids_to_extract.rds"))
saveRDS(matched_comms_df_for_plot, file = file.path("rds_objects", "matched_comms_df_for_plot.rds"))
saveRDS(p, file = file.path("rds_objects", "p.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"

library(dplyr)
library(stringr)
library(ggplot2)
library(plotly)
library(hms)
library(tidyr)

# -- Step 1: Define keywords
keywords <- c("permit", "1045", "7844")
pattern <- paste0("\\b(", paste(keywords, collapse = "|"), ")\\b")

# -- Step 2: Filter messages
search_target_content <- tolower(other_communications_df$content)

keyword_matches_df <- other_communications_df %>%
  filter(str_detect(search_target_content, pattern))

# -- Step 3: Extract keywords
plot_data <- keyword_matches_df %>%
  mutate(matched_keywords = str_extract_all(tolower(content), pattern)) %>%
  unnest(matched_keywords) %>%
  mutate(matched_keywords = str_to_title(matched_keywords)) %>%
  arrange(matched_keywords, timestamp) %>%
  mutate(
    timestamp = as.POSIXct(timestamp),
    comm_date = as.Date(timestamp),
    comm_time_of_day = hms::as_hms(format(timestamp, "%H:%M:%S")),
    wrapped_content = str_wrap(content, width = 50),
    tooltip_text = paste0(
      "<b>Date:</b> ", comm_date,
      "<br><b>Time:</b> ", comm_time_of_day,
      "<br><b>Event ID:</b> ", event_id,
      "<br><b>Content:</b><br>", wrapped_content
    )
  )

# -- Step 4: Plot
p <- ggplot(plot_data, aes(x = comm_date, y = comm_time_of_day)) +
  geom_point(aes(
    color = matched_keywords,
    shape = sender_sub_type,
    text = tooltip_text,
    group = matched_keywords  # ensures matched_keywords is in layer
  ), size = 2.5, alpha = 0.7, show.legend = TRUE) +
  scale_shape_manual(values = c(
    "Person" = 16,
    "Vessel" = 17,
    "Organization" = 15,
    "Location" = 18
  )) +
  facet_wrap(~ matched_keywords, ncol = 1, scales = "fixed") +
  scale_y_time(
    limits = hms::as_hms(c("08:00:00", "13:00:00")),
    breaks = hms::as_hms(c("08:00:00", "09:00:00", "10:00:00", "11:00:00", "12:00:00", "13:00:00")),
    labels = c("08:00", "09:00", "10:00", "11:00", "12:00", "13:00")
  ) +
  scale_x_date(date_breaks = "1 day", date_labels = "%d %b") +
  labs(
    title = "Interactive Timeline: Keyword Mentions by Day and Time",
    x = "Date",
    y = "Time of Day",
    shape = "Sender Type",
    color = " "
  ) +
  theme_grey() +
  theme(
    axis.text.y = element_text(size = 6),
    axis.title.y = element_text(size = 7),
    axis.ticks.y = element_line(),
    axis.text.x = element_text(size = 6, angle = 45, hjust = 1),
    axis.title.x = element_text(margin = margin(t = 10), size = 7),
    panel.spacing = unit(0.5, "lines"),  # Applies to both x and y spacing
    strip.text = element_text(size = 8, face = "bold"),
    legend.position = "bottom",
    legend.text = element_text(size = 6),
    legend.title = element_blank()
  )

# -- Step 5: Convert to plotly
ggplotly(p, tooltip = "text")

saveRDS(keyword_matches_df, file = file.path("rds_objects", "keyword_matches_df.rds"))
saveRDS(plot_data, file = file.path("rds_objects", "plot_data.rds"))
saveRDS(p, file = file.path("rds_objects", "p.rds"))
saveRDS(search_target_content, file = file.path("rds_objects", "search_target_content.rds"))
saveRDS(keywords, file = file.path("rds_objects", "keywords.rds"))
saveRDS(pattern, file = file.path("rds_objects", "pattern.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"

library(reactable)
library(dplyr)
library(stringr)
library(readr) # Used for read_lines

# 1. Prepare data from the text provided
data_text <- "
Date & Time | Event Description
------------|---------------------------------------------------------------------------------------------------------------
2 Oct (12:16pm) | Rodriguez was from the Sailor Shifts Team.
5 Oct (10:54am) | Jensen from City Council approved Nemo Reef permit.
6 Oct (9:57am) | Remora told Sailor Shifts Team that Nadia got Commissioner Torres to sign off the permit.
6 Oct (10:45am) | Mako acknowledged NR-1045 permit to Nemo Reef.
6 Oct (12:33pm) | Mako was lead vessel to Neptune and Remora as authorised by V. Miesel Shipping.
7 Oct (9:40am) | Mako was operating under permit NR-1045 for conservation research.
8 Oct (10:24am) | Mako was operating under V. Miesel's special marine research permit CR-7844 approved by Oceanus City Council.
8 Oct (10:30am) | Mako informed Remora that both of them were operating under permit CR-7844 and have a 5 day deadline.
8 Oct (10:40am) | Remora was approved by Paackland Harbor to operate with extended hours under permit NR-1045.
9 Oct (11:53am) | Mako requesed for additional crew from v Miesel Shipping for the 24hr operations over next 5 days.
11 Oct (6:00am) | Nemo Reef closure mandated by Oceanus City Council.
11 Oct (8:57am) | All research permits must be submitted within 72 hours.
11 Oct (10:05am) | V. Miesel Shipping informed Remora that 30% of her crew to be reassigned to Neptune.
12 Oct (10:01am) | Davis as Captain oversaw crew reallocation.
12 Oct (11:19am) | Nadia secured documentation for CR-7844.
12 Oct (12:52pm) | Harbor closure for 3 days from 13 Oct 06:00.
"

# Parse the data into a data frame
# Use read_lines to handle the multi-line string
data_lines <- read_lines(data_text)

# Remove the header separator line and any empty lines
data_lines <- data_lines[!grepl("^---|===", data_lines) & data_lines != ""]

# The first non-separator line is the header
col_names_raw <- str_split(data_lines[1], "\\|")[[1]]
col_names <- str_trim(col_names_raw)

# The rest are data rows
data_content <- data_lines[-1]

# Create data frame by splitting lines and putting into a tibble
df <- tibble(line = data_content) %>%
  mutate(
    `Date & Time` = str_trim(str_extract(line, "^[^|]+")),
    `Event Description` = str_trim(str_extract(line, "(?<=\\|).+$"))
  ) %>%
  select(`Date & Time`, `Event Description`) # Ensure correct column order and names

# 2. Create the reactable table with desired features

reactable(
  df,
  filterable = TRUE, # Enable column-specific filters (from the tutorial)
  searchable = TRUE, # Enable global search box (from the tutorial)
  paginationType = "numbers", # Display page numbers (corrected from "pages")
  defaultPageSize = 5, # Show 5 rows per page
  showPageSizeOptions = TRUE, # Allow users to change page size
  pageSizeOptions = c(5, 10, 15, 20, 50), # Options for page sizes
  striped = TRUE, # Add alternating row colors (from the tutorial)
  highlight = TRUE, # Highlight row on hover (from the tutorial)
  columns = list(
    `Date & Time` = colDef(
      name = "Date & Time",
      minWidth = 120, # Adjust width to fit content
      align = "left"
    ),
    `Event Description` = colDef(
      name = "Event Description",
      minWidth = 500, # Ensure enough width for event descriptions
      align = "left"
    )
  ),
  # Apply a custom theme for better aesthetics (inspired by the tutorial)
  theme = reactableTheme(
    borderColor = "#dfe2e5",
    stripedColor = "#f6f8fa",
    highlightColor = "#f0f5f9",
    cellPadding = "8px 12px",
    style = list(fontFamily = "Verdana, Geneva, sans-serif", fontSize = "14px"),
    headerStyle = list(
      "&.rt-th:hover" = list(backgroundColor = "#e0e6eb"),
      fontSize = "15px",
      fontWeight = 600,
      color = "#333", # Darker header text for contrast
      background = "#f7f7f7" # Slightly grey background for header
    ),
    rowSelectedStyle = list(backgroundColor = "#e6f2ff", "&:hover" = list(backgroundColor = "#e6f2ff")),
    # Styles for search/filter inputs (from tutorial's theme example)
    searchInputStyle = list(width = "100%", margin = "5px 0", padding = "5px"),
    filterInputStyle = list(width = "100%", margin = "2px 0", padding = "4px")
  )
)

saveRDS(col_names, file = file.path("rds_objects", "col_names.rds"))
saveRDS(df, file = file.path("rds_objects", "df.rds"))
saveRDS(col_names_raw, file = file.path("rds_objects", "col_names_raw.rds"))
saveRDS(data_text, file = file.path("rds_objects", "data_text.rds"))
saveRDS(data_lines, file = file.path("rds_objects", "data_lines.rds"))
saveRDS(data_content, file = file.path("rds_objects", "data_content.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
library(dplyr)
library(stringr)
library(ggplot2)
library(plotly)
library(hms)
library(tidyr)

# -- Step 1: Define keywords
keywords <- c("mine", "mining", "music video", "lighting", "reef operation")
pattern <- paste0("\\b(", paste(keywords, collapse = "|"), ")\\b")

# -- Step 2: Filter messages
search_target_content <- tolower(other_communications_df$content)

keyword_matches_df <- other_communications_df %>%
  filter(str_detect(search_target_content, pattern))

# -- Step 3: Extract keywords
plot_data <- keyword_matches_df %>%
  mutate(matched_keywords = str_extract_all(tolower(content), pattern)) %>%
  unnest(matched_keywords) %>%
  mutate(matched_keywords = str_to_title(matched_keywords)) %>%
  arrange(matched_keywords, timestamp) %>%
  mutate(
    timestamp = as.POSIXct(timestamp),
    comm_date = as.Date(timestamp),
    comm_time_of_day = hms::as_hms(format(timestamp, "%H:%M:%S")),
    wrapped_content = str_wrap(content, width = 50),
    tooltip_text = paste0(
      "<b>Date:</b> ", comm_date,
      "<br><b>Time:</b> ", comm_time_of_day,
      "<br><b>Event ID:</b> ", event_id,
      "<br><b>From:</b> ", sender_name,
      "<br><b>To:</b> ", recipient_name,
      "<br><b>Content:</b><br>", wrapped_content
    )
  )

# -- Step 4: Plot
p <- ggplot(plot_data, aes(x = comm_date, y = comm_time_of_day)) +
  geom_point(aes(
    color = matched_keywords,
    shape = sender_sub_type,
    text = tooltip_text,
    group = matched_keywords  # ensures matched_keywords is in layer
  ), size = 2.5, alpha = 0.7, show.legend = TRUE) +
  scale_shape_manual(values = c(
    "Person" = 16,
    "Vessel" = 17,
    "Organization" = 15,
    "Location" = 18
  )) +
  facet_wrap(~ matched_keywords, ncol = 1, scales = "fixed") +
  scale_y_time(
    limits = hms::as_hms(c("08:00:00", "13:00:00")),
    breaks = hms::as_hms(c("08:00:00", "09:00:00", "10:00:00", "11:00:00", "12:00:00", "13:00:00")),
    labels = c("08:00", "09:00", "10:00", "11:00", "12:00", "13:00")
  ) +
  scale_x_date(date_breaks = "1 day", date_labels = "%d %b") +
  labs(
    title = "Interactive Timeline: Keyword Mentions by Day and Time",
    x = "Date",
    y = "Time of Day",
    shape = "Sender Type",
    color = "Keyword"
  ) +
  theme_grey() +
  theme(
    axis.text.y = element_text(size = 6),
    axis.title.y = element_text(size = 7),
    axis.ticks.y = element_line(),
    axis.text.x = element_text(size = 6, angle = 45, hjust = 1),
    axis.title.x = element_text(margin = margin(t = 10), size = 7),
    panel.spacing = unit(0.5, "lines"),  # Applies to both x and y spacing
    strip.text = element_text(size = 8, face = "bold"),
    legend.position = "bottom",
    legend.text = element_text(size = 6),
    legend.title = element_blank()
  )

# -- Step 5: Convert to plotly
ggplotly(p, tooltip = "text")

saveRDS(keyword_matches_df, file = file.path("rds_objects", "keyword_matches_df.rds"))
saveRDS(plot_data, file = file.path("rds_objects", "plot_data.rds"))
saveRDS(p, file = file.path("rds_objects", "p.rds"))
saveRDS(search_target_content, file = file.path("rds_objects", "search_target_content.rds"))
saveRDS(keywords, file = file.path("rds_objects", "keywords.rds"))
saveRDS(pattern, file = file.path("rds_objects", "pattern.rds"))

# ------------------
#| code-fold: true
#| code-summary: "Show the code"
# --- Step 0: Define Event Data for Annotations ---
event_dates <- as.Date(c("2040-10-01", "2040-10-03", "2040-10-04", "2040-10-05",
                         "2040-10-06", "2040-10-08", "2040-10-09", "2040-10-11",
                         "2040-10-12", "2040-10-14"))

event_labels_full <- c(
  "The Middleman told The Boss (Nadia) about a potential loophole in Nemo Reef protection zone",
  "Tourism vessels were offering authorised access. Their violations were addressed at City Council meeting",
  "Nadia told Mako to abort Nemo Reef mission as conservation vessels were close",
  "News of Nadia assisting to get permit for Nemo Reef signed by Jensen from City Council on 4/10 spreaded",
  "V. Miesel Shipping HQ told Neptune that Mako is lead vessel",
  "Neptune told Mako they were under special marine research permit CR-7844",
  "Neptune's underwater concrete forms in fragile crates discovered",
  "37 'authorised maintenance vessels' documented despite Nemo Reef closure",
  "Crew reallocated from Remona to Neptune overseen by Captain Davis",
  "Music video production after Nemo Reef equipment transfer"
)

event_labels_short <- c(
  "Loophole", "Tourism", "Mako", "Permit\nResults", "#NR-1045",
  "#CR-7844", "Underwater\nStructure", "37 Vessels", "Shifted Crew", "MV\nProduction"
)

# --- Step 1: Calculate Daily Message Frequencies ---
freq_df <- other_communications_df %>%
  mutate(comm_date = as.Date(timestamp)) %>%
  count(comm_date, name = "message_count") %>%
  # Complete the date range to include all days, filling missing counts with 0
  complete(comm_date = seq(min(comm_date), max(comm_date), by = "day"),
           fill = list(message_count = 0))

# --- Step 2: Build Event Annotation Table ---
event_df <- tibble(
  date = event_dates,
  label = gsub("\n", "<br>", event_labels_short), # Replace \n with <br> for HTML tooltips
  tooltip_raw = str_wrap(event_labels_full, width = 40) # Wrap long text for tooltips
)

core_events <- event_df %>%
  left_join(freq_df, by = c("date" = "comm_date")) %>%
  group_by(date) %>%
  mutate(
    offset = row_number(), # For stacking multiple events on the same day
    # Adjust value for arrow positioning relative to message count
    value = as.numeric(message_count) + 0.5 * offset,
    date_jitter = date + (offset - 1) * 0.1 # Slight horizontal jitter if multiple events
  ) %>%
  ungroup() %>%
  mutate(
    # Create rich HTML tooltip content for Plotly
    tooltip = paste0(
      "📅 <b>Date:</b> ", date,
      "<br>✉️ <b>Messages:</b> ", message_count,
      "<br>📝 <b>Core Event:</b> ", tooltip_raw
    )
  )

# --- Step 3: Create the ggplot2 Plot ---
p <- ggplot(freq_df, aes(x = comm_date, y = message_count)) +
  # Line plot for message frequency
  geom_line(color = "navyblue") +
  # Segments (arrows) pointing from the line to the event annotations
  geom_segment(
    data = core_events,
    aes(x = date, xend = date, y = value - 1, yend = value + 0.5), # Adjust yend for arrow tip
    arrow = arrow(length = unit(0.15, "inches")), color = "grey40"
  ) +
  # Points for the events (these will have the interactive tooltips)
  geom_point(
    data = core_events,
    aes(x = date, y = value + 0.5, text = tooltip), # 'text' aesthetic is key for Plotly tooltips
    color = "firebrick", size = 3
  ) +
  # Text labels for the short event names
  geom_text(
    data = core_events,
    aes(x = date, y = value + 2, label = label), # Position text above the point/arrow
    color = "black", fontface = "bold", size = 2.5, vjust = 0
  ) +
  # X-axis scale for dates
  scale_x_date(date_breaks = "1 day", date_labels = "%b %d") +
  # Labels and titles
  labs(
    title = "Message Frequency Over Time",
    x = "Date", y = "Message Count"
  ) +
  # Ensure annotations are not clipped by plot limits
  coord_cartesian(clip = "off") +
  # Minimal theme for a clean look
  theme_grey(base_size = 10) +
  # Further theme adjustments
  theme(
    axis.text.x = element_text(angle = 45, hjust = 1),
    plot.title = element_text(face="bold"),
    plot.margin = margin(30, 30, 10, 10), # Adjust plot margins if needed
  )

# --- Step 4: Convert to Interactive Plotly Object ---
# The 'tooltip = "text"' argument tells Plotly to use the 'text' aesthetic
# for the hover information, which we created as 'tooltip' in core_events.
p_interactive <- ggplotly(p, tooltip = "text")

# --- Step 5: Display the Interactive Plot ---
p_interactive

saveRDS(event_df, file = file.path("rds_objects", "event_df.rds"))
saveRDS(event_labels_full, file = file.path("rds_objects", "event_labels_full.rds"))
saveRDS(p_interactive, file = file.path("rds_objects", "p_interactive.rds"))
saveRDS(core_events, file = file.path("rds_objects", "core_events.rds"))
saveRDS(p, file = file.path("rds_objects", "p.rds"))
saveRDS(event_dates, file = file.path("rds_objects", "event_dates.rds"))
saveRDS(event_labels_short, file = file.path("rds_objects", "event_labels_short.rds"))
saveRDS(freq_df, file = file.path("rds_objects", "freq_df.rds"))