First attempt at a Node trait with Domain node using Claude Code

Author: brainless

pixlie_ai/src/engine/node.rs

@@ -18,15 +18,28 @@ use crate::entity::web::link::Link;
 use crate::entity::web::web_metadata::WebMetadata;
 use crate::entity::web::web_page::WebPage;
 use crate::error::PiResult;
-use crate::{ExternalData, FetchError, FetchResponse};
+use crate::{ExternalData, FetchError, FetchRequest, FetchResponse};
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::cmp::Ordering;
 use std::sync::Arc;
 use strum::{Display, EnumString};
 use ts_rs::TS;
 use utoipa::ToSchema;
 
+pub trait Node: Send + Sync {
+    fn process(&self, node_id: NodeId, engine: Arc<&Engine>) -> PiResult<()>;
+    fn as_any(&self) -> &dyn Any;
+    fn get_labels(&self) -> Vec<NodeLabel>;
+}
+
+pub trait Fetchable: Node {
+    fn make_fetch_requests(&self, node_id: NodeId, engine: Arc<&Engine>) -> PiResult<Vec<FetchRequest>>;
+    fn handle_fetch_response(&self, node_id: NodeId, engine: Arc<&Engine>, response: FetchResponse) -> PiResult<()>;
+    fn handle_fetch_error(&self, node_id: NodeId, engine: Arc<&Engine>, error: FetchError) -> PiResult<()>;
+}
+
 #[derive(Clone, Display, Deserialize, Serialize)]
 pub enum Payload {
     Link(Link),
@@ -112,6 +125,118 @@ pub struct NodeItem {
     pub written_at: DateTime<Utc>,
 }
 
+pub struct TypedNodeItem<T: Node> {
+    pub id: NodeId,
+    pub node: T,
+    pub flags: NodeFlags,
+    pub written_at: DateTime<Utc>,
+}
+
+impl<T: Node> TypedNodeItem<T> {
+    pub fn new(id: NodeId, node: T) -> Self {
+        Self {
+            id,
+            node,
+            flags: NodeFlags::empty(),
+            written_at: Utc::now(),
+        }
+    }
+
+    pub fn process(&self, engine: Arc<&Engine>) -> PiResult<()> {
+        self.node.process(self.id, engine)
+    }
+
+    pub fn make_fetch_requests(&self, engine: Arc<&Engine>) -> PiResult<Vec<FetchRequest>> 
+    where 
+        T: Fetchable 
+    {
+        self.node.make_fetch_requests(self.id, engine)
+    }
+
+    pub fn handle_fetch_response(&self, engine: Arc<&Engine>, response: FetchResponse) -> PiResult<()> 
+    where 
+        T: Fetchable 
+    {
+        self.node.handle_fetch_response(self.id, engine, response)
+    }
+
+    pub fn handle_fetch_error(&self, engine: Arc<&Engine>, error: FetchError) -> PiResult<()> 
+    where 
+        T: Fetchable 
+    {
+        self.node.handle_fetch_error(self.id, engine, error)
+    }
+
+    pub fn get_labels(&self) -> Vec<NodeLabel> {
+        self.node.get_labels()
+    }
+
+    pub fn get_node(&self) -> &T {
+        &self.node
+    }
+
+    pub fn get_node_mut(&mut self) -> &mut T {
+        &mut self.node
+    }
+}
+
+pub enum AnyTypedNode {
+    Domain(TypedNodeItem<crate::entity::web::domain::Domain>),
+    // Link(TypedNodeItem<crate::entity::web::link::Link>),
+    // WebPage(TypedNodeItem<crate::entity::web::web_page::WebPage>),
+    // Add more as you implement them
+}
+
+impl AnyTypedNode {
+    pub fn process(&self, engine: Arc<&Engine>) -> PiResult<()> {
+        match self {
+            AnyTypedNode::Domain(node) => node.process(engine),
+            // AnyTypedNode::Link(node) => node.process(engine),
+            // AnyTypedNode::WebPage(node) => node.process(engine),
+        }
+    }
+
+    pub fn handle_fetch_response(&self, engine: Arc<&Engine>, response: FetchResponse) -> PiResult<()> {
+        match self {
+            AnyTypedNode::Domain(node) => node.handle_fetch_response(engine, response),
+        }
+    }
+
+    pub fn handle_fetch_error(&self, engine: Arc<&Engine>, error: FetchError) -> PiResult<()> {
+        match self {
+            AnyTypedNode::Domain(node) => node.handle_fetch_error(engine, error),
+        }
+    }
+
+    pub fn get_labels(&self) -> Vec<NodeLabel> {
+        match self {
+            AnyTypedNode::Domain(node) => node.get_labels(),
+        }
+    }
+
+    pub fn get_id(&self) -> NodeId {
+        match self {
+            AnyTypedNode::Domain(node) => node.id,
+        }
+    }
+}
+
+pub fn create_typed_node(id: NodeId, labels: &[NodeLabel], payload: &Payload) -> Option<AnyTypedNode> {
+    use crate::entity::web::domain::Domain;
+    
+    if labels.contains(&NodeLabel::DomainName) {
+        if let Payload::Text(domain_name) = payload {
+            return Some(AnyTypedNode::Domain(TypedNodeItem::new(id, Domain::new(domain_name.clone()))));
+        }
+    }
+    
+    // Add more entity types here as you implement them
+    // if labels.contains(&NodeLabel::Link) { ... }
+    // if labels.contains(&NodeLabel::WebPage) { ... }
+    
+    None
+}
+
 impl NodeItem {
     pub(super) fn process(&self, arced_engine: Arc<&Engine>) -> PiResult<()> {
         if self.labels.contains(&NodeLabel::DomainName) {
@@ -213,12 +338,6 @@ impl NodeItem {
     }
 }
 
-impl NodeItem {
-    pub fn get_label(&self) -> String {
-        self.payload.to_string()
-    }
-}
-
 impl Ord for NodeItem {
     fn cmp(&self, other: &Self) -> Ordering {
         self.id.cmp(&other.id)

pixlie_ai/src/entity/web/domain.rs

@@ -1,11 +1,85 @@
-use crate::engine::node::{ArcedNodeItem, NodeId, NodeItem, NodeLabel, Payload};
+use crate::engine::node::{ArcedNodeItem, Fetchable, Node, NodeId, NodeItem, NodeLabel, Payload};
 use crate::engine::{EdgeLabel, Engine, NodeFlags};
 use crate::error::{PiError, PiResult};
-use crate::{ExternalData, FetchRequest};
+use crate::{ExternalData, FetchError, FetchRequest, FetchResponse};
 use log::error;
+use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::sync::Arc;
 
-pub struct Domain;
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Domain {
+    pub name: String,
+}
+
+impl Node for Domain {
+    fn process(&self, node_id: NodeId, engine: Arc<&Engine>) -> PiResult<()> {
+        // Initial processing logic - could trigger fetch requests
+        let fetch_requests = self.make_fetch_requests(node_id, engine.clone())?;
+        for request in fetch_requests {
+            engine.fetch(request)?;
+        }
+        Ok(())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn get_labels(&self) -> Vec<NodeLabel> {
+        vec![NodeLabel::DomainName]
+    }
+}
+
+impl Fetchable for Domain {
+    fn make_fetch_requests(&self, node_id: NodeId, engine: Arc<&Engine>) -> PiResult<Vec<FetchRequest>> {
+        // Check if we already have robots.txt
+        // For now, always request it
+        Ok(vec![FetchRequest::new(node_id, "/robots.txt")])
+    }
+
+    fn handle_fetch_response(&self, node_id: NodeId, engine: Arc<&Engine>, response: FetchResponse) -> PiResult<()> {
+        // Handle robots.txt response
+        let content_node_id = engine
+            .get_or_add_node(
+                Payload::Text(response.contents.clone()),
+                vec![NodeLabel::RobotsTxt],
+                true,
+                None,
+            )?
+            .get_node_id();
+        engine.add_connection(
+            (node_id, content_node_id),
+            (EdgeLabel::OwnerOf, EdgeLabel::BelongsTo),
+        )?;
+        engine.toggle_flag(&node_id, NodeFlags::IS_PROCESSED)?;
+        Ok(())
+    }
+
+    fn handle_fetch_error(&self, node_id: NodeId, engine: Arc<&Engine>, error: FetchError) -> PiResult<()> {
+        // Handle robots.txt fetch error - save empty robots.txt
+        let content_node_id = engine
+            .get_or_add_node(
+                Payload::Text("".to_string()),
+                vec![NodeLabel::RobotsTxt],
+                true,
+                None,
+            )?
+            .get_node_id();
+        engine.add_connection(
+            (node_id, content_node_id),
+            (EdgeLabel::OwnerOf, EdgeLabel::BelongsTo),
+        )?;
+        engine.toggle_flag(&node_id, NodeFlags::IS_PROCESSED)?;
+        Ok(())
+    }
+}
+
+impl Domain {
+    pub fn new(name: String) -> Self {
+        Self { name }
+    }
+}
 
 pub enum FindDomainOf<'a> {
     DomainName(&'a str),