Resolved remaining bandit results

travis-bauer · travis-bauer · commit 3efea3f8a13a · 2025-09-04T20:58:45.000-06:00
diff --git a/src/talkpipe/app/chatterlang_serve.py b/src/talkpipe/app/chatterlang_serve.py
@@ -7,6 +7,7 @@
 import logging
 import argparse
 import yaml
+import html
 import asyncio
 from fastapi.responses import FileResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
@@ -638,11 +639,10 @@ def _get_stream_interface(self) -> str:
         
         form_fields = self._generate_form_fields()
         
-        return (  # nosec B608
-            f'''<!DOCTYPE html>
+        return f'''<!DOCTYPE html> 
         <html>
         <head>
-            <title>{self.title} - Stream</title>
+            <title>{html.escape(self.title)} - Stream</title>
             <style>
                 * {{
                     margin: 0;
@@ -984,7 +984,7 @@ def _get_stream_interface(self) -> str:
         </head>
         <body>
             <div class="header">
-                <h1>{self.form_config.title}</h1>
+                <h1>{html.escape(self.form_config.title)}</h1>
             </div>
             
             <div class="main-container">
@@ -1170,7 +1170,7 @@ def _get_stream_interface(self) -> str:
                     }}
                     
                     // Add user message to chat immediately for instant feedback
-                    const displayProperty = '{self.display_property}' || Object.keys(data)[0];
+                    const displayProperty = '{html.escape(str(self.display_property))}' || Object.keys(data)[0];
                     const userMessage = data[displayProperty] || JSON.stringify(data);
                     lastUserMessage = userMessage; // Store to detect duplicates from server
                     addMessage(userMessage, 'user', new Date().toISOString());
@@ -1228,8 +1228,7 @@ def _get_stream_interface(self) -> str:
             </script>
         </body>
         </html>
-        '''
-        )
+        '''  # nosec B608 
     
     def _get_html_interface(self) -> str:
         """Generate HTML interface with configurable form"""
@@ -1272,11 +1271,11 @@ def _get_html_interface(self) -> str:
         
         form_fields = self._generate_form_fields()
         
-        return (  # nosec B608
-            f'''<!DOCTYPE html>
+        # nosec B608 - HTML template with proper escaping, not SQL injection
+        return f'''<!DOCTYPE html>
         <html>
         <head>
-            <title>{self.title}</title>
+            <title>{html.escape(self.title)}</title>
             <style>
                 * {{
                     margin: 0;
@@ -1293,7 +1292,7 @@ def _get_html_interface(self) -> str:
                 }}
                 
                 .main-content {{
-                    height: calc(100vh - {height});
+                    height: calc(100vh - {html.escape(height)});
                     padding: 20px;
                     overflow-y: auto;
                     background-color: #f8f9fa;
@@ -1336,9 +1335,9 @@ def _get_html_interface(self) -> str:
                 
                 .form-panel {{
                     position: fixed;
-                    {form_style}
-                    background-color: {input_bg};
-                    border: 1px solid {border_color};
+                    {html.escape(form_style)}
+                    background-color: {html.escape(input_bg)};
+                    border: 1px solid {html.escape(border_color)};
                     padding: 20px;
                     box-shadow: 0 -2px 10px rgba(0,0,0,0.1);
                     overflow-y: auto;
@@ -1355,7 +1354,7 @@ def _get_html_interface(self) -> str:
                 }}
                 
                 .form-header h3 {{
-                    color: {text_color};
+                    color: {html.escape(text_color)};
                     margin: 0;
                 }}
                 
@@ -1485,9 +1484,9 @@ def _get_html_interface(self) -> str:
         <body>
             <div class="main-content">
                 <div class="info-section">
-                    <h1>{self.title}</h1>
+                    <h1>{html.escape(self.title)}</h1>
                     <p>Submit JSON data using the form below or send POST requests to:</p>
-                    <div class="endpoint-info">POST http://{self.host}:{self.port}/process</div>
+                    <div class="endpoint-info">POST http://{html.escape(self.host)}:{html.escape(str(self.port))}/process</div>
                     {"<p>Authentication required: Include 'X-API-Key' header</p>" if self.require_auth else ""}
                     <p>View API documentation at: <a href="/docs">/docs</a></p>
                     <p>View streaming interface at: <a href="/stream">/stream</a></p>
@@ -1504,7 +1503,7 @@ def _get_html_interface(self) -> str:
             <div class="form-panel" id="formPanel">
                 <div class="form-container">
                     <div class="form-header">
-                        <h3>{self.form_config.title}</h3>
+                        <h3>{html.escape(self.form_config.title)}</h3>
                     </div>
                     
                     {auth_header}
@@ -1644,9 +1643,8 @@ def _get_html_interface(self) -> str:
             </script>
         </body>
         </html>
-        '''
-        )
-    
+        '''  # nosec B608
+
     def set_processor_function(self, func: Callable[[Dict[str, Any]], Any]):
         """Set the function used to process incoming JSON data"""
         self.processor_function = func
diff --git a/src/talkpipe/pipe/basic.py b/src/talkpipe/pipe/basic.py
@@ -4,7 +4,7 @@
 import logging
 import time
 import sys
-import pickle
+import json
 import hashlib
 import copy
 import pandas as pd
@@ -478,14 +478,14 @@ def transform(self, input_iter: Iterable) -> Iterator:
         """
         yield from input_iter
 
-def hash_data(data, algorithm="MD5", field_list="_", use_repr=False, fail_on_missing=True, default=None):
+def hash_data(data, algorithm="MD5", field_list="_", use_repr=True, fail_on_missing=True, default=None):
     """Hash a single data item using the specified parameters.
     
     Args:
         data: The data item to hash
         algorithm (str): Hash algorithm to use. Options include SHA1, SHA224, SHA256, SHA384, SHA512, SHA-3, and MD5.
         fields (list): List of fields to include in the hash
-        use_repr (bool): Whether to use repr() or pickle
+        use_repr (bool): Whether to use repr() or JSON serialization. Defaults to True for security.
         fail_on_missing (bool): Whether to fail on missing fields
         
     Returns:
@@ -498,28 +498,40 @@ def hash_data(data, algorithm="MD5", field_list="_", use_repr=False, fail_on_mis
             if fail_on_missing:
                 raise ValueError(f"Field {field} value was None")
             else:
-                logging.warning(f"Field {field} value was None.  Ignoring")
+                logging.warning(f"Field {field} value was None. Ignoring")
                 continue
+        
         if use_repr:
-            hasher.update(repr(item).encode())
+            # Safe: repr() doesn't execute code and works with all objects
+            hasher.update(repr(item).encode('utf-8'))
         else:
-            hasher.update(pickle.dumps(item))
+            # Safe: JSON serialization instead of pickle
+            try:
+                # Use JSON with sorted keys for deterministic hashing
+                json_str = json.dumps(item, sort_keys=True, default=str, ensure_ascii=False)
+                hasher.update(json_str.encode('utf-8'))
+            except (TypeError, ValueError) as e:
+                # Fallback to repr for non-JSON-serializable objects
+                logging.debug(f"JSON serialization failed for {type(item)}, using repr: {e}")
+                hasher.update(repr(item).encode('utf-8'))
+    
     return hasher.hexdigest()
 
 @registry.register_segment("hash")
 class Hash(AbstractSegment):
     """Hashes the input data using the specified algorithm.
 
     This segment hashes the input data using the specified algorithm.
-    Strings will be encoded and hashed.  All other datatypes wil be hashed using either pickle or repr().
+    All datatypes are hashed using either repr() or JSON serialization for security.
 
     Args:
-        algorithm (str): Hash algorithm to use.  Options include SHA1, SHA224, SHA256, SHA384, SHA512, SHA-3, and MD5.
-        use_repr (bool): If True, the repr() version of the input data is hashed.  If False, the input data is hashed via 
-            pickling.  Using repr() will handle all object, even those that can't be pickled and won't be subject to
-            changes in pickling formats.  But the pickled version will include more state and generally be more reliable.
+        algorithm (str): Hash algorithm to use. Options include SHA1, SHA224, SHA256, SHA384, SHA512, SHA-3, and MD5.
+        use_repr (bool): If True, the repr() version of the input data is hashed. If False, JSON serialization 
+            is used with fallback to repr() for non-JSON-serializable objects. Using repr() will handle all 
+            objects consistently and won't be subject to changes in serialization formats. JSON serialization 
+            provides more structured representation but may not work with all Python objects.
     """
-    def __init__(self, algorithm: str="MD5", use_repr=False, field_list: str = "_", append_as=None, fail_on_missing: bool = True):
+    def __init__(self, algorithm: str="MD5", use_repr=True, field_list: str = "_", append_as=None, fail_on_missing: bool = True):
         super().__init__()
         self.algorithm = algorithm
         self.use_repr = use_repr
diff --git a/src/talkpipe/pipe/io.py b/src/talkpipe/pipe/io.py
@@ -3,7 +3,7 @@
 from typing import Optional, Iterable, Iterator
 import logging
 import os
-import pickle
+import pickle  # nosec B403 - Used only for write operations, not loading untrusted data
 import json
 from pprint import pformat
 from prompt_toolkit import PromptSession
diff --git a/src/talkpipe/search/simplevectordb.py b/src/talkpipe/search/simplevectordb.py
