added Safe_Str__Http__Text

Author: DinisCruz

osbot_utils/helpers/safe_str/http/Safe_Str__Http__ContentType.py renamed to osbot_utils/helpers/safe_str/http/Safe_Str__Http__Content_Type.py

@@ -4,7 +4,7 @@
 TYPE_SAFE_STR__HTTP__CONTENT_TYPE__REGEX = re.compile(r'[^a-zA-Z0-9/\-+.;= ]')
 TYPE_SAFE_STR__HTTP__CONTENT_TYPE__MAX_LENGTH = 256
 
-class Safe_Str__Http__ContentType(Safe_Str):
+class Safe_Str__Http__Content_Type(Safe_Str):
     regex                      = TYPE_SAFE_STR__HTTP__CONTENT_TYPE__REGEX
     max_length                 = TYPE_SAFE_STR__HTTP__CONTENT_TYPE__MAX_LENGTH
     allow_empty                = False

osbot_utils/helpers/safe_str/http/Safe_Str__Http__LastModified.py renamed to osbot_utils/helpers/safe_str/http/Safe_Str__Http__Last_Modified.py

@@ -4,7 +4,7 @@
 TYPE_SAFE_STR__HTTP__LAST_MODIFIED__REGEX = re.compile(r'[^a-zA-Z0-9:, -]')
 TYPE_SAFE_STR__HTTP__LAST_MODIFIED__MAX_LENGTH = 64
 
-class Safe_Str__Http__LastModified(Safe_Str):
+class Safe_Str__Http__Last_Modified(Safe_Str):
     regex                      = TYPE_SAFE_STR__HTTP__LAST_MODIFIED__REGEX
     max_length                 = TYPE_SAFE_STR__HTTP__LAST_MODIFIED__MAX_LENGTH
     allow_empty                = False

osbot_utils/helpers/safe_str/http/Safe_Str__Http__Text.py

@@ -0,0 +1,35 @@
+import re
+from osbot_utils.helpers.safe_str.Safe_Str import Safe_Str
+
+
+TYPE_SAFE_STR__TEXT__MAX_LENGTH = 1048576           # Define the size constant - 1 megabyte in bytes
+
+# A more permissive regex that primarily filters out:
+# - NULL byte (U+0000)
+# - Control characters (U+0001 to U+0008, U+000B to U+000C, U+000E to U+001F)
+# - Some potentially problematic characters in various contexts
+# But allows:
+# - All standard printable ASCII characters
+# - Tab (U+0009), Line Feed (U+000A), and Carriage Return (U+000D)
+# - A wide range of punctuation, symbols, and Unicode characters for international text
+
+TYPE_SAFE_STR__HTTP__TEXT__REGEX = re.compile(r'[\x00\x01-\x08\x0B\x0C\x0E-\x1F\x7F]')
+
+class Safe_Str__Http__Text(Safe_Str):
+    """
+    Safe string class for general text content with a 1MB limit.
+    Allows a wide range of characters suitable for natural language text,
+    including international characters, while filtering out control characters
+    and other potentially problematic sequences.
+    """
+    max_length                  = TYPE_SAFE_STR__TEXT__MAX_LENGTH
+    regex                       = TYPE_SAFE_STR__HTTP__TEXT__REGEX
+    trim_whitespace             = True              # Trim leading/trailing whitespace
+    normalize_newlines          = True              # Option to normalize different newline styles
+
+    def __new__(cls, value=None):
+
+        if cls.normalize_newlines and value is not None and isinstance(value, str):         # Handle newline normalization before passing to parent class
+            value = value.replace('\r\n', '\n').replace('\r', '\n')                         # Normalize different newline styles to \n
+
+        return super().__new__(cls, value)                                                  # Now call the parent implementation

tests/unit/helpers/safe_str/html/test_Safe_Str__Http__ContentType.py

@@ -1,58 +1,57 @@
 import pytest
-from unittest import TestCase
+from unittest                                                       import TestCase
+from osbot_utils.helpers.safe_str.http.Safe_Str__Http__Content_Type import Safe_Str__Http__Content_Type
 
-from osbot_utils.helpers.safe_str.http.Safe_Str__Http__ContentType import Safe_Str__Http__ContentType
 
-
-class test_Safe_Str__Http__ContentType(TestCase):
+class test_Safe_Str__Http__Content_Type(TestCase):
 
     def test_Safe_Str__Http__ContentType_class(self):
         # Standard MIME types
-        assert Safe_Str__Http__ContentType('text/html'       ) == 'text/html'
-        assert Safe_Str__Http__ContentType('application/json') == 'application/json'
-        assert Safe_Str__Http__ContentType('image/jpeg'      ) == 'image/jpeg'
-        assert Safe_Str__Http__ContentType('audio/mpeg'      ) == 'audio/mpeg'
-        assert Safe_Str__Http__ContentType('video/mp4'       ) == 'video/mp4'
+        assert Safe_Str__Http__Content_Type('text/html') == 'text/html'
+        assert Safe_Str__Http__Content_Type('application/json') == 'application/json'
+        assert Safe_Str__Http__Content_Type('image/jpeg') == 'image/jpeg'
+        assert Safe_Str__Http__Content_Type('audio/mpeg') == 'audio/mpeg'
+        assert Safe_Str__Http__Content_Type('video/mp4') == 'video/mp4'
 
         # With parameters
-        assert Safe_Str__Http__ContentType('text/html; charset=utf-8'       ) == 'text/html; charset=utf-8'
-        assert Safe_Str__Http__ContentType('application/json; charset=utf-8') == 'application/json; charset=utf-8'
-        assert Safe_Str__Http__ContentType('text/plain; charset=iso-8859-1' ) == 'text/plain; charset=iso-8859-1'
+        assert Safe_Str__Http__Content_Type('text/html; charset=utf-8') == 'text/html; charset=utf-8'
+        assert Safe_Str__Http__Content_Type('application/json; charset=utf-8') == 'application/json; charset=utf-8'
+        assert Safe_Str__Http__Content_Type('text/plain; charset=iso-8859-1') == 'text/plain; charset=iso-8859-1'
 
         # Complex content types
-        assert Safe_Str__Http__ContentType('application/vnd.api+json'       ) == 'application/vnd.api+json'
-        assert Safe_Str__Http__ContentType('application/ld+json'            ) == 'application/ld+json'
-        assert Safe_Str__Http__ContentType('application/vnd.ms-excel'       ) == 'application/vnd.ms-excel'
+        assert Safe_Str__Http__Content_Type('application/vnd.api+json') == 'application/vnd.api+json'
+        assert Safe_Str__Http__Content_Type('application/ld+json') == 'application/ld+json'
+        assert Safe_Str__Http__Content_Type('application/vnd.ms-excel') == 'application/vnd.ms-excel'
 
         # Whitespace handling (trim_whitespace = True)
-        assert Safe_Str__Http__ContentType('  text/html  '                    ) == 'text/html'
-        assert Safe_Str__Http__ContentType('application/json; charset=utf-8  ') == 'application/json; charset=utf-8'
+        assert Safe_Str__Http__Content_Type('  text/html  ') == 'text/html'
+        assert Safe_Str__Http__Content_Type('application/json; charset=utf-8  ') == 'application/json; charset=utf-8'
 
         # Numeric conversion
-        assert Safe_Str__Http__ContentType(12345) == '12345'
+        assert Safe_Str__Http__Content_Type(12345) == '12345'
 
         # Invalid characters get replaced
-        assert Safe_Str__Http__ContentType('text/html<script>'                ) == 'text/html_script_'
-        assert Safe_Str__Http__ContentType('text/html:invalid'                ) == 'text/html_invalid'
-        assert Safe_Str__Http__ContentType('text@html'                        ) == 'text_html'
+        assert Safe_Str__Http__Content_Type('text/html<script>') == 'text/html_script_'
+        assert Safe_Str__Http__Content_Type('text/html:invalid') == 'text/html_invalid'
+        assert Safe_Str__Http__Content_Type('text@html') == 'text_html'
 
         # Edge cases and exceptions
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__ContentType(None)
+            Safe_Str__Http__Content_Type(None)
         assert "Value cannot be None when allow_empty is False" in str(exc_info.value)
 
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__ContentType('')
+            Safe_Str__Http__Content_Type('')
         assert "Value cannot be empty when allow_empty is False" in str(exc_info.value)
 
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__ContentType('<?&*^?>')  # All invalid chars
+            Safe_Str__Http__Content_Type('<?&*^?>')  # All invalid chars
         assert "Sanitized value consists entirely of '_' characters" in str(exc_info.value)
 
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__ContentType('   ')  # Spaces only (will be trimmed)
+            Safe_Str__Http__Content_Type('   ')  # Spaces only (will be trimmed)
         assert "Value cannot be empty when allow_empty is False" in str(exc_info.value)
 
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__ContentType('a' * 257)  # Exceeds max length
+            Safe_Str__Http__Content_Type('a' * 257)  # Exceeds max length
         assert "Value exceeds maximum length of 256" in str(exc_info.value)

tests/unit/helpers/safe_str/html/test_Safe_Str__Http__LastModified.py

@@ -1,48 +1,48 @@
 import pytest
-from unittest                                                       import TestCase
-from osbot_utils.helpers.safe_str.http.Safe_Str__Http__LastModified import Safe_Str__Http__LastModified
+from unittest                                                        import TestCase
+from osbot_utils.helpers.safe_str.http.Safe_Str__Http__Last_Modified import Safe_Str__Http__Last_Modified
 
 
-class test_Safe_Str__Http__LastModified(TestCase):
+class test_Safe_Str__Http__Last_Modified(TestCase):
 
     def test_Safe_Str__Http__LastModified_class(self):
         # Standard RFC formats
-        assert Safe_Str__Http__LastModified('Wed, 21 Oct 2023 07:28:00 GMT') == 'Wed, 21 Oct 2023 07:28:00 GMT'
-        assert Safe_Str__Http__LastModified('Mon, 15 May 2024 12:30:45 GMT') == 'Mon, 15 May 2024 12:30:45 GMT'
-        assert Safe_Str__Http__LastModified('Sat, 01 Jan 2022 00:00:00 GMT') == 'Sat, 01 Jan 2022 00:00:00 GMT'
+        assert Safe_Str__Http__Last_Modified('Wed, 21 Oct 2023 07:28:00 GMT') == 'Wed, 21 Oct 2023 07:28:00 GMT'
+        assert Safe_Str__Http__Last_Modified('Mon, 15 May 2024 12:30:45 GMT') == 'Mon, 15 May 2024 12:30:45 GMT'
+        assert Safe_Str__Http__Last_Modified('Sat, 01 Jan 2022 00:00:00 GMT') == 'Sat, 01 Jan 2022 00:00:00 GMT'
 
         # Different date formats that might be used
-        assert Safe_Str__Http__LastModified('2023-10-21T07:28:00Z') == '2023-10-21T07:28:00Z'
-        assert Safe_Str__Http__LastModified('21 Oct 2023 07:28:00 GMT') == '21 Oct 2023 07:28:00 GMT'
+        assert Safe_Str__Http__Last_Modified('2023-10-21T07:28:00Z') == '2023-10-21T07:28:00Z'
+        assert Safe_Str__Http__Last_Modified('21 Oct 2023 07:28:00 GMT') == '21 Oct 2023 07:28:00 GMT'
 
         # Whitespace handling (trim_whitespace = True)
-        assert Safe_Str__Http__LastModified('  Wed, 21 Oct 2023 07:28:00 GMT  ') == 'Wed, 21 Oct 2023 07:28:00 GMT'
+        assert Safe_Str__Http__Last_Modified('  Wed, 21 Oct 2023 07:28:00 GMT  ') == 'Wed, 21 Oct 2023 07:28:00 GMT'
 
         # Invalid characters get replaced
-        assert Safe_Str__Http__LastModified('Wed, 21 Oct 2023<script>') == 'Wed, 21 Oct 2023_script_'
-        assert Safe_Str__Http__LastModified('Wed; 21 Oct 2023') == 'Wed_ 21 Oct 2023'
-        assert Safe_Str__Http__LastModified('Wed, 21/Oct/2023') == 'Wed, 21_Oct_2023'
+        assert Safe_Str__Http__Last_Modified('Wed, 21 Oct 2023<script>') == 'Wed, 21 Oct 2023_script_'
+        assert Safe_Str__Http__Last_Modified('Wed; 21 Oct 2023') == 'Wed_ 21 Oct 2023'
+        assert Safe_Str__Http__Last_Modified('Wed, 21/Oct/2023') == 'Wed, 21_Oct_2023'
 
         # Numeric conversion
-        assert Safe_Str__Http__LastModified(20231021) == '20231021'
+        assert Safe_Str__Http__Last_Modified(20231021) == '20231021'
 
         # Edge cases and exceptions
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__LastModified(None)
+            Safe_Str__Http__Last_Modified(None)
         assert "Value cannot be None when allow_empty is False" in str(exc_info.value)
 
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__LastModified('')
+            Safe_Str__Http__Last_Modified('')
         assert "Value cannot be empty when allow_empty is False" in str(exc_info.value)
 
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__LastModified('<?&*^?>')  # All invalid chars
+            Safe_Str__Http__Last_Modified('<?&*^?>')  # All invalid chars
         assert "Sanitized value consists entirely of '_' characters" in str(exc_info.value)
 
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__LastModified('   ')  # Spaces only (will be trimmed)
+            Safe_Str__Http__Last_Modified('   ')  # Spaces only (will be trimmed)
         assert "Value cannot be empty when allow_empty is False" in str(exc_info.value)
 
         with pytest.raises(ValueError) as exc_info:
-            Safe_Str__Http__LastModified('a' * 65)  # Exceeds max length
+            Safe_Str__Http__Last_Modified('a' * 65)  # Exceeds max length
         assert "Value exceeds maximum length of 64" in str(exc_info.value)

tests/unit/helpers/safe_str/html/test_Safe_Str__Http__Text.py

@@ -0,0 +1,115 @@
+import pytest
+from unittest                                               import TestCase
+from osbot_utils.helpers.safe_str.http.Safe_Str__Http__Text import Safe_Str__Http__Text, \
+    TYPE_SAFE_STR__HTTP__TEXT__REGEX, TYPE_SAFE_STR__TEXT__MAX_LENGTH
+from osbot_utils.utils.Str import trim
+
+
+class test_Safe_Str__Http__Text(TestCase):
+
+    def test_Safe_Str__Http__Text_basic(self):
+        # Basic text with various allowed characters
+        assert str(Safe_Str__Http__Text("Hello, world!"         )) == "Hello, world!"
+        assert str(Safe_Str__Http__Text("This is a test. 123"   )) == "This is a test. 123"
+        assert str(Safe_Str__Http__Text("Line 1\nLine 2"        )) == "Line 1\nLine 2"
+
+        # Text with various punctuation and special characters
+        assert str(Safe_Str__Http__Text("Symbols: !@#$%^&*()_+-=[]{}|;':\",./<>?")) == "Symbols: !@#$%^&*()_+-=[]{}|;':\",./<>?"
+        assert str(Safe_Str__Http__Text("Math: 5 + 5 = 10, 10 * 10 = 100")) == "Math: 5 + 5 = 10, 10 * 10 = 100"
+        assert str(Safe_Str__Http__Text("Currency: $100, €50, £75, ¥500")) == "Currency: $100, €50, £75, ¥500"
+
+        # Whitespace handling (trim_whitespace = True)
+        assert str(Safe_Str__Http__Text("  Hello  ")) == "Hello"
+        assert str(Safe_Str__Http__Text("\tTabbed text\t")) == "Tabbed text"
+        assert str(Safe_Str__Http__Text("\nText with newlines\n")) == "Text with newlines"
+
+        # Newline normalization (normalize_newlines = True)
+        assert str(Safe_Str__Http__Text("Windows\r\nLine\r\nBreaks" )) == "Windows\nLine\nBreaks"
+        assert str(Safe_Str__Http__Text("Mac\rLine\rBreaks"         )) == "Mac\nLine\nBreaks"
+        assert str(Safe_Str__Http__Text("Mixed\nLine\r\nBreaks\r"   )) == "Mixed\nLine\nBreaks"
+
+        # Empty and None handling (allow_empty = True)
+        assert str(Safe_Str__Http__Text("")) == ""
+        assert str(Safe_Str__Http__Text(None)) == ""
+
+        # Unicode text
+        assert str(Safe_Str__Http__Text("Unicode: ☺ ♥ ★ ☆ ☂ ☃ ♫ ♪")) == "Unicode: ☺ ♥ ★ ☆ ☂ ☃ ♫ ♪"
+        assert str(Safe_Str__Http__Text("Languages: English, Español, Français, Deutsch, 日本語, 中文, Русский")) == "Languages: English, Español, Français, Deutsch, 日本語, 中文, Русский"
+
+    def test_Safe_Str__Http__Text_control_chars(self):
+        # Text with control characters (should be filtered out)
+        input_with_control = "Text with control chars: \x00\x01\x02\x03"
+        expected = "Text with control chars: ____"
+        assert str(Safe_Str__Http__Text(input_with_control)) == expected
+
+        # Text with other problematic sequences
+        input_with_escape = "Text with escape sequences: \x1B[31mRed\x1B[0m" # ANSI color codes
+        expected = "Text with escape sequences: _[31mRed_[0m"
+        assert str(Safe_Str__Http__Text(input_with_escape)) == expected
+
+    def test_Safe_Str__Http__Text_length_limits(self):
+        # Text at the limit
+        text_at_limit = "a" * TYPE_SAFE_STR__TEXT__MAX_LENGTH
+        assert len(str(Safe_Str__Http__Text(text_at_limit))) == TYPE_SAFE_STR__TEXT__MAX_LENGTH
+
+        # Text exceeding the limit
+        with pytest.raises(ValueError) as exc_info:
+            Safe_Str__Http__Text("a" * (TYPE_SAFE_STR__TEXT__MAX_LENGTH + 1))
+        assert f"Value exceeds maximum length of {TYPE_SAFE_STR__TEXT__MAX_LENGTH}" in str(exc_info.value)
+
+    def test_Safe_Str__Http__Text_code_snippets(self):
+        # Python code snippet
+        python_code = """def hello_world():
+    print("Hello, world!")
+    return True
+"""
+        assert str(Safe_Str__Http__Text(python_code)) == trim(python_code)
+
+        # JavaScript code snippet
+        js_code = """function helloWorld() {
+    console.log("Hello, world!");
+    return true;
+}
+"""
+        assert str(Safe_Str__Http__Text(js_code)) == trim(js_code)
+
+        # HTML snippet
+        html_snippet = """<div class="container">
+    <h1>Hello, world!</h1>
+    <p>This is a paragraph.</p>
+</div>
+"""
+        assert str(Safe_Str__Http__Text(html_snippet)) == trim(html_snippet)
+
+        # SQL snippet
+        sql_snippet = """SELECT id, name, email 
+FROM users 
+WHERE status = 'active' 
+ORDER BY name ASC;
+"""
+        assert str(Safe_Str__Http__Text(sql_snippet)) == trim(sql_snippet)
+
+    def test_Safe_Str__Http__Text_multiline(self):
+        # Multiline text with various formatting
+        multiline = """# Title
+        
+## Subtitle
+
+This is a paragraph with *emphasis* and **strong** text.
+
+- List item 1
+- List item 2
+- List item 3
+
+1. Numbered item 1
+2. Numbered item 2
+3. Numbered item 3
+
+> This is a blockquote.
+
+```python
+def hello():
+    print("Hello, world!")
+```    
+"""
+        assert str(Safe_Str__Http__Text(multiline)) == multiline.strip()  # because trim_whitespace=True