Keep the link content of the image. Remove some dirty data

Author: chenghao.zhang

edgar/company_reports.py

@@ -259,10 +259,31 @@ def management_discussion(self):
     @property
     def directors_officers_and_governance(self):
         return self['Item 10']
+    
+    @property
+    @lru_cache(maxsize=1)
+    def chunked_document(self):
+        return ChunkedDocument(self._filing.html(), prefix_src=self._filing.base_dir)
 
     def __str__(self):
         return f"""TenK('{self.company}')"""
 
+    def __getitem__(self, item_or_part: str):
+        # Show the item or part from the filing document. e.g. Item 1 Business from 10-K or Part I from 10-Q
+        item_text = self.chunked_document[item_or_part]
+        if item_text:
+            item_text = item_text.rstrip()
+            last_line = item_text.split("\n")[-1]
+            if re.match(r'^\b(PART\s+[IVXLC]+)\b', last_line):
+                item_text = item_text.rstrip(last_line)
+        return item_text
+
+    def get_item_with_part(self, part: str, item: str):
+        # Show the item or part from the filing document. e.g. Item 1 Business from 10-K or Part I from 10-Q
+        item_text = self.chunked_document.get_item_with_part(part, item)
+        # remove first line or last line (redundant part information)
+        return item_text
+    
     def get_structure(self):
         # Create the main tree
         tree = Tree("📄 ")
@@ -397,6 +418,11 @@ def get_item_with_part(self, part: str, item: str):
         # remove first line or last line (redundant part information)
         return item_text
 
+    @property
+    @lru_cache(maxsize=1)
+    def chunked_document(self):
+        return ChunkedDocument(self._filing.html(), prefix_src=self._filing.base_dir)
+    
     def get_structure(self):
         # Create the main tree
         tree = Tree("📄 ")

edgar/files/html_documents.py

@@ -242,6 +242,30 @@ def __str__(self):
     def __repr__(self):
         return self.text
 
+class LinkBlock(Block):
+    
+    def __init__(self, text: str, tag:str, alt:str, src:str, **tags):
+        super().__init__(text, **tags)
+        self.tag = tag
+        self.alt = alt
+        self.src = src
+        self.inline: bool = True
+
+    def get_text(self):    
+        return f'<{self.tag} alt="{self.alt}" src="{self.src}">'
+        # return ''
+
+    def to_markdown(self, prefix_src:str=""):
+        return f"![alt  {self.alt}]({prefix_src}/{self.src})\n"
+    
+    def get_complete_text(self, prefix_src:str):
+        return f'<{self.tag} alt="{self.alt}" src="{prefix_src}/{self.src}">\n'
+
+    def __str__(self):
+        return "LinkBlock"
+
+    def __repr__(self):
+        return self.text
 
 class TextBlock(Block):
 
@@ -487,8 +511,8 @@ def generate_chunks(self, ignore_tables: bool = False) -> List[List[Block]]:
                 is_item_header = bool(re.match(item_pattern, block.text))
                 is_part_header = bool(re.match(part_pattern, block.text))
 
-                if is_item_header:
-                    # Yield the current chunk before starting a new one with the "Item" header
+                if is_part_header:
+                     # Yield the current chunk before starting a new one with the "Part" header
                     if current_chunk:
                         if any(block.text.strip() for block in current_chunk):  # Avoid emitting empty chunks
                             yield current_chunk
@@ -497,11 +521,11 @@ def generate_chunks(self, ignore_tables: bool = False) -> List[List[Block]]:
                     current_chunk = [block]
 
                     # Update flags accordingly
-                    item_header_detected = True
+                    item_header_detected = False
                     header_detected = True  # "Item" headers are considered regular headers for flag purposes
                     accumulating_regular_text = False  # Reset since we're starting a new section
-                elif is_part_header:
-                     # Yield the current chunk before starting a new one with the "Item" header
+                elif is_item_header:
+                    # Yield the current chunk before starting a new one with the "Item" header
                     if current_chunk:
                         if any(block.text.strip() for block in current_chunk):  # Avoid emitting empty chunks
                             yield current_chunk
@@ -535,6 +559,11 @@ def generate_chunks(self, ignore_tables: bool = False) -> List[List[Block]]:
                         header_detected = False
                         item_header_detected = False
                     current_chunk.append(block)
+            elif isinstance(block, LinkBlock):
+                analysis = False
+                is_regular_text = False
+                is_item_header = False
+                yield [block]
 
             # Check to yield the remaining chunk if it's the last block
             if i == len(self.blocks) - 1 and current_chunk:
@@ -547,12 +576,20 @@ def extract_and_format_content(element) -> List[Block]:
     Recursively extract and format content from an element,
     applying special formatting to tables and concatenating text for other elements.
     """
-
     if element.name == 'table':
         table_block = TableBlock(table_element=element, rows=len(element.find_all("tr")))
         return [table_block]
     elif element.name in ['ul', 'ol']:
         return [TextBlock(text=fixup(element.text), element=element.name, text_type='list')]
+    elif element.name in ["img", ]:
+        return [
+                LinkBlock(text=str(element),
+                          tag=element.name,
+                          element=element.name, 
+                          alt=element.get('alt'),
+                          src=element.get('src'),
+                          text_type='string')
+                ]
     else:
         inline = is_inline(element)
         blocks: List[Block] = []

edgar/files/htmltools.py

@@ -15,7 +15,7 @@
 from edgar.core import pandas_version
 
 from edgar.datatools import compress_dataframe
-from edgar.files.html_documents import HtmlDocument, Block, TableBlock, table_to_markdown
+from edgar.files.html_documents import HtmlDocument, Block, TableBlock, LinkBlock, table_to_markdown
 from edgar.richtools import repr_rich
 
 __all__ = [
@@ -285,7 +285,6 @@ def chunks2df(chunks: List[List[Block]],
     chunk_df.loc[chunk_df.Toc.notnull() & chunk_df.Toc, 'Item'] = ""
     # if item_adjuster:
     # chunk_df = item_adjuster(chunk_df, **{'item_structure': item_structure, 'item_detector': item_detector})
-
     # Foward fill item and parts
     # Handle deprecation warning in fillna(method='ffill')
     if pandas_version >= (2, 1, 0):
@@ -330,14 +329,17 @@ class ChunkedDocument:
 
     def __init__(self,
                  html: str,
-                 chunk_fn: Callable[[List], pd.DataFrame] = chunks2df):
+                 chunk_fn: Callable[[List], pd.DataFrame] = chunks2df,
+                 prefix_src: str = ""):
         """
         :param html: The filing html
         :param chunk_fn: A function that converts the chunks to a dataframe
+        :param file_path: The path to the filing
         """
         self.chunks = chunk(html)
         self._chunked_data = chunk_fn(self.chunks)
         self.chunk_fn = chunk_fn
+        self.prefix_src = prefix_src
 
     @lru_cache(maxsize=4)
     def as_dataframe(self):
@@ -445,21 +447,42 @@ def tables(self):
             for block in chunk:
                 if isinstance(block, TableBlock):
                     yield block
+    
+    def assemble_block_text(self, chunks: List[Block]):
+
+        if self.prefix_src:
+            for chunk in chunks:
+                for block in chunk:
+                    if isinstance(block, LinkBlock):
+                        yield block.to_markdown(prefix_src=self.prefix_src)
+                    else:
+                        yield block.get_text()
+        else:
+            for chunk in chunks:
+                yield "".join([block.get_text() for block in chunk])
 
     def get_item_with_part(self, part: str, item: str):
         if isinstance(part, str):
             chunks = list(self._chunks_mul_for(part, item))
-            return "".join(["".join(block.get_text()
-                                for block in chunk)
-                        for chunk in chunks])
+            return self.clean_part_line("".join([text for text in self.assemble_block_text(chunks)]))
         return ""
 
+    @staticmethod
+    def clean_part_line(text:str):
+        # clean last line dity data, example 'PART I — FINANCIAL INFORMATION'
+        res = text.rstrip("\n")
+        last_line = res.split("\n")[-1]
+        if re.match(r'^\b(PART\s+[IVXLC]+)\b', last_line):
+            res = res.rstrip(last_line)
+        return res
+    
     def get_signature(self):
         res = self.get_item_with_part("Signature", "Signature")
         last_line = res.split("\n")[-1]
         if re.match(r'^\b(PART\s+[IVXLC]+)\b', last_line):
             res = res.rstrip(last_line)
-        return res
+        return self.clean_part_line(res)
+
 
     def get_introduction(self):
         """
@@ -487,16 +510,12 @@ def get_introduction(self):
             return ""
 
         # Reuse __getitem__ to extract chunks up to min_index
-        res = "".join([
-                "".join(block.get_text() for block in self.chunks[idx])
-                for idx in range(intro_index)
-            ])
-        # clean last line dity data, example 'PART I — FINANCIAL INFORMATION'
-        res = res.rstrip("\n")
-        last_line = res.split("\n")[-1]
-        if re.match(r'^\b(PART\s+[IVXLC]+)\b', last_line):
-            res = res.rstrip(last_line)
-        return res
+        res = "".join(
+            [text for text in
+                self.assemble_block_text(
+                    [self.chunks[idx] for idx in range(intro_index)]
+            )])
+        return self.clean_part_line(res)
 
     def __len__(self):
         return len(self.chunks)
@@ -511,9 +530,7 @@ def __getitem__(self, item):
         if len(chunks) == 0:
             return None
         # render the nested List of List [str]
-        return "".join(["".join(block.get_text()
-                                for block in chunk)
-                        for chunk in chunks])
+        return "".join([text for text in self.assemble_block_text(chunks)])
 
     def __iter__(self):
         return iter(self.chunks)