Skip to content
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
db7d221
Create convertor.py
edwinjosechittilappilly Apr 23, 2025
0d8388b
Merge branch 'main' into convert_component
edwinjosechittilappilly Apr 24, 2025
52d667c
[autofix.ci] apply automated fixes
autofix-ci[bot] Apr 24, 2025
a12ef7f
Merge branch 'main' into convert_component
edwinjosechittilappilly Apr 24, 2025
4658c14
convert component
edwinjosechittilappilly Apr 24, 2025
bf30499
Merge branch 'convert_component' of https://github.com/langflow-ai/la…
edwinjosechittilappilly Apr 24, 2025
c3164f9
[autofix.ci] apply automated fixes
autofix-ci[bot] Apr 24, 2025
588fc41
add Type_conversion base class with dispatchers for performance based…
edwinjosechittilappilly Apr 29, 2025
765d40f
fix lint issues
edwinjosechittilappilly Apr 29, 2025
e65c60e
add type_convertor test
edwinjosechittilappilly Apr 29, 2025
87ef657
[autofix.ci] apply automated fixes
autofix-ci[bot] Apr 29, 2025
3dd3307
update tests
edwinjosechittilappilly Apr 29, 2025
2c0ba21
Merge branch 'convert_component' of https://github.com/langflow-ai/la…
edwinjosechittilappilly Apr 29, 2025
6ab11ff
Merge branch 'main' into convert_component
edwinjosechittilappilly Apr 29, 2025
00a4b16
Merge branch 'convert_component' of https://github.com/langflow-ai/la…
edwinjosechittilappilly Apr 29, 2025
7ff74ae
fix tests
edwinjosechittilappilly May 9, 2025
ec023e2
Merge branch 'main' into convert_component
edwinjosechittilappilly May 9, 2025
c02b3ef
Merge branch 'main' into convert_component
edwinjosechittilappilly May 12, 2025
c555554
update with auto conversion methods
edwinjosechittilappilly May 15, 2025
2c97cab
Merge branch 'main' into convert_component
edwinjosechittilappilly May 16, 2025
db08285
update function to component file
edwinjosechittilappilly May 19, 2025
b4e83cc
Merge branch 'main' into convert_component
edwinjosechittilappilly May 20, 2025
5343410
Merge branch 'main' into convert_component
edwinjosechittilappilly May 22, 2025
b779bd3
Merge branch 'main' into convert_component
edwinjosechittilappilly May 22, 2025
78b0701
Merge branch 'main' into convert_component
edwinjosechittilappilly May 29, 2025
d00eb5c
feat: enhance input validation for Data, DataFrame, and Message types
ogabrielluiz Apr 2, 2025
357d1e3
test: add unit tests for DataInput, MessageInput, and DataFrameInput …
ogabrielluiz Apr 2, 2025
f0ff9df
updated changes to use type classes
edwinjosechittilappilly May 29, 2025
13a8d70
[autofix.ci] apply automated fixes
autofix-ci[bot] May 29, 2025
08136b9
add convert logic
edwinjosechittilappilly May 29, 2025
7705c72
update to converter
edwinjosechittilappilly May 29, 2025
bd01612
update converts
edwinjosechittilappilly May 29, 2025
61370c1
Update converter.py
edwinjosechittilappilly May 29, 2025
d1a62ef
Merge branch 'main' into convert_component
edwinjosechittilappilly May 29, 2025
6e108ae
[autofix.ci] apply automated fixes
autofix-ci[bot] May 29, 2025
484faf6
revert converter.py
edwinjosechittilappilly May 30, 2025
54d66f5
Update inputs.py
edwinjosechittilappilly May 30, 2025
33f467b
Update test_inputs.py
edwinjosechittilappilly May 30, 2025
8613a67
update to logic
edwinjosechittilappilly May 30, 2025
9475ff6
Update test_type_convertor_component.py
edwinjosechittilappilly May 30, 2025
19dcfdc
update converter
edwinjosechittilappilly May 30, 2025
f7ef291
Merge branch 'main' into convert_component
edwinjosechittilappilly May 30, 2025
5953a6b
[autofix.ci] apply automated fixes
autofix-ci[bot] May 30, 2025
9d2ffae
Merge branch 'main' into convert_component
carlosrcoelho Jun 2, 2025
3903d7b
refactor: rename conversion functions for clarity
ogabrielluiz Jun 2, 2025
835b7ce
fix: add TYPE_CHECKING for conditional imports in message.py
ogabrielluiz Jun 2, 2025
821f9f0
refactor: simplify data conversion methods in Message class
ogabrielluiz Jun 2, 2025
5a2f9f6
refactor: enhance DataFrame methods for clarity and type safety
ogabrielluiz Jun 2, 2025
b7314f0
refactor: streamline Data class methods for improved clarity
ogabrielluiz Jun 2, 2025
53af8db
refactor: simplify conversion method calls by removing redundant argu…
ogabrielluiz Jun 2, 2025
84ed405
rename test file
ogabrielluiz Jun 2, 2025
dc498f6
refactor: remove obsolete test file for data conversion
ogabrielluiz Jun 2, 2025
619cd6a
refactor: add support for converting dictionary to DataFrame
ogabrielluiz Jun 2, 2025
1d8fc2c
Merge branch 'main' into convert_component
Yukiyukiyeah Jun 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file.
2 changes: 2 additions & 0 deletions src/backend/base/langflow/components/processing/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from .alter_metadata import AlterMetadataComponent
from .combine_text import CombineTextComponent
from .convertor import TypeConverterComponent
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: shouldn't it be converter?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will Do

from .create_data import CreateDataComponent
from .data_operations import DataOperationsComponent
from .extract_key import ExtractDataKeyComponent
Expand Down Expand Up @@ -38,5 +39,6 @@
"RegexExtractorComponent",
"SelectDataComponent",
"SplitTextComponent",
"TypeConverterComponent",
"UpdateDataComponent",
]
138 changes: 138 additions & 0 deletions src/backend/base/langflow/components/processing/convertor.py
Copy link
Contributor

@ogabrielluiz ogabrielluiz May 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I still think the conversion logic should be in methods. Soon we will need to use these in other places and once we move to serializers it will be troublesome to phase these out.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Earlier, the converter was at the base of processing.
Can I move to generic component utils in the langflow base?

Or should we add this to the core component logic so it can be utilised across all other components?

Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
from collections.abc import AsyncIterator, Iterator
from typing import Any

from langflow.custom import Component
from langflow.io import HandleInput, Output, TabInput
from langflow.schema import Data, DataFrame, Message
from langflow.services.database.models.message.model import MessageBase


def get_message_converter(v) -> Message:
# If v is a instance of Message, then its fine
if isinstance(v, dict):
return Message(**v)
if isinstance(v, Message):
return v
if isinstance(v, str | AsyncIterator | Iterator):
return Message(text=v)
if isinstance(v, MessageBase):
return Message(**v.model_dump())
if isinstance(v, DataFrame):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to centralize this df to text conversion logic somewhere in another PR

# Process DataFrame similar to the _safe_convert method
# Remove empty rows
processed_df = v.dropna(how="all")
# Remove empty lines in each cell
processed_df = processed_df.replace(r"^\s*$", "", regex=True)
# Replace multiple newlines with a single newline
processed_df = processed_df.replace(r"\n+", "\n", regex=True)
# Replace pipe characters to avoid markdown table issues
processed_df = processed_df.replace(r"\|", r"\\|", regex=True)
processed_df = processed_df.map(lambda x: str(x).replace("\n", "<br/>") if isinstance(x, str) else x)
# Convert to markdown and wrap in a Message
return Message(text=processed_df.to_markdown(index=False))
if isinstance(v, Data):
if v.text_key in v.data:
return Message(text=v.get_text())
return Message(text=str(v.data))
msg = f"Invalid value type {type(v)}"
raise ValueError(msg)


def get_data_converter(v: DataFrame | Data | Message | dict) -> Data:
"""Get the data conversion dispatcher."""
if isinstance(v, DataFrame):
# Convert DataFrame to a list of dictionaries and wrap in a Data object
dict_list = v.to_dict(orient="records")
return Data(data={"results": dict_list})
if isinstance(v, Message):
return Data(data=v.data)
if isinstance(v, dict):
return Data(data=v)
if not isinstance(v, Data):
msg = f"Invalid value type {type(v)} for input Expected Data."
raise ValueError(msg) # noqa: TRY004
return v


def get_dataframe_converter(v: DataFrame | Data | Message | dict) -> DataFrame:
"""Get the dataframe conversion dispatcher."""
if isinstance(v, Data):
data_dict = v.data
# If data contains only one key and the value is a list of dictionaries, convert to DataFrame
if (
len(data_dict) == 1
and isinstance(next(iter(data_dict.values())), list)
and all(isinstance(item, dict) for item in next(iter(data_dict.values())))
):
return DataFrame(data=next(iter(data_dict.values())))
return DataFrame(data=[v])
if isinstance(v, Message):
return DataFrame(data=[v])
if isinstance(v, dict):
return DataFrame(data=[v])
if not isinstance(v, DataFrame):
msg = f"Invalid value type {type(v)}. Expected DataFrame."
raise ValueError(msg) # noqa: TRY004
return v


class TypeConverterComponent(Component):
display_name = "Type Convert"
description = "Convert between different types (Message, Data, DataFrame)"
icon = "repeat"

inputs = [
HandleInput(
name="input_data",
display_name="Input",
input_types=["Message", "Data", "DataFrame"],
info="Accept Message, Data or DataFrame as input",
required=True,
),
TabInput(
name="output_type",
display_name="Output Type",
options=["Message", "Data", "DataFrame"],
info="Select the desired output data type",
real_time_refresh=True,
value="Message",
),
]

outputs = [Output(display_name="Message Output", name="message_output", method="convert_to_message")]

def update_outputs(self, frontend_node: dict, field_name: str, field_value: Any) -> dict:
"""Dynamically show only the relevant output based on the selected output type."""
if field_name == "output_type":
# Start with empty outputs
frontend_node["outputs"] = []

# Add only the selected output type
if field_value == "Message":
frontend_node["outputs"].append(
Output(display_name="Message Output", name="message_output", method="convert_to_message").to_dict()
)
elif field_value == "Data":
frontend_node["outputs"].append(
Output(display_name="Data Output", name="data_output", method="convert_to_data").to_dict()
)
elif field_value == "DataFrame":
frontend_node["outputs"].append(
Output(
display_name="DataFrame Output", name="dataframe_output", method="convert_to_dataframe"
).to_dict()
)

return frontend_node

def convert_to_message(self) -> Message:
"""Convert input to Message type."""
return get_message_converter(self.input_data)

def convert_to_data(self) -> Data:
"""Convert input to Data type."""
return get_data_converter(self.input_data)

def convert_to_dataframe(self) -> DataFrame:
"""Convert input to DataFrame type."""
return get_dataframe_converter(self.input_data)
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import pandas as pd
import pytest
from langflow.components.processing.convertor import TypeConverterComponent
from langflow.schema import Data, DataFrame, Message

from tests.base import ComponentTestBaseWithoutClient


class TestTypeConverterComponent(ComponentTestBaseWithoutClient):
@pytest.fixture
def component_class(self):
"""Return the component class to test."""
return TypeConverterComponent

@pytest.fixture
def file_names_mapping(self):
"""Return an empty list since this component doesn't have version-specific files."""
return []

# Message to other types
def test_message_to_message(self, component_class):
"""Test converting Message to Message."""
component = component_class(input_data=Message(text="Hello World"), output_type="Message")
result = component.convert_to_message()
assert isinstance(result, Message)
assert result.text == "Hello World"

def test_message_to_data(self, component_class):
"""Test converting Message to Data."""
component = component_class(input_data=Message(text="Hello"), output_type="Data")
result = component.convert_to_data()
assert isinstance(result, Data)
assert "text" in result.data
assert result.data["text"] == "Hello"

def test_message_to_dataframe(self, component_class):
"""Test converting Message to DataFrame."""
component = component_class(input_data=Message(text="Hello"), output_type="DataFrame")
result = component.convert_to_dataframe()
assert isinstance(result, DataFrame)
assert "text" in result.columns
assert result.iloc[0]["text"] == "Hello"

# Data to other types
def test_data_to_message(self, component_class):
"""Test converting Data to Message."""
component = component_class(input_data=Data(data={"text": "Hello World"}), output_type="Message")
result = component.convert_to_message()
assert isinstance(result, Message)
assert result.text == "Hello World"

def test_data_to_data(self, component_class):
"""Test converting Data to Data."""
component = component_class(input_data=Data(data={"key": "value"}), output_type="Data")
result = component.convert_to_data()
assert isinstance(result, Data)
assert result.data == {"key": "value"}

def test_data_to_dataframe(self, component_class):
"""Test converting Data to DataFrame."""
component = component_class(input_data=Data(data={"text": "Hello World"}), output_type="DataFrame")
result = component.convert_to_dataframe()
assert isinstance(result, DataFrame)
assert "text" in result.columns
assert result.iloc[0]["text"] == "Hello World"

# DataFrame to other types
def test_dataframe_to_message(self, component_class):
"""Test converting DataFrame to Message."""
df_data = pd.DataFrame({"col1": ["Hello"], "col2": ["World"]})
component = component_class(input_data=DataFrame(data=df_data), output_type="Message")
result = component.convert_to_message()
assert isinstance(result, Message)
assert result.text == "| col1 | col2 |\n|:-------|:-------|\n| Hello | World |"

def test_dataframe_to_data(self, component_class):
"""Test converting DataFrame to Data."""
df_data = pd.DataFrame({"col1": ["Hello"]})
component = component_class(input_data=DataFrame(data=df_data), output_type="Data")
result = component.convert_to_data()
assert isinstance(result, Data)
assert isinstance(result.data, dict)

def test_dataframe_to_dataframe(self, component_class):
"""Test converting DataFrame to DataFrame."""
df_data = pd.DataFrame({"col1": ["Hello"], "col2": ["World"]})
component = component_class(input_data=DataFrame(data=df_data), output_type="DataFrame")
result = component.convert_to_dataframe()
assert isinstance(result, DataFrame)
assert "col1" in result.columns
assert "col2" in result.columns
assert result.iloc[0]["col1"] == "Hello"
assert result.iloc[0]["col2"] == "World"

def test_update_outputs(self, component_class):
"""Test the update_outputs method."""
component = component_class(input_data=Message(text="Hello"), output_type="Message")
frontend_node = {"outputs": []}

# Test with Message output
updated = component.update_outputs(frontend_node, "output_type", "Message")
assert len(updated["outputs"]) == 1
assert updated["outputs"][0]["name"] == "message_output"

# Test with Data output
updated = component.update_outputs(frontend_node, "output_type", "Data")
assert len(updated["outputs"]) == 1
assert updated["outputs"][0]["name"] == "data_output"

# Test with DataFrame output
updated = component.update_outputs(frontend_node, "output_type", "DataFrame")
assert len(updated["outputs"]) == 1
assert updated["outputs"][0]["name"] == "dataframe_output"
Loading