Skip to content

[bug] OutOfMemoryError bug in the workflow editor when iterating with loop over large number of items #3258

@ivicac

Description

@ivicac

Sample workflow:

sample_10000.csv

{
    "label": "workflow10000",
    "description": "",
    "inputs": [],
    "triggers": [
        {
            "description": "",
            "label": "Manual",
            "name": "trigger_1",
            "type": "manual/v1/manual"
        }
    ],
    "tasks": [
        {
            "label": "Filesystem",
            "name": "filesystem_1",
            "parameters": {
                "filename": "/Volumes/Data/Users/ivicac2/Downloads/sample_10000.csv"
            },
            "type": "filesystem/v1/readFile",
            "metadata": {
                "ui": {
                    "dynamicPropertyTypes": {
                        "filename": "STRING"
                    }
                }
            }
        },
        {
            "label": "Filesystem",
            "name": "filesystem_2",
            "parameters": {
                "filename": "/Volumes/Data/Users/ivicac2/Downloads/result2.csv"
            },
            "type": "filesystem/v1/readFile",
            "metadata": {
                "ui": {
                    "dynamicPropertyTypes": {
                        "filename": "STRING"
                    }
                }
            }
        },
        {
            "label": "CSV File",
            "name": "csvFile_1",
            "parameters": {
                "delimiter": ",",
                "headerRow": true,
                "includeEmptyCells": false,
                "readAsString": false,
                "fileEntry": "${filesystem_1}"
            },
            "type": "csvFile/v1/read",
            "metadata": {
                "ui": {
                    "dynamicPropertyTypes": {
                        "fileEntry": "FILE_ENTRY"
                    }
                }
            }
        },
        {
            "label": "Loop",
            "name": "loop_1",
            "parameters": {
                "loopForever": false,
                "items": "${csvFile_1}",
                "iteratee": [
                    {
                        "label": "CSV File",
                        "name": "csvFile_2",
                        "parameters": {
                            "fileEntry": "${filesystem_2}",
                            "rows": [
                                "${loop_1.item}"
                            ]
                        },
                        "type": "csvFile/v1/append",
                        "metadata": {
                            "ui": {
                                "dynamicPropertyTypes": {
                                    "fileEntry": "FILE_ENTRY",
                                    "rows[0]": "OBJECT"
                                }
                            }
                        }
                    }
                ]
            },
            "type": "loop/v1",
            "metadata": {
                "ui": {
                    "dynamicPropertyTypes": {
                        "items": "ARRAY"
                    }
                }
            }
        },
        {
            "label": "Filesystem",
            "name": "filesystem_3",
            "parameters": {
                "fileEntry": "${filesystem_2}",
                "filename": "/Volumes/Data/Users/ivicac2/Downloads/result2.csv"
            },
            "type": "filesystem/v1/writeFile",
            "metadata": {
                "ui": {
                    "dynamicPropertyTypes": {
                        "fileEntry": "FILE_ENTRY",
                        "filename": "STRING"
                    }
                }
            }
        }
    ]
}

The solution is not to use SyncMessageBroker in JobSyncExecutor, but instead to create a new, simple, in-memory AsyncMessageBroker, then start the execution in the new thread an wait in the main thread until execution finishes to preserve sync behaviour

Metadata

Metadata

Labels

backendConcerning any and all backend issuesbugSomething isn't working

Type

Projects

Status

V 0.1

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions