CudaRunTimeError

[2518298329]

in cpu , run extract_single_partiton, dask workers will raise CudaRunTimeError ,could someone answer me
my code:
def main():

if get_all_files_paths_under(config.cleaned_output_dir):
    logger.warning("Files is already exists, skipping...")
    sys.exit(0)
    
with get_client(scheduler_address=config.cpu_scheduler_address) as client:
    worker_num = get_num_workers(client)
    logger.info(f"Num CPU Workers = {worker_num}", flush=True)

    t0 = time.time()
    dataset = extract_markdown(config.input_dirs,
                            config.extract_output_dir)
    #dataset = add_id_and_clean_text(dataset, config.prefix)


    dataset = dataset.repartition(npartitions=config.repartition_num)
    
    dataset.df.to_parquet(config.cleaned_output_dir, write_index=False)

    logger.info(f"Extract and initialize data done took {time.time() - t0:.2f} s")

def extract_markdown(input_dirs,
extract_output_dir,
data_format = "markdown",
output_type="parquet",
dump = True):
logger.info(f"Now starting extract {data_format} data in {input_dirs[:1]} files!\n")

if dump:
    os.makedirs(extract_output_dir, exist_ok=True)

output_paths = list(
    map(
        lambda f: os.path.join(
            extract_output_dir, os.path.basename(f).split(".")[0] + f".{output_type}"
        ),
        input_dirs,
    )
)

iterator = MarkdownIterator(1000)
extractor = MarkdownExtractor()

df = dd.from_map(extract_single_partition,
                zip(input_dirs, output_paths),
                iterator=iterator,
                extractor=extractor,
                output_type=output_type,
                dump=dump)
return DocumentDataset(df)

Error:Traceback (most recent call last):
File "", line 1, in
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/nemo_curator/init.py", line 17, in
from .modules import *
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/nemo_curator/modules/init.py", line 24, in
from .add_id import AddId
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/nemo_curator/modules/add_id.py", line 21, in
from nemo_curator.datasets import DocumentDataset
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/nemo_curator/datasets/init.py", line 17, in
from .doc_dataset import DocumentDataset
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/nemo_curator/datasets/doc_dataset.py", line 22, in
from nemo_curator.utils.distributed_utils import read_data, write_to_disk
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/nemo_curator/utils/distributed_utils.py", line 23, in
from nemo_curator._compat import DASK_CUDF_PARQUET_READ_INCONSISTENT_SCHEMA
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/nemo_curator/_compat.py", line 28, in
import dask_cudf
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/dask_cudf/init.py", line 7, in
import cudf
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/cudf/init.py", line 20, in
validate_setup()
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/cudf/utils/gpu_utils.py", line 55, in validate_setup
raise e
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/cudf/utils/gpu_utils.py", line 52, in validate_setup
gpus_count = getDeviceCount()
File "/home/jovyan/yujia-text-cfs/workspace-0/venvs/nemo-curator/lib/python3.10/site-packages/rmm/_cuda/gpu.py", line 102, in getDeviceCount
raise CUDARuntimeError(status)
rmm._cuda.gpu.CUDARuntimeError: cudaErrorInsufficientDriver: CUDA driver version is insufficient for CUDA runtime version