async def crawl_local_files(files: List[str], max_concurrent: int = 5):
    browser_config = BrowserConfig(
        headless=True,
        verbose=False,
        extra_args=["--disable-gpu", "--disable-dev-shm-usage", "--no-sandbox"],
    )
    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, capture_console_messages = False)
    crawler = AsyncWebCrawler(config=browser_config)
    await crawler.start()

    try:
        semaphore = asyncio.Semaphore(max_concurrent)
        async def process_local_file(fileurl: str):
            async with semaphore:
                result = await crawler.arun(
                    url=fileurl,
                    config=crawler_config
                )
                if result.success:
                    # print(result.markdown)
                    await process_and_store_document(fileurl, result.markdown)
                else:
                    print(f"Failed : {fileurl} - Error: {result.error_message}")
        await asyncio.gather(*[process_local_file(fileurl) for fileurl in files])
    finally:
        await crawler.close()


def get_file_paths() -> List[str]:
    """
    
    Get a list of file paths from the local directory.
    """
    print("Getting file paths from the local directory...")    
    file_paths = []
    for file in os.listdir("./assets/"):
        print(file)
        if file.endswith(".html"):
            file_path = os.path.join("./assets/", file)
            abs_path = os.path.abspath(file_path)
            fileurl = f"file://{abs_path}"
            file_paths.append(fileurl)
    return file_paths


async def main():

    print("Starting the main function...")
    files = get_file_paths()

    await crawl_local_files(files)


if __name__ == "__main__":
    asyncio.run(main())