Skip to content

Commit 78c973f

Browse files
richard-torichard-to
authored
Add script for add code/api snippets into docs. (#823)
* Add script for add code/api snippets into docs. This script will integrate go through the markdown documentation and inserting code examples and api docstrings into the docs. python transform_markdown.py python transform_markdown.py --cache_files_list python transform_markdown.py --use_cached_files_list You can use the cached_files list to create a block list to ignore specific files that are not relevant. This script is not integrated with the existing workflow yet.
1 parent 33e4c87 commit 78c973f

File tree

4 files changed

+276
-0
lines changed

4 files changed

+276
-0
lines changed

‎ai/prompt_context/markdown_docs_blocklist.txt

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
../../docs/comparison.md
2+
../../docs/showcase.md
3+
../../docs/demo.md
4+
../../docs/index.md
5+
../../docs/goals.md
6+
../../docs/codelab/5.md
7+
../../docs/codelab/4.md
8+
../../docs/codelab/index.md
9+
../../docs/codelab/3.md
10+
../../docs/codelab/2.md
11+
../../docs/internal/type-checking.md
12+
../../docs/internal/toolchain.md
13+
../../docs/internal/architecture.md
14+
../../docs/internal/publishing.md
15+
../../docs/internal/testing.md
16+
../../docs/internal/hot-reload.md
17+
../../docs/internal/codespaces.md
18+
../../docs/internal/contributing.md
19+
../../docs/internal/new-component.md
20+
../../docs/internal/vs-code-remote-container.md
21+
../../docs/internal/modes.md
22+
../../docs/internal/development.md
23+
../../docs/internal/ci.md
24+
../../docs/blog/index.md
25+
../../docs/blog/posts/visual-editor.md
26+
../../docs/blog/posts/hello-mesop.md
27+
../../docs/blog/posts/why-mesop.md
28+
../../docs/blog/posts/web-components.md
29+
../../docs/web-components/api.md
30+
../../docs/web-components/troubleshooting.md
31+
../../docs/web-components/quickstart.md
32+
../../docs/web-components/index.md
33+
../../docs/getting-started/installing.md
34+
../../docs/guides/testing.md
35+
../../docs/guides/performance.md
36+
../../docs/guides/deployment.md
37+
../../docs/guides/auth.md
38+
../../docs/guides/debugging.md
39+
../../docs/guides/labs.md
40+
../../docs/api/config.md

‎ai/prompt_context/markdown_docs_context.txt

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
../../docs/comparison.md
2+
../../docs/showcase.md
3+
../../docs/faq.md
4+
../../docs/demo.md
5+
../../docs/index.md
6+
../../docs/goals.md
7+
../../docs/codelab/5.md
8+
../../docs/codelab/4.md
9+
../../docs/codelab/index.md
10+
../../docs/codelab/3.md
11+
../../docs/codelab/2.md
12+
../../docs/internal/type-checking.md
13+
../../docs/internal/toolchain.md
14+
../../docs/internal/architecture.md
15+
../../docs/internal/publishing.md
16+
../../docs/internal/testing.md
17+
../../docs/internal/hot-reload.md
18+
../../docs/internal/codespaces.md
19+
../../docs/internal/contributing.md
20+
../../docs/internal/new-component.md
21+
../../docs/internal/vs-code-remote-container.md
22+
../../docs/internal/modes.md
23+
../../docs/internal/development.md
24+
../../docs/internal/ci.md
25+
../../docs/blog/index.md
26+
../../docs/blog/posts/visual-editor.md
27+
../../docs/blog/posts/hello-mesop.md
28+
../../docs/blog/posts/why-mesop.md
29+
../../docs/blog/posts/web-components.md
30+
../../docs/web-components/api.md
31+
../../docs/web-components/troubleshooting.md
32+
../../docs/web-components/quickstart.md
33+
../../docs/web-components/index.md
34+
../../docs/components/text.md
35+
../../docs/components/input.md
36+
../../docs/components/markdown.md
37+
../../docs/components/radio.md
38+
../../docs/components/slider.md
39+
../../docs/components/chat.md
40+
../../docs/components/button.md
41+
../../docs/components/image.md
42+
../../docs/components/video.md
43+
../../docs/components/slide-toggle.md
44+
../../docs/components/checkbox.md
45+
../../docs/components/select.md
46+
../../docs/components/audio.md
47+
../../docs/components/table.md
48+
../../docs/components/autocomplete.md
49+
../../docs/components/plot.md
50+
../../docs/components/index.md
51+
../../docs/components/embed.md
52+
../../docs/components/sidenav.md
53+
../../docs/components/icon.md
54+
../../docs/components/link.md
55+
../../docs/components/box.md
56+
../../docs/components/textarea.md
57+
../../docs/components/text-to-text.md
58+
../../docs/components/uploader.md
59+
../../docs/components/code.md
60+
../../docs/components/tooltip.md
61+
../../docs/components/badge.md
62+
../../docs/components/progress-bar.md
63+
../../docs/components/progress-spinner.md
64+
../../docs/components/html.md
65+
../../docs/components/text-to-image.md
66+
../../docs/components/divider.md
67+
../../docs/getting-started/quickstart.md
68+
../../docs/getting-started/installing.md
69+
../../docs/getting-started/core-concepts.md
70+
../../docs/guides/multi-pages.md
71+
../../docs/guides/web-security.md
72+
../../docs/guides/state-management.md
73+
../../docs/guides/layouts.md
74+
../../docs/guides/testing.md
75+
../../docs/guides/interactivity.md
76+
../../docs/guides/performance.md
77+
../../docs/guides/event-handlers.md
78+
../../docs/guides/theming.md
79+
../../docs/guides/deployment.md
80+
../../docs/guides/auth.md
81+
../../docs/guides/debugging.md
82+
../../docs/guides/labs.md
83+
../../docs/api/query-params.md
84+
../../docs/api/viewport-size.md
85+
../../docs/api/config.md
86+
../../docs/api/style.md
87+
../../docs/api/page.md
88+
../../docs/api/commands/scroll-into-view.md
89+
../../docs/api/commands/focus-component.md
90+
../../docs/api/commands/navigate.md

‎ai/prompt_context/transform_markdown.py

+140
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
"""
2+
This script replaces mkdocs code and api placeholders with the actual code examples and
3+
API docstrings and writes the modified output to a file.
4+
5+
This code needs to import mesop, so your terminal needs to have mesop imported via pip.
6+
It does not seem to work when .cli.venv is loaded.
7+
"""
8+
9+
import argparse
10+
import inspect
11+
import os
12+
13+
import mesop as me
14+
import mesop.labs as mel
15+
from mesop.features.theme import ThemeVar
16+
17+
DOCS_PATH = "../../docs/"
18+
MARKDOWN_DOCS_CONTEXT_FILE = "markdown_docs_context.txt"
19+
MARKDOWN_DOCS_BLOCKLIST_FILE = "markdown_docs_blocklist.txt"
20+
OUTPUT_DIR = "../gen/extracted_markdown/"
21+
22+
23+
def main():
24+
parser = argparse.ArgumentParser(
25+
prog="Transform Markdown",
26+
description="Transform markdown documents for prompt context.",
27+
)
28+
parser.add_argument("--use_cached_files_list", action="store_true")
29+
parser.add_argument("--cache_files_list", action="store_true")
30+
args = parser.parse_args()
31+
32+
if args.use_cached_files_list and args.cache_files_list:
33+
print(
34+
"The flags --use_cached_files_list and --cache_files_list cannot be enabled at the same time"
35+
)
36+
return
37+
38+
if args.use_cached_files_list:
39+
file_paths = read_file_paths_from_file(MARKDOWN_DOCS_CONTEXT_FILE)
40+
else:
41+
file_paths = get_file_paths_recursively(DOCS_PATH)
42+
43+
if args.cache_files_list:
44+
write_file_paths_to_file(MARKDOWN_DOCS_CONTEXT_FILE, file_paths)
45+
46+
filtered_file_paths = filter_file_paths(file_paths)
47+
process_documentation(filtered_file_paths)
48+
49+
50+
def get_file_paths_recursively(directory):
51+
file_paths = []
52+
53+
for root, _, files in os.walk(directory):
54+
for file in files:
55+
if file.endswith(".md"):
56+
file_path = os.path.relpath(os.path.join(root, file), directory)
57+
file_paths.append(directory + file_path)
58+
59+
return file_paths
60+
61+
62+
def write_file_paths_to_file(output_file, file_paths):
63+
with open(output_file, "w") as f:
64+
f.write("\n".join(file_paths).strip())
65+
66+
67+
def read_file_paths_from_file(input_file):
68+
with open(input_file) as f:
69+
return list(filter(None, f.read().split("\n")))
70+
71+
72+
def filter_file_paths(file_paths):
73+
blocklist = set(read_file_paths_from_file(MARKDOWN_DOCS_BLOCKLIST_FILE))
74+
return [file_path for file_path in file_paths if file_path not in blocklist]
75+
76+
77+
def process_documentation(file_paths):
78+
if not os.path.exists(OUTPUT_DIR):
79+
os.makedirs(OUTPUT_DIR)
80+
81+
total_char_count = 0
82+
processed_files = 0
83+
for file_path in file_paths:
84+
with open(file_path) as f:
85+
updated_lines = []
86+
for line in f.readlines():
87+
# Handle code snippet imports
88+
if line.startswith("--8<--"):
89+
with open(
90+
"../../" + line.removeprefix("--8<--").strip().replace('"', "")
91+
) as f2:
92+
updated_lines.append(f2.read())
93+
# Handle doc strings
94+
elif line.startswith("::: "):
95+
try:
96+
symbol_name = line.removeprefix(":::").strip().split(".")[-1]
97+
if symbol_name == "ThemeVar":
98+
updated_lines.append("### ThemeVar \n")
99+
updated_lines.append("attr ThemeVar = " + str(ThemeVar) + "\n")
100+
else:
101+
if "mesop.labs." in line:
102+
symbol = getattr(mel, symbol_name)
103+
else:
104+
symbol = getattr(me, symbol_name)
105+
106+
signature = inspect.signature(symbol)
107+
updated_lines.append("### " + symbol.__name__ + "\n")
108+
type_keyword = (
109+
"def" if "function" in str(symbol.__class__) else "class"
110+
)
111+
updated_lines.append("```python")
112+
updated_lines.append(
113+
type_keyword + " " + symbol.__name__ + str(signature) + ":"
114+
)
115+
updated_lines.append(" " + symbol.__doc__.strip())
116+
updated_lines.append("```\n")
117+
except Exception:
118+
print("Failed to process: " + line)
119+
updated_lines.append(line.strip())
120+
else:
121+
updated_lines.append(line.strip())
122+
123+
file_contents = "\n".join(updated_lines).strip()
124+
char_count = len(file_contents)
125+
total_char_count += char_count
126+
processed_files += 1
127+
print(f"Processed {file_path}: {char_count:,} characters")
128+
129+
with open(
130+
OUTPUT_DIR + file_path.replace("../../", "").replace("/", "_"), "w"
131+
) as fw:
132+
fw.write(file_contents)
133+
134+
print(f"Text extraction complete. Output written to {OUTPUT_DIR}")
135+
print(f"Total files processed: {processed_files}")
136+
print(f"Total characters extracted: {total_char_count:,}")
137+
138+
139+
if __name__ == "__main__":
140+
main()

‎mesop/__init__.py

+6
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@
8686
from mesop.components.plot.plot import (
8787
plot as plot,
8888
)
89+
from mesop.components.progress_bar.progress_bar import (
90+
ProgressBarAnimationEndEvent as ProgressBarAnimationEndEvent,
91+
)
8992
from mesop.components.progress_bar.progress_bar import (
9093
progress_bar as progress_bar,
9194
)
@@ -101,6 +104,9 @@
101104
from mesop.components.radio.radio import (
102105
radio as radio,
103106
)
107+
from mesop.components.select.select import (
108+
SelectOpenedChangeEvent as SelectOpenedChangeEvent,
109+
)
104110
from mesop.components.select.select import (
105111
SelectOption as SelectOption,
106112
)

0 commit comments

Comments
 (0)