Skip to content

Commit 8537c14

Browse files
auscasterauscaster
committed
feat(n8n): improve docs build orchestration
1 parent 127792e commit 8537c14

2 files changed

Lines changed: 130 additions & 16 deletions

File tree

‎test/sourcey-retrieval.test.mjs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,28 @@ Loaded from llms-full.
123123
assert.equal(fromHtml.pageContent, 'HTML Guide Fallback text.');
124124
});
125125

126+
it('loads pages from sitemap when llms-full is missing', async () => {
127+
const results = await loadAllSourceyDocs({
128+
httpGet: fixtureHttpGet({
129+
'/reference/llms-full.txt': notFound(),
130+
'/reference/sitemap.xml': `<urlset>
131+
<url><loc>https://docs.example.com/reference/guides/sitemap.html</loc></url>
132+
</urlset>`,
133+
'/reference/guides/sitemap.html': '<html><head><title>Sitemap Guide</title></head><body><h1>Sitemap Guide</h1><p>Loaded from sitemap fallback.</p></body></html>',
134+
}),
135+
siteUrl: SITE_URL,
136+
outputMode: 'page',
137+
includeContent: true,
138+
maxPages: 10,
139+
chunkSize: 4000,
140+
});
141+
142+
assert.equal(results.length, 1);
143+
assert.equal(results[0].title, 'Sitemap Guide');
144+
assert.equal(results[0].path, 'guides/sitemap');
145+
assert.equal(results[0].pageContent, 'Sitemap Guide Loaded from sitemap fallback.');
146+
});
147+
126148
it('loads all docs as chunk items for vector-store ingestion', async () => {
127149
const results = await loadAllSourceyDocs({
128150
httpGet: fixtureHttpGet({

‎workflows/build-and-index-sourcey-docs.json‎

Lines changed: 108 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,56 +9,148 @@
99
"typeVersion": 1,
1010
"position": [0, 0]
1111
},
12+
{
13+
"parameters": {
14+
"assignments": {
15+
"assignments": [
16+
{
17+
"id": "repo-owner",
18+
"name": "repoOwner",
19+
"type": "string",
20+
"value": "your-org"
21+
},
22+
{
23+
"id": "repo-name",
24+
"name": "repository",
25+
"type": "string",
26+
"value": "your-docs-repo"
27+
},
28+
{
29+
"id": "workflow-id",
30+
"name": "workflowId",
31+
"type": "string",
32+
"value": "sourcey-build-docs.yml"
33+
},
34+
{
35+
"id": "workflow-ref",
36+
"name": "ref",
37+
"type": "string",
38+
"value": "main"
39+
},
40+
{
41+
"id": "fallback-site-url",
42+
"name": "fallbackSiteUrl",
43+
"type": "string",
44+
"value": "https://docs.example.com"
45+
},
46+
{
47+
"id": "max-pages",
48+
"name": "maxPages",
49+
"type": "number",
50+
"value": 100
51+
},
52+
{
53+
"id": "chunk-size",
54+
"name": "chunkSize",
55+
"type": "number",
56+
"value": 4000
57+
}
58+
]
59+
}
60+
},
61+
"id": "build-inputs",
62+
"name": "Build Inputs",
63+
"type": "n8n-nodes-base.set",
64+
"typeVersion": 3,
65+
"position": [240, 0]
66+
},
1267
{
1368
"parameters": {
1469
"resource": "workflow",
1570
"operation": "dispatch",
16-
"owner": "sourcey",
17-
"repository": "example-docs",
18-
"workflowId": "deploy-docs.yml",
19-
"ref": "main"
71+
"owner": "={{ $json.repoOwner }}",
72+
"repository": "={{ $json.repository }}",
73+
"workflowId": "={{ $json.workflowId }}",
74+
"ref": "={{ $json.ref }}"
2075
},
2176
"id": "dispatch-build",
2277
"name": "Dispatch Sourcey Build",
2378
"type": "n8n-nodes-base.github",
2479
"typeVersion": 1,
25-
"position": [240, 0]
80+
"position": [480, 0]
2681
},
2782
{
2883
"parameters": {
29-
"amount": 60,
30-
"unit": "seconds"
84+
"resume": "webhook",
85+
"options": {}
3186
},
32-
"id": "wait-for-deploy",
33-
"name": "Wait for Deployment",
87+
"id": "wait-for-deployment-url",
88+
"name": "Wait for Deployment URL",
3489
"type": "n8n-nodes-base.wait",
3590
"typeVersion": 1,
36-
"position": [480, 0]
91+
"position": [720, 0]
92+
},
93+
{
94+
"parameters": {
95+
"assignments": {
96+
"assignments": [
97+
{
98+
"id": "resolved-site-url",
99+
"name": "siteUrl",
100+
"type": "string",
101+
"value": "={{ $json.deployment_status?.environment_url || $json.deployment?.environment_url || $json.deployment?.payload?.web_url || $json.target_url || $json.fallbackSiteUrl }}"
102+
},
103+
{
104+
"id": "resolved-max-pages",
105+
"name": "maxPages",
106+
"type": "number",
107+
"value": "={{ $json.maxPages }}"
108+
},
109+
{
110+
"id": "resolved-chunk-size",
111+
"name": "chunkSize",
112+
"type": "number",
113+
"value": "={{ $json.chunkSize }}"
114+
}
115+
]
116+
}
117+
},
118+
"id": "deployment-url",
119+
"name": "Deployment URL",
120+
"type": "n8n-nodes-base.set",
121+
"typeVersion": 3,
122+
"position": [960, 0]
37123
},
38124
{
39125
"parameters": {
40126
"operation": "loadAll",
41-
"siteUrl": "https://example.com/docs",
127+
"siteUrl": "={{ $json.siteUrl }}",
42128
"outputMode": "chunk",
43129
"includeContent": true,
44-
"maxPages": 100,
45-
"chunkSize": 4000
130+
"maxPages": "={{ $json.maxPages }}",
131+
"chunkSize": "={{ $json.chunkSize }}"
46132
},
47133
"id": "sourcey-load-all",
48134
"name": "Sourcey Load All",
49135
"type": "n8n-nodes-sourcey.sourcey",
50136
"typeVersion": 1,
51-
"position": [720, 0]
137+
"position": [1200, 0]
52138
}
53139
],
54140
"connections": {
55141
"Manual Trigger": {
142+
"main": [[{ "node": "Build Inputs", "type": "main", "index": 0 }]]
143+
},
144+
"Build Inputs": {
56145
"main": [[{ "node": "Dispatch Sourcey Build", "type": "main", "index": 0 }]]
57146
},
58147
"Dispatch Sourcey Build": {
59-
"main": [[{ "node": "Wait for Deployment", "type": "main", "index": 0 }]]
148+
"main": [[{ "node": "Wait for Deployment URL", "type": "main", "index": 0 }]]
149+
},
150+
"Wait for Deployment URL": {
151+
"main": [[{ "node": "Deployment URL", "type": "main", "index": 0 }]]
60152
},
61-
"Wait for Deployment": {
153+
"Deployment URL": {
62154
"main": [[{ "node": "Sourcey Load All", "type": "main", "index": 0 }]]
63155
}
64156
}

0 commit comments

Comments
 (0)