Skip to content

Commit 519c1c2

Browse files
adhityanadhityan
authored
Merge pull request #184 from llm-tools/confluence_updates
Confluence updates
2 parents 84d6be8 + 5b7b3ba commit 519c1c2

File tree

35 files changed

+315
-299
lines changed

35 files changed

+315
-299
lines changed

‎core/embedjs-interfaces/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-interfaces",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Interfaces for extending the embedjs ecosystem",
55
"dependencies": {
6-
"@langchain/core": "^0.3.23",
6+
"@langchain/core": "^0.3.25",
77
"debug": "^4.4.0",
88
"md5": "^2.3.0",
99
"uuid": "^11.0.3"

‎core/embedjs-utils/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-utils",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Useful util functions when extending the embedjs ecosystem",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24"
6+
"@llm-tools/embedjs-interfaces": "0.1.25"
77
},
88
"type": "module",
99
"main": "./src/index.js",

‎core/embedjs/package.json

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
{
22
"type": "module",
33
"name": "@llm-tools/embedjs",
4-
"version": "0.1.24",
4+
"version": "0.1.25",
55
"description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
66
"dependencies": {
77
"@langchain/textsplitters": "^0.1.0",
8-
"@llm-tools/embedjs-interfaces": "0.1.24",
9-
"@llm-tools/embedjs-utils": "0.1.24",
8+
"@llm-tools/embedjs-interfaces": "0.1.25",
9+
"@llm-tools/embedjs-utils": "0.1.25",
1010
"debug": "^4.4.0",
1111
"langchain": "^0.3.7",
1212
"md5": "^2.3.0",
13-
"mime": "^4.0.4",
13+
"mime": "^4.0.6",
1414
"stream-mime-type": "^2.0.0"
1515
},
1616
"devDependencies": {

‎databases/embedjs-astra/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-astradb",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add AstraDB support to embedjs",
55
"dependencies": {
66
"@datastax/astra-db-ts": "^1.5.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.24",
7+
"@llm-tools/embedjs-interfaces": "0.1.25",
88
"debug": "^4.4.0"
99
},
1010
"type": "module",

‎databases/embedjs-cosmos/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-cosmos",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add CosmosDB support to embedjs",
55
"dependencies": {
66
"@azure/cosmos": "^4.2.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.24",
7+
"@llm-tools/embedjs-interfaces": "0.1.25",
88
"debug": "^4.4.0"
99
},
1010
"type": "module",

‎databases/embedjs-hnswlib/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-hnswlib",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add HNSWLib support to embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
77
"debug": "^4.4.0",
88
"hnswlib-node": "^3.0.0"
99
},

‎databases/embedjs-lancedb/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-lancedb",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add LanceDb support to embedjs",
55
"dependencies": {
66
"@lancedb/lancedb": "^0.14.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.24",
7+
"@llm-tools/embedjs-interfaces": "0.1.25",
88
"compute-cosine-similarity": "^1.1.0",
99
"debug": "^4.4.0"
1010
},

‎databases/embedjs-libsql/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
{
22
"name": "@llm-tools/embedjs-libsql",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add LibSQL support to embedjs",
55
"dependencies": {
66
"@libsql/client": "^0.14.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.24",
8-
"@llm-tools/embedjs-utils": "0.1.24",
7+
"@llm-tools/embedjs-interfaces": "0.1.25",
8+
"@llm-tools/embedjs-utils": "0.1.25",
99
"debug": "^4.4.0"
1010
},
1111
"type": "module",

‎databases/embedjs-lmdb/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-lmdb",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add LMDB support to embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
77
"debug": "^4.4.0",
88
"lmdb": "^3.2.0"
99
},

‎databases/embedjs-mongodb/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-mongodb",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add MongoDB support to embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
77
"debug": "^4.4.0",
88
"mongodb": "^6.12.0"
99
},

‎databases/embedjs-pinecone/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-pinecone",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add Pinecone support to embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
77
"@pinecone-database/pinecone": "^4.0.0",
88
"debug": "^4.4.0"
99
},

‎databases/embedjs-qdrant/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-qdrant",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add Qdrant support to embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
77
"@qdrant/js-client-rest": "^1.12.0",
88
"debug": "^4.4.0",
99
"uuid": "^11.0.3"

‎databases/embedjs-redis/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-redis",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add Redis support to embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
77
"ioredis": "^5.4.1"
88
},
99
"type": "module",

‎databases/embedjs-weaviate/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"name": "@llm-tools/embedjs-weaviate",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Add Weaviate support to embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
77
"compute-cosine-similarity": "^1.1.0",
88
"debug": "^4.4.0",
99
"weaviate-ts-client": "^2.2.0"

‎examples/confluence/src/main.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,5 @@ const llmApplication = await new RAGApplicationBuilder()
1313
.setEmbeddingModel(new OpenAiEmbeddings())
1414
.build();
1515

16-
await llmApplication.addLoader(new ConfluenceLoader({ spaceNames: ['DEMO'] }));
16+
await llmApplication.addLoader(new ConfluenceLoader({ spaceName: 'DEMO' }));
1717
console.log(await llmApplication.query('Who founded Tesla?'));

‎loaders/embedjs-loader-confluence/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-loader-confluence",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Confluence loader for embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
7-
"@llm-tools/embedjs-loader-web": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
7+
"@llm-tools/embedjs-loader-web": "0.1.25",
88
"confluence.js": "^1.7.4",
99
"debug": "^4.4.0",
1010
"md5": "^2.3.0"

‎loaders/embedjs-loader-confluence/src/confluence-loader.ts

+36-24
Original file line numberDiff line numberDiff line change
@@ -10,32 +10,34 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
1010

1111
private readonly confluence: ConfluenceClient;
1212
private readonly confluenceBaseUrl: string;
13-
private readonly spaceNames: string[];
13+
private readonly spaceName: string;
14+
15+
private readonly lastUpdatedFilter?: Date;
1416

1517
constructor({
16-
spaceNames,
18+
spaceName,
1719
confluenceBaseUrl,
1820
confluenceUsername,
1921
confluenceToken,
2022
chunkSize,
2123
chunkOverlap,
24+
options,
2225
}: {
23-
spaceNames: [string, ...string[]];
26+
spaceName: string;
2427
confluenceBaseUrl?: string;
2528
confluenceUsername?: string;
2629
confluenceToken?: string;
2730
chunkSize?: number;
2831
chunkOverlap?: number;
32+
options?: {
33+
lastUpdatedFilter: Date;
34+
};
2935
}) {
30-
super(
31-
`ConfluenceLoader_${md5(spaceNames.sort().join(','))}`,
32-
{ spaceNames },
33-
chunkSize ?? 2000,
34-
chunkOverlap ?? 200,
35-
);
36-
37-
this.spaceNames = spaceNames;
36+
super(`ConfluenceLoader_${md5(spaceName)}`, { spaceName }, chunkSize ?? 2000, chunkOverlap ?? 200);
37+
38+
this.spaceName = spaceName;
3839
this.confluenceBaseUrl = confluenceBaseUrl ?? process.env.CONFLUENCE_BASE_URL;
40+
this.lastUpdatedFilter = options?.lastUpdatedFilter ?? null;
3941

4042
this.confluence = new ConfluenceClient({
4143
host: this.confluenceBaseUrl,
@@ -49,16 +51,13 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
4951
}
5052

5153
override async *getUnfilteredChunks() {
52-
for (const spaceKey of this.spaceNames) {
53-
let count = 0;
54-
55-
for await (const result of this.processSpace(spaceKey)) {
56-
yield result;
57-
count++;
58-
}
59-
60-
this.debug(`Space '${spaceKey}' had ${count} new pages`);
54+
let count = 0;
55+
for await (const result of this.processSpace(this.spaceName)) {
56+
yield result;
57+
count++;
6158
}
59+
60+
this.debug(`Space '${this.spaceName}' had ${count} new pages`);
6261
}
6362

6463
private async *processSpace(spaceKey: string) {
@@ -84,13 +83,26 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
8483
let confluenceVersion = 0;
8584

8685
try {
87-
const spaceProperties = await this.confluence.content.getContentById({
86+
const pageProperties = await this.confluence.content.getContentById({
8887
id: pageId,
89-
expand: ['version'],
88+
expand: ['version', 'history'],
9089
});
9190

92-
if (!spaceProperties.version.number) throw new Error('Version number not found in space properties...');
93-
confluenceVersion = spaceProperties.version.number;
91+
if (this.lastUpdatedFilter) {
92+
const pageLastEditDate = new Date(pageProperties.history.lastUpdated.when);
93+
94+
if (pageLastEditDate > this.lastUpdatedFilter) {
95+
this.debug(`Page '${title}' has last edit date ${pageLastEditDate}. Continuing...`);
96+
} else {
97+
this.debug(
98+
`Page '${title}' has last edit date ${pageLastEditDate}, which is less than filter date. Skipping...`,
99+
);
100+
return;
101+
}
102+
}
103+
104+
if (!pageProperties.version.number) throw new Error('Version number not found in page properties...');
105+
confluenceVersion = pageProperties.version.number;
94106
} catch (e) {
95107
this.debug('Could not get page properties. Page will be SKIPPED!', title, e.response);
96108
return;

‎loaders/embedjs-loader-csv/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-loader-csv",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "CSV loader for embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
7-
"@llm-tools/embedjs-utils": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
7+
"@llm-tools/embedjs-utils": "0.1.25",
88
"csv-parse": "^5.6.0",
99
"debug": "^4.4.0",
1010
"md5": "^2.3.0"

‎loaders/embedjs-loader-markdown/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-loader-markdown",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "XML loader for embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
7-
"@llm-tools/embedjs-loader-web": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
7+
"@llm-tools/embedjs-loader-web": "0.1.25",
88
"debug": "^4.4.0",
99
"md5": "^2.3.0",
1010
"micromark": "^4.0.1",

‎loaders/embedjs-loader-msoffice/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
{
22
"name": "@llm-tools/embedjs-loader-msoffice",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Word, PPT and Excel loader for embedjs",
55
"dependencies": {
66
"@langchain/textsplitters": "^0.1.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.24",
8-
"@llm-tools/embedjs-utils": "0.1.24",
7+
"@llm-tools/embedjs-interfaces": "0.1.25",
8+
"@llm-tools/embedjs-utils": "0.1.25",
99
"md5": "^2.3.0",
1010
"office-text-extractor": "^3.0.3"
1111
},

‎loaders/embedjs-loader-pdf/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
{
22
"name": "@llm-tools/embedjs-loader-pdf",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "PDF loader for embedjs",
55
"dependencies": {
66
"@langchain/textsplitters": "^0.1.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.24",
8-
"@llm-tools/embedjs-utils": "0.1.24",
7+
"@llm-tools/embedjs-interfaces": "0.1.25",
8+
"@llm-tools/embedjs-utils": "0.1.25",
99
"md5": "^2.3.0",
1010
"office-text-extractor": "^3.0.3"
1111
},

‎loaders/embedjs-loader-sitemap/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"name": "@llm-tools/embedjs-loader-sitemap",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Sitemap recursive loader for embedjs",
55
"dependencies": {
6-
"@llm-tools/embedjs-interfaces": "0.1.24",
7-
"@llm-tools/embedjs-loader-web": "0.1.24",
6+
"@llm-tools/embedjs-interfaces": "0.1.25",
7+
"@llm-tools/embedjs-loader-web": "0.1.25",
88
"debug": "^4.4.0",
99
"md5": "^2.3.0",
1010
"sitemapper": "^3.2.18"

‎loaders/embedjs-loader-web/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
{
22
"name": "@llm-tools/embedjs-loader-web",
3-
"version": "0.1.24",
3+
"version": "0.1.25",
44
"description": "Web page loader for embedjs",
55
"dependencies": {
66
"@langchain/textsplitters": "^0.1.0",
7-
"@llm-tools/embedjs-interfaces": "0.1.24",
8-
"@llm-tools/embedjs-utils": "0.1.24",
7+
"@llm-tools/embedjs-interfaces": "0.1.25",
8+
"@llm-tools/embedjs-utils": "0.1.25",
99
"debug": "^4.4.0",
1010
"html-to-text": "^9.0.5",
1111
"md5": "^2.3.0"

0 commit comments

Comments
 (0)