Skip to content

Commit 53913db

Browse files
dlucdluc
authored
Merge branch 'main' into rabbitmq_client_v7.0
2 parents d55f6e0 + 0961ec7 commit 53913db

File tree

6 files changed

+190
-7
lines changed

6 files changed

+190
-7
lines changed

‎Directory.Packages.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
<PackageVersion Include="DocumentFormat.OpenXml" Version="3.2.0" />
1515
<PackageVersion Include="HtmlAgilityPack" Version="1.11.72" />
1616
<PackageVersion Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
17+
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.0" />
1718
<PackageVersion Include="Microsoft.Data.SqlClient" Version="5.2.2" />
1819
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="9.0.1" />
1920
<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="9.0.1" />

‎extensions/Chunkers/Chunkers/PlainTextChunker.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@ internal enum SeparatorTypes
6060
"!!!!", "????", "!!!", "???", "?!?", "!?!", "!?", "?!", "!!", "??", "....", "...", "..",
6161
// 1 char separators without space
6262
".", "?", "!", "⁉", "⁈", "⁇", "…",
63-
]);
63+
// Chinese punctuation
64+
"。", "？", "！", "；", "："
65+
]);
6466

6567
// Prioritized list of characters to split inside a sentence.
6668
private static readonly SeparatorTrie s_potentialSeparators = new([
@@ -69,12 +71,16 @@ internal enum SeparatorTypes
6971
") ", ")\t", ")\n",
7072
"] ", "]\t", "]\n",
7173
")", "]",
74+
// Chinese punctuation
75+
"，", "、", "（", "）", "【", "】", "《", "》"
7276
]);
7377

7478
// Prioritized list of characters to split inside a sentence when other splits are not found.
7579
private static readonly SeparatorTrie s_weakSeparators1 = new([
7680
": ", ":", // note: \n \t make no difference with this char
7781
", ", ",", // note: \n \t make no difference with this char
82+
// Chinese punctuation
83+
"：", "，"
7884
]);
7985

8086
// Prioritized list of characters to split inside a sentence when other splits are not found.
@@ -84,6 +90,8 @@ internal enum SeparatorTypes
8490
"' ", "'", // note: \n \t make no difference with this char
8591
"\" ", "\"", // note: \n \t make no difference with this char
8692
" ", // note: \n \t make no difference with this char
93+
// Chinese punctuation
94+
"“", "”", "‘", "’"
8795
]);
8896

8997
// Prioritized list of characters to split inside a sentence when other splits are not found.
@@ -93,6 +101,8 @@ internal enum SeparatorTypes
93101
"|", // note: \n \t make no difference with this char
94102
"@", // note: \n \t make no difference with this char
95103
"=", // note: \n \t make no difference with this char
104+
// Chinese punctuation
105+
"·", "—", "～"
96106
]);
97107

98108
public PlainTextChunker(ITextTokenizer? tokenizer = null)

‎service/Service.AspNetCore/Service.AspNetCore.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
<ItemGroup>
1414
<ProjectReference Include="..\..\extensions\KM\KernelMemory\KernelMemory.csproj" />
15+
<PackageReference Include="Microsoft.AspNetCore.OpenApi" />
1516
</ItemGroup>
1617

1718
<PropertyGroup>

‎service/Service.AspNetCore/WebAPIEndpoints.cs‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
using Microsoft.KernelMemory.DocumentStorage;
2121
using Microsoft.KernelMemory.HTTP;
2222
using Microsoft.KernelMemory.Service.AspNetCore.Models;
23+
using Microsoft.OpenApi.Models;
2324

2425
namespace Microsoft.KernelMemory.Service.AspNetCore;
2526

@@ -107,8 +108,63 @@ public static RouteHandlerBuilder AddPostUploadEndpoint(
107108
})
108109
.WithName("UploadDocument")
109110
.WithDisplayName("UploadDocument")
110-
.WithDescription("Upload a new document to the knowledge base and extract memories from it.")
111-
.WithSummary("Upload a document to the knowledge base and extract memories from it. If a document with the same ID already exists, it will be overwritten and the memories previously extracted will be updated. Note: a document can contain multiple files.")
111+
.WithOpenApi(
112+
operation =>
113+
{
114+
operation.Summary = "Upload a new document to the knowledge base";
115+
operation.Description = "Upload a document consisting of one or more files to extract memories from. The extraction process happens asynchronously. If a document with the same ID already exists, it will be overwritten and the memories previously extracted will be updated.";
116+
operation.RequestBody = new OpenApiRequestBody
117+
{
118+
Content =
119+
{
120+
["multipart/form-data"] = new OpenApiMediaType
121+
{
122+
Schema = new OpenApiSchema
123+
{
124+
Type = "object",
125+
Properties =
126+
{
127+
["index"] = new OpenApiSchema
128+
{
129+
Type = "string",
130+
Description = "Name of the index where to store memories generated by the files."
131+
},
132+
["documentId"] = new OpenApiSchema
133+
{
134+
Type = "string",
135+
Description = "Unique ID used for import pipeline and document ID."
136+
},
137+
["tags"] = new OpenApiSchema
138+
{
139+
Type = "object",
140+
AdditionalProperties = new OpenApiSchema { Type = "string" },
141+
Description = "Tags to apply to the memories extracted from the files."
142+
},
143+
["steps"] = new OpenApiSchema
144+
{
145+
Type = "array",
146+
Items = new OpenApiSchema { Type = "string" },
147+
Description = "How to process the files, e.g. how to extract/chunk etc."
148+
},
149+
["files"] = new OpenApiSchema
150+
{
151+
Type = "array",
152+
Items = new OpenApiSchema
153+
{
154+
Type = "string",
155+
Format = "binary"
156+
},
157+
Description = "Files to process and extract memories from."
158+
}
159+
}
160+
}
161+
}
162+
},
163+
Description = "Document to upload and extract memories from"
164+
};
165+
return operation;
166+
}
167+
)
112168
.Produces<UploadAccepted>(StatusCodes.Status202Accepted)
113169
.Produces<ProblemDetails>(StatusCodes.Status400BadRequest)
114170
.Produces<ProblemDetails>(StatusCodes.Status401Unauthorized)

‎service/Service/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ public static void Main(string[] args)
206206
const double AspnetDefaultMaxUploadSize = 30000000d / 1024 / 1024;
207207
Console.WriteLine("* Web service auth : " + (config.ServiceAuthorization.Enabled ? "Enabled" : "Disabled"));
208208
Console.WriteLine("* Max HTTP req size : " + (config.Service.MaxUploadSizeMb ?? AspnetDefaultMaxUploadSize).ToString("0.#", CultureInfo.CurrentCulture) + " Mb");
209-
Console.WriteLine("* OpenAPI swagger : " + (config.Service.OpenApiEnabled ? "Enabled" : "Disabled"));
209+
Console.WriteLine("* OpenAPI swagger : " + (config.Service.OpenApiEnabled ? "Enabled (/swagger/index.html)" : "Disabled"));
210210
}
211211

212212
Console.WriteLine("* Memory Db : " + app.Services.GetService<IMemoryDb>()?.GetType().FullName);

‎swagger.json‎

Lines changed: 118 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,10 +94,58 @@
9494
"tags": [
9595
"Microsoft.KernelMemory.ServiceAssembly"
9696
],
97-
"summary": "Upload a document to the knowledge base and extract memories from it. If a document with the same ID already exists, it will be overwritten and the memories previously extracted will be updated. Note: a document can contain multiple files.",
98-
"description": "Upload a new document to the knowledge base and extract memories from it.",
97+
"summary": "Upload a new document to the knowledge base",
98+
"description": "Upload a document consisting of one or more files to extract memories from. The extraction process happens asynchronously. If a document with the same ID already exists, it will be overwritten and the memories previously extracted will be updated.",
9999
"operationId": "UploadDocument",
100+
"requestBody": {
101+
"description": "Document to upload and extract memories from",
102+
"content": {
103+
"multipart/form-data": {
104+
"schema": {
105+
"type": "object",
106+
"properties": {
107+
"index": {
108+
"type": "string",
109+
"description": "Name of the index where to store memories generated by the files."
110+
},
111+
"documentId": {
112+
"type": "string",
113+
"description": "Unique ID used for import pipeline and document ID."
114+
},
115+
"tags": {
116+
"type": "object",
117+
"additionalProperties": {
118+
"type": "string"
119+
},
120+
"description": "Tags to apply to the memories extracted from the files."
121+
},
122+
"steps": {
123+
"type": "array",
124+
"items": {
125+
"type": "string"
126+
},
127+
"description": "How to process the files, e.g. how to extract/chunk etc."
128+
},
129+
"files": {
130+
"type": "array",
131+
"items": {
132+
"type": "string",
133+
"format": "binary"
134+
},
135+
"description": "Files to process and extract memories from."
136+
}
137+
}
138+
}
139+
}
140+
}
141+
},
100142
"responses": {
143+
"200": {
144+
"description": "OK",
145+
"content": {
146+
"application/json": {}
147+
}
148+
},
101149
"202": {
102150
"description": "Accepted",
103151
"content": {
@@ -669,6 +717,13 @@
669717
"type": "string",
670718
"nullable": true
671719
},
720+
"tokenUsage": {
721+
"type": "array",
722+
"items": {
723+
"$ref": "#/components/schemas/TokenUsage"
724+
},
725+
"nullable": true
726+
},
672727
"relevantSources": {
673728
"type": "array",
674729
"items": {
@@ -855,6 +910,53 @@
855910
],
856911
"type": "string"
857912
},
913+
"TokenUsage": {
914+
"type": "object",
915+
"properties": {
916+
"timestamp": {
917+
"type": "string",
918+
"format": "date-time"
919+
},
920+
"serviceType": {
921+
"type": "string",
922+
"nullable": true
923+
},
924+
"modelType": {
925+
"type": "string",
926+
"nullable": true
927+
},
928+
"modelName": {
929+
"type": "string",
930+
"nullable": true
931+
},
932+
"tokenizerTokensIn": {
933+
"type": "integer",
934+
"format": "int32",
935+
"nullable": true
936+
},
937+
"tokenizerTokensOut": {
938+
"type": "integer",
939+
"format": "int32",
940+
"nullable": true
941+
},
942+
"serviceTokensIn": {
943+
"type": "integer",
944+
"format": "int32",
945+
"nullable": true
946+
},
947+
"serviceTokensOut": {
948+
"type": "integer",
949+
"format": "int32",
950+
"nullable": true
951+
},
952+
"serviceReasoningTokens": {
953+
"type": "integer",
954+
"format": "int32",
955+
"nullable": true
956+
}
957+
},
958+
"additionalProperties": false
959+
},
858960
"UploadAccepted": {
859961
"type": "object",
860962
"properties": {
@@ -873,6 +975,19 @@
873975
},
874976
"additionalProperties": false
875977
}
978+
},
979+
"securitySchemes": {
980+
"auth": {
981+
"type": "apiKey",
982+
"description": "The API key to access the API",
983+
"name": "Authorization",
984+
"in": "header"
985+
}
986+
}
987+
},
988+
"security": [
989+
{
990+
"auth": []
876991
}
877-
}
992+
]
878993
}

0 commit comments

Comments
 (0)