Merge pull request #134 from cs-util-com/feature/jsonSchemaSpec2020

cs-util · web-flow · commit 3521a3b82ce0 · 2025-09-04T17:13:50.000+02:00
Feature/json schema spec2020
diff --git a/CsCore/.github/workflows/dotnet-tests.yml b/CsCore/.github/workflows/dotnet-tests.yml
@@ -0,0 +1,66 @@
+name: .NET Tests (Safe Caching + Reporting + Exclude IntegrationTests)
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read  # always minimal
+  checks: write   # needed to post check run
+  pull-requests: none
+
+jobs:
+  build-test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: none
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '8.0.x'
+          cache: true
+          cache-dependency-path: |
+            **/*.csproj
+            Directory.Packages.props
+
+      - name: Restore dependencies
+        run: dotnet restore CsCore/CsCore.sln
+
+      - name: Build solution
+        run: dotnet build CsCore/CsCore.sln --no-restore
+
+      - name: Run xUnit tests (exclude integrationTests)
+        run: >
+          dotnet test CsCore/xUnitTests/xUnitTests.csproj
+          --no-build
+          --verbosity normal
+          --logger "trx;LogFileName=test-results.trx"
+          --filter "FullyQualifiedName\!~integrationTests"
+
+  report:
+    needs: build-test
+    if: github.event_name != 'pull_request'
+        || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Upload and annotate test results
+        uses: dorny/test-reporter@v1
+        with:
+          name: Test Results
+          path: '**/*.trx'
+          reporter: dotnet-trx
+          fail-on-error: 'true'
+
diff --git a/CsCore/CsCoreUnity/Plugins/CsCoreUnity/com/csutil/ui/jsonschema/JsonModelExtensions.cs b/CsCore/CsCoreUnity/Plugins/CsCoreUnity/com/csutil/ui/jsonschema/JsonModelExtensions.cs
@@ -210,7 +210,7 @@ public static async Task LoadModelList(this ListFieldView self, JObject root, Js
 
         private static void SetupButtons(ListFieldView listView, JObject root, JsonSchemaToView viewGenerator, JArray modelArray, Dictionary<FieldView, JToken> map) {
             listView.add.SetOnClickAction(async delegate {
-                JToken entry = listView.field.items.First().NewDefaultJInstance();
+                JToken entry = listView.field.items.anyOf.First().NewDefaultJInstance();
                 modelArray.Add(entry);
                 var fieldName = "" + (modelArray.Count - 1);
                 var fv = await CreateChildEntryView(listView, root, viewGenerator, entry, fieldName);
@@ -256,7 +256,7 @@ private static IEnumerable<ListEntryView> GetSelectedViews(ListFieldView self) {
 
         private static async Task<FieldView> CreateChildEntryView(
                 ListFieldView self, JObject root, JsonSchemaToView viewGenerator, JToken modelEntry, string fieldName) {
-            JsonSchema newEntryVm = GetMatchingSchema(modelEntry, self.field.items);
+            JsonSchema newEntryVm = GetMatchingSchema(modelEntry, self.field.items.anyOf);
             GameObject childView = await AddChildEntryView(self, viewGenerator, fieldName, newEntryVm);
             await childView.LinkToJsonModel(root, viewGenerator);
             return childView.GetComponentInChildren<FieldView>();
diff --git a/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/extensions/EnumExtensions.cs b/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/extensions/EnumExtensions.cs
@@ -68,11 +68,21 @@ private static void AssertEnumEntriesMustBePowerOfTwo<T>() where T : struct {
                 int entryAsInt = (int)(object)entry;
                 if (entryAsInt == 0) { continue; } // 0 is always a power of two
                 if ((entryAsInt & (entryAsInt - 1)) != 0) { // Check if the entry is a power of two:
-                    Log.w($"Enum {typeof(T)} cant be used with .ContainsFlag() because not all entries are a power of two, e.g. {entry}={entryAsInt}! "
-                        + $"This error should only be ignored if you have enum entries that are composed from other enum entries (eg MyEnum.A = MyEnum.B | MyEnum.C)");
+                    var enumType = typeof(T);
+                    if (!IsOnBlacklist(enumType)) {
+                        Log.w($"Enum {enumType} cant be used with .ContainsFlag() because not all entries are a power of two, e.g. {entry}={entryAsInt}! "
+                            + $"This error should only be ignored if you have enum entries that are composed from other enum entries (eg MyEnum.A = MyEnum.B | MyEnum.C)");
+                    }
                 }
             }
         }
+        
+        private static bool IsOnBlacklist(Type enumType) {
+            // Anything in the System namespace is on the blacklist, as the dev has no control over these enums anyways:
+            if (enumType.Namespace.StartsWith("System")) { return true; }
+            // Add more types to the blacklist here if needed
+            return false; 
+        }
 
         public static string GetEntryName<T>(this T entry) where T : struct
 #if CSHARP_7_3 // Using Enum as a generic type constraint is only available in C# 7.3+
diff --git a/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/http/apis/OpenAi.cs b/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/http/apis/OpenAi.cs
@@ -7,6 +7,8 @@
 using Newtonsoft.Json;
 using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
+using com.csutil.model;
 using Zio;
 
 namespace com.csutil.http.apis {
@@ -210,11 +212,13 @@ public class Request {
             /// <summary> https://platform.openai.com/docs/api-reference/chat/create#chat-create-temperature </summary>
             public double? temperature { get; set; }
 
-            /// <summary> The maximum number of tokens to generate in the completion.
-            /// The token count of your prompt plus max_tokens cannot exceed the model's context length.
-            /// Most models have a context length of 2048 tokens (except for the newest models, which support 4096). </summary>
-            public int max_tokens { get; set; }
-
+            /// <summary>
+            /// An upper bound for the number of tokens that can be generated for a completion,
+            /// including visible output tokens and reasoning tokens. See also
+            /// https://platform.openai.com/docs/api-reference/chat/create#chat-create-max_completion_tokens
+            /// </summary>
+            public int? max_completion_tokens { get; set; } = null;
+            
             /// <summary> Number of desired <see cref="Response.choices"/> to be returned, defaults to 1.
             /// See also https://platform.openai.com/docs/api-reference/chat/create#chat-create-n </summary>
             public int? n { get; set; } = null;
@@ -230,13 +234,8 @@ public class Request {
             /// <summary> typically null, but if the AI e.g. should respond only with json it should be ChatGpt.Request.ResponseFormat.json </summary>
             public ResponseFormat response_format { get; set; }
 
-            public Request(List<Message> messages, int max_tokens = 4096) {
-                var tokenCountForMessages = JsonWriter.GetWriter(this).Write(messages).Length;
-                if (max_tokens + tokenCountForMessages > 4096) {
-                    max_tokens = 4096 - tokenCountForMessages;
-                }
+            public Request(List<Message> messages) {
                 this.messages = messages;
-                this.max_tokens = max_tokens;
             }
 
             /// <summary> https://platform.openai.com/docs/guides/audio/quickstart?audio-generation-quickstart-example=audio-out </summary>
@@ -257,10 +256,15 @@ public class ResponseFormat {
                 public static ResponseFormat json => new ResponseFormat() { type = "json_object" };
 
                 /// <summary> See https://platform.openai.com/docs/guides/structured-outputs/how-to-use </summary>
-                public static ResponseFormat NewJsonSchema(string name, JsonSchema schema) => new ResponseFormat() {
-                    type = "json_schema",
-                    json_schema = new JsonSchemaResponse(name, schema, strict: true)
-                };
+                public static ResponseFormat NewJsonSchema(string name, JsonSchema schema) {
+                    if (schema.additionalProperties) {
+                        throw new ArgumentException($"Invalid schema: {name} has 'additionalProperties' set to true, but this is not supported in strict json schema mode.");
+                    }
+                    return new ResponseFormat() {
+                        type = "json_schema",
+                        json_schema = new JsonSchemaResponse(name, schema, strict: true)
+                    };
+                }
 
                 public string type { get; set; }
 
@@ -282,6 +286,12 @@ public JsonSchemaResponse(string name, JsonSchema schema, bool strict) {
                         this.name = name;
                         this.schema = schema;
                         this.strict = strict;
+                        if (!Regex.IsMatch(name, "^[a-zA-Z0-9_-]+$")) { // name must match the pattern '^[a-zA-Z0-9_-]+$' so verify that here:
+                            throw new ArgumentException($"Invalid 'response_format.json_schema.name': {name} does not match pattern. Expected a string that matches the pattern '^[a-zA-Z0-9_-]+$'.");
+                        }
+                        if (schema.type != "object") {
+                            throw new ArgumentException($"Invalid schema.type: {schema.type} is not supported, expected 'object'.");
+                        }
                     }
 
                 }
@@ -359,20 +369,18 @@ public class Request {
             /// <summary> See https://beta.openai.com/docs/models/overview </summary>
             public string model = "gpt-4o";
 
-            /// <summary> The maximum number of tokens to generate in the completion.
-            /// The token count of your prompt plus max_tokens cannot exceed the model's context length.
-            /// Most models have a context length of 2048 tokens (except for the newest models, which support 4096). </summary>
-            public int max_tokens { get; set; }
+            /// <summary>
+            /// An upper bound for the number of tokens that can be generated for a completion,
+            /// including visible output tokens and reasoning tokens. See also
+            /// https://platform.openai.com/docs/api-reference/chat/create#chat-create-max_completion_tokens
+            /// </summary>
+            public int? max_completion_tokens { get; set; } = null;
             public List<Line> messages { get; set; }
 
-            public Request(List<Line> messages, int max_tokens = 4096) {
-                var tokenCountForMessages = JsonWriter.GetWriter(this).Write(messages).Length;
-                if (max_tokens + tokenCountForMessages > 4096) {
-                    max_tokens = 4096 - tokenCountForMessages;
-                }
+            public Request(List<Line> messages) {
                 this.messages = messages;
-                this.max_tokens = max_tokens;
             }
+            
         }
 
         public class Response {
@@ -440,6 +448,14 @@ public static void SetResponseFormatToJsonSchema<T>(this ChatGpt.Request self, T
             self.response_format = ChatGpt.Request.ResponseFormat.NewJsonSchema(schemaName, jsonSchema);
         }
         
+        public static void SetResponseFormatToJsonSchema<T>(this ChatGpt.Request self) {
+            ModelToJsonSchema schemaGenerator = new ModelToJsonSchema(nullValueHandling: NullValueHandling.Ignore);
+            Type type = typeof(T);
+            string className = type.Name;
+            JsonSchema jsonSchema = schemaGenerator.ToJsonSchema(className, type);
+            self.response_format = ChatGpt.Request.ResponseFormat.NewJsonSchema(className, jsonSchema);
+        }
+        
         public static JsonSchema CreateJsonSchema<T>(T exampleResponse) {
             var schemaGenerator = new ModelToJsonSchema(nullValueHandling: NullValueHandling.Ignore);
             var className = typeof(T).Name;
diff --git a/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/RegexTemplates.cs b/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/RegexTemplates.cs
@@ -46,10 +46,10 @@ public static class RegexTemplates {
 
     public static class RegexUtil {
 
-        private static Regex camelCaseSplitter = new Regex(@"(\B[A-Z]+?(?=[A-Z][^A-Z])|\B[A-Z]+?(?=[^A-Z]))");
+        private static Regex camelCaseSplitter = new Regex(@"(?<!^)(?=[A-Z][a-z])|(?<=[a-z])(?=[A-Z])|(?<=[A-Za-z])(?=[0-9])|(?<=[0-9])(?=[A-Za-z])", RegexOptions.Compiled);
 
         public static string SplitCamelCaseString(string camelCaseString) {
-            return camelCaseSplitter.Replace(camelCaseString, " $1").ToFirstCharUpperCase();
+            return camelCaseSplitter.Replace(camelCaseString, " $0").ToFirstCharUpperCase();
         }
 
         /// <summary> Combines multiple regex via AND </summary>
diff --git a/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/ecs/ExpensiveRegularAsyncTaskInBackground.cs b/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/ecs/ExpensiveRegularAsyncTaskInBackground.cs
@@ -36,6 +36,20 @@ public Task SetTaskFor(string contextId, Action taskToDoInBackground) {
                 return Task.CompletedTask;
             }
         }
+
+        public async Task WaitTillDone(int millisecondsDelay = 10) {
+            while (!runningTasks.IsEmpty || !waitingTasks.IsEmpty) {
+                var tasksToWaitFor = runningTasks.Values;
+                if (tasksToWaitFor.IsEmpty()) {
+                    // If there are no running tasks but still waiting tasks,
+                    // a short delay is needed to allow the system to schedule the next task.
+                    await TaskV2.Delay(millisecondsDelay);
+                } else {
+                    await Task.WhenAll(tasksToWaitFor);
+                }
+            }
+        }
+
     }
 
 }
diff --git a/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/ecs/TemplatesIO.cs b/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/ecs/TemplatesIO.cs
@@ -92,6 +92,8 @@ public Task SaveChanges(T instance) {
             });
         }
 
+        public Task WaitForAllSavedChanges(int millisecondsDelay = 10) { return _taskQueue.WaitTillDone(millisecondsDelay); }
+
         // [Conditional("DEBUG")]
         // private void LogSaveChangesIfThereIsAQueue(T instance) {
         //     if (_taskQueue.GetRemainingScheduledTaskCount() % 50 == 0) {
diff --git a/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/jsonschema/BaseJsonSchemaToView.cs b/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/jsonschema/BaseJsonSchemaToView.cs
@@ -63,8 +63,8 @@ public async Task<V> AddViewForJsonSchemaField(V parentView, JsonSchema field, s
             }
             if (type == JTokenType.Array) {
                 var e = field.items;
-                if (e.Count == 1) {
-                    JsonSchema item = e.First();
+                if (e.anyOf.Count == 1) {
+                    JsonSchema item = e.anyOf.First();
                     var childJType = item.GetJTokenType();
                     if (schemaGenerator.IsSimpleType(childJType)) {
                         return await AddAndInit(parentView, field, fieldName, await NewListFieldView(field));
diff --git a/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/jsonschema/JsonSchema.cs b/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/jsonschema/JsonSchema.cs
@@ -1,6 +1,6 @@
 using Newtonsoft.Json;
-using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace com.csutil.model.jsonschema {
 
@@ -64,7 +64,7 @@ public class JsonSchema {
 
         /// <summary> If the field is an object it has a view model itself, see also
         /// https://json-schema.org/understanding-json-schema/reference/array.html#items </summary>
-        public List<JsonSchema> items;
+        public Items items;
 
         /// <summary> If true items is a set so it can only contain unique items, see also
         /// https://json-schema.org/understanding-json-schema/reference/array.html#uniqueness </summary>
@@ -79,10 +79,23 @@ public class JsonSchema {
         /// https://json-schema.org/understanding-json-schema/reference/generic.html#enumerated-values </summary>
         [JsonProperty("enum")]
         public string[] contentEnum;
-        
+
         public bool additionalProperties;
 
-        public static string ToTitle(string varName) { return RegexUtil.SplitCamelCaseString(varName); }
+        public static string ToTitle(string varName) {
+            var words = RegexUtil.SplitCamelCaseString(varName)
+                .Split(' ').SelectMany(x => x.Split('_'))
+                .Where(x => !x.IsNullOrEmpty())
+                .Select(x => char.ToUpper(x[0]) + x.Substring(1));
+            return string.Join(" ", words);
+        }
+
+    }
+
+    public class Items {
+
+        /// <summary> The instance is valid if it matches at least one of the listed schemas. Multiple matches are fine </summary>
+        public List<JsonSchema> anyOf;
 
     }
 
diff --git a/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/jsonschema/ModelToJsonSchema.cs b/CsCore/PlainNetClassLib/src/Plugins/CsCore/com/csutil/model/jsonschema/ModelToJsonSchema.cs
diff --git a/CsCore/xUnitTests/src/Plugins/CsCoreXUnitTests/com/csutil/tests/http/apis/OpenAiTests.cs b/CsCore/xUnitTests/src/Plugins/CsCoreXUnitTests/com/csutil/tests/http/apis/OpenAiTests.cs
diff --git a/CsCore/xUnitTests/src/Plugins/CsCoreXUnitTests/com/csutil/tests/model/esc/EntityComponentSystemTests.cs b/CsCore/xUnitTests/src/Plugins/CsCoreXUnitTests/com/csutil/tests/model/esc/EntityComponentSystemTests.cs
diff --git a/CsCore/xUnitTests/src/Plugins/CsCoreXUnitTests/com/csutil/tests/model/jsonschema/JsonSchemaTests.cs b/CsCore/xUnitTests/src/Plugins/CsCoreXUnitTests/com/csutil/tests/model/jsonschema/JsonSchemaTests.cs

Original file line number	Diff line number	Diff line change
`@@ -46,10 +46,10 @@ public static class RegexTemplates {`
`46`	`46`
`47`	`47`	`public static class RegexUtil {`
`48`	`48`
`49`		`- private static Regex camelCaseSplitter = new Regex(@"(\B[A-Z]+?(?=[A-Z][^A-Z])\|\B[A-Z]+?(?=[^A-Z]))");`
	`49`	`+ private static Regex camelCaseSplitter = new Regex(@"(?<!^)(?=[A-Z][a-z])\|(?<=[a-z])(?=[A-Z])\|(?<=[A-Za-z])(?=[0-9])\|(?<=[0-9])(?=[A-Za-z])", RegexOptions.Compiled);`
`50`	`50`
`51`	`51`	`public static string SplitCamelCaseString(string camelCaseString) {`
`52`		`- return camelCaseSplitter.Replace(camelCaseString, " $1").ToFirstCharUpperCase();`
	`52`	`+ return camelCaseSplitter.Replace(camelCaseString, " $0").ToFirstCharUpperCase();`
`53`	`53`	`}`
`54`	`54`
`55`	`55`	`/// <summary> Combines multiple regex via AND </summary>`
Original file line number	Diff line number	Diff line change
`@@ -36,6 +36,20 @@ public Task SetTaskFor(string contextId, Action taskToDoInBackground) {`
`36`	`36`	`return Task.CompletedTask;`
`37`	`37`	`}`
`38`	`38`	`}`
	`39`	`+`
	`40`	`+ public async Task WaitTillDone(int millisecondsDelay = 10) {`
	`41`	`+ while (!runningTasks.IsEmpty \|\| !waitingTasks.IsEmpty) {`
	`42`	`+ var tasksToWaitFor = runningTasks.Values;`
	`43`	`+ if (tasksToWaitFor.IsEmpty()) {`
	`44`	`+ // If there are no running tasks but still waiting tasks,`
	`45`	`+ // a short delay is needed to allow the system to schedule the next task.`
	`46`	`+ await TaskV2.Delay(millisecondsDelay);`
	`47`	`+ } else {`
	`48`	`+ await Task.WhenAll(tasksToWaitFor);`
	`49`	`+ }`
	`50`	`+ }`
	`51`	`+ }`
	`52`	`+`
`39`	`53`	`}`
`40`	`54`
`41`	`55`	`}`
Original file line number	Diff line number	Diff line change
`@@ -92,6 +92,8 @@ public Task SaveChanges(T instance) {`
`92`	`92`	`});`
`93`	`93`	`}`
`94`	`94`
	`95`	`+ public Task WaitForAllSavedChanges(int millisecondsDelay = 10) { return _taskQueue.WaitTillDone(millisecondsDelay); }`
	`96`	`+`
`95`	`97`	`// [Conditional("DEBUG")]`
`96`	`98`	`// private void LogSaveChangesIfThereIsAQueue(T instance) {`
`97`	`99`	`// if (_taskQueue.GetRemainingScheduledTaskCount() % 50 == 0) {`