surreal: normalize nulls/dates/ids, refactor schema & autoincrement, and update tests

- Add helpers to detect Surreal record IDs and ISO-like datetimes; avoid quoting record IDs and emit datetime literals consistently.
- Use NONE as SurrealDB null representation (VALUE/ASSERT/WHERE emit NONE), and adjust query generator (IS NONE).
- Rename migration/history table to `_allographer_migrations`, add createMigrationTable(isReset) to allow reset/removal, and store created_at in ISO8601 'T...Z' format.
- Rework autoincrement flow: centralize _autoincrement_sequences usage, compute next-index expression, update event/field definitions to use the sequence, and change migration primary column from `id` to `index`.
- Append "RETURN AFTER" to INSERT builders and simplify insertId paths to reliably obtain returned IDs.
- Refactor create_column_query: introduce helpers (addAssertPart/addValueClause/addAssertClause/autoIncrementValueExpr) to centralize value/default/assert logic and improve nullable/default handling across types (string, int, decimal, datetime, json, record).
- Fix/adjust query behaviors (use math::mean for avg), improve avg parsing for multiple JSON numeric kinds, and tighten JSON/record field syntax.
- Add test utilities and env defaults (envIntDefault/envStringDefault, isNullish, normalizeInfoDef) and update tests to match new behavior (table/index renames, null checks, function name changes, datetime formats).
- Miscellaneous SurrealDB compatibility and normalization fixes across impl, exec, schema utils, and tests.

Author: dumblepy

src/allographer/query_builder/libs/surreal/surreal_lib.nim

@@ -2,10 +2,64 @@ import std/strutils
 import std/json
 
 
+proc looksLikeRecordId(s: string): bool =
+  let colonPos = s.find(':')
+  if colonPos <= 0 or colonPos >= s.high:
+    return false
+  if s.find(':', colonPos + 1) >= 0:
+    return false
+  if s.contains(' ') or s.contains('"') or s.contains('\''):
+    return false
+  if not (s[0].isAlphaAscii or s[0] == '_'):
+    return false
+  for ch in s[1 ..< colonPos]:
+    if not (ch.isAlphaNumeric or ch == '_'):
+      return false
+  for ch in s[colonPos + 1 .. ^1]:
+    if ch.isSpaceAscii:
+      return false
+  return true
+
+
+proc looksLikeDateTimeValue(s: string): bool =
+  if s.len < 10:
+    return false
+
+  for idx in [0, 1, 2, 3, 5, 6, 8, 9]:
+    if idx >= s.len or not s[idx].isDigit:
+      return false
+
+  if s[4] != '-' or s[7] != '-':
+    return false
+
+  if s.len == 10:
+    return true
+
+  if s.len < 19:
+    return false
+
+  if s[10] notin {'T', ' '}:
+    return false
+
+  for idx in [11, 12, 14, 15, 17, 18]:
+    if idx >= s.len or not s[idx].isDigit:
+      return false
+
+  if s[13] != ':' or s[16] != ':':
+    return false
+
+  if s.len == 19:
+    return true
+
+  return s[19] in {'.', 'Z', '+', '-'}
+
+
 proc dbQuote(s:string):string =
   ## DB quotes the string.
   if s == "null":
     return "NULL"
+  if looksLikeRecordId(s):
+    return s
   result = newStringOfCap(s.len + 2)
   result.add "'"
   for c in items(s):
@@ -93,14 +147,20 @@ proc appendJsonLetClause(result: var string, idx: int, arg: JsonNode, quoteStrin
     result.add($arg.getFloat)
   of JString:
     let val = arg.getStr().replace("\"", "\\\"")
-    if quoteString:
+    if looksLikeRecordId(val):
+      result.add(val)
+    elif looksLikeDateTimeValue(val):
+      result.add("<datetime>\"")
+      result.add(val)
+      result.add("\"")
+    elif quoteString:
       result.add('"')
       result.add(val)
       result.add('"')
     else:
       result.add(val)
   of JNull:
-    result.add("null")
+    result.add("NONE")
   of JArray, JObject:
     result.add($arg)
   result.add("; ")

src/allographer/query_builder/models/surreal/query/surreal_generator.nim

@@ -214,9 +214,9 @@ proc whereNullSql*(self: SurrealQuery): SurrealQuery =
       column = quote(column)
 
       if self.queryString.contains("WHERE"):
-        self.queryString.add(&" OR {column} IS NULL")
+        self.queryString.add(&" OR {column} IS NONE")
       else:
-        self.queryString.add(&" WHERE {column} IS NULL")
+        self.queryString.add(&" WHERE {column} IS NONE")
 
   return self
 
@@ -375,7 +375,7 @@ proc selectCountSql*(self: SurrealQuery): SurrealQuery =
 proc selectAvgSql*(self: SurrealQuery, column:string): SurrealQuery =
   var column = column
   column = quote(column)
-  self.queryString = &"SELECT math::trimean({column}) AS avg"
+  self.queryString = &"SELECT math::mean({column}) AS avg"
   return self
 
 

src/allographer/query_builder/models/surreal/surreal_exec.nim

@@ -391,18 +391,15 @@ proc insert*(self:SurrealQuery, items:seq[JsonNode]) {.async.} =
 
 proc insertId*(self:SurrealQuery, items:JsonNode, key="id"):Future[SurrealId] {.async.} =
   ## https://surrealdb.com/docs/surrealql/statements/insert
-  let sql = self.insertValueBuilder(items)
+  var sql = self.insertValueBuilder(items) & " RETURN AFTER"
   self.log.logger(sql)
   let res = self.getRow(sql).await
-  if res.isSome():
-    return SurrealId.new(res.get()[key].getStr())
-  else:
-    return SurrealId.new()
+  return SurrealId.new(res.get()[key].getStr)
 
 
 proc insertId*(self: SurrealQuery, items: seq[JsonNode], key="id"):Future[seq[SurrealId]] {.async.} =
   result = newSeq[SurrealId](items.len)
-  var sql = self.insertValuesBuilder(items)
+  var sql = self.insertValuesBuilder(items) & " RETURN AFTER"
   self.log.logger(sql)
   let res = self.getAllRows(sql).await
   var i = 0
@@ -431,21 +428,18 @@ proc insert*[T](self:SurrealQuery, items:seq[T]) {.async.} =
 
 proc insertId*[T](self:SurrealQuery, items:T, key="id"):Future[SurrealId] {.async.} =
   ## https://surrealdb.com/docs/surrealql/statements/insert
-  let sql = self.insertValueBuilder(%items)
+  var sql = self.insertValueBuilder(%items) & " RETURN AFTER"
   self.log.logger(sql)
   let res = self.getRow(sql).await
-  if res.isSome():
-    return SurrealId.new(res.get()[key].getStr())
-  else:
-    return SurrealId.new()
+  return SurrealId.new(res.get()[key].getStr)
 
 
 proc insertId*[T](self: SurrealQuery, items: seq[T], key="id"):Future[seq[SurrealId]] {.async.} =
   result = newSeq[SurrealId](items.len)
   var jsonItems = newSeq[JsonNode](items.len)
   for i, item in items:
     jsonItems[i] = %item
-  var sql = self.insertValuesBuilder(jsonItems)
+  var sql = self.insertValuesBuilder(jsonItems) & " RETURN AFTER"
   self.log.logger(sql)
   let res = self.getAllRows(sql).await
   var i = 0
@@ -573,7 +567,16 @@ proc avg*(self:SurrealQuery, column:string):Future[float]{.async.} =
   self.log.logger(sql)
   let response =  await self.getRow(sql)
   if response.isSome:
-    return response.get["avg"].getStr().parseFloat()
+    let value = response.get["avg"]
+    case value.kind
+    of JInt:
+      return value.getInt.float
+    of JFloat:
+      return value.getFloat()
+    of JString:
+      return value.getStr().parseFloat()
+    else:
+      return 0.0
   else:
     return 0.0
 

src/allographer/schema_builder/queries/surreal/create_migration_table.nim

@@ -13,7 +13,7 @@ import ./surreal_query_type
 import ./sub/create_column_query
 
 
-proc createMigrationTable*(self: SurrealSchema) =
+proc createMigrationTable*(self: SurrealSchema, isReset: bool = false) =
   let logDisplay = self.rdb.log.shouldDisplayLog
   let logFile = self.rdb.log.shouldOutputLogFile
   self.rdb.log.shouldDisplayLog = false
@@ -23,7 +23,12 @@ proc createMigrationTable*(self: SurrealSchema) =
     self.rdb.log.shouldOutputLogFile = logFile
 
   let info = self.rdb.raw("INFO FOR DB").info().waitFor()
-  if not info[0]["result"]["tables"].contains("_autoincrement_migrations"):
+  var hasMigrationTable = info[0]["result"]["tables"].contains("_allographer_migrations")
+  if isReset and hasMigrationTable:
+    self.rdb.raw("REMOVE TABLE `_allographer_migrations`").exec().waitFor()
+    hasMigrationTable = false
+
+  if not hasMigrationTable:
     var queries:seq[string]
     queries.add(&"DEFINE TABLE `{self.table.name}` SCHEMAFULL")
 

src/allographer/schema_builder/queries/surreal/schema_utils.nim

@@ -75,7 +75,7 @@ proc execThenSaveHistory*(rdb:SurrealConnections, tableName:string, queries:seq[
     rdb.log.shouldOutputLogFile = logFile
 
   let tableQuery = queries.join("; ")
-  let createdAt = now().utc.format("yyyy-MM-dd HH:mm:ss'.'fff")
+  let createdAt = now().utc.format("yyyy-MM-dd'T'HH:mm:ss'.'fff'Z'")
   rdb.table("_allographer_migrations").insert(%*{
     "name": tableName,
     "query": tableQuery,
@@ -107,7 +107,7 @@ proc execThenSaveHistory*(rdb:SurrealConnections, tableName:string, query:string
     rdb.log.shouldDisplayLog = logDisplay
     rdb.log.shouldOutputLogFile = logFile
 
-  let createdAt = now().utc.format("yyyy-MM-dd HH:mm:ss'.'fff")
+  let createdAt = now().utc.format("yyyy-MM-dd'T'HH:mm:ss'.'fff'Z'")
   rdb.table("_allographer_migrations").insert(%*{
     "name": tableName,
     "query": query,