add: replace std/base64 with simdutf

Fixes #37

Author: xTrayambak

ferus.nimble

@@ -17,7 +17,7 @@ requires "stylus >= 0.1.3"
 requires "https://github.com/ferus-web/sanchar >= 2.0.2"
 requires "https://git.sr.ht/~bptato/chame >= 1.0.1"
 requires "seccomp >= 0.2.1"
-requires "simdutf >= 5.5.0"
+requires "simdutf >= 6.5.1"
 requires "https://github.com/ferus-web/bali#master"
 requires "results >= 0.5.0"
 requires "pretty >= 0.1.0"

src/components/ipc/shared.nim

@@ -1,4 +1,5 @@
-import std/[times, net, logging, options, base64]
+import std/[times, net, logging, options]
+import pkg/simdutf/base64
 
 proc `*`[T](opt: Option[T]): bool {.inline, noSideEffect, gcsafe.} =
   # dumb hacks to make code look less yucky
@@ -336,8 +337,8 @@ type
 proc `==`*(a, b: FerusProcess): bool {.inline.} =
   a.worker == b.worker and a.kind == b.kind and a.socket == b.socket
 
-func content*(packet: DataTransferResult): string {.inline.} =
-  packet.data.decode()
+proc content*(packet: DataTransferResult): string {.inline.} =
+  packet.data.decode(urlSafe = true)
 
 proc magicFromStr*(s: string): Option[FerusMagic] =
   case s

src/components/js/process.nim

@@ -1,15 +1,18 @@
-import std/[logging, json, base64, net]
-import ../../components/ipc/client/prelude
-import bali/grammar/prelude
-import bali/runtime/prelude
-import bali/stdlib/console
-import jsony
-import ../../components/shared/[nix, sugar]
-import ../../components/web/[window]
-import ../../components/web/document as jsdoc
-import ../../components/web/websockets as jswebsocket
+import std/[logging, json, net]
+import
+  pkg/jsony,
+  pkg/simdutf/base64,
+  pkg/bali/grammar/prelude,
+  pkg/bali/runtime/prelude,
+  pkg/bali/stdlib/console
+import
+  ../../components/shared/[nix, sugar],
+  ../../components/ipc/client/prelude,
+  ../../components/web/[window],
+  ../../components/web/document as jsdoc,
+  ../../components/web/websockets as jswebsocket,
+  ./ipc
 from ../../components/parsers/html/document import HTMLDocument
-import ./ipc
 
 type JSProcess* = object
   ipc*: IPCClient
@@ -32,8 +35,8 @@ proc jsExecBuffer*(js: var JSProcess, data: string) =
   js.ipc.setState(Processing)
   let data = &tryParseJson(data, JSExecPacket)
 
-  js.parser = newParser(data.buffer.decode())
-  js.runtime = newRuntime(data.name.decode(), js.parser.parse())
+  js.parser = newParser(data.buffer.decode(urlSafe = true))
+  js.runtime = newRuntime(data.name.decode(urlSafe = true), js.parser.parse())
   window.generateIR(js.runtime, js.ipc)
   jsdoc.generateIR(js.runtime)
   jswebsocket.generateBindings(js.runtime, js.ipc)

src/components/master/master.nim

@@ -2,25 +2,20 @@
 ## It essentially allows you to do anything with your own group of processes (renderer, JS runtime, CSS/HTML parsers, etc.)
 ## This includes summoning them, telling them to do a task, telling them to exit, et cetera.
 
+import std/[os, logging, osproc, strutils, options, net, sets, terminal, posix, tables]
 import
-  std/
-    [os, logging, osproc, strutils, options, base64, net, sets, terminal, posix, tables]
-import ../../components/ipc/server/prelude
-import jsony
-import ./summon
-import pretty
+  pkg/[jsony, pretty],
+  pkg/sanchar/parse/url,
+  pkg/sanchar/proto/http/shared,
+  pkg/simdutf/base64
 
-# Process specific imports
-
-# Network
-import sanchar/parse/url
-import sanchar/proto/http/shared
-
-import ../../components/shared/[nix, sugar]
 import
+  ../../components/shared/[nix, sugar],
   ../../components/
-    [network/ipc, renderer/ipc, parsers/html/ipc, parsers/html/document, js/ipc]
-import ../../components/web/cookie/parsed_cookie
+    [network/ipc, renderer/ipc, parsers/html/ipc, parsers/html/document, js/ipc],
+  ../../components/web/cookie/parsed_cookie,
+  ../../components/ipc/server/prelude,
+  ./summon
 
 when defined(unix):
   import std/posix
@@ -139,7 +134,9 @@ proc parseHTML*(
 
   info ("Sending group $1 HTML parser process a request to parse some HTML" % [$group])
 
-  master.server.send((&process).socket, ParseHTMLPacket(source: encode(source)))
+  master.server.send(
+    (&process).socket, ParseHTMLPacket(source: encode(source, urlSafe = true))
+  )
 
   info ("Waiting for response from group $1 HTML parser process" % [$group])
 
@@ -217,7 +214,8 @@ proc executeJS*(
   var prc = &process
   assert prc.kind == JSRuntime
   master.server.send(
-    prc.socket, JSExecPacket(name: name.encode(), buffer: code.encode())
+    prc.socket,
+    JSExecPacket(name: name.encode(urlSafe = true), buffer: code.encode(urlSafe = true)),
   )
 
   info "Dispatching JS code to runtime process."
@@ -236,7 +234,7 @@ proc setWindowTitle*(master: MasterProcess, title: string) {.inline.} =
   master.server.send(
     (&process).socket,
     RendererSetWindowTitle(
-      title: title.encode(safe = true) # So that we can get spaces
+      title: title.encode(urlSafe = true) # So that we can get spaces
     ),
   )
 
@@ -288,7 +286,7 @@ proc loadFont*(
   let encoded = encode(
     # encode the data in base64 to ensure that it doesn't mess up the JSON packet
     readFile file,
-    safe = true,
+    urlSafe = true,
   )
   master.server.send(
     (&process).socket,

src/components/network/ipc.nim

@@ -1,6 +1,11 @@
-import std/[base64, options]
-import ../../components/shared/[sugar]
-import sanchar/parse/url, sanchar/proto/http/shared, ../../components/ipc/shared
+#!fmt: off
+import std/[options]
+import pkg/sanchar/parse/url,
+       pkg/sanchar/proto/http/shared,
+       pkg/simdutf/base64
+import ../../components/ipc/shared,
+       ../../components/shared/[sugar]
+#!fmt: on
 
 type
   NetworkFetchPacket* = object
@@ -20,8 +25,8 @@ type
     kind*: FerusMagic = feNetworkWebSocketCreationResult
     error*: Option[string]
 
-func content*(res: NetworkFetchResult): string {.inline.} =
+proc content*(res: NetworkFetchResult): string {.inline.} =
   if !res.response:
     return
 
-  (&res.response).content.decode()
+  (&res.response).content.decode(urlSafe = true)

src/components/network/process.nim

@@ -1,17 +1,19 @@
-import std/[base64, strutils, sequtils, importutils, logging, options, os, json, net]
-import pkg/sanchar/[http, proto/http/shared], pkg/sanchar/parse/url
-import pkg/pretty
-import pkg/whisky
+import std/[strutils, sequtils, importutils, logging, options, os, json, net]
+import
+  pkg/[pretty, whisky, jsony],
+  pkg/sanchar/[http, proto/http/shared],
+  pkg/sanchar/parse/url,
+  pkg/simdutf/base64
 
 when defined(ferusUseCurl):
-  import pkg/webby/httpheaders
-  import pkg/curly
+  #!fmt:off
+  import pkg/webby/httpheaders, pkg/curly #!fmt:on
 
-import pkg/jsony
-import ../../components/shared/[nix, sugar]
-import ../../components/network/[websocket, types, ipc]
-import ../../components/build_utils
-import ../../components/ipc/client/prelude
+import
+  ../../components/shared/[nix, sugar],
+  ../../components/network/[websocket, types, ipc],
+  ../../components/build_utils,
+  ../../components/ipc/client/prelude
 
 privateAccess(WebSocket)
 
@@ -125,7 +127,7 @@ proc talk(client: FerusNetworkClient, process: FerusProcess) {.inline.} =
       return
 
     var resp = &odata.response
-    resp.content = resp.content.encode()
+    resp.content = resp.content.encode(urlSafe = true)
 
     odata.response = some(move(resp))
 

src/components/parsers/html/document.nim

@@ -1,11 +1,10 @@
 ## I love chame
 
-import std/[options, logging, strutils, tables, base64]
-import chagashi/charset
-import sanchar/parse/url
-import ../../shared/sugar
-import ../../web/dom
-import pretty
+#!fmt:off
+import std/[options, logging, strutils, tables]
+import pkg/[pretty], pkg/chagashi/charset, pkg/sanchar/parse/url, pkg/simdutf/base64
+import ../../shared/sugar, ../../web/dom
+#!fmt:on
 
 type
   HTMLElement* = ref object
@@ -55,15 +54,15 @@ func findAll*(
 
   res
 
-func text*(elem: HTMLElement): Option[string] {.inline.} =
+proc text*(elem: HTMLElement): Option[string] {.inline.} =
   if *elem.text:
-    return some(decode(&elem.text))
+    return some(decode(&elem.text, urlSafe = true))
 
-func attribute*(elem: HTMLElement, name: string): Option[string] {.inline.} =
+proc attribute*(elem: HTMLElement, name: string): Option[string] {.inline.} =
   if name in elem.attributes:
-    return some(decode(elem.attributes[name]))
+    return some(decode(elem.attributes[name], urlSafe = true))
 
-func classes*(elem: HTMLElement): seq[string] {.inline.} =
+proc classes*(elem: HTMLElement): seq[string] {.inline.} =
   let classAttr = elem.attribute("class")
   if !classAttr:
     return
@@ -79,7 +78,7 @@ proc parseHTMLElement*(document: Document, element: Element): HTMLElement =
   for (prefix, namespace, name, value) in element.attrs:
     # we currently don't really care about the namespace
     let mappedName = document.factory.atomToStr(name)
-    elem.attributes[mappedName] = encode(value)
+    elem.attributes[mappedName] = encode(value, urlSafe = true)
 
   case tag
   of TAG_P,
@@ -97,7 +96,7 @@ proc parseHTMLElement*(document: Document, element: Element): HTMLElement =
     for txt in element.textNodes:
       text &= txt.data
 
-    elem.text = some(encode(text))
+    elem.text = some(encode(text, urlSafe = true))
   else:
     discard
 

src/components/parsers/html/process.nim

@@ -1,10 +1,10 @@
-import std/[base64, options, json, logging, monotimes, net, os]
-import ./ipc
-import ./document
-import ../../web/dom
-import ../../shared/[nix, sugar]
-import ../../../components/ipc/client/prelude
-import jsony
+import std/[options, json, logging, monotimes, net, os]
+import pkg/jsony, pkg/simdutf/base64
+import
+  ./[document, ipc],
+  ../../web/dom,
+  ../../shared/[nix, sugar],
+  ../../../components/ipc/client/prelude
 
 proc htmlParse*(oparsingData: Option[ParseHTMLPacket]): HTMLParseResult =
   if !oparsingData:
@@ -17,7 +17,7 @@ proc htmlParse*(oparsingData: Option[ParseHTMLPacket]): HTMLParseResult =
 
   let
     startTime = getMonoTime()
-    document = parseHTML(newStringStream(decode(parsingData.source)))
+    document = parseHTML(newStringStream(decode(parsingData.source, urlSafe = true)))
     endTime = getMonoTime()
 
   info "Parsed HTML in " & $(endTime - startTime)

src/components/renderer/ipc.nim

@@ -1,9 +1,7 @@
-import std/base64
 import ../../components/ipc/shared
 import pkg/vmath
 from pkg/ferusgfx import Scene
-import pkg/ferusgfx/displaylist
-
+import pkg/ferusgfx/displaylist, pkg/simdutf/base64
 import ../parsers/html/document
 
 type
@@ -58,17 +56,24 @@ proc parseHook*(s: string, i: int, v2: ptr Scene) {.inline.} =
 
 proc newTextNode*(content: string, position: Vec2, font: string): Drawable {.inline.} =
   Drawable(
-    kind: TextNode, content: content.encode(safe = true), position: position, font: font
+    kind: TextNode,
+    content: content.encode(urlSafe = true),
+    position: position,
+    font: font,
   )
 
 proc newImageNode*(path: string, position: Vec2): Drawable {.inline.} =
   Drawable(
-    kind: ImageNode, imgContent: path.readFile().encode(safe = true), position: position
+    kind: ImageNode,
+    imgContent: path.readFile().encode(urlSafe = true),
+    position: position,
   )
 
 proc newGIFNode*(path: string, position: Vec2): Drawable {.inline.} =
   Drawable(
-    kind: GIFNode, gifContent: path.readFile().encode(safe = true), position: position
+    kind: GIFNode,
+    gifContent: path.readFile().encode(urlSafe = true),
+    position: position,
   )
 
 proc newDisplayList*(clearAll: bool = false): IPCDisplayList {.inline.} =

src/components/renderer/process.nim

@@ -1,7 +1,8 @@
-import std/[options, json, logging, base64, importutils, logging, posix, net]
-import jsony
-import ferusgfx/[displaylist, fontmgr, textnode, imagenode, gifnode]
-import pixie
+import std/[options, json, logging, importutils, logging, posix, net]
+import
+  pkg/ferusgfx/[displaylist, fontmgr, textnode, imagenode, gifnode],
+  pkg/[jsony, pixie],
+  pkg/simdutf/base64
 when defined(linux):
   import ../../components/sandbox/linux
 
@@ -45,7 +46,7 @@ proc loadFont*(
     font: Option[Font]
 
   try:
-    typeface = some decode(data).readTypeface(packet.format)
+    typeface = some cast[string](decodeSeq(data, urlSafe = true)).parseTtf()
   except PixieError as exc:
     error "Failed to load typeface: " & exc.msg & ": fmt=" & packet.format
     return
@@ -135,7 +136,7 @@ proc talk(
       return
 
     client.setState(Processing)
-    renderer.setWindowTitle((&reinterpreted).title.decode())
+    renderer.setWindowTitle((&reinterpreted).title.decode(urlSafe = true))
     client.setState(Idling)
   of feRendererRenderDocument:
     let reinterpreted = tryParseJson(data, RendererRenderDocument)