build and test fixes

Author: RJKeevil

Dockerfile

@@ -2,7 +2,7 @@
 
 ARG GO_VERSION=1.25.5
 ARG ONNXRUNTIME_VERSION=1.23.2
-ARG ONNXRUNTIME_GENAI_VERSION=0.11.14
+ARG ONNXRUNTIME_GENAI_VERSION=0.11.4
 ARG GOPJRT_VERSION=0.83.1
 ARG BUILD_PLATFORM=linux/amd64
 
@@ -12,12 +12,14 @@ ARG BUILD_PLATFORM=linux/amd64
 FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS hugot-runtime
 ARG GO_VERSION
 ARG ONNXRUNTIME_VERSION
+ARG ONNXRUNTIME_GENAI_VERSION
 ARG GOPJRT_VERSION
 
 ENV PATH="$PATH:/usr/local/go/bin" \
     GOPJRT_NOSUDO=1
 
 COPY ./scripts/download-onnxruntime.sh /download-onnxruntime.sh
+COPY ./scripts/download-onnxruntime-genai.sh /download-onnxruntime-genai.sh
 RUN --mount=src=./go.mod,dst=/go.mod \
     dnf --allowerasing -y install gcc jq bash tar xz gzip glibc-static libstdc++ wget zip git dirmngr sudo which && \
     ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so && \

README.md

@@ -82,7 +82,7 @@ To use Hugot as a library in your application, you can directly import it and fo
 
 #### Backends
 
-- if using Onnx Runtime, the onnxruntime.so file should be obtained from the releases section of this page. If you want to use alternative architectures from `linux/amd64` you will have to download it from [the ONNX Runtime releases page](https://github.com/microsoft/onnxruntime/releases/), see the [dockerfile](./Dockerfile) as an example. Hugot looks for this file at /usr/lib/onnxruntime.so or /usr/lib64/onnxruntime.so by default. A different location can be specified by passing the `WithOnnxLibraryPath()` option to `NewORTSession()`, e.g:
+- if using Onnx Runtime, the onnxruntime.so file should be obtained from the releases section of this page. If you want to use alternative architectures from `linux/amd64` you will have to download it from [the ONNX Runtime releases page](https://github.com/microsoft/onnxruntime/releases/), see the [dockerfile](./Dockerfile) as an example. Hugot looks for this file at /usr/lib/libonnxruntime.so by default. A different location can be specified by passing the `WithOnnxLibraryPath()` option to `NewORTSession()`, e.g:
 
 ```
 session, err := NewORTSession(
@@ -274,12 +274,11 @@ To use Hugot with Nvidia gpu acceleration, you need to have the following:
 
 - The Nvidia driver for your graphics card (if running in Docker and WSL2, starting with --gpus all should inherit the drivers from the host OS)
 - ONNX Runtime:
-    - The cuda gpu version of ONNX Runtime on the machine/docker container. You can see how we get that by looking at the [Dockerfile](./Dockerfile). You can also get the ONNX Runtime libraries that we use for testing from the release. Just download the gpu .so libraries and put them in /usr/lib64.
+    - The cuda gpu version of ONNX Runtime on the machine/docker container. You can see how we get that by looking at the [Dockerfile](./Dockerfile). You can also get the ONNX Runtime libraries that we use for testing from the release. Just download the gpu .so libraries and put them in /usr/lib.
     - The required CUDA libraries installed on your system that are compatible with the ONNX Runtime gpu version you use. See [here](https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html). For instance, for onnxruntime-gpu 19.0, we need CUDA 12.x (any minor version should be compatible) and cuDNN 9.x.
     - Start a session with the following:
       ```
       opts := []options.WithOption{
-        options.WithOnnxLibraryPath("/usr/lib64/onnxruntime-gpu/libonnxruntime.so"),
         options.WithCuda(map[string]string{
           "device_id": "0",
         }),

backends/model_ort.go

@@ -37,35 +37,35 @@ func mapORTOptions(options *options.Options) ([]string, map[string]map[string]st
 
 	// CUDA
 	if ortOptions.CudaOptions != nil {
-		providers = append(providers, "NvTensorRtRtxExecutionProvider")
-		providerOptions["NvTensorRtRtxExecutionProvider"] = ortOptions.CudaOptions
+		providers = append(providers, "cuda")
+		providerOptions["cuda"] = ortOptions.CudaOptions
 	}
 
 	// CoreML
 	if ortOptions.CoreMLOptions != nil {
-		providers = append(providers, "CoreMLExecutionProvider")
-		providerOptions["CoreMLExecutionProvider"] = ortOptions.CoreMLOptions
+		providers = append(providers, "CoreML")
+		providerOptions["CoreML"] = ortOptions.CoreMLOptions
 	}
 
 	// DirectML
 	if ortOptions.DirectMLOptions != nil {
-		providers = append(providers, "DirectMLExecutionProvider")
+		providers = append(providers, "DML")
 		// Map device id to string map expected by advanced session
-		providerOptions["DirectMLExecutionProvider"] = map[string]string{
+		providerOptions["DML"] = map[string]string{
 			"device_id": strconv.Itoa(*ortOptions.DirectMLOptions),
 		}
 	}
 
 	// OpenVINO
 	if ortOptions.OpenVINOOptions != nil {
-		providers = append(providers, "OpenVINOExecutionProvider")
-		providerOptions["OpenVINOExecutionProvider"] = ortOptions.OpenVINOOptions
+		providers = append(providers, "OpenVINO")
+		providerOptions["OpenVINO"] = ortOptions.OpenVINOOptions
 	}
 
 	// TensorRT
 	if ortOptions.TensorRTOptions != nil {
-		providers = append(providers, "TensorRTExecutionProvider")
-		providerOptions["TensorRTExecutionProvider"] = ortOptions.TensorRTOptions
+		providers = append(providers, "NvTensorRtRtx")
+		providerOptions["NvTensorRtRtx"] = ortOptions.TensorRTOptions
 	}
 	return providers, providerOptions, nil
 }

cmd/main_test.go

@@ -15,8 +15,6 @@ import (
 	"github.com/knights-analytics/hugot/util/fileutil"
 )
 
-const onnxRuntimeSharedLibrary = "/usr/lib64/onnxruntime.so"
-
 func TestTextClassificationCli(t *testing.T) {
 	app := &cli.Command{
 		Name:     "hugot",
@@ -43,7 +41,6 @@ func TestTextClassificationCli(t *testing.T) {
 
 	args := append(baseArgs, "run",
 		fmt.Sprintf("--input=%s", testDataDir),
-		fmt.Sprintf("--s=%s", onnxRuntimeSharedLibrary),
 		fmt.Sprintf("--model=%s", testModel),
 		"--type=textClassification")
 	if err := app.Run(context.Background(), args); err != nil {
@@ -73,7 +70,6 @@ func TestTokenClassificationCli(t *testing.T) {
 
 	args := append(baseArgs, "run",
 		fmt.Sprintf("--input=%s", path.Join(testDataDir, "test-token-classification.jsonl")),
-		fmt.Sprintf("--s=%s", onnxRuntimeSharedLibrary),
 		fmt.Sprintf("--model=%s", testModel),
 		"--type=tokenClassification",
 		fmt.Sprintf("--output=%s", testDataDir))
@@ -105,7 +101,6 @@ func TestFeatureExtractionCli(t *testing.T) {
 
 	args := append(baseArgs, "run",
 		fmt.Sprintf("--input=%s", path.Join(testDataDir, "test-feature-extraction.jsonl")),
-		fmt.Sprintf("--s=%s", onnxRuntimeSharedLibrary),
 		fmt.Sprintf("--model=%s", testModel),
 		fmt.Sprintf("--onnxFilename=%s", "model.onnx"),
 		"--type=featureExtraction",
@@ -146,7 +141,6 @@ func TestModelChain(t *testing.T) {
 	// try to download the model to hugo folder and run it
 	args := append(baseArgs, "run",
 		fmt.Sprintf("--input=%s", testDataDir),
-		fmt.Sprintf("--s=%s", onnxRuntimeSharedLibrary),
 		fmt.Sprintf("--model=%s", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english"),
 		"--type=textClassification")
 	if err := app.Run(context.Background(), args); err != nil {
@@ -156,7 +150,6 @@ func TestModelChain(t *testing.T) {
 	// run it again. This time the model should be read from the hugot folder without re-downloading it.
 	args = append(baseArgs, "run", fmt.Sprintf("--input=%s", testDataDir),
 		fmt.Sprintf("--model=%s", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english"),
-		fmt.Sprintf("--s=%s", onnxRuntimeSharedLibrary),
 		"--type=textClassification")
 	if err := app.Run(context.Background(), args); err != nil {
 		check(t, err)

cuda.Dockerfile

@@ -2,6 +2,7 @@
 
 ARG GO_VERSION=1.25.5
 ARG ONNXRUNTIME_VERSION=1.23.2
+ARG ONNXRUNTIME_GENAI_VERSION=0.11.4
 ARG GOPJRT_VERSION=0.83.1
 ARG JAX_CUDA_VERSION=0.8.1
 ARG BUILD_PLATFORM=linux/amd64
@@ -11,13 +12,15 @@ ARG BUILD_PLATFORM=linux/amd64
 FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS hugot-runtime
 ARG GO_VERSION
 ARG ONNXRUNTIME_VERSION
+ARG ONNXRUNTIME_GENAI_VERSION
 ARG GOPJRT_VERSION
 ARG JAX_CUDA_VERSION
 
 ENV PATH="$PATH:/usr/local/go/bin" \
     GOPJRT_NOSUDO=1
 
-COPY ./scripts/download-onnxruntime-gpu.sh /download-onnxruntime-gpu.sh
+COPY ./scripts/download-onnxruntime.sh /download-onnxruntime.sh
+COPY ./scripts/download-onnxruntime-genai.sh /download-onnxruntime-genai.sh
 RUN --mount=src=./go.mod,dst=/go.mod \
     dnf --allowerasing -y install gcc jq bash tar xz gzip glibc-static libstdc++ wget zip git dirmngr sudo which && \
     ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so && \

downloader.go

@@ -68,7 +68,7 @@ func DownloadModel(modelName string, destination string, options DownloadOptions
 	}
 
 	// make sure it's an onnx model with tokenizer
-	downloadFiles, err := validateDownloadHfModel(repo, options)
+	downloadFiles, err := validateDownloadedHFModel(repo, options)
 	if err != nil {
 		return "", err
 	}
@@ -104,7 +104,7 @@ func DownloadModel(modelName string, destination string, options DownloadOptions
 	return "", fmt.Errorf("failed to download %s after %d attempts", modelName, options.MaxRetries)
 }
 
-func validateDownloadHfModel(repo *hub.Repo, options DownloadOptions) ([]string, error) {
+func validateDownloadedHFModel(repo *hub.Repo, options DownloadOptions) ([]string, error) {
 	for i := 0; i < options.MaxRetries; i++ {
 		err := repo.DownloadInfo(false)
 		if err != nil {

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/gomlx/go-huggingface v0.3.1
 	github.com/gomlx/gomlx v0.26.0-rc2
 	github.com/gomlx/onnx-gomlx v0.3.4-rc0.0.20251216070438-403f89ce7f75
-	github.com/knights-analytics/ortgenai v0.0.0-20251215144038-8c17ce42e8d5
+	github.com/knights-analytics/ortgenai v0.0.0-20251217113036-5abc6d6b37b1
 	github.com/stretchr/testify v1.11.1
 	github.com/sugarme/tokenizer v0.3.0
 	github.com/urfave/cli/v3 v3.6.1
@@ -17,8 +17,6 @@ require (
 	golang.org/x/image v0.34.0
 )
 
-replace github.com/knights-analytics/ortgenai => /home/rpinosio/repositories/knights/ortgenai
-
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect

go.sum

@@ -41,6 +41,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/janpfeifer/go-benchmarks v0.1.1 h1:gLLy07/JrOKSnMWeUxSnjTdhkglgmrNR2IBDnR4kRqw=
 github.com/janpfeifer/go-benchmarks v0.1.1/go.mod h1:5AagXCOUzevvmYFQalcgoa4oWPyH1IkZNckolGWfiSM=
+github.com/knights-analytics/ortgenai v0.0.0-20251217113036-5abc6d6b37b1 h1:1CCq4Rux7SyRm+HG5rWydGeluI1NzBRvgbSj91GoFUg=
+github.com/knights-analytics/ortgenai v0.0.0-20251217113036-5abc6d6b37b1/go.mod h1:zolhST8PioaCCsnwHBUesIBKU4CW/ZrF169YjFfg7BY=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=

hugot_go_test.go

@@ -176,25 +176,28 @@ func TestObjectDetectionPipelineValidationGo(t *testing.T) {
 }
 
 // text generation
-// func TestTextGenerationPipelineGo(t *testing.T) {
-// 	session, err := NewGoSession()
-// 	checkT(t, err)
-// 	defer func(session *Session) {
-// 		destroyErr := session.Destroy()
-// 		checkT(t, destroyErr)
-// 	}(session)
-// 	textGenerationPipeline(t, session)
-// }
-//
-// func TestTextGenerationPipelineValidationGo(t *testing.T) {
-// 	session, err := NewGoSession()
-// 	checkT(t, err)
-// 	defer func(session *Session) {
-// 		destroyErr := session.Destroy()
-// 		checkT(t, destroyErr)
-// 	}(session)
-// 	textGenerationPipelineValidation(t, session)
-// }
+
+func TestTextGenerationPipelineGo(t *testing.T) {
+	t.Skip("Generative models are not supported yet for Go")
+	session, err := NewGoSession()
+	checkT(t, err)
+	defer func(session *Session) {
+		destroyErr := session.Destroy()
+		checkT(t, destroyErr)
+	}(session)
+	textGenerationPipeline(t, session)
+}
+
+func TestTextGenerationPipelineValidationGo(t *testing.T) {
+	t.Skip("Generative models are not supported yet for Go")
+	session, err := NewGoSession()
+	checkT(t, err)
+	defer func(session *Session) {
+		destroyErr := session.Destroy()
+		checkT(t, destroyErr)
+	}(session)
+	textGenerationPipelineValidation(t, session)
+}
 
 // No same name