Skip to content

Commit 8411fdb

Browse files
committed
update onnxruntime_go, goMLX and gopjrt
1 parent 030c5e5 commit 8411fdb

7 files changed

Lines changed: 62 additions & 113 deletions

File tree

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
ARG GO_VERSION=1.25.5
44
ARG ONNXRUNTIME_VERSION=1.22.0
5-
ARG GOPJRT_VERSION=0.9.1
5+
ARG GOPJRT_VERSION=0.10.0
66
ARG BUILD_PLATFORM=linux/amd64
77

88
#--- runtime layer with all hugot dependencies for cpu

backends/model_gomlx.go

Lines changed: 18 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -306,10 +306,14 @@ func createInputTensorsGoMLX(batch *PipelineBatch, model *Model, padBatchDimensi
306306
batch.InputValues = inputTensors
307307
batch.PaddingMask = paddingMasks
308308
batch.DestroyInputs = func() error {
309+
var err error
309310
for _, t := range inputTensors {
310-
t.FinalizeAll()
311+
tensorErr := t.FinalizeAll()
312+
if err == nil {
313+
err = tensorErr
314+
}
311315
}
312-
return nil
316+
return err
313317
}
314318
return nil
315319
}
@@ -523,76 +527,6 @@ func createGenerativeCallFunc(model *Model, modelParsed *onnx.Model, outputNames
523527
}
524528
}
525529

526-
func RunGenerativeGoMLXSessionOnBatch(batch *PipelineBatch, p *BasePipeline) error {
527-
// Generative models
528-
batchSize := len(batch.Input)
529-
finished := make([]bool, batchSize)
530-
generatedTokens := make([][]int64, batchSize)
531-
532-
var outputTensors []*tensors.Tensor
533-
var err error
534-
defer func() {
535-
for _, t := range outputTensors {
536-
t.FinalizeAll()
537-
}
538-
}()
539-
540-
for step := 0; step < batch.MaxNewTokens; step++ {
541-
if step == 0 {
542-
p.Model.FirstIteration = true
543-
} else {
544-
p.Model.FirstIteration = false
545-
}
546-
modelInputs := batch.InputValues.([]*tensors.Tensor)
547-
548-
outputTensors, err = p.Model.GoMLXModel.Exec.Exec(modelInputs)
549-
if err != nil {
550-
return err
551-
}
552-
553-
genTensor := outputTensors[0]
554-
var flatTokens []int64
555-
tensors.ConstFlatData(genTensor, func(flat []int64) {
556-
flatTokens = flat
557-
})
558-
for i := range batchSize {
559-
if !finished[i] {
560-
generatedTokens[i] = append(generatedTokens[i], flatTokens[i])
561-
}
562-
}
563-
564-
eosMatchTensor := outputTensors[len(outputTensors)-1]
565-
var doneFlags []bool
566-
tensors.ConstFlatData(eosMatchTensor, func(flat []bool) {
567-
doneFlags = flat
568-
})
569-
for i, flag := range doneFlags {
570-
if flag {
571-
finished[i] = true
572-
}
573-
}
574-
575-
allDone := true
576-
for _, isDone := range finished {
577-
if !isDone {
578-
allDone = false
579-
break
580-
}
581-
}
582-
if allDone {
583-
break
584-
}
585-
batch.InputValues = outputTensors
586-
}
587-
588-
batch.OutputValues = make([]any, batchSize)
589-
for i := range generatedTokens {
590-
batch.OutputValues[i] = generatedTokens[i]
591-
}
592-
593-
return nil
594-
}
595-
596530
func runGoMLXSessionOnBatch(batch *PipelineBatch, p *BasePipeline) error {
597531
// Non-generative models
598532
outputTensors, err := p.Model.GoMLXModel.Exec.Exec(batch.InputValues.([]*tensors.Tensor))
@@ -601,21 +535,24 @@ func runGoMLXSessionOnBatch(batch *PipelineBatch, p *BasePipeline) error {
601535
}
602536
defer func() {
603537
for _, t := range outputTensors {
604-
t.FinalizeAll()
538+
err = errors.Join(t.FinalizeAll())
605539
}
606540
}()
607541

608542
convertedOutput := make([]any, len(outputTensors))
609543
for i, t := range outputTensors {
610544
var rawOutput []float32
611-
tensors.ConstFlatData(t, func(flat []float32) {
545+
err = tensors.ConstFlatData(t, func(flat []float32) {
612546
rawOutput = flat
613547
})
548+
if err != nil {
549+
return err
550+
}
614551
convertedOutput[i] = ReshapeOutput(rawOutput, p.Model.OutputsMeta[i], batch.Size, batch.PaddingMask, batch.MaxSequenceLength)
615552
}
616553

617554
batch.OutputValues = convertedOutput
618-
return nil
555+
return err
619556
}
620557

621558
func nextPowerOf2(n int) int {
@@ -669,10 +606,14 @@ func createImageTensorsGoXLA(batch *PipelineBatch, preprocessed [][][][]float32)
669606
inputTensors := []*tensors.Tensor{tensors.FromFlatDataAndDimensions(backing, n, c, h, w)}
670607
batch.InputValues = inputTensors
671608
batch.DestroyInputs = func() error {
609+
var err error
672610
for _, input := range inputTensors {
673-
input.FinalizeAll()
611+
tensorErr := input.FinalizeAll()
612+
if tensorErr != nil {
613+
err = tensorErr
614+
}
674615
}
675-
return nil
616+
return err
676617
}
677618
return nil
678619
}

cuda.Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
ARG GO_VERSION=1.25.5
44
ARG ONNXRUNTIME_VERSION=1.22.0
5-
ARG GOPJRT_VERSION=0.9.1
6-
ARG JAX_CUDA_VERSION=0.8.0
5+
ARG GOPJRT_VERSION=0.10.0
6+
ARG JAX_CUDA_VERSION=0.8.1
77
ARG BUILD_PLATFORM=linux/amd64
88

99
#--- runtime layer with all hugot dependencies for cpu and gpu ---

go.mod

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@ require (
66
github.com/daulet/tokenizers v1.24.0
77
github.com/gomlx/exceptions v0.0.3
88
github.com/gomlx/go-huggingface v0.3.1
9-
github.com/gomlx/gomlx v0.24.1
10-
github.com/gomlx/gopjrt v0.9.1
11-
github.com/gomlx/onnx-gomlx v0.3.2
9+
github.com/gomlx/gomlx v0.25.0
10+
github.com/gomlx/gopjrt v0.10.0
11+
github.com/gomlx/onnx-gomlx v0.3.3
1212
github.com/stretchr/testify v1.11.1
1313
github.com/sugarme/tokenizer v0.3.0
1414
github.com/urfave/cli/v3 v3.6.1
1515
github.com/viant/afs v1.29.0
16-
github.com/yalue/onnxruntime_go v1.23.0
16+
github.com/yalue/onnxruntime_go v1.24.0
1717
)
1818

1919
require (
@@ -23,7 +23,7 @@ require (
2323
github.com/go-errors/errors v1.5.1 // indirect
2424
github.com/go-logr/logr v1.4.3 // indirect
2525
github.com/gofrs/flock v0.13.0 // indirect
26-
github.com/gomlx/stablehlo v0.1.0 // indirect
26+
github.com/gomlx/stablehlo v0.2.0 // indirect
2727
github.com/google/uuid v1.6.0 // indirect
2828
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
2929
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect

go.sum

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@ github.com/gomlx/exceptions v0.0.3 h1:HKnTgEjj4jlmhr8zVFkTP9qmV1ey7ypYYosQ8GzXWu
2929
github.com/gomlx/exceptions v0.0.3/go.mod h1:uHL0TQwJ0xaV2/snJOJV6hSE4yRmhhfymuYgNredGxU=
3030
github.com/gomlx/go-huggingface v0.3.1 h1:kMXA0ecTKywh4xo3WpklCDqUPmyg4ihsyftgCZNYGaA=
3131
github.com/gomlx/go-huggingface v0.3.1/go.mod h1:j1cd4gD0A2pwdYHZfkPMYfYr1C+KHCc2YA1kTEBWlTo=
32-
github.com/gomlx/gomlx v0.24.1 h1:52Gdjk9hgfrA2Oah5roa7pmmD7WEm4D8UfHaXcg8ZhY=
33-
github.com/gomlx/gomlx v0.24.1/go.mod h1:IFuL3ILLfJgY6XHkgwQtV6R0Nn61+tt7MV84xkzFj9A=
34-
github.com/gomlx/gopjrt v0.9.1 h1:R+grf72WYfPk53jLl5oLxqZU2DsE3kA8OIH6tICYC34=
35-
github.com/gomlx/gopjrt v0.9.1/go.mod h1:c8UENVGnxIDdihEL5HinlAdgR7RxTbEPLBppiMQF1ew=
36-
github.com/gomlx/onnx-gomlx v0.3.2 h1:GzYvrCqAuxAID7OEPlDwqqkYMKUyEbDTX37Qc8v17M0=
37-
github.com/gomlx/onnx-gomlx v0.3.2/go.mod h1:wqD8V9UgefKA7gjLT2YTXPs1m+THV4phS/bh71VnfrM=
38-
github.com/gomlx/stablehlo v0.1.0 h1:vcjBWL29mLpwOTNkktXve7N2H3ORdSv/KZC0j7hyjK4=
39-
github.com/gomlx/stablehlo v0.1.0/go.mod h1:xBBM/uNpu9SAzZW1rtMvKI832wwZ2DPJVHOsbwTNfgE=
32+
github.com/gomlx/gomlx v0.25.0 h1:hXqM95WhweRuzYdUt8omOdXwHoAMccVDvLjOmCmPzqQ=
33+
github.com/gomlx/gomlx v0.25.0/go.mod h1:x7sLYdFcW9WDmBOtQI1/nlDD9E/uWamwAJ0Ud/4ZZnM=
34+
github.com/gomlx/gopjrt v0.10.0 h1:mb/LeIk9xkaW5hh9oB5hEGuCbnqr81D0WmJsl+o1gY0=
35+
github.com/gomlx/gopjrt v0.10.0/go.mod h1:cUcwwDra6Uf9RTysQyv51YidUOk12dcz2HbjVi2J0S0=
36+
github.com/gomlx/onnx-gomlx v0.3.3 h1:0GjQVbwsjgqGgyBiVReaVbqhKOYeUHMCZntyD1igXPU=
37+
github.com/gomlx/onnx-gomlx v0.3.3/go.mod h1:qfQVZ28/9INXi4ztONVhowm+83CqHya7CfwATquyAAU=
38+
github.com/gomlx/stablehlo v0.2.0 h1:OOKA7APi9BFh7f0u33aGPeMfZpDXzJiCIYzs0sRd+HM=
39+
github.com/gomlx/stablehlo v0.2.0/go.mod h1:OAWBbjjuS4c53wPsw2L+wtDVS/5RQY2OZLA3FYFPeJk=
4040
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
4141
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
4242
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -89,8 +89,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
8989
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
9090
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
9191
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
92-
github.com/yalue/onnxruntime_go v1.23.0 h1:Hin0mFphwGOeT7xEQrAIi/p2O6ngmSy4uz0yXkC9yCw=
93-
github.com/yalue/onnxruntime_go v1.23.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
92+
github.com/yalue/onnxruntime_go v1.24.0 h1:IdgJLxxyotlsUTmL1UnHZgBzXJGgY51LZ4vQ5rZeOXU=
93+
github.com/yalue/onnxruntime_go v1.24.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
9494
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
9595
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
9696
golang.org/x/exp v0.0.0-20251125195548-87e1e737ad39 h1:DHNhtq3sNNzrvduZZIiFyXWOL9IWaDPHqTnLJp+rCBY=

hugot_training_gomlx.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,10 @@ func TrainGoMLX(s *TrainingSession) error {
157157
if s.config.Verbose {
158158
fmt.Printf("Running evaluation for epoch %d\n on trainEvalDataset", loop.Epoch)
159159
}
160-
lossAndMetrics := gomlxTrainer.Eval(s.config.TrainEvalDataset)
160+
lossAndMetrics, err := gomlxTrainer.Eval(s.config.TrainEvalDataset)
161+
if err != nil {
162+
return err
163+
}
161164
meanTrainLoss := lossAndMetrics[1].Value().(float32)
162165
trainLosses = append(trainLosses, meanTrainLoss)
163166
}
@@ -166,7 +169,10 @@ func TrainGoMLX(s *TrainingSession) error {
166169
if s.config.Verbose {
167170
fmt.Printf("Running evaluation for epoch %d\n on evalDataset", loop.Epoch)
168171
}
169-
lossAndMetrics := gomlxTrainer.Eval(s.config.EvalDataset)
172+
lossAndMetrics, err := gomlxTrainer.Eval(s.config.EvalDataset)
173+
if err != nil {
174+
return err
175+
}
170176
meanLoss := lossAndMetrics[1].Value().(float32)
171177
evalLosses = append(evalLosses, meanLoss)
172178

hugot_training_test.go

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,10 @@ func round3decimals(x float64) float64 {
8282

8383
func trainSimilarity(t *testing.T,
8484
config TrainingConfig,
85-
examplesLhs,
86-
examplesRhs []string) []float64 {
85+
examplesLHS,
86+
examplesRHS []string,
87+
) []float64 {
88+
t.Helper()
8789
// Create a new GoMLX training session. Currently, training is only possible by loading an onnx model
8890
// into GoMLX, fine-tuning it, and then writing it back to onnx. Hugot deals with the details
8991
// for you here.
@@ -118,7 +120,7 @@ func trainSimilarity(t *testing.T,
118120
}()
119121

120122
// we now load the newly trained onnx model and generate the predictions with onnxruntime backend
121-
return runModel(t, "ORT", examplesLhs, examplesRhs, "./models/testTrain")
123+
return runModel(t, "ORT", examplesLHS, examplesRHS, "./models/testTrain")
122124
}
123125

124126
func TestTrainSemanticSimilarity(t *testing.T) {
@@ -130,8 +132,8 @@ func TestTrainSemanticSimilarity(t *testing.T) {
130132
checkT(t, err)
131133
lines := bytes.Split(data, []byte("\n"))
132134

133-
var examplesLhs []string
134-
var examplesRhs []string
135+
var examplesLHS []string
136+
var examplesRHS []string
135137
var scores []float64
136138

137139
for _, line := range lines {
@@ -140,16 +142,16 @@ func TestTrainSemanticSimilarity(t *testing.T) {
140142
if err != nil {
141143
t.Fatal(err)
142144
}
143-
examplesLhs = append(examplesLhs, example["sentence1"].(string))
144-
examplesRhs = append(examplesRhs, example["sentence2"].(string))
145+
examplesLHS = append(examplesLHS, example["sentence1"].(string))
146+
examplesRHS = append(examplesRHS, example["sentence2"].(string))
145147
scores = append(scores, example["score"].(float64))
146148
}
147149

148150
// first we run the untrained onnx model with onnxruntime backend
149-
similaritiesOnnxruntime := runModel(t, "ORT", examplesLhs, examplesRhs, modelPath)
151+
similaritiesOnnxruntime := runModel(t, "ORT", examplesLHS, examplesRHS, modelPath)
150152

151153
// we do the same for GoMLX and check the forward pass results match
152-
similaritiesGoMLX := runModel(t, "XLA", examplesLhs, examplesRhs, modelPath)
154+
similaritiesGoMLX := runModel(t, "XLA", examplesLHS, examplesRHS, modelPath)
153155

154156
for i := range similaritiesOnnxruntime {
155157
assert.Equal(t, round3decimals(similaritiesOnnxruntime[i]), round3decimals(similaritiesGoMLX[i]))
@@ -185,7 +187,7 @@ func TestTrainSemanticSimilarity(t *testing.T) {
185187
},
186188
Verbose: true,
187189
}
188-
similaritiesGoMLXTrained := trainSimilarity(t, trainingConfig, examplesLhs, examplesRhs)
190+
similaritiesGoMLXTrained := trainSimilarity(t, trainingConfig, examplesLHS, examplesRHS)
189191

190192
fmt.Println("GoMLX trained model predictions:")
191193
for i := range similaritiesGoMLXTrained {
@@ -202,24 +204,24 @@ func TestTrainSemanticSimilarity(t *testing.T) {
202204

203205
// we can also train a model using an in memory dataset. For this we create the slice of examples manually.
204206
var examples []datasets.SemanticSimilarityExample
205-
for i := 0; i < len(examplesLhs); i++ {
207+
for i := 0; i < len(examplesLHS); i++ {
206208
examples = append(examples, datasets.SemanticSimilarityExample{
207-
Sentence1: examplesLhs[i],
208-
Sentence2: examplesRhs[i],
209+
Sentence1: examplesLHS[i],
210+
Sentence2: examplesRHS[i],
209211
Score: float32(scores[i]),
210212
})
211213
}
212214
inMemoryDataset, err := datasets.NewInMemorySemanticSimilarityDataset(examples, 1, nil)
213215
checkT(t, err)
214216
trainingConfig.TrainDataset = inMemoryDataset
215-
similaritiesGoMLXTrainedInMemory := trainSimilarity(t, trainingConfig, examplesLhs, examplesRhs)
217+
similaritiesGoMLXTrainedInMemory := trainSimilarity(t, trainingConfig, examplesLHS, examplesRHS)
216218
for i := range similaritiesGoMLXTrainedInMemory {
217219
assert.Equal(t, round3decimals(similaritiesGoMLXTrained[i]), round3decimals(similaritiesGoMLXTrainedInMemory[i]))
218220
}
219221

220222
// we can also freeze layers
221223
trainingConfig.Options = append(trainingConfig.Options, WithFreezeLayers([]int{-1})) // freeze all layers but the last one
222-
similaritiesGoMLXTrainedFrozen := trainSimilarity(t, trainingConfig, examplesLhs, examplesRhs)
224+
similaritiesGoMLXTrainedFrozen := trainSimilarity(t, trainingConfig, examplesLHS, examplesRHS)
223225

224226
fmt.Println("GoMLX trained model predictions freezing all layers but the last one:")
225227
for i := range similaritiesGoMLXTrainedFrozen {
@@ -230,7 +232,7 @@ func TestTrainSemanticSimilarity(t *testing.T) {
230232
func rmse(predictions []float64, labels []float64) float64 {
231233
var sum float64
232234
for i := 0; i < len(predictions); i++ {
233-
sum += math.Pow(predictions[i]-labels[i], 2)
235+
sum += (predictions[i] - labels[i]) * (predictions[i] - labels[i])
234236
}
235237
return math.Sqrt(sum / float64(len(predictions)))
236238
}

0 commit comments

Comments
 (0)