Skip to content

Commit 7c51350

Browse files
kercre123kercre123
authored
Merge pull request #469 from kercre123/more-lua-functions
More lua functions
2 parents 3e38bc7 + aa64eff commit 7c51350

File tree

7 files changed

+226
-27
lines changed

7 files changed

+226
-27
lines changed

‎chipper/pkg/scripting/display.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package scripting
2+
3+
// taken from https://github.com/fforchino/vector-go-sdk
4+
5+
import "image"
6+
7+
func ConvertPixesTo16BitRGB(r uint32, g uint32, b uint32, a uint32, opacityPercentage uint16) uint16 {
8+
R, G, B := uint16(r/257), uint16(g/8193), uint16(b/257)
9+
10+
R = R * opacityPercentage / 100
11+
G = G * opacityPercentage / 100
12+
B = B * opacityPercentage / 100
13+
14+
//The format appears to be: 000bbbbbrrrrrggg
15+
16+
var Br uint16 = (uint16(B & 0xF8)) << 5 // 5 bits for blue [8..12]
17+
var Rr uint16 = (uint16(R & 0xF8)) // 5 bits for red [3..7]
18+
var Gr uint16 = (uint16(G)) // 3 bits for green [0..2]
19+
20+
out := uint16(Br | Rr | Gr)
21+
//println(fmt.Sprintf("%d,%d,%d -> R: %016b G: %016b B: %016b = %016b", R, G, B, Rr, Gr, Br, out))
22+
return out
23+
}
24+
25+
func ConvertPixelsToRawBitmap(image image.Image, opacityPercentage int) []uint16 {
26+
imgHeight, imgWidth := image.Bounds().Max.Y, image.Bounds().Max.X
27+
bitmap := make([]uint16, imgWidth*imgHeight)
28+
29+
for y := 0; y < imgHeight; y++ {
30+
for x := 0; x < imgWidth; x++ {
31+
r, g, b, a := image.At(x, y).RGBA()
32+
bitmap[(y)*imgWidth+(x)] = ConvertPixesTo16BitRGB(r, g, b, a, uint16(opacityPercentage))
33+
}
34+
}
35+
return bitmap
36+
}

‎chipper/pkg/scripting/scripting.go‎

Lines changed: 165 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
package scripting
22

33
import (
4+
"bytes"
45
"context"
6+
"encoding/binary"
57
"encoding/json"
68
"fmt"
9+
"image"
710
"io"
811
"net/http"
12+
"os"
913
"time"
1014

1115
"github.com/fforchino/vector-go-sdk/pkg/vector"
@@ -26,6 +30,24 @@ releaseBehaviorControl()
2630
<goroutine determines whether the function blocks or not>
2731
sayText(text string, goroutine bool)
2832
playAnimation(animation string, goroutine bool)
33+
sleep(milliseconds int)
34+
moveLift(radpersecond int)
35+
moveHead(radpersecond int)
36+
37+
// leftWheelmmps2 and rightWheelmmps2 are what you want the wheels to accelerate to. if you want
38+
// the wheels to accelerate immediately, just set leftWheelmmps2 and rightWheelmmps2 to 0
39+
moveWheels(leftWheelmmps, rightWheelmmps, leftWheelmmps2, rightWheelmmps2 int)
40+
41+
// won't block
42+
showImage(filePath string, durationMs int)
43+
44+
// wrapper for http.Post and http.Get respectively
45+
// timeout is in seconds
46+
// set timeout to 0 for default
47+
// each return response as strings
48+
postHTTPRequest(url, contentType, body string, timeout int) (resp string)
49+
getHTTPRequest(url, timeout int) (resp string)
50+
2951
3052
*/
3153

@@ -44,7 +66,7 @@ func sayText(L *lua.LState) int {
4466
executeWithGoroutine(L, func(L *lua.LState) error {
4567
_, err := gRfLS(L).Conn.SayText(L.Context(), &vectorpb.SayTextRequest{Text: textToSay, UseVectorVoice: true, DurationScalar: 1.0})
4668
return err
47-
})
69+
}, false)
4870
return 0
4971
}
5072

@@ -53,10 +75,135 @@ func playAnimation(L *lua.LState) int {
5375
executeWithGoroutine(L, func(L *lua.LState) error {
5476
_, err := gRfLS(L).Conn.PlayAnimation(L.Context(), &vectorpb.PlayAnimationRequest{Animation: &vectorpb.Animation{Name: animToPlay}, Loops: 1})
5577
return err
56-
})
78+
}, false)
79+
return 0
80+
}
81+
82+
func sleep(L *lua.LState) int {
83+
sleepInMS := L.ToInt(1)
84+
time.Sleep(time.Millisecond * time.Duration(sleepInMS))
85+
return 0
86+
}
87+
88+
func moveHead(L *lua.LState) int {
89+
headSpeed := L.ToInt(1)
90+
headSpeedF := float32(headSpeed) / float32(100)
91+
executeWithGoroutine(L, func(L *lua.LState) error {
92+
_, err := gRfLS(L).Conn.MoveHead(L.Context(), &vectorpb.MoveHeadRequest{SpeedRadPerSec: headSpeedF})
93+
return err
94+
}, true)
95+
return 0
96+
}
97+
98+
func moveLift(L *lua.LState) int {
99+
liftSpeed := L.ToInt(1)
100+
liftSpeedF := float32(liftSpeed) / float32(100)
101+
executeWithGoroutine(L, func(L *lua.LState) error {
102+
_, err := gRfLS(L).Conn.MoveLift(L.Context(), &vectorpb.MoveLiftRequest{SpeedRadPerSec: liftSpeedF})
103+
return err
104+
}, true)
105+
return 0
106+
}
107+
108+
func moveWheels(L *lua.LState) int {
109+
leftWheelSpeed := L.ToInt(1)
110+
rightWheelSpeed := L.ToInt(2)
111+
leftWheelSpeed2 := L.ToInt(3)
112+
rightWheelSpeed2 := L.ToInt(4)
113+
executeWithGoroutine(L, func(L *lua.LState) error {
114+
_, err := gRfLS(L).Conn.DriveWheels(L.Context(),
115+
&vectorpb.DriveWheelsRequest{
116+
LeftWheelMmps: float32(leftWheelSpeed),
117+
RightWheelMmps: float32(rightWheelSpeed),
118+
LeftWheelMmps2: float32(leftWheelSpeed2),
119+
RightWheelMmps2: float32(rightWheelSpeed2),
120+
})
121+
return err
122+
}, true)
123+
return 0
124+
}
125+
126+
// later
127+
func showImageOnScreen(L *lua.LState) int {
128+
filePath := L.ToString(1)
129+
duration := L.ToInt(2)
130+
f, err := os.Open(filePath)
131+
if err != nil {
132+
logger.LogUI("Lua error: unable to open image file:", err)
133+
return 0
134+
}
135+
img, _, err := image.Decode(f)
136+
if err != nil {
137+
logger.LogUI("Lua error: file is not an image:", err)
138+
return 0
139+
}
140+
pixels := ConvertPixelsToRawBitmap(img, 100)
141+
buf := new(bytes.Buffer)
142+
for _, ui := range pixels {
143+
binary.Write(buf, binary.LittleEndian, ui)
144+
}
145+
executeWithGoroutine(L, func(L *lua.LState) error {
146+
_, err := gRfLS(L).Conn.DisplayFaceImageRGB(L.Context(),
147+
&vectorpb.DisplayFaceImageRGBRequest{
148+
FaceData: buf.Bytes(),
149+
DurationMs: uint32(duration),
150+
InterruptRunning: true,
151+
})
152+
return err
153+
}, true)
57154
return 0
58155
}
59156

157+
func postHTTPRequest(L *lua.LState) int {
158+
url := L.ToString(1)
159+
contentType := L.ToString(2)
160+
body := L.ToString(3)
161+
timeout := L.ToInt(4)
162+
r := bytes.NewReader([]byte(body))
163+
cl := http.DefaultClient
164+
if timeout > 0 {
165+
cl.Timeout = time.Second * time.Duration(timeout)
166+
}
167+
resp, err := cl.Post(url, contentType, r)
168+
if err != nil {
169+
logger.LogUI("Lua postHTTPRequest error:", err)
170+
L.Push(lua.LString("http error: " + err.Error()))
171+
return 1
172+
}
173+
b, err := io.ReadAll(resp.Body)
174+
L.Push(lua.LString(string(b)))
175+
if err != nil {
176+
logger.LogUI("Lua postHTTPRequest error:", err)
177+
L.Push(lua.LString("http error: " + err.Error()))
178+
} else {
179+
L.Push(lua.LString(string(b)))
180+
}
181+
return 1
182+
}
183+
184+
func getHTTPRequest(L *lua.LState) int {
185+
url := L.ToString(1)
186+
timeout := L.ToInt(2)
187+
cl := http.DefaultClient
188+
if timeout > 0 {
189+
cl.Timeout = time.Second * time.Duration(timeout)
190+
}
191+
resp, err := cl.Get(url)
192+
if err != nil {
193+
logger.LogUI("Lua getHTTPRequest error:", err)
194+
L.Push(lua.LString("http error: " + err.Error()))
195+
return 1
196+
}
197+
b, err := io.ReadAll(resp.Body)
198+
if err != nil {
199+
logger.LogUI("Lua getHTTPRequest error:", err)
200+
L.Push(lua.LString("http error: " + err.Error()))
201+
} else {
202+
L.Push(lua.LString(string(b)))
203+
}
204+
return 1
205+
}
206+
60207
// get robot from LState
61208
func gRfLS(L *lua.LState) *vector.Vector {
62209
ud := L.GetGlobal("bot").(*lua.LUserData)
@@ -70,6 +217,13 @@ func MakeLuaState(esn string, validating bool) (*lua.LState, error) {
70217
L.SetContext(context.Background())
71218
L.SetGlobal("sayText", L.NewFunction(sayText))
72219
L.SetGlobal("playAnimation", L.NewFunction(playAnimation))
220+
L.SetGlobal("sleep", L.NewFunction(sleep))
221+
L.SetGlobal("moveHead", L.NewFunction(moveHead))
222+
L.SetGlobal("moveLift", L.NewFunction(moveLift))
223+
L.SetGlobal("moveWheels", L.NewFunction(moveWheels))
224+
L.SetGlobal("showImage", L.NewFunction(showImageOnScreen))
225+
L.SetGlobal("postHTTPRequest", L.NewFunction(postHTTPRequest))
226+
L.SetGlobal("getHTTPRequest", L.NewFunction(getHTTPRequest))
73227
SetBControlFunctions(L)
74228
ud := L.NewUserData()
75229
if !validating {
@@ -89,19 +243,24 @@ func MakeLuaState(esn string, validating bool) (*lua.LState, error) {
89243
return L, nil
90244
}
91245

92-
func executeWithGoroutine(L *lua.LState, fn func(L *lua.LState) error) {
93-
goroutine := L.ToBool(2)
246+
func executeWithGoroutine(L *lua.LState, fn func(L *lua.LState) error, force bool) {
247+
var goroutine bool
248+
if force {
249+
goroutine = true
250+
} else {
251+
goroutine = L.ToBool(2)
252+
}
94253
if goroutine {
95254
go func() {
96255
err := fn(L)
97256
if err != nil {
98-
logger.Println("LUA: failure: " + err.Error())
257+
logger.LogUI("LUA: failure: " + err.Error())
99258
}
100259
}()
101260
} else {
102261
err := fn(L)
103262
if err != nil {
104-
logger.Println("LUA: failure: " + err.Error())
263+
logger.LogUI("LUA: failure: " + err.Error())
105264
}
106265
}
107266
}

‎chipper/pkg/wirepod/stt/whisper.cpp/WhisperCpp.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ func Init() error {
6262
logger.Println("Opening Whisper model (" + modelPath + ")")
6363
//logger.Println(whisper.Whisper_print_system_info())
6464
context = whisper.Whisper_init(modelPath)
65-
params = context.Whisper_full_default_params(whisper.SAMPLING_GREEDY)
65+
params = context.Whisper_full_default_params(whisper.SamplingStrategy(whisper.SAMPLING_GREEDY))
6666
params.SetTranslate(false)
6767
params.SetPrintSpecial(false)
6868
params.SetPrintProgress(false)

‎chipper/pkg/wirepod/ttr/kgsim.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -239,19 +239,20 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo
239239
var fullRespSlice []string
240240
var isDone bool
241241
var c *openai.Client
242-
if vars.APIConfig.Knowledge.Provider == "together" {
242+
switch vars.APIConfig.Knowledge.Provider {
243+
case "together":
243244
if vars.APIConfig.Knowledge.Model == "" {
244245
vars.APIConfig.Knowledge.Model = "meta-llama/Llama-3-70b-chat-hf"
245246
vars.WriteConfigToDisk()
246247
}
247248
conf := openai.DefaultConfig(vars.APIConfig.Knowledge.Key)
248249
conf.BaseURL = "https://api.together.xyz/v1"
249250
c = openai.NewClientWithConfig(conf)
250-
} else if vars.APIConfig.Knowledge.Provider == "custom" {
251+
case "custom":
251252
conf := openai.DefaultConfig(vars.APIConfig.Knowledge.Key)
252253
conf.BaseURL = vars.APIConfig.Knowledge.Endpoint
253254
c = openai.NewClientWithConfig(conf)
254-
} else if vars.APIConfig.Knowledge.Provider == "openai" {
255+
case "openai":
255256
c = openai.NewClient(vars.APIConfig.Knowledge.Key)
256257
}
257258
speakReady := make(chan string)
@@ -261,7 +262,7 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo
261262

262263
stream, err := c.CreateChatCompletionStream(ctx, aireq)
263264
if err != nil {
264-
log.Printf("Error creating chat completion stream: %v", err)
265+
log.Printf("Error creating chat completion stream: %v", err)
265266
if strings.Contains(err.Error(), "does not exist") && vars.APIConfig.Knowledge.Provider == "openai" {
266267
logger.Println("GPT-4 model cannot be accessed with this API key. You likely need to add more than $5 dollars of funds to your OpenAI account.")
267268
logger.LogUI("GPT-4 model cannot be accessed with this API key. You likely need to add more than $5 dollars of funds to your OpenAI account.")
@@ -345,10 +346,10 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo
345346
return
346347
}
347348

348-
if (len(response.Choices) == 0) {
349-
logger.Println("Empty response")
350-
return
351-
}
349+
if len(response.Choices) == 0 {
350+
logger.Println("Empty response")
351+
return
352+
}
352353

353354
fullfullRespText = fullfullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content)
354355
fullRespText = fullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content)

‎chipper/pkg/wirepod/ttr/kgsim_cmds.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ func ModelIsSupported(cmd LLMCommand, model string) bool {
152152
func CreatePrompt(origPrompt string, model string, isKG bool) string {
153153
prompt := origPrompt + "\n\n" + "Keep in mind, user input comes from speech-to-text software, so respond accordingly. No special characters, especially these: & ^ * # @ - . No lists. No formatting."
154154
if vars.APIConfig.Knowledge.CommandsEnable {
155-
prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:"
155+
prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:"
156156
for _, cmd := range ValidLLMCommands {
157157
if ModelIsSupported(cmd, model) {
158158
promptAppendage := "\n\nCommand Name: " + cmd.Command + "\nDescription: " + cmd.Description + "\nParameter choices: " + cmd.ParamChoices
@@ -466,21 +466,22 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector
466466
var fullRespSlice []string
467467
var isDone bool
468468
var c *openai.Client
469-
if vars.APIConfig.Knowledge.Provider == "together" {
469+
switch vars.APIConfig.Knowledge.Provider {
470+
case "together":
470471
if vars.APIConfig.Knowledge.Model == "" {
471472
vars.APIConfig.Knowledge.Model = "meta-llama/Llama-2-70b-chat-hf"
472473
vars.WriteConfigToDisk()
473474
}
474475
conf := openai.DefaultConfig(vars.APIConfig.Knowledge.Key)
475476
conf.BaseURL = "https://api.together.xyz/v1"
476477
c = openai.NewClientWithConfig(conf)
477-
} else if vars.APIConfig.Knowledge.Provider == "openai" {
478+
case "openai":
478479
c = openai.NewClient(vars.APIConfig.Knowledge.Key)
479-
} else if vars.APIConfig.Knowledge.Provider == "custom" {
480-
conf := openai.DefaultConfig(vars.APIConfig.Knowledge.Key)
480+
case "custom":
481+
conf := openai.DefaultConfig(vars.APIConfig.Knowledge.Key)
481482
conf.BaseURL = vars.APIConfig.Knowledge.Endpoint
482483
c = openai.NewClientWithConfig(conf)
483-
}
484+
}
484485
ctx := context.Background()
485486
speakReady := make(chan string)
486487

‎chipper/pkg/wirepod/ttr/matchIntentSend.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -127,13 +127,14 @@ func customIntentHandler(req interface{}, voiceText string, botSerial string) bo
127127

128128
var args []string
129129
for _, arg := range c.ExecArgs {
130-
if arg == "!botSerial" {
130+
switch arg {
131+
case "!botSerial":
131132
arg = botSerial
132-
} else if arg == "!speechText" {
133+
case "!speechText":
133134
arg = "\"" + voiceText + "\""
134-
} else if arg == "!intentName" {
135+
case "!intentName":
135136
arg = c.Name
136-
} else if arg == "!locale" {
137+
case "!locale":
137138
arg = vars.APIConfig.STT.Language
138139
}
139140
args = append(args, arg)

‎chipper/pkg/wirepod/ttr/weather.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,10 +184,11 @@ func getWeather(location string, botUnits string, hoursFromNow int) (string, str
184184
}
185185
if weatherEnabled {
186186
if botUnits != "" {
187-
if botUnits == "F" {
187+
switch botUnits {
188+
case "F":
188189
logger.Println("Weather units set to F")
189190
weatherAPIUnit = "F"
190-
} else if botUnits == "C" {
191+
case "C":
191192
logger.Println("Weather units set to C")
192193
weatherAPIUnit = "C"
193194
}

0 commit comments

Comments
 (0)