refactor(emulate-elastic): delegate system client access - #689
Conversation
donhardman
commented
Jun 29, 2026
- Move system client retrieval logic to HTTPClient
- Replace InternalBuddyClientTrait with direct client methods
- Update entity, cat, and Kibana handlers to use manticoreClient
- Streamline entity alias management via new client method
- Move system client retrieval logic to HTTPClient - Replace InternalBuddyClientTrait with direct client methods - Update entity, cat, and Kibana handlers to use manticoreClient - Streamline entity alias management via new client method
clt❌ CLT tests in Failed tests:🔧 Edit failed tests in UI: test/clt-tests/sharding/rollback/master-takeover-while-rollback.rec––– input –––
export INSTANCE=1
––– output –––
OK
––– input –––
mkdir -p /var/{run,lib,log}/manticore-${INSTANCE}
––– output –––
OK
––– input –––
stdbuf -oL searchd -c test/clt-tests/base/searchd-with-flexible-ports.conf > /dev/null
––– output –––
OK
––– input –––
if timeout 30 grep -qm1 '\[BUDDY\] started' <(tail -n 1000 -f /var/log/manticore-${INSTANCE}/searchd.log); then echo 'Buddy started!'; else echo 'Timeout or failed!'; cat /var/log/manticore-${INSTANCE}/searchd.log; fi
––– output –––
OK
––– input –––
export INSTANCE=2
––– output –––
OK
––– input –––
mkdir -p /var/{run,lib,log}/manticore-${INSTANCE}
––– output –––
OK
––– input –––
stdbuf -oL searchd -c test/clt-tests/base/searchd-with-flexible-ports.conf > /dev/null
––– output –––
OK
––– input –––
if timeout 30 grep -qm1 '\[BUDDY\] started' <(tail -n 1000 -f /var/log/manticore-${INSTANCE}/searchd.log); then echo 'Buddy started!'; else echo 'Timeout or failed!'; cat /var/log/manticore-${INSTANCE}/searchd.log; fi
––– output –––
OK
––– input –––
export INSTANCE=3
––– output –––
OK
––– input –––
mkdir -p /var/{run,lib,log}/manticore-${INSTANCE}
––– output –––
OK
––– input –––
stdbuf -oL searchd -c test/clt-tests/base/searchd-with-flexible-ports.conf > /dev/null
––– output –––
OK
––– input –––
if timeout 30 grep -qm1 '\[BUDDY\] started' <(tail -n 1000 -f /var/log/manticore-${INSTANCE}/searchd.log); then echo 'Buddy started!'; else echo 'Timeout or failed!'; cat /var/log/manticore-${INSTANCE}/searchd.log; fi
––– output –––
OK
––– input –––
export CLUSTER_NAME=c TABLE_NAME=t
––– output –––
OK
––– input –––
mysql -h0 -P1306 -e "create cluster ${CLUSTER_NAME}"
––– output –––
OK
––– input –––
mysql -h0 -P1306 -e "show status like 'cluster_${CLUSTER_NAME}_status'\G"
––– output –––
OK
––– input –––
for n in `seq 2 $INSTANCE`; do mysql -h0 -P${n}306 -e "join cluster ${CLUSTER_NAME} at '127.0.0.1:1312'"; done;
––– output –––
OK
––– input –––
mysql -h0 -P${INSTANCE}306 -e "show status like 'cluster_${CLUSTER_NAME}_status'\G"
––– output –––
OK
––– input –––
show_shards() {
# Capture the whole SHOW CREATE TABLE output: the render is multi-line, so the agent=
# clauses live on a later line than "Create Table:" — filtering to that line drops them.
shards=$(mysql -h0 -P$1 -e "show create table $2 option force=1\G")
(
# Handle local shards
echo "$shards" | grep -oP "local='[^']*'" | sed "s/local='//g; s/'//g" | tr ',' '\n' | while read -r tbl; do
[ -n "$tbl" ] && echo "$tbl: local"
done
# Handle agent shards. Strip the [retry_count=...,ha_strategy=...] options suffix so all
# mirrors of a shard group under the same table name instead of a polluted awk key.
echo "$shards" | grep -oP "agent='[^']*'" | sed "s/agent='//g; s/'//g; s/\[[^]]*\]//g" | tr '|' '\n' | awk -F: '{agents[$3] = agents[$3] ? agents[$3] " " $1 ":" $2 : $1 ":" $2} END {for (t in agents) print t ": " agents[t]}'
) | sort
}
––– output –––
OK
––– input –––
check_cluster_status() {
mysql -h0 -P$1 -e "SHOW STATUS LIKE 'cluster_${2}_status'\\G" | grep -q "Value: primary" && echo "OK" || echo "FAIL"
}
––– output –––
OK
––– input –––
mysql -h0 -P1306 -e "CREATE TABLE ${CLUSTER_NAME}:${TABLE_NAME} (id bigint, value string) shards='2' rf='2'"; echo $?
––– output –––
OK
––– input –––
show_shards 1306 "$TABLE_NAME"
––– output –––
OK
––– input –––
export INSTANCE=1; stdbuf -oL searchd --stopwait -c test/clt-tests/base/searchd-with-flexible-ports.conf > /dev/null; echo "Node 1 killed"
––– output –––
OK
––– input –––
timeout 30 bash -c 'while lsof -i :${INSTANCE}306 &>/dev/null; do sleep 1; done'
––– output –––
OK
––– input –––
timeout 10 grep -qm1 'becoming master' <(tail -n 1000 -f /var/log/manticore-{2,3}/searchd.log 2>/dev/null); echo $?
––– output –––
OK
––– input –––
for i in 2 3; do check_cluster_status ${i}306 "$CLUSTER_NAME"; done | uniq
––– output –––
OK
––– input –––
show_shards 2306 "$TABLE_NAME"
––– output –––
OK
––– input –––
export INSTANCE=1
––– output –––
OK
––– input –––
mkdir -p /var/{run,lib,log}/manticore-${INSTANCE}
––– output –––
OK
––– input –––
stdbuf -oL searchd -c test/clt-tests/base/searchd-with-flexible-ports.conf > /dev/null
––– output –––
OK
––– input –––
if timeout 30 grep -qm1 '\[BUDDY\] started' <(tail -n 1000 -f /var/log/manticore-${INSTANCE}/searchd.log); then echo 'Buddy started!'; else echo 'Timeout or failed!'; cat /var/log/manticore-${INSTANCE}/searchd.log; fi
––– output –––
OK
––– input –––
sleep 2
––– output –––
OK
––– input –––
for i in 1 2 3; do check_cluster_status ${i}306 "$CLUSTER_NAME"; done | uniq
––– output –––
OK
––– input –––
show_shards 2306 "$TABLE_NAME"
––– output –––
- system.t_s0: 127.0.0.1:2312 127.0.0.1:3312
+ ERROR 1064 (42000) at line 1: You have an error in your query. Please, double-check it.
- system.t_s1: 127.0.0.1:2312 127.0.0.1:3312 |
🐳 Docker Images PushedBuddy Test-Kit Images:
Base image used: These images contain the buddy code from this PR and can be used for testing. Built from commit: eb2b8f6 |
clt❌ CLT tests in Failed tests:🔧 Edit failed tests in UI: test/clt-tests/mcl/auto-embeddings-jina-remote.rec––– input –––
rm -f /var/log/manticore/searchd.log; stdbuf -oL searchd --stopwait > /dev/null; stdbuf -oL searchd ${SEARCHD_ARGS:-} > /dev/null
––– output –––
OK
––– input –––
if timeout 10 grep -qm1 'accepting connections' <(tail -n 1000 -f /var/log/manticore/searchd.log); then echo 'Accepting connections!'; else echo 'Timeout or failed!'; fi
––– output –––
OK
––– input –––
cosine_similarity() {
local file1="$1" file2="$2"
awk '
NR==FNR { a[NR]=$1; suma2+=$1*$1; next }
{
dot += a[FNR]*$1
sumb2 += $1*$1
}
END {
print dot / (sqrt(suma2) * sqrt(sumb2))
}' "$file1" "$file2"
}
––– output –––
OK
––– input –––
export -f cosine_similarity
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_invalid_model (title TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/invalid-model-name-12345' FROM = 'title') " 2>&1
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_valid_model_no_api_key (title TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/jina-embeddings-v2-base-en' FROM = 'title') " 2>&1
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_jina_remote (title TEXT, content TEXT, description TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/jina-embeddings-v4' FROM = 'title, content' API_KEY='${JINA_API_KEY}') "; echo $?
––– output –––
- 0
+ ERROR 1064 (42000) at line 1: error adding table 'test_jina_remote': prealloc: Failed to send request to remote model
+ 1
––– input –––
mysql -h0 -P9306 -E -e "SHOW CREATE TABLE test_jina_remote"
––– output –––
- *************************** 1. row ***************************
+ ERROR 1064 (42000) at line 1: You have an error in your query. Please, double-check it.
- Table: test_jina_remote
- Create Table: CREATE TABLE test_jina_remote (
- id bigint,
- title text,
- content text,
- description text,
- embedding float_vector knn_type='hnsw' hnsw_similarity='L2' model_name='jina/jina-embeddings-v4' from='title, content' api_timeout='10'
- )
––– input –––
mysql -h0 -P9306 -e "INSERT INTO test_jina_remote (id, title, content, description) VALUES(1, 'machine learning algorithms', 'deep neural networks and artificial intelligence', 'advanced AI research')"; echo $?
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "SELECT COUNT(*) as record_count FROM test_jina_remote WHERE id=1"
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "INSERT INTO test_jina_remote (id, title, content, description) VALUES(2, 'machine learning algorithms', 'deep neural networks and artificial intelligence', 'different description')"
mysql -h0 -P9306 -e "SELECT embedding FROM test_jina_remote WHERE id=1" | \
grep -v embedding | \
sed 's/[0-9]\+\(\.[0-9]\+\)\?/\n&\n/g' | \
grep -E '^[0-9]+(\.[0-9]+)?$' | \
awk '{printf "%.5f\n", $1}' > /tmp/vector1.txt
mysql -h0 -P9306 -e "SELECT embedding FROM test_jina_remote WHERE id=2" | \
grep -v embedding | \
sed 's/[0-9]\+\(\.[0-9]\+\)\?/\n&\n/g' | \
grep -E '^[0-9]+(\.[0-9]+)?$' | \
awk '{printf "%.5f\n", $1}' > /tmp/vector2.txt
SIMILARITY=$(cosine_similarity /tmp/vector1.txt /tmp/vector2.txt)
echo "Cosine similarity: $SIMILARITY"
RESULT=$(awk -v sim="$SIMILARITY" 'BEGIN {
if (sim > 0.99)
print "SUCCESS: Same FROM fields produce similar vectors (similarity: " sim ")"
else
print "FAIL: Different vectors (FROM does not include description field and should not change generated vector value) (similarity: " sim ")"
}')
echo "$RESULT"
rm -f /tmp/vector1.txt /tmp/vector2.txt
––– output –––
- Cosine similarity: #!/(1|0\.[0-9]+)/!#
+ ERROR 1064 (42000) at line 1: table test_jina_remote: parse error: unknown column: embedding
- SUCCESS: Same FROM fields produce similar vectors (similarity: #!/(1|0\.[0-9]+)/!#)
+ ERROR 1064 (42000) at line 1: table test_jina_remote: parse error: unknown column: embedding
+ Cosine similarity: -nan
+ FAIL: Different vectors (FROM does not include description field and should not change generated vector value) (similarity: -nan)
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_jina_title_only (title TEXT, content TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/jina-embeddings-v4' FROM = 'title' API_KEY='${JINA_API_KEY}') "; mysql -h0 -P9306 -e "INSERT INTO test_jina_title_only (id, title, content) VALUES(1, 'machine learning algorithms', 'completely different content here')"; MD5_MULTI=$(mysql -h0 -P9306 -e "SELECT embedding FROM test_jina_remote WHERE id=1" | grep -v embedding | md5sum | awk '{print $1}'); MD5_SINGLE=$(mysql -h0 -P9306 -e "SELECT embedding FROM test_jina_title_only WHERE id=1" | grep -v embedding | md5sum | awk '{print $1}'); echo "multi_field_md5: $MD5_MULTI"; echo "single_field_md5: $MD5_SINGLE"; if [ "$MD5_MULTI" != "$MD5_SINGLE" ]; then echo "SUCCESS: Different FROM specifications produce different vectors"; else echo "INFO: FROM field comparison result"; fi
––– output –––
- multi_field_md5: #!/[0-9a-f]{32}/!#
+ ERROR 1064 (42000) at line 1: error adding table 'test_jina_title_only': prealloc: Failed to send request to remote model
- single_field_md5: #!/[0-9a-f]{32}/!#
+ ERROR 1064 (42000) at line 1: table test_jina_remote: parse error: unknown column: embedding
- SUCCESS: Different FROM specifications produce different vectors
+ ERROR 1064 (42000) at line 1: table test_jina_title_only: parse error: unknown column: embedding
+ multi_field_md5: d41d8cd98f00b204e9800998ecf8427e
+ single_field_md5: d41d8cd98f00b204e9800998ecf8427e
+ INFO: FROM field comparison result
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_jina_invalid_field (title TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/text-embedding-ada-002' FROM = 'nonexistent_field') " 2>&1
––– output –––
OK
––– input –––
if mysql -h0 -P9306 -e "SHOW TABLES LIKE 'test_jina_no_from'" | grep -q test_jina_no_from; then mysql -h0 -P9306 -e "INSERT INTO test_jina_no_from (id, title, embedding) VALUES(1, 'test title', '(0.1, 0.2, 0.3, 0.4, 0.5)')"; echo "insert_result: $?"; else echo "insert_result: skipped (table not created)"; fi
––– output –––
OK
––– input –––
if mysql -h0 -P9306 -e "SHOW TABLES LIKE 'test_jina_no_from'" | grep -q test_jina_no_from; then mysql -h0 -P9306 -e "SHOW CREATE TABLE test_jina_no_from"; else echo "table_structure: skipped (table not created)"; fi
––– output –––
OK
––– input –––
if [ -n "$JINA_API_KEY" ] && [ "$JINA_API_KEY" != "dummy_key_for_testing" ]; then echo "API key is available for testing"; else echo "API key not available - using dummy for error testing"; fi
––– output –––
OK |
🐳 Docker Images PushedBuddy Test-Kit Images:
Base image used: These images contain the buddy code from this PR and can be used for testing. Built from commit: d9a8cae |
1 similar comment
🐳 Docker Images PushedBuddy Test-Kit Images:
Base image used: These images contain the buddy code from this PR and can be used for testing. Built from commit: d9a8cae |
- upgrade symfony cache, console, dependency-injection, expression-language, finder, and var-exporter to v6.4.42 - upgrade symfony cache-contracts, deprecation-contracts, and service-contracts to v3.7.1 - update buddy-core reference - update composer plugin api version to 2.9.0
clt❌ CLT tests in Failed tests:🔧 Edit failed tests in UI: test/clt-tests/mcl/auto-embeddings-jina-remote.rec––– input –––
rm -f /var/log/manticore/searchd.log; stdbuf -oL searchd --stopwait > /dev/null; stdbuf -oL searchd ${SEARCHD_ARGS:-} > /dev/null
––– output –––
OK
––– input –––
if timeout 10 grep -qm1 'accepting connections' <(tail -n 1000 -f /var/log/manticore/searchd.log); then echo 'Accepting connections!'; else echo 'Timeout or failed!'; fi
––– output –––
OK
––– input –––
cosine_similarity() {
local file1="$1" file2="$2"
awk '
NR==FNR { a[NR]=$1; suma2+=$1*$1; next }
{
dot += a[FNR]*$1
sumb2 += $1*$1
}
END {
print dot / (sqrt(suma2) * sqrt(sumb2))
}' "$file1" "$file2"
}
––– output –––
OK
––– input –––
export -f cosine_similarity
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_invalid_model (title TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/invalid-model-name-12345' FROM = 'title') " 2>&1
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_valid_model_no_api_key (title TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/jina-embeddings-v2-base-en' FROM = 'title') " 2>&1
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_jina_remote (title TEXT, content TEXT, description TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/jina-embeddings-v4' FROM = 'title, content' API_KEY='${JINA_API_KEY}') "; echo $?
––– output –––
OK
––– input –––
mysql -h0 -P9306 -E -e "SHOW CREATE TABLE test_jina_remote"
––– output –––
OK
––– input –––
mysql -h0 -P9306 -e "INSERT INTO test_jina_remote (id, title, content, description) VALUES(1, 'machine learning algorithms', 'deep neural networks and artificial intelligence', 'advanced AI research')"; echo $?
––– output –––
- 0
+ ERROR 1064 (42000) at line 1: Failed to send request to remote model
+ 1
––– input –––
mysql -h0 -P9306 -e "SELECT COUNT(*) as record_count FROM test_jina_remote WHERE id=1"
––– output –––
+--------------+
| record_count |
+--------------+
- | 1 |
+ | 0 |
+--------------+
––– input –––
mysql -h0 -P9306 -e "INSERT INTO test_jina_remote (id, title, content, description) VALUES(2, 'machine learning algorithms', 'deep neural networks and artificial intelligence', 'different description')"
mysql -h0 -P9306 -e "SELECT embedding FROM test_jina_remote WHERE id=1" | \
grep -v embedding | \
sed 's/[0-9]\+\(\.[0-9]\+\)\?/\n&\n/g' | \
grep -E '^[0-9]+(\.[0-9]+)?$' | \
awk '{printf "%.5f\n", $1}' > /tmp/vector1.txt
mysql -h0 -P9306 -e "SELECT embedding FROM test_jina_remote WHERE id=2" | \
grep -v embedding | \
sed 's/[0-9]\+\(\.[0-9]\+\)\?/\n&\n/g' | \
grep -E '^[0-9]+(\.[0-9]+)?$' | \
awk '{printf "%.5f\n", $1}' > /tmp/vector2.txt
SIMILARITY=$(cosine_similarity /tmp/vector1.txt /tmp/vector2.txt)
echo "Cosine similarity: $SIMILARITY"
RESULT=$(awk -v sim="$SIMILARITY" 'BEGIN {
if (sim > 0.99)
print "SUCCESS: Same FROM fields produce similar vectors (similarity: " sim ")"
else
print "FAIL: Different vectors (FROM does not include description field and should not change generated vector value) (similarity: " sim ")"
}')
echo "$RESULT"
rm -f /tmp/vector1.txt /tmp/vector2.txt
––– output –––
- Cosine similarity: #!/(1|0\.[0-9]+)/!#
+ Cosine similarity: -nan
- SUCCESS: Same FROM fields produce similar vectors (similarity: #!/(1|0\.[0-9]+)/!#)
+ FAIL: Different vectors (FROM does not include description field and should not change generated vector value) (similarity: -nan)
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_jina_title_only (title TEXT, content TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/jina-embeddings-v4' FROM = 'title' API_KEY='${JINA_API_KEY}') "; mysql -h0 -P9306 -e "INSERT INTO test_jina_title_only (id, title, content) VALUES(1, 'machine learning algorithms', 'completely different content here')"; MD5_MULTI=$(mysql -h0 -P9306 -e "SELECT embedding FROM test_jina_remote WHERE id=1" | grep -v embedding | md5sum | awk '{print $1}'); MD5_SINGLE=$(mysql -h0 -P9306 -e "SELECT embedding FROM test_jina_title_only WHERE id=1" | grep -v embedding | md5sum | awk '{print $1}'); echo "multi_field_md5: $MD5_MULTI"; echo "single_field_md5: $MD5_SINGLE"; if [ "$MD5_MULTI" != "$MD5_SINGLE" ]; then echo "SUCCESS: Different FROM specifications produce different vectors"; else echo "INFO: FROM field comparison result"; fi
––– output –––
- multi_field_md5: #!/[0-9a-f]{32}/!#
+ ERROR 1064 (42000) at line 1: Failed to send request to remote model
- single_field_md5: #!/[0-9a-f]{32}/!#
+ multi_field_md5: d41d8cd98f00b204e9800998ecf8427e
- SUCCESS: Different FROM specifications produce different vectors
+ single_field_md5: d41d8cd98f00b204e9800998ecf8427e
+ INFO: FROM field comparison result
––– input –––
mysql -h0 -P9306 -e "CREATE TABLE test_jina_invalid_field (title TEXT, embedding FLOAT_VECTOR KNN_TYPE='hnsw' HNSW_SIMILARITY='l2' MODEL_NAME = 'jina/text-embedding-ada-002' FROM = 'nonexistent_field') " 2>&1
––– output –––
OK
––– input –––
if mysql -h0 -P9306 -e "SHOW TABLES LIKE 'test_jina_no_from'" | grep -q test_jina_no_from; then mysql -h0 -P9306 -e "INSERT INTO test_jina_no_from (id, title, embedding) VALUES(1, 'test title', '(0.1, 0.2, 0.3, 0.4, 0.5)')"; echo "insert_result: $?"; else echo "insert_result: skipped (table not created)"; fi
––– output –––
OK
––– input –––
if mysql -h0 -P9306 -e "SHOW TABLES LIKE 'test_jina_no_from'" | grep -q test_jina_no_from; then mysql -h0 -P9306 -e "SHOW CREATE TABLE test_jina_no_from"; else echo "table_structure: skipped (table not created)"; fi
––– output –––
OK
––– input –––
if [ -n "$JINA_API_KEY" ] && [ "$JINA_API_KEY" != "dummy_key_for_testing" ]; then echo "API key is available for testing"; else echo "API key not available - using dummy for error testing"; fi
––– output –––
OK |
🐳 Docker Images PushedBuddy Test-Kit Images:
Base image used: These images contain the buddy code from this PR and can be used for testing. Built from commit: fb02ca2 |
1 similar comment
🐳 Docker Images PushedBuddy Test-Kit Images:
Base image used: These images contain the buddy code from this PR and can be used for testing. Built from commit: fb02ca2 |