Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

A/B - Performance gain change #967

Merged
merged 6 commits into from
Nov 13, 2024
Merged

A/B - Performance gain change #967

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a20c1d9
Change showing performance gain for the very fast and very slow times
nablaone Nov 11, 2024
be2575a
Merge branch 'main' into pr-ab-perf-display
nablaone Nov 12, 2024
cd2d726
Fixes
nablaone Nov 12, 2024
a905bee
Update quesma/quesma/ui/ab_testing.go
nablaone Nov 13, 2024
8c123e9
rebranding
nablaone Nov 13, 2024
fc30d9b
Merge branch 'main' into pr-ab-perf-display
nablaone Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 74 additions & 23 deletions quesma/quesma/ui/ab_testing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"quesma/elasticsearch"
"quesma/jsondiff"
"quesma/logger"
"quesma/quesma/config"
"quesma/quesma/ui/internal/builder"
"strings"
"time"
Expand Down Expand Up @@ -75,7 +76,6 @@ If the performance gain is positive, it means that the second backend connector
buffer.Html(`<select id="order_by" name="order_by">`)
buffer.Html(`<option value="default">Default</option>`)
buffer.Html(`<option value="response_similarity">Response similarity</option>`)
buffer.Html(`<option value="performance_gain">Performance gain</option>`)
buffer.Html(`<option value="count">Count</option>`)
buffer.Html(`</select>`)
buffer.Html(`<br>`)
Expand All @@ -86,7 +86,7 @@ If the performance gain is positive, it means that the second backend connector
buffer.Html(`<div class="menu">`)
buffer.Html("\n</div>")
} else {
buffer.Html(`<p>A/B Testing results are not available.</p>`)
buffer.Html(`<p>CR results are not available.</p>`)
}

buffer.Html("\n</main>\n\n")
Expand Down Expand Up @@ -258,17 +258,19 @@ func formatJSON(in *string) string {
}

type abTestingReportRow struct {
dashboardId string
panelId string
dashboardUrl string
detailsUrl string
dashboardName string
panelName string
aName string
bName string
successRate *float64
performanceGain *float64
count int
dashboardId string
panelId string
dashboardUrl string
detailsUrl string
dashboardName string
panelName string
aName string
bName string
aTime *float64
bTime *float64
successRate *float64
responseTimeDelta *float64
count int
}

func (qmc *QuesmaManagementConsole) abTestingReadReport(kibanaUrl, orderBy string) ([]abTestingReportRow, error) {
Expand All @@ -281,7 +283,6 @@ func (qmc *QuesmaManagementConsole) abTestingReadReport(kibanaUrl, orderBy strin
orderByToSQL := map[string]string{
"default": "dashboard_id, panel_id, a_name, b_name",
"response_similarity": "response_similarity DESC, dashboard_id, panel_id, a_name, b_name",
"performance_gain": "performance_gain DESC,dashboard_id, panel_id, a_name, b_name",
"count": "count DESC,dashboard_id, panel_id, a_name, b_name",
}

Expand Down Expand Up @@ -311,8 +312,10 @@ SELECT
a_name,
b_name,
(sumIf(c,ok)/ sum(c)) * 100 as response_similarity,
((avgIf(a_time,ok)- avgIf(b_time,ok))/avgIf(a_time,ok))*100.0 as performance_gain,
sum(c) as count

sum(c) as count,
avgIf(a_time,ok) as a_time,
avgIf(b_time,ok) as b_time
FROM
subresults
GROUP BY
Expand All @@ -331,7 +334,7 @@ GROUP BY

for rows.Next() {
row := abTestingReportRow{}
err := rows.Scan(&row.dashboardId, &row.panelId, &row.aName, &row.bName, &row.successRate, &row.performanceGain, &row.count)
err := rows.Scan(&row.dashboardId, &row.panelId, &row.aName, &row.bName, &row.successRate, &row.count, &row.aTime, &row.bTime)
if err != nil {
return nil, err
}
Expand All @@ -341,6 +344,26 @@ GROUP BY
row.dashboardName = kibanaDashboards.dashboardName(row.dashboardId)
row.panelName = kibanaDashboards.panelName(row.dashboardId, row.panelId)

if row.aTime != nil && row.bTime != nil {

var clickhouseTime, elasticTime float64

if row.aName == config.ElasticsearchTarget {
elasticTime = *row.aTime
clickhouseTime = *row.bTime
} else {
elasticTime = *row.bTime
clickhouseTime = *row.aTime
}

if elasticTime == 0 {
row.responseTimeDelta = nil
} else {
delta := (elasticTime - clickhouseTime) / elasticTime * 100
row.responseTimeDelta = &delta
}
}

result = append(result, row)
}

Expand All @@ -361,7 +384,7 @@ func (qmc *QuesmaManagementConsole) generateABTestingReport(kibanaUrl, orderBy s
buffer.Html(`<th class="key">Panel</th>` + "\n")
buffer.Html(`<th class="key">Count <br> <small>(since start)</small></th>` + "\n")
buffer.Html(`<th class="key">Response similarity</th>` + "\n")
buffer.Html(`<th class="key">Performance gain</th>` + "\n")
buffer.Html(`<th class="key">Response time delta</th>` + "\n")
buffer.Html(`<th class="key"></th>` + "\n")
buffer.Html("</tr>\n")
buffer.Html("</thead>\n")
Expand Down Expand Up @@ -405,8 +428,36 @@ func (qmc *QuesmaManagementConsole) generateABTestingReport(kibanaUrl, orderBy s
buffer.Html(`</td>`)

buffer.Html(`<td>`)
if row.performanceGain != nil {
buffer.Text(fmt.Sprintf("%.01f%%", *row.performanceGain))

const minTime = 0.05 // 50ms, we don't show performance gain for queries faster than this
const maxTime = 5.0 // if a query takes longer than this, we show the name of the slowest backend

if row.responseTimeDelta != nil {
buffer.Html(`<span title="`)
buffer.Html(fmt.Sprintf("%s=%.03fs, %s=%.03fs", row.aName, *row.aTime, row.bName, *row.bTime))
buffer.Html(`">`)

switch {

case *row.aTime < minTime && *row.bTime < minTime:
buffer.Text("both < 50ms")

case *row.aTime > maxTime && *row.bTime < maxTime:
buffer.Text(fmt.Sprintf("%s is over %0.02fs", row.aName, *row.aTime))

case *row.bTime > maxTime && *row.aTime < maxTime:
buffer.Text(fmt.Sprintf("%s is over %0.02fs", row.bName, *row.bTime))

default:

if *row.responseTimeDelta > 0 {
buffer.Text(fmt.Sprintf("%.01f%% faster", *row.responseTimeDelta))
} else {
buffer.Text(fmt.Sprintf("%.01f%% slower", -(*row.responseTimeDelta)))
}

}
buffer.Html(`</span>`)
} else {
buffer.Text("n/a")
}
Expand Down Expand Up @@ -536,7 +587,7 @@ func (qmc *QuesmaManagementConsole) generateABPanelDetails(dashboardId, panelId

buffer.Html(`<main id="ab_testing_dashboard">`)

buffer.Html(`<h2>A/B Testing - Panel Details</h2>`)
buffer.Html(`<h2>Compatibility report - Panel Details</h2>`)
buffer.Html(`<h3>`)
buffer.Text(fmt.Sprintf("Dashboard: %s", dashboardName))
buffer.Html(`</h3>`)
Expand Down Expand Up @@ -671,7 +722,7 @@ func (qmc *QuesmaManagementConsole) generateABMismatchDetails(dashboardId, panel

buffer.Html(`<main id="ab_testing_dashboard">`)

buffer.Html(`<h2>A/B Testing - Panel requests</h2>`)
buffer.Html(`<h2>Compatibility report - Panel requests</h2>`)

buffer.Html(`<h3>`)
buffer.Text(fmt.Sprintf("Dashboard: %s", dashboardName))
Expand Down Expand Up @@ -796,7 +847,7 @@ func (qmc *QuesmaManagementConsole) generateABSingleRequest(requestId string) []
buffer := newBufferWithHead()
buffer.Html(`<main id="ab_testing_dashboard">`)

buffer.Html(`<h2>A/B Testing - Request Results </h2>`)
buffer.Html(`<h2>Compatibility report - Request Results </h2>`)

fmtAny := func(value any) string {
if value == nil {
Expand Down
4 changes: 2 additions & 2 deletions quesma/quesma/ui/html_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ func (qmc *QuesmaManagementConsole) generateTopNavigation(target string) []byte
if target == "ab-testing-dashboard" {
buffer.Html(` class="active"`)
}
buffer.Html(`><a href="`)
buffer.Html(`><a title="Compatibility Report" href="`)
buffer.Html(abTestingPath)
buffer.Html(`">A/B</a></li>`)
buffer.Html(`">CR</a></li>`)
Copy link
Member

@avelanarius avelanarius Nov 12, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't forget that there are instances of "A/B testing" still left in ab_testing.go (such as A/B Testing - Panel Details).

It's OK if you want to do it in a followup PR.


if qmc.isAuthEnabled {
buffer.Html(`<li><a href="/logout">Logout</a></li>`)
Expand Down
Loading