From db7bc59ac536eef18d38a60d1908515b3a5dedfd Mon Sep 17 00:00:00 2001 From: Ritwik Ranjan Date: Mon, 23 Sep 2024 12:56:58 +0000 Subject: [PATCH 1/6] chore/tests: add network perf tests for Retina - Added new performance tests for iperf throughput (TCP and UDP) - Metrics include CPU Utilization Host, CPU Utilization Remote, Max RTT, Mean RTT, Min RTT, Retransmits, and Total Throughput This commit introduces new performance tests to measure iperf throughput under various conditions for the Retina project. Signed-off-by: Ritwik Ranjan --- .gitignore | 2 + go.mod | 21 ++- go.sum | 33 ++-- test/e2e/jobs/jobs.go | 64 +++++++ test/e2e/retina_perf_test.go | 82 +++++++++ .../perf/get-network-performance-measures.go | 84 +++++++++ .../perf/get-perf-regression-results.go | 168 ++++++++++++++++++ 7 files changed, 433 insertions(+), 21 deletions(-) create mode 100644 test/e2e/retina_perf_test.go create mode 100644 test/e2e/scenarios/perf/get-network-performance-measures.go create mode 100644 test/e2e/scenarios/perf/get-perf-regression-results.go diff --git a/.gitignore b/.gitignore index 27e2f7832c..cc081e115b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ bin/ image-metadata-*.json *packetmonitorsupport*/ +*.pem +*results*.json diff --git a/go.mod b/go.mod index 7c078872ce..34a7942bf4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/microsoft/retina -go 1.22.7 +go 1.23 toolchain go1.23.1 @@ -10,7 +10,7 @@ require ( github.com/prometheus/client_golang v1.20.4 github.com/spf13/cobra v1.8.1 go.uber.org/zap v1.27.0 - k8s.io/client-go v0.30.3 + k8s.io/client-go v0.31.1 sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.49 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.20 ) @@ -98,6 +98,7 @@ require ( github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect @@ -168,7 +169,7 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/moby v26.0.0+incompatible // indirect - github.com/moby/spdystream v0.2.0 // indirect + github.com/moby/spdystream v0.4.0 // indirect github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/signal v0.7.0 // indirect @@ -189,7 +190,6 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/s3rj1k/go-fanotify/fanotify v0.0.0-20210917134616-9c00a300bb7a // indirect @@ -209,6 +209,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/vishvananda/netns v0.0.4 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -232,6 +233,7 @@ require ( google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect k8s.io/apiserver v0.30.3 // indirect @@ -264,11 +266,11 @@ require ( google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.30.3 - k8s.io/apimachinery v0.30.3 - k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/api v0.31.1 + k8s.io/apimachinery v0.31.1 + k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240102154912-e7106e64919e + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 @@ -312,6 +314,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.59.1 + github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240921231226-2a794a9dcf0e github.com/safchain/ethtool v0.4.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 @@ -332,4 +335,6 @@ require ( sigs.k8s.io/controller-runtime v0.18.5 ) +require github.com/rogpeppe/go-internal v1.12.0 // indirect + replace github.com/vishvananda/netns => github.com/inspektor-gadget/netns v0.0.5-0.20230524185006-155d84c555d6 diff --git a/go.sum b/go.sum index 117e675f3f..2a8bf6f7e2 100644 --- a/go.sum +++ b/go.sum @@ -306,6 +306,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= @@ -443,7 +445,6 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= @@ -656,8 +657,8 @@ github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/moby v26.0.0+incompatible h1:2n9/cIWkxiEI1VsWgTGgXhxIWUbv42PyxEP9L+RReC0= github.com/moby/moby v26.0.0+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= +github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= @@ -759,6 +760,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240921231226-2a794a9dcf0e h1:V13W3zJXaQ98OKFqMWnodX1/JQS1V9isnKKkFdsx8dA= +github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240921231226-2a794a9dcf0e/go.mod h1:53va7JZ55oNRxC/vGA1sukvZRJQWXtmOdY9Kczv0T98= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -843,6 +846,8 @@ github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vishvananda/netlink v1.2.1-beta.2.0.20240524165444-4d4ba1473f21 h1:tcHUxOT8j/R+0S+A1j8D2InqguXFNxAiij+8QFOlX7Y= github.com/vishvananda/netlink v1.2.1-beta.2.0.20240524165444-4d4ba1473f21/go.mod h1:whJevzBpTrid75eZy99s3DqCmy05NfibNaF2Ol5Ox5A= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1139,6 +1144,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/dnaeon/go-vcr.v3 v3.2.0 h1:Rltp0Vf+Aq0u4rQXgmXgtgoRDStTnFN83cWgSGSoRzM= gopkg.in/dnaeon/go-vcr.v3 v3.2.0/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -1167,30 +1174,30 @@ helm.sh/helm/v3 v3.15.4 h1:UFHd6oZ1IN3FsUZ7XNhOQDyQ2QYknBNWRHH57e9cbHY= helm.sh/helm/v3 v3.15.4/go.mod h1:phOwlxqGSgppCY/ysWBNRhG3MtnpsttOzxaTK+Mt40E= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= -k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= +k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= +k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= -k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= -k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= +k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/apiserver v0.30.3 h1:QZJndA9k2MjFqpnyYv/PH+9PE0SHhx3hBho4X0vE65g= k8s.io/apiserver v0.30.3/go.mod h1:6Oa88y1CZqnzetd2JdepO0UXzQX4ZnOekx2/PtEjrOg= k8s.io/cli-runtime v0.30.3 h1:aG69oRzJuP2Q4o8dm+f5WJIX4ZBEwrvdID0+MXyUY6k= k8s.io/cli-runtime v0.30.3/go.mod h1:hwrrRdd9P84CXSKzhHxrOivAR9BRnkMt0OeP5mj7X30= -k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= -k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= +k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= +k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= k8s.io/component-base v0.30.3 h1:Ci0UqKWf4oiwy8hr1+E3dsnliKnkMLZMVbWzeorlk7s= k8s.io/component-base v0.30.3/go.mod h1:C1SshT3rGPCuNtBs14RmVD2xW0EhRSeLvBh7AGk1quA= k8s.io/cri-api v0.30.1 h1:AUM78wiC56B1WJ2c795AS0IG5T57CkEdkn0IuC+miAE= k8s.io/cri-api v0.30.1/go.mod h1://4/umPJSW1ISNSNng4OwjpkvswJOQwU8rnkvO8P+xg= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/kubectl v0.30.3 h1:YIBBvMdTW0xcDpmrOBzcpUVsn+zOgjMYIu7kAq+yqiI= k8s.io/kubectl v0.30.3/go.mod h1:IcR0I9RN2+zzTRUa1BzZCm4oM0NLOawE6RzlDvd1Fpo= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.49 h1:3AsgRzy3mz/AOsofFsuzQFs2paHACDl2J0Ibd6dh+5Q= diff --git a/test/e2e/jobs/jobs.go b/test/e2e/jobs/jobs.go index eee000c415..345530e490 100644 --- a/test/e2e/jobs/jobs.go +++ b/test/e2e/jobs/jobs.go @@ -1,6 +1,9 @@ package retina import ( + "fmt" + "time" + "github.com/microsoft/retina/test/e2e/framework/azure" "github.com/microsoft/retina/test/e2e/framework/generic" "github.com/microsoft/retina/test/e2e/framework/kubernetes" @@ -8,6 +11,7 @@ import ( "github.com/microsoft/retina/test/e2e/scenarios/dns" "github.com/microsoft/retina/test/e2e/scenarios/drop" "github.com/microsoft/retina/test/e2e/scenarios/latency" + "github.com/microsoft/retina/test/e2e/scenarios/perf" tcp "github.com/microsoft/retina/test/e2e/scenarios/tcp" ) @@ -50,6 +54,26 @@ func CreateTestInfra(subID, clusterName, location, kubeConfigFilePath string) *t return job } +func RegisterExistingInfra(subID, clusterName, resGroupName, location, kubeConfigFilePath string) *types.Job { + job := types.NewJob("Register existing e2e test infrastructure") + + job.AddStep(&azure.GetAKSKubeConfig{ + SubscriptionID: subID, + ResourceGroupName: resGroupName, + ClusterName: clusterName, + Location: location, + KubeConfigFilePath: kubeConfigFilePath, + }, nil) + + job.AddStep(&generic.LoadFlags{ + TagEnv: generic.DefaultTagEnv, + ImageNamespaceEnv: generic.DefaultImageNamespace, + ImageRegistryEnv: generic.DefaultImageRegistry, + }, nil) + + return job +} + func DeleteTestInfra(subID, clusterName, location string) *types.Job { job := types.NewJob("Delete e2e test infrastructure") @@ -186,3 +210,43 @@ func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFi return job } + +func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job { + job := types.NewJob("Run performance tests") + + benchmarkFile := fmt.Sprintf("netperf-benchmark-%s.json", time.Now().Format("20060102150405")) + resultFile := fmt.Sprintf("netperf-result-%s.json", time.Now().Format("20060102150405")) + + job.AddStep(&perf.GetNetworkPerformanceMeasures{ + KubeConfigFilePath: kubeConfigFilePath, + ResultTag: "no-retina", + JsonOutputFile: benchmarkFile, + }, &types.StepOptions{ + SkipSavingParametersToJob: true, + }) + + job.AddStep(&kubernetes.InstallHelmChart{ + Namespace: "kube-system", + ReleaseName: "retina", + KubeConfigFilePath: kubeConfigFilePath, + ChartPath: chartPath, + TagEnv: generic.DefaultTagEnv, + }, nil) + + job.AddStep(&perf.GetNetworkPerformanceMeasures{ + KubeConfigFilePath: kubeConfigFilePath, + ResultTag: "retina", + JsonOutputFile: resultFile, + }, &types.StepOptions{ + SkipSavingParametersToJob: true, + }) + + job.AddStep(&perf.GetNetworkRegressionResults{ + BaseResultsFile: benchmarkFile, + NewResultsFile: resultFile, + }, &types.StepOptions{ + SkipSavingParametersToJob: true, + }) + + return job +} diff --git a/test/e2e/retina_perf_test.go b/test/e2e/retina_perf_test.go new file mode 100644 index 0000000000..be4bb9738a --- /dev/null +++ b/test/e2e/retina_perf_test.go @@ -0,0 +1,82 @@ +package retina + +import ( + "crypto/rand" + "math/big" + "os" + "os/user" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/microsoft/retina/test/e2e/common" + "github.com/microsoft/retina/test/e2e/framework/types" + jobs "github.com/microsoft/retina/test/e2e/jobs" + "github.com/stretchr/testify/require" +) + +// TestE2ERetina tests all e2e scenarios for retina +func TestPerfRetina(t *testing.T) { + curuser, err := user.Current() + require.NoError(t, err) + + clusterName := curuser.Username + common.NetObsRGtag + strconv.FormatInt(time.Now().Unix(), 10) + + subID := os.Getenv("AZURE_SUBSCRIPTION_ID") + require.NotEmpty(t, subID) + + location := os.Getenv("AZURE_LOCATION") + if location == "" { + var nBig *big.Int + nBig, err = rand.Int(rand.Reader, big.NewInt(int64(len(locations)))) + if err != nil { + t.Fatalf("Failed to generate a secure random index: %v", err) + } + location = locations[nBig.Int64()] + } + + cwd, err := os.Getwd() + require.NoError(t, err) + + // Get to root of the repo by going up two directories + rootDir := filepath.Dir(filepath.Dir(cwd)) + + chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina") + kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem") + + useExistingInfra, err := strconv.ParseBool(os.Getenv("USE_EXISTING_INFRA")) + if err != nil { + useExistingInfra = false + } + if useExistingInfra { + clusterName = os.Getenv("CLUSTER_NAME") + require.NotEmpty(t, clusterName) + resGroupName := os.Getenv("RESOURCE_GROUP_NAME") + require.NotEmpty(t, resGroupName) + // RegisterExistingInfra + registerExistingInfra := types.NewRunner(t, jobs.RegisterExistingInfra(subID, clusterName, resGroupName, location, kubeConfigFilePath)) + registerExistingInfra.Run() + } else { + // CreateTestInfra + createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, clusterName, location, kubeConfigFilePath)) + createTestInfra.Run() + } + + // sleep for 2 minutes to ensure that the cluster is up and running + time.Sleep(2 * time.Minute) + + // Hacky way to ensure that the test infra is deleted even if the test panics + defer func() { + if r := recover(); r != nil { + t.Logf("Recovered in TestE2ERetina, %v", r) + } + if !useExistingInfra { + _ = jobs.DeleteTestInfra(subID, clusterName, location).Run() + } + }() + + // Gather benchmark results then install retina and run the performance tests + runner := types.NewRunner(t, jobs.RunPerfTest(kubeConfigFilePath, chartPath)) + runner.Run() +} diff --git a/test/e2e/scenarios/perf/get-network-performance-measures.go b/test/e2e/scenarios/perf/get-network-performance-measures.go new file mode 100644 index 0000000000..4ccce4a3cf --- /dev/null +++ b/test/e2e/scenarios/perf/get-network-performance-measures.go @@ -0,0 +1,84 @@ +package perf + +import ( + "fmt" + "io" + "os" + + "github.com/ritwikranjan/perf-tests/network/benchmarks/netperf/lib" +) + +type GetNetworkPerformanceMeasures struct { + ResultTag string + KubeConfigFilePath string + JsonOutputFile string +} + +func (v *GetNetworkPerformanceMeasures) Prevalidate() error { + return nil +} + +func (v *GetNetworkPerformanceMeasures) Run() error { + results, err := lib.PerformTests(lib.TestParams{ + Iterations: 1, + Image: "ghcr.io/ritwikranjan/nptest", + TestNamespace: "netperf", + TestFrom: 13, + TestTo: 17, + JsonOutput: true, + Tag: v.ResultTag, + KubeConfig: v.KubeConfigFilePath, + }) + if err != nil { + return fmt.Errorf("failed to get network performance measures: %v", err) + } + sourceJsonOutputFile := results[0].JsonResultFile + err = copyFile(sourceJsonOutputFile, v.JsonOutputFile) + if err != nil { + return fmt.Errorf("failed to copy json output file: %v", err) + } + defer deleteFile(sourceJsonOutputFile) + defer deleteFile(results[0].CsvResultFile) + return nil +} + +func (v *GetNetworkPerformanceMeasures) Stop() error { + return nil +} + +func copyFile(src, dst string) error { + // Open the source file + sourceFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open source file: %v", err) + } + defer sourceFile.Close() + + // Create the destination file + destinationFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("failed to create destination file: %v", err) + } + defer destinationFile.Close() + + // Copy the contents from source to destination + _, err = io.Copy(destinationFile, sourceFile) + if err != nil { + return fmt.Errorf("failed to copy file: %v", err) + } + + // Flush the destination file to ensure all data is written + err = destinationFile.Sync() + if err != nil { + return fmt.Errorf("failed to sync destination file: %v", err) + } + + return nil +} + +func deleteFile(filePath string) { + err := os.Remove(filePath) + if err != nil { + fmt.Println("Failed to delete file: ", err) + } +} diff --git a/test/e2e/scenarios/perf/get-perf-regression-results.go b/test/e2e/scenarios/perf/get-perf-regression-results.go new file mode 100644 index 0000000000..213fd734a2 --- /dev/null +++ b/test/e2e/scenarios/perf/get-perf-regression-results.go @@ -0,0 +1,168 @@ +package perf + +import ( + "encoding/json" + "fmt" + "io" + "os" + "time" +) + +type CPUUtilization struct { + Host float64 `json:"host"` + Remote float64 `json:"remote"` +} + +type TestResult struct { + TotalThroughput float64 `json:"total_throughput"` + MeanRTT int `json:"mean_rtt,omitempty"` + MinRTT int `json:"min_rtt,omitempty"` + MaxRTT int `json:"max_rtt,omitempty"` + Retransmits int `json:"retransmits,omitempty"` + JitterMs float64 `json:"jitter_ms,omitempty"` + LostPackets int `json:"lost_packets,omitempty"` + TotalPackets int `json:"total_packets,omitempty"` + LostPercent float64 `json:"lost_percent,omitempty"` + CPUUtilization CPUUtilization `json:"cpu_utilization"` +} + +type TestCase struct { + Label string `json:"label"` + Result TestResult `json:"result"` +} + +type RegressionResult struct { + Label string `json:"label"` + Metric string `json:"metric"` + BaseValue float64 `json:"base_value"` + NewValue float64 `json:"new_value"` + Regression float64 `json:"regression"` +} + +type AggregatedResult struct { + Label string `json:"label"` + BaseMetrics map[string]float64 `json:"base_metrics"` + Metrics map[string]float64 `json:"metrics"` + Regressions map[string]float64 `json:"regressions"` +} + +type GetNetworkRegressionResults struct { + BaseResultsFile string + NewResultsFile string +} + +func (v *GetNetworkRegressionResults) Prevalidate() error { + return nil +} + +func (v *GetNetworkRegressionResults) Run() error { + testCases1, err := readJSONFile(v.BaseResultsFile) + if err != nil { + return fmt.Errorf("error reading file %s: %v", v.BaseResultsFile, err) + } + + testCases2, err := readJSONFile(v.NewResultsFile) + if err != nil { + return fmt.Errorf("error reading file %s: %v", v.NewResultsFile, err) + } + + if len(testCases1) != len(testCases2) { + return fmt.Errorf("number of test cases in the two files do not match") + } + + aggregatedResults := make(map[string]*AggregatedResult) + + for i := range testCases1 { + tc1 := testCases1[i] + tc2 := testCases2[i] + + if _, exists := aggregatedResults[tc1.Label]; !exists { + aggregatedResults[tc1.Label] = &AggregatedResult{ + Label: tc1.Label, + BaseMetrics: make(map[string]float64), + Metrics: make(map[string]float64), + Regressions: make(map[string]float64), + } + } + + metrics := []struct { + name string + oldVal float64 + newVal float64 + }{ + {"Total Throughput", tc1.Result.TotalThroughput, tc2.Result.TotalThroughput}, + {"Mean RTT", float64(tc1.Result.MeanRTT), float64(tc2.Result.MeanRTT)}, + {"Min RTT", float64(tc1.Result.MinRTT), float64(tc2.Result.MinRTT)}, + {"Max RTT", float64(tc1.Result.MaxRTT), float64(tc2.Result.MaxRTT)}, + {"Retransmits", float64(tc1.Result.Retransmits), float64(tc2.Result.Retransmits)}, + {"Jitter (ms)", tc1.Result.JitterMs, tc2.Result.JitterMs}, + {"Lost Packets", float64(tc1.Result.LostPackets), float64(tc2.Result.LostPackets)}, + {"Lost Percent", tc1.Result.LostPercent, tc2.Result.LostPercent}, + {"CPU Utilization Host", tc1.Result.CPUUtilization.Host, tc2.Result.CPUUtilization.Host}, + {"CPU Utilization Remote", tc1.Result.CPUUtilization.Remote, tc2.Result.CPUUtilization.Remote}, + } + + for _, metric := range metrics { + if metric.oldVal != 0 || metric.newVal != 0 { + aggregatedResults[tc1.Label].BaseMetrics[metric.name] = metric.oldVal + aggregatedResults[tc1.Label].Metrics[metric.name] = metric.newVal + aggregatedResults[tc1.Label].Regressions[metric.name] = calculateRegression(metric.oldVal, metric.newVal) + } + } + } + + var results []AggregatedResult + for _, result := range aggregatedResults { + results = append(results, *result) + } + + outputFile := fmt.Sprintf("network-regression-results-%s.json", time.Now().Format("20060102150405")) + file, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("error creating file %s: %v", outputFile, err) + } + defer file.Close() + + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + if err := encoder.Encode(results); err != nil { + return fmt.Errorf("error encoding results to JSON: %v", err) + } + + defer deleteFile(v.BaseResultsFile) + defer deleteFile(v.NewResultsFile) + + return nil +} + +func (v *GetNetworkRegressionResults) Stop() error { + return nil +} + +func readJSONFile(filename string) ([]TestCase, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + byteValue, err := io.ReadAll(file) + if err != nil { + return nil, err + } + + var testCases []TestCase + err = json.Unmarshal(byteValue, &testCases) + if err != nil { + return nil, err + } + + return testCases, nil +} + +func calculateRegression(old, new float64) float64 { + if old == 0 { + return 0 + } + return ((new - old) / old) * 100 +} From 8d90d10f1a4644023312b68d12882af896b29017 Mon Sep 17 00:00:00 2001 From: Ritwik Ranjan Date: Fri, 27 Sep 2024 12:48:40 +0000 Subject: [PATCH 2/6] address PR comments and update perf-tests dependency Signed-off-by: Ritwik Ranjan --- Makefile | 3 + go.mod | 16 ++- go.sum | 28 ++-- test/e2e/common/common.go | 2 + test/e2e/retina_perf_test.go | 32 ++--- .../perf/get-network-performance-measures.go | 28 ++-- .../perf/get-perf-regression-results.go | 136 +++++++++--------- 7 files changed, 123 insertions(+), 122 deletions(-) diff --git a/Makefile b/Makefile index 80298f45b8..ac278a5d9b 100644 --- a/Makefile +++ b/Makefile @@ -537,3 +537,6 @@ quick-deploy-hubble: .PHONY: simplify-dashboards simplify-dashboards: cd deploy/testutils && go test ./... -tags=dashboard,simplifydashboard -v && cd $(REPO_ROOT) + +run-perf-test: + go test -v ./test/e2e/retina_perf_test.go -timeout 2h -tags=e2e -count=1 -args -image-tag=${TAG} -image-registry=${IMAGE_REGISTRY} -image-namespace=${IMAGE_NAMESPACE} diff --git a/go.mod b/go.mod index 34a7942bf4..f7c30af56d 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.2 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect @@ -185,7 +185,7 @@ require ( github.com/opencontainers/selinux v1.11.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -258,8 +258,8 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/oauth2 v0.22.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.8.0 golang.org/x/sys v0.25.0 golang.org/x/term v0.24.0 // indirect @@ -269,8 +269,8 @@ require ( k8s.io/api v0.31.1 k8s.io/apimachinery v0.31.1 k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect + k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 @@ -314,7 +314,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.59.1 - github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240921231226-2a794a9dcf0e github.com/safchain/ethtool v0.4.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 @@ -332,9 +331,12 @@ require ( k8s.io/apiextensions-apiserver v0.30.3 k8s.io/cli-runtime v0.30.3 k8s.io/kubectl v0.30.3 + k8s.io/perf-tests/network/benchmarks/netperf v0.0.0-00010101000000-000000000000 sigs.k8s.io/controller-runtime v0.18.5 ) require github.com/rogpeppe/go-internal v1.12.0 // indirect replace github.com/vishvananda/netns => github.com/inspektor-gadget/netns v0.0.5-0.20230524185006-155d84c555d6 + +replace k8s.io/perf-tests/network/benchmarks/netperf => github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240926175158-d0a61c755609 diff --git a/go.sum b/go.sum index 2a8bf6f7e2..758b2ab1c9 100644 --- a/go.sum +++ b/go.sum @@ -271,8 +271,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+BtnqkLAU= -github.com/emicklei/go-restful/v3 v3.11.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -717,8 +717,8 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible h1:FhnA4iH8T/yYW+AolPONZjGE897wxj3MAzfEbrZkSYw= +github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= @@ -760,8 +760,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240921231226-2a794a9dcf0e h1:V13W3zJXaQ98OKFqMWnodX1/JQS1V9isnKKkFdsx8dA= -github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240921231226-2a794a9dcf0e/go.mod h1:53va7JZ55oNRxC/vGA1sukvZRJQWXtmOdY9Kczv0T98= +github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240926175158-d0a61c755609 h1:YTQYpu9p6MjFIRJNZpCV3EQ2DTyQboKNQWV3WOQYLYU= +github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240926175158-d0a61c755609/go.mod h1:PUa9msVr70tIWPb+i5a68SNFbyiEBEcoQbgFHaTEEL0= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -972,12 +972,12 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= -golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1192,12 +1192,12 @@ k8s.io/cri-api v0.30.1 h1:AUM78wiC56B1WJ2c795AS0IG5T57CkEdkn0IuC+miAE= k8s.io/cri-api v0.30.1/go.mod h1://4/umPJSW1ISNSNng4OwjpkvswJOQwU8rnkvO8P+xg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= k8s.io/kubectl v0.30.3 h1:YIBBvMdTW0xcDpmrOBzcpUVsn+zOgjMYIu7kAq+yqiI= k8s.io/kubectl v0.30.3/go.mod h1:IcR0I9RN2+zzTRUa1BzZCm4oM0NLOawE6RzlDvd1Fpo= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY9mD9fNT47QO6HI= +k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.49 h1:3AsgRzy3mz/AOsofFsuzQFs2paHACDl2J0Ibd6dh+5Q= diff --git a/test/e2e/common/common.go b/test/e2e/common/common.go index 85f394f8b1..d12511b237 100644 --- a/test/e2e/common/common.go +++ b/test/e2e/common/common.go @@ -9,3 +9,5 @@ const ( // netObsRGtag is used to tag resources created by this test suite NetObsRGtag = "-e2e-netobs-" ) + +var AzureLocations = []string{"eastus2", "centralus", "southcentralus", "uksouth", "centralindia", "westus2"} diff --git a/test/e2e/retina_perf_test.go b/test/e2e/retina_perf_test.go index be4bb9738a..6c67c5e47c 100644 --- a/test/e2e/retina_perf_test.go +++ b/test/e2e/retina_perf_test.go @@ -29,11 +29,11 @@ func TestPerfRetina(t *testing.T) { location := os.Getenv("AZURE_LOCATION") if location == "" { var nBig *big.Int - nBig, err = rand.Int(rand.Reader, big.NewInt(int64(len(locations)))) + nBig, err = rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations)))) if err != nil { t.Fatalf("Failed to generate a secure random index: %v", err) } - location = locations[nBig.Int64()] + location = common.AzureLocations[nBig.Int64()] } cwd, err := os.Getwd() @@ -45,34 +45,18 @@ func TestPerfRetina(t *testing.T) { chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina") kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem") - useExistingInfra, err := strconv.ParseBool(os.Getenv("USE_EXISTING_INFRA")) - if err != nil { - useExistingInfra = false - } - if useExistingInfra { - clusterName = os.Getenv("CLUSTER_NAME") - require.NotEmpty(t, clusterName) - resGroupName := os.Getenv("RESOURCE_GROUP_NAME") - require.NotEmpty(t, resGroupName) - // RegisterExistingInfra - registerExistingInfra := types.NewRunner(t, jobs.RegisterExistingInfra(subID, clusterName, resGroupName, location, kubeConfigFilePath)) - registerExistingInfra.Run() - } else { - // CreateTestInfra - createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, clusterName, location, kubeConfigFilePath)) - createTestInfra.Run() - } - - // sleep for 2 minutes to ensure that the cluster is up and running - time.Sleep(2 * time.Minute) + // CreateTestInfra + createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, clusterName, location, kubeConfigFilePath)) + createTestInfra.Run() // Hacky way to ensure that the test infra is deleted even if the test panics defer func() { if r := recover(); r != nil { t.Logf("Recovered in TestE2ERetina, %v", r) } - if !useExistingInfra { - _ = jobs.DeleteTestInfra(subID, clusterName, location).Run() + err := jobs.DeleteTestInfra(subID, clusterName, location).Run() + if err != nil { + t.Logf("Failed to delete test infrastructure: %v", err) } }() diff --git a/test/e2e/scenarios/perf/get-network-performance-measures.go b/test/e2e/scenarios/perf/get-network-performance-measures.go index 4ccce4a3cf..bd26e4915a 100644 --- a/test/e2e/scenarios/perf/get-network-performance-measures.go +++ b/test/e2e/scenarios/perf/get-network-performance-measures.go @@ -5,7 +5,7 @@ import ( "io" "os" - "github.com/ritwikranjan/perf-tests/network/benchmarks/netperf/lib" + "k8s.io/perf-tests/network/benchmarks/netperf/lib" ) type GetNetworkPerformanceMeasures struct { @@ -33,12 +33,10 @@ func (v *GetNetworkPerformanceMeasures) Run() error { return fmt.Errorf("failed to get network performance measures: %v", err) } sourceJsonOutputFile := results[0].JsonResultFile - err = copyFile(sourceJsonOutputFile, v.JsonOutputFile) + err = moveFile(sourceJsonOutputFile, v.JsonOutputFile) if err != nil { return fmt.Errorf("failed to copy json output file: %v", err) } - defer deleteFile(sourceJsonOutputFile) - defer deleteFile(results[0].CsvResultFile) return nil } @@ -46,8 +44,21 @@ func (v *GetNetworkPerformanceMeasures) Stop() error { return nil } +func moveFile(src, dst string) error { + err := copyFile(src, dst) + if err != nil { + return fmt.Errorf("failed to move file: %v", err) + } + + err = os.Remove(src) + if err != nil { + fmt.Printf("warning: failed to delete source file %s: %v\n", src, err) + } + + return nil +} + func copyFile(src, dst string) error { - // Open the source file sourceFile, err := os.Open(src) if err != nil { return fmt.Errorf("failed to open source file: %v", err) @@ -75,10 +86,3 @@ func copyFile(src, dst string) error { return nil } - -func deleteFile(filePath string) { - err := os.Remove(filePath) - if err != nil { - fmt.Println("Failed to delete file: ", err) - } -} diff --git a/test/e2e/scenarios/perf/get-perf-regression-results.go b/test/e2e/scenarios/perf/get-perf-regression-results.go index 213fd734a2..c5db9b1dd0 100644 --- a/test/e2e/scenarios/perf/get-perf-regression-results.go +++ b/test/e2e/scenarios/perf/get-perf-regression-results.go @@ -8,41 +8,42 @@ import ( "time" ) +type TestInfo struct { + Protocol string `json:"protocol"` + Streams int `json:"streams"` + Blksize int `json:"blksize"` + Duration int `json:"duration"` +} + type CPUUtilization struct { - Host float64 `json:"host"` - Remote float64 `json:"remote"` + HostTotal float64 `json:"host_total"` + RemoteTotal float64 `json:"remote_total"` } -type TestResult struct { +type Result struct { + TestInfo TestInfo `json:"test_info"` TotalThroughput float64 `json:"total_throughput"` - MeanRTT int `json:"mean_rtt,omitempty"` - MinRTT int `json:"min_rtt,omitempty"` - MaxRTT int `json:"max_rtt,omitempty"` - Retransmits int `json:"retransmits,omitempty"` - JitterMs float64 `json:"jitter_ms,omitempty"` - LostPackets int `json:"lost_packets,omitempty"` - TotalPackets int `json:"total_packets,omitempty"` - LostPercent float64 `json:"lost_percent,omitempty"` + MeanRTT float64 `json:"mean_rtt"` + MinRTT float64 `json:"min_rtt"` + MaxRTT float64 `json:"max_rtt"` + Retransmits int `json:"retransmits"` CPUUtilization CPUUtilization `json:"cpu_utilization"` + JitterMs float64 `json:"jitter_ms"` + LostPackets int `json:"lost_packets"` + LostPercent float64 `json:"lost_percent"` + OutofOrder int `json:"out_of_order"` } -type TestCase struct { - Label string `json:"label"` - Result TestResult `json:"result"` +type TestResult struct { + Label string `json:"label"` + Result Result `json:"result"` } type RegressionResult struct { - Label string `json:"label"` - Metric string `json:"metric"` - BaseValue float64 `json:"base_value"` - NewValue float64 `json:"new_value"` - Regression float64 `json:"regression"` -} - -type AggregatedResult struct { Label string `json:"label"` - BaseMetrics map[string]float64 `json:"base_metrics"` - Metrics map[string]float64 `json:"metrics"` + TestInfo TestInfo `json:"test_info"` + Benchmark map[string]float64 `json:"benchmark"` + Result map[string]float64 `json:"result"` Regressions map[string]float64 `json:"regressions"` } @@ -56,63 +57,69 @@ func (v *GetNetworkRegressionResults) Prevalidate() error { } func (v *GetNetworkRegressionResults) Run() error { - testCases1, err := readJSONFile(v.BaseResultsFile) + benchmarkResults, err := readJSONFile(v.BaseResultsFile) if err != nil { - return fmt.Errorf("error reading file %s: %v", v.BaseResultsFile, err) + return fmt.Errorf("failed to read benchmark results file: %v", err) } - testCases2, err := readJSONFile(v.NewResultsFile) + newResults, err := readJSONFile(v.NewResultsFile) if err != nil { - return fmt.Errorf("error reading file %s: %v", v.NewResultsFile, err) + return fmt.Errorf("failed to read new results file: %v", err) } - if len(testCases1) != len(testCases2) { - return fmt.Errorf("number of test cases in the two files do not match") + if len(benchmarkResults) != len(newResults) { + return fmt.Errorf("number of tests in benchmark results and new results do not match") } - aggregatedResults := make(map[string]*AggregatedResult) + regressionResults := make(map[string]*RegressionResult) - for i := range testCases1 { - tc1 := testCases1[i] - tc2 := testCases2[i] + for i := range benchmarkResults { + benchmarkResult := benchmarkResults[i] + newResult := newResults[i] - if _, exists := aggregatedResults[tc1.Label]; !exists { - aggregatedResults[tc1.Label] = &AggregatedResult{ - Label: tc1.Label, - BaseMetrics: make(map[string]float64), - Metrics: make(map[string]float64), + if benchmarkResult.Label != newResult.Label { + return fmt.Errorf("test labels do not match") + } + + if _, exists := regressionResults[benchmarkResults[i].Label]; !exists { + regressionResults[benchmarkResults[i].Label] = &RegressionResult{ + Label: benchmarkResults[i].Label, + TestInfo: benchmarkResults[i].Result.TestInfo, + Benchmark: make(map[string]float64), + Result: make(map[string]float64), Regressions: make(map[string]float64), } } metrics := []struct { - name string - oldVal float64 - newVal float64 + name string + benchmark float64 + result float64 }{ - {"Total Throughput", tc1.Result.TotalThroughput, tc2.Result.TotalThroughput}, - {"Mean RTT", float64(tc1.Result.MeanRTT), float64(tc2.Result.MeanRTT)}, - {"Min RTT", float64(tc1.Result.MinRTT), float64(tc2.Result.MinRTT)}, - {"Max RTT", float64(tc1.Result.MaxRTT), float64(tc2.Result.MaxRTT)}, - {"Retransmits", float64(tc1.Result.Retransmits), float64(tc2.Result.Retransmits)}, - {"Jitter (ms)", tc1.Result.JitterMs, tc2.Result.JitterMs}, - {"Lost Packets", float64(tc1.Result.LostPackets), float64(tc2.Result.LostPackets)}, - {"Lost Percent", tc1.Result.LostPercent, tc2.Result.LostPercent}, - {"CPU Utilization Host", tc1.Result.CPUUtilization.Host, tc2.Result.CPUUtilization.Host}, - {"CPU Utilization Remote", tc1.Result.CPUUtilization.Remote, tc2.Result.CPUUtilization.Remote}, + {"total_throughput", benchmarkResult.Result.TotalThroughput, newResult.Result.TotalThroughput}, + {"mean_rtt", benchmarkResult.Result.MeanRTT, newResult.Result.MeanRTT}, + {"min_rtt", benchmarkResult.Result.MinRTT, newResult.Result.MinRTT}, + {"max_rtt", benchmarkResult.Result.MaxRTT, newResult.Result.MaxRTT}, + {"retransmits", float64(benchmarkResult.Result.Retransmits), float64(newResult.Result.Retransmits)}, + {"jitter_ms", benchmarkResult.Result.JitterMs, newResult.Result.JitterMs}, + {"lost_packets", float64(benchmarkResult.Result.LostPackets), float64(newResult.Result.LostPackets)}, + {"lost_percent", benchmarkResult.Result.LostPercent, newResult.Result.LostPercent}, + {"out_of_order", float64(benchmarkResult.Result.OutofOrder), float64(newResult.Result.OutofOrder)}, + {"host_total_cpu", benchmarkResult.Result.CPUUtilization.HostTotal, newResult.Result.CPUUtilization.HostTotal}, + {"remote_total_cpu", benchmarkResult.Result.CPUUtilization.RemoteTotal, newResult.Result.CPUUtilization.RemoteTotal}, } for _, metric := range metrics { - if metric.oldVal != 0 || metric.newVal != 0 { - aggregatedResults[tc1.Label].BaseMetrics[metric.name] = metric.oldVal - aggregatedResults[tc1.Label].Metrics[metric.name] = metric.newVal - aggregatedResults[tc1.Label].Regressions[metric.name] = calculateRegression(metric.oldVal, metric.newVal) + if metric.benchmark != 0 && metric.result != 0 { + regressionResults[benchmarkResult.Label].Benchmark[metric.name] = metric.benchmark + regressionResults[benchmarkResult.Label].Result[metric.name] = metric.result + regressionResults[benchmarkResult.Label].Regressions[metric.name] = calculateRegression(metric.benchmark, metric.result) } } } - var results []AggregatedResult - for _, result := range aggregatedResults { + var results []RegressionResult + for _, result := range regressionResults { results = append(results, *result) } @@ -125,12 +132,11 @@ func (v *GetNetworkRegressionResults) Run() error { encoder := json.NewEncoder(file) encoder.SetIndent("", " ") - if err := encoder.Encode(results); err != nil { - return fmt.Errorf("error encoding results to JSON: %v", err) - } - defer deleteFile(v.BaseResultsFile) - defer deleteFile(v.NewResultsFile) + err = encoder.Encode(results) + if err != nil { + return fmt.Errorf("error encoding regression results: %v", err) + } return nil } @@ -139,7 +145,7 @@ func (v *GetNetworkRegressionResults) Stop() error { return nil } -func readJSONFile(filename string) ([]TestCase, error) { +func readJSONFile(filename string) ([]TestResult, error) { file, err := os.Open(filename) if err != nil { return nil, err @@ -151,7 +157,7 @@ func readJSONFile(filename string) ([]TestCase, error) { return nil, err } - var testCases []TestCase + var testCases []TestResult err = json.Unmarshal(byteValue, &testCases) if err != nil { return nil, err From 34b158ac8451aa1cd1588aeaa2ad43fefe46058b Mon Sep 17 00:00:00 2001 From: Ritwik Ranjan Date: Tue, 1 Oct 2024 10:47:37 +0000 Subject: [PATCH 3/6] fix: remove go.mod version bump --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ca23b1f68a..280abb7d0e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/microsoft/retina -go 1.23 +go 1.22.7 toolchain go1.23.1 @@ -339,4 +339,4 @@ require github.com/rogpeppe/go-internal v1.12.0 // indirect replace github.com/vishvananda/netns => github.com/inspektor-gadget/netns v0.0.5-0.20230524185006-155d84c555d6 -replace k8s.io/perf-tests/network/benchmarks/netperf => github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240926175158-d0a61c755609 +replace k8s.io/perf-tests/network/benchmarks/netperf => github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001103703-41323364781f diff --git a/go.sum b/go.sum index a660395bf3..8fd53d3005 100644 --- a/go.sum +++ b/go.sum @@ -760,8 +760,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240926175158-d0a61c755609 h1:YTQYpu9p6MjFIRJNZpCV3EQ2DTyQboKNQWV3WOQYLYU= -github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20240926175158-d0a61c755609/go.mod h1:PUa9msVr70tIWPb+i5a68SNFbyiEBEcoQbgFHaTEEL0= +github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001103703-41323364781f h1:fbcRfs8KpPJdjYBU2xuP53K90rVisQgWqsfUnmSuhcg= +github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001103703-41323364781f/go.mod h1:uNVSmziO8eBGgZ9UKBt3Yv9Uvw97T/1nh08FjgWN/fM= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= From 043200388c5bb7715a0f72764eaea3e7a3afbddb Mon Sep 17 00:00:00 2001 From: Ritwik Ranjan Date: Tue, 1 Oct 2024 12:49:07 +0000 Subject: [PATCH 4/6] fix: downgrading the accidently upgraded k8s modules --- go.mod | 11 ++++------- go.sum | 22 ++++++++-------------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 280abb7d0e..ffdb0327e3 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/prometheus/client_golang v1.20.4 github.com/spf13/cobra v1.8.1 go.uber.org/zap v1.27.0 - k8s.io/client-go v0.31.1 + k8s.io/client-go v0.30.3 sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.49 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.20 ) @@ -98,7 +98,6 @@ require ( github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect @@ -209,7 +208,6 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/vishvananda/netns v0.0.4 // indirect - github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -233,7 +231,6 @@ require ( google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect k8s.io/apiserver v0.30.3 // indirect @@ -266,8 +263,8 @@ require ( google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.31.1 - k8s.io/apimachinery v0.31.1 + k8s.io/api v0.30.3 + k8s.io/apimachinery v0.30.3 k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 @@ -339,4 +336,4 @@ require github.com/rogpeppe/go-internal v1.12.0 // indirect replace github.com/vishvananda/netns => github.com/inspektor-gadget/netns v0.0.5-0.20230524185006-155d84c555d6 -replace k8s.io/perf-tests/network/benchmarks/netperf => github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001103703-41323364781f +replace k8s.io/perf-tests/network/benchmarks/netperf => github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001122546-074cd3ab166f diff --git a/go.sum b/go.sum index 8fd53d3005..44f205b2a3 100644 --- a/go.sum +++ b/go.sum @@ -306,8 +306,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= @@ -760,8 +758,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001103703-41323364781f h1:fbcRfs8KpPJdjYBU2xuP53K90rVisQgWqsfUnmSuhcg= -github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001103703-41323364781f/go.mod h1:uNVSmziO8eBGgZ9UKBt3Yv9Uvw97T/1nh08FjgWN/fM= +github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001122546-074cd3ab166f h1:pa7oaoNcLP3ipdLvvGpNKLrj/o1Plg9VQ69Rsp2tGp0= +github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001122546-074cd3ab166f/go.mod h1:jeV6A8q9uDVDwffTt5KBk+5g7bXfpEImYW6qLKn0E+I= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -846,8 +844,6 @@ github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vishvananda/netlink v1.2.1-beta.2.0.20240524165444-4d4ba1473f21 h1:tcHUxOT8j/R+0S+A1j8D2InqguXFNxAiij+8QFOlX7Y= github.com/vishvananda/netlink v1.2.1-beta.2.0.20240524165444-4d4ba1473f21/go.mod h1:whJevzBpTrid75eZy99s3DqCmy05NfibNaF2Ol5Ox5A= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1144,8 +1140,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/dnaeon/go-vcr.v3 v3.2.0 h1:Rltp0Vf+Aq0u4rQXgmXgtgoRDStTnFN83cWgSGSoRzM= gopkg.in/dnaeon/go-vcr.v3 v3.2.0/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -1174,18 +1168,18 @@ helm.sh/helm/v3 v3.15.4 h1:UFHd6oZ1IN3FsUZ7XNhOQDyQ2QYknBNWRHH57e9cbHY= helm.sh/helm/v3 v3.15.4/go.mod h1:phOwlxqGSgppCY/ysWBNRhG3MtnpsttOzxaTK+Mt40E= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= -k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= +k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= +k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= -k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= -k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= +k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/apiserver v0.30.3 h1:QZJndA9k2MjFqpnyYv/PH+9PE0SHhx3hBho4X0vE65g= k8s.io/apiserver v0.30.3/go.mod h1:6Oa88y1CZqnzetd2JdepO0UXzQX4ZnOekx2/PtEjrOg= k8s.io/cli-runtime v0.30.3 h1:aG69oRzJuP2Q4o8dm+f5WJIX4ZBEwrvdID0+MXyUY6k= k8s.io/cli-runtime v0.30.3/go.mod h1:hwrrRdd9P84CXSKzhHxrOivAR9BRnkMt0OeP5mj7X30= -k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= -k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= +k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= +k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= k8s.io/component-base v0.30.3 h1:Ci0UqKWf4oiwy8hr1+E3dsnliKnkMLZMVbWzeorlk7s= k8s.io/component-base v0.30.3/go.mod h1:C1SshT3rGPCuNtBs14RmVD2xW0EhRSeLvBh7AGk1quA= k8s.io/cri-api v0.30.1 h1:AUM78wiC56B1WJ2c795AS0IG5T57CkEdkn0IuC+miAE= From 4de9504041dc7236427681d09427369e31922470 Mon Sep 17 00:00:00 2001 From: Ritwik Ranjan Date: Wed, 2 Oct 2024 14:17:52 +0000 Subject: [PATCH 5/6] fix: address PR comments Signed-off-by: Ritwik Ranjan --- go.mod | 2 +- go.sum | 4 ++-- test/e2e/retina_perf_test.go | 14 ++++++-------- .../perf/get-network-performance-measures.go | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index ffdb0327e3..382dc1eda4 100644 --- a/go.mod +++ b/go.mod @@ -336,4 +336,4 @@ require github.com/rogpeppe/go-internal v1.12.0 // indirect replace github.com/vishvananda/netns => github.com/inspektor-gadget/netns v0.0.5-0.20230524185006-155d84c555d6 -replace k8s.io/perf-tests/network/benchmarks/netperf => github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001122546-074cd3ab166f +replace k8s.io/perf-tests/network/benchmarks/netperf => github.com/Azure/perf-tests/network/benchmarks/netperf v0.0.0-20241002134946-01238fe018fc diff --git a/go.sum b/go.sum index 44f205b2a3..c43bdf2f75 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/perf-tests/network/benchmarks/netperf v0.0.0-20241002134946-01238fe018fc h1:XpAgbFWmvKlNTuk2ogOc3eV1/0I02JzBGeC68/2dOtk= +github.com/Azure/perf-tests/network/benchmarks/netperf v0.0.0-20241002134946-01238fe018fc/go.mod h1:jeV6A8q9uDVDwffTt5KBk+5g7bXfpEImYW6qLKn0E+I= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -758,8 +760,6 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001122546-074cd3ab166f h1:pa7oaoNcLP3ipdLvvGpNKLrj/o1Plg9VQ69Rsp2tGp0= -github.com/ritwikranjan/perf-tests/network/benchmarks/netperf v0.0.0-20241001122546-074cd3ab166f/go.mod h1:jeV6A8q9uDVDwffTt5KBk+5g7bXfpEImYW6qLKn0E+I= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= diff --git a/test/e2e/retina_perf_test.go b/test/e2e/retina_perf_test.go index 6c67c5e47c..8230677d9a 100644 --- a/test/e2e/retina_perf_test.go +++ b/test/e2e/retina_perf_test.go @@ -16,8 +16,10 @@ import ( "github.com/stretchr/testify/require" ) -// TestE2ERetina tests all e2e scenarios for retina -func TestPerfRetina(t *testing.T) { +// This test creates a new k8s cluster runs some network performance tests +// saves the data as benchmark information and then installs retina and runs the performance tests +// to compare the results and publishes a json with regression information. +func TestE2EPerfRetina(t *testing.T) { curuser, err := user.Current() require.NoError(t, err) @@ -49,16 +51,12 @@ func TestPerfRetina(t *testing.T) { createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, clusterName, location, kubeConfigFilePath)) createTestInfra.Run() - // Hacky way to ensure that the test infra is deleted even if the test panics - defer func() { - if r := recover(); r != nil { - t.Logf("Recovered in TestE2ERetina, %v", r) - } + t.Cleanup(func() { err := jobs.DeleteTestInfra(subID, clusterName, location).Run() if err != nil { t.Logf("Failed to delete test infrastructure: %v", err) } - }() + }) // Gather benchmark results then install retina and run the performance tests runner := types.NewRunner(t, jobs.RunPerfTest(kubeConfigFilePath, chartPath)) diff --git a/test/e2e/scenarios/perf/get-network-performance-measures.go b/test/e2e/scenarios/perf/get-network-performance-measures.go index bd26e4915a..2f133a6eb3 100644 --- a/test/e2e/scenarios/perf/get-network-performance-measures.go +++ b/test/e2e/scenarios/perf/get-network-performance-measures.go @@ -21,7 +21,7 @@ func (v *GetNetworkPerformanceMeasures) Prevalidate() error { func (v *GetNetworkPerformanceMeasures) Run() error { results, err := lib.PerformTests(lib.TestParams{ Iterations: 1, - Image: "ghcr.io/ritwikranjan/nptest", + Image: "ghcr.io/azure/nptest:latest", // We would use the versioned image once we make a release in Azure/perf-tests TestNamespace: "netperf", TestFrom: 13, TestTo: 17, From 1d1da2c97991ac8647d753672097a641298e8947 Mon Sep 17 00:00:00 2001 From: Ritwik Ranjan Date: Thu, 3 Oct 2024 12:49:18 +0000 Subject: [PATCH 6/6] fixed stuff after merge from main Signed-off-by: Ritwik Ranjan --- .gitignore | 2 ++ test/e2e/common/common.go | 2 +- test/e2e/jobs/jobs.go | 20 -------------------- test/e2e/retina_e2e_test.go | 5 ++--- test/e2e/retina_perf_test.go | 2 +- 5 files changed, 6 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 42b39c102e..026134d26a 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ image-metadata-*.json *packetmonitorsupport*/ *.pem *results*.json +netperf-*.json +netperf-*.csv diff --git a/test/e2e/common/common.go b/test/e2e/common/common.go index d12511b237..554336dbe6 100644 --- a/test/e2e/common/common.go +++ b/test/e2e/common/common.go @@ -10,4 +10,4 @@ const ( NetObsRGtag = "-e2e-netobs-" ) -var AzureLocations = []string{"eastus2", "centralus", "southcentralus", "uksouth", "centralindia", "westus2"} +var AzureLocations = []string{"eastus2", "centralus", "northeurope", "uksouth", "centralindia", "westus2"} diff --git a/test/e2e/jobs/jobs.go b/test/e2e/jobs/jobs.go index 0c9069063d..1fd91c5d28 100644 --- a/test/e2e/jobs/jobs.go +++ b/test/e2e/jobs/jobs.go @@ -66,26 +66,6 @@ func CreateTestInfra(subID, clusterName, location, kubeConfigFilePath string, cr return job } -func RegisterExistingInfra(subID, clusterName, resGroupName, location, kubeConfigFilePath string) *types.Job { - job := types.NewJob("Register existing e2e test infrastructure") - - job.AddStep(&azure.GetAKSKubeConfig{ - SubscriptionID: subID, - ResourceGroupName: resGroupName, - ClusterName: clusterName, - Location: location, - KubeConfigFilePath: kubeConfigFilePath, - }, nil) - - job.AddStep(&generic.LoadFlags{ - TagEnv: generic.DefaultTagEnv, - ImageNamespaceEnv: generic.DefaultImageNamespace, - ImageRegistryEnv: generic.DefaultImageRegistry, - }, nil) - - return job -} - func DeleteTestInfra(subID, clusterName, location string) *types.Job { job := types.NewJob("Delete e2e test infrastructure") diff --git a/test/e2e/retina_e2e_test.go b/test/e2e/retina_e2e_test.go index e69dab3e61..9e4b40e2c6 100644 --- a/test/e2e/retina_e2e_test.go +++ b/test/e2e/retina_e2e_test.go @@ -18,7 +18,6 @@ import ( ) var ( - locations = []string{"eastus2", "centralus", "southcentralus", "uksouth", "centralindia", "westus2"} createInfra = flag.Bool("create-infra", true, "create a Resource group, vNET and AKS cluster for testing") deleteInfra = flag.Bool("delete-infra", true, "delete a Resource group, vNET and AKS cluster for testing") ) @@ -41,11 +40,11 @@ func TestE2ERetina(t *testing.T) { location := os.Getenv("AZURE_LOCATION") if location == "" { var nBig *big.Int - nBig, err = rand.Int(rand.Reader, big.NewInt(int64(len(locations)))) + nBig, err = rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations)))) if err != nil { t.Fatalf("Failed to generate a secure random index: %v", err) } - location = locations[nBig.Int64()] + location = common.AzureLocations[nBig.Int64()] } cwd, err := os.Getwd() diff --git a/test/e2e/retina_perf_test.go b/test/e2e/retina_perf_test.go index 8230677d9a..4e50f20f1c 100644 --- a/test/e2e/retina_perf_test.go +++ b/test/e2e/retina_perf_test.go @@ -48,7 +48,7 @@ func TestE2EPerfRetina(t *testing.T) { kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem") // CreateTestInfra - createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, clusterName, location, kubeConfigFilePath)) + createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, clusterName, location, kubeConfigFilePath, true)) createTestInfra.Run() t.Cleanup(func() {