From 75c0322a5c9ab5e8436fc2bb3e495444f5b11c2f Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Mon, 7 Apr 2025 14:56:22 +0500 Subject: [PATCH 01/31] EVEREST-1925 WIP blocklist --- api/everest-server.gen.go | 310 ++++++++++++---------- client/everest-client.gen.go | 413 +++++++++++++++++++---------- docs/spec/openapi.yml | 28 ++ internal/server/everest.go | 34 ++- internal/server/session.go | 22 ++ pkg/common/constants.go | 2 + pkg/session/blocklist.go | 121 +++++++++ pkg/session/data_processor.go | 83 ++++++ pkg/session/data_processor_test.go | 124 +++++++++ pkg/session/jwt.go | 65 +++++ pkg/session/jwt_test.go | 100 +++++++ 11 files changed, 1007 insertions(+), 295 deletions(-) create mode 100644 pkg/session/blocklist.go create mode 100644 pkg/session/data_processor.go create mode 100644 pkg/session/data_processor_test.go create mode 100644 pkg/session/jwt.go create mode 100644 pkg/session/jwt_test.go diff --git a/api/everest-server.gen.go b/api/everest-server.gen.go index ed6ae4c47..7123f44e8 100644 --- a/api/everest-server.gen.go +++ b/api/everest-server.gen.go @@ -1697,6 +1697,9 @@ type ServerInterface interface { // Cluster resources // (GET /resources) GetKubernetesClusterResources(ctx echo.Context) error + // Everest UI Logout + // (DELETE /session) + DeleteSession(ctx echo.Context) error // Everest UI Login // (POST /session) CreateSession(ctx echo.Context) error @@ -2519,6 +2522,17 @@ func (w *ServerInterfaceWrapper) GetKubernetesClusterResources(ctx echo.Context) return err } +// DeleteSession converts echo context to params. +func (w *ServerInterfaceWrapper) DeleteSession(ctx echo.Context) error { + var err error + + ctx.Set(BearerAuthScopes, []string{}) + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.DeleteSession(ctx) + return err +} + // CreateSession converts echo context to params. func (w *ServerInterfaceWrapper) CreateSession(ctx echo.Context) error { var err error @@ -2611,6 +2625,7 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL router.PATCH(baseURL+"/namespaces/:namespace/monitoring-instances/:name", wrapper.UpdateMonitoringInstance) router.GET(baseURL+"/permissions", wrapper.GetUserPermissions) router.GET(baseURL+"/resources", wrapper.GetKubernetesClusterResources) + router.DELETE(baseURL+"/session", wrapper.DeleteSession) router.POST(baseURL+"/session", wrapper.CreateSession) router.GET(baseURL+"/settings", wrapper.GetSettings) router.GET(baseURL+"/version", wrapper.VersionInfo) @@ -2618,12 +2633,12 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9/XMbuZHov4JirmptH0nJ3t28i94PebLsbHS7XqskOVfvTL0YnAFJnGaACYCRzHX8", + "H4sIAAAAAAAC/+x9/XMbuZHov4Jiriq2j6Rk727erd4PebLsbHS7XqskOVfvTL0YnAFJnGaACYCRzHX8", "v79CN4D5wlBDfdhylqnKWpzB4KPR391AfxolMi+kYMLo0cGnkU5WLKfw50uaXJbFmZGKLpl9QNOUGy4F", "zU6ULJgynOnRwYJmmo1HKdOJ4oV9Pzpw3xKNHxMuFlLlFF6OR0Xt608jmmXymqW/0pzpgib4MGWFYgk1", "LB0dGFV2+v+Fa0PkgojwFXH9ECNJqRkxK67JvDGN0XjEDcthALMu2OhgpI3iYjn6PPYPqFJ0bX/Py+SS", "GTuraPPGdCLvF1Il7ISa1ZlZZwyXtKBlZgLA3CdzKTNGhf1G9A0WVtl9Ox59nCzlxD6c6EteTGSBWzQp", - "JBeGKYTf5/FIsWV0ssN7wO8+jZgo89HB+5H+fjQe0d9KxUYX4+6sS5VFV3PFFF+sz385a0AFd7kNFJj3", + "JBeGKYTf5/FIsWV0ssN7wO8+jZgo89HB+5H+bjQe0d9KxUYX4+6sS5VFV3PFFF+sz385a0AFd7kNFJj3", "P0quLCK8Rwg19sZ9Uo0v5//DEmPHaeCvthhjBwwY8G+KLUYHoz/sVQSw57B/r4n6Eew4Uowa1mh2QhXF", "nm9PJ4XtgxmmdJdMkoRp/TNbR2H6TRBRc/TzFSNJJss0rB5b7yVSGMoFU0TUdvhLEV9zkocWDIqkbMEF", "szO1Q8C8LODMitVYHPx89esZvkaGR1bGFPpgb++ynDMlmGF6yuVeKhNt15mwwug9ecXUFWfXe9dSXXKx", @@ -2641,102 +2656,102 @@ var swaggerSpec = []string{ "O43zUgMqIboGRNK296DFEYpjRsirQ+a3Qgk70Sj8yyKTND22wuOKZmcxJvGu3YSIMp8zZYGjWSJFqsmc", "mWvGcGlzLjK51AS7rrEqK6KWTHXUAb+imJQPyNmd15l/hSvOnGrs6Sp8WNN+o1vvUbtFl/5xA/+mXwjF", "jk6R49WY8Ux4vTWTyC2mjxffYEgHwdFw3b0PON2u6uqyQRl5JAsew5PTZoPQf0Bit+MJvjaSKGatmRFY", - "JTk1iLrfv4hgcoWg/fgZGJmSYsNKWkTRxatqK8Zegw69xUinadl9jrSw2sIZKFBx1QDfBSSkoCwTp3JZ", + "JTk1iLrfvYhgcoWg/fgZGJmSYsNKWkTRxatqK8Zegw69xUinadl9jrSw2sIZKFBx1QDfBSSkoCwTp3JZ", "GTuX0mijaGG1MkoEuyZOj+6jk57RXtbetgnRaXh2WywFMFDevhAdghYCK0WZ+mVIrqBmFRGK1Kz8jG0L", "bwA4OC14xvZSrlhipFpPb4VgMHAMl9K5my+uPA7fVy87jWIQfvXSI4mfendvuyC5UU8AlWDCxaShEjTZ", - "dwdrrCIfxf0w83fnRxbtHQJCp9YeIBYNrM1aGMSQnJoDMhu92N//42T/+WT/xfnzHw/2fzjY//G/Z6Po", + "dwdrrCIfxf0w83fnRxbtHQJCp9YeIBYNrM1aGMSQnJoDMhu92N//02T/+WT/xfnzHw72vz/Y/+G/Z6Po", "Lns7PNjOOJu2y+d8XYTJ2E8sGP3qpsAU0Ix3H6M5GLHkuwwgxhKYWHLBYszePvfz8EYzweY3KLG4Bd0+", "Ue/2fbqu2vvVAVuiei3xo1NviPOm/eJscY+BVrqit+yaWzqywrUUKVPZ2jIyO3dqpLIG3oKUwq2OpWPC", "rpjF1IlvgtYC+t0cxfuxHL3XOpuJX9+evz4g76z9iHYs18TBak0KCWa8NjTLUNm1RmvGKKjSFEiEKuMX", "kWxgIIoVGU9oVBjim64UdPAPn0akX84Fzy22PY9JwsrYj4zqXhHqNOdguWccbG3LY8HSaE4Dt8BaY5qZ", "cecr25t9yfNCahCMLcwrSjBKxfrtYnTw/lN31h3H1kWb/o5O3nlg2T/DFBwvzSEMA6zTMGU/+H9PZrN/", - "/+fk6Z+fPHm/P/nTxb8/mc2m8Nezp39++s/w69+fPn3y5P3Pb346P3l9wZ/+870o80v89c8n79nri+H9", - "PH36538D/2Dls5xYbijVxK3LuwZzlku1vjNQ3kA3Hi7Y6bcNmhgz1FUgraXaeQ9yg3V5/XyzyEkyqiMk", - "cmQf+w5DT/DQ8SrvsSwsg9FWpyVXMitzaMajUlPz39id9/qM/xZWajsMNnjvPL6VDa+rQwCqfjX60wap", - "7LYfGlbyuPiYWFBIbZaK6X9k9ofO03ncya6ZOgOvt47rVu+aDaJGErwmLhbj/aTgL8NXUa/hVZ8wbYlS", - "t0jf/Cbtsgo99Trwcym4kbgj7cHfhHeBx1RPNtNX1RD1izg830RatYFKSbsvcnTqLID29/dvBAwSp940", - "awpG5wv1DKNaxTTGjXgeZ0c81+BUqYCiUfd0g49DjI0L0ACn/hV+PJ4J8GFYIQ121HyNGk+IFoJOdG4f", - "cU2oIDQrVtT5f6lIvRxx/jWH0TPxai1ozhMPhcPMO0TIglHwzy6pYVXn2KEdJc9LY03oKTk24ESWIluD", - "o5eh0zhMDVy7PX6j0/oyiWILppiwuyGFpRNjBaMgJzI9s0BptNbdHdjgCQGcyqlJVg28bAxTyHQaAT6R", - "Cwt+ZqcRHJZ1WNgdATDk9BIcTNRUWESvKM8soGaCC81TRmht1+LYCrGSGLDgRYO2kpXUTADAqY+yeIIJ", - "4ExRnKAGyPLCrFH9XpuVxYQQwYFWtvucprWZj4k0K6auuWYzAdvs1M4yM7VQHIx9s7EMm3Sjk6UldSzx", - "THJaTC7ZWtd76bZy3eS0sJ2idtufl7C1QP9GlNN2rgPo+Phw7iJSOf1oTRBCc1kK2MhE5kVpKosiZETE", - "A3KbovoNwbKXU0GXbBL6nVTMYW8UQQUfLvy979upD5u2dg6Nx40750kOiT50xDWROTfO01LnRWPCwUlO", - "ywzitMQhDV8gR+OasI/WkuQmW5PKkJ+JwB3AuBbWhMzAYoHNn3jRBtHnaTWVBKPA7GPCWOpG+7KINsyP", - "U1DL4GNORBDFDZ+9NrKouxTigTolP64j/dnHwQUHPxrOoCmp2+9WxhdW/ClODZuJyAfoUZkz2zDjbsdt", - "50t+xYRTQ6fkcCYSmecY3CUJdfaRZqbyrARZV4tEggLBPrpcCUw68Y7U4NVK+qLbwzxZuKobHVnsYyF1", - "zNUGz5udYdsbNF/uHOinVCxjauPxSf29H8DHzY5PvKtd4fsnR8evTu3ewWhPZ8ISimWtHmxWBDf314Ci", - "wTURsq6J9qtSjSnVMjfsbGiaKqa1nakgjbkQcLyZlSwNRB1MTvXlBh9rld3W9bn6vJmNflcHfvv1GPTG", - "OasSbqQiHqFq5l+t3/B2iFP2ds47xJKv7btrzGLnutu57r6e6+5mrw0ia8tpk0uxlHbhK4oCzwk+579Z", - "zmUpEqYGUrJeUZVG/Rtn7k3wq/nfzWwDcnL25tXLiTVfemQR5rf1SSR8W+er/YMRjY2dCO2mMw/nS3UV", - "r5rG1mypZYOF8S+icasbshS8p4QvmjCosneiag+00z0bqBvJcrWsGPzobstt7G899u96v4jpgc20Ggjl", - "RbNqDDWlvjkfEJo1FinngCZbpQQmhl+xsz5f+mH9ddsBjsqqCAHjJ+BCBbfN02hwUAo0vHSUJNw7bz+0", - "llR9HELV3bX1KDKh86rvlBnKMxSPUjBCrZ5bhe9KpSDZ1MMRVNbDk2PiBW4XkhnV5lxRoWGkcx5zHHXb", - "BEWPaoP5ci6tzk3YhNZW21Yyh9kiiqBxBLbS1HnTXGLyHPLg0GdTi51W3SYrq9OlU2I1RG+MWYl/KeS1", - "AF3RKu/eTw0TCz1aOKD67roBiwXC7eC/q9NWSg2DoH88KVbrKNq5F2gJrcqcCqIYTSF7KbwTKVglYhk2", - "k86t0gkTDmDzkMkp+BKpQHeVS2C2c83px1+YWJrV6OD7F//rj/8RmajHwp+YYH0ps902bdY+9UnA02XV", - "JuTOVptzTTX4PC1yp6QsYBF/kQrjzyJhY8soo71x7XE3W5PnL8Zk7gAyRZSZVmT0/uPFNDJnrsmfxq0J", - "WRO/BOJIwTKEwLxiSDLOPouQDAsTnrbY7R9/qLPb/bjSS3UMzPi8ImRqdYWlonlODU8IT5kwfMGZqiMI", - "KsbwobdYw+q+04746ihzAvnJTAGz8SZwnSzXBUOcQv5rjRCWmJC9Dx7ynFFhhbUb0xu945mwb69XzFIu", - "HkdwHymYl+YpUywllCxLqqgwjKVw8gGjG5hzX1E6rdLcPVY3fOt2li5lGlC/hfPP91/8AJsRHjQ0y/eH", - "k/+mk98unrg/9id/+vv44OJZ7ecFqoLdAGePIHPcy/NaD9QxsDa5IOeqZGPyFzhZRN4JYEn1ZBr7fjQe", - "QYPReORaREN3cU3TZ+rUMLx2VoAApZGFlFN3JGiayHyvOkvQ4hnP/9hUxd8jWC6evJ+4v575R0//DCr0", - "pgZPn+2B+h3Ae/F+UoF6ahXx2run/3ajdzwilyrOG+gs7NaGmGDbXt8m2SfI8W62D6gRPteHxFJ94mfW", - "gOdH1CQnDAolr3jKNFmUWUaaOFcW2ihG86C6UGAkGeWCGPbRREdcSW3i8aC/ujd+sb5lLRndD+T8E8qa", - "5HWtdoBQfFMJRfbRKFo/BVwTfR0/4XZi7G1UJGCkUsNRJ8vlayIn7GzgchHFrMP8uwy/kCoSoTqRylRJ", - "hMoMAemAxGCrTaxjthJN110HDrQmhYybDbHeE5nnTKQsDYQQG6zbyo9d66E3Pw59ON61Z58LxlLQCqtz", - "UCieuQ69zNlCKvt6qWjqZWMnqa7WqdXBMoSA01Qik5tuSnDpz1gx0tCs7ikbDOI+2eKsomCpNCRNH2UM", - "89q30Pplz0GiaLNh5xtdHvPXPeVI7vGQI7nhjCP5Fz/iSO7rhCPpHnAkjfON5Fs/3uiy9rc95IifTb/W", - "iYOoZuLT8W9IxK8PKRVfcks77TAXTOZ25wWa87iDp8nDYHt/U9/uJNKq1SbmEjzyr4KMaPge/kfOwT4O", - "PQz3Nrjkr8iQPiusGlAbmhcdbRGh/J3GPDIn9oYNnjJtuOjRuV5VL/0kQGntHiSJItySFpFN/IkWujKH", - "vW9VMbAy7SckZQZtVpfdAwc2MrnUUWcrcvlTOApC5xmLe7h+ibSqfFwgJ5yXixqvuQWqggm4wyaDIQu4", - "F1cEwsgeLcP1INQMICqA68XtdQN/RcoA4oKrQzDPzkkRBFDdFepjwRjz5BpdX21+UeNMO/3hQfWH4Gwe", - "dAVOXHuMWNU7teSLqCUDqPjI7+KRT/mx/cQTRCM5l87C7HJSd1aofuVP07JRTkxt8KgNCHD2rSYiKyp8", - "JYplIAwBbDUk78Q3XRLUbQkgAtwIMQwGbyM1+76hW/kRbwJ7/RognHvvNsSW226rGMhvmnW3rAq7kzB2", - "Z48Egzs23uEtQdUFR/7gw8HeXqmZOsAjCP/n+f7+tPb/gx9/qFvf9SOwWl9LlTY7VVKaUc/xCb+PN7Ue", - "gMeDpOq9ydOdIH3kgnQnQh+zCD2JngzvOQ3eEj2ty3aoyjjT5pXT9itO8mL/xfeT5y8m3z8/f/H9wY9/", - "OvjxT/892HqI204udNi2mgpuFBhILfuJLozff3do3pqohl4yscGUap7W78wMG93rcgds2Kmzvm5isK7d", - "ML+mM+l2js2dY/P359h0lLK1Z9N9N41di3G3i2GQHDdfmfStXwWzu7lld3PLI7q5ZauYQJ1L1MMAtQ29", - "GQ9rXOIeQwGemd0iFtDLzxrBgK0TB4f6g2szb5xlCdNtccX7CBG7MQdZrLW29+MI9krXTuF63Aas17h3", - "duxjtGNf91y51Xx/gxmEyf4782dn/vyOzB+kDDB7EOz2Lzwh3rqhbtpXw8ThfpO1bnGMtHtHHmh92lCR", - "VjewVPdIt+alp+SUL1eGCHlNuPlO440kxccEaABOu0zJX+U1u3KH3V1Au9BjUiyhERVrvOuCVOdGNitu", - "vem3N6loDuDbqGav++DvL+qo70D0BiKrQKmyQR3VNR+eUWl3dKBxLWElGfuM0E13NXSTRqCvSlGqJ8c6", - "Xal3BtMAEPK69cpvaevbcfUAjypaXJIy04TnWHjErCKaruKGJzSLhwXhy79SvYpiObw9cRbs3QKDG+6V", - "3IH7C4A73NbQexHJbhcefhe6D+xSdtvyuLYl1sRnq7+DHPaIrH/bbNC0nps54eGuWEyIZ9PqzjPNDAp8", - "dyr5g7tfdlowlUhB4VSQ+yzcOTsx8gMBnS6k8zm52N0Cd53sSUbFqTUXO/dvNN6jFhVu4PJKeq2RV1T9", - "LXdewemscZtrzhyc3Lhm++t0BlVjgn9m4vztq7cH5DBNnc5UarYoMzzHpqekMpXGxKqsY1Ly9M8DnDWt", - "ewpyWvhLvaiROU9u8ikVKxq7DMbh14l92z7sCZ/0YllPIqPVQg/NcD+YoWrJTK/5eF5/7W1UfxDESHK9", - "4u72ujDB6lihm2o6HRZH9D3UJtMFIxMpF8sWeTbV+y0oOX7+6WZs39HdY6K7R4TDbUuyz+KqLK24K9nJ", - "dC4IJZf/oTfcBr6dWxnH3exOrtrczY3sTeCdv+pxeo+dY3LnNX5MXuPXSslIPBUew4UJUsCR9dalyL2a", - "R2yMnwM/dQGEY7GQG/NDfUTIQjFyxzC8PI8nuIZr1uEGdCiHuk0l1OZV6XhLcbh2uHITufMnnk3ORL0a", - "5/vRsngxGo+Wxfeji22KpdZnzoYT2Fnts2ilnsY9QDXoxWB1MWQDT/vvd4vsYp2X9HjtIvnaRfmGZxmv", - "Qw6P3dZTlkcHoxIPaH8ej1KuL8/cCd5hX+B1ZS/Xhg0eZkgCdQDPYVjf5/EooQVNuFn/i671yC+vg3H+", - "xbi23zE0qy5BP3a3sDjPurudbhMNdL99STX7L25WkEAQubeudiW8+6JVpLzj4sbitS7F/yI64ZdRq+vm", - "sb5cQfS8O5etCjq3y/0Wed7NMRleW9iVA25e5nLbzlo1hFvOf3zl/SRVLdDzX872zs5+IfC1v2d2FK06", - "PABlG2h3R/SFCxiH2F/fRpHqIs8nNZy7nz2/h8r43Y29BbcYgBp4ardWJf5eONt4289P3rwZuEJXE/fu", - "bNEO2ZF6lnN0HtKCu0LjteLmBb+E4t33gzHxs0Dh6R14mcbzdrWZpzkXt+5xiPg9efOmC+6zgiVD+RVU", - "D7snpHxQZERrq4GM0QVp720YpDtHdI6I0AuSuNP3jfLy7fGro6Oee75fo3ue2Db+9id1Y70na5oeR+xl", - "6AWuOXdXn7umr6ImvNYlU+9Of+npJ8wGabtrZCWyiKn/8LF7OVyt6Ngobo31eYYxY6rjmb9lthfGy0zO", - "adZ/Ha3kaVLt0yaUqe1oe+K1TqKzrBtqfVXAsHJXs9hBYcmTGaa0r1gAjWr2qn2MjsWTUEfrb1BGS4cg", - "HNa5oVm2xu1FDxcUtjlrWJ7uNm0xCcI1/d/d+lANS5kmicQbcK1iZRIskCPIW1j8G2bo9Fd3L0hE18WZ", - "vv5YUBF308VaEb2S17pxK2FrTnC7vCsmxuxnabxqeN1n0rmTQaRUpc6j9J0m4Q7Lh/bzhEkNcrgiG28c", - "Majx9GgYeAHXFcaLyntAVogXuYc3YVo7Yd2h74c0YeaNOW5lvczL5JKZeEr7OTh0ZJmG1WPrvXB0nLg6", - "ybG7UqqOItNYSJVA+PnMrDPWd8Z+2fc5HnbuA7UzoWKl2CpraIgxU4tOt2QO3pTaH/Go3aYaYhq9pd3G", - "A4JIvpfbhBAtn2IfTfsWujAxi0ouOGckuu4GODF95DSjYkuS8hFBHYYtbCdtenKhxsPEdBIIN0kjN69z", - "qi9jCF/GIpYD+htW8bsGlMPCypTYuX/IThByIgvvl3f1wiAKovhyyeLRRwxHBWbQ2KrOHAAAHczdFCK/", - "GQvbJxSi55Fw2/zwrWNJ+JIYqi87Oda1Xr0zBe+IGI+ENKfuT3c5xChs5ev2PewbsVbX72SIXItRN4Y2", - "3oMwcLATpnKuQwZm60R0dat+pCJQ88tujGmga6pHp/Rjx/SyXl7ilUfPSqLxiEWZZUcyz7m5vQsC+rTT", - "iR+u3soFFk9m2MLorIOtPq2q93F90TGIcgl6EC14TpOVlZ3raXG5tA80XAM+vXo+teLeaoaRO7ncm5oa", - "7NUhjMbotTArZnhSq/YF5Q1X9IqNCRdJVgLlYcVJKlJyRRWXpQ7ZVOi3mpLDKnaY0zV0gElU7m7nT5UK", - "OyZ+Yp+jxZwMFyWL3VWDb/yN7poZn6HlyqkaQrGQDZGidZMroD9RzJRKsBRjw9Vh/3AXu6u2saKa5FYv", - "BVAFdzueUsT4KddEFvQfJQthZn+hmZEEzC1/RX+opeii1bUQqd0CzNAGjQwC83DWzyjOrvC2WxDCkJK2", - "qEXOAtyPECp2kyjeIY7Ff6Gv2g3ghdSaQ8xtUV9ps9iIXbe/2l8qBIFZUSs/Fuya5FyUFlywuZblsRRB", - "4rfe5wDg/eMe2niJny8q4EqNwk4iKH1tMbyzLqGZh5SDNO7lgiu4hhhjqWNSioxpTdayxPkoljAeQGnk", - "JRMYlqaCMIjDOinWUykzx/K8x4blR7IUJnYDfbtN98ZgXc613W77DlDOzR62Ay3LUGMJqMvfwue33y9w", - "igUamH+KKOR16JSAd9huEsJaswwOKmtXwqF9j7GbuZ+UJiVeng7Yi+C13fityNjC4HXA0MDX+UtLcCZp", - "pjjN+G9VLbkwUV7d0EieMA74P2cJtVYHD8UmklUpLuGm7+qtcdmXoSgGNHparcfdxSEk4mV7TbiQcPn9", - "rVbisxtkloIyRQW5ej59/iNJpa+ZVRsDcd9yfbgRudS15K0Ypjxj2vAcChM8a9TttoSbZXhl05Qcgdsm", - "pL/YcRUDRtrXN178DDxCuR/sI03MwAu4W9QbM98V0i419YslKzbyna4l39TthSqJpHPv43ztPGuQxpNa", + "/+fk6Z+fPHm/P/nx4t+fzGZT+OvZ0z8//Wf49e9Pnz558v7nNz+dn7y+4E//+V6U+SX++ueT9+z1xfB+", + "nj7987+Bf7DyWU4sN5Rq4tblXYM5y6Va3xkob6AbDxfs9NsGTYwZ6iqQ1lLtvAe5wbq8fr5Z5CQZ1RES", + "ObKPfYehJ3joeJX3WBaWwWir05IrmZU5NONRqan5b+zOe33GfwsrtR0GG7x3Ht/KhtfVIQBVvxr9aYNU", + "dtsPDSt5XHxMLCikNkvF9D8y+0Pn6TzuZNdMnYHXW8d1q3fNBlEjCV4TF4vxflLwl+GrqNfwqk+YtkSp", + "W6RvfpN2WYWeeh34uRTcSNyR9uBvwrvAY6onm+mraoj6RRyebyKt2kClpN0XOTp1FkD7+/s3AgaJU2+a", + "NQWj84V6hlGtYhrjRjyPsyOea3CqVEDRqHu6wcchxsYFaIBT/wo/Hs8E+DCskAY7ar5GjSdEC0EnOreP", + "uCZUEJoVK+r8v1SkXo44/5rD6Jl4tRY054mHwmHmHSJkwSj4Z5fUsKpz7NCOkuelsSb0lBwbcCJLka3B", + "0cvQaRymBq7dHr/RaX2ZRLEFU0zY3ZDC0omxglGQE5meWaA0WuvuDmzwhABO5dQkqwZeNoYpZDqNAJ/I", + "hQU/s9MIDss6LOyOABhyegkOJmoqLKJXlGcWUDPBheYpI7S2a3FshVhJDFjwokFbyUpqJgDg1EdZPMEE", + "cKYoTlADZHlh1qh+r83KYkKI4EAr231O09rMx0SaFVPXXLOZgG12ameZmVooDsa+2ViGTbrRydKSOpZ4", + "JjktJpdsreu9dFu5bnJa2E5Ru+3PS9haoH8jymk71wF0fHw4dxGpnH60JgihuSwFbGQi86I0lUURMiLi", + "AblNUf2GYNnLqaBLNgn9TirmsDeKoIIPF/7e9+3Uh01bO4fG48ad8ySHRB864prInBvnaanzojHh4CSn", + "ZQZxWuKQhi+Qo3FN2EdrSXKTrUllyM9E4A5gXAtrQmZgscDmT7xog+jztJpKglFg9jFhLHWjfVlEG+bH", + "Kahl8DEnIojihs9eG1nUXQrxQJ2SH9eR/uzj4IKDHw1n0JTU7Xcr4wsr/hSnhs1E5AP0qMyZbZhxt+O2", + "8yW/YsKpoVNyOBOJzHMM7pKEOvtIM1N5VoKsq0UiQYFgH12uBCadeEdq8GolfdHtYZ4sXNWNjiz2sZA6", + "5mqD583OsO0Nmi93DvRTKpYxtfH4pP7eD+DjZscn3tWu8P2To+NXp3bvYLSnM2EJxbJWDzYrgpv7a0DR", + "4JoIWddE+1WpxpRqmRt2NjRNFdPazlSQxlwION7MSpYGog4mp/pyg4+1ym7r+lx93sxGv6sDv/16DHrj", + "nFUJN1IRj1A186/Wb3g7xCl7O+cdYsnX9t01ZrFz3e1cd1/PdXez1waRteW0yaVYSrvwFUWB5wSf898s", + "57IUCVMDKVmvqEqj/o0z9yb41fzvZrYBOTl78+rlxJovPbII89v6JBK+rfPV/sGIxsZOhHbTmYfzpbqK", + "V01ja7bUssHC+BfRuNUNWQreU8IXTRhU2TtRtQfa6Z4N1I1kuVpWDH50t+U29rce+3e9X8T0wGZaDYTy", + "olk1hppS35wPCM0ai5RzQJOtUgITw6/YWZ8v/bD+uu0AR2VVhIDxE3ChgtvmaTQ4KAUaXjpKEu6dtx9a", + "S6o+DqHq7tp6FJnQedV3ygzlGYpHKRihVs+twnelUpBs6uEIKuvhyTHxArcLyYxqc66o0DDSOY85jrpt", + "gqJHtcF8OZdW5yZsQmurbSuZw2wRRdA4Altp6rxpLjF5Dnlw6LOpxU6rbpOV1enSKbEaojfGrMS/FPJa", + "gK5olXfvp4aJhR4tHFB9d92AxQLhdvDf1WkrpYZB0D+eFKt1FO3cC7SEVmVOBVGMppC9FN6JFKwSsQyb", + "SedW6YQJB7B5yOQUfIlUoLvKJTDbueb04y9MLM1qdPDdi//1p/+ITNRj4U9MsL6U2W6bNmuf+iTg6bJq", + "E3Jnq825php8nha5U1IWsIi/SIXxZ5GwsWWU0d649ribrcnzF2MydwCZIspMKzJ6//FiGpkz1+THcWtC", + "1sQvgThSsAwhMK8YkoyzzyIkw8KEpy12+6fv6+x2P670Uh0DMz6vCJlaXWGpaJ5TwxPCUyYMX3Cm6giC", + "ijF86C3WsLo/akd8dZQ5gfxkpoDZeBO4TpbrgiFOIf+1RghLTMjeBw95zqiwwtqN6Y3e8UzYt9crZikX", + "jyO4jxTMS/OUKZYSSpYlVVQYxlI4+YDRDcy5ryidVmnuHqsbvnU7S5cyDajfwvnn+y++h80IDxqa5fvD", + "yX/TyW8XT9wf+5Mf/z4+uHhW+3mBqmA3wNkjyBz38rzWA3UMrE0uyLkq2Zj8BU4WkXcCWFI9mca+H41H", + "0GA0HrkW0dBdXNP0mTo1DK+dFSBAaWQh5dQdCZomMt+rzhK0eMbzPzVV8fcIlosn7yfur2f+0dM/gwq9", + "qcHTZ3ugfgfwXryfVKCeWkW89u7pv93oHY/IpYrzBjoLu7UhJti217dJ9glyvJvtA2qEz/UhsVSf+Jk1", + "4PkRNckJg0LJK54yTRZllpEmzpWFNorRPKguFBhJRrkghn000RFXUpt4POiv7o1frG9ZS0b3Azn/hLIm", + "eV2rHSAU31RCkX00itZPAddEX8dPuJ0YexsVCRip1HDUyXL5msgJOxu4XEQx6zD/LsMvpIpEqE6kMlUS", + "oTJDQDogMdhqE+uYrUTTddeBA61JIeNmQ6z3ROY5EylLAyHEBuu28mPXeujNj0Mfjnft2eeCsRS0wuoc", + "FIpnrkMvc7aQyr5eKpp62dhJqqt1anWwDCHgNJXI5KabElz6M1aMNDSre8oGg7hPtjirKFgqDUnTRxnD", + "vPYttH7Zc5Ao2mzY+UaXx/x1TzmSezzkSG4440j+xY84kvs64Ui6BxxJ43wj+daPN7qs/W0POeJn0691", + "4iCqmfh0/BsS8etDSsWX3NJOO8wFk7ndeYHmPO7gafIw2N7f1Lc7ibRqtYm5BI/8qyAjGr6H/5FzsI9D", + "D8O9DS75KzKkzwqrBtSG5kVHW0Qo/1FjHpkTe8MGT5k2XPToXK+ql34SoLR2D5JEEW5Ji8gm/kQLXZnD", + "3reqGFiZ9hOSMoM2q8vugQMbmVzqqLMVufwpHAWh84zFPVy/RFpVPi6QE87LRY3X3AJVwQTcYZPBkAXc", + "iysCYWSPluF6EGoGEBXA9eL2uoG/ImUAccHVIZhn56QIAqjuCvWxYIx5co2urza/qHGmnf7woPpDcDYP", + "ugInrj1GrOqdWvJF1JIBVHzkd/HIp/zYfuIJopGcS2dhdjmpOytUv/KnadkoJ6Y2eNQGBDj7VhORFRW+", + "EsUyEIYAthqSd+KbLgnqtgQQAW6EGAaDt5Gafd/QrfyIN4G9fg0Qzr13G2LLbbdVDOQ3zbpbVoXdSRi7", + "s0eCwR0b7/CWoOqCI3/w4WBvr9RMHeARhP/zfH9/Wvv/wQ/f163v+hFYra+lSpudKinNqOf4hN/Hm1oP", + "wONBUvXe5OlOkD5yQboToY9ZhJ5ET4b3nAZviZ7WZTtUZZxp88pp+xUnebH/4rvJ8xeT756fv/ju4Icf", + "D3748b8HWw9x28mFDttWU8GNAgOpZT/RhfH77w7NWxPV0EsmNphSzdP6nZlho3td7oANO3XW100M1rUb", + "5td0Jt3OsblzbP7+HJuOUrb2bLrvprFrMe52MQyS4+Yrk771q2B2N7fsbm55RDe3bBUTqHOJehigtqE3", + "42GNS9xjKMAzs1vEAnr5WSMYsHXi4FB/cG3mjbMsYbotrngfIWI35iCLtdb2fhzBXunaKVyP24D1GvfO", + "jn2Mduzrniu3mu9vMIMw2X9n/uzMn9+R+YOUAWYPgt3+hSfEWzfUTftqmDjcb7LWLY6Rdu/IA61PGyrS", + "6gaW6h7p1rz0lJzy5coQIa8JN3/UeCNJ8TEBGoDTLlPyV3nNrtxhdxfQLvSYFEtoRMUa77og1bmRzYpb", + "b/rtTSqaA/g2qtnrPvj7izrqOxC9gcgqUKpsUEd1zYdnVNodHWhcS1hJxj4jdNNdDd2kEeirUpTqybFO", + "V+qdwTQAhLxuvfJb2vp2XD3Ao4oWl6TMNOE5Fh4xq4imq7jhCc3iYUH48q9Ur6JYDm9PnAV7t8Dghnsl", + "d+D+AuAOtzX0XkSy24WH34XuA7uU3bY8rm2JNfHZ6u8ghz0i6982GzSt52ZOeLgrFhPi2bS680wzgwLf", + "nUr+4O6XnRZMJVJQOBXkPgt3zk6M/EBApwvpfE4udrfAXSd7klFxas3Fzv0bjfeoRYUbuLySXmvkFVV/", + "y51XcDpr3OaaMwcnN67Z/jqdQdWY4J+ZOH/76u0BOUxTpzOVmi3KDM+x6SmpTKUxsSrrmJQ8/fMAZ03r", + "noKcFv5SL2pkzpObfErFisYug3H4dWLftg97wie9WNaTyGi10EMz3A9mqFoy02s+ntdfexvVHwQxklyv", + "uLu9LkywOlbopppOh8URfQ+1yXTByETKxbJFnk31fgtKjp9/uhnbd3T3mOjuEeFw25Lss7gqSyvuSnYy", + "nQtCyeV/6A23gW/nVsZxN7uTqzZ3cyN7E3jnr3qc3mPnmNx5jR+T1/i1UjIST4XHcGGCFHBkvXUpcq/m", + "ERvj58BPXQDhWCzkxvxQHxGyUIzcMQwvz+MJruGadbgBHcqhblMJtXlVOt5SHK4drtxE7vyJZ5MzUa/G", + "+X60LF6MxqNl8d3oYptiqfWZs+EEdlb7LFqpp3EPUA16MVhdDNnA0/773SK7WOclPV67SL52Ub7hWcbr", + "kMNjt/WU5dHBqMQD2p/Ho5TryzN3gnfYF3hd2cu1YYOHGZJAHcBzGNb3eTxKaEETbtb/oms98svrYJx/", + "Ma7tdwzNqkvQj90tLM6z7m6n20QD3W9fUs3+i5sVJBBE7q2rXQnvvmgVKe+4uLF4rUvxv4hO+GXU6rp5", + "rC9XED3vzmWrgs7tcr9FnndzTIbXFnblgJuXudy2s1YN4ZbzH195P0lVC/T8l7O9s7NfCHzt75kdRasO", + "D0DZBtrdEX3hAsYh9te3UaS6yPNJDefuZ8/voTJ+d2NvwS0GoAae2q1Vib8Xzjbe9vOTN28GrtDVxL07", + "W7RDdqSe5Rydh7TgrtB4rbh5wS+hePf9YEz8LFB4egdepvG8XW3mac7FrXscIn5P3rzpgvusYMlQfgXV", + "w+4JKR8UGdHaaiBjdEHaexsG6c4RnSMi9IIk7vR9o7x8e/zq6Kjnnu/X6J4nto2//UndWO/JmqbHEXsZ", + "eoFrzt3V567pq6gJr3XJ1LvTX3r6CbNB2u4aWYksYuo/fOxeDlcrOjaKW2N9nmHMmOp45m+Z7YXxMpNz", + "mvVfRyt5mlT7tAllajvannitk+gs64ZaXxUwrNzVLHZQWPJkhintKxZAo5q9ah+jY/Ek1NH6G5TR0iEI", + "h3VuaJatcXvRwwWFbc4alqe7TVtMgnBN/3e3PlTDUqZJIvEGXKtYmQQL5AjyFhb/hhk6/dXdCxLRdXGm", + "rz8WVMTddLFWRK/ktW7cStiaE9wu74qJMftZGq8aXveZdO5kEClVqfMo/VGTcIflQ/t5wqQGOVyRjTeO", + "GNR4ejQMvIDrCuNF5T0gK8SL3MObMK2dsO7Q90OaMPPGHLeyXuZlcslMPKX9HBw6skzD6rH1Xjg6Tlyd", + "5NhdKVVHkWkspEog/Hxm1hnrO2O/7PscDzv3gdqZULFSbJU1NMSYqUWnWzIHb0rtj3jUblMNMY3e0m7j", + "AUEk38ttQoiWT7GPpn0LXZiYRSUXnDMSXXcDnJg+cppRsSVJ+YigDsMWtpM2PblQ42FiOgmEm6SRm9c5", + "1ZcxhC9jEcsB/Q2r+F0DymFhZUrs3D9kJwg5kYX3y7t6YRAFUXy5ZPHoI4ajAjNobFVnDgCADuZuCpHf", + "jIXtEwrR80i4bX741rEkfEkM1ZedHOtar96ZgndEjEdCmlP3p7scYhS28nX7HvaNWKvrdzJErsWoG0Mb", + "70EYONgJUznXIQOzdSK6ulU/UhGo+WU3xjTQNdWjU/qxY3pZLy/xyqNnJdF4xKLMsiOZ59zc3gUBfdrp", + "xA9Xb+UCiyczbGF01sFWn1bV+7i+6BhEuQQ9iBY8p8nKys71tLhc2gcargGfXj2fWnFvNcPInVzuTU0N", + "9uoQRmP0WpgVMzypVfuC8oYresXGhIskK4HysOIkFSm5oorLUodsKvRbTclhFTvM6Ro6wCQqd7fzp0qF", + "HRM/sc/RYk6Gi5LF7qrBN/5Gd82Mz9By5VQNoVjIhkjRuskV0J8oZkolWIqx4eqwf7iL3VXbWFFNcquX", + "AqiCux1PKWL8lGsiC/qPkoUws7/QzEgC5pa/oj/UUnTR6lqI1G4BZmiDRgaBeTjrZxRnV3jbLQhhSElb", + "1CJnAe5HCBW7SRTvEMfiv9BX7QbwQmrNIea2qK+0WWzErttf7S8VgsCsqJUfC3ZNci5KCy7YXMvyWIog", + "8VvvcwDw/nEPbbzEzxcVcKVGYScRlL62GN5Zl9DMQ8pBGvdywRVcQ4yx1DEpRca0JmtZ4nwUSxgPoDTy", + "kgkMS1NBGMRhnRTrqZSZY3neY8PyI1kKE7uBvt2me2OwLufabrd9ByjnZg/bgZZlqLEE1OVv4fPb7xc4", + "xQINzD9FFPI6dErAO2w3CWGtWQYHlbUr4dC+x9jN3E9KkxIvTwfsRfDabvxWZGxh8DpgaODr/KUlOJM0", + "U5xm/LeqllyYKK9uaCRPGAf8n7OEWquDh2ITyaoUl3DTd/XWuOzLUBQDGj2t1uPu4hAS8bK9JlxIuPz+", + "Vivx2Q0yS0GZooJcPZ8+/4Gk0tfMqo2BuG+5PtyIXOpa8lYMU54xbXgOhQmeNep2W8LNMryyaUqOwG0T", + "0l/suIoBI+3rGy9+Bh6h3A/2kSZm4AXcLeqNme8KaZea+sWSFRv5o64l39TthSqJpHPv43ztPGuQxpNa", "6zjnwt346dgbUnaoWfI34Ae+prNxWXc0cOJal3BOBzgUKUUuUygwCSavZy448yk5kUWJ18/MsRqtXmvD", "8imxqiMUeXpwH0UiBdp9yXriSiJOqEgngZ0n66jzkGWLX7iIKMz+Deb9vDv9pZ3uE/Zl0PpnYiZevT45", "fX10eP76FanyGJDKoFKlleJ0STt1HgV5Pn2xbzGYWWW9yW64BiNOoNSEu9xzecX8Z8/9ZwPT+AapS3hO", @@ -2752,62 +2767,63 @@ var swaggerSpec = []string{ "coWdUcPSaoifuUi/duqv6A1rQNzuzvAhT64riwbZDhfLzHWPNqKPNTq/Tfq0h3MbtT5cGKbOWCJFrCLm", "8aJWX726Vp8LovETXxem8hy7mKA7SYG+iHRKzuyOOvUFs7/Re1LP9Ab+Y+gllNujGVgEhhEKlg2ZON+t", "1KEj05Reoc+VvCaZxDDoNeUmzJJe+nz1dvfTYfVjSh5B/nfHr9q7Oe3dprDffVvVxt94QlipmZosS56y", - "vap2nf5DyWNYeUcxuEH++VNOplTCCWy7SwnNsiA8xHfGt0CPlvc+7c6IPPQZkUTGzrmelcslcs6/np+f", - "+L2xbatiXMh5xmSf8FDweyCN1Eqs3ZMMrOlhu4Mq93xQ5Q4WRf1INDi0We89aM30jjujRQha3MkAuV6t", - "WzN3+TJ2cbPRX1APnI3cQu9gmZBDr6knGVXo/6ICyc9BEchvXlqGydDNKa+YUlbL5Ga6TZ3Ms8hJdY6K", - "ldU6DshsdFZC3oi1RVV9pQ+OjlabAOdUs2zkDScbNUtKxc0a7slCUfGSUcXUYYmXCQDyQCgTHlfd2jWM", - "Pts+ePS40x+I7QIDB/bRTBxmWZ2CiY8+Hp4c+0L65IP9SCrn/TggOBkyK/f3v08gdgB/sg9kBYYzKnSU", - "gInjggtcYOHICRSOnAks0orvnFIg585bP1+7+Ie/SyAxmWtqOZb54JQJ+OFL5Nm34IZR3Bp3PESQdKIY", - "Ey6Qzw3kyJ7g3QRhtUiNtWDjwej5dH+6745sC1rw0cHo++n+9IW7qBR2Zc9F0yce2ktmenIRLDyXfrb+", - "kBIYlN7J18gjY/qGg18Bz4/T0cHoJ2biB97AXw0GNEz4xf6+DxsyDNrUcjf3/scxFgeNGzhXfEBAvrb8", - "BepblFlFnRawP9zjZPBIYWTwd0L3DP/jlxj+2GtQzvHBXMPxSJd5TtV6dDA6ah48NHSpRwfvawfgRhf2", - "gz3RyGvbjGqo5GgI8Xby23Iq6BLpzBFADKes0Kml0j0gJjUznQdjUAOIb9yaRH3GHpRYpRUdTHBa4OPE", - "cZaJV418bnvt+ybM9z6Fvz/vYTbgxJHsgP1wKQFZ1kokBKugC/dGTiWkQVY5kQfvY+lDMDEvBLvJilBP", - "gJqVPzJRW2jjeAVmSFbb1hZXFw+IBs1Fb4cLO27iCQHyWVtIViMFl2vroAzEUEi9CXXxQI5lJYJdt3oG", - "Qf7smQ8nPHsGAYUPHz7Yfz7Z/xAyC7rwbHTgH1ZRB6uf6e89Kc1G42YDQFFs5Ug2NPk89gNYNavVuUVc", - "33mj0yobF1/j7+eNNiHNGJvgz79f2t+1ViFD1o0DPzutMMXWraCcJEwYRbPJ89movorPAW63AiD9rVTs", - "AWEI/W8EY8hX3ghJN8O/0wSieX/HFWyAaat9HbhtwHUYKR59a3CVx8ZJQe9+KbFG173wjsiiXU5+hJ+c", - "d1YYEhAgwOxrgbbX9flLSYGdALiFOokHKDuYu0EC9KtDbUVnuE6E7z6jYMmYYRtEDDbQEYqr/PE+gvjB", - "dvuhqza9gj62pvZtCX0rGh8/Kk3th5hrdEdLm2gJkWorWhroAoihecI7eO5t/yW/YgKfASpECOAnZnbY", - "/8XtlJ2E2p6qfmJmK5IqqElWG4gKQw1biQ/yVmQuIyW0cDkgPlfEBzAimmXk4OOO2u5fl+0/XzpMl4UN", - "0dvs9U7T/Zb4COLHl9d0/WG6iXf/u3JqcPhsG2dKT1l6L/QrdK0L/l5LN147fQu+VCf+x84begrFx/lC", - "H5y/urE7eBV9rODF/vMvP5kjd5bAMQicx4svP49DVwFvxxM71n8PxneY4w1MsZfT3YI73tYh0Ee8Paod", - "xFNv4Jdo1j1Ofjne5nC4gwXk3VgetpClSF1C8RvnNH7vHcUXoQZmbOE+Weyh1NHjBdHMjN2hlaCQspSU", - "BZ4chWyjlnb6j5KpdTWNJGNUlEVb8+5Mo7py4iENwS1zCnca3m39L1txs4EOmAdgKz8xs+MpD8hTLh6z", - "JrYj2cq585i0D1/g9e7Gma8SfS/Wma9n+vswz0L11oH2mQf1YzPQNqzjK1hoG2bzZU20DRPZ2WjDbTQV", - "eIJnkx6wW/LJwPNuwyjvzU7zRHzfhtpjYZ3baVW+JP2d1KrTBl/8FvSqnY30tWykzdzktlbSPRB110za", - "UfS3ayndQiXaUe4GU2kz2RalGRgIfwjKxYDbjni/APF+GyaZi5vvTLLtTbJFme14YSeW/7hsoq0O9rSn", - "rruOota1wt1zPy1s0o/DPfRlCHl34OcOB346yFcjmFAp3QF6+0M/HarcDrOjDtDfiedzsHx9bK7ORyJQ", - "h0nSbP3AHs6da/NOrs2buNFwOb6d/N775MU/+DT3aol6txXrLpaltw4DReT7Szedb8p0upvJtNlWqu/W", - "4w4N77SVe9RWPE19jQBxh0fUA8a3ZhK+E1c6q/P+Dk6YCB859VPeMZJviJG4XdtxkvvkJKoiha/hMLi3", - "4Ol9B013rGGXyroL0z6+MO1NltFt47T3Gp/dMY9vIRK7o8r7CcHe6DodFIO9X6U/GnndkeUjj7Hezvn7", - "CIKqO1ZybxHMr+f6RHdGtcwtLij1dQurj3sTKe5V0TiqJrvjbd+AylHbrx3HuJ/8r6ROAl+XczRr+A69", - "27j6qrf68D0zjdo8d1zjW+AaYcN2XOO+uEaDBu6JbUzqvd6GgxTcqC1Yx4nkwky4mJzzHErUyisGZeIW", - "8guxkhM74R0P+QZ4COzUjnvcinvcQGtfWu9gYsnFLeOt7ts7JWO8duP/HnItca27kON9hBxZwJsOuSCY", - "h1KL72gLYtkri6WiKZsUGRVDKadgAkrtInClIq4T3bxltJ7LOROHoT5nth4TbgjNtIzUl/Cdu/J1WGYc", - "SqIJFioWF0wtpMpZSmbCFamzcpourCzC2WBt/gBkP1c/Fyx3fPV8+ny6P3ZF0i33ynOsK24kCfXR7cqt", - "3tBZ7xSrz2DhcvfQtsaqvCkrFEuwWraoaqPXqi1fPZ++mO7HNYp32N2J3Zd/ZY5SX+eOldxKDnvMKxBX", - "PBd569BVfyn+sUeLQskrmg24IyOwjIgYDoR2w9GHb4CQDwEi7NER80NcshqWeOjRIILTpzg0bEPFqBsW", - "SRsJhgYwdoxjuzADYvkmsH9RTlJlPG2bq+Bmfj8WvFO5vg3jnfnJfitWt4PuTtDfzV0X9n2TxXCLM953", - "p6RmgsHvnJgeLjGgn44ed17Ajv7vKy1gEAu4H1GdS8GNtIg94UIbKpLtvGzV9yR8b7Vm2nEURP1rb8Ln", - "x2H030clw8jKdy63O7jcYohYo6AK3NsfbY50jRZq7E1VOh2wTJMPFqs+OP6smZnOxEuqWUok2r/+/cpa", - "nVY6G37FyCVbY4nkRIoFX5auGLdgLNWNvs7KZEWoHhO+wK4OSJHnH8a2Q0E+2L+hs/qX1oTj1oB2RZgb", - "Y/Sfzu6i7L9+rbzumhEWm4uMvOnHi693eDuyfTtmc9vTyxHK7+c2/aI6Kn63FNe3PU8UY15bltK7HUfw", - "zCAOwy9T6OjNNmP/virr/YBzfdjhYxxSSIMpCo/xUE4LWQXdRPADvVx3osCfmLkb+b35PZHfTozuaDvu", - "eNtKkm9TZvBO1I0ugZ18/draPu7DZm0/v0nb/yqlA3d86l+HTzkH4UMbHQVTOdeaSzHABxhL7wmfh1zc", - "UjOFKT5ck6RUigmTrUkml0sIr4Mj5dnrjzQvMnbwbCYOtS5zPDC/kFkmr+1qT18eHpFCZjxZjyFSYbvV", - "5APNeOJjF3M5/3AwE1DnvxgTJTN2kLKrceWC1GOiGE3H5FmrRdthOibPxuTZXm8zn7rYaDeX841NlmMC", - "0616dJO1LMQCFHIPEKqt5bcB69btV/tpJgiZjWqtZqMD8t4+Jf4f+7/ZCL6bjcb1ZxV4Wi8srFqPns1G", - "+PNiPLD3Nmi7HTZ/791hCA/zLcaw/1zMxGcHyUOR3gT6OpoNB/xczh9u1tEUM83USY2cHzLLqzXUzql0", - "u0wvyymLxpZ5zn5YmhUTxk2MzMr9/Rd/JPapVPw3XM6F7XHPs/qtTmPRgibcrDHN8oryjM4zcEZjV16l", - "+7mcMyXAfeQPFsRxr2pY3WLkZvWAaLhh1B1Gbu/mrO5KClvn0bGCtMM6zQBlB6QGcq3LEFf5z/86J0Ze", - "MgGc1aoEqNng1TgW5V6jBmM/dWl8IXQB5JKX2pAVdRfsfMjkkosPgNBznnGz7o9lnLkpP1DCnG4eOewx", - "HGANzWNZ92sfFMqu3XD8GmAdNef8E7RadvQymF5YUipu1qOD9xd16vF4++6Y/GJx8la8XDNjuFhuoYpD", - "Oq37ynNtPxXQ9LMMQ3wxrn3mh3tAHh3GGIxhG4Bcm7AH7k9MMEUzPKKEULxiyvOm4UB0H7VhaJshDsQY", - "y9/wo2M8HvVgMHTDbAfCADT/dT/MmhD/NHrJqGLKIqjdgM92CwAE6H0qVTY6GO1dPR/ZN67PNowt/NZm", - "Zbm7YhkkWxvZ1ilq99k7O7UmZ7oep/4+2wfSaj12zqrdqt/qMFi7W58CdYfZktpFna77UDDiLt1W9wi7", - "Xn350i06fdmO3je6Iv6GuqFdVn6IqquaE2NoN7TJUUGLbbDT0PkQ3tsdtU4gKneDzGVpevlrNWKDuO6A", - "bORtLXXb9V09+nzx+f8HAAD//5OT678uSQEA", + "vap2nf5DyWNYeUcxuEH++VNOplTCCWy7SwnNsiA8xB+Nb4EeLe992p0ReegzIomMnXM9K5dL5Jx/PT8/", + "8Xtj21bFuJDzjMk+4aHg90AaqZVYuycZWNPDdgdV7vmgyh0sivqRaHBos9570JrpHXdGixC0uJMBcr1a", + "t2bu8mXs4majv6AeOBu5hd7BMiGHXlNPMqrQ/0UFkp+DIpDfvLQMk6GbU14xpayWyc10mzqZZ5GT6hwV", + "K6t1HJDZ6KyEvBFri6r6Sh8cHa02Ac6pZtnIG042apaUips13JOFouIlo4qpwxIvEwDkgVAmPK66tWsY", + "fbZ98Ohxpz8Q2wUGDuyjmTjMsjoFEx99PDw59oX0yQf7kVTO+3FAcDJkVu7vf5dA7AD+ZB/ICgxnVOgo", + "ARPHBRe4wMKREygcORNYpBXfOaVAzp23fr528Q9/l0BiMtfUcizzwSkT8MOXyLNvwQ2juDXueIgg6UQx", + "JlwgnxvIkT3BuwnCapEaa8HGg9Hz6f503x3ZFrTgo4PRd9P96Qt3USnsyp6Lpk88tJfM9OQiWHgu/Wz9", + "ISUwKL2Tr5FHxvQNB78Cnh+no4PRT8zED7yBvxoMaJjwi/19HzZkGLSp5W7u/Y9jLA4aN3Cu+ICAfG35", + "C9S3KLOKOi1gv7/HyeCRwsjg74TuGf6HLzH8sdegnOODuYbjkS7znKr16GB01Dx4aOhSjw7e1w7AjS7s", + "B3uikde2GdVQydEQ4u3kt+VU0CXSmSOAGE5ZoVNLpXtATGpmOg/GoAYQ37g1ifqMPSixSis6mOC0wMeJ", + "4ywTrxr53Pba902Y730Kf3/ew2zAiSPZAfvhUgKyrJVICFZBF+6NnEpIg6xyIg/ex9KHYGJeCHaTFaGe", + "ADUrf2SittDG8QrMkKy2rS2uLh4QDZqL3g4XdtzEEwLks7aQrEYKLtfWQRmIoZB6E+rigRzLSgS7bvUM", + "gvzZMx9OePYMAgofPnyw/3yy/yFkFnTh2ejAP6yiDlY/0995UpqNxs0GgKLYypFsaPJ57Aewalarc4u4", + "vvNGp1U2Lr7G388bbUKaMTbBn3+/tL9rrUKGrBsHfnZaYYqtW0E5SZgwimaT57NRfRWfA9xuBUD6W6nY", + "A8IQ+t8IxpCvvBGSboZ/pwlE8/6OK9gA01b7OnDbgOswUjz61uAqj42Tgt79UmKNrnvhHZFFu5z8CD85", + "76wwJCBAgNnXAm2v6/OXkgI7AXALdRIPUHYwd4ME6FeH2orOcJ0I331GwZIxwzaIGGygIxRX+eN9BPGD", + "7fZDV216BX1sTe3bEvpWND5+VJra9zHX6I6WNtESItVWtDTQBRBD84R38Nzb/kt+xQQ+A1SIEMBPzOyw", + "/4vbKTsJtT1V/cTMViRVUJOsNhAVhhq2Eh/krchcRkpo4XJAfK6ID2BENMvIwccdtd2/Ltt/vnSYLgsb", + "orfZ652m+y3xEcSPL6/p+sN0E+/+d+XU4PDZNs6UnrL0XuhX6FoX/L2Wbrx2+hZ8qU78j5039BSKj/OF", + "Pjh/dWN38Cr6WMGL/edffjJH7iyBYxA4jxdffh6HrgLejid2rP8ejO8wxxuYYi+nuwV3vK1DoI94e1Q7", + "iKfewC/RrHuc/HK8zeFwBwvIu7E8bCFLkbqE4jfOafzeO4ovQg3M2MJ9sthDqaPHC6KZGbtDK0EhZSkp", + "Czw5CtlGLe30HyVT62oaScaoKIu25t2ZRnXlxEMaglvmFO40vNv6X7biZgMdMA/AVn5iZsdTHpCnXDxm", + "TWxHspVz5zFpH77A692NM18l+l6sM1/P9PdhnoXqrQPtMw/qx2agbVjHV7DQNszmy5poGyays9GG22gq", + "8ATPJj1gt+STgefdhlHem53mifi+DbXHwjq306p8Sfo7qVWnDb74LehVOxvpa9lIm7nJba2keyDqrpm0", + "o+hv11K6hUq0o9wNptJmsi1KMzAQ/hCUiwG3HfF+AeL9NkwyFzffmWTbm2SLMtvxwk4s/3HZRFsd7GlP", + "XXcdRa1rhbvnflrYpB+He+jLEPLuwM8dDvx0kK9GMKFSugP09od+OlS5HWZHHaC/E8/nYPn62Fydj0Sg", + "DpOk2fqBPZw71+adXJs3caPhcnw7+b33yYt/8Gnu1RL1bivWXSxLbx0Gisj3l24635TpdDeTabOtVN+t", + "xx0a3mkr96iteJr6GgHiDo+oB4xvzSR8J650Vuf9HZwwET5y6qe8YyTfECNxu7bjJPfJSVRFCl/DYXBv", + "wdP7DpruWMMulXUXpn18YdqbLKPbxmnvNT67Yx7fQiR2R5X3E4K90XU6KAZ7v0p/NPK6I8tHHmO9nfP3", + "EQRVd6zk3iKYX8/1ie6MaplbXFDq6xZWH/cmUtyronFUTXbH274BlaO2XzuOcT/5X0mdBL4u52jW8B16", + "t3H1VW/14XtmGrV57rjGt8A1wobtuMZ9cY0GDdwT25jUe70NBym4UVuwjhPJhZlwMTnnOZSolVcMysQt", + "5BdiJSd2wjse8g3wENipHfe4Ffe4gda+tN7BxJKLW8Zb3bd3SsZ47cb/PeRa4lp3Icf7CDmygDcdckEw", + "D6UW39EWxLJXFktFUzYpMiqGUk7BBJTaReBKRVwnunnLaD2XcyYOQ33ObD0m3BCaaRmpL+E7d+XrsMw4", + "lEQTLFQsLphaSJWzlMyEK1Jn5TRdWFmEs8Ha/AHIfq5+Llju+Or59Pl0f+yKpFvuledYV9xIEuqj25Vb", + "vaGz3ilWn8HC5e6hbY1VeVNWKJZgtWxR1UavVVu+ej59Md2PaxTvsLsTuy//yhylvs4dK7mVHPaYVyCu", + "eC7y1qGr/lL8Y48WhZJXNBtwR0ZgGRExHAjthqMP3wAhHwJE2KMj5oe4ZDUs8dCjQQSnT3Fo2IaKUTcs", + "kjYSDA1g7BjHdmEGxPJNYP+inKTKeNo2V8HN/H4seKdyfRvGO/OT/VasbgfdnaC/m7su7Psmi+EWZ7zv", + "TknNBIPfOTE9XGJAPx097ryAHf3fV1rAIBZwP6I6l4IbaRF7woU2VCTbedmq70n43mrNtOMoiPrX3oTP", + "j8Pov49KhpGV71xud3C5xRCxRkEVuLc/2hzpGi3U2JuqdDpgmSYfLFZ9cPxZMzOdiZdUs5RItH/9+5W1", + "Oq10NvyKkUu2xhLJiRQLvixdMW7BWKobfZ2VyYpQPSZ8gV0dkCLPP4xth4J8sH9DZ/UvrQnHrQHtijA3", + "xug/nd1F2X/9WnndNSMsNhcZedOPF1/v8HZk+3bM5ranlyOU389t+kV1VPxuKa5ve54oxry2LKV3O47g", + "mUEchl+m0NGbbcb+fVXW+x7n+rDDxzikkAZTFB7joZwWsgq6ieAHernuRIE/MXM38nvzeyK/nRjd0Xbc", + "8baVJN+mzOCdqBtdAjv5+rW1fdyHzdp+fpO2/1VKB+741L8On3IOwoc2Ogqmcq41l2KADzCW3hM+D7m4", + "pWYKU3y4JkmpFBMmW5NMLpcQXgdHyrPXH2leZOzg2Uwcal3meGB+IbNMXtvVnr48PCKFzHiyHkOkwnar", + "yQea8cTHLuZy/uFgJqDOfzEmSmbsIGVX48oFqcdEMZqOybNWi7bDdEyejcmzvd5mPnWx0W4u5xubLMcE", + "plv16CZrWYgFKOQeIFRby28D1q3br/bTTBAyG9VazUYH5L19Svw/9n+zEXw3G43rzyrwtF5YWLUePZuN", + "8OfFeGDvbdB2O2z+3rvDEB7mW4xh/7mYic8OkocivQn0dTQbDvi5nD/crKMpZpqpkxo5P2SWV2uonVPp", + "dplellMWjS3znP2wNCsmjJsYmZX7+y/+ROxTqfhvuJwL2+OeZ/VbncaiBU24WWOa5RXlGZ1n4IzGrrxK", + "93M5Z0qA+8gfLIjjXtWwusXIzeoB0XDDqDuM3N7NWd2VFLbOo2MFaYd1mgHKDvJAcnFFM45W0mtUUOD5", + "f/7XOTHykon+64rO3DB3ShF78ePDA/hcSpJTsSbUGJYXRj+qrfVQf3dMfpFLWZqtOc2N8TOudRnCZ2Fn", + "QYBazQ8VWLwByXKWGh64bM0QoQKumJfakBV19yh9yOSSiw/At+Y842bdH7Kqo8wD5EXq5snSHvsQ1tA8", + "fXe/ZmCh7NoNx68B1lGr3T9B4/RbMgd/t1TLklJxsx4dvL/op2EubqUsaGYMF8stbD3I13ZfebXATwVM", + "ySzDGHJMLTjzwz2gEhDGGIzbG4Bcm7AH7k9MMEUzPAOHULxiygu/4UB0H7VhaJshDsRY2t/wo2M8f/dg", + "MHTDbAfCADT/dT/MmhD/NHrJqGLKIqjdgM92CwAE6N4sVTY6GO1dPR/ZN67PNowt/NZmZeWKYhlk8xvZ", + "VlprBROcI6SmyHRdmv19tk881nrsHIa8Vb/VacN2tz7H7g6zJbWbYF33oSLJXbqtLqp2vfr6uFt0+rKd", + "HtLoivgrEId2WTm6qq5qXrKh3dAmRwUzqcFOQ+dDeG931DqBqNwNMpel6eWv1YgN4roDspG3tbMBru/q", + "0eeLz/8/AAD//2SptQSPSwEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/client/everest-client.gen.go b/client/everest-client.gen.go index 9670c56e9..c647c617c 100644 --- a/client/everest-client.gen.go +++ b/client/everest-client.gen.go @@ -1791,6 +1791,9 @@ type ClientInterface interface { // GetKubernetesClusterResources request GetKubernetesClusterResources(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) + // DeleteSession request + DeleteSession(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) + // CreateSessionWithBody request with any body CreateSessionWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2367,6 +2370,18 @@ func (c *Client) GetKubernetesClusterResources(ctx context.Context, reqEditors . return c.Client.Do(req) } +func (c *Client) DeleteSession(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewDeleteSessionRequest(c.Server) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) CreateSessionWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewCreateSessionRequestWithBody(c.Server, contentType, body) if err != nil { @@ -3941,6 +3956,33 @@ func NewGetKubernetesClusterResourcesRequest(server string) (*http.Request, erro return req, nil } +// NewDeleteSessionRequest generates requests for DeleteSession +func NewDeleteSessionRequest(server string) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/session") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("DELETE", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewCreateSessionRequest calls the generic CreateSession builder with application/json body func NewCreateSessionRequest(server string, body CreateSessionJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -4208,6 +4250,9 @@ type ClientWithResponsesInterface interface { // GetKubernetesClusterResourcesWithResponse request GetKubernetesClusterResourcesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetKubernetesClusterResourcesResponse, error) + // DeleteSessionWithResponse request + DeleteSessionWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*DeleteSessionResponse, error) + // CreateSessionWithBodyWithResponse request with any body CreateSessionWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateSessionResponse, error) @@ -5089,6 +5134,29 @@ func (r GetKubernetesClusterResourcesResponse) StatusCode() int { return 0 } +type DeleteSessionResponse struct { + Body []byte + HTTPResponse *http.Response + JSON429 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r DeleteSessionResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r DeleteSessionResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type CreateSessionResponse struct { Body []byte HTTPResponse *http.Response @@ -5096,6 +5164,7 @@ type CreateSessionResponse struct { Token *string `json:"token,omitempty"` } JSON400 *Error + JSON429 *Error JSON500 *Error } @@ -5571,6 +5640,15 @@ func (c *ClientWithResponses) GetKubernetesClusterResourcesWithResponse(ctx cont return ParseGetKubernetesClusterResourcesResponse(rsp) } +// DeleteSessionWithResponse request returning *DeleteSessionResponse +func (c *ClientWithResponses) DeleteSessionWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*DeleteSessionResponse, error) { + rsp, err := c.DeleteSession(ctx, reqEditors...) + if err != nil { + return nil, err + } + return ParseDeleteSessionResponse(rsp) +} + // CreateSessionWithBodyWithResponse request with arbitrary body returning *CreateSessionResponse func (c *ClientWithResponses) CreateSessionWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateSessionResponse, error) { rsp, err := c.CreateSessionWithBody(ctx, contentType, body, reqEditors...) @@ -7080,6 +7158,39 @@ func ParseGetKubernetesClusterResourcesResponse(rsp *http.Response) (*GetKuberne return response, nil } +// ParseDeleteSessionResponse parses an HTTP response from a DeleteSessionWithResponse call +func ParseDeleteSessionResponse(rsp *http.Response) (*DeleteSessionResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &DeleteSessionResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON429 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseCreateSessionResponse parses an HTTP response from a CreateSessionWithResponse call func ParseCreateSessionResponse(rsp *http.Response) (*CreateSessionResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -7110,6 +7221,13 @@ func ParseCreateSessionResponse(rsp *http.Response) (*CreateSessionResponse, err } response.JSON400 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON429 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -7174,12 +7292,12 @@ func ParseVersionInfoResponse(rsp *http.Response) (*VersionInfoResponse, error) // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9/XMbuZHov4JirmptH0nJ3t28i94PebLsbHS7XqskOVfvTL0YnAFJnGaACYCRzHX8", + "H4sIAAAAAAAC/+x9/XMbuZHov4Jiriq2j6Rk727erd4PebLsbHS7XqskOVfvTL0YnAFJnGaACYCRzHX8", "v79CN4D5wlBDfdhylqnKWpzB4KPR391AfxolMi+kYMLo0cGnkU5WLKfw50uaXJbFmZGKLpl9QNOUGy4F", "zU6ULJgynOnRwYJmmo1HKdOJ4oV9Pzpw3xKNHxMuFlLlFF6OR0Xt608jmmXymqW/0pzpgib4MGWFYgk1", "LB0dGFV2+v+Fa0PkgojwFXH9ECNJqRkxK67JvDGN0XjEDcthALMu2OhgpI3iYjn6PPYPqFJ0bX/Py+SS", "GTuraPPGdCLvF1Il7ISa1ZlZZwyXtKBlZgLA3CdzKTNGhf1G9A0WVtl9Ox59nCzlxD6c6EteTGSBWzQp", - "JBeGKYTf5/FIsWV0ssN7wO8+jZgo89HB+5H+fjQe0d9KxUYX4+6sS5VFV3PFFF+sz385a0AFd7kNFJj3", + "JBeGKYTf5/FIsWV0ssN7wO8+jZgo89HB+5H+bjQe0d9KxUYX4+6sS5VFV3PFFF+sz385a0AFd7kNFJj3", "P0quLCK8Rwg19sZ9Uo0v5//DEmPHaeCvthhjBwwY8G+KLUYHoz/sVQSw57B/r4n6Eew4Uowa1mh2QhXF", "nm9PJ4XtgxmmdJdMkoRp/TNbR2H6TRBRc/TzFSNJJss0rB5b7yVSGMoFU0TUdvhLEV9zkocWDIqkbMEF", "szO1Q8C8LODMitVYHPx89esZvkaGR1bGFPpgb++ynDMlmGF6yuVeKhNt15mwwug9ecXUFWfXe9dSXXKx", @@ -7197,102 +7315,102 @@ var swaggerSpec = []string{ "O43zUgMqIboGRNK296DFEYpjRsirQ+a3Qgk70Sj8yyKTND22wuOKZmcxJvGu3YSIMp8zZYGjWSJFqsmc", "mWvGcGlzLjK51AS7rrEqK6KWTHXUAb+imJQPyNmd15l/hSvOnGrs6Sp8WNN+o1vvUbtFl/5xA/+mXwjF", "jk6R49WY8Ux4vTWTyC2mjxffYEgHwdFw3b0PON2u6uqyQRl5JAsew5PTZoPQf0Bit+MJvjaSKGatmRFY", - "JTk1iLrfv4hgcoWg/fgZGJmSYsNKWkTRxatqK8Zegw69xUinadl9jrSw2sIZKFBx1QDfBSSkoCwTp3JZ", + "JTk1iLrfvYhgcoWg/fgZGJmSYsNKWkTRxatqK8Zegw69xUinadl9jrSw2sIZKFBx1QDfBSSkoCwTp3JZ", "GTuX0mijaGG1MkoEuyZOj+6jk57RXtbetgnRaXh2WywFMFDevhAdghYCK0WZ+mVIrqBmFRGK1Kz8jG0L", "bwA4OC14xvZSrlhipFpPb4VgMHAMl9K5my+uPA7fVy87jWIQfvXSI4mfendvuyC5UU8AlWDCxaShEjTZ", - "dwdrrCIfxf0w83fnRxbtHQJCp9YeIBYNrM1aGMSQnJoDMhu92N//42T/+WT/xfnzHw/2fzjY//G/Z6Po", + "dwdrrCIfxf0w83fnRxbtHQJCp9YeIBYNrM1aGMSQnJoDMhu92N//02T/+WT/xfnzHw72vz/Y/+G/Z6Po", "Lns7PNjOOJu2y+d8XYTJ2E8sGP3qpsAU0Ix3H6M5GLHkuwwgxhKYWHLBYszePvfz8EYzweY3KLG4Bd0+", "Ue/2fbqu2vvVAVuiei3xo1NviPOm/eJscY+BVrqit+yaWzqywrUUKVPZ2jIyO3dqpLIG3oKUwq2OpWPC", "rpjF1IlvgtYC+t0cxfuxHL3XOpuJX9+evz4g76z9iHYs18TBak0KCWa8NjTLUNm1RmvGKKjSFEiEKuMX", "kWxgIIoVGU9oVBjim64UdPAPn0akX84Fzy22PY9JwsrYj4zqXhHqNOdguWccbG3LY8HSaE4Dt8BaY5qZ", "cecr25t9yfNCahCMLcwrSjBKxfrtYnTw/lN31h3H1kWb/o5O3nlg2T/DFBwvzSEMA6zTMGU/+H9PZrN/", - "/+fk6Z+fPHm/P/nTxb8/mc2m8Nezp39++s/w69+fPn3y5P3Pb346P3l9wZ/+870o80v89c8n79nri+H9", - "PH36538D/2Dls5xYbijVxK3LuwZzlku1vjNQ3kA3Hi7Y6bcNmhgz1FUgraXaeQ9yg3V5/XyzyEkyqiMk", - "cmQf+w5DT/DQ8SrvsSwsg9FWpyVXMitzaMajUlPz39id9/qM/xZWajsMNnjvPL6VDa+rQwCqfjX60wap", - "7LYfGlbyuPiYWFBIbZaK6X9k9ofO03ncya6ZOgOvt47rVu+aDaJGErwmLhbj/aTgL8NXUa/hVZ8wbYlS", - "t0jf/Cbtsgo99Trwcym4kbgj7cHfhHeBx1RPNtNX1RD1izg830RatYFKSbsvcnTqLID29/dvBAwSp940", - "awpG5wv1DKNaxTTGjXgeZ0c81+BUqYCiUfd0g49DjI0L0ACn/hV+PJ4J8GFYIQ121HyNGk+IFoJOdG4f", - "cU2oIDQrVtT5f6lIvRxx/jWH0TPxai1ozhMPhcPMO0TIglHwzy6pYVXn2KEdJc9LY03oKTk24ESWIluD", - "o5eh0zhMDVy7PX6j0/oyiWILppiwuyGFpRNjBaMgJzI9s0BptNbdHdjgCQGcyqlJVg28bAxTyHQaAT6R", - "Cwt+ZqcRHJZ1WNgdATDk9BIcTNRUWESvKM8soGaCC81TRmht1+LYCrGSGLDgRYO2kpXUTADAqY+yeIIJ", - "4ExRnKAGyPLCrFH9XpuVxYQQwYFWtvucprWZj4k0K6auuWYzAdvs1M4yM7VQHIx9s7EMm3Sjk6UldSzx", - "THJaTC7ZWtd76bZy3eS0sJ2idtufl7C1QP9GlNN2rgPo+Phw7iJSOf1oTRBCc1kK2MhE5kVpKosiZETE", - "A3KbovoNwbKXU0GXbBL6nVTMYW8UQQUfLvy979upD5u2dg6Nx40750kOiT50xDWROTfO01LnRWPCwUlO", - "ywzitMQhDV8gR+OasI/WkuQmW5PKkJ+JwB3AuBbWhMzAYoHNn3jRBtHnaTWVBKPA7GPCWOpG+7KINsyP", - "U1DL4GNORBDFDZ+9NrKouxTigTolP64j/dnHwQUHPxrOoCmp2+9WxhdW/ClODZuJyAfoUZkz2zDjbsdt", - "50t+xYRTQ6fkcCYSmecY3CUJdfaRZqbyrARZV4tEggLBPrpcCUw68Y7U4NVK+qLbwzxZuKobHVnsYyF1", - "zNUGz5udYdsbNF/uHOinVCxjauPxSf29H8DHzY5PvKtd4fsnR8evTu3ewWhPZ8ISimWtHmxWBDf314Ci", - "wTURsq6J9qtSjSnVMjfsbGiaKqa1nakgjbkQcLyZlSwNRB1MTvXlBh9rld3W9bn6vJmNflcHfvv1GPTG", - "OasSbqQiHqFq5l+t3/B2iFP2ds47xJKv7btrzGLnutu57r6e6+5mrw0ia8tpk0uxlHbhK4oCzwk+579Z", - "zmUpEqYGUrJeUZVG/Rtn7k3wq/nfzWwDcnL25tXLiTVfemQR5rf1SSR8W+er/YMRjY2dCO2mMw/nS3UV", - "r5rG1mypZYOF8S+icasbshS8p4QvmjCosneiag+00z0bqBvJcrWsGPzobstt7G899u96v4jpgc20Ggjl", - "RbNqDDWlvjkfEJo1FinngCZbpQQmhl+xsz5f+mH9ddsBjsqqCAHjJ+BCBbfN02hwUAo0vHSUJNw7bz+0", - "llR9HELV3bX1KDKh86rvlBnKMxSPUjBCrZ5bhe9KpSDZ1MMRVNbDk2PiBW4XkhnV5lxRoWGkcx5zHHXb", - "BEWPaoP5ci6tzk3YhNZW21Yyh9kiiqBxBLbS1HnTXGLyHPLg0GdTi51W3SYrq9OlU2I1RG+MWYl/KeS1", - "AF3RKu/eTw0TCz1aOKD67roBiwXC7eC/q9NWSg2DoH88KVbrKNq5F2gJrcqcCqIYTSF7KbwTKVglYhk2", - "k86t0gkTDmDzkMkp+BKpQHeVS2C2c83px1+YWJrV6OD7F//rj/8RmajHwp+YYH0ps902bdY+9UnA02XV", - "JuTOVptzTTX4PC1yp6QsYBF/kQrjzyJhY8soo71x7XE3W5PnL8Zk7gAyRZSZVmT0/uPFNDJnrsmfxq0J", - "WRO/BOJIwTKEwLxiSDLOPouQDAsTnrbY7R9/qLPb/bjSS3UMzPi8ImRqdYWlonlODU8IT5kwfMGZqiMI", - "KsbwobdYw+q+04746ihzAvnJTAGz8SZwnSzXBUOcQv5rjRCWmJC9Dx7ynFFhhbUb0xu945mwb69XzFIu", - "HkdwHymYl+YpUywllCxLqqgwjKVw8gGjG5hzX1E6rdLcPVY3fOt2li5lGlC/hfPP91/8AJsRHjQ0y/eH", - "k/+mk98unrg/9id/+vv44OJZ7ecFqoLdAGePIHPcy/NaD9QxsDa5IOeqZGPyFzhZRN4JYEn1ZBr7fjQe", - "QYPReORaREN3cU3TZ+rUMLx2VoAApZGFlFN3JGiayHyvOkvQ4hnP/9hUxd8jWC6evJ+4v575R0//DCr0", - "pgZPn+2B+h3Ae/F+UoF6ahXx2run/3ajdzwilyrOG+gs7NaGmGDbXt8m2SfI8W62D6gRPteHxFJ94mfW", - "gOdH1CQnDAolr3jKNFmUWUaaOFcW2ihG86C6UGAkGeWCGPbRREdcSW3i8aC/ujd+sb5lLRndD+T8E8qa", - "5HWtdoBQfFMJRfbRKFo/BVwTfR0/4XZi7G1UJGCkUsNRJ8vlayIn7GzgchHFrMP8uwy/kCoSoTqRylRJ", - "hMoMAemAxGCrTaxjthJN110HDrQmhYybDbHeE5nnTKQsDYQQG6zbyo9d66E3Pw59ON61Z58LxlLQCqtz", - "UCieuQ69zNlCKvt6qWjqZWMnqa7WqdXBMoSA01Qik5tuSnDpz1gx0tCs7ikbDOI+2eKsomCpNCRNH2UM", - "89q30Pplz0GiaLNh5xtdHvPXPeVI7vGQI7nhjCP5Fz/iSO7rhCPpHnAkjfON5Fs/3uiy9rc95IifTb/W", - "iYOoZuLT8W9IxK8PKRVfcks77TAXTOZ25wWa87iDp8nDYHt/U9/uJNKq1SbmEjzyr4KMaPge/kfOwT4O", - "PQz3Nrjkr8iQPiusGlAbmhcdbRGh/J3GPDIn9oYNnjJtuOjRuV5VL/0kQGntHiSJItySFpFN/IkWujKH", - "vW9VMbAy7SckZQZtVpfdAwc2MrnUUWcrcvlTOApC5xmLe7h+ibSqfFwgJ5yXixqvuQWqggm4wyaDIQu4", - "F1cEwsgeLcP1INQMICqA68XtdQN/RcoA4oKrQzDPzkkRBFDdFepjwRjz5BpdX21+UeNMO/3hQfWH4Gwe", - "dAVOXHuMWNU7teSLqCUDqPjI7+KRT/mx/cQTRCM5l87C7HJSd1aofuVP07JRTkxt8KgNCHD2rSYiKyp8", - "JYplIAwBbDUk78Q3XRLUbQkgAtwIMQwGbyM1+76hW/kRbwJ7/RognHvvNsSW226rGMhvmnW3rAq7kzB2", - "Z48Egzs23uEtQdUFR/7gw8HeXqmZOsAjCP/n+f7+tPb/gx9/qFvf9SOwWl9LlTY7VVKaUc/xCb+PN7Ue", - "gMeDpOq9ydOdIH3kgnQnQh+zCD2JngzvOQ3eEj2ty3aoyjjT5pXT9itO8mL/xfeT5y8m3z8/f/H9wY9/", - "OvjxT/892HqI204udNi2mgpuFBhILfuJLozff3do3pqohl4yscGUap7W78wMG93rcgds2Kmzvm5isK7d", - "ML+mM+l2js2dY/P359h0lLK1Z9N9N41di3G3i2GQHDdfmfStXwWzu7lld3PLI7q5ZauYQJ1L1MMAtQ29", - "GQ9rXOIeQwGemd0iFtDLzxrBgK0TB4f6g2szb5xlCdNtccX7CBG7MQdZrLW29+MI9krXTuF63Aas17h3", - "duxjtGNf91y51Xx/gxmEyf4782dn/vyOzB+kDDB7EOz2Lzwh3rqhbtpXw8ThfpO1bnGMtHtHHmh92lCR", - "VjewVPdIt+alp+SUL1eGCHlNuPlO440kxccEaABOu0zJX+U1u3KH3V1Au9BjUiyhERVrvOuCVOdGNitu", - "vem3N6loDuDbqGav++DvL+qo70D0BiKrQKmyQR3VNR+eUWl3dKBxLWElGfuM0E13NXSTRqCvSlGqJ8c6", - "Xal3BtMAEPK69cpvaevbcfUAjypaXJIy04TnWHjErCKaruKGJzSLhwXhy79SvYpiObw9cRbs3QKDG+6V", - "3IH7C4A73NbQexHJbhcefhe6D+xSdtvyuLYl1sRnq7+DHPaIrH/bbNC0nps54eGuWEyIZ9PqzjPNDAp8", - "dyr5g7tfdlowlUhB4VSQ+yzcOTsx8gMBnS6k8zm52N0Cd53sSUbFqTUXO/dvNN6jFhVu4PJKeq2RV1T9", - "LXdewemscZtrzhyc3Lhm++t0BlVjgn9m4vztq7cH5DBNnc5UarYoMzzHpqekMpXGxKqsY1Ly9M8DnDWt", - "ewpyWvhLvaiROU9u8ikVKxq7DMbh14l92z7sCZ/0YllPIqPVQg/NcD+YoWrJTK/5eF5/7W1UfxDESHK9", - "4u72ujDB6lihm2o6HRZH9D3UJtMFIxMpF8sWeTbV+y0oOX7+6WZs39HdY6K7R4TDbUuyz+KqLK24K9nJ", - "dC4IJZf/oTfcBr6dWxnH3exOrtrczY3sTeCdv+pxeo+dY3LnNX5MXuPXSslIPBUew4UJUsCR9dalyL2a", - "R2yMnwM/dQGEY7GQG/NDfUTIQjFyxzC8PI8nuIZr1uEGdCiHuk0l1OZV6XhLcbh2uHITufMnnk3ORL0a", - "5/vRsngxGo+Wxfeji22KpdZnzoYT2Fnts2ilnsY9QDXoxWB1MWQDT/vvd4vsYp2X9HjtIvnaRfmGZxmv", - "Qw6P3dZTlkcHoxIPaH8ej1KuL8/cCd5hX+B1ZS/Xhg0eZkgCdQDPYVjf5/EooQVNuFn/i671yC+vg3H+", - "xbi23zE0qy5BP3a3sDjPurudbhMNdL99STX7L25WkEAQubeudiW8+6JVpLzj4sbitS7F/yI64ZdRq+vm", - "sb5cQfS8O5etCjq3y/0Wed7NMRleW9iVA25e5nLbzlo1hFvOf3zl/SRVLdDzX872zs5+IfC1v2d2FK06", - "PABlG2h3R/SFCxiH2F/fRpHqIs8nNZy7nz2/h8r43Y29BbcYgBp4ardWJf5eONt4289P3rwZuEJXE/fu", - "bNEO2ZF6lnN0HtKCu0LjteLmBb+E4t33gzHxs0Dh6R14mcbzdrWZpzkXt+5xiPg9efOmC+6zgiVD+RVU", - "D7snpHxQZERrq4GM0QVp720YpDtHdI6I0AuSuNP3jfLy7fGro6Oee75fo3ue2Db+9id1Y70na5oeR+xl", - "6AWuOXdXn7umr6ImvNYlU+9Of+npJ8wGabtrZCWyiKn/8LF7OVyt6Ngobo31eYYxY6rjmb9lthfGy0zO", - "adZ/Ha3kaVLt0yaUqe1oe+K1TqKzrBtqfVXAsHJXs9hBYcmTGaa0r1gAjWr2qn2MjsWTUEfrb1BGS4cg", - "HNa5oVm2xu1FDxcUtjlrWJ7uNm0xCcI1/d/d+lANS5kmicQbcK1iZRIskCPIW1j8G2bo9Fd3L0hE18WZ", - "vv5YUBF308VaEb2S17pxK2FrTnC7vCsmxuxnabxqeN1n0rmTQaRUpc6j9J0m4Q7Lh/bzhEkNcrgiG28c", - "Majx9GgYeAHXFcaLyntAVogXuYc3YVo7Yd2h74c0YeaNOW5lvczL5JKZeEr7OTh0ZJmG1WPrvXB0nLg6", - "ybG7UqqOItNYSJVA+PnMrDPWd8Z+2fc5HnbuA7UzoWKl2CpraIgxU4tOt2QO3pTaH/Go3aYaYhq9pd3G", - "A4JIvpfbhBAtn2IfTfsWujAxi0ouOGckuu4GODF95DSjYkuS8hFBHYYtbCdtenKhxsPEdBIIN0kjN69z", - "qi9jCF/GIpYD+htW8bsGlMPCypTYuX/IThByIgvvl3f1wiAKovhyyeLRRwxHBWbQ2KrOHAAAHczdFCK/", - "GQvbJxSi55Fw2/zwrWNJ+JIYqi87Oda1Xr0zBe+IGI+ENKfuT3c5xChs5ev2PewbsVbX72SIXItRN4Y2", - "3oMwcLATpnKuQwZm60R0dat+pCJQ88tujGmga6pHp/Rjx/SyXl7ilUfPSqLxiEWZZUcyz7m5vQsC+rTT", - "iR+u3soFFk9m2MLorIOtPq2q93F90TGIcgl6EC14TpOVlZ3raXG5tA80XAM+vXo+teLeaoaRO7ncm5oa", - "7NUhjMbotTArZnhSq/YF5Q1X9IqNCRdJVgLlYcVJKlJyRRWXpQ7ZVOi3mpLDKnaY0zV0gElU7m7nT5UK", - "OyZ+Yp+jxZwMFyWL3VWDb/yN7poZn6HlyqkaQrGQDZGidZMroD9RzJRKsBRjw9Vh/3AXu6u2saKa5FYv", - "BVAFdzueUsT4KddEFvQfJQthZn+hmZEEzC1/RX+opeii1bUQqd0CzNAGjQwC83DWzyjOrvC2WxDCkJK2", - "qEXOAtyPECp2kyjeIY7Ff6Gv2g3ghdSaQ8xtUV9ps9iIXbe/2l8qBIFZUSs/Fuya5FyUFlywuZblsRRB", - "4rfe5wDg/eMe2niJny8q4EqNwk4iKH1tMbyzLqGZh5SDNO7lgiu4hhhjqWNSioxpTdayxPkoljAeQGnk", - "JRMYlqaCMIjDOinWUykzx/K8x4blR7IUJnYDfbtN98ZgXc613W77DlDOzR62Ay3LUGMJqMvfwue33y9w", - "igUamH+KKOR16JSAd9huEsJaswwOKmtXwqF9j7GbuZ+UJiVeng7Yi+C13fityNjC4HXA0MDX+UtLcCZp", - "pjjN+G9VLbkwUV7d0EieMA74P2cJtVYHD8UmklUpLuGm7+qtcdmXoSgGNHparcfdxSEk4mV7TbiQcPn9", - "rVbisxtkloIyRQW5ej59/iNJpa+ZVRsDcd9yfbgRudS15K0Ypjxj2vAcChM8a9TttoSbZXhl05Qcgdsm", - "pL/YcRUDRtrXN178DDxCuR/sI03MwAu4W9QbM98V0i419YslKzbyna4l39TthSqJpHPv43ztPGuQxpNa", + "/+fk6Z+fPHm/P/nx4t+fzGZT+OvZ0z8//Wf49e9Pnz558v7nNz+dn7y+4E//+V6U+SX++ueT9+z1xfB+", + "nj7987+Bf7DyWU4sN5Rq4tblXYM5y6Va3xkob6AbDxfs9NsGTYwZ6iqQ1lLtvAe5wbq8fr5Z5CQZ1RES", + "ObKPfYehJ3joeJX3WBaWwWir05IrmZU5NONRqan5b+zOe33GfwsrtR0GG7x3Ht/KhtfVIQBVvxr9aYNU", + "dtsPDSt5XHxMLCikNkvF9D8y+0Pn6TzuZNdMnYHXW8d1q3fNBlEjCV4TF4vxflLwl+GrqNfwqk+YtkSp", + "W6RvfpN2WYWeeh34uRTcSNyR9uBvwrvAY6onm+mraoj6RRyebyKt2kClpN0XOTp1FkD7+/s3AgaJU2+a", + "NQWj84V6hlGtYhrjRjyPsyOea3CqVEDRqHu6wcchxsYFaIBT/wo/Hs8E+DCskAY7ar5GjSdEC0EnOreP", + "uCZUEJoVK+r8v1SkXo44/5rD6Jl4tRY054mHwmHmHSJkwSj4Z5fUsKpz7NCOkuelsSb0lBwbcCJLka3B", + "0cvQaRymBq7dHr/RaX2ZRLEFU0zY3ZDC0omxglGQE5meWaA0WuvuDmzwhABO5dQkqwZeNoYpZDqNAJ/I", + "hQU/s9MIDss6LOyOABhyegkOJmoqLKJXlGcWUDPBheYpI7S2a3FshVhJDFjwokFbyUpqJgDg1EdZPMEE", + "cKYoTlADZHlh1qh+r83KYkKI4EAr231O09rMx0SaFVPXXLOZgG12ameZmVooDsa+2ViGTbrRydKSOpZ4", + "JjktJpdsreu9dFu5bnJa2E5Ru+3PS9haoH8jymk71wF0fHw4dxGpnH60JgihuSwFbGQi86I0lUURMiLi", + "AblNUf2GYNnLqaBLNgn9TirmsDeKoIIPF/7e9+3Uh01bO4fG48ad8ySHRB864prInBvnaanzojHh4CSn", + "ZQZxWuKQhi+Qo3FN2EdrSXKTrUllyM9E4A5gXAtrQmZgscDmT7xog+jztJpKglFg9jFhLHWjfVlEG+bH", + "Kahl8DEnIojihs9eG1nUXQrxQJ2SH9eR/uzj4IKDHw1n0JTU7Xcr4wsr/hSnhs1E5AP0qMyZbZhxt+O2", + "8yW/YsKpoVNyOBOJzHMM7pKEOvtIM1N5VoKsq0UiQYFgH12uBCadeEdq8GolfdHtYZ4sXNWNjiz2sZA6", + "5mqD583OsO0Nmi93DvRTKpYxtfH4pP7eD+DjZscn3tWu8P2To+NXp3bvYLSnM2EJxbJWDzYrgpv7a0DR", + "4JoIWddE+1WpxpRqmRt2NjRNFdPazlSQxlwION7MSpYGog4mp/pyg4+1ym7r+lx93sxGv6sDv/16DHrj", + "nFUJN1IRj1A186/Wb3g7xCl7O+cdYsnX9t01ZrFz3e1cd1/PdXez1waRteW0yaVYSrvwFUWB5wSf898s", + "57IUCVMDKVmvqEqj/o0z9yb41fzvZrYBOTl78+rlxJovPbII89v6JBK+rfPV/sGIxsZOhHbTmYfzpbqK", + "V01ja7bUssHC+BfRuNUNWQreU8IXTRhU2TtRtQfa6Z4N1I1kuVpWDH50t+U29rce+3e9X8T0wGZaDYTy", + "olk1hppS35wPCM0ai5RzQJOtUgITw6/YWZ8v/bD+uu0AR2VVhIDxE3ChgtvmaTQ4KAUaXjpKEu6dtx9a", + "S6o+DqHq7tp6FJnQedV3ygzlGYpHKRihVs+twnelUpBs6uEIKuvhyTHxArcLyYxqc66o0DDSOY85jrpt", + "gqJHtcF8OZdW5yZsQmurbSuZw2wRRdA4Altp6rxpLjF5Dnlw6LOpxU6rbpOV1enSKbEaojfGrMS/FPJa", + "gK5olXfvp4aJhR4tHFB9d92AxQLhdvDf1WkrpYZB0D+eFKt1FO3cC7SEVmVOBVGMppC9FN6JFKwSsQyb", + "SedW6YQJB7B5yOQUfIlUoLvKJTDbueb04y9MLM1qdPDdi//1p/+ITNRj4U9MsL6U2W6bNmuf+iTg6bJq", + "E3Jnq825php8nha5U1IWsIi/SIXxZ5GwsWWU0d649ribrcnzF2MydwCZIspMKzJ6//FiGpkz1+THcWtC", + "1sQvgThSsAwhMK8YkoyzzyIkw8KEpy12+6fv6+x2P670Uh0DMz6vCJlaXWGpaJ5TwxPCUyYMX3Cm6giC", + "ijF86C3WsLo/akd8dZQ5gfxkpoDZeBO4TpbrgiFOIf+1RghLTMjeBw95zqiwwtqN6Y3e8UzYt9crZikX", + "jyO4jxTMS/OUKZYSSpYlVVQYxlI4+YDRDcy5ryidVmnuHqsbvnU7S5cyDajfwvnn+y++h80IDxqa5fvD", + "yX/TyW8XT9wf+5Mf/z4+uHhW+3mBqmA3wNkjyBz38rzWA3UMrE0uyLkq2Zj8BU4WkXcCWFI9mca+H41H", + "0GA0HrkW0dBdXNP0mTo1DK+dFSBAaWQh5dQdCZomMt+rzhK0eMbzPzVV8fcIlosn7yfur2f+0dM/gwq9", + "qcHTZ3ugfgfwXryfVKCeWkW89u7pv93oHY/IpYrzBjoLu7UhJti217dJ9glyvJvtA2qEz/UhsVSf+Jk1", + "4PkRNckJg0LJK54yTRZllpEmzpWFNorRPKguFBhJRrkghn000RFXUpt4POiv7o1frG9ZS0b3Azn/hLIm", + "eV2rHSAU31RCkX00itZPAddEX8dPuJ0YexsVCRip1HDUyXL5msgJOxu4XEQx6zD/LsMvpIpEqE6kMlUS", + "oTJDQDogMdhqE+uYrUTTddeBA61JIeNmQ6z3ROY5EylLAyHEBuu28mPXeujNj0Mfjnft2eeCsRS0wuoc", + "FIpnrkMvc7aQyr5eKpp62dhJqqt1anWwDCHgNJXI5KabElz6M1aMNDSre8oGg7hPtjirKFgqDUnTRxnD", + "vPYttH7Zc5Ao2mzY+UaXx/x1TzmSezzkSG4440j+xY84kvs64Ui6BxxJ43wj+daPN7qs/W0POeJn0691", + "4iCqmfh0/BsS8etDSsWX3NJOO8wFk7ndeYHmPO7gafIw2N7f1Lc7ibRqtYm5BI/8qyAjGr6H/5FzsI9D", + "D8O9DS75KzKkzwqrBtSG5kVHW0Qo/1FjHpkTe8MGT5k2XPToXK+ql34SoLR2D5JEEW5Ji8gm/kQLXZnD", + "3reqGFiZ9hOSMoM2q8vugQMbmVzqqLMVufwpHAWh84zFPVy/RFpVPi6QE87LRY3X3AJVwQTcYZPBkAXc", + "iysCYWSPluF6EGoGEBXA9eL2uoG/ImUAccHVIZhn56QIAqjuCvWxYIx5co2urza/qHGmnf7woPpDcDYP", + "ugInrj1GrOqdWvJF1JIBVHzkd/HIp/zYfuIJopGcS2dhdjmpOytUv/KnadkoJ6Y2eNQGBDj7VhORFRW+", + "EsUyEIYAthqSd+KbLgnqtgQQAW6EGAaDt5Gafd/QrfyIN4G9fg0Qzr13G2LLbbdVDOQ3zbpbVoXdSRi7", + "s0eCwR0b7/CWoOqCI3/w4WBvr9RMHeARhP/zfH9/Wvv/wQ/f163v+hFYra+lSpudKinNqOf4hN/Hm1oP", + "wONBUvXe5OlOkD5yQboToY9ZhJ5ET4b3nAZviZ7WZTtUZZxp88pp+xUnebH/4rvJ8xeT756fv/ju4Icf", + "D3748b8HWw9x28mFDttWU8GNAgOpZT/RhfH77w7NWxPV0EsmNphSzdP6nZlho3td7oANO3XW100M1rUb", + "5td0Jt3OsblzbP7+HJuOUrb2bLrvprFrMe52MQyS4+Yrk771q2B2N7fsbm55RDe3bBUTqHOJehigtqE3", + "42GNS9xjKMAzs1vEAnr5WSMYsHXi4FB/cG3mjbMsYbotrngfIWI35iCLtdb2fhzBXunaKVyP24D1GvfO", + "jn2Mduzrniu3mu9vMIMw2X9n/uzMn9+R+YOUAWYPgt3+hSfEWzfUTftqmDjcb7LWLY6Rdu/IA61PGyrS", + "6gaW6h7p1rz0lJzy5coQIa8JN3/UeCNJ8TEBGoDTLlPyV3nNrtxhdxfQLvSYFEtoRMUa77og1bmRzYpb", + "b/rtTSqaA/g2qtnrPvj7izrqOxC9gcgqUKpsUEd1zYdnVNodHWhcS1hJxj4jdNNdDd2kEeirUpTqybFO", + "V+qdwTQAhLxuvfJb2vp2XD3Ao4oWl6TMNOE5Fh4xq4imq7jhCc3iYUH48q9Ur6JYDm9PnAV7t8Dghnsl", + "d+D+AuAOtzX0XkSy24WH34XuA7uU3bY8rm2JNfHZ6u8ghz0i6982GzSt52ZOeLgrFhPi2bS680wzgwLf", + "nUr+4O6XnRZMJVJQOBXkPgt3zk6M/EBApwvpfE4udrfAXSd7klFxas3Fzv0bjfeoRYUbuLySXmvkFVV/", + "y51XcDpr3OaaMwcnN67Z/jqdQdWY4J+ZOH/76u0BOUxTpzOVmi3KDM+x6SmpTKUxsSrrmJQ8/fMAZ03r", + "noKcFv5SL2pkzpObfErFisYug3H4dWLftg97wie9WNaTyGi10EMz3A9mqFoy02s+ntdfexvVHwQxklyv", + "uLu9LkywOlbopppOh8URfQ+1yXTByETKxbJFnk31fgtKjp9/uhnbd3T3mOjuEeFw25Lss7gqSyvuSnYy", + "nQtCyeV/6A23gW/nVsZxN7uTqzZ3cyN7E3jnr3qc3mPnmNx5jR+T1/i1UjIST4XHcGGCFHBkvXUpcq/m", + "ERvj58BPXQDhWCzkxvxQHxGyUIzcMQwvz+MJruGadbgBHcqhblMJtXlVOt5SHK4drtxE7vyJZ5MzUa/G", + "+X60LF6MxqNl8d3oYptiqfWZs+EEdlb7LFqpp3EPUA16MVhdDNnA0/773SK7WOclPV67SL52Ub7hWcbr", + "kMNjt/WU5dHBqMQD2p/Ho5TryzN3gnfYF3hd2cu1YYOHGZJAHcBzGNb3eTxKaEETbtb/oms98svrYJx/", + "Ma7tdwzNqkvQj90tLM6z7m6n20QD3W9fUs3+i5sVJBBE7q2rXQnvvmgVKe+4uLF4rUvxv4hO+GXU6rp5", + "rC9XED3vzmWrgs7tcr9FnndzTIbXFnblgJuXudy2s1YN4ZbzH195P0lVC/T8l7O9s7NfCHzt75kdRasO", + "D0DZBtrdEX3hAsYh9te3UaS6yPNJDefuZ8/voTJ+d2NvwS0GoAae2q1Vib8Xzjbe9vOTN28GrtDVxL07", + "W7RDdqSe5Rydh7TgrtB4rbh5wS+hePf9YEz8LFB4egdepvG8XW3mac7FrXscIn5P3rzpgvusYMlQfgXV", + "w+4JKR8UGdHaaiBjdEHaexsG6c4RnSMi9IIk7vR9o7x8e/zq6Kjnnu/X6J4nto2//UndWO/JmqbHEXsZ", + "eoFrzt3V567pq6gJr3XJ1LvTX3r6CbNB2u4aWYksYuo/fOxeDlcrOjaKW2N9nmHMmOp45m+Z7YXxMpNz", + "mvVfRyt5mlT7tAllajvannitk+gs64ZaXxUwrNzVLHZQWPJkhintKxZAo5q9ah+jY/Ek1NH6G5TR0iEI", + "h3VuaJatcXvRwwWFbc4alqe7TVtMgnBN/3e3PlTDUqZJIvEGXKtYmQQL5AjyFhb/hhk6/dXdCxLRdXGm", + "rz8WVMTddLFWRK/ktW7cStiaE9wu74qJMftZGq8aXveZdO5kEClVqfMo/VGTcIflQ/t5wqQGOVyRjTeO", + "GNR4ejQMvIDrCuNF5T0gK8SL3MObMK2dsO7Q90OaMPPGHLeyXuZlcslMPKX9HBw6skzD6rH1Xjg6Tlyd", + "5NhdKVVHkWkspEog/Hxm1hnrO2O/7PscDzv3gdqZULFSbJU1NMSYqUWnWzIHb0rtj3jUblMNMY3e0m7j", + "AUEk38ttQoiWT7GPpn0LXZiYRSUXnDMSXXcDnJg+cppRsSVJ+YigDsMWtpM2PblQ42FiOgmEm6SRm9c5", + "1ZcxhC9jEcsB/Q2r+F0DymFhZUrs3D9kJwg5kYX3y7t6YRAFUXy5ZPHoI4ajAjNobFVnDgCADuZuCpHf", + "jIXtEwrR80i4bX741rEkfEkM1ZedHOtar96ZgndEjEdCmlP3p7scYhS28nX7HvaNWKvrdzJErsWoG0Mb", + "70EYONgJUznXIQOzdSK6ulU/UhGo+WU3xjTQNdWjU/qxY3pZLy/xyqNnJdF4xKLMsiOZ59zc3gUBfdrp", + "xA9Xb+UCiyczbGF01sFWn1bV+7i+6BhEuQQ9iBY8p8nKys71tLhc2gcargGfXj2fWnFvNcPInVzuTU0N", + "9uoQRmP0WpgVMzypVfuC8oYresXGhIskK4HysOIkFSm5oorLUodsKvRbTclhFTvM6Ro6wCQqd7fzp0qF", + "HRM/sc/RYk6Gi5LF7qrBN/5Gd82Mz9By5VQNoVjIhkjRuskV0J8oZkolWIqx4eqwf7iL3VXbWFFNcquX", + "AqiCux1PKWL8lGsiC/qPkoUws7/QzEgC5pa/oj/UUnTR6lqI1G4BZmiDRgaBeTjrZxRnV3jbLQhhSElb", + "1CJnAe5HCBW7SRTvEMfiv9BX7QbwQmrNIea2qK+0WWzErttf7S8VgsCsqJUfC3ZNci5KCy7YXMvyWIog", + "8VvvcwDw/nEPbbzEzxcVcKVGYScRlL62GN5Zl9DMQ8pBGvdywRVcQ4yx1DEpRca0JmtZ4nwUSxgPoDTy", + "kgkMS1NBGMRhnRTrqZSZY3neY8PyI1kKE7uBvt2me2OwLufabrd9ByjnZg/bgZZlqLEE1OVv4fPb7xc4", + "xQINzD9FFPI6dErAO2w3CWGtWQYHlbUr4dC+x9jN3E9KkxIvTwfsRfDabvxWZGxh8DpgaODr/KUlOJM0", + "U5xm/LeqllyYKK9uaCRPGAf8n7OEWquDh2ITyaoUl3DTd/XWuOzLUBQDGj2t1uPu4hAS8bK9JlxIuPz+", + "Vivx2Q0yS0GZooJcPZ8+/4Gk0tfMqo2BuG+5PtyIXOpa8lYMU54xbXgOhQmeNep2W8LNMryyaUqOwG0T", + "0l/suIoBI+3rGy9+Bh6h3A/2kSZm4AXcLeqNme8KaZea+sWSFRv5o64l39TthSqJpHPv43ztPGuQxpNa", "6zjnwt346dgbUnaoWfI34Ae+prNxWXc0cOJal3BOBzgUKUUuUygwCSavZy448yk5kUWJ18/MsRqtXmvD", "8imxqiMUeXpwH0UiBdp9yXriSiJOqEgngZ0n66jzkGWLX7iIKMz+Deb9vDv9pZ3uE/Zl0PpnYiZevT45", "fX10eP76FanyGJDKoFKlleJ0STt1HgV5Pn2xbzGYWWW9yW64BiNOoNSEu9xzecX8Z8/9ZwPT+AapS3hO", @@ -7308,62 +7426,63 @@ var swaggerSpec = []string{ "coWdUcPSaoifuUi/duqv6A1rQNzuzvAhT64riwbZDhfLzHWPNqKPNTq/Tfq0h3MbtT5cGKbOWCJFrCLm", "8aJWX726Vp8LovETXxem8hy7mKA7SYG+iHRKzuyOOvUFs7/Re1LP9Ab+Y+gllNujGVgEhhEKlg2ZON+t", "1KEj05Reoc+VvCaZxDDoNeUmzJJe+nz1dvfTYfVjSh5B/nfHr9q7Oe3dprDffVvVxt94QlipmZosS56y", - "vap2nf5DyWNYeUcxuEH++VNOplTCCWy7SwnNsiA8xHfGt0CPlvc+7c6IPPQZkUTGzrmelcslcs6/np+f", - "+L2xbatiXMh5xmSf8FDweyCN1Eqs3ZMMrOlhu4Mq93xQ5Q4WRf1INDi0We89aM30jjujRQha3MkAuV6t", - "WzN3+TJ2cbPRX1APnI3cQu9gmZBDr6knGVXo/6ICyc9BEchvXlqGydDNKa+YUlbL5Ga6TZ3Ms8hJdY6K", - "ldU6DshsdFZC3oi1RVV9pQ+OjlabAOdUs2zkDScbNUtKxc0a7slCUfGSUcXUYYmXCQDyQCgTHlfd2jWM", - "Pts+ePS40x+I7QIDB/bRTBxmWZ2CiY8+Hp4c+0L65IP9SCrn/TggOBkyK/f3v08gdgB/sg9kBYYzKnSU", - "gInjggtcYOHICRSOnAks0orvnFIg585bP1+7+Ie/SyAxmWtqOZb54JQJ+OFL5Nm34IZR3Bp3PESQdKIY", - "Ey6Qzw3kyJ7g3QRhtUiNtWDjwej5dH+6745sC1rw0cHo++n+9IW7qBR2Zc9F0yce2ktmenIRLDyXfrb+", - "kBIYlN7J18gjY/qGg18Bz4/T0cHoJ2biB97AXw0GNEz4xf6+DxsyDNrUcjf3/scxFgeNGzhXfEBAvrb8", - "BepblFlFnRawP9zjZPBIYWTwd0L3DP/jlxj+2GtQzvHBXMPxSJd5TtV6dDA6ah48NHSpRwfvawfgRhf2", - "gz3RyGvbjGqo5GgI8Xby23Iq6BLpzBFADKes0Kml0j0gJjUznQdjUAOIb9yaRH3GHpRYpRUdTHBa4OPE", - "cZaJV418bnvt+ybM9z6Fvz/vYTbgxJHsgP1wKQFZ1kokBKugC/dGTiWkQVY5kQfvY+lDMDEvBLvJilBP", - "gJqVPzJRW2jjeAVmSFbb1hZXFw+IBs1Fb4cLO27iCQHyWVtIViMFl2vroAzEUEi9CXXxQI5lJYJdt3oG", - "Qf7smQ8nPHsGAYUPHz7Yfz7Z/xAyC7rwbHTgH1ZRB6uf6e89Kc1G42YDQFFs5Ug2NPk89gNYNavVuUVc", - "33mj0yobF1/j7+eNNiHNGJvgz79f2t+1ViFD1o0DPzutMMXWraCcJEwYRbPJ89movorPAW63AiD9rVTs", - "AWEI/W8EY8hX3ghJN8O/0wSieX/HFWyAaat9HbhtwHUYKR59a3CVx8ZJQe9+KbFG173wjsiiXU5+hJ+c", - "d1YYEhAgwOxrgbbX9flLSYGdALiFOokHKDuYu0EC9KtDbUVnuE6E7z6jYMmYYRtEDDbQEYqr/PE+gvjB", - "dvuhqza9gj62pvZtCX0rGh8/Kk3th5hrdEdLm2gJkWorWhroAoihecI7eO5t/yW/YgKfASpECOAnZnbY", - "/8XtlJ2E2p6qfmJmK5IqqElWG4gKQw1biQ/yVmQuIyW0cDkgPlfEBzAimmXk4OOO2u5fl+0/XzpMl4UN", - "0dvs9U7T/Zb4COLHl9d0/WG6iXf/u3JqcPhsG2dKT1l6L/QrdK0L/l5LN147fQu+VCf+x84begrFx/lC", - "H5y/urE7eBV9rODF/vMvP5kjd5bAMQicx4svP49DVwFvxxM71n8PxneY4w1MsZfT3YI73tYh0Ee8Paod", - "xFNv4Jdo1j1Ofjne5nC4gwXk3VgetpClSF1C8RvnNH7vHcUXoQZmbOE+Weyh1NHjBdHMjN2hlaCQspSU", - "BZ4chWyjlnb6j5KpdTWNJGNUlEVb8+5Mo7py4iENwS1zCnca3m39L1txs4EOmAdgKz8xs+MpD8hTLh6z", - "JrYj2cq585i0D1/g9e7Gma8SfS/Wma9n+vswz0L11oH2mQf1YzPQNqzjK1hoG2bzZU20DRPZ2WjDbTQV", - "eIJnkx6wW/LJwPNuwyjvzU7zRHzfhtpjYZ3baVW+JP2d1KrTBl/8FvSqnY30tWykzdzktlbSPRB110za", - "UfS3ayndQiXaUe4GU2kz2RalGRgIfwjKxYDbjni/APF+GyaZi5vvTLLtTbJFme14YSeW/7hsoq0O9rSn", - "rruOota1wt1zPy1s0o/DPfRlCHl34OcOB346yFcjmFAp3QF6+0M/HarcDrOjDtDfiedzsHx9bK7ORyJQ", - "h0nSbP3AHs6da/NOrs2buNFwOb6d/N775MU/+DT3aol6txXrLpaltw4DReT7Szedb8p0upvJtNlWqu/W", - "4w4N77SVe9RWPE19jQBxh0fUA8a3ZhK+E1c6q/P+Dk6YCB859VPeMZJviJG4XdtxkvvkJKoiha/hMLi3", - "4Ol9B013rGGXyroL0z6+MO1NltFt47T3Gp/dMY9vIRK7o8r7CcHe6DodFIO9X6U/GnndkeUjj7Hezvn7", - "CIKqO1ZybxHMr+f6RHdGtcwtLij1dQurj3sTKe5V0TiqJrvjbd+AylHbrx3HuJ/8r6ROAl+XczRr+A69", - "27j6qrf68D0zjdo8d1zjW+AaYcN2XOO+uEaDBu6JbUzqvd6GgxTcqC1Yx4nkwky4mJzzHErUyisGZeIW", - "8guxkhM74R0P+QZ4COzUjnvcinvcQGtfWu9gYsnFLeOt7ts7JWO8duP/HnItca27kON9hBxZwJsOuSCY", - "h1KL72gLYtkri6WiKZsUGRVDKadgAkrtInClIq4T3bxltJ7LOROHoT5nth4TbgjNtIzUl/Cdu/J1WGYc", - "SqIJFioWF0wtpMpZSmbCFamzcpourCzC2WBt/gBkP1c/Fyx3fPV8+ny6P3ZF0i33ynOsK24kCfXR7cqt", - "3tBZ7xSrz2DhcvfQtsaqvCkrFEuwWraoaqPXqi1fPZ++mO7HNYp32N2J3Zd/ZY5SX+eOldxKDnvMKxBX", - "PBd569BVfyn+sUeLQskrmg24IyOwjIgYDoR2w9GHb4CQDwEi7NER80NcshqWeOjRIILTpzg0bEPFqBsW", - "SRsJhgYwdoxjuzADYvkmsH9RTlJlPG2bq+Bmfj8WvFO5vg3jnfnJfitWt4PuTtDfzV0X9n2TxXCLM953", - "p6RmgsHvnJgeLjGgn44ed17Ajv7vKy1gEAu4H1GdS8GNtIg94UIbKpLtvGzV9yR8b7Vm2nEURP1rb8Ln", - "x2H030clw8jKdy63O7jcYohYo6AK3NsfbY50jRZq7E1VOh2wTJMPFqs+OP6smZnOxEuqWUok2r/+/cpa", - "nVY6G37FyCVbY4nkRIoFX5auGLdgLNWNvs7KZEWoHhO+wK4OSJHnH8a2Q0E+2L+hs/qX1oTj1oB2RZgb", - "Y/Sfzu6i7L9+rbzumhEWm4uMvOnHi693eDuyfTtmc9vTyxHK7+c2/aI6Kn63FNe3PU8UY15bltK7HUfw", - "zCAOwy9T6OjNNmP/virr/YBzfdjhYxxSSIMpCo/xUE4LWQXdRPADvVx3osCfmLkb+b35PZHfTozuaDvu", - "eNtKkm9TZvBO1I0ugZ18/draPu7DZm0/v0nb/yqlA3d86l+HTzkH4UMbHQVTOdeaSzHABxhL7wmfh1zc", - "UjOFKT5ck6RUigmTrUkml0sIr4Mj5dnrjzQvMnbwbCYOtS5zPDC/kFkmr+1qT18eHpFCZjxZjyFSYbvV", - "5APNeOJjF3M5/3AwE1DnvxgTJTN2kLKrceWC1GOiGE3H5FmrRdthOibPxuTZXm8zn7rYaDeX841NlmMC", - "0616dJO1LMQCFHIPEKqt5bcB69btV/tpJgiZjWqtZqMD8t4+Jf4f+7/ZCL6bjcb1ZxV4Wi8srFqPns1G", - "+PNiPLD3Nmi7HTZ/791hCA/zLcaw/1zMxGcHyUOR3gT6OpoNB/xczh9u1tEUM83USY2cHzLLqzXUzql0", - "u0wvyymLxpZ5zn5YmhUTxk2MzMr9/Rd/JPapVPw3XM6F7XHPs/qtTmPRgibcrDHN8oryjM4zcEZjV16l", - "+7mcMyXAfeQPFsRxr2pY3WLkZvWAaLhh1B1Gbu/mrO5KClvn0bGCtMM6zQBlB6QGcq3LEFf5z/86J0Ze", - "MgGc1aoEqNng1TgW5V6jBmM/dWl8IXQB5JKX2pAVdRfsfMjkkosPgNBznnGz7o9lnLkpP1DCnG4eOewx", - "HGANzWNZ92sfFMqu3XD8GmAdNef8E7RadvQymF5YUipu1qOD9xd16vF4++6Y/GJx8la8XDNjuFhuoYpD", - "Oq37ynNtPxXQ9LMMQ3wxrn3mh3tAHh3GGIxhG4Bcm7AH7k9MMEUzPKKEULxiyvOm4UB0H7VhaJshDsQY", - "y9/wo2M8HvVgMHTDbAfCADT/dT/MmhD/NHrJqGLKIqjdgM92CwAE6H0qVTY6GO1dPR/ZN67PNowt/NZm", - "Zbm7YhkkWxvZ1ilq99k7O7UmZ7oep/4+2wfSaj12zqrdqt/qMFi7W58CdYfZktpFna77UDDiLt1W9wi7", - "Xn350i06fdmO3je6Iv6GuqFdVn6IqquaE2NoN7TJUUGLbbDT0PkQ3tsdtU4gKneDzGVpevlrNWKDuO6A", - "bORtLXXb9V09+nzx+f8HAAD//5OT678uSQEA", + "vap2nf5DyWNYeUcxuEH++VNOplTCCWy7SwnNsiA8xB+Nb4EeLe992p0ReegzIomMnXM9K5dL5Jx/PT8/", + "8Xtj21bFuJDzjMk+4aHg90AaqZVYuycZWNPDdgdV7vmgyh0sivqRaHBos9570JrpHXdGixC0uJMBcr1a", + "t2bu8mXs4majv6AeOBu5hd7BMiGHXlNPMqrQ/0UFkp+DIpDfvLQMk6GbU14xpayWyc10mzqZZ5GT6hwV", + "K6t1HJDZ6KyEvBFri6r6Sh8cHa02Ac6pZtnIG042apaUips13JOFouIlo4qpwxIvEwDkgVAmPK66tWsY", + "fbZ98Ohxpz8Q2wUGDuyjmTjMsjoFEx99PDw59oX0yQf7kVTO+3FAcDJkVu7vf5dA7AD+ZB/ICgxnVOgo", + "ARPHBRe4wMKREygcORNYpBXfOaVAzp23fr528Q9/l0BiMtfUcizzwSkT8MOXyLNvwQ2juDXueIgg6UQx", + "JlwgnxvIkT3BuwnCapEaa8HGg9Hz6f503x3ZFrTgo4PRd9P96Qt3USnsyp6Lpk88tJfM9OQiWHgu/Wz9", + "ISUwKL2Tr5FHxvQNB78Cnh+no4PRT8zED7yBvxoMaJjwi/19HzZkGLSp5W7u/Y9jLA4aN3Cu+ICAfG35", + "C9S3KLOKOi1gv7/HyeCRwsjg74TuGf6HLzH8sdegnOODuYbjkS7znKr16GB01Dx4aOhSjw7e1w7AjS7s", + "B3uikde2GdVQydEQ4u3kt+VU0CXSmSOAGE5ZoVNLpXtATGpmOg/GoAYQ37g1ifqMPSixSis6mOC0wMeJ", + "4ywTrxr53Pba902Y730Kf3/ew2zAiSPZAfvhUgKyrJVICFZBF+6NnEpIg6xyIg/ex9KHYGJeCHaTFaGe", + "ADUrf2SittDG8QrMkKy2rS2uLh4QDZqL3g4XdtzEEwLks7aQrEYKLtfWQRmIoZB6E+rigRzLSgS7bvUM", + "gvzZMx9OePYMAgofPnyw/3yy/yFkFnTh2ejAP6yiDlY/0995UpqNxs0GgKLYypFsaPJ57Aewalarc4u4", + "vvNGp1U2Lr7G388bbUKaMTbBn3+/tL9rrUKGrBsHfnZaYYqtW0E5SZgwimaT57NRfRWfA9xuBUD6W6nY", + "A8IQ+t8IxpCvvBGSboZ/pwlE8/6OK9gA01b7OnDbgOswUjz61uAqj42Tgt79UmKNrnvhHZFFu5z8CD85", + "76wwJCBAgNnXAm2v6/OXkgI7AXALdRIPUHYwd4ME6FeH2orOcJ0I331GwZIxwzaIGGygIxRX+eN9BPGD", + "7fZDV216BX1sTe3bEvpWND5+VJra9zHX6I6WNtESItVWtDTQBRBD84R38Nzb/kt+xQQ+A1SIEMBPzOyw", + "/4vbKTsJtT1V/cTMViRVUJOsNhAVhhq2Eh/krchcRkpo4XJAfK6ID2BENMvIwccdtd2/Ltt/vnSYLgsb", + "orfZ652m+y3xEcSPL6/p+sN0E+/+d+XU4PDZNs6UnrL0XuhX6FoX/L2Wbrx2+hZ8qU78j5039BSKj/OF", + "Pjh/dWN38Cr6WMGL/edffjJH7iyBYxA4jxdffh6HrgLejid2rP8ejO8wxxuYYi+nuwV3vK1DoI94e1Q7", + "iKfewC/RrHuc/HK8zeFwBwvIu7E8bCFLkbqE4jfOafzeO4ovQg3M2MJ9sthDqaPHC6KZGbtDK0EhZSkp", + "Czw5CtlGLe30HyVT62oaScaoKIu25t2ZRnXlxEMaglvmFO40vNv6X7biZgMdMA/AVn5iZsdTHpCnXDxm", + "TWxHspVz5zFpH77A692NM18l+l6sM1/P9PdhnoXqrQPtMw/qx2agbVjHV7DQNszmy5poGyays9GG22gq", + "8ATPJj1gt+STgefdhlHem53mifi+DbXHwjq306p8Sfo7qVWnDb74LehVOxvpa9lIm7nJba2keyDqrpm0", + "o+hv11K6hUq0o9wNptJmsi1KMzAQ/hCUiwG3HfF+AeL9NkwyFzffmWTbm2SLMtvxwk4s/3HZRFsd7GlP", + "XXcdRa1rhbvnflrYpB+He+jLEPLuwM8dDvx0kK9GMKFSugP09od+OlS5HWZHHaC/E8/nYPn62Fydj0Sg", + "DpOk2fqBPZw71+adXJs3caPhcnw7+b33yYt/8Gnu1RL1bivWXSxLbx0Gisj3l24635TpdDeTabOtVN+t", + "xx0a3mkr96iteJr6GgHiDo+oB4xvzSR8J650Vuf9HZwwET5y6qe8YyTfECNxu7bjJPfJSVRFCl/DYXBv", + "wdP7DpruWMMulXUXpn18YdqbLKPbxmnvNT67Yx7fQiR2R5X3E4K90XU6KAZ7v0p/NPK6I8tHHmO9nfP3", + "EQRVd6zk3iKYX8/1ie6MaplbXFDq6xZWH/cmUtyronFUTXbH274BlaO2XzuOcT/5X0mdBL4u52jW8B16", + "t3H1VW/14XtmGrV57rjGt8A1wobtuMZ9cY0GDdwT25jUe70NBym4UVuwjhPJhZlwMTnnOZSolVcMysQt", + "5BdiJSd2wjse8g3wENipHfe4Ffe4gda+tN7BxJKLW8Zb3bd3SsZ47cb/PeRa4lp3Icf7CDmygDcdckEw", + "D6UW39EWxLJXFktFUzYpMiqGUk7BBJTaReBKRVwnunnLaD2XcyYOQ33ObD0m3BCaaRmpL+E7d+XrsMw4", + "lEQTLFQsLphaSJWzlMyEK1Jn5TRdWFmEs8Ha/AHIfq5+Llju+Or59Pl0f+yKpFvuledYV9xIEuqj25Vb", + "vaGz3ilWn8HC5e6hbY1VeVNWKJZgtWxR1UavVVu+ej59Md2PaxTvsLsTuy//yhylvs4dK7mVHPaYVyCu", + "eC7y1qGr/lL8Y48WhZJXNBtwR0ZgGRExHAjthqMP3wAhHwJE2KMj5oe4ZDUs8dCjQQSnT3Fo2IaKUTcs", + "kjYSDA1g7BjHdmEGxPJNYP+inKTKeNo2V8HN/H4seKdyfRvGO/OT/VasbgfdnaC/m7su7Psmi+EWZ7zv", + "TknNBIPfOTE9XGJAPx097ryAHf3fV1rAIBZwP6I6l4IbaRF7woU2VCTbedmq70n43mrNtOMoiPrX3oTP", + "j8Pov49KhpGV71xud3C5xRCxRkEVuLc/2hzpGi3U2JuqdDpgmSYfLFZ9cPxZMzOdiZdUs5RItH/9+5W1", + "Oq10NvyKkUu2xhLJiRQLvixdMW7BWKobfZ2VyYpQPSZ8gV0dkCLPP4xth4J8sH9DZ/UvrQnHrQHtijA3", + "xug/nd1F2X/9WnndNSMsNhcZedOPF1/v8HZk+3bM5ranlyOU389t+kV1VPxuKa5ve54oxry2LKV3O47g", + "mUEchl+m0NGbbcb+fVXW+x7n+rDDxzikkAZTFB7joZwWsgq6ieAHernuRIE/MXM38nvzeyK/nRjd0Xbc", + "8baVJN+mzOCdqBtdAjv5+rW1fdyHzdp+fpO2/1VKB+741L8On3IOwoc2Ogqmcq41l2KADzCW3hM+D7m4", + "pWYKU3y4JkmpFBMmW5NMLpcQXgdHyrPXH2leZOzg2Uwcal3meGB+IbNMXtvVnr48PCKFzHiyHkOkwnar", + "yQea8cTHLuZy/uFgJqDOfzEmSmbsIGVX48oFqcdEMZqOybNWi7bDdEyejcmzvd5mPnWx0W4u5xubLMcE", + "plv16CZrWYgFKOQeIFRby28D1q3br/bTTBAyG9VazUYH5L19Svw/9n+zEXw3G43rzyrwtF5YWLUePZuN", + "8OfFeGDvbdB2O2z+3rvDEB7mW4xh/7mYic8OkocivQn0dTQbDvi5nD/crKMpZpqpkxo5P2SWV2uonVPp", + "dplellMWjS3znP2wNCsmjJsYmZX7+y/+ROxTqfhvuJwL2+OeZ/VbncaiBU24WWOa5RXlGZ1n4IzGrrxK", + "93M5Z0qA+8gfLIjjXtWwusXIzeoB0XDDqDuM3N7NWd2VFLbOo2MFaYd1mgHKDvJAcnFFM45W0mtUUOD5", + "f/7XOTHykon+64rO3DB3ShF78ePDA/hcSpJTsSbUGJYXRj+qrfVQf3dMfpFLWZqtOc2N8TOudRnCZ2Fn", + "QYBazQ8VWLwByXKWGh64bM0QoQKumJfakBV19yh9yOSSiw/At+Y842bdH7Kqo8wD5EXq5snSHvsQ1tA8", + "fXe/ZmCh7NoNx68B1lGr3T9B4/RbMgd/t1TLklJxsx4dvL/op2EubqUsaGYMF8stbD3I13ZfebXATwVM", + "ySzDGHJMLTjzwz2gEhDGGIzbG4Bcm7AH7k9MMEUzPAOHULxiygu/4UB0H7VhaJshDsRY2t/wo2M8f/dg", + "MHTDbAfCADT/dT/MmhD/NHrJqGLKIqjdgM92CwAE6N4sVTY6GO1dPR/ZN67PNowt/NZmZeWKYhlk8xvZ", + "VlprBROcI6SmyHRdmv19tk881nrsHIa8Vb/VacN2tz7H7g6zJbWbYF33oSLJXbqtLqp2vfr6uFt0+rKd", + "HtLoivgrEId2WTm6qq5qXrKh3dAmRwUzqcFOQ+dDeG931DqBqNwNMpel6eWv1YgN4roDspG3tbMBru/q", + "0eeLz/8/AAD//2SptQSPSwEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/docs/spec/openapi.yml b/docs/spec/openapi.yml index 3ffae329b..8e99fb316 100644 --- a/docs/spec/openapi.yml +++ b/docs/spec/openapi.yml @@ -59,6 +59,12 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + '429': + description: Too many attempts + content: + application/json: + schema: + $ref: '#/components/schemas/Error' '500': description: Internal server error content: @@ -72,6 +78,28 @@ paths: application/json: schema: $ref: '#/components/schemas/UserCredentials' + delete: + tags: + - Authentication & Authorization + summary: Everest UI Logout + description: | + This API invalidates Everest API JWT token. + operationId: deleteSession + responses: + '200': + description: Successful operation + '429': + description: Too many attempts + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' '/permissions': get: tags: diff --git a/internal/server/everest.go b/internal/server/everest.go index da7105092..6fa180432 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -27,6 +27,7 @@ import ( "net/http" "slices" + "github.com/AlekSi/pointer" "github.com/getkin/kin-openapi/openapi3filter" "github.com/golang-jwt/jwt/v5" echojwt "github.com/labstack/echo-jwt/v4" @@ -61,6 +62,7 @@ type EverestServer struct { sessionMgr *session.Manager attemptsStore *RateLimiterMemoryStore handler handlers.Handler + blacklist session.Blocklist oidcProvider *oidc.ProviderConfig } @@ -117,6 +119,7 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar kubeClient: kubeClient, sessionMgr: sessMgr, attemptsStore: store, + blacklist: session.NewBlocklist(kubeClient, l), oidcProvider: oidcProvider, } e.echo.HTTPErrorHandler = e.errorHandlerChain() @@ -202,6 +205,12 @@ func (e *EverestServer) initHTTPServer(ctx context.Context) error { } apiGroup.Use(jwtMW) + blocklistMW, err := e.blocklistMiddleWare() + if err != nil { + return err + } + apiGroup.Use(blocklistMW) + apiGroup.Use(e.checkOperatorUpgradeState) api.RegisterHandlers(apiGroup, e) @@ -363,7 +372,7 @@ func (e *EverestServer) getBodyFromContext(ctx echo.Context, into any) error { func sessionRateLimiter(limit int) (echo.MiddlewareFunc, *RateLimiterMemoryStore) { allButSession := func(c echo.Context) bool { - return c.Request().Method != echo.POST || c.Request().URL.Path != "/v1/session" + return c.Request().URL.Path != "/v1/session" } config := echomiddleware.DefaultRateLimiterConfig config.Skipper = allButSession @@ -414,3 +423,26 @@ func everestErrorHandler(next echo.HTTPErrorHandler) echo.HTTPErrorHandler { next(err, c) } } + +func (e *EverestServer) blocklistMiddleWare() (echo.MiddlewareFunc, error) { + skipper, err := newSkipperFunc() + if err != nil { + return nil, err + } + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if skipper(c) { + return next(c) + } + if allow, err := e.blacklist.Allow(c.Request().Context()); err != nil { + e.l.Error(err) + return err + } else if !allow { + return c.JSON(http.StatusUnauthorized, api.Error{ + Message: pointer.ToString("Invalid token"), + }) + } + return next(c) + } + }, nil +} diff --git a/internal/server/session.go b/internal/server/session.go index ece540f0d..8f8c5b84d 100644 --- a/internal/server/session.go +++ b/internal/server/session.go @@ -23,11 +23,13 @@ import ( "time" "github.com/AlekSi/pointer" + "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/percona/everest/api" "github.com/percona/everest/pkg/accounts" + "github.com/percona/everest/pkg/common" ) const ( @@ -66,6 +68,26 @@ func (e *EverestServer) CreateSession(ctx echo.Context) error { return ctx.JSON(http.StatusOK, map[string]string{"token": jwtToken}) } +// DeleteSession deletes session. +func (e *EverestServer) DeleteSession(ctx echo.Context) error { + e.attemptsStore.IncreaseTimeout(ctx.RealIP()) + // add token to blacklist + c := ctx.Request().Context() + token, ok := c.Value(common.UserCtxKey).(*jwt.Token) + if !ok { + return ctx.JSON(http.StatusUnauthorized, errors.New("failed to get token from context")) + } + if token != nil { + err := e.blacklist.Add(c, token) + if err != nil { + return ctx.JSON(http.StatusRequestTimeout, api.Error{ + Message: pointer.To("Incorrect username or password provided"), + }) + } + } + return ctx.JSON(http.StatusOK, nil) +} + func sessionErrToHTTPRes(ctx echo.Context, err error) error { if errors.Is(err, accounts.ErrAccountNotFound) || errors.Is(err, accounts.ErrIncorrectPassword) { diff --git a/pkg/common/constants.go b/pkg/common/constants.go index b0f0619f6..e8981a21b 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -66,6 +66,8 @@ const ( EverestAccountsSecretName = "everest-accounts" // EverestJWTSecretName is the name of the secret that holds JWT secret. EverestJWTSecretName = "everest-jwt" + // EverestBlocklistSecretName is the name of the secret that holds JWT blocklist. + EverestBlocklistSecretName = "everest-blocklist" // EverestJWTPrivateKeyFile is the path to the JWT private key. EverestJWTPrivateKeyFile = "/etc/jwt/id_rsa" // EverestJWTPublicKeyFile is the path to the JWT public key. diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go new file mode 100644 index 000000000..89a47a8ba --- /dev/null +++ b/pkg/session/blocklist.go @@ -0,0 +1,121 @@ +package session + +import ( + "context" + + "github.com/golang-jwt/jwt/v5" + "github.com/pkg/errors" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/percona/everest/pkg/common" +) + +const ( + dataKey = "list" + maxRetries = 10 +) + +type Blocklist interface { + Add(ctx context.Context, token *jwt.Token) error + Allow(ctx context.Context) (bool, error) +} + +type blocklist struct { + secretsManager SecretsManager + content Content + l *zap.SugaredLogger +} + +type SecretsManager interface { + // GetSecret returns a secret by name. + GetSecret(ctx context.Context, namespace, name string) (*corev1.Secret, error) + // CreateSecret creates a secret. + CreateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) + // UpdateSecret updates a secret. + UpdateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) +} + +type Content interface { + Add(l *zap.SugaredLogger, secret *corev1.Secret, tokenData string) (*corev1.Secret, bool) + IsIn(secret *corev1.Secret, tokenData string) bool +} + +func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { + shrunkToken, err := shrinkToken(token) + if err != nil { + return err + } + + for attempts := 0; attempts < maxRetries; attempts++ { + secret, err := b.secretsManager.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) + if err != nil { + if k8serrors.IsNotFound(err) { + _, err = b.secretsManager.CreateSecret(ctx, blockListSecretTemplate(shrunkToken)) + if err != nil { + b.l.Errorf("failed to create %s secret: %v", common.EverestBlocklistSecretName, err) + continue + } + return nil + } + } + secret, retryNeeded := b.content.Add(b.l, secret, shrunkToken) + if retryNeeded { + continue + } + if _, err := b.secretsManager.UpdateSecret(ctx, secret); err != nil { + b.l.Errorf("failed to update %s secret: %v", common.EverestBlocklistSecretName, err) + continue + } + return nil + } + return nil +} + +func blockListSecretTemplate(stringData string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: common.EverestBlocklistSecretName, + Namespace: common.SystemNamespace, + }, + StringData: map[string]string{ + dataKey: stringData, + }, + } +} + +func (b *blocklist) Allow(ctx context.Context) (bool, error) { + token, ok := ctx.Value(common.UserCtxKey).(*jwt.Token) + if !ok { + return false, errors.New("failed to get token from context") + } + + secret, err := b.secretsManager.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) + if err != nil { + if k8serrors.IsNotFound(err) { + return true, nil + } + return false, errors.Wrap(err, "failed to get secret") + } + + shrunkToken, err := shrinkToken(token) + if err != nil { + return false, errors.Wrap(err, "failed to shrink token") + } + + return !b.content.IsIn(secret, shrunkToken), nil +} + +func NewBlocklist(secretsManager SecretsManager, logger *zap.SugaredLogger) Blocklist { + return &blocklist{ + secretsManager: secretsManager, + content: newDataProcessor(), + l: logger, + } +} diff --git a/pkg/session/data_processor.go b/pkg/session/data_processor.go new file mode 100644 index 000000000..0439dad6d --- /dev/null +++ b/pkg/session/data_processor.go @@ -0,0 +1,83 @@ +package session + +import ( + "strconv" + "strings" + "sync" + "time" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" +) + +const ( + sep = "," + timestampLen = 10 +) + +type content struct { + mutex sync.Mutex +} + +func newDataProcessor() Content { + return &content{} +} + +func (d *content) Add(l *zap.SugaredLogger, secret *corev1.Secret, tokenData string) (*corev1.Secret, bool) { + if d.mutex.TryLock() { + defer d.mutex.Unlock() + return addDataToSecret(l, secret, tokenData, time.Now().UTC()), false + } + return secret, true +} + +func (d *content) IsIn(secret *corev1.Secret, tokenData string) bool { + list, ok := secret.Data[dataKey] + return ok && strings.Contains(string(list), tokenData) +} + +func addDataToSecret(l *zap.SugaredLogger, secret *corev1.Secret, newData string, thresholdDate time.Time) *corev1.Secret { + byteArr, ok := secret.Data[dataKey] + if !ok { + secret.StringData = map[string]string{ + dataKey: newData, + } + return secret + } + list := string(byteArr) + list = cleanupOld(l, list, thresholdDate) + list += sep + newData + secret.StringData = map[string]string{ + dataKey: list, + } + return secret +} + +func cleanupOld(l *zap.SugaredLogger, list string, thresholdDate time.Time) string { + // the timestamps are naturally sorted from old to new when being added to the list. + // so to clean up the old records we cut string from the beginning and check if the timestamp of each piece + // is younger than the threshold. Once we hit a younger token we stop cutting and return the rest. + for len(list) > 0 { + shrunkToken, newList, ok := strings.Cut(list, sep) + if !ok { + // the last shrunktoken ???? + } + length := len(shrunkToken) + if length < timestampLen { + l.Info("blocklist contains irregular data format") + continue + } + ts := shrunkToken[length-10 : length] + tsInt, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + l.Infof("failed to parse timestamp %v", tsInt) + continue + } + timeObj := time.Unix(tsInt, 0) + if timeObj.After(thresholdDate) { + return list + } + list = newList + } + return list +} diff --git a/pkg/session/data_processor_test.go b/pkg/session/data_processor_test.go new file mode 100644 index 000000000..515e07070 --- /dev/null +++ b/pkg/session/data_processor_test.go @@ -0,0 +1,124 @@ +package session + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + + "github.com/percona/everest/pkg/logger" +) + +func TestCleanupOld(t *testing.T) { + type tcase struct { + name string + data string + thresholdDate time.Time + expected string + } + logger := logger.MustInitLogger(true, "everest") + l := logger.Sugar() + tcases := []tcase{ + { + name: "one outdated", + data: "id123AAA1743687192,id2323BBB1743687194", + thresholdDate: time.Date(2025, 4, 3, 13, 33, 13, 0, time.UTC), + expected: "id2323BBB1743687194", + }, + { + name: "two outdated", + data: "id123AAA1743687192,id2323BBB1743687193,id2323BBB1743687194", + thresholdDate: time.Date(2025, 4, 3, 13, 33, 13, 0, time.UTC), + expected: "id2323BBB1743687194", + }, + { + name: "all outdated", + data: "id123AAA1743687191,id2323BBB1743687192,id2323BBB1743687193", + thresholdDate: time.Date(2025, 4, 3, 13, 33, 13, 0, time.UTC), + expected: "", + }, + { + name: "all fresh", + data: "id123AAA1743687194,id2323BBB1743687195,id2323BBB1743687196", + thresholdDate: time.Date(2025, 4, 3, 13, 33, 13, 0, time.UTC), + expected: "id123AAA1743687194,id2323BBB1743687195,id2323BBB1743687196", + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.expected, cleanupOld(l, tc.data, tc.thresholdDate)) + }) + } +} + +func TestAddDataToSecret(t *testing.T) { + type tcase struct { + name string + data string + secret *corev1.Secret + thresholdDate time.Time + expected *corev1.Secret + } + logger := logger.MustInitLogger(true, "everest") + l := logger.Sugar() + tcases := []tcase{ + { + name: "no data in secret", + data: "id123AAA1743687192", + secret: &corev1.Secret{}, + thresholdDate: time.Date(2025, 4, 3, 13, 33, 10, 0, time.UTC), + expected: &corev1.Secret{ + StringData: map[string]string{ + dataKey: "id123AAA1743687192", + }, + }, + }, + { + name: "nothing to delete, add newer data", + data: "id123AAA1743687192", + secret: &corev1.Secret{ + Data: map[string][]byte{ + dataKey: []byte("id123AAA1743687191"), + }, + }, + thresholdDate: time.Date(2025, 4, 3, 13, 33, 10, 0, time.UTC), + expected: &corev1.Secret{ + Data: map[string][]byte{ + dataKey: []byte("id123AAA1743687191"), + }, + StringData: map[string]string{ + dataKey: "id123AAA1743687191,id123AAA1743687192", + }, + }, + }, + { + name: "deleted old data, add newer data", + data: "id123AAA1743687194", + secret: &corev1.Secret{ + Data: map[string][]byte{ + dataKey: []byte("id123AAA1743687191,id123AAA1743687193"), + }, + }, + thresholdDate: time.Date(2025, 4, 3, 13, 33, 12, 0, time.UTC), + expected: &corev1.Secret{ + Data: map[string][]byte{ + dataKey: []byte("id123AAA1743687191,id123AAA1743687193"), + }, + StringData: map[string]string{ + dataKey: "id123AAA1743687193,id123AAA1743687194", + }, + }, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + result := addDataToSecret(l, tc.secret, tc.data, tc.thresholdDate) + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/pkg/session/jwt.go b/pkg/session/jwt.go new file mode 100644 index 000000000..e2908a9fd --- /dev/null +++ b/pkg/session/jwt.go @@ -0,0 +1,65 @@ +package session + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/golang-jwt/jwt/v5" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +var ( + errEmptyToken = errors.New("token is empty") + errExtractJti = errors.New("could not extract jti") + errExtractExp = errors.New("could not extract exp") + errUnsupportedClaim = func(claims any) error { + return errors.New(fmt.Sprintf("unsupported claims type: %T", claims)) + } +) + +type JWTContent struct { + Payload map[string]interface{} `json:"payload"` +} + +func shrinkToken(token *jwt.Token) (string, error) { + l := log.FromContext(context.Background()) + content, err := extractContent(token) + if err != nil { + l.Error(err, "could not extract content") + return "", err + } + l.Info("payload", "content", content.Payload) + jti, ok := content.Payload["jti"].(string) + if !ok { + l.Error(err, "could not extract jti") + return "", errExtractJti + } + exp, ok := content.Payload["exp"].(float64) + if !ok { + l.Error(err, "could not extract exp") + return "", errExtractExp + } + return jti + strconv.FormatFloat(exp, 'f', 0, 64), nil +} + +func extractContent(token *jwt.Token) (*JWTContent, error) { + if token == nil { + return nil, errEmptyToken + } + claimsMap := make(map[string]interface{}) + + switch claims := token.Claims.(type) { + case jwt.MapClaims: + for key, val := range claims { + claimsMap[key] = val + } + default: + return nil, errUnsupportedClaim(claims) + } + + return &JWTContent{ + Payload: claimsMap, + }, nil +} diff --git a/pkg/session/jwt_test.go b/pkg/session/jwt_test.go new file mode 100644 index 000000000..fa607e9c9 --- /dev/null +++ b/pkg/session/jwt_test.go @@ -0,0 +1,100 @@ +package session + +import ( + "testing" + + "github.com/golang-jwt/jwt/v5" + "github.com/stretchr/testify/assert" +) + +func TestShrinkToken(t *testing.T) { + type tcase struct { + name string + claims jwt.MapClaims + shrunkToken string + error error + } + tcases := []tcase{ + { + name: "valid", + claims: jwt.MapClaims{ + "jti": "9d1c1f98-a479-41e3-8939-c7cb3edefa", + "exp": float64(331743679478), + }, + shrunkToken: "9d1c1f98-a479-41e3-8939-c7cb3edefa331743679478", + error: nil, + }, + { + name: "no jti", + claims: jwt.MapClaims{ + "exp": float64(331743679478), + }, + shrunkToken: "", + error: errExtractJti, + }, + { + name: "no exp", + claims: jwt.MapClaims{ + "jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", + }, + shrunkToken: "", + error: errExtractExp, + }, + } + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + result, err := shrinkToken(jwt.NewWithClaims(jwt.SigningMethodHS256, tc.claims)) + assert.Equal(t, tc.error, err) + assert.Equal(t, tc.shrunkToken, result) + }) + } +} + +func TestExtractContent(t *testing.T) { + type tcase struct { + name string + token *jwt.Token + error error + result *JWTContent + } + tokenUnsupportedClaims := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.RegisteredClaims{}) + tcases := []tcase{ + { + name: "empty token", + token: nil, + result: nil, + error: errEmptyToken, + }, + { + name: "unsupported claims", + token: tokenUnsupportedClaims, + result: nil, + error: errUnsupportedClaim(tokenUnsupportedClaims.Claims), + }, + { + name: "valid empty payload", + token: jwt.New(jwt.SigningMethodHS256), + result: &JWTContent{ + Payload: make(map[string]interface{}), + }, + error: nil, + }, + { + name: "valid with payload", + token: jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", "exp": float64(331743679478)}), + result: &JWTContent{ + Payload: map[string]interface{}{"exp": float64(331743679478), "jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a"}, + }, + error: nil, + }, + } + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + result, err := extractContent(tc.token) + assert.Equal(t, tc.error, err) + assert.Equal(t, tc.result, result) + }) + } +} From 1d66a76ef7be16bc3931903df1b853d653deaa68 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Mon, 7 Apr 2025 19:32:03 +0500 Subject: [PATCH 02/31] changes in cleanup, renamings --- api/everest-server.gen.go | 16 ++++---- client/everest-client.gen.go | 16 ++++---- docs/spec/openapi.yml | 2 +- internal/server/session.go | 3 +- pkg/session/blocklist.go | 18 ++++----- ...data_processor.go => content_processor.go} | 36 +++++++++--------- ...ssor_test.go => content_processor_test.go} | 38 +++++++++++++++++++ pkg/session/jwt.go | 2 +- pkg/session/jwt_test.go | 26 ++++++------- 9 files changed, 96 insertions(+), 61 deletions(-) rename pkg/session/{data_processor.go => content_processor.go} (57%) rename pkg/session/{data_processor_test.go => content_processor_test.go} (74%) diff --git a/api/everest-server.gen.go b/api/everest-server.gen.go index 7123f44e8..8da6074f5 100644 --- a/api/everest-server.gen.go +++ b/api/everest-server.gen.go @@ -2816,14 +2816,14 @@ var swaggerSpec = []string{ "8OfFeGDvbdB2O2z+3rvDEB7mW4xh/7mYic8OkocivQn0dTQbDvi5nD/crKMpZpqpkxo5P2SWV2uonVPp", "dplellMWjS3znP2wNCsmjJsYmZX7+y/+ROxTqfhvuJwL2+OeZ/VbncaiBU24WWOa5RXlGZ1n4IzGrrxK", "93M5Z0qA+8gfLIjjXtWwusXIzeoB0XDDqDuM3N7NWd2VFLbOo2MFaYd1mgHKDvJAcnFFM45W0mtUUOD5", - "f/7XOTHykon+64rO3DB3ShF78ePDA/hcSpJTsSbUGJYXRj+qrfVQf3dMfpFLWZqtOc2N8TOudRnCZ2Fn", - "QYBazQ8VWLwByXKWGh64bM0QoQKumJfakBV19yh9yOSSiw/At+Y842bdH7Kqo8wD5EXq5snSHvsQ1tA8", - "fXe/ZmCh7NoNx68B1lGr3T9B4/RbMgd/t1TLklJxsx4dvL/op2EubqUsaGYMF8stbD3I13ZfebXATwVM", - "ySzDGHJMLTjzwz2gEhDGGIzbG4Bcm7AH7k9MMEUzPAOHULxiygu/4UB0H7VhaJshDsRY2t/wo2M8f/dg", - "MHTDbAfCADT/dT/MmhD/NHrJqGLKIqjdgM92CwAE6N4sVTY6GO1dPR/ZN67PNowt/NZmZeWKYhlk8xvZ", - "VlprBROcI6SmyHRdmv19tk881nrsHIa8Vb/VacN2tz7H7g6zJbWbYF33oSLJXbqtLqp2vfr6uFt0+rKd", - "HtLoivgrEId2WTm6qq5qXrKh3dAmRwUzqcFOQ+dDeG931DqBqNwNMpel6eWv1YgN4roDspG3tbMBru/q", - "0eeLz/8/AAD//2SptQSPSwEA", + "f/7XOTHykon+64rO3DB38q29+PHhAXwuJcmpWBNqDMsLox/V1nqovzsmv8ilLM3WnObG+BnXugzhs7Cz", + "IECt5ocKLN6AZDlLDQ9ctmaIUAFXzEttyIq6e5Q+ZHLJxQfgW3OecbPuD1nVUeYB8iJ182Rpj30Ia2ie", + "vrtfM7BQdu2G49cA66jV7p+gcfotmYO/W6plSam4WY8O3l/00zAXt1IWNDOGi+UWth7ka7uvvFrgpwKm", + "ZJZhDDmmFpz54R5QCQhjDMbtDUCuTdgD9ycmmKIZnoFDKF4x5YXfcCC6j9owtM0QB2Is7W/40TGev3sw", + "GLphtgNhAJr/uh9mTYh/Gr1kVDFlEdRuwGe7BQACdG+WKhsdjPauno/sG9dnG8YWfmuzsnJFsQyy+Y1s", + "K621ggnOEVJTZLouzf4+2yceaz12DkPeqt/qtGG7W59jd4fZktpNsK77UJHkLt1WF1W7Xn193C06fdlO", + "D2l0RfwViEO7rBxdVVc1L9nQbmiTo4KZ1GCnofMhvLc7ap1AVO4GmcvS9PLXasQGcd0B2cjb2tkA13f1", + "6PPF5/8fAAD//z/XWzePSwEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/client/everest-client.gen.go b/client/everest-client.gen.go index c647c617c..898ab273a 100644 --- a/client/everest-client.gen.go +++ b/client/everest-client.gen.go @@ -7475,14 +7475,14 @@ var swaggerSpec = []string{ "8OfFeGDvbdB2O2z+3rvDEB7mW4xh/7mYic8OkocivQn0dTQbDvi5nD/crKMpZpqpkxo5P2SWV2uonVPp", "dplellMWjS3znP2wNCsmjJsYmZX7+y/+ROxTqfhvuJwL2+OeZ/VbncaiBU24WWOa5RXlGZ1n4IzGrrxK", "93M5Z0qA+8gfLIjjXtWwusXIzeoB0XDDqDuM3N7NWd2VFLbOo2MFaYd1mgHKDvJAcnFFM45W0mtUUOD5", - "f/7XOTHykon+64rO3DB3ShF78ePDA/hcSpJTsSbUGJYXRj+qrfVQf3dMfpFLWZqtOc2N8TOudRnCZ2Fn", - "QYBazQ8VWLwByXKWGh64bM0QoQKumJfakBV19yh9yOSSiw/At+Y842bdH7Kqo8wD5EXq5snSHvsQ1tA8", - "fXe/ZmCh7NoNx68B1lGr3T9B4/RbMgd/t1TLklJxsx4dvL/op2EubqUsaGYMF8stbD3I13ZfebXATwVM", - "ySzDGHJMLTjzwz2gEhDGGIzbG4Bcm7AH7k9MMEUzPAOHULxiygu/4UB0H7VhaJshDsRY2t/wo2M8f/dg", - "MHTDbAfCADT/dT/MmhD/NHrJqGLKIqjdgM92CwAE6N4sVTY6GO1dPR/ZN67PNowt/NZmZeWKYhlk8xvZ", - "VlprBROcI6SmyHRdmv19tk881nrsHIa8Vb/VacN2tz7H7g6zJbWbYF33oSLJXbqtLqp2vfr6uFt0+rKd", - "HtLoivgrEId2WTm6qq5qXrKh3dAmRwUzqcFOQ+dDeG931DqBqNwNMpel6eWv1YgN4roDspG3tbMBru/q", - "0eeLz/8/AAD//2SptQSPSwEA", + "f/7XOTHykon+64rO3DB38q29+PHhAXwuJcmpWBNqDMsLox/V1nqovzsmv8ilLM3WnObG+BnXugzhs7Cz", + "IECt5ocKLN6AZDlLDQ9ctmaIUAFXzEttyIq6e5Q+ZHLJxQfgW3OecbPuD1nVUeYB8iJ182Rpj30Ia2ie", + "vrtfM7BQdu2G49cA66jV7p+gcfotmYO/W6plSam4WY8O3l/00zAXt1IWNDOGi+UWth7ka7uvvFrgpwKm", + "ZJZhDDmmFpz54R5QCQhjDMbtDUCuTdgD9ycmmKIZnoFDKF4x5YXfcCC6j9owtM0QB2Is7W/40TGev3sw", + "GLphtgNhAJr/uh9mTYh/Gr1kVDFlEdRuwGe7BQACdG+WKhsdjPauno/sG9dnG8YWfmuzsnJFsQyy+Y1s", + "K621ggnOEVJTZLouzf4+2yceaz12DkPeqt/qtGG7W59jd4fZktpNsK77UJHkLt1WF1W7Xn193C06fdlO", + "D2l0RfwViEO7rBxdVVc1L9nQbmiTo4KZ1GCnofMhvLc7ap1AVO4GmcvS9PLXasQGcd0B2cjb2tkA13f1", + "6PPF5/8fAAD//z/XWzePSwEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/docs/spec/openapi.yml b/docs/spec/openapi.yml index 8e99fb316..24c8c4aec 100644 --- a/docs/spec/openapi.yml +++ b/docs/spec/openapi.yml @@ -86,7 +86,7 @@ paths: This API invalidates Everest API JWT token. operationId: deleteSession responses: - '200': + '204': description: Successful operation '429': description: Too many attempts diff --git a/internal/server/session.go b/internal/server/session.go index 8f8c5b84d..f18d0adfa 100644 --- a/internal/server/session.go +++ b/internal/server/session.go @@ -71,7 +71,6 @@ func (e *EverestServer) CreateSession(ctx echo.Context) error { // DeleteSession deletes session. func (e *EverestServer) DeleteSession(ctx echo.Context) error { e.attemptsStore.IncreaseTimeout(ctx.RealIP()) - // add token to blacklist c := ctx.Request().Context() token, ok := c.Value(common.UserCtxKey).(*jwt.Token) if !ok { @@ -85,7 +84,7 @@ func (e *EverestServer) DeleteSession(ctx echo.Context) error { }) } } - return ctx.JSON(http.StatusOK, nil) + return ctx.NoContent(http.StatusNoContent) } func sessionErrToHTTPRes(ctx echo.Context, err error) error { diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 89a47a8ba..610ce6bdf 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -25,7 +25,7 @@ type Blocklist interface { type blocklist struct { secretsManager SecretsManager - content Content + content ContentProcessor l *zap.SugaredLogger } @@ -38,13 +38,13 @@ type SecretsManager interface { UpdateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) } -type Content interface { +type ContentProcessor interface { Add(l *zap.SugaredLogger, secret *corev1.Secret, tokenData string) (*corev1.Secret, bool) - IsIn(secret *corev1.Secret, tokenData string) bool + IsBlocked(secret *corev1.Secret, tokenData string) bool } func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { - shrunkToken, err := shrinkToken(token) + shortenedToken, err := shortenToken(token) if err != nil { return err } @@ -53,7 +53,7 @@ func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { secret, err := b.secretsManager.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) if err != nil { if k8serrors.IsNotFound(err) { - _, err = b.secretsManager.CreateSecret(ctx, blockListSecretTemplate(shrunkToken)) + _, err = b.secretsManager.CreateSecret(ctx, blockListSecretTemplate(shortenedToken)) if err != nil { b.l.Errorf("failed to create %s secret: %v", common.EverestBlocklistSecretName, err) continue @@ -61,7 +61,7 @@ func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { return nil } } - secret, retryNeeded := b.content.Add(b.l, secret, shrunkToken) + secret, retryNeeded := b.content.Add(b.l, secret, shortenedToken) if retryNeeded { continue } @@ -104,18 +104,18 @@ func (b *blocklist) Allow(ctx context.Context) (bool, error) { return false, errors.Wrap(err, "failed to get secret") } - shrunkToken, err := shrinkToken(token) + shortenedToken, err := shortenToken(token) if err != nil { return false, errors.Wrap(err, "failed to shrink token") } - return !b.content.IsIn(secret, shrunkToken), nil + return !b.content.IsBlocked(secret, shortenedToken), nil } func NewBlocklist(secretsManager SecretsManager, logger *zap.SugaredLogger) Blocklist { return &blocklist{ secretsManager: secretsManager, - content: newDataProcessor(), + content: newContentProcessor(), l: logger, } } diff --git a/pkg/session/data_processor.go b/pkg/session/content_processor.go similarity index 57% rename from pkg/session/data_processor.go rename to pkg/session/content_processor.go index 0439dad6d..9bdf5bd1b 100644 --- a/pkg/session/data_processor.go +++ b/pkg/session/content_processor.go @@ -13,25 +13,27 @@ import ( const ( sep = "," timestampLen = 10 + // expiration = time.Hour * 24 + expiration = time.Second * 30 ) -type content struct { +type contentProcessor struct { mutex sync.Mutex } -func newDataProcessor() Content { - return &content{} +func newContentProcessor() ContentProcessor { + return &contentProcessor{} } -func (d *content) Add(l *zap.SugaredLogger, secret *corev1.Secret, tokenData string) (*corev1.Secret, bool) { +func (d *contentProcessor) Add(l *zap.SugaredLogger, secret *corev1.Secret, tokenData string) (*corev1.Secret, bool) { if d.mutex.TryLock() { defer d.mutex.Unlock() - return addDataToSecret(l, secret, tokenData, time.Now().UTC()), false + return addDataToSecret(l, secret, tokenData, time.Now().Add(-expiration).UTC()), false } return secret, true } -func (d *content) IsIn(secret *corev1.Secret, tokenData string) bool { +func (d *contentProcessor) IsBlocked(secret *corev1.Secret, tokenData string) bool { list, ok := secret.Data[dataKey] return ok && strings.Contains(string(list), tokenData) } @@ -54,30 +56,26 @@ func addDataToSecret(l *zap.SugaredLogger, secret *corev1.Secret, newData string } func cleanupOld(l *zap.SugaredLogger, list string, thresholdDate time.Time) string { - // the timestamps are naturally sorted from old to new when being added to the list. - // so to clean up the old records we cut string from the beginning and check if the timestamp of each piece - // is younger than the threshold. Once we hit a younger token we stop cutting and return the rest. - for len(list) > 0 { - shrunkToken, newList, ok := strings.Cut(list, sep) - if !ok { - // the last shrunktoken ???? - } - length := len(shrunkToken) + tokens := strings.Split(list, sep) + newList := make([]string, 0, len(tokens)) + for _, shortenedToken := range tokens { + length := len(shortenedToken) if length < timestampLen { l.Info("blocklist contains irregular data format") continue } - ts := shrunkToken[length-10 : length] + ts := shortenedToken[length-10 : length] tsInt, err := strconv.ParseInt(ts, 10, 64) if err != nil { l.Infof("failed to parse timestamp %v", tsInt) continue } timeObj := time.Unix(tsInt, 0) + l.Info("time", "from token: ", timeObj, "treshold: ", thresholdDate) + // only keep the tokens which natural expiration time is not over yet if timeObj.After(thresholdDate) { - return list + newList = append(newList, shortenedToken) } - list = newList } - return list + return strings.Join(newList, sep) } diff --git a/pkg/session/data_processor_test.go b/pkg/session/content_processor_test.go similarity index 74% rename from pkg/session/data_processor_test.go rename to pkg/session/content_processor_test.go index 515e07070..2825b8161 100644 --- a/pkg/session/data_processor_test.go +++ b/pkg/session/content_processor_test.go @@ -1,10 +1,13 @@ package session import ( + "fmt" + "strings" "testing" "time" "github.com/stretchr/testify/assert" + "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "github.com/percona/everest/pkg/logger" @@ -122,3 +125,38 @@ func TestAddDataToSecret(t *testing.T) { }) } } + +/* +This benchmark measures how much time does it take to clean up a long lists of tokens. +On an Apple M3 Pro it takes ~0.69 ms to perform a cleanup for a list of 10,000 tokens, which is acceptable. + +goos: darwin +goarch: arm64 +pkg: github.com/percona/everest/pkg/session +cpu: Apple M3 Pro +BenchmarkCleanupOld +BenchmarkCleanupOld-12 1500 671899 ns/op +*/ +func BenchmarkCleanupOld(b *testing.B) { + numTokens := 10000 + list := generateTestList(numTokens) + thresholdDate := time.Date(2025, 4, 3, 13, 33, 1, 0, time.UTC) + l := zap.L().Sugar() + b.ResetTimer() + for i := 0; i < b.N; i++ { + cleanupOld(l, list, thresholdDate) + } +} + +func generateTestList(numTokens int) string { + var builder strings.Builder + expDate := time.Date(2070, 4, 3, 13, 33, 1, 0, time.UTC).Unix() + for i := 0; i < numTokens; i++ { + // expiration date is year 2070 which is long ahead, so all the tokens should be kept + builder.WriteString("21669bd9-2374-4dc1-9238-77d5cad01fed" + fmt.Sprintf("%d", expDate)) + if i < numTokens-1 { + builder.WriteString(sep) + } + } + return builder.String() +} diff --git a/pkg/session/jwt.go b/pkg/session/jwt.go index e2908a9fd..dd6a368e3 100644 --- a/pkg/session/jwt.go +++ b/pkg/session/jwt.go @@ -23,7 +23,7 @@ type JWTContent struct { Payload map[string]interface{} `json:"payload"` } -func shrinkToken(token *jwt.Token) (string, error) { +func shortenToken(token *jwt.Token) (string, error) { l := log.FromContext(context.Background()) content, err := extractContent(token) if err != nil { diff --git a/pkg/session/jwt_test.go b/pkg/session/jwt_test.go index fa607e9c9..f9581c28f 100644 --- a/pkg/session/jwt_test.go +++ b/pkg/session/jwt_test.go @@ -7,12 +7,12 @@ import ( "github.com/stretchr/testify/assert" ) -func TestShrinkToken(t *testing.T) { +func TestShortenToken(t *testing.T) { type tcase struct { - name string - claims jwt.MapClaims - shrunkToken string - error error + name string + claims jwt.MapClaims + shortenedToken string + error error } tcases := []tcase{ { @@ -21,32 +21,32 @@ func TestShrinkToken(t *testing.T) { "jti": "9d1c1f98-a479-41e3-8939-c7cb3edefa", "exp": float64(331743679478), }, - shrunkToken: "9d1c1f98-a479-41e3-8939-c7cb3edefa331743679478", - error: nil, + shortenedToken: "9d1c1f98-a479-41e3-8939-c7cb3edefa331743679478", + error: nil, }, { name: "no jti", claims: jwt.MapClaims{ "exp": float64(331743679478), }, - shrunkToken: "", - error: errExtractJti, + shortenedToken: "", + error: errExtractJti, }, { name: "no exp", claims: jwt.MapClaims{ "jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", }, - shrunkToken: "", - error: errExtractExp, + shortenedToken: "", + error: errExtractExp, }, } for _, tc := range tcases { t.Run(tc.name, func(t *testing.T) { t.Parallel() - result, err := shrinkToken(jwt.NewWithClaims(jwt.SigningMethodHS256, tc.claims)) + result, err := shortenToken(jwt.NewWithClaims(jwt.SigningMethodHS256, tc.claims)) assert.Equal(t, tc.error, err) - assert.Equal(t, tc.shrunkToken, result) + assert.Equal(t, tc.shortenedToken, result) }) } } From f6be2c3b545484d173b08ee26c7e29533fd23766 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 8 Apr 2025 14:39:01 +0500 Subject: [PATCH 03/31] tests, improvements --- .github/workflows/dev-be-ci.yaml | 4 +- api-tests/tests/auth.spec.ts | 63 ++++++++++++++++-------- internal/server/everest.go | 11 +++-- internal/server/session.go | 2 +- pkg/session/blocklist.go | 69 ++++++++++++++------------- pkg/session/content_processor.go | 48 +++++++++++-------- pkg/session/content_processor_test.go | 26 +++++++++- 7 files changed, 142 insertions(+), 81 deletions(-) diff --git a/.github/workflows/dev-be-ci.yaml b/.github/workflows/dev-be-ci.yaml index 11829eea4..eb040f11d 100644 --- a/.github/workflows/dev-be-ci.yaml +++ b/.github/workflows/dev-be-ci.yaml @@ -349,8 +349,10 @@ jobs: run: | kubectl port-forward --namespace everest-system deployment/everest-server 8080:8080 & - - name: Create Everest test user + - name: Create Everest test users run: | + ./bin/everestctl accounts create -u test -p password + echo "API_TOKEN_TEST=$(curl --location -s 'localhost:8080/v1/session' --header 'Content-Type: application/json' --data '{"username": "test","password": "password"}' | jq -r .token)" >> $GITHUB_ENV ./bin/everestctl accounts create -u everest_ci -p password echo "API_TOKEN=$(curl --location -s 'localhost:8080/v1/session' --header 'Content-Type: application/json' --data '{"username": "everest_ci","password": "password"}' | jq -r .token)" >> $GITHUB_ENV diff --git a/api-tests/tests/auth.spec.ts b/api-tests/tests/auth.spec.ts index 7a7ace10c..5394ba22e 100644 --- a/api-tests/tests/auth.spec.ts +++ b/api-tests/tests/auth.spec.ts @@ -12,30 +12,51 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import { expect, test } from '@fixtures'; -import { checkError } from '@tests/tests/helpers'; +import {expect, test} from '@fixtures'; -test('auth header fails with invalid token', async ({ request }) => { - const version = await request.get('/v1/version', { - headers: { - Authorization: 'Bearer 123', - }, - }); +test('auth header fails with invalid token', async ({request}) => { + const version = await request.get('/v1/version', { + headers: { + Authorization: 'Bearer 123', + }, + }); - expect(version.status()).toEqual(401); + expect(version.status()).toEqual(401); }); test.describe('no authorization header', () => { - test.use({ - extraHTTPHeaders: { - 'Content-Type': 'application/json', - Accept: 'application/json', - }, - }); - - test('auth header fails with no content', async ({ request }) => { - const version = await request.get('/v1/version'); - - expect(version.status()).toEqual(400); - }); + test.use({ + extraHTTPHeaders: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + }); + + test('auth header fails with no content', async ({request}) => { + const version = await request.get('/v1/version'); + + expect(version.status()).toEqual(400); + }); +}); + +test.describe('logout', () => { + test.use({ + extraHTTPHeaders: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${process.env.API_TOKEN_TEST}`, + }, + }); + + test('authenticated api request fails after logout', async ({request}) => { + const versionBeforeLogout = await request.get('/v1/version'); + // NOTE: this test is сontext-dependent, the API_TOKEN_TEST needs to be valid before each run + expect(versionBeforeLogout.status()).toEqual(200); + + const response = await request.delete('/v1/session'); + expect(response.status()).toEqual(204); + + const versionAfterLogout = await request.get('/v1/version'); + expect(versionAfterLogout.status()).toEqual(401); + }); }); diff --git a/internal/server/everest.go b/internal/server/everest.go index 6fa180432..526aa0547 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -62,7 +62,7 @@ type EverestServer struct { sessionMgr *session.Manager attemptsStore *RateLimiterMemoryStore handler handlers.Handler - blacklist session.Blocklist + blocklist session.Blocklist oidcProvider *oidc.ProviderConfig } @@ -112,6 +112,11 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar return nil, errors.Join(err, errors.New("failed to get OIDC provider config")) } + blockList, err := session.NewBlocklist(ctx, kubeClient, l) + if err != nil { + return nil, errors.Join(err, errors.New("failed to configure tokens blocklist")) + } + e := &EverestServer{ config: c, l: l, @@ -119,7 +124,7 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar kubeClient: kubeClient, sessionMgr: sessMgr, attemptsStore: store, - blacklist: session.NewBlocklist(kubeClient, l), + blocklist: blockList, oidcProvider: oidcProvider, } e.echo.HTTPErrorHandler = e.errorHandlerChain() @@ -434,7 +439,7 @@ func (e *EverestServer) blocklistMiddleWare() (echo.MiddlewareFunc, error) { if skipper(c) { return next(c) } - if allow, err := e.blacklist.Allow(c.Request().Context()); err != nil { + if allow, err := e.blocklist.IsAllowed(c.Request().Context()); err != nil { e.l.Error(err) return err } else if !allow { diff --git a/internal/server/session.go b/internal/server/session.go index f18d0adfa..42ef632f9 100644 --- a/internal/server/session.go +++ b/internal/server/session.go @@ -77,7 +77,7 @@ func (e *EverestServer) DeleteSession(ctx echo.Context) error { return ctx.JSON(http.StatusUnauthorized, errors.New("failed to get token from context")) } if token != nil { - err := e.blacklist.Add(c, token) + err := e.blocklist.Add(c, token) if err != nil { return ctx.JSON(http.StatusRequestTimeout, api.Error{ Message: pointer.To("Incorrect username or password provided"), diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 610ce6bdf..5a1671c8e 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -2,6 +2,8 @@ package session import ( "context" + "github.com/percona/everest/pkg/kubernetes" + "github.com/percona/everest/pkg/kubernetes/informer" "github.com/golang-jwt/jwt/v5" "github.com/pkg/errors" @@ -20,27 +22,21 @@ const ( type Blocklist interface { Add(ctx context.Context, token *jwt.Token) error - Allow(ctx context.Context) (bool, error) + IsAllowed(ctx context.Context) (bool, error) } type blocklist struct { - secretsManager SecretsManager - content ContentProcessor - l *zap.SugaredLogger -} - -type SecretsManager interface { - // GetSecret returns a secret by name. - GetSecret(ctx context.Context, namespace, name string) (*corev1.Secret, error) - // CreateSecret creates a secret. - CreateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) - // UpdateSecret updates a secret. - UpdateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) + kubeClient kubernetes.KubernetesConnector + content ContentProcessor + informer *informer.Informer + cachedSecret *corev1.Secret + l *zap.SugaredLogger } type ContentProcessor interface { Add(l *zap.SugaredLogger, secret *corev1.Secret, tokenData string) (*corev1.Secret, bool) - IsBlocked(secret *corev1.Secret, tokenData string) bool + IsBlocked(shortenedToken string) bool + UpdateCache(secret *corev1.Secret) } func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { @@ -50,10 +46,10 @@ func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { } for attempts := 0; attempts < maxRetries; attempts++ { - secret, err := b.secretsManager.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) + secret, err := b.kubeClient.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) if err != nil { if k8serrors.IsNotFound(err) { - _, err = b.secretsManager.CreateSecret(ctx, blockListSecretTemplate(shortenedToken)) + _, err = b.kubeClient.CreateSecret(ctx, blockListSecretTemplate(shortenedToken)) if err != nil { b.l.Errorf("failed to create %s secret: %v", common.EverestBlocklistSecretName, err) continue @@ -61,14 +57,17 @@ func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { return nil } } + b.cachedSecret = secret secret, retryNeeded := b.content.Add(b.l, secret, shortenedToken) if retryNeeded { continue } - if _, err := b.secretsManager.UpdateSecret(ctx, secret); err != nil { - b.l.Errorf("failed to update %s secret: %v", common.EverestBlocklistSecretName, err) + updatedSecret, updateErr := b.kubeClient.UpdateSecret(ctx, secret) + if updateErr != nil { + b.l.Errorf("failed to update %s secret: %v", common.EverestBlocklistSecretName, updateErr) continue } + b.content.UpdateCache(updatedSecret) return nil } return nil @@ -90,32 +89,36 @@ func blockListSecretTemplate(stringData string) *corev1.Secret { } } -func (b *blocklist) Allow(ctx context.Context) (bool, error) { +func (b *blocklist) IsAllowed(ctx context.Context) (bool, error) { token, ok := ctx.Value(common.UserCtxKey).(*jwt.Token) if !ok { return false, errors.New("failed to get token from context") } - secret, err := b.secretsManager.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) - if err != nil { - if k8serrors.IsNotFound(err) { - return true, nil - } - return false, errors.Wrap(err, "failed to get secret") - } - shortenedToken, err := shortenToken(token) if err != nil { return false, errors.Wrap(err, "failed to shrink token") } - return !b.content.IsBlocked(secret, shortenedToken), nil + return !b.content.IsBlocked(shortenedToken), nil } -func NewBlocklist(secretsManager SecretsManager, logger *zap.SugaredLogger) Blocklist { - return &blocklist{ - secretsManager: secretsManager, - content: newContentProcessor(), - l: logger, +func NewBlocklist(ctx context.Context, kubeClient kubernetes.KubernetesConnector, logger *zap.SugaredLogger) (Blocklist, error) { + // read the existing blocklist token or create it + secret, err := kubeClient.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) + if err != nil { + if !k8serrors.IsNotFound(err) { + return nil, errors.Wrap(err, "failed to get secret") + } + var createErr error + secret, createErr = kubeClient.CreateSecret(ctx, blockListSecretTemplate("")) + if createErr != nil { + return nil, errors.Wrap(createErr, "failed to create secret") + } } + return &blocklist{ + kubeClient: kubeClient, + content: newContentProcessor(secret), + l: logger, + }, nil } diff --git a/pkg/session/content_processor.go b/pkg/session/content_processor.go index 9bdf5bd1b..7d317f48e 100644 --- a/pkg/session/content_processor.go +++ b/pkg/session/content_processor.go @@ -13,49 +13,56 @@ import ( const ( sep = "," timestampLen = 10 - // expiration = time.Hour * 24 - expiration = time.Second * 30 ) type contentProcessor struct { - mutex sync.Mutex + mutex sync.Mutex + cachedSecret *corev1.Secret } -func newContentProcessor() ContentProcessor { - return &contentProcessor{} +func newContentProcessor(secret *corev1.Secret) ContentProcessor { + return &contentProcessor{ + cachedSecret: secret, + } +} + +func (d *contentProcessor) UpdateCache(secret *corev1.Secret) { + d.cachedSecret = secret } -func (d *contentProcessor) Add(l *zap.SugaredLogger, secret *corev1.Secret, tokenData string) (*corev1.Secret, bool) { +func (d *contentProcessor) Add(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string) (*corev1.Secret, bool) { if d.mutex.TryLock() { defer d.mutex.Unlock() - return addDataToSecret(l, secret, tokenData, time.Now().Add(-expiration).UTC()), false + updatedSecret := addDataToSecret(l, secret, shortenedToken, time.Now().UTC()) + return updatedSecret, false } return secret, true } -func (d *contentProcessor) IsBlocked(secret *corev1.Secret, tokenData string) bool { - list, ok := secret.Data[dataKey] - return ok && strings.Contains(string(list), tokenData) +func (d *contentProcessor) IsBlocked(shortenedToken string) bool { + if d.cachedSecret == nil { + return false + } + list, ok := d.cachedSecret.Data[dataKey] + return ok && strings.Contains(string(list), shortenedToken) } -func addDataToSecret(l *zap.SugaredLogger, secret *corev1.Secret, newData string, thresholdDate time.Time) *corev1.Secret { - byteArr, ok := secret.Data[dataKey] +func addDataToSecret(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string, now time.Time) *corev1.Secret { + existingList, ok := secret.Data[dataKey] if !ok { secret.StringData = map[string]string{ - dataKey: newData, + dataKey: shortenedToken, } return secret } - list := string(byteArr) - list = cleanupOld(l, list, thresholdDate) - list += sep + newData + list := append(cleanupOld(l, string(existingList), now), shortenedToken) secret.StringData = map[string]string{ - dataKey: list, + dataKey: strings.Join(list, sep), } return secret } -func cleanupOld(l *zap.SugaredLogger, list string, thresholdDate time.Time) string { +func cleanupOld(l *zap.SugaredLogger, list string, now time.Time) []string { tokens := strings.Split(list, sep) newList := make([]string, 0, len(tokens)) for _, shortenedToken := range tokens { @@ -71,11 +78,10 @@ func cleanupOld(l *zap.SugaredLogger, list string, thresholdDate time.Time) stri continue } timeObj := time.Unix(tsInt, 0) - l.Info("time", "from token: ", timeObj, "treshold: ", thresholdDate) // only keep the tokens which natural expiration time is not over yet - if timeObj.After(thresholdDate) { + if timeObj.After(now) { newList = append(newList, shortenedToken) } } - return strings.Join(newList, sep) + return newList } diff --git a/pkg/session/content_processor_test.go b/pkg/session/content_processor_test.go index 2825b8161..226658024 100644 --- a/pkg/session/content_processor_test.go +++ b/pkg/session/content_processor_test.go @@ -69,7 +69,7 @@ func TestAddDataToSecret(t *testing.T) { l := logger.Sugar() tcases := []tcase{ { - name: "no data in secret", + name: "empty secret", data: "id123AAA1743687192", secret: &corev1.Secret{}, thresholdDate: time.Date(2025, 4, 3, 13, 33, 10, 0, time.UTC), @@ -89,6 +89,8 @@ func TestAddDataToSecret(t *testing.T) { }, thresholdDate: time.Date(2025, 4, 3, 13, 33, 10, 0, time.UTC), expected: &corev1.Secret{ + // the Data field is updated only when the object is applied to k8s, so for this test + // only the write-only StringData field is expected to be changed. Data: map[string][]byte{ dataKey: []byte("id123AAA1743687191"), }, @@ -107,6 +109,8 @@ func TestAddDataToSecret(t *testing.T) { }, thresholdDate: time.Date(2025, 4, 3, 13, 33, 12, 0, time.UTC), expected: &corev1.Secret{ + // the Data field is updated only when the object is applied to k8s, so for this test + // only the write-only StringData field is expected to be changed. Data: map[string][]byte{ dataKey: []byte("id123AAA1743687191,id123AAA1743687193"), }, @@ -115,6 +119,26 @@ func TestAddDataToSecret(t *testing.T) { }, }, }, + { + name: "deleted all old data, add newer data", + data: "id123AAA1743687195", + secret: &corev1.Secret{ + Data: map[string][]byte{ + dataKey: []byte("id123AAA1743687191,id123AAA1743687193"), + }, + }, + thresholdDate: time.Date(2025, 4, 3, 13, 33, 14, 0, time.UTC), + expected: &corev1.Secret{ + // the Data field is updated only when the object is applied to k8s, so for this test + // only the write-only StringData field is expected to be changed. + Data: map[string][]byte{ + dataKey: []byte("id123AAA1743687191,id123AAA1743687193"), + }, + StringData: map[string]string{ + dataKey: "id123AAA1743687195", + }, + }, + }, } for _, tc := range tcases { From 40392b7dd8d9ddda126c85c5566a6e26f4d7f3b6 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 8 Apr 2025 14:43:51 +0500 Subject: [PATCH 04/31] format tests --- api-tests/tests/auth.spec.ts | 41 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/api-tests/tests/auth.spec.ts b/api-tests/tests/auth.spec.ts index 5394ba22e..89709b62c 100644 --- a/api-tests/tests/auth.spec.ts +++ b/api-tests/tests/auth.spec.ts @@ -12,31 +12,32 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import {expect, test} from '@fixtures'; +import { expect, test } from '@fixtures'; +import { checkError } from '@tests/tests/helpers'; -test('auth header fails with invalid token', async ({request}) => { - const version = await request.get('/v1/version', { - headers: { - Authorization: 'Bearer 123', - }, - }); +test('auth header fails with invalid token', async ({ request }) => { + const version = await request.get('/v1/version', { + headers: { + Authorization: 'Bearer 123', + }, + }); - expect(version.status()).toEqual(401); + expect(version.status()).toEqual(401); }); test.describe('no authorization header', () => { - test.use({ - extraHTTPHeaders: { - 'Content-Type': 'application/json', - Accept: 'application/json', - }, - }); - - test('auth header fails with no content', async ({request}) => { - const version = await request.get('/v1/version'); - - expect(version.status()).toEqual(400); - }); + test.use({ + extraHTTPHeaders: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + }); + + test('auth header fails with no content', async ({ request }) => { + const version = await request.get('/v1/version'); + + expect(version.status()).toEqual(400); + }); }); test.describe('logout', () => { From 0c172a09813d7ea2ef10b7b81885c3761f725a37 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 8 Apr 2025 14:58:11 +0500 Subject: [PATCH 05/31] remove using github.com/pkg/errors --- pkg/session/blocklist.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 5a1671c8e..ef767f1cd 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -2,11 +2,11 @@ package session import ( "context" + "fmt" "github.com/percona/everest/pkg/kubernetes" "github.com/percona/everest/pkg/kubernetes/informer" "github.com/golang-jwt/jwt/v5" - "github.com/pkg/errors" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -92,12 +92,12 @@ func blockListSecretTemplate(stringData string) *corev1.Secret { func (b *blocklist) IsAllowed(ctx context.Context) (bool, error) { token, ok := ctx.Value(common.UserCtxKey).(*jwt.Token) if !ok { - return false, errors.New("failed to get token from context") + return false, fmt.Errorf("failed to get token from context") } shortenedToken, err := shortenToken(token) if err != nil { - return false, errors.Wrap(err, "failed to shrink token") + return false, fmt.Errorf("failed to shorten token: %w", err) } return !b.content.IsBlocked(shortenedToken), nil @@ -108,12 +108,12 @@ func NewBlocklist(ctx context.Context, kubeClient kubernetes.KubernetesConnector secret, err := kubeClient.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) if err != nil { if !k8serrors.IsNotFound(err) { - return nil, errors.Wrap(err, "failed to get secret") + return nil, fmt.Errorf("failed to get secret: %w", err) } var createErr error secret, createErr = kubeClient.CreateSecret(ctx, blockListSecretTemplate("")) if createErr != nil { - return nil, errors.Wrap(createErr, "failed to create secret") + return nil, fmt.Errorf("failed to create secret: %w", createErr) } } return &blocklist{ From cc30ee0035c8a5d8f1ffaa0f4390cb6968d8be44 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 8 Apr 2025 15:01:16 +0500 Subject: [PATCH 06/31] fix tests --- pkg/session/content_processor_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/session/content_processor_test.go b/pkg/session/content_processor_test.go index 226658024..9baa7de14 100644 --- a/pkg/session/content_processor_test.go +++ b/pkg/session/content_processor_test.go @@ -18,7 +18,7 @@ func TestCleanupOld(t *testing.T) { name string data string thresholdDate time.Time - expected string + expected []string } logger := logger.MustInitLogger(true, "everest") l := logger.Sugar() @@ -27,25 +27,25 @@ func TestCleanupOld(t *testing.T) { name: "one outdated", data: "id123AAA1743687192,id2323BBB1743687194", thresholdDate: time.Date(2025, 4, 3, 13, 33, 13, 0, time.UTC), - expected: "id2323BBB1743687194", + expected: []string{"id2323BBB1743687194"}, }, { name: "two outdated", data: "id123AAA1743687192,id2323BBB1743687193,id2323BBB1743687194", thresholdDate: time.Date(2025, 4, 3, 13, 33, 13, 0, time.UTC), - expected: "id2323BBB1743687194", + expected: []string{"id2323BBB1743687194"}, }, { name: "all outdated", data: "id123AAA1743687191,id2323BBB1743687192,id2323BBB1743687193", thresholdDate: time.Date(2025, 4, 3, 13, 33, 13, 0, time.UTC), - expected: "", + expected: []string{}, }, { name: "all fresh", data: "id123AAA1743687194,id2323BBB1743687195,id2323BBB1743687196", thresholdDate: time.Date(2025, 4, 3, 13, 33, 13, 0, time.UTC), - expected: "id123AAA1743687194,id2323BBB1743687195,id2323BBB1743687196", + expected: []string{"id123AAA1743687194", "id2323BBB1743687195", "id2323BBB1743687196"}, }, } From e6fb101d63b0e695c7df1d4fc364bb0b91eeef3e Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 8 Apr 2025 15:08:05 +0500 Subject: [PATCH 07/31] remove informer link --- pkg/session/blocklist.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index ef767f1cd..184d0d8dc 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "github.com/percona/everest/pkg/kubernetes" - "github.com/percona/everest/pkg/kubernetes/informer" "github.com/golang-jwt/jwt/v5" "go.uber.org/zap" @@ -28,7 +27,6 @@ type Blocklist interface { type blocklist struct { kubeClient kubernetes.KubernetesConnector content ContentProcessor - informer *informer.Informer cachedSecret *corev1.Secret l *zap.SugaredLogger } From 316e230a2b1317d06f3e5de9de47e7ab295c164a Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 8 Apr 2025 16:11:08 +0500 Subject: [PATCH 08/31] format, add comments, refactor --- internal/server/everest.go | 4 +- internal/server/session.go | 23 ++---- pkg/session/blocklist.go | 122 ++++++++++++++++--------------- pkg/session/content_processor.go | 23 ++++-- pkg/session/jwt.go | 7 -- 5 files changed, 92 insertions(+), 87 deletions(-) diff --git a/internal/server/everest.go b/internal/server/everest.go index 526aa0547..cb6eaf8fd 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -439,10 +439,10 @@ func (e *EverestServer) blocklistMiddleWare() (echo.MiddlewareFunc, error) { if skipper(c) { return next(c) } - if allow, err := e.blocklist.IsAllowed(c.Request().Context()); err != nil { + if isBlocked, err := e.blocklist.IsBlocked(c.Request().Context()); err != nil { e.l.Error(err) return err - } else if !allow { + } else if isBlocked { return c.JSON(http.StatusUnauthorized, api.Error{ Message: pointer.ToString("Invalid token"), }) diff --git a/internal/server/session.go b/internal/server/session.go index 42ef632f9..9707e6774 100644 --- a/internal/server/session.go +++ b/internal/server/session.go @@ -23,13 +23,11 @@ import ( "time" "github.com/AlekSi/pointer" - "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/percona/everest/api" "github.com/percona/everest/pkg/accounts" - "github.com/percona/everest/pkg/common" ) const ( @@ -68,22 +66,17 @@ func (e *EverestServer) CreateSession(ctx echo.Context) error { return ctx.JSON(http.StatusOK, map[string]string{"token": jwtToken}) } -// DeleteSession deletes session. +// DeleteSession invalidates the user token by adding it to the blocklist func (e *EverestServer) DeleteSession(ctx echo.Context) error { e.attemptsStore.IncreaseTimeout(ctx.RealIP()) - c := ctx.Request().Context() - token, ok := c.Value(common.UserCtxKey).(*jwt.Token) - if !ok { - return ctx.JSON(http.StatusUnauthorized, errors.New("failed to get token from context")) - } - if token != nil { - err := e.blocklist.Add(c, token) - if err != nil { - return ctx.JSON(http.StatusRequestTimeout, api.Error{ - Message: pointer.To("Incorrect username or password provided"), - }) - } + err := e.blocklist.Block(ctx.Request().Context()) + if err != nil { + e.l.Errorf("blocklist error: %v", err) + return ctx.JSON(http.StatusInternalServerError, api.Error{ + Message: pointer.To("Failed to logout user"), + }) } + return ctx.NoContent(http.StatusNoContent) } diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 184d0d8dc..787e55192 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -3,7 +3,6 @@ package session import ( "context" "fmt" - "github.com/percona/everest/pkg/kubernetes" "github.com/golang-jwt/jwt/v5" "go.uber.org/zap" @@ -12,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/percona/everest/pkg/common" + "github.com/percona/everest/pkg/kubernetes" ) const ( @@ -19,25 +19,48 @@ const ( maxRetries = 10 ) +// Blocklist represents interface to block JWT tokens and check if a token is blocked. type Blocklist interface { - Add(ctx context.Context, token *jwt.Token) error - IsAllowed(ctx context.Context) (bool, error) + // Block invalidates the token from the context by adding it to blocklist. + Block(ctx context.Context) error + // IsBlocked checks if the token from the context is blocked. + IsBlocked(ctx context.Context) (bool, error) } type blocklist struct { - kubeClient kubernetes.KubernetesConnector - content ContentProcessor - cachedSecret *corev1.Secret - l *zap.SugaredLogger + kubeClient kubernetes.KubernetesConnector + contentProcessor ContentProcessor + cachedSecret *corev1.Secret + l *zap.SugaredLogger } -type ContentProcessor interface { - Add(l *zap.SugaredLogger, secret *corev1.Secret, tokenData string) (*corev1.Secret, bool) - IsBlocked(shortenedToken string) bool - UpdateCache(secret *corev1.Secret) +// NewBlocklist creates a new block list +func NewBlocklist(ctx context.Context, kubeClient kubernetes.KubernetesConnector, logger *zap.SugaredLogger) (Blocklist, error) { + // read the existing blocklist token or create it + secret, err := kubeClient.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) + if err != nil { + if !k8serrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get secret: %w", err) + } + var createErr error + secret, createErr = kubeClient.CreateSecret(ctx, blockListSecretTemplate("")) + if createErr != nil { + return nil, fmt.Errorf("failed to create secret: %w", createErr) + } + } + return &blocklist{ + kubeClient: kubeClient, + contentProcessor: newContentProcessor(secret), + l: logger, + }, nil } -func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { +// Block invalidates the token from the context by adding it to blocklist. +func (b *blocklist) Block(ctx context.Context) error { + token, err := extractToken(ctx) + if err != nil { + return err + } shortenedToken, err := shortenToken(token) if err != nil { return err @@ -46,31 +69,48 @@ func (b *blocklist) Add(ctx context.Context, token *jwt.Token) error { for attempts := 0; attempts < maxRetries; attempts++ { secret, err := b.kubeClient.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) if err != nil { - if k8serrors.IsNotFound(err) { - _, err = b.kubeClient.CreateSecret(ctx, blockListSecretTemplate(shortenedToken)) - if err != nil { - b.l.Errorf("failed to create %s secret: %v", common.EverestBlocklistSecretName, err) - continue - } - return nil - } + b.l.Errorf("failed to get %s secret: %v", common.EverestBlocklistSecretName, err) + return err } - b.cachedSecret = secret - secret, retryNeeded := b.content.Add(b.l, secret, shortenedToken) + + secret, retryNeeded := b.contentProcessor.Block(b.l, secret, shortenedToken) if retryNeeded { + b.l.Infof("failed to block token, retrying") continue } updatedSecret, updateErr := b.kubeClient.UpdateSecret(ctx, secret) if updateErr != nil { - b.l.Errorf("failed to update %s secret: %v", common.EverestBlocklistSecretName, updateErr) + b.l.Errorf("failed to update %s secret, retrying: %v", common.EverestBlocklistSecretName, updateErr) continue } - b.content.UpdateCache(updatedSecret) + b.contentProcessor.UpdateCache(updatedSecret) return nil } return nil } +// IsBlocked checks if the token from the context is blocked. +func (b *blocklist) IsBlocked(ctx context.Context) (bool, error) { + token, err := extractToken(ctx) + if err != nil { + return false, err + } + shortenedToken, err := shortenToken(token) + if err != nil { + return false, fmt.Errorf("failed to shorten token: %w", err) + } + + return b.contentProcessor.IsBlocked(shortenedToken), nil +} + +func extractToken(ctx context.Context) (*jwt.Token, error) { + token, ok := ctx.Value(common.UserCtxKey).(*jwt.Token) + if !ok { + return nil, fmt.Errorf("failed to get token from context") + } + return token, nil +} + func blockListSecretTemplate(stringData string) *corev1.Secret { return &corev1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -86,37 +126,3 @@ func blockListSecretTemplate(stringData string) *corev1.Secret { }, } } - -func (b *blocklist) IsAllowed(ctx context.Context) (bool, error) { - token, ok := ctx.Value(common.UserCtxKey).(*jwt.Token) - if !ok { - return false, fmt.Errorf("failed to get token from context") - } - - shortenedToken, err := shortenToken(token) - if err != nil { - return false, fmt.Errorf("failed to shorten token: %w", err) - } - - return !b.content.IsBlocked(shortenedToken), nil -} - -func NewBlocklist(ctx context.Context, kubeClient kubernetes.KubernetesConnector, logger *zap.SugaredLogger) (Blocklist, error) { - // read the existing blocklist token or create it - secret, err := kubeClient.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) - if err != nil { - if !k8serrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get secret: %w", err) - } - var createErr error - secret, createErr = kubeClient.CreateSecret(ctx, blockListSecretTemplate("")) - if createErr != nil { - return nil, fmt.Errorf("failed to create secret: %w", createErr) - } - } - return &blocklist{ - kubeClient: kubeClient, - content: newContentProcessor(secret), - l: logger, - }, nil -} diff --git a/pkg/session/content_processor.go b/pkg/session/content_processor.go index 7d317f48e..e51a0f8f2 100644 --- a/pkg/session/content_processor.go +++ b/pkg/session/content_processor.go @@ -15,6 +15,16 @@ const ( timestampLen = 10 ) +// ContentProcessor knows how to work with actual data in the blocklist. +type ContentProcessor interface { + // Block adds the token to the blacklist, cleans up the outdated data from the secret and returns the updated secret + Block(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string) (*corev1.Secret, bool) + // IsBlocked returns true if the given shortened token is in the blocklist. + IsBlocked(shortenedToken string) bool + // UpdateCache updates the cached secret which is used for reading. + UpdateCache(secret *corev1.Secret) +} + type contentProcessor struct { mutex sync.Mutex cachedSecret *corev1.Secret @@ -26,11 +36,8 @@ func newContentProcessor(secret *corev1.Secret) ContentProcessor { } } -func (d *contentProcessor) UpdateCache(secret *corev1.Secret) { - d.cachedSecret = secret -} - -func (d *contentProcessor) Add(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string) (*corev1.Secret, bool) { +// Block adds the token to the blacklist, cleans up the outdated data from the secret and returns the updated secret +func (d *contentProcessor) Block(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string) (*corev1.Secret, bool) { if d.mutex.TryLock() { defer d.mutex.Unlock() updatedSecret := addDataToSecret(l, secret, shortenedToken, time.Now().UTC()) @@ -39,6 +46,7 @@ func (d *contentProcessor) Add(l *zap.SugaredLogger, secret *corev1.Secret, shor return secret, true } +// IsBlocked returns true if the given shortened token is in the blocklist. func (d *contentProcessor) IsBlocked(shortenedToken string) bool { if d.cachedSecret == nil { return false @@ -47,6 +55,11 @@ func (d *contentProcessor) IsBlocked(shortenedToken string) bool { return ok && strings.Contains(string(list), shortenedToken) } +// UpdateCache updates the cached secret which is used for reading. +func (d *contentProcessor) UpdateCache(secret *corev1.Secret) { + d.cachedSecret = secret +} + func addDataToSecret(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string, now time.Time) *corev1.Secret { existingList, ok := secret.Data[dataKey] if !ok { diff --git a/pkg/session/jwt.go b/pkg/session/jwt.go index dd6a368e3..b40ebe937 100644 --- a/pkg/session/jwt.go +++ b/pkg/session/jwt.go @@ -1,13 +1,11 @@ package session import ( - "context" "errors" "fmt" "strconv" "github.com/golang-jwt/jwt/v5" - "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -24,21 +22,16 @@ type JWTContent struct { } func shortenToken(token *jwt.Token) (string, error) { - l := log.FromContext(context.Background()) content, err := extractContent(token) if err != nil { - l.Error(err, "could not extract content") return "", err } - l.Info("payload", "content", content.Payload) jti, ok := content.Payload["jti"].(string) if !ok { - l.Error(err, "could not extract jti") return "", errExtractJti } exp, ok := content.Payload["exp"].(float64) if !ok { - l.Error(err, "could not extract exp") return "", errExtractExp } return jti + strconv.FormatFloat(exp, 'f', 0, 64), nil From ba3b7f4e43bb548d29fb56583976c65db6628e55 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 8 Apr 2025 16:53:18 +0500 Subject: [PATCH 09/31] new tests, refactoring --- pkg/session/blocklist.go | 4 +-- pkg/session/content_processor.go | 10 ++++--- pkg/session/content_processor_test.go | 41 +++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 787e55192..e929a5552 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -73,8 +73,8 @@ func (b *blocklist) Block(ctx context.Context) error { return err } - secret, retryNeeded := b.contentProcessor.Block(b.l, secret, shortenedToken) - if retryNeeded { + secret, ok := b.contentProcessor.Block(b.l, secret, shortenedToken) + if !ok { b.l.Infof("failed to block token, retrying") continue } diff --git a/pkg/session/content_processor.go b/pkg/session/content_processor.go index e51a0f8f2..6de0a7317 100644 --- a/pkg/session/content_processor.go +++ b/pkg/session/content_processor.go @@ -17,7 +17,8 @@ const ( // ContentProcessor knows how to work with actual data in the blocklist. type ContentProcessor interface { - // Block adds the token to the blacklist, cleans up the outdated data from the secret and returns the updated secret + // Block adds the token to the blacklist, cleans up the outdated data from the secret. + // The method returns the updated secret and a bool that indicates if the attempt was successful. Block(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string) (*corev1.Secret, bool) // IsBlocked returns true if the given shortened token is in the blocklist. IsBlocked(shortenedToken string) bool @@ -36,14 +37,15 @@ func newContentProcessor(secret *corev1.Secret) ContentProcessor { } } -// Block adds the token to the blacklist, cleans up the outdated data from the secret and returns the updated secret +// Block adds the token to the blacklist, cleans up the outdated data from the secret. +// The method returns the updated secret and a bool that indicates if the attempt was successful. func (d *contentProcessor) Block(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string) (*corev1.Secret, bool) { if d.mutex.TryLock() { defer d.mutex.Unlock() updatedSecret := addDataToSecret(l, secret, shortenedToken, time.Now().UTC()) - return updatedSecret, false + return updatedSecret, true } - return secret, true + return secret, false } // IsBlocked returns true if the given shortened token is in the blocklist. diff --git a/pkg/session/content_processor_test.go b/pkg/session/content_processor_test.go index 9baa7de14..2621e1fcc 100644 --- a/pkg/session/content_processor_test.go +++ b/pkg/session/content_processor_test.go @@ -2,6 +2,7 @@ package session import ( "fmt" + "go.uber.org/zap/zaptest" "strings" "testing" "time" @@ -184,3 +185,43 @@ func generateTestList(numTokens int) string { } return builder.String() } + +func TestContentProcessor_Block_ReturnsOriginalSecretWhenLocked(t *testing.T) { + l := zaptest.NewLogger(t).Sugar() + secret := &corev1.Secret{} + shortenedToken := "test-token" + processor := &contentProcessor{cachedSecret: blockListSecretTemplate("")} + + // Acquire the lock directly to simulate contention + processor.mutex.Lock() + defer processor.mutex.Unlock() + + // Call Block while the lock is held + returnedSecret, locked := processor.Block(l, secret, shortenedToken) + + if locked { + t.Errorf("Expected Block to return false when the lock is already held, but it returned true.") + } + + if returnedSecret != secret { + t.Errorf("Expected Block to return the original secret when the lock is already held, but it returned a different secret.") + } +} + +func TestContentProcessor_Block_ReturnsUpdatedSecretWhenUnlocked(t *testing.T) { + l := zaptest.NewLogger(t).Sugar() + secret := &corev1.Secret{} + shortenedToken := "test-token" + processor := &contentProcessor{cachedSecret: blockListSecretTemplate("")} + + // Call Block when the lock is free + returnedSecret, locked := processor.Block(l, secret, shortenedToken) + + if !locked { + t.Errorf("Expected Block to return true when the lock is free, but it returned false.") + } + + if returnedSecret == nil { + t.Errorf("Expected Block to return a non-nil secret, but it returned nil.") + } +} From a24c580c24c7476cff96aa7deba74022362913c5 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 8 Apr 2025 18:29:17 +0500 Subject: [PATCH 10/31] format, update tests --- pkg/session/content_processor_test.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pkg/session/content_processor_test.go b/pkg/session/content_processor_test.go index 2621e1fcc..0f4b216d3 100644 --- a/pkg/session/content_processor_test.go +++ b/pkg/session/content_processor_test.go @@ -2,16 +2,14 @@ package session import ( "fmt" - "go.uber.org/zap/zaptest" "strings" "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/zap" + "go.uber.org/zap/zaptest" corev1 "k8s.io/api/core/v1" - - "github.com/percona/everest/pkg/logger" ) func TestCleanupOld(t *testing.T) { @@ -21,8 +19,7 @@ func TestCleanupOld(t *testing.T) { thresholdDate time.Time expected []string } - logger := logger.MustInitLogger(true, "everest") - l := logger.Sugar() + l := zaptest.NewLogger(t).Sugar() tcases := []tcase{ { name: "one outdated", @@ -66,8 +63,7 @@ func TestAddDataToSecret(t *testing.T) { thresholdDate time.Time expected *corev1.Secret } - logger := logger.MustInitLogger(true, "everest") - l := logger.Sugar() + l := zaptest.NewLogger(t).Sugar() tcases := []tcase{ { name: "empty secret", @@ -153,7 +149,7 @@ func TestAddDataToSecret(t *testing.T) { /* This benchmark measures how much time does it take to clean up a long lists of tokens. -On an Apple M3 Pro it takes ~0.69 ms to perform a cleanup for a list of 10,000 tokens, which is acceptable. +On an Apple M3 Pro it takes ~0.67 ms to perform a cleanup for a list of 10,000 tokens, which is acceptable. goos: darwin goarch: arm64 From 969e117d83fda5f984ad5618dee75336652425d9 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Mon, 21 Apr 2025 20:05:37 +0500 Subject: [PATCH 11/31] use cache for everest API server --- internal/server/everest.go | 4 +-- pkg/kubernetes/kubernetes.go | 51 +++++++++++++++++++++++++++++++++--- pkg/session/blocklist.go | 5 ++-- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/internal/server/everest.go b/internal/server/everest.go index bde471b9f..39b2cbe17 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -91,7 +91,7 @@ func getOIDCProviderConfig(ctx context.Context, kubeClient kubernetes.Kubernetes // NewEverestServer creates and configures everest API. func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.SugaredLogger) (*EverestServer, error) { - kubeConnector, err := kubernetes.NewInCluster(l) + kubeConnector, err := kubernetes.NewInClusterWithCache(l, ctx) if err != nil { return nil, errors.Join(err, errors.New("failed creating Kubernetes client")) } @@ -112,7 +112,7 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar return nil, errors.Join(err, errors.New("failed to get OIDC provider config")) } - blockList, err := session.NewBlocklist(ctx, kubeClient, l) + blockList, err := session.NewBlocklist(ctx, kubeConnector, l) if err != nil { return nil, errors.Join(err, errors.New("failed to configure tokens blocklist")) } diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index b34a4efa0..ba7f0863e 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -38,6 +38,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1" @@ -132,9 +133,7 @@ func New(kubeconfigPath string, l *zap.SugaredLogger) (KubernetesConnector, erro // NewInCluster creates a new kubernetes client using incluster authentication. func NewInCluster(l *zap.SugaredLogger) (KubernetesConnector, error) { - restConfig := ctrl.GetConfigOrDie() - restConfig.QPS = defaultQPSLimit - restConfig.Burst = defaultBurstLimit + restConfig := inClusterRestConfig() k8sclient, err := ctrlclient.New(restConfig, getKubernetesClientOptions()) if err != nil { @@ -148,6 +147,42 @@ func NewInCluster(l *zap.SugaredLogger) (KubernetesConnector, error) { }, nil } +// NewInClusterWithCache creates a new kubernetes client using incluster authentication with cache enabled +func NewInClusterWithCache(l *zap.SugaredLogger, ctx context.Context) (KubernetesConnector, error) { + restConfig := inClusterRestConfig() + cacheOptions := cache.Options{ + Scheme: CreateScheme(), + } + k8sCache, err := cache.New(restConfig, cacheOptions) + if err != nil { + panic(err) + } + go func() { + l.Info("starting cache") + if err := k8sCache.Start(ctx); err != nil { + l.Errorf("error starting pod cache: %s", err) + os.Exit(1) + } + }() + k8sclient, err := ctrlclient.New(restConfig, getKubernetesClientOptionsWithCache(k8sCache)) + if err != nil { + return nil, err + } + + return &Kubernetes{ + k8sClient: k8sclient, + l: l.With("component", "kubernetes"), + restConfig: restConfig, + }, nil +} + +func inClusterRestConfig() *rest.Config { + restConfig := ctrl.GetConfigOrDie() + restConfig.QPS = defaultQPSLimit + restConfig.Burst = defaultBurstLimit + return restConfig +} + // CreateScheme creates a new runtime.Scheme. // It registers all necessary types: // - standard client-go types @@ -170,6 +205,16 @@ func getKubernetesClientOptions() ctrlclient.Options { } } +func getKubernetesClientOptionsWithCache(k8sCache cache.Cache) ctrlclient.Options { + return ctrlclient.Options{ + Scheme: CreateScheme(), + Cache: &ctrlclient.CacheOptions{ + Reader: k8sCache, + Unstructured: false, + }, + } +} + func (k *Kubernetes) getDiscoveryClient() discovery.DiscoveryInterface { once.Do(func() { httpClient, err := rest.HTTPClientFor(k.restConfig) diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index e929a5552..16f8e428e 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "github.com/percona/everest/pkg/common" "github.com/percona/everest/pkg/kubernetes" @@ -37,7 +38,7 @@ type blocklist struct { // NewBlocklist creates a new block list func NewBlocklist(ctx context.Context, kubeClient kubernetes.KubernetesConnector, logger *zap.SugaredLogger) (Blocklist, error) { // read the existing blocklist token or create it - secret, err := kubeClient.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) + secret, err := kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err != nil { if !k8serrors.IsNotFound(err) { return nil, fmt.Errorf("failed to get secret: %w", err) @@ -67,7 +68,7 @@ func (b *blocklist) Block(ctx context.Context) error { } for attempts := 0; attempts < maxRetries; attempts++ { - secret, err := b.kubeClient.GetSecret(ctx, common.SystemNamespace, common.EverestBlocklistSecretName) + secret, err := b.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err != nil { b.l.Errorf("failed to get %s secret: %v", common.EverestBlocklistSecretName, err) return err From 02dbbd5f9809e5ab6a60fd2fc72ddb73683704e1 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Mon, 21 Apr 2025 21:42:23 +0500 Subject: [PATCH 12/31] remove cached secret, use token store --- pkg/session/blocklist.go | 51 +++----- pkg/session/content_processor.go | 102 --------------- pkg/session/token_store.go | 118 ++++++++++++++++++ ..._processor_test.go => token_store_test.go} | 40 ------ 4 files changed, 133 insertions(+), 178 deletions(-) delete mode 100644 pkg/session/content_processor.go create mode 100644 pkg/session/token_store.go rename pkg/session/{content_processor_test.go => token_store_test.go} (80%) diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 16f8e428e..32b366d67 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -7,9 +7,7 @@ import ( "github.com/golang-jwt/jwt/v5" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "github.com/percona/everest/pkg/common" "github.com/percona/everest/pkg/kubernetes" @@ -29,30 +27,24 @@ type Blocklist interface { } type blocklist struct { - kubeClient kubernetes.KubernetesConnector - contentProcessor ContentProcessor - cachedSecret *corev1.Secret - l *zap.SugaredLogger + tokenStore TokenStore + l *zap.SugaredLogger +} + +type TokenStore interface { + Add(ctx context.Context, shortenedToken string) error + Exists(ctx context.Context, shortenedToken string) (bool, error) } // NewBlocklist creates a new block list func NewBlocklist(ctx context.Context, kubeClient kubernetes.KubernetesConnector, logger *zap.SugaredLogger) (Blocklist, error) { - // read the existing blocklist token or create it - secret, err := kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + store, err := newTokenStore(ctx, kubeClient, logger) if err != nil { - if !k8serrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get secret: %w", err) - } - var createErr error - secret, createErr = kubeClient.CreateSecret(ctx, blockListSecretTemplate("")) - if createErr != nil { - return nil, fmt.Errorf("failed to create secret: %w", createErr) - } + return nil, err } return &blocklist{ - kubeClient: kubeClient, - contentProcessor: newContentProcessor(secret), - l: logger, + tokenStore: store, + l: logger, }, nil } @@ -68,26 +60,13 @@ func (b *blocklist) Block(ctx context.Context) error { } for attempts := 0; attempts < maxRetries; attempts++ { - secret, err := b.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) - if err != nil { - b.l.Errorf("failed to get %s secret: %v", common.EverestBlocklistSecretName, err) - return err - } - - secret, ok := b.contentProcessor.Block(b.l, secret, shortenedToken) - if !ok { - b.l.Infof("failed to block token, retrying") - continue - } - updatedSecret, updateErr := b.kubeClient.UpdateSecret(ctx, secret) - if updateErr != nil { - b.l.Errorf("failed to update %s secret, retrying: %v", common.EverestBlocklistSecretName, updateErr) + if err := b.tokenStore.Add(ctx, shortenedToken); err != nil { + b.l.Errorf("failed to add token to the blocklist: %v", err) continue } - b.contentProcessor.UpdateCache(updatedSecret) return nil } - return nil + return fmt.Errorf("failed to block token after %d attempts", maxRetries) } // IsBlocked checks if the token from the context is blocked. @@ -101,7 +80,7 @@ func (b *blocklist) IsBlocked(ctx context.Context) (bool, error) { return false, fmt.Errorf("failed to shorten token: %w", err) } - return b.contentProcessor.IsBlocked(shortenedToken), nil + return b.tokenStore.Exists(ctx, shortenedToken) } func extractToken(ctx context.Context) (*jwt.Token, error) { diff --git a/pkg/session/content_processor.go b/pkg/session/content_processor.go deleted file mode 100644 index 6de0a7317..000000000 --- a/pkg/session/content_processor.go +++ /dev/null @@ -1,102 +0,0 @@ -package session - -import ( - "strconv" - "strings" - "sync" - "time" - - "go.uber.org/zap" - corev1 "k8s.io/api/core/v1" -) - -const ( - sep = "," - timestampLen = 10 -) - -// ContentProcessor knows how to work with actual data in the blocklist. -type ContentProcessor interface { - // Block adds the token to the blacklist, cleans up the outdated data from the secret. - // The method returns the updated secret and a bool that indicates if the attempt was successful. - Block(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string) (*corev1.Secret, bool) - // IsBlocked returns true if the given shortened token is in the blocklist. - IsBlocked(shortenedToken string) bool - // UpdateCache updates the cached secret which is used for reading. - UpdateCache(secret *corev1.Secret) -} - -type contentProcessor struct { - mutex sync.Mutex - cachedSecret *corev1.Secret -} - -func newContentProcessor(secret *corev1.Secret) ContentProcessor { - return &contentProcessor{ - cachedSecret: secret, - } -} - -// Block adds the token to the blacklist, cleans up the outdated data from the secret. -// The method returns the updated secret and a bool that indicates if the attempt was successful. -func (d *contentProcessor) Block(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string) (*corev1.Secret, bool) { - if d.mutex.TryLock() { - defer d.mutex.Unlock() - updatedSecret := addDataToSecret(l, secret, shortenedToken, time.Now().UTC()) - return updatedSecret, true - } - return secret, false -} - -// IsBlocked returns true if the given shortened token is in the blocklist. -func (d *contentProcessor) IsBlocked(shortenedToken string) bool { - if d.cachedSecret == nil { - return false - } - list, ok := d.cachedSecret.Data[dataKey] - return ok && strings.Contains(string(list), shortenedToken) -} - -// UpdateCache updates the cached secret which is used for reading. -func (d *contentProcessor) UpdateCache(secret *corev1.Secret) { - d.cachedSecret = secret -} - -func addDataToSecret(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string, now time.Time) *corev1.Secret { - existingList, ok := secret.Data[dataKey] - if !ok { - secret.StringData = map[string]string{ - dataKey: shortenedToken, - } - return secret - } - list := append(cleanupOld(l, string(existingList), now), shortenedToken) - secret.StringData = map[string]string{ - dataKey: strings.Join(list, sep), - } - return secret -} - -func cleanupOld(l *zap.SugaredLogger, list string, now time.Time) []string { - tokens := strings.Split(list, sep) - newList := make([]string, 0, len(tokens)) - for _, shortenedToken := range tokens { - length := len(shortenedToken) - if length < timestampLen { - l.Info("blocklist contains irregular data format") - continue - } - ts := shortenedToken[length-10 : length] - tsInt, err := strconv.ParseInt(ts, 10, 64) - if err != nil { - l.Infof("failed to parse timestamp %v", tsInt) - continue - } - timeObj := time.Unix(tsInt, 0) - // only keep the tokens which natural expiration time is not over yet - if timeObj.After(now) { - newList = append(newList, shortenedToken) - } - } - return newList -} diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go new file mode 100644 index 000000000..13f34f307 --- /dev/null +++ b/pkg/session/token_store.go @@ -0,0 +1,118 @@ +package session + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + "github.com/percona/everest/pkg/common" + "github.com/percona/everest/pkg/kubernetes" +) + +const ( + sep = "," + timestampLen = 10 +) + +func newTokenStore(ctx context.Context, kubeClient kubernetes.KubernetesConnector, logger *zap.SugaredLogger) (TokenStore, error) { + store := &tokenStore{ + l: logger, + kubeClient: kubeClient, + } + _, err := kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + if err == nil { + return store, nil + } + if !k8serrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get secret: %w", err) + } + var createErr error + _, createErr = kubeClient.CreateSecret(ctx, blockListSecretTemplate("")) + if createErr != nil { + return nil, fmt.Errorf("failed to create secret: %w", createErr) + } + return store, nil +} + +type tokenStore struct { + kubeClient kubernetes.KubernetesConnector + l *zap.SugaredLogger + mutex sync.Mutex +} + +func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { + ts.mutex.Lock() + defer ts.mutex.Unlock() + + secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + if err != nil { + ts.l.Errorf("failed to get %s secret: %v", common.EverestBlocklistSecretName, err) + return err + } + + addDataToSecret(ts.l, secret, shortenedToken, time.Now().UTC()) + _, updateErr := ts.kubeClient.UpdateSecret(ctx, secret) + if updateErr != nil { + ts.l.Errorf("failed to update %s secret, retrying: %v", common.EverestBlocklistSecretName, updateErr) + return err + } + return nil +} + +func (ts *tokenStore) Exists(ctx context.Context, shortenedToken string) (bool, error) { + // no worries about k8s requests overhead - the controller-runtime cache is enabled for Everest API server + secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + if err != nil { + ts.l.Errorf("failed to get %s secret: %v", common.EverestBlocklistSecretName, err) + return false, err + } + list, ok := secret.Data[dataKey] + return ok && strings.Contains(string(list), shortenedToken), nil +} + +func addDataToSecret(l *zap.SugaredLogger, secret *corev1.Secret, shortenedToken string, now time.Time) *corev1.Secret { + existingList, ok := secret.Data[dataKey] + if !ok { + secret.StringData = map[string]string{ + dataKey: shortenedToken, + } + return secret + } + list := append(cleanupOld(l, string(existingList), now), shortenedToken) + secret.StringData = map[string]string{ + dataKey: strings.Join(list, sep), + } + return secret +} + +func cleanupOld(l *zap.SugaredLogger, list string, now time.Time) []string { + tokens := strings.Split(list, sep) + newList := make([]string, 0, len(tokens)) + for _, shortenedToken := range tokens { + length := len(shortenedToken) + if length < timestampLen { + l.Info("blocklist contains irregular data format") + continue + } + ts := shortenedToken[length-10 : length] + tsInt, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + l.Infof("failed to parse timestamp %v", tsInt) + continue + } + timeObj := time.Unix(tsInt, 0) + // only keep the tokens which natural expiration time is not over yet + if timeObj.After(now) { + newList = append(newList, shortenedToken) + } + } + return newList +} diff --git a/pkg/session/content_processor_test.go b/pkg/session/token_store_test.go similarity index 80% rename from pkg/session/content_processor_test.go rename to pkg/session/token_store_test.go index 0f4b216d3..11def18be 100644 --- a/pkg/session/content_processor_test.go +++ b/pkg/session/token_store_test.go @@ -181,43 +181,3 @@ func generateTestList(numTokens int) string { } return builder.String() } - -func TestContentProcessor_Block_ReturnsOriginalSecretWhenLocked(t *testing.T) { - l := zaptest.NewLogger(t).Sugar() - secret := &corev1.Secret{} - shortenedToken := "test-token" - processor := &contentProcessor{cachedSecret: blockListSecretTemplate("")} - - // Acquire the lock directly to simulate contention - processor.mutex.Lock() - defer processor.mutex.Unlock() - - // Call Block while the lock is held - returnedSecret, locked := processor.Block(l, secret, shortenedToken) - - if locked { - t.Errorf("Expected Block to return false when the lock is already held, but it returned true.") - } - - if returnedSecret != secret { - t.Errorf("Expected Block to return the original secret when the lock is already held, but it returned a different secret.") - } -} - -func TestContentProcessor_Block_ReturnsUpdatedSecretWhenUnlocked(t *testing.T) { - l := zaptest.NewLogger(t).Sugar() - secret := &corev1.Secret{} - shortenedToken := "test-token" - processor := &contentProcessor{cachedSecret: blockListSecretTemplate("")} - - // Call Block when the lock is free - returnedSecret, locked := processor.Block(l, secret, shortenedToken) - - if !locked { - t.Errorf("Expected Block to return true when the lock is free, but it returned false.") - } - - if returnedSecret == nil { - t.Errorf("Expected Block to return a non-nil secret, but it returned nil.") - } -} From d7eb85177a0659cefeae269c9dfa861d5562c2e7 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Tue, 22 Apr 2025 12:54:15 +0500 Subject: [PATCH 13/31] use RWMutex --- pkg/session/token_store.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index 13f34f307..cd0bb8faa 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -45,7 +45,7 @@ func newTokenStore(ctx context.Context, kubeClient kubernetes.KubernetesConnecto type tokenStore struct { kubeClient kubernetes.KubernetesConnector l *zap.SugaredLogger - mutex sync.Mutex + mutex sync.RWMutex } func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { @@ -68,6 +68,8 @@ func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { } func (ts *tokenStore) Exists(ctx context.Context, shortenedToken string) (bool, error) { + ts.mutex.RLock() + defer ts.mutex.RUnlock() // no worries about k8s requests overhead - the controller-runtime cache is enabled for Everest API server secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err != nil { From 246833ab1f992c8ab257b848a8f42709fa56ff70 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Wed, 23 Apr 2025 18:29:08 +0500 Subject: [PATCH 14/31] Update pkg/kubernetes/kubernetes.go Co-authored-by: Maxim Kondratenko --- pkg/kubernetes/kubernetes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index ba7f0863e..578997f75 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -158,7 +158,7 @@ func NewInClusterWithCache(l *zap.SugaredLogger, ctx context.Context) (Kubernete panic(err) } go func() { - l.Info("starting cache") + l.Info("starting session blocklist cache") if err := k8sCache.Start(ctx); err != nil { l.Errorf("error starting pod cache: %s", err) os.Exit(1) From 23019cd338c22bb7ab788dbd3ed81e2cc42fe26e Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Wed, 23 Apr 2025 18:29:36 +0500 Subject: [PATCH 15/31] Update pkg/kubernetes/kubernetes.go Co-authored-by: Maxim Kondratenko --- pkg/kubernetes/kubernetes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index 578997f75..ca8a7ea26 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -160,7 +160,7 @@ func NewInClusterWithCache(l *zap.SugaredLogger, ctx context.Context) (Kubernete go func() { l.Info("starting session blocklist cache") if err := k8sCache.Start(ctx); err != nil { - l.Errorf("error starting pod cache: %s", err) + l.Errorf("error starting session blocklist cache: %s", err) os.Exit(1) } }() From cea26e12aec774b8eb71fed8ee820dd50fc1c625 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Wed, 23 Apr 2025 19:41:55 +0500 Subject: [PATCH 16/31] address review comments --- api/everest-server.gen.go | 20 ++++++------- client/everest-client.gen.go | 16 +++++----- docs/spec/openapi.yml | 4 +-- pkg/session/blocklist.go | 37 ++++++++++++----------- pkg/session/jwt.go | 15 ++++++++++ pkg/session/jwt_test.go | 15 ++++++++++ pkg/session/token_store.go | 52 +++++++++++++++++++++++++-------- pkg/session/token_store_test.go | 15 ++++++++++ 8 files changed, 123 insertions(+), 51 deletions(-) diff --git a/api/everest-server.gen.go b/api/everest-server.gen.go index 8da6074f5..6cc956bc0 100644 --- a/api/everest-server.gen.go +++ b/api/everest-server.gen.go @@ -1697,10 +1697,10 @@ type ServerInterface interface { // Cluster resources // (GET /resources) GetKubernetesClusterResources(ctx echo.Context) error - // Everest UI Logout + // Everest API Logout // (DELETE /session) DeleteSession(ctx echo.Context) error - // Everest UI Login + // Everest API Login // (POST /session) CreateSession(ctx echo.Context) error // Settings @@ -2816,14 +2816,14 @@ var swaggerSpec = []string{ "8OfFeGDvbdB2O2z+3rvDEB7mW4xh/7mYic8OkocivQn0dTQbDvi5nD/crKMpZpqpkxo5P2SWV2uonVPp", "dplellMWjS3znP2wNCsmjJsYmZX7+y/+ROxTqfhvuJwL2+OeZ/VbncaiBU24WWOa5RXlGZ1n4IzGrrxK", "93M5Z0qA+8gfLIjjXtWwusXIzeoB0XDDqDuM3N7NWd2VFLbOo2MFaYd1mgHKDvJAcnFFM45W0mtUUOD5", - "f/7XOTHykon+64rO3DB38q29+PHhAXwuJcmpWBNqDMsLox/V1nqovzsmv8ilLM3WnObG+BnXugzhs7Cz", - "IECt5ocKLN6AZDlLDQ9ctmaIUAFXzEttyIq6e5Q+ZHLJxQfgW3OecbPuD1nVUeYB8iJ182Rpj30Ia2ie", - "vrtfM7BQdu2G49cA66jV7p+gcfotmYO/W6plSam4WY8O3l/00zAXt1IWNDOGi+UWth7ka7uvvFrgpwKm", - "ZJZhDDmmFpz54R5QCQhjDMbtDUCuTdgD9ycmmKIZnoFDKF4x5YXfcCC6j9owtM0QB2Is7W/40TGev3sw", - "GLphtgNhAJr/uh9mTYh/Gr1kVDFlEdRuwGe7BQACdG+WKhsdjPauno/sG9dnG8YWfmuzsnJFsQyy+Y1s", - "K621ggnOEVJTZLouzf4+2yceaz12DkPeqt/qtGG7W59jd4fZktpNsK77UJHkLt1WF1W7Xn193C06fdlO", - "D2l0RfwViEO7rBxdVVc1L9nQbmiTo4KZ1GCnofMhvLc7ap1AVO4GmcvS9PLXasQGcd0B2cjb2tkA13f1", - "6PPF5/8fAAD//z/XWzePSwEA", + "f/7XOTHykon+64rO3DB38q29+PHhAXwuJcmpWBNqDMsLox/V1tah/otcytJszWpuDKBxrcsQPwtbCxLU", + "qn6oweIVSJa11Kbk0jVDiArYYl5qQ1bUXaT0IZNLLj4A45rzjJt1f8yqjjMPkBipm0dLewxEWEPz+N39", + "2oGFsms3HL8GWEfNdv8ErdNvyR783ZItS0rFzXp08P5iAxFzcSt1QTNjuFhuYe1Bxrb7yisGfi5gTGYZ", + "RpFjisGZH+4B1YAwxmDk3gDl2oQ9cH9igima4Sk4hOIVU178DQei+6gNQ9sMkSDG0/6GHx3jCbwHg6Eb", + "ZjsQBqD5r/th1oT4p9FLRhVTFkHtBny2WwAgQAdnqbLRwWjv6vnIvnF9tmFs4bc2KytYFMsgn9/Ittpa", + "K5ngXCE1Vabr1Ozvs33msdZj5zjkrfqtzhu2u/VZdneYLandBeu6DzVJ7tJtdVW169VXyN2i05ftBJFG", + "V8Rfgji0y8rVVXVV85MN7YY2OSoYSg12Gjofwnu7o9YJROVukLksTS9/rUZsENcdkI28rZ0OcH1Xjz5f", + "fP7/AQAA//+k8Tx8kUsBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/client/everest-client.gen.go b/client/everest-client.gen.go index 898ab273a..b94cc5b5b 100644 --- a/client/everest-client.gen.go +++ b/client/everest-client.gen.go @@ -7475,14 +7475,14 @@ var swaggerSpec = []string{ "8OfFeGDvbdB2O2z+3rvDEB7mW4xh/7mYic8OkocivQn0dTQbDvi5nD/crKMpZpqpkxo5P2SWV2uonVPp", "dplellMWjS3znP2wNCsmjJsYmZX7+y/+ROxTqfhvuJwL2+OeZ/VbncaiBU24WWOa5RXlGZ1n4IzGrrxK", "93M5Z0qA+8gfLIjjXtWwusXIzeoB0XDDqDuM3N7NWd2VFLbOo2MFaYd1mgHKDvJAcnFFM45W0mtUUOD5", - "f/7XOTHykon+64rO3DB38q29+PHhAXwuJcmpWBNqDMsLox/V1nqovzsmv8ilLM3WnObG+BnXugzhs7Cz", - "IECt5ocKLN6AZDlLDQ9ctmaIUAFXzEttyIq6e5Q+ZHLJxQfgW3OecbPuD1nVUeYB8iJ182Rpj30Ia2ie", - "vrtfM7BQdu2G49cA66jV7p+gcfotmYO/W6plSam4WY8O3l/00zAXt1IWNDOGi+UWth7ka7uvvFrgpwKm", - "ZJZhDDmmFpz54R5QCQhjDMbtDUCuTdgD9ycmmKIZnoFDKF4x5YXfcCC6j9owtM0QB2Is7W/40TGev3sw", - "GLphtgNhAJr/uh9mTYh/Gr1kVDFlEdRuwGe7BQACdG+WKhsdjPauno/sG9dnG8YWfmuzsnJFsQyy+Y1s", - "K621ggnOEVJTZLouzf4+2yceaz12DkPeqt/qtGG7W59jd4fZktpNsK77UJHkLt1WF1W7Xn193C06fdlO", - "D2l0RfwViEO7rBxdVVc1L9nQbmiTo4KZ1GCnofMhvLc7ap1AVO4GmcvS9PLXasQGcd0B2cjb2tkA13f1", - "6PPF5/8fAAD//z/XWzePSwEA", + "f/7XOTHykon+64rO3DB38q29+PHhAXwuJcmpWBNqDMsLox/V1tah/otcytJszWpuDKBxrcsQPwtbCxLU", + "qn6oweIVSJa11Kbk0jVDiArYYl5qQ1bUXaT0IZNLLj4A45rzjJt1f8yqjjMPkBipm0dLewxEWEPz+N39", + "2oGFsms3HL8GWEfNdv8ErdNvyR783ZItS0rFzXp08P5iAxFzcSt1QTNjuFhuYe1Bxrb7yisGfi5gTGYZ", + "RpFjisGZH+4B1YAwxmDk3gDl2oQ9cH9igima4Sk4hOIVU178DQei+6gNQ9sMkSDG0/6GHx3jCbwHg6Eb", + "ZjsQBqD5r/th1oT4p9FLRhVTFkHtBny2WwAgQAdnqbLRwWjv6vnIvnF9tmFs4bc2KytYFMsgn9/Ittpa", + "K5ngXCE1Vabr1Ozvs33msdZj5zjkrfqtzhu2u/VZdneYLandBeu6DzVJ7tJtdVW169VXyN2i05ftBJFG", + "V8Rfgji0y8rVVXVV85MN7YY2OSoYSg12Gjofwnu7o9YJROVukLksTS9/rUZsENcdkI28rZ0OcH1Xjz5f", + "fP7/AQAA//+k8Tx8kUsBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/docs/spec/openapi.yml b/docs/spec/openapi.yml index 24c8c4aec..ddd78ff07 100644 --- a/docs/spec/openapi.yml +++ b/docs/spec/openapi.yml @@ -38,7 +38,7 @@ paths: tags: - Authentication & Authorization security: [] - summary: Everest UI Login + summary: Everest API Login description: | This API issues a new JWT token for logging in from the Everest API. The provided user must have the `login` capability. @@ -81,7 +81,7 @@ paths: delete: tags: - Authentication & Authorization - summary: Everest UI Logout + summary: Everest API Logout description: | This API invalidates Everest API JWT token. operationId: deleteSession diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 32b366d67..6417e2d7f 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -1,3 +1,18 @@ +// everest +// Copyright (C) 2025 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package session import ( @@ -6,8 +21,6 @@ import ( "github.com/golang-jwt/jwt/v5" "go.uber.org/zap" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/percona/everest/pkg/common" "github.com/percona/everest/pkg/kubernetes" @@ -32,7 +45,9 @@ type blocklist struct { } type TokenStore interface { + // Add adds the shortened token to the blocklist Add(ctx context.Context, shortenedToken string) error + // Exists checks if the shortened token is in the blocklist Exists(ctx context.Context, shortenedToken string) (bool, error) } @@ -61,7 +76,7 @@ func (b *blocklist) Block(ctx context.Context) error { for attempts := 0; attempts < maxRetries; attempts++ { if err := b.tokenStore.Add(ctx, shortenedToken); err != nil { - b.l.Errorf("failed to add token to the blocklist: %v", err) + b.l.Errorf("failed to add shortened token %s to the blocklist: %v", shortenedToken, err) continue } return nil @@ -90,19 +105,3 @@ func extractToken(ctx context.Context) (*jwt.Token, error) { } return token, nil } - -func blockListSecretTemplate(stringData string) *corev1.Secret { - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: common.EverestBlocklistSecretName, - Namespace: common.SystemNamespace, - }, - StringData: map[string]string{ - dataKey: stringData, - }, - } -} diff --git a/pkg/session/jwt.go b/pkg/session/jwt.go index b40ebe937..8d1f3f6cf 100644 --- a/pkg/session/jwt.go +++ b/pkg/session/jwt.go @@ -1,3 +1,18 @@ +// everest +// Copyright (C) 2025 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package session import ( diff --git a/pkg/session/jwt_test.go b/pkg/session/jwt_test.go index f9581c28f..69d4aa3ac 100644 --- a/pkg/session/jwt_test.go +++ b/pkg/session/jwt_test.go @@ -1,3 +1,18 @@ +// everest +// Copyright (C) 2025 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package session import ( diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index cd0bb8faa..a3721b42b 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -1,3 +1,18 @@ +// everest +// Copyright (C) 2025 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package session import ( @@ -5,12 +20,12 @@ import ( "fmt" "strconv" "strings" - "sync" "time" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "github.com/percona/everest/pkg/common" @@ -35,9 +50,10 @@ func newTokenStore(ctx context.Context, kubeClient kubernetes.KubernetesConnecto return nil, fmt.Errorf("failed to get secret: %w", err) } var createErr error - _, createErr = kubeClient.CreateSecret(ctx, blockListSecretTemplate("")) + secret := getBlockListSecretTemplate("") + _, createErr = kubeClient.CreateSecret(ctx, secret) if createErr != nil { - return nil, fmt.Errorf("failed to create secret: %w", createErr) + return nil, fmt.Errorf("failed to create secret %s in namespace %s: %w", secret.Name, secret.Namespace, createErr) } return store, nil } @@ -45,31 +61,27 @@ func newTokenStore(ctx context.Context, kubeClient kubernetes.KubernetesConnecto type tokenStore struct { kubeClient kubernetes.KubernetesConnector l *zap.SugaredLogger - mutex sync.RWMutex } +// Add adds the shortened token to the blocklist func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { - ts.mutex.Lock() - defer ts.mutex.Unlock() - secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err != nil { - ts.l.Errorf("failed to get %s secret: %v", common.EverestBlocklistSecretName, err) + ts.l.Errorf("failed to get %s secret in the %s namespace: %v", secret.Name, secret.Namespace, err) return err } - addDataToSecret(ts.l, secret, shortenedToken, time.Now().UTC()) + secret = addDataToSecret(ts.l, secret, shortenedToken, time.Now().UTC()) _, updateErr := ts.kubeClient.UpdateSecret(ctx, secret) if updateErr != nil { - ts.l.Errorf("failed to update %s secret, retrying: %v", common.EverestBlocklistSecretName, updateErr) + ts.l.Errorf("failed to update %s secret in the %s namespace, retrying: %v", secret.Name, secret.Namespace, updateErr) return err } return nil } +// Exists checks if the shortened token is in the blocklist func (ts *tokenStore) Exists(ctx context.Context, shortenedToken string) (bool, error) { - ts.mutex.RLock() - defer ts.mutex.RUnlock() // no worries about k8s requests overhead - the controller-runtime cache is enabled for Everest API server secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err != nil { @@ -118,3 +130,19 @@ func cleanupOld(l *zap.SugaredLogger, list string, now time.Time) []string { } return newList } + +func getBlockListSecretTemplate(stringData string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: common.EverestBlocklistSecretName, + Namespace: common.SystemNamespace, + }, + StringData: map[string]string{ + dataKey: stringData, + }, + } +} diff --git a/pkg/session/token_store_test.go b/pkg/session/token_store_test.go index 11def18be..7dcfde3a6 100644 --- a/pkg/session/token_store_test.go +++ b/pkg/session/token_store_test.go @@ -1,3 +1,18 @@ +// everest +// Copyright (C) 2025 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package session import ( From fb874f55b2f8270236cefb7f11f0fbbceed79a3e Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Wed, 23 Apr 2025 20:14:19 +0500 Subject: [PATCH 17/31] add tests --- pkg/session/blocklist_test.go | 107 ++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 pkg/session/blocklist_test.go diff --git a/pkg/session/blocklist_test.go b/pkg/session/blocklist_test.go new file mode 100644 index 000000000..3f9095bd5 --- /dev/null +++ b/pkg/session/blocklist_test.go @@ -0,0 +1,107 @@ +// everest +// Copyright (C) 2025 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package session + +import ( + "context" + "testing" + + "github.com/golang-jwt/jwt/v5" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/percona/everest/pkg/common" + "github.com/percona/everest/pkg/kubernetes" +) + +func TestBlocklist_Block(t *testing.T) { + objs := []ctrlclient.Object{ + getBlockListSecretTemplate(""), + } + + mockClient := fakeclient.NewClientBuilder().WithScheme(kubernetes.CreateScheme()) + mockClient.WithObjects(objs...) + l := zap.NewNop().Sugar() + k := kubernetes.NewEmpty(l).WithKubernetesClient(mockClient.Build()) + ctx := context.WithValue( + context.Background(), + common.UserCtxKey, + jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", "exp": float64(331743679478)}), + ) + + b, err := NewBlocklist(ctx, k, l) + assert.NoError(t, err) + + err = b.Block(ctx) + assert.NoError(t, err) + + secret, err := k.GetSecret(ctx, ctrlclient.ObjectKey{ + Name: common.EverestBlocklistSecretName, + Namespace: common.SystemNamespace, + }) + assert.NoError(t, err) + // the mocked client does not do this StringData -> Data transformation in Secrets which the actual k8a API do, so + // we only check the StringData field + assert.Equal(t, "9d1c1f98-a479-41e3-8939-c7cb3e049a331743679478", secret.StringData[dataKey]) +} + +func TestBlocklist_IsBlocked(t *testing.T) { + secret := getBlockListSecretTemplate("the-blocked-jti331743679478") + // when writing Secrets, the mocked client does not do this StringData -> Data transformation which the actual k8a API do, + // so we need to set the Data field manually + secret.Data = map[string][]byte{dataKey: []byte("the-blocked-jti331743679478")} + + objs := []ctrlclient.Object{secret} + + mockClient := fakeclient.NewClientBuilder().WithScheme(kubernetes.CreateScheme()) + mockClient.WithObjects(objs...) + l := zap.NewNop().Sugar() + k := kubernetes.NewEmpty(l).WithKubernetesClient(mockClient.Build()) + + t.Run("blocked token in context", func(t *testing.T) { + ctx := context.WithValue( + context.Background(), + common.UserCtxKey, + jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "the-blocked-jti", "exp": float64(331743679478)}), + ) + + b, err := NewBlocklist(ctx, k, l) + assert.NoError(t, err) + + blocked, err := b.IsBlocked(ctx) + assert.NoError(t, err) + assert.True(t, blocked) + + }) + + t.Run("not blocked token in context", func(t *testing.T) { + ctx := context.WithValue( + context.Background(), + common.UserCtxKey, + jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "some-other-jti", "exp": float64(331743679478)}), + ) + + b, err := NewBlocklist(ctx, k, l) + assert.NoError(t, err) + + blocked, err := b.IsBlocked(ctx) + assert.NoError(t, err) + assert.False(t, blocked) + }) + +} From 749b2a31d079fb9eed791cba120ec20ecc28a23c Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Thu, 24 Apr 2025 15:54:27 +0500 Subject: [PATCH 18/31] address review comments --- pkg/session/blocklist.go | 25 ++++++++------ pkg/session/blocklist_test.go | 63 ++++++++++++++++++++++------------- pkg/session/token_store.go | 2 +- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 6417e2d7f..37d9575a1 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -18,7 +18,9 @@ package session import ( "context" "fmt" + "time" + "github.com/cenkalti/backoff/v4" "github.com/golang-jwt/jwt/v5" "go.uber.org/zap" @@ -27,8 +29,9 @@ import ( ) const ( - dataKey = "list" - maxRetries = 10 + dataKey = "list" + maxRetries = 10 + backoffInterval = 500 * time.Millisecond ) // Blocklist represents interface to block JWT tokens and check if a token is blocked. @@ -74,14 +77,16 @@ func (b *blocklist) Block(ctx context.Context) error { return err } - for attempts := 0; attempts < maxRetries; attempts++ { - if err := b.tokenStore.Add(ctx, shortenedToken); err != nil { - b.l.Errorf("failed to add shortened token %s to the blocklist: %v", shortenedToken, err) - continue - } - return nil - } - return fmt.Errorf("failed to block token after %d attempts", maxRetries) + var bOff backoff.BackOff + bOff = backoff.NewConstantBackOff(backoffInterval) + bOff = backoff.WithMaxRetries(bOff, maxRetries) + bOff = backoff.WithContext(bOff, ctx) + return backoff.Retry( + func() error { + return b.tokenStore.Add(ctx, shortenedToken) + }, + bOff, + ) } // IsBlocked checks if the token from the context is blocked. diff --git a/pkg/session/blocklist_test.go b/pkg/session/blocklist_test.go index 3f9095bd5..fa922bf95 100644 --- a/pkg/session/blocklist_test.go +++ b/pkg/session/blocklist_test.go @@ -22,6 +22,7 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/stretchr/testify/assert" "go.uber.org/zap" + k8serrors "k8s.io/apimachinery/pkg/api/errors" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -30,27 +31,36 @@ import ( ) func TestBlocklist_Block(t *testing.T) { - objs := []ctrlclient.Object{ - getBlockListSecretTemplate(""), - } - mockClient := fakeclient.NewClientBuilder().WithScheme(kubernetes.CreateScheme()) - mockClient.WithObjects(objs...) l := zap.NewNop().Sugar() k := kubernetes.NewEmpty(l).WithKubernetesClient(mockClient.Build()) - ctx := context.WithValue( - context.Background(), - common.UserCtxKey, - jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", "exp": float64(331743679478)}), - ) + ctx := backgroundContextWithToken(jwt.MapClaims{"jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", "exp": float64(331743679478)}) + + // check there is no blocklist secret before the blocklist creation + secret, err := k.GetSecret(ctx, ctrlclient.ObjectKey{ + Name: common.EverestBlocklistSecretName, + Namespace: common.SystemNamespace, + }) + assert.True(t, k8serrors.IsNotFound(err)) + assert.Nil(t, secret) b, err := NewBlocklist(ctx, k, l) assert.NoError(t, err) + // blocklist secret appears after the blocklist creation + secret, err = k.GetSecret(ctx, ctrlclient.ObjectKey{ + Name: common.EverestBlocklistSecretName, + Namespace: common.SystemNamespace, + }) + assert.NoError(t, err) + assert.NotNil(t, secret) + assert.Equal(t, "", secret.StringData[dataKey]) + + // block the token from the context and check the secret has been changed accordingly err = b.Block(ctx) assert.NoError(t, err) - secret, err := k.GetSecret(ctx, ctrlclient.ObjectKey{ + secret, err = k.GetSecret(ctx, ctrlclient.ObjectKey{ Name: common.EverestBlocklistSecretName, Namespace: common.SystemNamespace, }) @@ -58,14 +68,21 @@ func TestBlocklist_Block(t *testing.T) { // the mocked client does not do this StringData -> Data transformation in Secrets which the actual k8a API do, so // we only check the StringData field assert.Equal(t, "9d1c1f98-a479-41e3-8939-c7cb3e049a331743679478", secret.StringData[dataKey]) + + // deleting secret to test the backoff + err = k.DeleteSecret(ctx, secret) + assert.NoError(t, err) + + // after deleting secret - try to block again, get the NotFound error + err = b.Block(ctx) + assert.Equal(t, true, k8serrors.IsNotFound(err)) } func TestBlocklist_IsBlocked(t *testing.T) { secret := getBlockListSecretTemplate("the-blocked-jti331743679478") // when writing Secrets, the mocked client does not do this StringData -> Data transformation which the actual k8a API do, - // so we need to set the Data field manually + // so we set the Data field manually secret.Data = map[string][]byte{dataKey: []byte("the-blocked-jti331743679478")} - objs := []ctrlclient.Object{secret} mockClient := fakeclient.NewClientBuilder().WithScheme(kubernetes.CreateScheme()) @@ -74,11 +91,7 @@ func TestBlocklist_IsBlocked(t *testing.T) { k := kubernetes.NewEmpty(l).WithKubernetesClient(mockClient.Build()) t.Run("blocked token in context", func(t *testing.T) { - ctx := context.WithValue( - context.Background(), - common.UserCtxKey, - jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "the-blocked-jti", "exp": float64(331743679478)}), - ) + ctx := backgroundContextWithToken(jwt.MapClaims{"jti": "the-blocked-jti", "exp": float64(331743679478)}) b, err := NewBlocklist(ctx, k, l) assert.NoError(t, err) @@ -86,15 +99,10 @@ func TestBlocklist_IsBlocked(t *testing.T) { blocked, err := b.IsBlocked(ctx) assert.NoError(t, err) assert.True(t, blocked) - }) t.Run("not blocked token in context", func(t *testing.T) { - ctx := context.WithValue( - context.Background(), - common.UserCtxKey, - jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "some-other-jti", "exp": float64(331743679478)}), - ) + ctx := backgroundContextWithToken(jwt.MapClaims{"jti": "some-other-jti", "exp": float64(331743679478)}) b, err := NewBlocklist(ctx, k, l) assert.NoError(t, err) @@ -103,5 +111,12 @@ func TestBlocklist_IsBlocked(t *testing.T) { assert.NoError(t, err) assert.False(t, blocked) }) +} +func backgroundContextWithToken(claims jwt.MapClaims) context.Context { + return context.WithValue( + context.Background(), + common.UserCtxKey, + jwt.NewWithClaims(jwt.SigningMethodHS256, claims), + ) } diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index a3721b42b..187977d15 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -67,7 +67,7 @@ type tokenStore struct { func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err != nil { - ts.l.Errorf("failed to get %s secret in the %s namespace: %v", secret.Name, secret.Namespace, err) + ts.l.Errorf("failed to get %s secret in the %s namespace: %v", common.EverestBlocklistSecretName, common.SystemNamespace, err) return err } From bea23f45a4479d8336dd9ffd29b1db56828116cd Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Thu, 24 Apr 2025 16:40:11 +0500 Subject: [PATCH 19/31] do not use controller-runtime cache --- internal/server/everest.go | 2 +- pkg/kubernetes/kubernetes.go | 51 +++--------------------------------- pkg/session/token_store.go | 18 ++++++++----- 3 files changed, 15 insertions(+), 56 deletions(-) diff --git a/internal/server/everest.go b/internal/server/everest.go index 39b2cbe17..d1d7b7702 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -91,7 +91,7 @@ func getOIDCProviderConfig(ctx context.Context, kubeClient kubernetes.Kubernetes // NewEverestServer creates and configures everest API. func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.SugaredLogger) (*EverestServer, error) { - kubeConnector, err := kubernetes.NewInClusterWithCache(l, ctx) + kubeConnector, err := kubernetes.NewInCluster(l) if err != nil { return nil, errors.Join(err, errors.New("failed creating Kubernetes client")) } diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index ba7f0863e..b34a4efa0 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -38,7 +38,6 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1" @@ -133,7 +132,9 @@ func New(kubeconfigPath string, l *zap.SugaredLogger) (KubernetesConnector, erro // NewInCluster creates a new kubernetes client using incluster authentication. func NewInCluster(l *zap.SugaredLogger) (KubernetesConnector, error) { - restConfig := inClusterRestConfig() + restConfig := ctrl.GetConfigOrDie() + restConfig.QPS = defaultQPSLimit + restConfig.Burst = defaultBurstLimit k8sclient, err := ctrlclient.New(restConfig, getKubernetesClientOptions()) if err != nil { @@ -147,42 +148,6 @@ func NewInCluster(l *zap.SugaredLogger) (KubernetesConnector, error) { }, nil } -// NewInClusterWithCache creates a new kubernetes client using incluster authentication with cache enabled -func NewInClusterWithCache(l *zap.SugaredLogger, ctx context.Context) (KubernetesConnector, error) { - restConfig := inClusterRestConfig() - cacheOptions := cache.Options{ - Scheme: CreateScheme(), - } - k8sCache, err := cache.New(restConfig, cacheOptions) - if err != nil { - panic(err) - } - go func() { - l.Info("starting cache") - if err := k8sCache.Start(ctx); err != nil { - l.Errorf("error starting pod cache: %s", err) - os.Exit(1) - } - }() - k8sclient, err := ctrlclient.New(restConfig, getKubernetesClientOptionsWithCache(k8sCache)) - if err != nil { - return nil, err - } - - return &Kubernetes{ - k8sClient: k8sclient, - l: l.With("component", "kubernetes"), - restConfig: restConfig, - }, nil -} - -func inClusterRestConfig() *rest.Config { - restConfig := ctrl.GetConfigOrDie() - restConfig.QPS = defaultQPSLimit - restConfig.Burst = defaultBurstLimit - return restConfig -} - // CreateScheme creates a new runtime.Scheme. // It registers all necessary types: // - standard client-go types @@ -205,16 +170,6 @@ func getKubernetesClientOptions() ctrlclient.Options { } } -func getKubernetesClientOptionsWithCache(k8sCache cache.Cache) ctrlclient.Options { - return ctrlclient.Options{ - Scheme: CreateScheme(), - Cache: &ctrlclient.CacheOptions{ - Reader: k8sCache, - Unstructured: false, - }, - } -} - func (k *Kubernetes) getDiscoveryClient() discovery.DiscoveryInterface { once.Do(func() { httpClient, err := rest.HTTPClientFor(k.restConfig) diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index 187977d15..bc109bb70 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -61,6 +61,7 @@ func newTokenStore(ctx context.Context, kubeClient kubernetes.KubernetesConnecto type tokenStore struct { kubeClient kubernetes.KubernetesConnector l *zap.SugaredLogger + secret *corev1.Secret } // Add adds the shortened token to the blocklist @@ -72,23 +73,26 @@ func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { } secret = addDataToSecret(ts.l, secret, shortenedToken, time.Now().UTC()) - _, updateErr := ts.kubeClient.UpdateSecret(ctx, secret) + updatedSecret, updateErr := ts.kubeClient.UpdateSecret(ctx, secret) if updateErr != nil { ts.l.Errorf("failed to update %s secret in the %s namespace, retrying: %v", secret.Name, secret.Namespace, updateErr) return err } + ts.secret = updatedSecret return nil } // Exists checks if the shortened token is in the blocklist func (ts *tokenStore) Exists(ctx context.Context, shortenedToken string) (bool, error) { - // no worries about k8s requests overhead - the controller-runtime cache is enabled for Everest API server - secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) - if err != nil { - ts.l.Errorf("failed to get %s secret: %v", common.EverestBlocklistSecretName, err) - return false, err + if ts.secret == nil { + secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + if err != nil { + ts.l.Errorf("failed to get %s secret in the %s namespace: %v", common.EverestBlocklistSecretName, common.SystemNamespace, err) + return false, err + } + ts.secret = secret } - list, ok := secret.Data[dataKey] + list, ok := ts.secret.Data[dataKey] return ok && strings.Contains(string(list), shortenedToken), nil } From f3bb4e39af86ae70241b083c4ad8d2fae5019171 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Thu, 24 Apr 2025 16:50:40 +0500 Subject: [PATCH 20/31] fix merge conflicts --- pkg/kubernetes/kubernetes.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index 911aa735f..b34a4efa0 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -148,42 +148,6 @@ func NewInCluster(l *zap.SugaredLogger) (KubernetesConnector, error) { }, nil } -// NewInClusterWithCache creates a new kubernetes client using incluster authentication with cache enabled -func NewInClusterWithCache(l *zap.SugaredLogger, ctx context.Context) (KubernetesConnector, error) { - restConfig := inClusterRestConfig() - cacheOptions := cache.Options{ - Scheme: CreateScheme(), - } - k8sCache, err := cache.New(restConfig, cacheOptions) - if err != nil { - panic(err) - } - go func() { - l.Info("starting session blocklist cache") - if err := k8sCache.Start(ctx); err != nil { - l.Errorf("error starting session blocklist cache: %s", err) - os.Exit(1) - } - }() - k8sclient, err := ctrlclient.New(restConfig, getKubernetesClientOptionsWithCache(k8sCache)) - if err != nil { - return nil, err - } - - return &Kubernetes{ - k8sClient: k8sclient, - l: l.With("component", "kubernetes"), - restConfig: restConfig, - }, nil -} - -func inClusterRestConfig() *rest.Config { - restConfig := ctrl.GetConfigOrDie() - restConfig.QPS = defaultQPSLimit - restConfig.Burst = defaultBurstLimit - return restConfig -} - // CreateScheme creates a new runtime.Scheme. // It registers all necessary types: // - standard client-go types From ded35e69da0fea018ca866c72d0b7b40f624969e Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Thu, 24 Apr 2025 17:01:10 +0500 Subject: [PATCH 21/31] upd error message --- pkg/session/token_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index bc109bb70..fb6605bb1 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -47,7 +47,7 @@ func newTokenStore(ctx context.Context, kubeClient kubernetes.KubernetesConnecto return store, nil } if !k8serrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get secret: %w", err) + return nil, fmt.Errorf("failed to get %s secret in the %s namespace: %w", common.EverestBlocklistSecretName, common.SystemNamespace, err) } var createErr error secret := getBlockListSecretTemplate("") From 3d0f4ee1c2345ef3ee770db56801df1552c8c495 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Thu, 24 Apr 2025 19:05:53 +0500 Subject: [PATCH 22/31] use restricted controller-runtime cache --- internal/server/everest.go | 13 ++++++++++++- pkg/cli/upgrade/upgrade.go | 2 +- pkg/kubernetes/kubernetes.go | 32 +++++++++++++++++++++++++++----- pkg/session/token_store.go | 18 +++++++----------- 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/internal/server/everest.go b/internal/server/everest.go index d1d7b7702..a0220df7c 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -37,8 +37,12 @@ import ( "github.com/unrolled/secure" "go.uber.org/zap" "golang.org/x/time/rate" + corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/percona/everest/api" "github.com/percona/everest/cmd/config" @@ -91,7 +95,14 @@ func getOIDCProviderConfig(ctx context.Context, kubeClient kubernetes.Kubernetes // NewEverestServer creates and configures everest API. func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.SugaredLogger) (*EverestServer, error) { - kubeConnector, err := kubernetes.NewInCluster(l) + options := &cache.Options{ + ByObject: map[ctrlclient.Object]cache.ByObject{ + &corev1.Secret{}: { + Field: fields.SelectorFromSet(fields.Set{"metadata.name": common.EverestBlocklistSecretName}), + }, + }, + } + kubeConnector, err := kubernetes.NewInCluster(l, ctx, options) if err != nil { return nil, errors.Join(err, errors.New("failed creating Kubernetes client")) } diff --git a/pkg/cli/upgrade/upgrade.go b/pkg/cli/upgrade/upgrade.go index 8e84b0b5a..4fcd42c93 100644 --- a/pkg/cli/upgrade/upgrade.go +++ b/pkg/cli/upgrade/upgrade.go @@ -102,7 +102,7 @@ func NewUpgrade(cfg *Config, l *zap.SugaredLogger) (*Upgrade, error) { var kubeClient kubernetes.KubernetesConnector if cfg.InCluster { - k, err := kubernetes.NewInCluster(cli.l) + k, err := kubernetes.NewInCluster(cli.l, context.Background(), nil) if err != nil { return nil, fmt.Errorf("could not create in-cluster kubernetes client: %w", err) } diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index b34a4efa0..8e8418967 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -38,6 +38,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1" @@ -117,7 +118,7 @@ func New(kubeconfigPath string, l *zap.SugaredLogger) (KubernetesConnector, erro restConfig.QPS = defaultQPSLimit restConfig.Burst = defaultBurstLimit - k8client, err := ctrlclient.New(restConfig, getKubernetesClientOptions()) + k8client, err := ctrlclient.New(restConfig, getKubernetesClientOptions(nil)) if err != nil { return nil, err } @@ -131,12 +132,28 @@ func New(kubeconfigPath string, l *zap.SugaredLogger) (KubernetesConnector, erro } // NewInCluster creates a new kubernetes client using incluster authentication. -func NewInCluster(l *zap.SugaredLogger) (KubernetesConnector, error) { +func NewInCluster(l *zap.SugaredLogger, ctx context.Context, cacheOptions *cache.Options) (KubernetesConnector, error) { restConfig := ctrl.GetConfigOrDie() restConfig.QPS = defaultQPSLimit restConfig.Burst = defaultBurstLimit - k8sclient, err := ctrlclient.New(restConfig, getKubernetesClientOptions()) + var k8sCache cache.Cache + var err error + if cacheOptions != nil { + k8sCache, err = cache.New(restConfig, *cacheOptions) + if err != nil { + panic(err) + } + go func() { + l.Info("starting incluster client cache") + if err := k8sCache.Start(ctx); err != nil { + l.Errorf("error starting incluster client cache: %s", err) + os.Exit(1) + } + }() + } + + k8sclient, err := ctrlclient.New(restConfig, getKubernetesClientOptions(k8sCache)) if err != nil { return nil, err } @@ -163,10 +180,15 @@ func CreateScheme() *runtime.Scheme { return scheme } -func getKubernetesClientOptions() ctrlclient.Options { +func getKubernetesClientOptions(cache cache.Cache) ctrlclient.Options { + var cacheOptions *ctrlclient.CacheOptions + if cache != nil { + cacheOptions = &ctrlclient.CacheOptions{Reader: cache} + } + return ctrlclient.Options{ Scheme: CreateScheme(), - Cache: nil, // disable cache + Cache: cacheOptions, } } diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index fb6605bb1..95f71e9c6 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -61,7 +61,6 @@ func newTokenStore(ctx context.Context, kubeClient kubernetes.KubernetesConnecto type tokenStore struct { kubeClient kubernetes.KubernetesConnector l *zap.SugaredLogger - secret *corev1.Secret } // Add adds the shortened token to the blocklist @@ -73,26 +72,23 @@ func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { } secret = addDataToSecret(ts.l, secret, shortenedToken, time.Now().UTC()) - updatedSecret, updateErr := ts.kubeClient.UpdateSecret(ctx, secret) + _, updateErr := ts.kubeClient.UpdateSecret(ctx, secret) if updateErr != nil { ts.l.Errorf("failed to update %s secret in the %s namespace, retrying: %v", secret.Name, secret.Namespace, updateErr) return err } - ts.secret = updatedSecret return nil } // Exists checks if the shortened token is in the blocklist func (ts *tokenStore) Exists(ctx context.Context, shortenedToken string) (bool, error) { - if ts.secret == nil { - secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) - if err != nil { - ts.l.Errorf("failed to get %s secret in the %s namespace: %v", common.EverestBlocklistSecretName, common.SystemNamespace, err) - return false, err - } - ts.secret = secret + // no worries about overwhelming k8s API - the secret is cached + secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + if err != nil { + ts.l.Errorf("failed to get %s secret in the %s namespace: %v", common.EverestBlocklistSecretName, common.SystemNamespace, err) + return false, err } - list, ok := ts.secret.Data[dataKey] + list, ok := secret.Data[dataKey] return ok && strings.Contains(string(list), shortenedToken), nil } From 1485c76d21ec2f55b9954d30c270d2d8ab5c5444 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:18:21 +0500 Subject: [PATCH 23/31] Update pkg/session/token_store.go Co-authored-by: Maxim Kondratenko --- pkg/session/token_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index fb6605bb1..17cc917ab 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -117,7 +117,7 @@ func cleanupOld(l *zap.SugaredLogger, list string, now time.Time) []string { for _, shortenedToken := range tokens { length := len(shortenedToken) if length < timestampLen { - l.Info("blocklist contains irregular data format") + l.Warn("blocklist contains irregular data format") continue } ts := shortenedToken[length-10 : length] From 5fb3c5561ef0511d696c9de896e067b0373a2eb8 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:19:37 +0500 Subject: [PATCH 24/31] Update pkg/session/token_store.go Co-authored-by: Maxim Kondratenko --- pkg/session/token_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index 17cc917ab..2f445389d 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -123,7 +123,7 @@ func cleanupOld(l *zap.SugaredLogger, list string, now time.Time) []string { ts := shortenedToken[length-10 : length] tsInt, err := strconv.ParseInt(ts, 10, 64) if err != nil { - l.Infof("failed to parse timestamp %v", tsInt) + l.Warnf("failed to parse timestamp %v", tsInt) continue } timeObj := time.Unix(tsInt, 0) From 9ba1ba9e5b20c6925f1b219b9dce6db67d17f3c7 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Fri, 25 Apr 2025 18:31:41 +0500 Subject: [PATCH 25/31] use separate controller-runtime cache & address comments --- internal/server/everest.go | 40 ++++++----- internal/server/session.go | 8 ++- pkg/common/accounts.go | 12 ++++ pkg/kubernetes/kubernetes.go | 4 +- pkg/session/blocklist.go | 93 +++++++++++++++++++------ pkg/session/blocklist_test.go | 118 ++++++++++++++++++++++++++++---- pkg/session/jwt.go | 73 -------------------- pkg/session/jwt_test.go | 115 ------------------------------- pkg/session/manager.go | 4 +- pkg/session/token_store.go | 45 +++++++----- pkg/session/token_store_test.go | 49 +++++++++++++ 11 files changed, 301 insertions(+), 260 deletions(-) delete mode 100644 pkg/session/jwt.go delete mode 100644 pkg/session/jwt_test.go diff --git a/internal/server/everest.go b/internal/server/everest.go index a0220df7c..ad7db877e 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -42,7 +42,6 @@ import ( "k8s.io/apimachinery/pkg/fields" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/percona/everest/api" "github.com/percona/everest/cmd/config" @@ -66,7 +65,6 @@ type EverestServer struct { sessionMgr *session.Manager attemptsStore *RateLimiterMemoryStore handler handlers.Handler - blocklist session.Blocklist oidcProvider *oidc.ProviderConfig } @@ -95,23 +93,34 @@ func getOIDCProviderConfig(ctx context.Context, kubeClient kubernetes.Kubernetes // NewEverestServer creates and configures everest API. func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.SugaredLogger) (*EverestServer, error) { + kubeConnector, err := kubernetes.NewInCluster(l, ctx, nil) + if err != nil { + return nil, errors.Join(err, errors.New("failed creating Kubernetes client")) + } + + echoServer := echo.New() + echoServer.Use(echomiddleware.RateLimiter(echomiddleware.NewRateLimiterMemoryStore(rate.Limit(c.APIRequestsRateLimit)))) + middleware, store := sessionRateLimiter(c.CreateSessionRateLimit) + echoServer.Use(middleware) + options := &cache.Options{ - ByObject: map[ctrlclient.Object]cache.ByObject{ + ByObject: map[client.Object]cache.ByObject{ &corev1.Secret{}: { Field: fields.SelectorFromSet(fields.Set{"metadata.name": common.EverestBlocklistSecretName}), }, }, } - kubeConnector, err := kubernetes.NewInCluster(l, ctx, options) + blocklistClient, err := kubernetes.NewInCluster(l, ctx, options) if err != nil { - return nil, errors.Join(err, errors.New("failed creating Kubernetes client")) + return nil, err } - echoServer := echo.New() - echoServer.Use(echomiddleware.RateLimiter(echomiddleware.NewRateLimiterMemoryStore(rate.Limit(c.APIRequestsRateLimit)))) - middleware, store := sessionRateLimiter(c.CreateSessionRateLimit) - echoServer.Use(middleware) + blockList, err := session.NewBlocklist(ctx, blocklistClient, l) + if err != nil { + return nil, errors.Join(err, errors.New("failed to configure tokens blocklist")) + } sessMgr, err := session.New( + blockList, session.WithAccountManager(kubeConnector.Accounts()), ) if err != nil { @@ -123,11 +132,6 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar return nil, errors.Join(err, errors.New("failed to get OIDC provider config")) } - blockList, err := session.NewBlocklist(ctx, kubeConnector, l) - if err != nil { - return nil, errors.Join(err, errors.New("failed to configure tokens blocklist")) - } - e := &EverestServer{ config: c, l: l, @@ -135,7 +139,6 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar kubeConnector: kubeConnector, sessionMgr: sessMgr, attemptsStore: store, - blocklist: blockList, oidcProvider: oidcProvider, } e.echo.HTTPErrorHandler = e.errorHandlerChain() @@ -452,7 +455,12 @@ func (e *EverestServer) blocklistMiddleWare() (echo.MiddlewareFunc, error) { if skipper(c) { return next(c) } - if isBlocked, err := e.blocklist.IsBlocked(c.Request().Context()); err != nil { + ctx := c.Request().Context() + token, tErr := common.ExtractToken(ctx) + if tErr != nil { + return tErr + } + if isBlocked, err := e.sessionMgr.IsBlocked(ctx, token); err != nil { e.l.Error(err) return err } else if isBlocked { diff --git a/internal/server/session.go b/internal/server/session.go index 9707e6774..e8f03b7ef 100644 --- a/internal/server/session.go +++ b/internal/server/session.go @@ -28,6 +28,7 @@ import ( "github.com/percona/everest/api" "github.com/percona/everest/pkg/accounts" + "github.com/percona/everest/pkg/common" ) const ( @@ -69,7 +70,12 @@ func (e *EverestServer) CreateSession(ctx echo.Context) error { // DeleteSession invalidates the user token by adding it to the blocklist func (e *EverestServer) DeleteSession(ctx echo.Context) error { e.attemptsStore.IncreaseTimeout(ctx.RealIP()) - err := e.blocklist.Block(ctx.Request().Context()) + c := ctx.Request().Context() + token, err := common.ExtractToken(c) + if err != nil { + return err + } + err = e.sessionMgr.Block(c, token) if err != nil { e.l.Errorf("blocklist error: %v", err) return ctx.JSON(http.StatusInternalServerError, api.Error{ diff --git a/pkg/common/accounts.go b/pkg/common/accounts.go index de9c64282..49a2f5d30 100644 --- a/pkg/common/accounts.go +++ b/pkg/common/accounts.go @@ -5,6 +5,9 @@ import ( "crypto/rand" "encoding/hex" "errors" + "fmt" + + "github.com/golang-jwt/jwt/v5" "github.com/percona/everest/pkg/accounts" ) @@ -40,3 +43,12 @@ func CreateInitialAdminAccount( // Create the admin account. return c.SetPassword(ctx, EverestAdminUser, pass, false) } + +// ExtractToken extracts token from context +func ExtractToken(ctx context.Context) (*jwt.Token, error) { + token, ok := ctx.Value(UserCtxKey).(*jwt.Token) + if !ok { + return nil, fmt.Errorf("failed to get token from context") + } + return token, nil +} diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index 8e8418967..dcc9f7d9c 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -183,7 +183,9 @@ func CreateScheme() *runtime.Scheme { func getKubernetesClientOptions(cache cache.Cache) ctrlclient.Options { var cacheOptions *ctrlclient.CacheOptions if cache != nil { - cacheOptions = &ctrlclient.CacheOptions{Reader: cache} + cacheOptions = &ctrlclient.CacheOptions{ + Reader: cache, + } } return ctrlclient.Options{ diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 37d9575a1..db5f40b36 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -17,15 +17,16 @@ package session import ( "context" + "errors" "fmt" + "strconv" "time" "github.com/cenkalti/backoff/v4" "github.com/golang-jwt/jwt/v5" "go.uber.org/zap" - - "github.com/percona/everest/pkg/common" - "github.com/percona/everest/pkg/kubernetes" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -34,12 +35,21 @@ const ( backoffInterval = 500 * time.Millisecond ) +var ( + errExtractJti = errors.New("could not extract jti") + errExtractExp = errors.New("could not extract exp") + errEmptyToken = errors.New("token is empty") + errUnsupportedClaim = func(claims any) error { + return errors.New(fmt.Sprintf("unsupported claims type: %T", claims)) + } +) + // Blocklist represents interface to block JWT tokens and check if a token is blocked. type Blocklist interface { // Block invalidates the token from the context by adding it to blocklist. - Block(ctx context.Context) error + Block(ctx context.Context, token *jwt.Token) error // IsBlocked checks if the token from the context is blocked. - IsBlocked(ctx context.Context) (bool, error) + IsBlocked(ctx context.Context, token *jwt.Token) (bool, error) } type blocklist struct { @@ -47,6 +57,7 @@ type blocklist struct { l *zap.SugaredLogger } +// TokenStore represents an abstraction for storage, hiding details about how the data is actually stored. type TokenStore interface { // Add adds the shortened token to the blocklist Add(ctx context.Context, shortenedToken string) error @@ -54,9 +65,21 @@ type TokenStore interface { Exists(ctx context.Context, shortenedToken string) (bool, error) } +// BlocklistClient supports only the k8s API methods that are needed for blocklist management. +// A separate client is needed to apply the controller-runtime cache only to the related objects. +// Using the controller-runtime client is also beneficial because it supports HA mode. +type BlocklistClient interface { + // GetSecret returns a secret that matches the criteria. + GetSecret(ctx context.Context, key client.ObjectKey) (*corev1.Secret, error) + // CreateSecret creates a secret. + CreateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) + // UpdateSecret updates a secret. + UpdateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) +} + // NewBlocklist creates a new block list -func NewBlocklist(ctx context.Context, kubeClient kubernetes.KubernetesConnector, logger *zap.SugaredLogger) (Blocklist, error) { - store, err := newTokenStore(ctx, kubeClient, logger) +func NewBlocklist(ctx context.Context, bc BlocklistClient, logger *zap.SugaredLogger) (Blocklist, error) { + store, err := newTokenStore(ctx, bc, logger) if err != nil { return nil, err } @@ -67,11 +90,7 @@ func NewBlocklist(ctx context.Context, kubeClient kubernetes.KubernetesConnector } // Block invalidates the token from the context by adding it to blocklist. -func (b *blocklist) Block(ctx context.Context) error { - token, err := extractToken(ctx) - if err != nil { - return err - } +func (b *blocklist) Block(ctx context.Context, token *jwt.Token) error { shortenedToken, err := shortenToken(token) if err != nil { return err @@ -90,11 +109,7 @@ func (b *blocklist) Block(ctx context.Context) error { } // IsBlocked checks if the token from the context is blocked. -func (b *blocklist) IsBlocked(ctx context.Context) (bool, error) { - token, err := extractToken(ctx) - if err != nil { - return false, err - } +func (b *blocklist) IsBlocked(ctx context.Context, token *jwt.Token) (bool, error) { shortenedToken, err := shortenToken(token) if err != nil { return false, fmt.Errorf("failed to shorten token: %w", err) @@ -103,10 +118,46 @@ func (b *blocklist) IsBlocked(ctx context.Context) (bool, error) { return b.tokenStore.Exists(ctx, shortenedToken) } -func extractToken(ctx context.Context) (*jwt.Token, error) { - token, ok := ctx.Value(common.UserCtxKey).(*jwt.Token) +// shortenToken contains only the "jti" and the "exp" claims from the token, so the format of shortened token is +// , for example "9d1c1f98-a479-41e3-8939-c7cb3edefa331743679478", +// where last 10 digits represent the expiration timestamp. +func shortenToken(token *jwt.Token) (string, error) { + content, err := extractContent(token) + if err != nil { + return "", err + } + jti, ok := content.Payload["jti"].(string) + if !ok { + return "", errExtractJti + } + exp, ok := content.Payload["exp"].(float64) if !ok { - return nil, fmt.Errorf("failed to get token from context") + return "", errExtractExp + } + return jti + strconv.FormatFloat(exp, 'f', 0, 64), nil +} + +// JWTContent represents the JWT token structure that is used by blocklist. +type JWTContent struct { + Payload map[string]interface{} `json:"payload"` +} + +func extractContent(token *jwt.Token) (*JWTContent, error) { + if token == nil { + return nil, errEmptyToken + } + claimsMap := make(map[string]interface{}) + + switch claims := token.Claims.(type) { + case jwt.MapClaims: + for key, val := range claims { + claimsMap[key] = val + } + default: + return nil, errUnsupportedClaim(claims) } - return token, nil + + return &JWTContent{ + Payload: claimsMap, + }, nil } diff --git a/pkg/session/blocklist_test.go b/pkg/session/blocklist_test.go index fa922bf95..869b3d00e 100644 --- a/pkg/session/blocklist_test.go +++ b/pkg/session/blocklist_test.go @@ -30,11 +30,105 @@ import ( "github.com/percona/everest/pkg/kubernetes" ) +func TestShortenToken(t *testing.T) { + type tcase struct { + name string + claims jwt.MapClaims + shortenedToken string + error error + } + tcases := []tcase{ + { + name: "valid", + claims: jwt.MapClaims{ + "jti": "9d1c1f98-a479-41e3-8939-c7cb3edefa", + "exp": float64(331743679478), + }, + shortenedToken: "9d1c1f98-a479-41e3-8939-c7cb3edefa331743679478", + error: nil, + }, + { + name: "no jti", + claims: jwt.MapClaims{ + "exp": float64(331743679478), + }, + shortenedToken: "", + error: errExtractJti, + }, + { + name: "no exp", + claims: jwt.MapClaims{ + "jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", + }, + shortenedToken: "", + error: errExtractExp, + }, + } + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + result, err := shortenToken(jwt.NewWithClaims(jwt.SigningMethodHS256, tc.claims)) + assert.Equal(t, tc.error, err) + assert.Equal(t, tc.shortenedToken, result) + }) + } +} + +func TestExtractContent(t *testing.T) { + type tcase struct { + name string + token *jwt.Token + error error + result *JWTContent + } + tokenUnsupportedClaims := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.RegisteredClaims{}) + tcases := []tcase{ + { + name: "empty token", + token: nil, + result: nil, + error: errEmptyToken, + }, + { + name: "unsupported claims", + token: tokenUnsupportedClaims, + result: nil, + error: errUnsupportedClaim(tokenUnsupportedClaims.Claims), + }, + { + name: "valid empty payload", + token: jwt.New(jwt.SigningMethodHS256), + result: &JWTContent{ + Payload: make(map[string]interface{}), + }, + error: nil, + }, + { + name: "valid with payload", + token: jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", "exp": float64(331743679478)}), + result: &JWTContent{ + Payload: map[string]interface{}{"exp": float64(331743679478), "jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a"}, + }, + error: nil, + }, + } + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + result, err := extractContent(tc.token) + assert.Equal(t, tc.error, err) + assert.Equal(t, tc.result, result) + }) + } +} + func TestBlocklist_Block(t *testing.T) { mockClient := fakeclient.NewClientBuilder().WithScheme(kubernetes.CreateScheme()) l := zap.NewNop().Sugar() k := kubernetes.NewEmpty(l).WithKubernetesClient(mockClient.Build()) - ctx := backgroundContextWithToken(jwt.MapClaims{"jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", "exp": float64(331743679478)}) + + ctx := context.Background() + jwt := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", "exp": float64(331743679478)}) // check there is no blocklist secret before the blocklist creation secret, err := k.GetSecret(ctx, ctrlclient.ObjectKey{ @@ -57,7 +151,7 @@ func TestBlocklist_Block(t *testing.T) { assert.Equal(t, "", secret.StringData[dataKey]) // block the token from the context and check the secret has been changed accordingly - err = b.Block(ctx) + err = b.Block(ctx, jwt) assert.NoError(t, err) secret, err = k.GetSecret(ctx, ctrlclient.ObjectKey{ @@ -74,7 +168,7 @@ func TestBlocklist_Block(t *testing.T) { assert.NoError(t, err) // after deleting secret - try to block again, get the NotFound error - err = b.Block(ctx) + err = b.Block(ctx, jwt) assert.Equal(t, true, k8serrors.IsNotFound(err)) } @@ -91,32 +185,26 @@ func TestBlocklist_IsBlocked(t *testing.T) { k := kubernetes.NewEmpty(l).WithKubernetesClient(mockClient.Build()) t.Run("blocked token in context", func(t *testing.T) { - ctx := backgroundContextWithToken(jwt.MapClaims{"jti": "the-blocked-jti", "exp": float64(331743679478)}) + ctx := context.Background() + jwt := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "the-blocked-jti", "exp": float64(331743679478)}) b, err := NewBlocklist(ctx, k, l) assert.NoError(t, err) - blocked, err := b.IsBlocked(ctx) + blocked, err := b.IsBlocked(ctx, jwt) assert.NoError(t, err) assert.True(t, blocked) }) t.Run("not blocked token in context", func(t *testing.T) { - ctx := backgroundContextWithToken(jwt.MapClaims{"jti": "some-other-jti", "exp": float64(331743679478)}) + ctx := context.Background() b, err := NewBlocklist(ctx, k, l) assert.NoError(t, err) - blocked, err := b.IsBlocked(ctx) + jwt := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "some-other-jti", "exp": float64(331743679478)}) + blocked, err := b.IsBlocked(ctx, jwt) assert.NoError(t, err) assert.False(t, blocked) }) } - -func backgroundContextWithToken(claims jwt.MapClaims) context.Context { - return context.WithValue( - context.Background(), - common.UserCtxKey, - jwt.NewWithClaims(jwt.SigningMethodHS256, claims), - ) -} diff --git a/pkg/session/jwt.go b/pkg/session/jwt.go deleted file mode 100644 index 8d1f3f6cf..000000000 --- a/pkg/session/jwt.go +++ /dev/null @@ -1,73 +0,0 @@ -// everest -// Copyright (C) 2025 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package session - -import ( - "errors" - "fmt" - "strconv" - - "github.com/golang-jwt/jwt/v5" -) - -var ( - errEmptyToken = errors.New("token is empty") - errExtractJti = errors.New("could not extract jti") - errExtractExp = errors.New("could not extract exp") - errUnsupportedClaim = func(claims any) error { - return errors.New(fmt.Sprintf("unsupported claims type: %T", claims)) - } -) - -type JWTContent struct { - Payload map[string]interface{} `json:"payload"` -} - -func shortenToken(token *jwt.Token) (string, error) { - content, err := extractContent(token) - if err != nil { - return "", err - } - jti, ok := content.Payload["jti"].(string) - if !ok { - return "", errExtractJti - } - exp, ok := content.Payload["exp"].(float64) - if !ok { - return "", errExtractExp - } - return jti + strconv.FormatFloat(exp, 'f', 0, 64), nil -} - -func extractContent(token *jwt.Token) (*JWTContent, error) { - if token == nil { - return nil, errEmptyToken - } - claimsMap := make(map[string]interface{}) - - switch claims := token.Claims.(type) { - case jwt.MapClaims: - for key, val := range claims { - claimsMap[key] = val - } - default: - return nil, errUnsupportedClaim(claims) - } - - return &JWTContent{ - Payload: claimsMap, - }, nil -} diff --git a/pkg/session/jwt_test.go b/pkg/session/jwt_test.go deleted file mode 100644 index 69d4aa3ac..000000000 --- a/pkg/session/jwt_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// everest -// Copyright (C) 2025 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package session - -import ( - "testing" - - "github.com/golang-jwt/jwt/v5" - "github.com/stretchr/testify/assert" -) - -func TestShortenToken(t *testing.T) { - type tcase struct { - name string - claims jwt.MapClaims - shortenedToken string - error error - } - tcases := []tcase{ - { - name: "valid", - claims: jwt.MapClaims{ - "jti": "9d1c1f98-a479-41e3-8939-c7cb3edefa", - "exp": float64(331743679478), - }, - shortenedToken: "9d1c1f98-a479-41e3-8939-c7cb3edefa331743679478", - error: nil, - }, - { - name: "no jti", - claims: jwt.MapClaims{ - "exp": float64(331743679478), - }, - shortenedToken: "", - error: errExtractJti, - }, - { - name: "no exp", - claims: jwt.MapClaims{ - "jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", - }, - shortenedToken: "", - error: errExtractExp, - }, - } - for _, tc := range tcases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - result, err := shortenToken(jwt.NewWithClaims(jwt.SigningMethodHS256, tc.claims)) - assert.Equal(t, tc.error, err) - assert.Equal(t, tc.shortenedToken, result) - }) - } -} - -func TestExtractContent(t *testing.T) { - type tcase struct { - name string - token *jwt.Token - error error - result *JWTContent - } - tokenUnsupportedClaims := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.RegisteredClaims{}) - tcases := []tcase{ - { - name: "empty token", - token: nil, - result: nil, - error: errEmptyToken, - }, - { - name: "unsupported claims", - token: tokenUnsupportedClaims, - result: nil, - error: errUnsupportedClaim(tokenUnsupportedClaims.Claims), - }, - { - name: "valid empty payload", - token: jwt.New(jwt.SigningMethodHS256), - result: &JWTContent{ - Payload: make(map[string]interface{}), - }, - error: nil, - }, - { - name: "valid with payload", - token: jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a", "exp": float64(331743679478)}), - result: &JWTContent{ - Payload: map[string]interface{}{"exp": float64(331743679478), "jti": "9d1c1f98-a479-41e3-8939-c7cb3e049a"}, - }, - error: nil, - }, - } - for _, tc := range tcases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - result, err := extractContent(tc.token) - assert.Equal(t, tc.error, err) - assert.Equal(t, tc.result, result) - }) - } -} diff --git a/pkg/session/manager.go b/pkg/session/manager.go index 6ca541b03..60181eced 100644 --- a/pkg/session/manager.go +++ b/pkg/session/manager.go @@ -41,13 +41,14 @@ const ( type Manager struct { accountManager accounts.Interface signingKey *rsa.PrivateKey + Blocklist } // Option is a function that modifies a SessionManager. type Option func(*Manager) // New creates a new session manager with the given options. -func New(options ...Option) (*Manager, error) { +func New(blocklist Blocklist, options ...Option) (*Manager, error) { m := &Manager{} for _, opt := range options { opt(m) @@ -57,6 +58,7 @@ func New(options ...Option) (*Manager, error) { return nil, errors.Join(err, errors.New("failed to get private key")) } m.signingKey = privKey + m.Blocklist = blocklist return m, nil } diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index 999da0ce2..c566fb967 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/percona/everest/pkg/common" - "github.com/percona/everest/pkg/kubernetes" ) const ( @@ -37,44 +36,56 @@ const ( timestampLen = 10 ) -func newTokenStore(ctx context.Context, kubeClient kubernetes.KubernetesConnector, logger *zap.SugaredLogger) (TokenStore, error) { - store := &tokenStore{ - l: logger, - kubeClient: kubeClient, +func newTokenStore(ctx context.Context, client BlocklistClient, logger *zap.SugaredLogger) (TokenStore, error) { + s := &tokenStore{ + l: logger, + client: client, } - _, err := kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + err := s.init(ctx) + if err != nil { + return nil, err + } + return s, nil +} + +func (ts *tokenStore) init(ctx context.Context) error { + _, err := ts.client.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err == nil { - return store, nil + return err } if !k8serrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get %s secret in the %s namespace: %w", common.EverestBlocklistSecretName, common.SystemNamespace, err) + err = fmt.Errorf("failed to get %s secret in the %s namespace: %w", common.EverestBlocklistSecretName, common.SystemNamespace, err) + ts.l.Error(err) + return err } var createErr error secret := getBlockListSecretTemplate("") - _, createErr = kubeClient.CreateSecret(ctx, secret) + _, createErr = ts.client.CreateSecret(ctx, secret) if createErr != nil { - return nil, fmt.Errorf("failed to create secret %s in namespace %s: %w", secret.Name, secret.Namespace, createErr) + err = fmt.Errorf("failed to create secret %s in namespace %s: %w", secret.Name, secret.Namespace, createErr) + ts.l.Error(err) + return err } - return store, nil + return nil } type tokenStore struct { - kubeClient kubernetes.KubernetesConnector - l *zap.SugaredLogger + client BlocklistClient + l *zap.SugaredLogger } // Add adds the shortened token to the blocklist func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { - secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + secret, err := ts.client.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err != nil { ts.l.Errorf("failed to get %s secret in the %s namespace: %v", common.EverestBlocklistSecretName, common.SystemNamespace, err) return err } secret = addDataToSecret(ts.l, secret, shortenedToken, time.Now().UTC()) - _, updateErr := ts.kubeClient.UpdateSecret(ctx, secret) + _, updateErr := ts.client.UpdateSecret(ctx, secret) if updateErr != nil { - ts.l.Errorf("failed to update %s secret in the %s namespace, retrying: %v", secret.Name, secret.Namespace, updateErr) + ts.l.Errorf("failed to update %s secret in the %s namespace withe the %s shortened token, retrying: %v", secret.Name, secret.Namespace, shortenedToken, updateErr) return err } return nil @@ -83,7 +94,7 @@ func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { // Exists checks if the shortened token is in the blocklist func (ts *tokenStore) Exists(ctx context.Context, shortenedToken string) (bool, error) { // no worries about overwhelming k8s API - the secret is cached - secret, err := ts.kubeClient.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) + secret, err := ts.client.GetSecret(ctx, types.NamespacedName{Namespace: common.SystemNamespace, Name: common.EverestBlocklistSecretName}) if err != nil { ts.l.Errorf("failed to get %s secret in the %s namespace: %v", common.EverestBlocklistSecretName, common.SystemNamespace, err) return false, err diff --git a/pkg/session/token_store_test.go b/pkg/session/token_store_test.go index 7dcfde3a6..6a51c86eb 100644 --- a/pkg/session/token_store_test.go +++ b/pkg/session/token_store_test.go @@ -196,3 +196,52 @@ func generateTestList(numTokens int) string { } return builder.String() } + +const ( + stringSizeMB = 1 // Size of the string in MB +) + +// generateLargeString creates a string of the specified size in MB. +func generateLargeString(sizeMB int) string { + size := sizeMB * 1024 * 1024 // Convert MB to bytes + builder := strings.Builder{} + builder.Grow(size) + + // Repeat a simple pattern to fill the string. This is important. + // Random data will compress differently and may not represent realistic + // data patterns. + pattern := "abcdefghijklmnopqrstuvwxyz" + patternLen := len(pattern) + + for i := 0; i < size; i++ { + builder.WriteByte(pattern[i%patternLen]) + } + + return builder.String() +} + +// BenchmarkStringsContains measures the time it takes to run strings.Contains on a 1MB string. +func BenchmarkStringsContains(b *testing.B) { + largeString := generateLargeString(stringSizeMB) + // Substring to search for. Choose a substring that's *likely* to be found for a realistic test. + substring := "xyz" + + // Reset the timer to exclude the setup time. + b.ResetTimer() + + for i := 0; i < b.N; i++ { + strings.Contains(largeString, substring) + } +} + +// BenchmarkStringsContainsNotFound measures the time it takes to run strings.Contains on a 1MB string when the substring is NOT present. +func BenchmarkStringsContainsNotFound(b *testing.B) { + largeString := generateLargeString(stringSizeMB) + substring := "thisstringisdefinitelynotpresent" // Substring that is NOT present. + + b.ResetTimer() // Reset the timer to exclude the setup time. + + for i := 0; i < b.N; i++ { + strings.Contains(largeString, substring) + } +} From 888a264158742237fc40d8dce5967683844efb6e Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Mon, 28 Apr 2025 13:01:55 +0500 Subject: [PATCH 26/31] Update pkg/session/token_store.go Co-authored-by: Maxim Kondratenko --- pkg/session/token_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index c566fb967..61a73e72c 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -124,7 +124,7 @@ func cleanupOld(l *zap.SugaredLogger, list string, now time.Time) []string { for _, shortenedToken := range tokens { length := len(shortenedToken) if length < timestampLen { - l.Warn("blocklist contains irregular data format") + l.Warnf("blocklist token='%s' contains irregular data format", shortenedToken) continue } ts := shortenedToken[length-10 : length] From 508fcb76c55bfed69c5f9869a832831e7a8b0cca Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Mon, 28 Apr 2025 13:44:37 +0500 Subject: [PATCH 27/31] Update pkg/session/token_store.go Co-authored-by: Maxim Kondratenko --- pkg/session/token_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index 61a73e72c..31349ddcf 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -85,7 +85,7 @@ func (ts *tokenStore) Add(ctx context.Context, shortenedToken string) error { secret = addDataToSecret(ts.l, secret, shortenedToken, time.Now().UTC()) _, updateErr := ts.client.UpdateSecret(ctx, secret) if updateErr != nil { - ts.l.Errorf("failed to update %s secret in the %s namespace withe the %s shortened token, retrying: %v", secret.Name, secret.Namespace, shortenedToken, updateErr) + ts.l.Errorf("failed to update %s secret in the %s namespace with the %s shortened token, retrying: %v", secret.Name, secret.Namespace, shortenedToken, updateErr) return err } return nil From befc97897990c9af3910882f372521a29e6f0d10 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Mon, 28 Apr 2025 13:45:21 +0500 Subject: [PATCH 28/31] Update internal/server/everest.go Co-authored-by: Maxim Kondratenko --- internal/server/everest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/everest.go b/internal/server/everest.go index ad7db877e..3fed56be3 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -112,7 +112,7 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar } blocklistClient, err := kubernetes.NewInCluster(l, ctx, options) if err != nil { - return nil, err + return nil, errors.Join(err, errors.New("failed creating Kubernetes client for blockList")) } blockList, err := session.NewBlocklist(ctx, blocklistClient, l) From 1e463992e24955ae5143f3cfe2bcb309526e9859 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Mon, 28 Apr 2025 15:28:57 +0500 Subject: [PATCH 29/31] make blocklist dependent from tokenstore --- internal/server/everest.go | 21 +-------------------- pkg/cli/upgrade/upgrade.go | 2 +- pkg/session/blocklist.go | 35 +++++++++++++++++++++-------------- pkg/session/blocklist_test.go | 17 ++++++++++++++--- pkg/session/manager.go | 11 +++++++++-- pkg/session/token_store.go | 15 +++++++++++++-- 6 files changed, 59 insertions(+), 42 deletions(-) diff --git a/internal/server/everest.go b/internal/server/everest.go index ad7db877e..789d7fcc4 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -37,10 +37,7 @@ import ( "github.com/unrolled/secure" "go.uber.org/zap" "golang.org/x/time/rate" - corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/fields" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/percona/everest/api" @@ -103,24 +100,8 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar middleware, store := sessionRateLimiter(c.CreateSessionRateLimit) echoServer.Use(middleware) - options := &cache.Options{ - ByObject: map[client.Object]cache.ByObject{ - &corev1.Secret{}: { - Field: fields.SelectorFromSet(fields.Set{"metadata.name": common.EverestBlocklistSecretName}), - }, - }, - } - blocklistClient, err := kubernetes.NewInCluster(l, ctx, options) - if err != nil { - return nil, err - } - - blockList, err := session.NewBlocklist(ctx, blocklistClient, l) - if err != nil { - return nil, errors.Join(err, errors.New("failed to configure tokens blocklist")) - } sessMgr, err := session.New( - blockList, + ctx, l, session.WithAccountManager(kubeConnector.Accounts()), ) if err != nil { diff --git a/pkg/cli/upgrade/upgrade.go b/pkg/cli/upgrade/upgrade.go index 4fcd42c93..e887853a2 100644 --- a/pkg/cli/upgrade/upgrade.go +++ b/pkg/cli/upgrade/upgrade.go @@ -102,7 +102,7 @@ func NewUpgrade(cfg *Config, l *zap.SugaredLogger) (*Upgrade, error) { var kubeClient kubernetes.KubernetesConnector if cfg.InCluster { - k, err := kubernetes.NewInCluster(cli.l, context.Background(), nil) + k, err := kubernetes.NewInCluster(cli.l, nil, nil) if err != nil { return nil, fmt.Errorf("could not create in-cluster kubernetes client: %w", err) } diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index db5f40b36..226b5671e 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -26,7 +26,12 @@ import ( "github.com/golang-jwt/jwt/v5" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/percona/everest/pkg/common" + "github.com/percona/everest/pkg/kubernetes" ) const ( @@ -65,21 +70,23 @@ type TokenStore interface { Exists(ctx context.Context, shortenedToken string) (bool, error) } -// BlocklistClient supports only the k8s API methods that are needed for blocklist management. -// A separate client is needed to apply the controller-runtime cache only to the related objects. -// Using the controller-runtime client is also beneficial because it supports HA mode. -type BlocklistClient interface { - // GetSecret returns a secret that matches the criteria. - GetSecret(ctx context.Context, key client.ObjectKey) (*corev1.Secret, error) - // CreateSecret creates a secret. - CreateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) - // UpdateSecret updates a secret. - UpdateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) -} - // NewBlocklist creates a new block list -func NewBlocklist(ctx context.Context, bc BlocklistClient, logger *zap.SugaredLogger) (Blocklist, error) { - store, err := newTokenStore(ctx, bc, logger) +func NewBlocklist(ctx context.Context, logger *zap.SugaredLogger) (Blocklist, error) { + options := &cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Field: fields.SelectorFromSet(fields.Set{"metadata.name": common.EverestBlocklistSecretName}), + }, + }, + } + // A separate client is needed to apply the controller-runtime cache only to the related objects. + // Using the controller-runtime client is also beneficial because it supports HA mode. + tokenStoreClient, err := kubernetes.NewInCluster(logger, ctx, options) + if err != nil { + return nil, err + } + + store, err := newTokenStore(ctx, tokenStoreClient, logger) if err != nil { return nil, err } diff --git a/pkg/session/blocklist_test.go b/pkg/session/blocklist_test.go index 869b3d00e..32b5bc454 100644 --- a/pkg/session/blocklist_test.go +++ b/pkg/session/blocklist_test.go @@ -138,7 +138,7 @@ func TestBlocklist_Block(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) assert.Nil(t, secret) - b, err := NewBlocklist(ctx, k, l) + b, err := mockNewBlocklist(ctx, l, k) assert.NoError(t, err) // blocklist secret appears after the blocklist creation @@ -188,7 +188,7 @@ func TestBlocklist_IsBlocked(t *testing.T) { ctx := context.Background() jwt := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "the-blocked-jti", "exp": float64(331743679478)}) - b, err := NewBlocklist(ctx, k, l) + b, err := mockNewBlocklist(ctx, l, k) assert.NoError(t, err) blocked, err := b.IsBlocked(ctx, jwt) @@ -199,7 +199,7 @@ func TestBlocklist_IsBlocked(t *testing.T) { t.Run("not blocked token in context", func(t *testing.T) { ctx := context.Background() - b, err := NewBlocklist(ctx, k, l) + b, err := mockNewBlocklist(ctx, l, k) assert.NoError(t, err) jwt := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"jti": "some-other-jti", "exp": float64(331743679478)}) @@ -208,3 +208,14 @@ func TestBlocklist_IsBlocked(t *testing.T) { assert.False(t, blocked) }) } + +func mockNewBlocklist(ctx context.Context, logger *zap.SugaredLogger, mockClient TokenStoreClient) (Blocklist, error) { + store, err := newTokenStore(ctx, mockClient, logger) + if err != nil { + return nil, err + } + return &blocklist{ + tokenStore: store, + l: logger, + }, nil +} diff --git a/pkg/session/manager.go b/pkg/session/manager.go index 60181eced..0386753c1 100644 --- a/pkg/session/manager.go +++ b/pkg/session/manager.go @@ -27,6 +27,7 @@ import ( "time" "github.com/golang-jwt/jwt/v5" + "go.uber.org/zap" "github.com/percona/everest/pkg/accounts" "github.com/percona/everest/pkg/common" @@ -48,7 +49,7 @@ type Manager struct { type Option func(*Manager) // New creates a new session manager with the given options. -func New(blocklist Blocklist, options ...Option) (*Manager, error) { +func New(ctx context.Context, l *zap.SugaredLogger, options ...Option) (*Manager, error) { m := &Manager{} for _, opt := range options { opt(m) @@ -58,7 +59,13 @@ func New(blocklist Blocklist, options ...Option) (*Manager, error) { return nil, errors.Join(err, errors.New("failed to get private key")) } m.signingKey = privKey - m.Blocklist = blocklist + + blockList, err := NewBlocklist(ctx, l) + if err != nil { + return nil, errors.Join(err, errors.New("failed to configure tokens blocklist")) + } + + m.Blocklist = blockList return m, nil } diff --git a/pkg/session/token_store.go b/pkg/session/token_store.go index c566fb967..855c5c5ff 100644 --- a/pkg/session/token_store.go +++ b/pkg/session/token_store.go @@ -27,6 +27,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/percona/everest/pkg/common" ) @@ -36,7 +37,17 @@ const ( timestampLen = 10 ) -func newTokenStore(ctx context.Context, client BlocklistClient, logger *zap.SugaredLogger) (TokenStore, error) { +// TokenStoreClient contains the methods that are needed for the token store management. +type TokenStoreClient interface { + // GetSecret returns a secret that matches the criteria. + GetSecret(ctx context.Context, key client.ObjectKey) (*corev1.Secret, error) + // CreateSecret creates a secret. + CreateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) + // UpdateSecret updates a secret. + UpdateSecret(ctx context.Context, secret *corev1.Secret) (*corev1.Secret, error) +} + +func newTokenStore(ctx context.Context, client TokenStoreClient, logger *zap.SugaredLogger) (TokenStore, error) { s := &tokenStore{ l: logger, client: client, @@ -70,7 +81,7 @@ func (ts *tokenStore) init(ctx context.Context) error { } type tokenStore struct { - client BlocklistClient + client TokenStoreClient l *zap.SugaredLogger } From 3e4ca6fdf53809ffbdd1f30cb88fe5dbbddeb3b2 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Mon, 28 Apr 2025 15:32:40 +0500 Subject: [PATCH 30/31] update error message --- pkg/session/blocklist.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/session/blocklist.go b/pkg/session/blocklist.go index 226b5671e..ab1da607b 100644 --- a/pkg/session/blocklist.go +++ b/pkg/session/blocklist.go @@ -83,7 +83,7 @@ func NewBlocklist(ctx context.Context, logger *zap.SugaredLogger) (Blocklist, er // Using the controller-runtime client is also beneficial because it supports HA mode. tokenStoreClient, err := kubernetes.NewInCluster(logger, ctx, options) if err != nil { - return nil, err + return nil, errors.Join(err, errors.New("failed creating Kubernetes client for blockList")) } store, err := newTokenStore(ctx, tokenStoreClient, logger) From dfac7bb80d012a44f9a58631724fae572272325c Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko Date: Mon, 28 Apr 2025 16:38:38 +0500 Subject: [PATCH 31/31] move middleware func inside session manager --- internal/server/everest.go | 31 +------------------------------ pkg/session/manager.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/internal/server/everest.go b/internal/server/everest.go index 789d7fcc4..2466d2c98 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -27,7 +27,6 @@ import ( "slices" "text/template" - "github.com/AlekSi/pointer" "github.com/getkin/kin-openapi/openapi3filter" "github.com/golang-jwt/jwt/v5" echojwt "github.com/labstack/echo-jwt/v4" @@ -207,7 +206,7 @@ func (e *EverestServer) initHTTPServer(ctx context.Context) error { } apiGroup.Use(jwtMW) - blocklistMW, err := e.blocklistMiddleWare() + blocklistMW, err := e.sessionMgr.BlocklistMiddleWare(newSkipperFunc) if err != nil { return err } @@ -425,31 +424,3 @@ func everestErrorHandler(next echo.HTTPErrorHandler) echo.HTTPErrorHandler { next(err, c) } } - -func (e *EverestServer) blocklistMiddleWare() (echo.MiddlewareFunc, error) { - skipper, err := newSkipperFunc() - if err != nil { - return nil, err - } - return func(next echo.HandlerFunc) echo.HandlerFunc { - return func(c echo.Context) error { - if skipper(c) { - return next(c) - } - ctx := c.Request().Context() - token, tErr := common.ExtractToken(ctx) - if tErr != nil { - return tErr - } - if isBlocked, err := e.sessionMgr.IsBlocked(ctx, token); err != nil { - e.l.Error(err) - return err - } else if isBlocked { - return c.JSON(http.StatusUnauthorized, api.Error{ - Message: pointer.ToString("Invalid token"), - }) - } - return next(c) - } - }, nil -} diff --git a/pkg/session/manager.go b/pkg/session/manager.go index 0386753c1..7547e1361 100644 --- a/pkg/session/manager.go +++ b/pkg/session/manager.go @@ -23,12 +23,17 @@ import ( "encoding/pem" "errors" "fmt" + "net/http" "os" "time" + "github.com/AlekSi/pointer" "github.com/golang-jwt/jwt/v5" + "github.com/labstack/echo/v4" + echomiddleware "github.com/labstack/echo/v4/middleware" "go.uber.org/zap" + "github.com/percona/everest/api" "github.com/percona/everest/pkg/accounts" "github.com/percona/everest/pkg/common" ) @@ -43,6 +48,7 @@ type Manager struct { accountManager accounts.Interface signingKey *rsa.PrivateKey Blocklist + l *zap.SugaredLogger } // Option is a function that modifies a SessionManager. @@ -59,6 +65,7 @@ func New(ctx context.Context, l *zap.SugaredLogger, options ...Option) (*Manager return nil, errors.Join(err, errors.New("failed to get private key")) } m.signingKey = privKey + m.l = l blockList, err := NewBlocklist(ctx, l) if err != nil { @@ -144,3 +151,31 @@ func (mgr *Manager) KeyFunc() jwt.Keyfunc { return mgr.signingKey.Public(), nil } } + +func (mgr *Manager) BlocklistMiddleWare(skipperFunc func() (echomiddleware.Skipper, error)) (echo.MiddlewareFunc, error) { + skipper, err := skipperFunc() + if err != nil { + return nil, err + } + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if skipper(c) { + return next(c) + } + ctx := c.Request().Context() + token, tErr := common.ExtractToken(ctx) + if tErr != nil { + return tErr + } + if isBlocked, err := mgr.IsBlocked(ctx, token); err != nil { + mgr.l.Error(err) + return err + } else if isBlocked { + return c.JSON(http.StatusUnauthorized, api.Error{ + Message: pointer.ToString("Invalid token"), + }) + } + return next(c) + } + }, nil +}