json2code: support chunked_fifo

Currently json2code only supports std::vector as a list type, but this
results in unavoidable large allocations for even modest response sizes,
e.g., is is not uncommon for sizeof(elem_type) for these vectors to be
100 - 1000 bytes for simple to moderately complex response types, so
then a mere ~1200 to ~120 objects in the vector are enough to result in
allocations > 128K, a problem for seastar applications which must
avoid large allocations (because of fragmentation).

To avoid this problem, support chunked_fifo as a second type for lists
in json2code. Use the type "chunked_array" instead of "array" to use it
and the generated code will use chunked_fifo instead of vector.

Also adds tests for the new use case in the json2code test.

Author: travisdowns

include/seastar/json/json_elements.hh

@@ -26,11 +26,12 @@
 #include <vector>
 #endif
 
+#include <seastar/core/chunked_fifo.hh>
 #include <seastar/core/do_with.hh>
+#include <seastar/core/iostream.hh>
 #include <seastar/core/loop.hh>
-#include <seastar/json/formatter.hh>
 #include <seastar/core/sstring.hh>
-#include <seastar/core/iostream.hh>
+#include <seastar/json/formatter.hh>
 #include <seastar/util/modules.hh>
 
 namespace seastar {
@@ -149,13 +150,15 @@ private:
 };
 
 /**
- * json_list is based on std vector implementation.
+ * json_list_template is an array type based on a
+ * container type passed as a template parameter, as we want to
+ * have flavors based on both vector and chunked_fifo.
  *
  * When values are added with push it is set the "set" flag to true
  * hence will be included in the parsed object
  */
-template<class T>
-class json_list : public json_base_element {
+template <class T, class Container>
+class json_list_template : public json_base_element {
 public:
 
     /**
@@ -186,7 +189,7 @@ public:
      * iteration and that it's elements can be assigned to the list elements
      */
     template<class C>
-    json_list& operator=(const C& list) {
+    json_list_template& operator=(const C& list) {
         _elements.clear();
         for  (auto i : list) {
             push(i);
@@ -196,9 +199,16 @@ public:
     virtual future<> write(output_stream<char>& s) const override {
         return formatter::write(s, _elements);
     }
-    std::vector<T> _elements;
+
+    Container _elements;
 };
 
+template <typename T>
+using json_list = json_list_template<T, std::vector<T>>;
+
+template <typename T>
+using json_chunked_list = json_list_template<T, seastar::chunked_fifo<T>>;
+
 class jsonable {
 public:
     jsonable() = default;

scripts/seastar-json2code.py

@@ -97,9 +97,14 @@ def valid_type(param):
     trace_err("Type [", param, "] not defined")
     return param
 
+# Map from json list types to the C++ implementing type
+LIST_TO_IMPL = {"array": "json_list", "chunked_array": "json_chunked_list"}
 
-def type_change(param, member):
-    if param == "array":
+def is_array_type(type: str):
+    return type in LIST_TO_IMPL
+
+def type_change(param: str, member):
+    if is_array_type(param):
         if "items" not in member:
             trace_err("array without item declaration in ", param)
             return ""
@@ -111,7 +116,8 @@ def type_change(param, member):
         else:
             trace_err("array items with no type or ref declaration ", param)
             return ""
-        return "json_list< " + valid_type(t) + " >"
+        return LIST_TO_IMPL[param] + "< " + valid_type(t) + " >"
+
     return "json_element< " + valid_type(param) + " >"
 
 
@@ -371,7 +377,7 @@ def is_model_valid(name, model):
     properties = getitem(model[name], "properties", name)
     for var in properties:
         type = getitem(properties[var], "type", name + ":" + var)
-        if type == "array":
+        if is_array_type(type):
             items = getitem(properties[var], "items", name + ":" + var)
             try:
                 type = getitem(items, "type", name + ":" + var + ":items")

tests/unit/api.json

@@ -82,6 +82,18 @@
             "VAL2",
             "VAL3"
           ]
+        },
+        "array_var": {
+          "type": "array",
+          "items": {
+            "type": "int"
+          }
+        },
+        "chunked_array_var": {
+          "type": "chunked_array",
+          "items": {
+            "type": "int"
+          }
         }
       }
     }

tests/unit/json2code_test.py

@@ -79,6 +79,13 @@ def test_missing_path_param(self):
             self.assertEqual(response['message'], 'Not found')
             self.assertEqual(response['code'], 404)
 
+    def test_arrays(self):
+        response = self._do_query('x', 'x', 'VAL2')
+        expected = [1, 2, 3]
+
+        self.assertEqual(response['array_var'], expected)
+        self.assertEqual(response['chunked_array_var'], expected)
+
     def _do_query(self, var1: str, var2: str, query_enum: str):
         def query(streaming: bool = False):
             params = urllib.parse.urlencode({

tests/unit/rest_api_httpd.cc

@@ -50,7 +50,12 @@ void set_routes(routes& r) {
         query_enum v = str2query_enum(req.query_parameters.at("query_enum"));
         // This demonstrate enum conversion
         obj.enum_var = v;
-        stream_enum is_streaming =str2stream_enum(req.query_parameters.at("stream_enum"));
+
+        std::vector<int> vec123{1, 2, 3};
+        obj.array_var = vec123;
+        obj.chunked_array_var = vec123;
+
+        auto use_streaming = req.query_parameters.at("use_streaming");
 
         if (use_streaming != "false" && use_streaming != "true") {
             throw bad_param_exception("param use_streaming must be true or false");