started working datasets

Author: gf712

src/shogun/io/OpenMLFlow.cpp

@@ -49,6 +49,8 @@ const char* OpenMLReader::list_dataset_qualities = "/data/qualities/{}";
 const char* OpenMLReader::list_dataset_filter = "/data/list/{}";
 /* FLOW API */
 const char* OpenMLReader::flow_file = "/flow/{}";
+/* TASK API */
+const char* OpenMLReader::task_file = "/task/{}";
 
 const std::unordered_map<std::string, std::string>
     OpenMLReader::m_format_options = {{"xml", xml_server},
@@ -102,10 +104,10 @@ void OpenMLReader::openml_curl_error_helper(CURL* curl_handle, CURLcode code)
 #endif // HAVE_CURL
 
 /**
- * Checks the returned flow in JSON format
- * @param doc the parsed flow
+ * Checks the returned response from OpenML in JSON format
+ * @param doc the parsed OpenML JSON format response
  */
-static void check_flow_response(Document& doc)
+static void check_response(const Document& doc, const std::string& type)
 {
 	if (SG_UNLIKELY(doc.HasMember("error")))
 	{
@@ -115,7 +117,9 @@ static void check_flow_response(Document& doc)
 		    root["message"].GetString())
 		return;
 	}
-	REQUIRE(doc.HasMember("flow"), "Unexpected format of OpenML flow.\n");
+	REQUIRE(
+	    doc.HasMember(type.c_str()), "Unexpected format of OpenML %s.\n",
+	    type.c_str());
 }
 
 /**
@@ -142,8 +146,7 @@ static SG_FORCED_INLINE void emplace_string_to_map(
  * @param name the name of the key
  */
 static SG_FORCED_INLINE void emplace_string_to_map(
-    const GenericObject<
-        true, GenericValue<UTF8<char>>>& v,
+    const GenericObject<true, GenericValue<UTF8<char>>>& v,
     std::unordered_map<std::string, std::string>& param_dict,
     const std::string& name)
 {
@@ -167,7 +170,7 @@ std::shared_ptr<OpenMLFlow> OpenMLFlow::download_flow(
 	auto reader = OpenMLReader(api_key);
 	auto return_string = reader.get("flow_file", "json", flow_id);
 	document.Parse(return_string.c_str());
-	check_flow_response(document);
+	check_response(document, "flow");
 
 	// store root for convenience. We know it exists from previous check.
 	const Value& root = document["flow"];
@@ -248,10 +251,63 @@ std::shared_ptr<OpenMLFlow> OpenMLFlow::from_file()
 	return std::shared_ptr<OpenMLFlow>();
 }
 
+std::shared_ptr<OpenMLTask>
+OpenMLTask::get_dataset(const std::string& task_id, const std::string& api_key)
+{
+	Document document;
+	std::string task_name;
+	std::string task_type;
+	std::string task_type_id;
+	std::pair<std::shared_ptr<OpenMLData>, std::shared_ptr<OpenMLSplit>>
+	    task_descriptor;
+
+	auto reader = OpenMLReader(api_key);
+	auto return_string = reader.get("task_file", "json", task_id);
+
+	document.Parse(return_string.c_str());
+	check_response(document, "task");
+
+	const Value& root = document["flow"];
+
+	REQUIRE(
+	    task_id == root["task_id"].GetString(),
+	    "Expected downloaded task to have the same id as the requested task "
+	    "id.\n")
+
+	task_name = root["task_name"].GetString();
+	task_type = root["task_type"].GetString();
+	task_type_id = root["task_type_id"].GetString();
+
+	// expect two elements in input array: dataset and split
+	const Value& json_input = root["input"];
+
+	REQUIRE(
+	    json_input.IsArray(), "Currently the dataset reader can only handle "
+	                          "inputs with a dataset and split field")
+
+	auto input_array = json_input.GetArray();
+	REQUIRE(
+	    input_array.Size() == 2,
+	    "Currently the dataset reader can only handle inputs with a dataset "
+	    "and split field. Found %d elements.",
+	    input_array.Size())
+
+	// handle dataset
+	auto json_dataset = input_array[0].GetObject();
+
+	auto result = std::make_shared<OpenMLTask>(
+	    task_id, task_name, task_type, task_type_id, task_descriptor);
+
+	return result;
+}
+
 /**
  * Class using the Any visitor pattern to convert
  * a string to a C++ type that can be used as a parameter
- * in a Shogun model.
+ * in a Shogun model. If the string value is not "null" it will
+ * be put in its casted type in the given model with the provided parameter
+ * name. If the value is null nothing happens, i.e. no error is thrown
+ * and no value is put in model.
  */
 class StringToShogun : public AnyVisitor
 {
@@ -266,18 +322,20 @@ class StringToShogun : public AnyVisitor
 
 	void on(bool* v) final
 	{
+		SG_SDEBUG(
+				"bool: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 		if (!is_null())
 		{
-			SG_SDEBUG("bool: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 			bool result = strcmp(m_string_val.c_str(), "true") == 0;
 			m_model->put(m_parameter, result);
 		}
 	}
 	void on(int32_t* v) final
 	{
+		SG_SDEBUG(
+				"int32: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 		if (!is_null())
 		{
-			SG_SDEBUG("int32: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 			try
 			{
 				int32_t result = std::stoi(m_string_val);
@@ -299,84 +357,98 @@ class StringToShogun : public AnyVisitor
 	}
 	void on(int64_t* v) final
 	{
+		SG_SDEBUG(
+				"int64: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 		if (!is_null())
 		{
-			SG_SDEBUG("int64: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
+
 			int64_t result = std::stol(m_string_val);
 			m_model->put(m_parameter, result);
 		}
 	}
 	void on(float* v) final
 	{
+		SG_SDEBUG(
+				"float: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 		if (!is_null())
 		{
-			SG_SDEBUG("float: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
-			char* end;
-			float32_t result = std::strtof(m_string_val.c_str(), &end);
+			float32_t result = std::stof(m_string_val);
 			m_model->put(m_parameter, result);
 		}
 	}
 	void on(double* v) final
 	{
+		SG_SDEBUG(
+				"double: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 		if (!is_null())
 		{
-			SG_SDEBUG("double: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
-			char* end;
-			float64_t result = std::strtod(m_string_val.c_str(), &end);
+			float64_t result = std::stod(m_string_val);
 			m_model->put(m_parameter, result);
 		}
 	}
 	void on(long double* v)
 	{
+		SG_SDEBUG(
+			"long double: %s=%s\n", m_parameter.c_str(),
+			m_string_val.c_str())
 		if (!is_null())
 		{
-			SG_SDEBUG("long double: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
-			char* end;
-			floatmax_t result = std::strtold(m_string_val.c_str(), &end);
+			floatmax_t result = std::stold(m_string_val);
 			m_model->put(m_parameter, result);
 		}
 	}
 	void on(CSGObject** v) final
 	{
-		SG_SDEBUG("CSGObject: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
+		SG_SDEBUG(
+		    "CSGObject: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 	}
 	void on(SGVector<int>* v) final
 	{
-		SG_SDEBUG("SGVector<int>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
+		SG_SDEBUG(
+		    "SGVector<int>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 	}
 	void on(SGVector<float>* v) final
 	{
-		SG_SDEBUG("SGVector<float>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
+		SG_SDEBUG(
+		    "SGVector<float>: %s=%s\n", m_parameter.c_str(),
+		    m_string_val.c_str())
 	}
 	void on(SGVector<double>* v) final
 	{
-		SG_SDEBUG("SGVector<double>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
+		SG_SDEBUG(
+		    "SGVector<double>: %s=%s\n", m_parameter.c_str(),
+		    m_string_val.c_str())
 	}
 	void on(SGMatrix<int>* mat) final
 	{
-		SG_SDEBUG("SGMatrix<int>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
+		SG_SDEBUG(
+		    "SGMatrix<int>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
 	}
 	void on(SGMatrix<float>* mat) final
 	{
-		SG_SDEBUG("SGMatrix<float>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
-	}
-	void on(SGMatrix<double>* mat) final
-	{
-		SG_SDEBUG("SGMatrix<double>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())
+		SG_SDEBUG(
+		    "SGMatrix<float>: %s=%s\n", m_parameter.c_str(),
+		    m_string_val.c_str())
 	}
-
-	bool is_null()
+	void on(SGMatrix<double>* mat) final{SG_SDEBUG(
+	    "SGMatrix<double>: %s=%s\n", m_parameter.c_str(), m_string_val.c_str())}
+
+	/**
+	 * In OpenML "null" is an empty parameter value field.
+	 * @return whether the field is "null"
+	 */
+	SG_FORCED_INLINE bool is_null()
 	{
 		bool result = strcmp(m_string_val.c_str(), "null") == 0;
 		return result;
 	}
 
-	void set_parameter_name(const std::string& name)
+	SG_FORCED_INLINE void set_parameter_name(const std::string& name)
 	{
 		m_parameter = name;
 	}
 
-	void set_string_value(const std::string& value)
+	SG_FORCED_INLINE void set_string_value(const std::string& value)
 	{
 		m_string_val = value;
 	}
@@ -396,17 +468,16 @@ class StringToShogun : public AnyVisitor
 std::shared_ptr<CSGObject> instantiate_model_from_factory(
     const std::string& factory_name, const std::string& algo_name)
 {
-	std::shared_ptr<CSGObject> obj;
 	if (factory_name == "machine")
-		obj = std::shared_ptr<CSGObject>(machine(algo_name));
-	else if (factory_name == "kernel")
-		obj = std::shared_ptr<CSGObject>(kernel(algo_name));
-	else if (factory_name == "distance")
-		obj = std::shared_ptr<CSGObject>(distance(algo_name));
-	else
-		SG_SERROR("Unsupported factory \"%s\".\n", factory_name.c_str())
+		return std::shared_ptr<CSGObject>(machine(algo_name));
+	if (factory_name == "kernel")
+		return std::shared_ptr<CSGObject>(kernel(algo_name));
+	if (factory_name == "distance")
+		return std::shared_ptr<CSGObject>(distance(algo_name));
 
-	return obj;
+	SG_SERROR("Unsupported factory \"%s\".\n", factory_name.c_str())
+
+	return nullptr;
 }
 
 /**
@@ -426,19 +497,21 @@ void cast_and_put(
 		//  temporary fix until shared_ptr PR merged
 		auto* tmp_clone = dynamic_cast<CMachine*>(casted_obj->clone());
 		obj->put(parameter_name, tmp_clone);
+		return;
 	}
-	else if (auto casted_obj = std::dynamic_pointer_cast<CKernel>(nested_obj))
+	if (auto casted_obj = std::dynamic_pointer_cast<CKernel>(nested_obj))
 	{
 		auto* tmp_clone = dynamic_cast<CKernel*>(casted_obj->clone());
 		obj->put(parameter_name, tmp_clone);
+		return;
 	}
-	else if (auto casted_obj = std::dynamic_pointer_cast<CDistance>(nested_obj))
+	if (auto casted_obj = std::dynamic_pointer_cast<CDistance>(nested_obj))
 	{
 		auto* tmp_clone = dynamic_cast<CDistance*>(casted_obj->clone());
 		obj->put(parameter_name, tmp_clone);
+		return;
 	}
-	else
-		SG_SERROR("Could not cast SGObject.\n")
+	SG_SERROR("Could not cast SGObject.\n")
 }
 
 std::shared_ptr<CSGObject> ShogunOpenML::flow_to_model(
@@ -447,8 +520,8 @@ std::shared_ptr<CSGObject> ShogunOpenML::flow_to_model(
 	auto params = flow->get_parameters();
 	auto components = flow->get_components();
 	auto class_name = get_class_info(flow->get_class_name());
-	auto module_name = std::get<0>(class_name);
-	auto algo_name = std::get<1>(class_name);
+	auto module_name = class_name.first;
+	auto algo_name = class_name.second;
 
 	auto obj = instantiate_model_from_factory(module_name, algo_name);
 	auto obj_param = obj->get_params();
@@ -486,12 +559,12 @@ ShogunOpenML::model_to_flow(const std::shared_ptr<CSGObject>& model)
 	return std::shared_ptr<OpenMLFlow>();
 }
 
-std::tuple<std::string, std::string>
+std::pair<std::string, std::string>
 ShogunOpenML::get_class_info(const std::string& class_name)
 {
 	std::vector<std::string> class_components;
 	auto begin = class_name.begin();
-	std::tuple<std::string, std::string> result;
+	std::pair<std::string, std::string> result;
 
 	for (auto it = class_name.begin(); it != class_name.end(); ++it)
 	{
@@ -503,15 +576,16 @@ ShogunOpenML::get_class_info(const std::string& class_name)
 		if (std::next(it) == class_name.end())
 			class_components.emplace_back(std::string(begin, std::next(it)));
 	}
-	if (class_components[0] == "shogun")
-		result = std::make_tuple(class_components[1], class_components[2]);
+
+	if (class_components[0] == "shogun" && class_components.size() == 3)
+		result = std::make_pair(class_components[1], class_components[2]);
+	else if (class_components[0] == "shogun" && class_components.size() != 3)
+		SG_SERROR("Invalid class name format %s.\n", class_name.c_str())
 	else
 		SG_SERROR(
 		    "The provided flow is not meant for shogun deserialisation! The "
 		    "required library is \"%s\".\n",
 		    class_components[0].c_str())
-	if (class_components.size() != 3)
-		SG_SERROR("Invalid class name format %s.\n", class_name.c_str())
 
 	return result;
 }