Comment Support in JSON output (#79)

* wip

* wip

* wip

* first pass at a fleshed out `ProcessComment`

* roll up paragraph comments into their parents

* comments now included in standard declaration json as `inline_comments`

* removing uncessary calls

* tweak

* more post-processing of commands for json output

* formatting

* adding all `hyde` values under a top-level `hyde` YAML key

* starting to inject some doxygen comments into the YAML

* wip

* enumerations appear to be documented well enough at this point.

* incorporating inline support throughout the emitters

* progress into better docs for classes

* support for compiler-provided (implicit) implementation annotations among other tweaks

* more tweaks to example files; also cdor/dtor briefs are optional

* ownership inheritance and brief/description function roll-up

* tweak

* parameter directions and another sample file to flex the kinds of inline documentation we may see. Support should be extended to some of these commands more formally (e.g., throw or warning.)

* more sample documentation

* more ease-of-use features around implicit routines and ownership bubble-up

* idempotency: making sure the second in two back-to-back updates changes nothing

* formatting

* more bug fixes

* better `have` YAML processing

* working on =default or =delete -> optional instead of required

* better compiler-managed (implicit, `=default`, `=delete`) routine handling

* Moving the fixup functionality to its own standalone mode

Author: fosterbrereton

README.md

@@ -83,3 +83,17 @@ To validate pre-existing YAML:
 
 To output updated YAML:
 ```./hyde -use-system-clang -hyde-yaml-dir=/path/to/output -hyde-update ../test_files/classes.cpp```
+
+# Hyde 1 to Hyde 2 Format Conversion
+
+As of the Hyde 2 work, all subfields in the YAML output (except the Jekyll-required `layout` and `title` fields) must go under a top-level `hyde` subfield. This allows for other tools to include additional (possibly same-named) fields under their own top-level subfields in the YAML.
+
+Here is an example of updating from Hyde 1 to Hyde 2 formatted docs by scanning a directory for markdown-formatted files and passing them to `hyde` with the new `-hyde-fixup-subfield` mode:
+
+   find . -name '*.md' | xargs -I % -L 1 /path/to/hyde -hyde-fixup-subfield % --
+
+# Sass Updates
+
+Sometimes it may be necessary to clean up or "lint" the sass files. You can do so with:
+
+    bundle exec sass-convert -i /path/to/file.scss

docs/libraries/index.md

@@ -6,7 +6,7 @@ NOTICE: Adobe permits you to use, modify, and distribute this file in
 accordance with the terms of the Adobe license agreement accompanying
 it. If you have received this file from a source other than Adobe,
 then your use, modification, or distribution of it requires the prior
-written permission of Adobe. 
+written permission of Adobe.
 */
 
 #pragma once
@@ -47,36 +47,58 @@ struct file_checker {
 
 /**************************************************************************************************/
 
+inline bool has_json_flag(const json& j, const char* k) {
+    return j.count(k) && j.at(k).get<bool>();
+}
+
+/**************************************************************************************************/
+
 struct yaml_base_emitter {
 public:
     yaml_base_emitter(std::filesystem::path src_root,
                       std::filesystem::path dst_root,
                       yaml_mode mode,
                       emit_options options,
                       bool editable_title = false)
-        : _src_root(std::move(src_root)), _dst_root(std::move(dst_root)), _mode(mode), _options(std::move(options)), _editable_title{editable_title} {}
+        : _src_root(std::move(src_root)), _dst_root(std::move(dst_root)), _mode(mode),
+          _options(std::move(options)), _editable_title{editable_title} {}
 
-    virtual bool emit(const json& j, json& out_emitted) = 0;
+    /// @param matched The json given to us by the matcher engine
+    /// @param output The resulting output of this call
+    /// @param inherited Any inherited fields from parent constructs, e.g., passing
+    ///                  the owner of a class to its members
+    /// @return `true` if an error took place during emit; `false` otherwise.
+    virtual bool emit(const json& matched, json& output, const json& inherited) = 0;
 
 protected:
-    json base_emitter_node(std::string layout, std::string title, std::string tag);
+    json base_emitter_node(std::string layout, std::string title, std::string tag, bool implicit);
 
-    bool reconcile(json node, std::filesystem::path root_path, std::filesystem::path path, json& out_reconciled);
+    bool reconcile(json node,
+                   std::filesystem::path root_path,
+                   std::filesystem::path path,
+                   json& out_reconciled);
 
-    std::string defined_in_file(const std::string& src_path,
-                                const std::filesystem::path& src_root);
+    std::string defined_in_file(const std::string& src_path, const std::filesystem::path& src_root);
 
     std::filesystem::path subcomponent(const std::filesystem::path& src_path,
-                                         const std::filesystem::path& src_root);
+                                       const std::filesystem::path& src_root);
 
-    void maybe_annotate(const json& j, json& node); // make out arg?
+    // For some reason nlohmann's JSON types aren't happy with my moving them about
+    // (they always seem to show up null (~) in the final YAML output)
+    // so converting these to out-arg-based routines will have to wait until I can
+    // sort that out.
+    void insert_inherited(const json& inherited, json& node); // make out arg?
+    void insert_annotations(const json& j, json& node);       // make out arg?
+    void insert_doxygen(const json& j, json& node);           // make out arg?
 
     std::string format_template_parameters(const json& json, bool with_types);
 
     std::string filename_filter(std::string f);
     std::string filename_truncate(std::string s);
 
-    void insert_typedefs(const json& j, json& node);
+    void insert_typedefs(const json& j, json& node, const json& inherited);
+
+    void check_inline_comments(const json& expected, json& out_merged);
 
     bool check_typedefs(const std::string& filepath,
                         const json& have_node,
@@ -143,6 +165,8 @@ struct yaml_base_emitter {
                    const std::string& key,
                    const check_proc& proc);
 
+    static bool has_inline_field(const json& j, const char* field);
+
 private:
     template <typename Arg, typename... Args>
     std::filesystem::path dst_path_append(std::filesystem::path p, Arg&& arg, Args&&... args);
@@ -206,7 +230,9 @@ std::filesystem::path yaml_base_emitter::dst_path(const json& j, Args&&... args)
 /**************************************************************************************************/
 
 template <typename Arg, typename... Args>
-std::filesystem::path yaml_base_emitter::dst_path_append(std::filesystem::path p, Arg&& arg, Args&&... args) {
+std::filesystem::path yaml_base_emitter::dst_path_append(std::filesystem::path p,
+                                                         Arg&& arg,
+                                                         Args&&... args) {
     return dst_path_append(dst_path_append(std::move(p), arg), std::forward<Args>(args)...);
 }
 

emitters/yaml_base_emitter.cpp

@@ -6,15 +6,19 @@ NOTICE: Adobe permits you to use, modify, and distribute this file in
 accordance with the terms of the Adobe license agreement accompanying
 it. If you have received this file from a source other than Adobe,
 then your use, modification, or distribution of it requires the prior
-written permission of Adobe. 
+written permission of Adobe.
 */
 
 #pragma once
 
 // stdc++
+#include <filesystem>
 #include <stdexcept>
 #include <string>
 
+// application
+#include "json.hpp"
+
 /**************************************************************************************************/
 
 namespace hyde {
@@ -24,16 +28,12 @@ namespace hyde {
 static constexpr char const* tag_value_missing_k = "__MISSING__";
 static constexpr char const* tag_value_optional_k = "__OPTIONAL__";
 static constexpr char const* tag_value_deprecated_k = "__DEPRECATED__";
+static constexpr char const* tag_value_inlined_k = "__INLINED__";
 static constexpr char const* index_filename_k = "index.md";
 
 /**************************************************************************************************/
 
-enum class attribute_category {
-    disabled,
-    required,
-    optional,
-    deprecated
-};
+enum class attribute_category { disabled, required, optional, deprecated, inlined };
 
 static constexpr char const* get_tag(attribute_category c) {
     switch (c) {
@@ -43,25 +43,37 @@ static constexpr char const* get_tag(attribute_category c) {
             return tag_value_optional_k;
         case attribute_category::deprecated:
             return tag_value_deprecated_k;
+        case attribute_category::inlined:
+            return tag_value_inlined_k;
         default:
             throw std::invalid_argument("unexpected attribute category");
     }
 }
 
-static inline bool is_tag(const std::string& s) {
-    return s.substr(0, 2) == "__";
-}
+static inline bool is_tag(const std::string& s) { return s.substr(0, 2) == "__"; }
 
 /**************************************************************************************************/
 
 struct emit_options {
     attribute_category _tested_by{attribute_category::disabled};
     bool _ignore_extraneous_files{false};
-    bool _fixup_hyde_subfield{false};
 };
 
 /**************************************************************************************************/
 
+struct documentation {
+    json _json;
+    std::string _remainder;
+    bool _error{false};
+};
+
+documentation parse_documentation(const std::filesystem::path& path, bool fixup_subfield);
+
+/// @return `true` on failure to write, `false` otherwise.
+bool write_documentation(const documentation& docs, const std::filesystem::path& path);
+
+/**************************************************************************************************/
+
 } // namespace hyde
 
 /**************************************************************************************************/

emitters/yaml_base_emitter.hpp

@@ -6,7 +6,7 @@ NOTICE: Adobe permits you to use, modify, and distribute this file in
 accordance with the terms of the Adobe license agreement accompanying
 it. If you have received this file from a source other than Adobe,
 then your use, modification, or distribution of it requires the prior
-written permission of Adobe. 
+written permission of Adobe.
 */
 
 // identity
@@ -43,14 +43,18 @@ bool yaml_class_emitter::do_merge(const std::string& filepath,
             bool failure{false};
 
             failure |= check_scalar(filepath, have, expected, nodepath, out_merged, "type");
-            failure |= check_editable_scalar(filepath, have, expected, nodepath, out_merged, "description");
-            failure |= check_scalar_array(filepath, have, expected, nodepath, out_merged, "annotation");
+            failure |= check_editable_scalar(filepath, have, expected, nodepath, out_merged,
+                                             "description");
+            failure |=
+                check_scalar_array(filepath, have, expected, nodepath, out_merged, "annotation");
+
+            check_inline_comments(expected, out_merged);
 
             return failure;
         });
 
     failure |= check_typedefs(filepath, have, expected, "", out_merged);
-    
+
     failure |= check_object_array(
         filepath, have, expected, "", out_merged, "methods", "title",
         [this](const std::string& filepath, const json& have, const json& expected,
@@ -59,15 +63,20 @@ bool yaml_class_emitter::do_merge(const std::string& filepath,
             return function_emitter.do_merge(filepath, have, expected, out_merged);
         });
 
+    check_inline_comments(expected, out_merged);
+
     return failure;
 }
 
 /**************************************************************************************************/
 
-bool yaml_class_emitter::emit(const json& j, json& out_emitted) {
-    json node = base_emitter_node("class", j["name"], "class");
+bool yaml_class_emitter::emit(const json& j, json& out_emitted, const json& inherited) {
+    json node = base_emitter_node("class", j["name"], "class", has_json_flag(j, "implicit"));
     node["hyde"]["defined_in_file"] = defined_in_file(j["defined_in_file"], _src_root);
-    maybe_annotate(j, node);
+
+    insert_inherited(inherited, node["hyde"]);
+    insert_annotations(j, node["hyde"]);
+    insert_doxygen(j, node["hyde"]);
 
     std::string declaration = format_template_parameters(j, true) + '\n' +
                               static_cast<const std::string&>(j["kind"]) + " " +
@@ -84,26 +93,31 @@ bool yaml_class_emitter::emit(const json& j, json& out_emitted) {
         for (const auto& field : j["fields"]) {
             const std::string& key = field["name"];
             auto& field_node = node["hyde"]["fields"][key];
+            insert_annotations(field, field_node);
+            insert_doxygen(field, field_node);
+
             field_node["type"] = static_cast<const std::string&>(field["type"]);
-            field_node["description"] = tag_value_missing_k;
-            maybe_annotate(field, field_node);
+            const bool inline_description_exists =
+                field_node.count("inline") && field_node["inline"].count("description");
+            field_node["description"] =
+                inline_description_exists ? tag_value_inlined_k : tag_value_missing_k;
         }
     }
 
-    insert_typedefs(j, node);
+    insert_typedefs(j, node, inherited);
 
-    auto dst = dst_path(j,
-                        static_cast<const std::string&>(j["name"]));
+    auto dst = dst_path(j, static_cast<const std::string&>(j["name"]));
 
-    bool failure = reconcile(std::move(node), _dst_root, std::move(dst) / index_filename_k, out_emitted);
+    bool failure =
+        reconcile(std::move(node), _dst_root, std::move(dst) / index_filename_k, out_emitted);
 
     const auto& methods = j["methods"];
     yaml_function_emitter function_emitter(_src_root, _dst_root, _mode, _options, true);
 
     for (auto it = methods.begin(); it != methods.end(); ++it) {
         function_emitter.set_key(it.key());
         auto function_emitted = hyde::json::object();
-        failure |= function_emitter.emit(it.value(), function_emitted);
+        failure |= function_emitter.emit(it.value(), function_emitted, out_emitted.at("hyde"));
         out_emitted["methods"].push_back(std::move(function_emitted));
     }
 

emitters/yaml_base_emitter_fwd.hpp

@@ -6,7 +6,7 @@ NOTICE: Adobe permits you to use, modify, and distribute this file in
 accordance with the terms of the Adobe license agreement accompanying
 it. If you have received this file from a source other than Adobe,
 then your use, modification, or distribution of it requires the prior
-written permission of Adobe. 
+written permission of Adobe.
 */
 
 #pragma once
@@ -28,7 +28,7 @@ struct yaml_class_emitter : public yaml_base_emitter {
                                 emit_options options)
         : yaml_base_emitter(std::move(src_root), std::move(dst_root), mode, std::move(options)) {}
 
-    bool emit(const json& j, json& out_emitted) override;
+    bool emit(const json& matched, json& output, const json& inherited) override;
 
     bool do_merge(const std::string& filepath,
                   const json& have,