diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..916dbdca6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# Contributing to FlyteIDL + +## Docs structure +The index.rst files for protos are kept in parallel folder structure under the docs folder. +All the proto definitions are within protos/flyteidl and there corresponding docs are kept in protos/docs + +Each module in protos has same named module under the docs also. +eg : protos/flyteidl/core has same named doc structure placing it index and other documentation files in protos/docs/core + +## Docs Generation +Currently pseudomuto/protoc-gen-doc is being used for generating the MD files which are placed in protos/docs and corresponding folder based on the module + +protoc-gen-doc generates a single MD file if we include all the protos and doesn't provide a way to split these. +The MD files are then fed as input to sphinx for html doc generation +Inorder to modularize this and not have one single huge html file the MD files are being generated per module(eg :core,admin,plugin) +But due to this process if there references in html/md files which don't exist in the same file then they lead to broken links. + +eg: If we generate admin docs through this process , they would generate admin.md file in protos/docs/admin folder but if they reference docs from core +like for flyteidl.core.ResourceType then they would try to find it in the same MD/html file which wont be available + +Inorder to fix this issue an additional post build step has been added to fix such links. +Check conf.py for sphinx to see build-finished event handler + +Note : The plan is to move to generating RST file instead of MD files once this PR is checkedin https://github.com/pseudomuto/protoc-gen-doc/pull/440#issuecomment-845678919 +which would simplify the doc generation further + diff --git a/Makefile b/Makefile index 10a519031..48f0ab5a8 100644 --- a/Makefile +++ b/Makefile @@ -41,19 +41,7 @@ install-piptools: .PHONY: doc_gen_deps # these dependencies are required by protoc gen doc for the protos which have external library dependencies. # which includes grpc-gateway, googleapis, k8.io/api and apimachinery, protocolbuffers doc_gen_deps: - rm -rf tmp/doc_gen_deps tmp/protocolbuffers tmp/grpc-gateway tmp/k8s.io - git clone --depth 1 https://github.com/googleapis/googleapis tmp/doc_gen_deps - rm -rf tmp/doc_gen_deps/.git - git clone --depth 1 https://github.com/protocolbuffers/protobuf tmp/protocolbuffers - cp -r tmp/protocolbuffers/src/* tmp/doc_gen_deps/ - rm -rf tmp/protocolbuffers - git -c advice.detachedHead=false clone --depth 1 --branch v1.15.2 https://github.com/grpc-ecosystem/grpc-gateway tmp/grpc-gateway #v1.15.2 is used to keep the grpc-gateway version in sync with generated protos which is using the LYFT image - cp -r tmp/grpc-gateway/protoc-gen-swagger tmp/doc_gen_deps/ - rm -rf tmp/grpc-gateway - git clone --depth 1 https://github.com/kubernetes/api tmp/k8s.io/api - git clone --depth 1 https://github.com/kubernetes/apimachinery tmp/k8s.io/apimachinery - cp -r tmp/k8s.io tmp/doc_gen_deps/ - rm -rf tmp/k8s.io + ./scripts/doc_gen_deps.sh .PHONY: doc-requirements.txt doc-requirements.txt: doc-requirements.in install-piptools diff --git a/conf.py b/conf.py index d2edb7ddc..d9aea941d 100644 --- a/conf.py +++ b/conf.py @@ -15,8 +15,8 @@ import os import re import sys +import subprocess -sys.path.insert(0, os.path.abspath("gen/pb-protodoc/")) import recommonmark from recommonmark.transform import AutoStructify @@ -31,6 +31,9 @@ release = re.sub('^v', '', os.popen('git describe').read().strip()) version = release +# Project and scripts directories +PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) +post_process_docs_file=os.path.join(PROJECT_DIR, "scripts", "post_process_docs.sh") # -- General configuration --------------------------------------------------- @@ -192,6 +195,9 @@ intersphinx_mapping = { "python": ("https://docs.python.org/{.major}".format(sys.version_info), None), } +def build_finished_handler(app, exception): + process = subprocess.Popen([post_process_docs_file]) + process.wait() def setup(app): app.add_config_value('recommonmark_config', { @@ -201,3 +207,4 @@ def setup(app): 'enable_eval_rst': True, }, True) app.add_transform(AutoStructify) + app.connect('build-finished', build_finished_handler) diff --git a/protos/docs/admin/index.rst b/protos/docs/admin/index.rst index cb354d28a..b08c7e8f7 100644 --- a/protos/docs/admin/index.rst +++ b/protos/docs/admin/index.rst @@ -4,7 +4,7 @@ Flyte Admin Service entities These are the control plane entities that can be used to communication with the Flyte Admin service over gRPC or REST. The endpoint specification is defined in the -`Admin service spec `__ +`Admin raw protos `__ .. toctree:: :maxdepth: 1 diff --git a/protos/docs/core/index.rst b/protos/docs/core/index.rst index 2f427d1ee..a20d9f524 100644 --- a/protos/docs/core/index.rst +++ b/protos/docs/core/index.rst @@ -5,6 +5,8 @@ These set of protocol buffers provides details of some of the core data structures like Workflows, Tasks, Nodes, Literals. These are the specification of the various entities in Flyte as well as the type system. +`Core raw protos `__ + .. toctree:: :maxdepth: 1 :caption: core diff --git a/protos/docs/datacatalog/index.rst b/protos/docs/datacatalog/index.rst index aa9f8ff7c..4e203eb58 100644 --- a/protos/docs/datacatalog/index.rst +++ b/protos/docs/datacatalog/index.rst @@ -6,6 +6,8 @@ indexing parameterized, strongly-typed data artifacts across revisions. It is us to catalog artifacts generated by task executions. Outputs generated by a task can be stored as artifact data and tagged by the user so that it can be retrieved later by that tag. +`Datacatalog raw proto `__ + .. toctree:: :maxdepth: 1 :caption: datacatalog diff --git a/protos/docs/event/index.rst b/protos/docs/event/index.rst index e7ba039fc..a9541ba68 100644 --- a/protos/docs/event/index.rst +++ b/protos/docs/event/index.rst @@ -18,6 +18,8 @@ Flyte External Eventing - Event Egress This refers to the interface for all the event messages exiting from the Flyte **control plane** and delivered on the configured pubsub channel. +`Event raw proto `__ + .. toctree:: :maxdepth: 1 :caption: event diff --git a/protos/docs/plugins/index.rst b/protos/docs/plugins/index.rst index f10ce4161..90924ae45 100644 --- a/protos/docs/plugins/index.rst +++ b/protos/docs/plugins/index.rst @@ -4,6 +4,8 @@ Flyte Task Plugins These protocol buffer specifications provide information about the various Task Plugins available in the Flyte system. +`Plugins raw protos `__ + .. toctree:: :maxdepth: 1 :caption: plugins diff --git a/protos/docs/service/index.rst b/protos/docs/service/index.rst index d2d7a7644..8bcf45d9c 100644 --- a/protos/docs/service/index.rst +++ b/protos/docs/service/index.rst @@ -3,6 +3,8 @@ REST and gRPC interface for the Flyte Admin Service This section provides all endpoint defintions that are implemented by the Admin service. +`Admin service raw protos `__ + .. toctree:: :maxdepth: 1 :caption: service diff --git a/scripts/doc_gen_deps.sh b/scripts/doc_gen_deps.sh new file mode 100755 index 000000000..483270600 --- /dev/null +++ b/scripts/doc_gen_deps.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -x + +TMP_DEPS_FOLDER=tmp/doc_gen_deps +TMP_DEPS_PROTOBUF_FOLDER=tmp/protocolbuffers +TMP_DEPS_GRPC_GATEWAY_FOLDER=tmp/grpc-gateway +TMP_DEPS_K8S_IO=tmp/k8s.io + +# clear all the tmp deps folder +rm -rf $TMP_DEPS_FOLDER $TMP_DEPS_PROTOBUF_FOLDER $TMP_DEPS_GRPC_GATEWAY_FOLDER $TMP_DEPS_K8S_IO + +# clone deps and move them to TMP_DEPS_FOLDER +# googleapi deps +git clone --depth 1 https://github.com/googleapis/googleapis $TMP_DEPS_FOLDER +rm -rf $TMP_DEPS_FOLDER/.git + +# protobuf deps +git clone --depth 1 https://github.com/protocolbuffers/protobuf $TMP_DEPS_PROTOBUF_FOLDER +cp -r $TMP_DEPS_PROTOBUF_FOLDER/src/* $TMP_DEPS_FOLDER +rm -rf $TMP_DEPS_PROTOBUF_FOLDER + +# grpc-gateway deps +git -c advice.detachedHead=false clone --depth 1 --branch v1.15.2 https://github.com/grpc-ecosystem/grpc-gateway $TMP_DEPS_GRPC_GATEWAY_FOLDER #v1.15.2 is used to keep the grpc-gateway version in sync with generated protos which is using the LYFT image +cp -r $TMP_DEPS_GRPC_GATEWAY_FOLDER/protoc-gen-swagger $TMP_DEPS_FOLDER +rm -rf $TMP_DEPS_GRPC_GATEWAY_FOLDER + +# k8 dependencies +git clone --depth 1 https://github.com/kubernetes/api $TMP_DEPS_K8S_IO/api +git clone --depth 1 https://github.com/kubernetes/apimachinery $TMP_DEPS_K8S_IO/apimachinery +cp -r $TMP_DEPS_K8S_IO $TMP_DEPS_FOLDER +rm -rf $TMP_DEPS_K8S_IO diff --git a/scripts/post_process_docs.sh b/scripts/post_process_docs.sh new file mode 100755 index 000000000..dd122a1c6 --- /dev/null +++ b/scripts/post_process_docs.sh @@ -0,0 +1,19 @@ +#!/bin/bash +LINK_TO_FILE_PATH=`grep -r "a name=\"flyteidl" --include "*.html" . | sed 's/.*docs\/\([^:]*\).*a name="\([^"]*\).*/\2=..\/\1/'` + +echo "Fixing links in html files" +#use the link to file path map to fix the link with correct paths +SEDOPTION="-i" +if [[ "$OSTYPE" == "darwin"* ]]; then + SEDOPTION="-i ''" +fi + +for LINK_TO_FILE_ENTRY in $LINK_TO_FILE_PATH +do + LINK=$(echo $LINK_TO_FILE_ENTRY | cut -f1 -d"=") + ESCAPED_LINK=`echo $LINK|sed 's/\\//\\\\\//g'` + FILE_PATH=$(echo $LINK_TO_FILE_ENTRY | cut -f2 -d"=") + ESCAPED_FILE_PATH=`echo $FILE_PATH|sed 's/\\//\\\\\//g'` + find . -name "*.html" -exec sed $SEDOPTION "s/href=\"#$ESCAPED_LINK\"/href=\"$ESCAPED_FILE_PATH#$ESCAPED_LINK\"/" {} \; +done +echo "Links fixed"