Merge pull request #11585 from owncloud/fix_some_dev_docs_stuff

[docs-only] Fix docs build process

Author: mmattel

.drone.star

@@ -41,6 +41,9 @@ KEYCLOAK_IMAGE = "quay.io/keycloak/keycloak:26.2.5"
 POSTGRES_ALPINE_IMAGE = "postgres:alpine3.18"
 TRIVY_IMAGE = "aquasec/trivy:latest"
 
+# the hugo version needs to be the same as in owncloud.github.io
+OC_CI_HUGO_STATIC_IMAGE = "hugomods/hugo:base-0.129.0"
+
 DEFAULT_PHP_VERSION = "8.4"
 DEFAULT_NODEJS_VERSION = "22"
 
@@ -482,7 +485,7 @@ def main(ctx):
 
     build_release_helpers = \
         changelog() + \
-        docs() + \
+        documentation() + \
         licenseCheck(ctx)
 
     test_pipelines = \
@@ -2376,11 +2379,11 @@ def releaseDockerReadme(ctx, repo, build_type):
         },
     }
 
-def docs():
+def documentation():
     return [{
         "kind": "pipeline",
         "type": "docker",
-        "name": "docs",
+        "name": "documentation",
         "platform": {
             "os": "linux",
             "arch": "amd64",
@@ -2389,23 +2392,29 @@ def docs():
             {
                 "name": "docs-generate",
                 "image": OC_CI_GOLANG,
+                # "pull": "always",
                 "environment": DRONE_HTTP_PROXY_ENV,
                 "commands": ["make docs-generate"],
             },
             {
-                "name": "prepare",
+                "name": "docs-copy",
                 "image": OC_CI_GOLANG,
                 "environment": DRONE_HTTP_PROXY_ENV,
-                "commands": [
-                    "make -C docs docs-copy",
-                ],
+                "commands": ["make docs-copy"],
             },
             {
-                "name": "test",
+                "name": "docs-hugo-drone-prep",
                 "image": OC_CI_GOLANG,
                 "environment": DRONE_HTTP_PROXY_ENV,
+                "commands": ["make docs-hugo-drone-prep"],
+            },
+            {
+                "name": "docs-build",
+                "image": OC_CI_HUGO_STATIC_IMAGE,
+                "environment": DRONE_HTTP_PROXY_ENV,
                 "commands": [
-                    "make -C docs test",
+                    "cd hugo",
+                    "hugo",
                 ],
             },
             {
@@ -2437,6 +2446,7 @@ def docs():
                 "commands": [
                     "tree docs/hugo/public",
                     "rm -rf docs/hugo",
+                    "rm -rf hugo",
                 ],
             },
         ],

Makefile

@@ -81,9 +81,13 @@ endif
 include .make/recursion.mk
 
 .PHONY: help
+DEFAULT_GOAL := help
 help:
 	@echo "Please use 'make <target>' where <target> is one of the following:"
 	@echo
+	@echo -e "${GREEN}List all available .PHONY targets:${RESET}\n"
+	@echo -e "\tmake list\t\t${BLUE}sorted alphabetically${RESET}"
+	@echo -e "${BLACK}---------------------------------------------------------${RESET}"
 	@echo -e "${GREEN}Testing with test suite natively installed:${RESET}\n"
 	@echo -e "${PURPLE}\tdocs: https://owncloud.dev/ocis/development/testing/#testing-with-test-suite-natively-installed${RESET}\n"
 	@echo -e "\tmake test-acceptance-api\t\t${BLUE}run API acceptance tests${RESET}"
@@ -94,14 +98,13 @@ help:
 	@echo -e "${RED}You also should have a look at other available Makefiles:${RESET}"
 	@echo
 	@echo -e "${GREEN}oCIS:${RESET}\n"
-	@echo -e "${PURPLE}\tdocs: https://owncloud.dev/ocis/development/build/${RESET}\n"
+	@echo -e "${PURPLE}\tdocs: https://owncloud.dev/ocis/ocis/build-docs/${RESET}\n"
 	@echo -e "\tsee ./ocis/Makefile"
 	@echo -e "\tor run ${YELLOW}make -C ocis help${RESET}"
 	@echo
 	@echo -e "${GREEN}Documentation:${RESET}\n"
-	@echo -e "${PURPLE}\tdocs: https://owncloud.dev/ocis/development/build-docs/${RESET}\n"
-	@echo -e "\tsee ./docs/Makefile"
-	@echo -e "\tor run ${YELLOW}make -C docs help${RESET}"
+	@echo -e "${PURPLE}\tdocs: https://owncloud.dev/ocis/build-docs/${RESET}\n"
+	@echo -e "\trun ${YELLOW}make list | grep docs-\t\t${BLUE}note: run all docs command via this makefile${RESET}"
 	@echo
 	@echo -e "${GREEN}Testing with test suite in docker:${RESET}\n"
 	@echo -e "${PURPLE}\tdocs: https://owncloud.dev/ocis/development/testing/#testing-with-test-suite-in-docker${RESET}\n"
@@ -117,6 +120,12 @@ help:
 	@echo -e "\tmake test-gherkin-lint-fix\t${BLUE}apply lint fixes to gherkin feature files${RESET}"
 	@echo
 
+.PHONY: list
+list:
+	@echo -e 'Available .PHONY targets: \n'
+	@grep -P -o '(?<=^\.PHONY: )(.*)' Makefile | sort -u
+	@echo
+
 .PHONY: clean-tests
 clean-tests:
 	@rm -Rf vendor-bin/**/vendor vendor-bin/**/composer.lock tests/acceptance/output
@@ -159,33 +168,57 @@ clean:
         $(MAKE) --no-print-directory -C $$mod clean || exit 1; \
     done
 
-.PHONY: docs-generate
+# generate the docs
+# intents and comments are intentional...
+.PHONY: docs-generate          # 1. prepare docs
 docs-generate:
-	# empty the folders first to only have files that are generated without remnants
+	@echo 'Empty folders first to only have those files that are generated without remnants.'
 	find docs/services/_includes/ -type f \( -name "*" ! -name ".git*" \) -delete || exit 1
 
-	# generate the services md files for dev docs
+	@echo 'Generate content from services.'
 	@for mod in $(OCIS_MODULES); do \
 		$(MAKE) --no-print-directory -C $$mod docs-generate || exit 1; \
 	done
 
-	# generate envvar adoc and md tables for dev and admin docs
-	$(MAKE) --no-print-directory -C docs docs-generate || exit 1 
-
-	# only copy all added/removed/deprecated files that have been created for a release + .gitkeep
-	# note that files are locally ignored via _include/.gitignore for pushing, but required by the drone process
-	# for markdown, if e.g. an `_index.md` file would be present containing links, hugo thinks it must render and fails
-	# the -I flag followed by % cp % means that all passed arguments will concatenated to the last %
-	find docs/services/general-info/envvars/env-var-deltas/ -type f \( -name ".gitkeep" -o -name "*-added.*" -o -name "*-removed.*" -o -name "*-deprecated.*" \) | xargs -I % cp % docs/services/_includes/adoc/env-var-deltas/
-
-.PHONY: docs-drone-test
-	# imitate a full drone run to build docs without pushing to the web.
-	# this can help identify uncaught issues when running `make -C docs docs-serve` only.
-docs-drone-test:
-	$(MAKE) --no-print-directory docs-generate
-	$(MAKE) --no-print-directory -C docs docs-copy
-	$(MAKE) --no-print-directory -C docs test 
-
+	@$(MAKE) --no-print-directory -C docs docs-run-helpers || exit 1 
+
+# initialize the docs build environment
+	@$(MAKE) --no-print-directory -C docs docs-init
+
+# copy required resources into hugo/content
+.PHONY: docs-copy              # 2. copy required doc resources
+docs-copy:
+	@$(MAKE) --no-print-directory -C docs docs-copy
+
+# the docs-build|serve commands requires that docs-init was run first for the required data to exists
+# create a docs build
+.PHONY: docs-build             # 3. build prepared docs
+docs-build:
+	@$(MAKE) --no-print-directory -C docs docs-build
+
+# serve built docs with hugo
+.PHONY: docs-serve             # serve the docs build
+docs-serve:
+	@$(MAKE) --no-print-directory -C docs docs-serve
+
+# clean up doc build artifacts 
+.PHONY: docs-clean             # clean all docs artifacts, must be run as sudo
+docs-clean:
+	@$(MAKE) --no-print-directory -C docs docs-clean
+
+# imitate a full drone run locally to build docs without pushing to the web.
+# this can help identify uncaught issues when running `make docs-serve` only.
+.PHONY: docs-local             # run all steps as drone would do it (1, 2, 3)
+docs-local:
+	@$(MAKE) --no-print-directory docs-generate
+	@$(MAKE) --no-print-directory docs-copy
+	@$(MAKE) --no-print-directory docs-build 
+
+# prepare a link from the root to the hugo folder because the image requires it
+# note that on local building, the referenced container of inside the hugo/makefile is used
+.PHONY: docs-hugo-drone-prep   # only used for drone !
+docs-hugo-drone-prep:
+	@$(MAKE) --no-print-directory -C docs docs-hugo-drone-prep
 
 .PHONY: check-env-var-annotations
 check-env-var-annotations:

docs/.gitignore

@@ -1,5 +1,4 @@
 hugo/
 grpc_apis/
-mutagen.yml.lock
 helpers/output/*
 services/**/_index.md

docs/Makefile

@@ -1,55 +1,96 @@
 SHELL := bash
+# we need an override so that this variable is cleanly set in this makefile
+# else it might come from somewhere else
+override HUGO := $(CURDIR)/hugo
+override HDIR := $(HUGO)/.git
 
-include ../.bingo/Variables.mk
+# if the directory exists, print that we do not need to prepare
+hugo-exists:
+	@if [ -d $(HDIR) ]; then \
+		echo; \
+		echo 'Build environment for hugo exists, nothing to setup, continuing.'; \
+	fi
 
+# if the directory does not extist, initialize the framework and create the theme
+$(HDIR):
+	@echo 'Initialize the hugo framework and build the theme, no docs get built.'
+	@mkdir -p $(HUGO)/content/
+	@mkdir -p $(HUGO)/extensions/
+	@mkdir -p $(HUGO)/public/
+	@cd $(HUGO) && git init
+	@cd $(HUGO) && git config advice.detachedHead false
+	@cd $(HUGO) && git remote rm origin || true
+	@cd $(HUGO) && git remote add origin https://github.com/owncloud/owncloud.github.io
+	@cd $(HUGO) && git fetch --depth=1
+	@cd $(HUGO) && git checkout origin/main -f
+	@$(MAKE) -C $(HUGO) --no-print-directory theme
+
+# print available make targets
 .PHONY: help
+.DEFAULT_GOAL := help
 help:
-	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk -F'[:;##]' '{printf "\033[36m%-30s\033[0m %s\n", $$2, $$NF}'
+	@echo -e 'Available .PHONY targets: \n'
+	@grep -P -o '(?<=^\.PHONY: )(.*)' Makefile | sort -u
+	@echo
+
+# alternative target because we also use it in the root's makefile
+.PHONY:list
+list: help
 
-.PHONY: docs-generate
-docs-generate: ## run docs-generate for all oCIS services
+# for drone only. drone uses an image for hugo that needs the hugo folder in the root
+# CURDIR will work for make only but not when running manually in the shell
+.PHONY: docs-hugo-drone-prep
+docs-hugo-drone-prep:
+	@echo
+	@echo 'Linking directory hugo for drone.'
+	@rm -f ../hugo
+	@ln -s $(HUGO) ../hugo
+
+# generate data from sources such as services
+.PHONY: docs-run-helpers
+docs-run-helpers:
+	@echo
+	@echo 'Generate envvar adoc and md tables for all oCIS services used for dev and admin docs'
 	@pushd helpers && go run .; popd
 
+# create the docs build environment 
 .PHONY: docs-init
-docs-init:
-	@mkdir -p hugo/content/
-	@mkdir -p hugo/public/
-	@touch hugo/public/.nojekyll
-	@cd hugo && git init
-	@cd hugo && git config advice.detachedHead false
-	@cd hugo && git remote rm origin || true
-	@cd hugo && git remote add origin https://github.com/owncloud/owncloud.github.io
-	@cd hugo && git fetch --depth=1
-	@cd hugo && git checkout origin/main -f
-	@$(MAKE) -C hugo theme
+docs-init: hugo-exists $(HDIR)
+# if there is nothing to do, stop printing that
+# another way would be https://stackoverflow.com/questions/58039810/makefiles-what-is-an-order-only-prerequisite
+	@:
 
-.PHONY: docs-serve
-docs-serve: docs-init docs-generate docs-copy ## serve docs with hugo
-	@bash -c "trap 'trap - SIGINT SIGTERM ERR; $(MAKE) --no-print-directory docs-sync-stop; exit 0' SIGINT SIGTERM ERR; $(MAKE) --no-print-directory docs-sync-start && $(MAKE) --no-print-directory hugo-serve"
+# the docs-build|serve commands require that docs-init was run first for the required data to exists
+# create a docs build
+.PHONY: docs-build
+docs-build:
+	@echo
+	@echo 'Building ocis for owncloud.dev'
+	@$(MAKE) -C $(HUGO) --no-print-directory hugo-build
+	@echo 'To view the rendered docs in the browser, run: make docs-serve'
 
-.PHONY: test
-test: $(HUGO)
-	@cd hugo && $(HUGO)
-
-.PHONY: hugo-serve
-hugo-serve: $(HUGO)
-	@cd hugo && $(HUGO) server
+# serve built docs with hugo
+.PHONY: docs-serve
+docs-serve:
+	@$(MAKE) -C $(HUGO) --no-print-directory hugo-server
 
+# copy required resources into hugo
 .PHONY: docs-copy
-docs-copy: docs-init docs-sync-start docs-sync-stop
-
-.PHONY: docs-sync-start
-docs-sync-start: $(MUTAGEN)
-	@$(MUTAGEN) project terminate || true
-	@$(MUTAGEN) daemon stop || true
-	@$(MUTAGEN) project start
-	@$(MUTAGEN) project flush
-
-.PHONY: docs-sync-stop
-docs-sync-stop: $(MUTAGEN)
-	@$(MUTAGEN) project terminate
-	@$(MUTAGEN) daemon stop
-
-.PHONY: clean
-clean: ## clean up docs build artifacts
-	@rm -rf hugo
+docs-copy:
+	@echo
+	@echo 'Syncing required doc files and directories for the build process.'
+# brace expansion not with sh
+	@rsync --delete -ax --exclude={hugo,templates,Makefile,.gitignore,README.md} ./ $(HUGO)/content/
+
+# clean up docs build artifacts (removing the hugo folder)
+.PHONY: docs-clean
+docs-clean:
+# NOTE: the whitespace at the beginning of the ifeq/else/endif statements is intentional.
+	@echo
+ 	ifneq ($(shell id -u), 0)
+		@echo "You must run the clean command using sudo or as root because some doc files are created by root"
+ 	else
+		@echo 'Clean up docs build artifacts'
+		@rm -rf $(HUGO)
+		@echo "Removed folder: $(HUGO)"
+ 	endif