Skip to content

feat(RELEASE-1675): auto generate README.md files for tasks/pipelines (#1) #1157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
johnbieren merged 1 commit into from
Jul 9, 2025
Merged

feat(RELEASE-1675): auto generate README.md files for tasks/pipelines (#1) #1157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 160 additions & 0 deletions .github/scripts/check_readme.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
#!/bin/bash

# This script will verify that the README.md of all task and
# pipeline directories provided matches the output of hack/readme_generator.sh
# to ensure that README files are up to date.
#
# Task and pipeline directories are provided
# either via README_ITEMS env var, or as arguments
# when running the script.

CLI_README_ITEMS=()
FAILED_ITEMS=()
FAILED_PARAMS=()

Comment thread
seanconroy2021 marked this conversation as resolved.
show_help() {
echo "Usage: $0 [item1] [item2] [...]"
echo
echo Flags:
echo " --help: Show this help message"
echo
echo "Items are task or pipeline directories. They can be supplied"
echo "either as arguments or via the README_ITEMS environment variable."
Comment thread
FilipNikolovski marked this conversation as resolved.
echo
echo "Examples:"
echo " $0 tasks/managed/apply-mapping tasks/internal/*"
echo
echo " or"
echo
echo ' export README_ITEMS="tasks/managed/apply-mapping some/other/dir"'
echo " $0"
exit 1
}

while [[ $# -gt 0 ]]; do
case "$1" in
--help)
show_help
;;
--*)
show_help
;;
*)
CLI_README_ITEMS+=("$1")
shift
;;
esac
done

if [[ "${#CLI_README_ITEMS[@]}" -gt 0 ]]; then
README_ITEMS=("${CLI_README_ITEMS[@]}")
else
read -r -a README_ITEMS <<< "${README_ITEMS[@]}"
fi

if [ "${#README_ITEMS[@]}" -eq 0 ]
then
show_help
fi

# Check that all directories exist. If not, fail
for ITEM in "${README_ITEMS[@]}"
do
if [[ -d "$ITEM" ]]; then
true
else
echo "Error: Invalid file or directory: $ITEM"
exit 1
fi
done

for ITEM in "${README_ITEMS[@]}"
do
echo "Task/Pipeline item: $ITEM"
ITEM_NAME=$(basename "$ITEM")
ITEM_DIR=$(cut -d '/' -f -3 <<< "$ITEM")
echo " Task/Pipeline name: $ITEM_NAME"

ITEM_PATH=${ITEM_DIR}/${ITEM_NAME}.yaml
if [ ! -f "$ITEM_PATH" ]
then
echo " Error: Task/Pipeline file does not exist: $ITEM_PATH"
exit 1
fi

if [[ $(yq ".spec | has(\"description\")" "$ITEM_PATH") == "false" ]]; then
echo " Error in $ITEM: Field '.spec.description' is missing in task"
FAILED_ITEMS+=("$ITEM - no description")
fi

# Check description doesn't have '.' or ','
PARAMS=$(yq .spec.params "$ITEM_PATH")
for ((i=0; i < $(yq length <<< "$PARAMS"); i++)); do
# Get rid of newlines and remove trailing whitespace
DESCRIPTION="$(yq .[$i].description <<< "$PARAMS" | tr '\n' ' ' | sed 's/[[:space:]]*$//')"
NAME="$(yq .[$i].name <<< "$PARAMS" | tr '\n' ' ' | sed 's/[[:space:]]*$//')"

if [[ $(yq ".[$i] | has(\"description\")" <<< "$PARAMS") == "false" ]]; then
echo " Error in $ITEM: Field '.param.[$i].description' ($NAME) is missing in task"
FAILED_PARAMS+=("$ITEM_PATH: $NAME - missing description")
elif [[ "${DESCRIPTION: -1}" =~ [.,] ]]; then
echo " Error in $NAME: Description should not end with a '${DESCRIPTION: -1}'"
FAILED_PARAMS+=("$ITEM_PATH: $NAME - invalid description")
fi
done

README_PATH=${ITEM_DIR}/README.md
if [ ! -f "$README_PATH" ]
then
echo " Error: README does not exist in $ITEM_DIR"
FAILED_ITEMS+=("$ITEM - no README.md")
continue
fi

if ! diff -u <(.github/scripts/readme_generator.sh --dry-run "$ITEM_DIR") "$README_PATH"; then
echo " Error: README.md has not been updated. Please use hack/readme-generator.sh to" \
"generate a new README.md to replace $ITEM/README.md"
FAILED_ITEMS+=("$ITEM - outdated README.md")
else
echo " README.md for $ITEM_DIR is up to date"
fi

done

if [[ "${#FAILED_PARAMS[@]}" -eq 0 ]]; then
echo
echo "All task/pipeline parameter descriptions are valid."
else
echo
echo "Error: these task/pipeline parameter descriptions must be updated:"
for PARAM in "${FAILED_PARAMS[@]}"; do
echo " $PARAM"
done

if [[ "${#FAILED_ITEMS[@]}" -eq 0 ]]; then
exit 1
fi
fi

if [[ "${#FAILED_ITEMS[@]}" -eq 0 ]]; then
echo
echo "All README.md files are up to date"
else
echo
echo "Error: these task/pipeline README.md files must be updated:"
for ITEM in "${FAILED_ITEMS[@]}"; do
echo " $ITEM"
done
echo
if [[ "${#FAILED_PARAMS[@]}" -eq 0 ]]; then
echo "Add missing descriptions to tasks/pipelines, then try running the" \
"following command to fix this:"
else
echo "Fix any invalid or missing parameter descriptions and add missing descriptions" \
"to tasks/pipelines, then try running the following command to fix this:"
fi
# Get unique items in array
FAILED_ITEMS=($(printf "%s\n" "${FAILED_ITEMS[@]% - *}" | sort -u ))
echo "./.github/scripts/readme_generator.sh" "${FAILED_ITEMS[@]}"
exit 1
fi
212 changes: 212 additions & 0 deletions .github/scripts/readme_generator.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
#!/usr/bin/env bash

# Automatically generates a README.md for tasks and pipelines.
# Without the '--dry-run' flag, this will automatically replace the current task/pipeline README.md
# A check will be run on each pull request to make sure the README.md matches this script's output.
#
# This script takes the '.spec.description' and '.spec.params' fields
# from the associated task/pipeline to create the description and table in the README.md of this task/pipeline.
#
# If you wish to update the README.md description and table, please update the '.spec.description' or '.spec.table'
# field in the Tekton task/pipeline and run this script, instead of changing it manually in the README.md.
#
# Usage: ./readme_generator.sh tasks/apply-mapping

show_help() {
echo "Usage: $0 [--dry-run] [item1] [item2] [...]"
echo
echo Flags:
echo " --help: Show this help message"
echo " --dry-run: Print the updated README.md files without changing"
echo " the current README.md in each task and pipeline"
echo
echo "Items are task or pipeline directories. They can be supplied"
echo "either as arguments or via the README_ITEMS environment variable."
echo
echo "Examples:"
echo " $0 tasks/managed/apply-mapping"
echo " $0 --dry-run tasks/managed/apply-mapping"
echo
echo " or"
echo
echo ' export README_ITEMS="mydir/tasks/apply-mapping some/other/dir"'
echo " $0"
exit 1
}

DRY_RUN=false
CLI_README_ITEMS=()

while [[ $# -gt 0 ]]; do
case "$1" in
--dry-run)
DRY_RUN=true
shift
;;
--help)
show_help
;;
--*)
show_help
;;
*)
CLI_README_ITEMS+=("$1")
shift
;;
esac
done

if [[ "${#CLI_README_ITEMS[@]}" -gt 0 ]]; then
README_ITEMS=("${CLI_README_ITEMS[@]}")
else
read -r -a README_ITEMS <<< "${README_ITEMS[@]}"
fi

if [ "${#README_ITEMS[@]}" -eq 0 ]
then
show_help
fi

# Check that all directories exist. If not, fail
for ITEM in "${README_ITEMS[@]}"
do
if [[ ! -d "$ITEM" ]]; then
echo "Error: Invalid file or directory: $ITEM"
exit 1
fi

ITEM_NAME=$(basename "$ITEM")
ITEM_DIR=$(cut -d '/' -f -3 <<< "$ITEM")

ITEM_PATH=${ITEM_DIR}/${ITEM_NAME}.yaml
if [ ! -f "$ITEM_PATH" ]
then
echo "Error: Task/Pipeline file does not exist: $ITEM_PATH"
exit 1
fi
done

# Creating after checking all directories exist to simplify cleanup
TEMP_README=$(mktemp)

# Adds trailing whitespace/hyphens to table until num_spaces is reached
spaces() { # (num_spaces, string)
for ((i=0; i < $(($1-${#2})); i++)); do
echo -n " "
done
}

dashes() { # (num_spaces, string)
for ((i=0; i < $(($1-${#2})); i++)); do
echo -n "-"
done
}

# Add table
for ITEM in "${README_ITEMS[@]}"
do
ITEM_DIR=$(cut -d '/' -f -3 <<< "$ITEM")
BASE_DIR=$(cut -d '/' -f 1 <<< "$ITEM")
ITEM_NAME=$(basename "$ITEM")
ITEM_PATH=${ITEM_DIR}/${ITEM_NAME}.yaml

# Don't print any debug messages when dry run is true
$DRY_RUN || echo "Task/Pipeline item: $ITEM"
$DRY_RUN || echo " Task/Pipeline name: $ITEM_NAME"

# Variables for description of README.md
METADATA_NAME=$(yq .metadata.name "$ITEM_PATH")
SPEC_DESCRIPTION=$(yq .spec.description "$ITEM_PATH")

# Variables for table
PARAMS=$(yq .spec.params "$ITEM_PATH")

# Get the maximum length for each column of table
LONGEST_NAME=0
LONGEST_DESCRIPTION=0
LONGEST_DEFAULT=13
LONGEST_OPTIONAL=8

for ((i=0; i < $(yq length <<< "$PARAMS"); i++)); do
# Get rid of newlines and remove trailing whitespace
NAME="$(yq .[$i].name <<< "$PARAMS" | tr '\n' ' ' | sed 's/[[:space:]]*$//')"
DESCRIPTION="$(yq .[$i].description <<< "$PARAMS" | tr '\n' ' ' | sed 's/[[:space:]]*$//')"
DEFAULT="$(yq .[$i].default <<< "$PARAMS" | tr '\n' ' ' | sed 's/[[:space:]]*$//')"

if [[ ${#NAME} -gt $LONGEST_NAME ]]; then
LONGEST_NAME=${#NAME}
fi

if [[ ${#DESCRIPTION} -gt $LONGEST_DESCRIPTION ]]; then
LONGEST_DESCRIPTION=${#DESCRIPTION}
fi

if [[ ${#DEFAULT} -gt $LONGEST_DEFAULT ]]; then
LONGEST_DEFAULT=${#DEFAULT}
fi
done

# create table and write contents to file
{
if [[ "$BASE_DIR" == "pipelines" && "$ITEM_NAME" != *-pipeline ]]; then
echo "# $METADATA_NAME pipeline"
elif [[ "$BASE_DIR" == "pipelines" && "$ITEM_NAME" == *-pipeline ]]; then
echo "# ${METADATA_NAME%-pipeline} pipeline"
else
echo "# $METADATA_NAME"
fi
echo
echo "$SPEC_DESCRIPTION"
echo
echo "## Parameters"
echo

# Print first row of table
echo "| Name$(spaces "$LONGEST_NAME" "Name")" \
"| Description$(spaces "$LONGEST_DESCRIPTION" "Description")" \
"| Optional$(spaces "$LONGEST_OPTIONAL" "Optional")" \
"| Default value$(spaces "$LONGEST_DEFAULT" "Default value") |"

# Print second row of table
echo -n "|-$(dashes "$LONGEST_NAME" "")-|-$(dashes "$LONGEST_DESCRIPTION" "")"
echo "-|-$(dashes "$LONGEST_OPTIONAL" "")-|-$(dashes "$LONGEST_DEFAULT" "")-|"

# Print remaining rows of table
for ((i=0; i < $(yq length <<< "$PARAMS"); i++)); do
# Get rid of newlines and remove trailing whitespace
NAME="$(yq .[$i].name <<< "$PARAMS" | tr '\n' ' ' | sed 's/[[:space:]]*$//')"
DESCRIPTION="$(yq .[$i].description <<< "$PARAMS" | tr '\n' ' ' | sed 's/[[:space:]]*$//')"
DEFAULT="$(yq .[$i].default <<< "$PARAMS" | tr '\n' ' ' | sed 's/[[:space:]]*$//')"

# Check that default doesn't exist
if [[ $(yq ".[$i] | has(\"default\")" <<< "$PARAMS") == "false" ]]; then
echo "| $NAME$(spaces "$LONGEST_NAME" "$NAME")" \
"| $DESCRIPTION$(spaces "$LONGEST_DESCRIPTION" "$DESCRIPTION")" \
"| No$(spaces "$LONGEST_OPTIONAL" "No")" \
"| -$(spaces "$LONGEST_DEFAULT" "-") |"
else
# Special case to show default empty strings as "" in table
if [[ -z "$DEFAULT" ]]; then
DEFAULT="\"\""
fi

echo "| $NAME$(spaces "$LONGEST_NAME" "$NAME")" \
"| $DESCRIPTION$(spaces "$LONGEST_DESCRIPTION" "$DESCRIPTION")" \
"| Yes$(spaces "$LONGEST_OPTIONAL" "Yes")" \
"| $DEFAULT$(spaces "$LONGEST_DEFAULT" "$DEFAULT") |"
fi
done
} > "$TEMP_README"

if [[ $DRY_RUN == "true" ]]; then
cat "$TEMP_README"
else
cat "$TEMP_README" > "${ITEM_DIR}/README.md"
echo " README.md for $ITEM_DIR updated"
fi
done

# Cleanup
if [ -v TEMP_README ]; then
rm -f "$TEMP_README"
fi
22 changes: 22 additions & 0 deletions .github/workflows/lint.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,3 +111,25 @@ jobs:
run: .github/scripts/tkn_check_compute_resources.sh
env:
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
check-readme:
name: Check README.md files
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Get changed dirs
id: changed-dirs
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v45.0.6
with:
files: |
tasks/*/**
pipelines/*/**
dir_names: "true"
dir_names_max_depth: "3"
- name: Check README.md files
if: |
steps.changed-dirs.outputs.any_changed == 'true'
run: .github/scripts/check_readme.sh
env:
README_ITEMS: >-
${{ steps.changed-dirs.outputs.all_changed_files }}
Comment thread
seanconroy2021 marked this conversation as resolved.
Loading