Add sort and exclude options for folder auto-discovery in toc.yml

docs/configure/content-set/navigation.md

@@ -112,7 +112,7 @@ The TOC in principle follows the directory structure on disk.
   - folder: subsection
 ```
 
-If a folder does not explicitly define `children` all markdown files within that folder are included automatically 
+If a folder does not explicitly define `children` all markdown files within that folder are included automatically
 
 If a folder does define `children` all markdown files within that folder have to be included. `docs-builder` will error if it detects dangling documentation files.
 
@@ -125,6 +125,36 @@ If a folder does define `children` all markdown files within that folder have to
       - file: page-two.md
 ```
 
+##### `sort`
+
+When auto-discovering files (no explicit `children`), you can control the sort order with `sort`:
+
+```yaml
+  ...
+  - folder: api-versions
+    sort: desc
+```
+
+Valid values are `asc`, `ascending`, `desc`, and `descending`. The default is ascending (A-Z). When set to descending, files are listed Z-A, which is useful for version-numbered folders where the newest version should appear first. Sorting uses natural order, so version numbers sort correctly (`3_2_0` comes before `3_10_0`). `index.md` is always placed first regardless of sort order.
+
+The `sort` option has no effect when `children` are explicitly defined, since the order is determined by the `children` list.
+
+##### `exclude`
+
+When using auto-discovery (no explicit `children`), you can exclude specific files from being included:
+
+```yaml
+  ...
+  - folder: subsection
+    exclude:
+      - draft.md
+      - internal-notes.md
+```
+
+Excluded file names are matched case-insensitively. This is useful when a folder contains files that should not appear in the navigation, like drafts or internal documentation.
+
+The `exclude` option has no effect when `children` are explicitly defined, since all files must be listed manually.
+
 #### Virtual grouping
 
 A `file` element may include children to create a virtual grouping that 

src/Elastic.Documentation.Configuration/Toc/DocumentationSetFile.cs

@@ -478,31 +478,44 @@ private static ITableOfContentsItem ResolveFolderRef(IDiagnosticsCollector colle
 			? fullPath
 			: fullPath.Substring(containerPath.Length + 1);
 
+		// Parse and validate sort order
+		if (!SortOrderExtensions.TryParse(folderRef.Sort, out var sortOrder) && folderRef.Sort is not null)
+			collector.EmitError(
+				context,
+				$"Unknown sort order '{folderRef.Sort}' for folder '{folderRef.PathRelativeToDocumentationSet}'."
+				+ " Valid values are: asc, ascending, desc, descending."
+			);
+
 		// If children are explicitly defined, resolve them
 		if (folderRef.Children.Count > 0)
 		{
 			// For children of folders, the container remains the same as the folder's container
 			var resolvedChildren = ResolveTableOfContents(collector, folderRef.Children, baseDirectory, fileSystem, fullPath, containerPath, context, suppressDiagnostics);
-			return new FolderRef(fullPath, pathRelativeToContainer, resolvedChildren, context);
+			// Exclude is intentionally not passed through — it only applies to auto-discovery
+			return new FolderRef(fullPath, pathRelativeToContainer, resolvedChildren, context, folderRef.Sort);
 		}
 
 		// No children defined - auto-discover .md files in the folder
-		var autoDiscoveredChildren = AutoDiscoverFolderFiles(collector, fullPath, containerPath, baseDirectory, fileSystem, context);
-		return new FolderRef(fullPath, pathRelativeToContainer, autoDiscoveredChildren, context);
+		// null preserves the default alphabetical sorting; non-null enables natural sort for version numbers
+		var explicitSortOrder = folderRef.Sort is not null ? sortOrder : (SortOrder?)null;
+		var autoDiscoveredChildren = AutoDiscoverFolderFiles(collector, fullPath, containerPath, baseDirectory, fileSystem, context, explicitSortOrder, folderRef.Exclude);
+		return new FolderRef(fullPath, pathRelativeToContainer, autoDiscoveredChildren, context, folderRef.Sort, folderRef.Exclude);
 	}
 
 	/// <summary>
 	/// Auto-discovers .md files in a folder directory and creates FileRef items for them.
-	/// If index.md exists, it's placed first. Otherwise, files are sorted alphabetically.
-	/// Files starting with '_' or '.' are excluded.
+	/// If index.md exists, it's placed first. Other files are sorted according to the specified sort order.
+	/// Files starting with '_' or '.' are excluded, as well as any files listed in <paramref name="exclude"/>.
 	/// </summary>
 	private static TableOfContents AutoDiscoverFolderFiles(
 		IDiagnosticsCollector collector,
 		string folderPath,
 		string containerPath,
 		IDirectoryInfo baseDirectory,
 		IFileSystem fileSystem,
-		string context)
+		string context,
+		SortOrder? sortOrder,
+		IReadOnlyCollection<string>? exclude)
 	{
 		var directoryPath = fileSystem.Path.Combine(baseDirectory.FullName, folderPath);
 		var directory = fileSystem.DirectoryInfo.New(directoryPath);
@@ -511,33 +524,40 @@ private static TableOfContents AutoDiscoverFolderFiles(
 			return [];
 
 		// Find all .md files in the directory (not recursive)
+		var excludeSet = exclude is { Count: > 0 }
+			? new HashSet<string>(exclude, StringComparer.OrdinalIgnoreCase)
+			: null;
 		var mdFiles = fileSystem.Directory
 			.GetFiles(directoryPath, "*.md")
 			.Select(f => fileSystem.FileInfo.New(f))
 			.Where(f => !f.Name.StartsWith('_') && !f.Name.StartsWith('.'))
-			.OrderBy(f => f.Name)
+			.Where(f => excludeSet is null || !excludeSet.Contains(f.Name))
 			.ToList();
 
 		if (mdFiles.Count == 0)
 			return [];
 
 		// Separate index.md from other files
 		var indexFile = mdFiles.FirstOrDefault(f => f.Name.Equals("index.md", StringComparison.OrdinalIgnoreCase));
-		var otherFiles = mdFiles.Where(f => !f.Name.Equals("index.md", StringComparison.OrdinalIgnoreCase)).ToList();
+		var otherFiles = mdFiles.Where(f => !f.Name.Equals("index.md", StringComparison.OrdinalIgnoreCase));
+
+		// When sort is explicitly set (non-null), use natural sort order (handles version numbers correctly: 3_2 < 3_10)
+		// When null, preserve the original alphabetical sorting behavior
+		var sortedFiles = sortOrder switch
+		{
+			SortOrder.Descending => otherFiles.OrderByDescending(f => f.Name, NaturalStringComparer.Instance).ToList(),
+			SortOrder.Ascending => otherFiles.OrderBy(f => f.Name, NaturalStringComparer.Instance).ToList(),
+			_ => otherFiles.OrderBy(f => f.Name).ToList()
+		};
 
 		var children = new TableOfContents();
 
 		// Add index.md first if it exists
 		if (indexFile != null)
-		{
-			var indexRef = indexFile.Name.Equals("index.md", StringComparison.OrdinalIgnoreCase)
-				? new IndexFileRef(indexFile.Name, indexFile.Name, false, [], context)
-				: new FileRef(indexFile.Name, indexFile.Name, false, [], context);
-			children.Add(indexRef);
-		}
+			children.Add(new IndexFileRef(indexFile.Name, indexFile.Name, false, [], context));
 
-		// Add other files sorted alphabetically
-		foreach (var file in otherFiles)
+		// Add other files sorted according to the specified order
+		foreach (var file in sortedFiles)
 		{
 			var fileRef = new FileRef(file.Name, file.Name, false, [], context);
 			children.Add(fileRef);

src/Elastic.Documentation.Configuration/Toc/TableOfContentsItems.cs

@@ -2,8 +2,94 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Globalization;
+
 namespace Elastic.Documentation.Configuration.Toc;
 
+/// <summary>
+/// Specifies the sort order for auto-discovered files in a folder.
+/// </summary>
+public enum SortOrder
+{
+	/// <summary>
+	/// Sort files in ascending alphabetical order (A-Z). This is the default.
+	/// </summary>
+	Ascending,
+
+	/// <summary>
+	/// Sort files in descending alphabetical order (Z-A).
+	/// Useful for version-numbered folders where newest should appear first.
+	/// </summary>
+	Descending
+}
+
+/// <summary>Parsing helpers for <see cref="SortOrder"/>.</summary>
+public static class SortOrderExtensions
+{
+	/// <summary>Tries to parse a YAML sort value (asc, ascending, desc, descending) into a <see cref="SortOrder"/>.</summary>
+	public static bool TryParse(string? value, out SortOrder result)
+	{
+		var normalized = value?.ToLowerInvariant();
+		(result, var valid) = normalized switch
+		{
+			"desc" or "descending" => (SortOrder.Descending, true),
+			"asc" or "ascending" => (SortOrder.Ascending, true),
+			_ => (SortOrder.Ascending, false)
+		};
+		return valid;
+	}
+}
+
+/// <summary>Compares strings using natural sort order, where numeric segments are compared as integers ("3_2" &lt; "3_10").</summary>
+public sealed class NaturalStringComparer : IComparer<string>
+{
+	public static NaturalStringComparer Instance { get; } = new();
+
+	public int Compare(string? x, string? y)
+	{
+		if (ReferenceEquals(x, y))
+			return 0;
+		if (x is null)
+			return -1;
+		if (y is null)
+			return 1;
+
+		var ix = 0;
+		var iy = 0;
+
+		while (ix < x.Length && iy < y.Length)
+		{
+			if (char.IsDigit(x[ix]) && char.IsDigit(y[iy]))
+			{
+				// Compare numeric segments as integers
+				var nx = ParseNumber(x, ref ix);
+				var ny = ParseNumber(y, ref iy);
+				var cmp = nx.CompareTo(ny);
+				if (cmp != 0)
+					return cmp;
+			}
+			else
+			{
+				var cmp = x[ix].CompareTo(y[iy]);
+				if (cmp != 0)
+					return cmp;
+				ix++;
+				iy++;
+			}
+		}
+
+		return x.Length.CompareTo(y.Length);
+	}
+
+	private static long ParseNumber(string s, ref int index)
+	{
+		var start = index;
+		while (index < s.Length && char.IsDigit(s[index]))
+			index++;
+		return long.Parse(s[start..index], CultureInfo.InvariantCulture);
+	}
+}
+
 /// <summary>
 /// Represents an item in a table of contents (file, folder, or TOC reference).
 /// </summary>
@@ -56,7 +142,9 @@ public record CrossLinkRef(Uri CrossLinkUri, string? Title, bool Hidden, IReadOn
 	public string PathRelativeToContainer => CrossLinkUri.ToString();
 }
 
-public record FolderRef(string PathRelativeToDocumentationSet, string PathRelativeToContainer, IReadOnlyCollection<ITableOfContentsItem> Children, string Context)
+/// <param name="Sort">Raw YAML sort value, parsed and validated during resolution via <see cref="SortOrderExtensions.TryParse"/>.</param>
+/// <param name="Exclude">File names to exclude from auto-discovery (like "draft.md", "internal.md").</param>
+public record FolderRef(string PathRelativeToDocumentationSet, string PathRelativeToContainer, IReadOnlyCollection<ITableOfContentsItem> Children, string Context, string? Sort = null, IReadOnlyCollection<string>? Exclude = null)
 	: ITableOfContentsItem;
 
 public record IsolatedTableOfContentsRef(string PathRelativeToDocumentationSet, string PathRelativeToContainer, IReadOnlyCollection<ITableOfContentsItem> Children, string Context)

src/Elastic.Documentation.Configuration/Toc/TableOfContentsYamlConverters.cs

@@ -72,7 +72,7 @@ public class TocItemYamlConverter : IYamlTypeConverter
 					}
 					value = childrenList;
 				}
-				else if (key.Value == "detection_rules")
+				else if (key.Value is "detection_rules" or "exclude")
 				{
 					// Parse the children list manually
 					var childrenList = new List<string>();
@@ -105,6 +105,12 @@ public class TocItemYamlConverter : IYamlTypeConverter
 		// Context will be set during LoadAndResolve, use empty string as placeholder during deserialization
 		const string placeholderContext = "";
 
+		// Capture raw sort value; parsing and validation happen during resolution
+		var sort = dictionary.TryGetValue("sort", out var sortValue) && sortValue is string sortStr ? sortStr : null;
+
+		// Capture exclude list for folder auto-discovery
+		var exclude = dictionary.TryGetValue("exclude", out var excludeObj) && excludeObj is string[] excludeArr ? excludeArr : null;
+
 		// Check for folder+file combination (e.g., folder: getting-started, file: getting-started.md)
 		// This represents a folder with a specific index file
 		// The file becomes a child of the folder (as FolderIndexFileRef), and user-specified children follow
@@ -124,7 +130,7 @@ public class TocItemYamlConverter : IYamlTypeConverter
 			// Return a FolderRef with the index file and children
 			// The folder path can be deep (e.g., "guides/getting-started"), that's OK
 			// PathRelativeToContainer will be set during resolution
-			return new FolderRef(folder, folder, folderChildren, placeholderContext);
+			return new FolderRef(folder, folder, folderChildren, placeholderContext, sort, exclude);
 		}
 		if (dictionary.TryGetValue("detection_rules", out var detectionRulesObj) && detectionRulesObj is string[] detectionRulesFolders &&
 			dictionary.TryGetValue("file", out var detectionRulesFilePath) && detectionRulesFilePath is string detectionRulesFile)
@@ -159,7 +165,7 @@ public class TocItemYamlConverter : IYamlTypeConverter
 		// Check for folder reference
 		// PathRelativeToContainer will be set during resolution
 		if (dictionary.TryGetValue("folder", out var folderPathOnly) && folderPathOnly is string folderOnly)
-			return new FolderRef(folderOnly, folderOnly, children, placeholderContext);
+			return new FolderRef(folderOnly, folderOnly, children, placeholderContext, sort, exclude);
 
 		// Check for toc reference
 		// PathRelativeToContainer will be set during resolution