Merge pull request #402 from Shazwazza/website-updates-jan-2021

WIP Website updates jan 2021

Author: Shazwazza

src/docs/convert.ps1

@@ -6,9 +6,9 @@
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the ""License""); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an ""AS IS"" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -51,41 +51,8 @@ if (-not (test-path $releaseLuceneFolder)) {
     Write-Error "Could not detect Java release files in folder $releaseLuceneFolder" -ErrorAction Stop
 }
 
-# ensure we have NuGet
-New-Item "$ToolsFolder\nuget" -type directory -force
-$nuget = "$ToolsFolder\nuget\nuget.exe"
-if (-not (test-path $nuget)) {
-    Write-Host "Download NuGet..."
-    Invoke-WebRequest "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" -OutFile $nuget -TimeoutSec 60
-}
-
-# ensure we have vswhere
-New-Item "$ToolsFolder\vswhere" -type directory -force
-$vswhere = "$ToolsFolder\vswhere\vswhere.exe"
-if (-not (test-path $vswhere)) {
-    Write-Host "Download VsWhere..."
-    $path = "$ToolsFolder\tmp"
-    &$nuget install vswhere -OutputDirectory $path
-    $dir = ls "$path\vswhere.*" | sort -property Name -descending | select -first 1
-    $file = ls -path "$dir" -name vswhere.exe -recurse
-    mv "$dir\$file" $vswhere   
-}
-
 Remove-Item  -Recurse -Force "$ToolsFolder\tmp"
 
-# Get MSBuild
-
-$MSBuild = &$vswhere -latest -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe | select-object -first 1
-if (-not (test-path $MSBuild)) {
-    throw "MSBuild not found!"
-}
-
-# Build the solution
-$sln = (Join-Path -Path $PSScriptFilePath "DocumentationTools.sln")
-& $nuget restore $sln
-& $msbuild $sln
-
-# Execute the program
-$exe = (Join-Path -Path $PSScriptFilePath "JavaDocToMarkdownConverter\bin\Debug\JavaDocToMarkdownConverter.exe")
-& $exe $releaseLuceneFolder $RepoRoot
-
+# TODO: Execute the JavaDocToMarkdownConverter dotnet tool
+# TODO: Validate that this only executes within a git branch: `docs/markdown-converted/*`
+#

websites/site/contributing/community-links.md

@@ -16,6 +16,11 @@ A very detailed how-to guide for working with Lucene.Net.
 
 > "This is one of the best posts I have ever seen about Lucene.NET"
 
+### Great intro posts about Lucene:
+
+* [Analysis of Lucene - Basic Concepts](https://www.alibabacloud.com/blog/analysis-of-lucene---basic-concepts_594672)
+* [Apache Lucene: free search for your website](https://www.ionos.com/digitalguide/server/configuration/apache-lucene/)
+
 ### Introductory Lucene.Net Series
 
 An excellent introductory series from [Simone Chiaretta](http://codeclimber.net.nz/)

websites/site/contributing/documentation.md

@@ -54,11 +54,9 @@ The file/folder structure is within `/websites/site`:
 
 ### Deploy the website
 
-- The website is deployed via git
-- Checkout the Git repo that hosts the documentation: https://github.com/apache/lucenenet-site/tree/asf-site _(ensure you have `asf-site` branch checked out, not `master`)_
-- Copy the build output of the website to the root. The build output will be all of the files in the `/websites/site/_site` in your main Lucene.NET checked out Git repository.
-- Commit and push these changes
-- The new version of the website will be live. If the amount of new files committed is large, the new files may take some time to become live.
+- The website is deployed via GitHub and is hosted by static files here: https://github.com/apache/lucenenet-site/tree/asf-site _(ensure you have `asf-site` branch checked out, not `master`)_
+- Any file changes made in the `master` branch of the Lucene.Net repository under the path `/websites/site/*` will trigger a GitHub action to build the site and publish a Pull Request to the https://github.com/apache/lucenenet-site repository where it can be accepted
+- Review and merge the Pull Request. The new version of the website will be live. If the amount of new files committed is large, the new files may take some time to become live.
 
 ## API Docs
 
@@ -95,39 +93,62 @@ The script parameters are:
 
 The file/folder structure is within `/websites/apidocs`:
 
-- `docs.ps1` - the build script
-- `docfx.json` - the DocFx configuration file _(see docfx manual for further info)_
-- `lucenetemplate/*` - the custom template files to style the website
-- `*.md` - the root site content such as the index and download pages
-- `toc.yml` - these files determine the menu structures _(see docfx manual for further info)_
-- `tools/*` - during the build process some tools will be downloaded which are stored here
-- `_site` - this is the exported static site that is generated
-
-> [!NOTE]
-> The project is going through a trial run of using mini-sites instead of a single API docs site. Currently, the active DocFx configuration files are the `docfx.<project>.json` files, however both the mini-site and `docfx.json` configuration files need to be updated, since there is a chance we will need to switch back.
+- `docs.ps1` - The build script
+- `docfx.*.json` - The DocFx configuration files _(see docfx manual for further info)_
+  - `docfx.{library}.json` - Where {library} is an individual Lucene.NET project (i.e. `codecs`). Each library is built as it's own individual DocFx site and it's xref maps are exported to file to be shared between DocFx builds.
+  - `docfx.global.json` - Each library DocFx json references this file for global metadata. This is where all global metadata such as Title, Logo, Footer, etc... are declared.
+  - `docfx.global.subsite.json` - Each library DocFx json references this file for global metadata which denotes the [`_rel`](https://dotnet.github.io/docfx/tutorial/intro_template.html#system-generated-properties) (The relative path of the root output folder from current output file. i.e. the base URL). For example: `https://lucenenet.apache.org/docs/4.8.0-beta00009/`.
+  - `docfx.site.json` - Once each library is built and it's xref maps are exported, the main documentation site container is built with this definition.
+- `lucenetemplate/*` - The custom template files to style the website
+- `*.md` - The root site content such as the index and download pages
+- `toc.yml` - These files determine the menu structures _(see docfx manual for further info)_
+- `tools/*` - During the build process some tools will be downloaded which are stored here
+- `_site` - This is the exported static site that is generated
 
-### Process overview
+### Java to Markdown converter
 
 The documentation generation is a complex process because it needs to convert the Java Lucene project's documentation into a usable format to produce the output Lucene.NET's documentation.
 
-The process overview is:
+The `JavaDocToMarkdownConverter` dotnet tool to is used to convert the Java Lucene project's docs into a useable format for DocFx. This tool uses a release tag output of the Java Lucene project as it's source to convert against the Lucene.NET's source. This tool must **only** be executed against the current documentation branch, for example in 4.8.0 it is: `docs/markdown-converted/4.8.1`. This conversion process does not need to be executed everytime the docs are built, it is executed rarely when:
+- A new major or minor version of Lucene.Net is created and the conversion needs to be re-executed again the Java Lucene source. In this case a new documentation branch should be created from the previous documentation branch.
+- Updates to the `JavaDocToMarkdownConverter` are made and the conversion needs to be re-executed.
+
+#### Manual execution
+
+To use the dotnet tool you must download the current tag of the Java Lucene project, for example: ["4.8.1"](https://github.com/apache/lucene-solr/releases/tag/releases%2Flucene-solr%2F4.8.1)
+
+Then install the tool:
+
+```
+dotnet tool install javadoc2markdown --add-source https://pkgs.dev.azure.com/lucene-net/_packaging/lucene-net-tools/nuget/v3/index.json --tool-path ./
+```
+
+Then run the command:
+```
+javadoc2markdown <LUCENE DIRECTORY> <LUCENENET DIRECTORY>
+```
+
+Where `<LUCENE DIRECTORY>` is the `lucene` sub folder location of the Java Lucene tag downloaded above. The `<LUCENENET DIRECTORY>` is the folder of your locally checked out Lucene.Net git repository of the documentation tag (i.e. `docs/markdown-converted/4.8.1`).
+
+#### Automated execution
+
+A powershell script has been created to automate the above. Execute the `./src/docs/convert.ps1` script and enter the current Lucene version to convert from. For example, for Lucene.Net 4.8.0 we are converting from the Java Lucene build release of ["4.8.1"](https://github.com/apache/lucene-solr/releases/tag/releases%2Flucene-solr%2F4.8.1) so in this case enter: 4.8.1 at the prompt or call the whole script like `./src/docs/convert.ps1 -JavaLuceneVersion 4.8.1`
+
+#### Review, commit and merge
+
+Once the conversion is done, review, commit and push those changes. Many times there will just be whitespace changes in the files especially if this process has been executed before for the same source/destination version. If this is a new source/destination version there will be a **lot** of file changes, at least one file per folder. If there are formatting issues or irregularities in the converted output then these will need to be addressed by making changes to the conversion tool itself (generally only needed for new major version releases).
+
+Once pushed, you can merge those changes to the `master` branch. Doing this may trigger merge conflicts because the documentation files may have been manually edited. In these cases you will need to manually review and fix the merge conflicts with your favorite merge tool ensuring that the most recent manual changes done are kept.
 
-- Use the `JavaDocToMarkdownConverter` project within the `DocumentationTools.sln` solution to run the conversion of the Java Lucene projects docs into a useable format for DocFx. This tool takes uses a release tag output of the Java Lucene project as it's source to convert against the Lucene.NET's source.
-- Run the documentation build script to produce the documentation site
-- Publish the output to the [`lucenenet-site`](https://github.com/apache/lucenenet-site) repository into a corresponding named version directory
+#### Tool info
 
-We don't want to manually change the converted resulting markdown files (`.md`) because they would get overwritten again when the conversion process is re-executed. Therefore to fix any formatting issues or customized output of the project docs, these customizations/fixes/tweaks are built directly into the conversion process itself in the `JavaDocToMarkdownConverter.csproj` project.
+- [Source code is here](https://github.com/NightOwl888/lucenenet-javadoc2markdown)
+- Tool name: `javadoc2markdown`
+- [NuGet feed here](https://dev.azure.com/lucene-net/Lucene.NET/_packaging?_a=feed&feed=lucene-net-tools)
 
 ### Building the docs
 
 - Checkout the Lucene.Net release tag to build the docs against
-- Execute the `./src/docs/convert.ps1` script and enter the current Lucene version to convert from.
-  - For example, for Lucene.Net 4.8.0 we are converting from the Java Lucene build release of ["4.8.1"](https://github.com/apache/lucene-solr/releases/tag/releases%2Flucene-solr%2F4.8.1) so in this case enter: 4.8.1 at the prompt or call the whole script like `./src/docs/convert.ps1 -JavaLuceneVersion 4.8.1`
-  - This script will download and extract the Java Lucene release files, build the `DocumentationTools.sln` solution and execute the `JavaDocToMarkdownConverter.exe`
-- Review and commit the files changed
-  - Many times there will just be whitespace changes in the files especially if this process has been executed before for the same source/destination version.
-  - If this is a new source/destination version there will be a **lot** of file changes, at least one file per folder.
-  - If there are formatting issues or irregularities in the converted output then these will need to be addressed by making changes to the conversion tool itself `JavaDocToMarkdownConverter.csproj` (generally only needed for new major version releases)
 - Execute the `./websites/apidocs/docs.ps1` script to build and serve the api docs website locally for testing.
   - Example: `./websites/apidocs/docs.ps1 -LuceneNetVersion 4.8.0-beta00008`
   - will serve a website on [http://localhost:8080](http://localhost:8080)

websites/site/site.ps1

@@ -38,7 +38,7 @@ $ToolsFolder = Join-Path -Path $SiteFolder -ChildPath "tools";
 #ensure the /build/tools folder
 New-Item $ToolsFolder -type directory -force
 
-if ($Clean -eq 1) {
+if ($Clean) {
 	Write-Host "Cleaning tools..."
 	Remove-Item (Join-Path -Path $ToolsFolder "\*") -recurse -force -ErrorAction SilentlyContinue
 }
@@ -52,15 +52,15 @@ if (-not (test-path $DocFxExe))
 {
 	Write-Host "Retrieving docfx..."
 	$DocFxZip = "$ToolsFolder\tmp\docfx.zip"
-	Invoke-WebRequest "https://github.com/dotnet/docfx/releases/download/v2.50/docfx.zip" -OutFile $DocFxZip -TimeoutSec 60
+	Invoke-WebRequest "https://github.com/dotnet/docfx/releases/download/v2.56.6/docfx.zip" -OutFile $DocFxZip -TimeoutSec 60
 	#unzip
 	Expand-Archive $DocFxZip -DestinationPath (Join-Path -Path $ToolsFolder -ChildPath "docfx")
 }
 
  Remove-Item  -Recurse -Force "$ToolsFolder\tmp"
 
 # delete anything that already exists
-if ($Clean -eq 1) {
+if ($Clean) {
 	Write-Host "Cleaning..."
 	Remove-Item (Join-Path -Path $SiteFolder "_site\*") -recurse -force -ErrorAction SilentlyContinue
 	Remove-Item (Join-Path -Path $SiteFolder "_site") -force -ErrorAction SilentlyContinue
@@ -72,7 +72,7 @@ $DocFxJson = Join-Path -Path $SiteFolder "docfx.json"
 $DocFxLog = Join-Path -Path $SiteFolder "obj\docfx.log"
 
 if($?) {
-	if ($ServeDocs -eq 0){
+	if ($ServeDocs -eq $false) {
 		# build the output
 		Write-Host "Building docs..."
 		& $DocFxExe build $DocFxJson -l "$DocFxLog" --loglevel $LogLevel