+if ($env:MVNW_REPOURL) {
+ $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
+ $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
+}
+$distributionUrlName = $distributionUrl -replace '^.*/',''
+$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
+
+$MAVEN_M2_PATH = "$HOME/.m2"
+if ($env:MAVEN_USER_HOME) {
+ $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
+}
+
+if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
+ New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
+}
+
+$MAVEN_WRAPPER_DISTS = $null
+if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
+ $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
+} else {
+ $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
+}
+
+$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
+$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
+$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
+
+if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
+ Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+ Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
+ exit $?
+}
+
+if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
+ Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
+}
+
+# prepare tmp dir
+$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
+$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
+$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
+trap {
+ if ($TMP_DOWNLOAD_DIR.Exists) {
+ try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+ catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+ }
+}
+
+New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
+
+# Download and Install Apache Maven
+Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+Write-Verbose "Downloading from: $distributionUrl"
+Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+$webclient = New-Object System.Net.WebClient
+if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
+ $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
+}
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
+if ($distributionSha256Sum) {
+ if ($USE_MVND) {
+ Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
+ }
+ Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
+ if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
+ Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
+ }
+}
+
+# unzip and move
+Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+$actualDistributionDir = ""
+
+# First try the expected directory name (for regular distributions)
+$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
+$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
+if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
+ $actualDistributionDir = $distributionUrlNameMain
+}
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if (!$actualDistributionDir) {
+ Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
+ $testPath = Join-Path $_.FullName "bin/$MVN_CMD"
+ if (Test-Path -Path $testPath -PathType Leaf) {
+ $actualDistributionDir = $_.Name
+ }
+ }
+}
+
+if (!$actualDistributionDir) {
+ Write-Error "Could not find Maven distribution directory in extracted archive"
+}
+
+Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
+try {
+ Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
+} catch {
+ if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
+ Write-Error "fail to move MAVEN_HOME"
+ }
+} finally {
+ try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+ catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+}
+
+Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
diff --git a/src/java/index-compat/pom.xml b/src/java/index-compat/pom.xml
new file mode 100644
index 0000000000..0eff83b243
--- /dev/null
+++ b/src/java/index-compat/pom.xml
@@ -0,0 +1,102 @@
+
+
+
+ 4.0.0
+
+ org.apache.lucenenet
+ index-compat
+ 4.8.1
+ jar
+
+ Lucene.NET index compatibility harness
+
+ Generates and reads Apache Lucene 4.8.1 indexes to verify two-way index/codec
+ compatibility with Lucene.NET. See README.md and issue #270.
+
+
+
+ UTF-8
+
+ 8
+ 4.8.1
+
+
+
+
+
+
+ org.apache.lucene
+ lucene-core
+ ${lucene.version}
+
+
+ org.apache.lucene
+ lucene-analyzers-common
+ ${lucene.version}
+
+
+ org.apache.lucene
+ lucene-codecs
+ ${lucene.version}
+
+
+ junit
+ junit
+ 4.13.2
+ test
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.13.0
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 3.2.5
+
+
+
+ ${lucenenet.index.dir}
+
+
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+ 3.2.0
+
+ org.apache.lucenenet.compat.GenerateIndex
+
+
+
+
+
diff --git a/src/java/index-compat/run-compat.bat b/src/java/index-compat/run-compat.bat
new file mode 100644
index 0000000000..8090afaf98
--- /dev/null
+++ b/src/java/index-compat/run-compat.bat
@@ -0,0 +1,30 @@
+@echo off
+GOTO endcommentblock
+:: -----------------------------------------------------------------------------------
+::
+:: Licensed to the Apache Software Foundation (ASF) under one or more
+:: contributor license agreements. See the NOTICE file distributed with
+:: this work for additional information regarding copyright ownership.
+:: The ASF licenses this file to You under the Apache License, Version 2.0
+:: (the "License"); you may not use this file except in compliance with
+:: the License. You may obtain a copy of the License at
+::
+:: http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing, software
+:: distributed under the License is distributed on an "AS IS" BASIS,
+:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+:: See the License for the specific language governing permissions and
+:: limitations under the License.
+::
+:: -----------------------------------------------------------------------------------
+::
+:: Thin wrapper that forwards to run-compat.ps1, which contains all the business
+:: logic. See run-compat.ps1 for what the harness does (both directions of the
+:: Lucene 4.8.1 <-> Lucene.NET index compatibility check, issue #270) and the
+:: COMPAT_TFM environment variable.
+::
+:: -----------------------------------------------------------------------------------
+:endcommentblock
+where pwsh >nul 2>nul
+if %ERRORLEVEL% NEQ 0 (echo "PowerShell could not be found. Please install version 3 or higher." & exit /b 1) else (pwsh -ExecutionPolicy bypass -Command "& '%~dpn0.ps1'" %*)
diff --git a/src/java/index-compat/run-compat.ps1 b/src/java/index-compat/run-compat.ps1
new file mode 100644
index 0000000000..afa335ce5e
--- /dev/null
+++ b/src/java/index-compat/run-compat.ps1
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Runs both directions of the Lucene 4.8.1 <-> Lucene.NET index compatibility
+# check (issue #270). Cross-platform (Windows / macOS / Linux PowerShell). Requires
+# a JDK and the .NET SDK. All generated indexes go under the gitignored work/
+# folder; nothing is committed.
+#
+# By default the test shard builds with its own default target framework. Set
+# $env:COMPAT_TFM (e.g. net10.0, net8.0) to force a specific one.
+
+$ErrorActionPreference = 'Stop'
+
+# Guarantee a non-zero exit code on any failure so CI fails the job. A bare throw
+# is not always reflected in the process exit code across PowerShell hosts/versions.
+trap {
+ Write-Host "==> Compatibility check FAILED: $_" -ForegroundColor Red
+ exit 1
+}
+
+$Here = Split-Path -Parent $MyInvocation.MyCommand.Path
+$RepoRoot = Resolve-Path (Join-Path $Here '..' '..' '..')
+$Work = Join-Path $Here 'work'
+$Shard = Join-Path $RepoRoot 'src' 'Lucene.Net.Tests._J-S' 'Lucene.Net.Tests._J-S.csproj'
+
+$JavaIndex = Join-Path $Work 'java'
+$DotNetIndex = Join-Path $Work 'dotnet'
+
+# Pick the correct Maven wrapper for the OS (mvnw.cmd is the Windows batch file).
+$Mvnw = if ($IsWindows) { Join-Path $Here 'mvnw.cmd' } else { Join-Path $Here 'mvnw' }
+
+# Only pass -f when the caller explicitly forces a target framework.
+$TfmArgs = if ($env:COMPAT_TFM) { @('-f', $env:COMPAT_TFM) } else { @() }
+
+function Invoke-DotNetTest([string]$Filter) {
+ # dotnet test exits 0 when a --filter matches nothing, which would hide a
+ # misconfiguration. Capture output, surface it, and fail on "matched 0".
+ $output = & dotnet test $Shard @TfmArgs -c Release --no-build --filter $Filter 2>&1
+ $exit = $LASTEXITCODE
+ $output | ForEach-Object { Write-Host $_ }
+ if ($exit -ne 0) { throw "dotnet test failed for filter '$Filter'" }
+ if ($output -match 'no test is available|matches the given testcase filter|total:\s*0\b') {
+ throw "No tests ran for filter '$Filter'. Is the shard built for this target framework?"
+ }
+}
+
+function Invoke-Maven([string[]]$MvnArgs) {
+ Push-Location $Here
+ try {
+ & $Mvnw @MvnArgs
+ if ($LASTEXITCODE -ne 0) { throw "Maven failed: $($MvnArgs -join ' ')" }
+ } finally {
+ Pop-Location
+ }
+}
+
+Write-Host "==> Building the .NET test shard$(if ($env:COMPAT_TFM) { " ($env:COMPAT_TFM)" })"
+& dotnet build $Shard @TfmArgs -c Release
+if ($LASTEXITCODE -ne 0) { throw "dotnet build failed" }
+
+Write-Host ""
+Write-Host "==> Direction 1: .NET writes, Java reads"
+Write-Host " .NET writing index into $DotNetIndex"
+[Environment]::SetEnvironmentVariable('lucenenet.compat.write.dir', $DotNetIndex, 'Process')
+try {
+ Invoke-DotNetTest 'FullyQualifiedName~TestJavaCompatibility.TestWriteIndexForJava'
+} finally {
+ [Environment]::SetEnvironmentVariable('lucenenet.compat.write.dir', $null, 'Process')
+}
+
+foreach ($variant in @('index.481.nocfs', 'index.481.cfs')) {
+ $indexDir = Join-Path $DotNetIndex $variant
+ Write-Host " Java reading $indexDir"
+ Invoke-Maven @('-q', 'test', "-Dlucenenet.index.dir=$indexDir")
+}
+
+Write-Host ""
+Write-Host "==> Direction 2: Java writes, .NET reads"
+Write-Host " Java writing index into $JavaIndex"
+Invoke-Maven @('-q', 'compile', 'exec:java', "-Dexec.args=$JavaIndex")
+
+Write-Host " .NET reading from $JavaIndex"
+[Environment]::SetEnvironmentVariable('lucenenet.compat.read.dir', $JavaIndex, 'Process')
+try {
+ Invoke-DotNetTest 'FullyQualifiedName~TestJavaCompatibility.TestReadJavaIndex'
+} finally {
+ [Environment]::SetEnvironmentVariable('lucenenet.compat.read.dir', $null, 'Process')
+}
+
+Write-Host ""
+Write-Host "==> Both directions passed."
+exit 0
diff --git a/src/java/index-compat/run-compat.sh b/src/java/index-compat/run-compat.sh
new file mode 100755
index 0000000000..d15b1589c5
--- /dev/null
+++ b/src/java/index-compat/run-compat.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Thin wrapper that forwards to run-compat.ps1, which contains all the business
+# logic. This keeps the cross-platform logic in a single place; the .sh, .bat,
+# and .ps1 entry points differ only in how they locate and invoke PowerShell.
+#
+# See run-compat.ps1 for what the harness does (both directions of the Lucene
+# 4.8.1 <-> Lucene.NET index compatibility check, issue #270) and the
+# COMPAT_TFM environment variable.
+
+if ! command -v pwsh &> /dev/null
+then
+ echo "PowerShell Core could not be found. Please install version 3 or higher."
+ exit 1
+fi
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+pwsh -ExecutionPolicy bypass -Command "& '$HERE/run-compat.ps1'" "$@"
diff --git a/src/java/index-compat/src/main/java/org/apache/lucenenet/compat/CompatDocs.java b/src/java/index-compat/src/main/java/org/apache/lucenenet/compat/CompatDocs.java
new file mode 100644
index 0000000000..33c3d51d68
--- /dev/null
+++ b/src/java/index-compat/src/main/java/org/apache/lucenenet/compat/CompatDocs.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucenenet.compat;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleDocValuesField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatDocValuesField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.*;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
+
+import java.io.PrintStream;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TreeSet;
+
+/**
+ * The single source of truth for the deterministic document set shared between
+ * Java (Apache Lucene 4.8.1) and Lucene.NET. This intentionally mirrors, field
+ * for field, the document schema produced by
+ * {@code TestBackwardsCompatibility.AddDoc} / {@code AddNoProxDoc} and the read
+ * back verification in {@code TestBackwardsCompatibility.SearchIndex} on the
+ * .NET side, so that an index written by either runtime can be validated by the
+ * other. See issue #270.
+ *
+ * Nothing here may use randomness: both runtimes must produce semantically
+ * identical indexes from the same inputs.
+ */
+public final class CompatDocs {
+
+ private CompatDocs() {
+ }
+
+ /** Number of "normal" (prox) documents indexed by {@link #writeIndex}. */
+ public static final int DOC_COUNT = 35;
+
+ /** The id that is deleted, matching the .NET harness. */
+ public static final int DELETED_ID = 7;
+
+ // Exactly matches the .NET literal: Lu + U+1D11E + ce + U+1D160 + ne + NUL + skull + astral + cd.
+ public static final String UTF8_VALUE = "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\uD917\uDC17cd";
+ public static final String CONTENT2_VALUE = "here is more content with aaa aaa aaa";
+ public static final String NON_ASCII_FIELD_NAME = "fie\u2C77ld";
+ public static final String NON_ASCII_FIELD_VALUE = "field with non-ascii name";
+
+ /**
+ * Writes the deterministic compatibility index into {@code dir}. The result
+ * has 35 documents, with id 7 deleted, term vectors, offsets, norms, and the
+ * full DocValues matrix, in either compound-file or non-compound-file form.
+ */
+ public static void writeIndex(Directory dir, boolean useCompoundFile) throws Exception {
+ Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_48);
+
+ LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
+ mp.setNoCFSRatio(useCompoundFile ? 1.0 : 0.0);
+ mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
+
+ IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_48, analyzer)
+ .setUseCompoundFile(useCompoundFile)
+ .setMaxBufferedDocs(10)
+ .setMergePolicy(mp);
+ IndexWriter writer = new IndexWriter(dir, conf);
+ for (int i = 0; i < DOC_COUNT; i++) {
+ addDoc(writer, i);
+ }
+ writer.close();
+
+ // Delete id 7 in a fresh writer so the layout matches the .NET harness.
+ conf = new IndexWriterConfig(Version.LUCENE_48, analyzer)
+ .setUseCompoundFile(useCompoundFile)
+ .setMaxBufferedDocs(10)
+ .setOpenMode(IndexWriterConfig.OpenMode.APPEND);
+ writer = new IndexWriter(dir, conf);
+ writer.deleteDocuments(new Term("id", Integer.toString(DELETED_ID)));
+ writer.close();
+ }
+
+ private static void addDoc(IndexWriter writer, int id) throws Exception {
+ Document doc = new Document();
+ doc.add(new TextField("content", "aaa", Field.Store.NO));
+ doc.add(new StringField("id", Integer.toString(id), Field.Store.YES));
+
+ FieldType customType2 = new FieldType(TextField.TYPE_STORED);
+ customType2.setStoreTermVectors(true);
+ customType2.setStoreTermVectorPositions(true);
+ customType2.setStoreTermVectorOffsets(true);
+ doc.add(new Field("autf8", UTF8_VALUE, customType2));
+ doc.add(new Field("utf8", UTF8_VALUE, customType2));
+ doc.add(new Field("content2", CONTENT2_VALUE, customType2));
+ doc.add(new Field(NON_ASCII_FIELD_NAME, NON_ASCII_FIELD_VALUE, customType2));
+
+ // numeric fields
+ doc.add(new IntField("trieInt", id, Field.Store.NO));
+ doc.add(new LongField("trieLong", (long) id, Field.Store.NO));
+
+ // docvalues fields
+ doc.add(new NumericDocValuesField("dvByte", (byte) id));
+ byte[] bytes = new byte[] {
+ (byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id
+ };
+ BytesRef ref = new BytesRef(bytes);
+ doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref));
+ doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref));
+ doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref));
+ doc.add(new SortedDocValuesField("dvBytesSortedVar", ref));
+ doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref));
+ doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref));
+ doc.add(new DoubleDocValuesField("dvDouble", (double) id));
+ doc.add(new FloatDocValuesField("dvFloat", (float) id));
+ doc.add(new NumericDocValuesField("dvInt", id));
+ doc.add(new NumericDocValuesField("dvLong", id));
+ doc.add(new NumericDocValuesField("dvPacked", id));
+ doc.add(new NumericDocValuesField("dvShort", (short) id));
+ doc.add(new SortedSetDocValuesField("dvSortedSet", ref));
+
+ // a field with both offsets and term vectors for a cross-check
+ FieldType customType3 = new FieldType(TextField.TYPE_STORED);
+ customType3.setStoreTermVectors(true);
+ customType3.setStoreTermVectorPositions(true);
+ customType3.setStoreTermVectorOffsets(true);
+ customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ doc.add(new Field("content5", CONTENT2_VALUE, customType3));
+
+ // a field that omits only positions
+ FieldType customType4 = new FieldType(TextField.TYPE_STORED);
+ customType4.setStoreTermVectors(true);
+ customType4.setStoreTermVectorPositions(false);
+ customType4.setStoreTermVectorOffsets(true);
+ customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+ doc.add(new Field("content6", CONTENT2_VALUE, customType4));
+
+ writer.addDocument(doc);
+ }
+
+ /**
+ * Runs the Lucene {@link CheckIndex} tool against {@code dir} and throws if
+ * the index reports any problem. This is the codec integrity gate; it
+ * validates the stored per file checksums written by the codec.
+ */
+ public static void checkIndex(Directory dir, PrintStream infoStream) throws Exception {
+ CheckIndex checker = new CheckIndex(dir);
+ checker.setCrossCheckTermVectors(true);
+ if (infoStream != null) {
+ checker.setInfoStream(infoStream);
+ }
+ CheckIndex.Status status = checker.checkIndex();
+ if (!status.clean) {
+ throw new IllegalStateException("CheckIndex reported the index at " + dir + " is not clean");
+ }
+ }
+
+ /**
+ * The sorted, unique set of terms that {@code StandardAnalyzer(LUCENE_48)}
+ * produces for {@link #UTF8_VALUE}. An index written by either runtime must
+ * contain exactly this term set in the {@code utf8} field, which is the
+ * cross-runtime contract for the UTF-8 edge cases. Computed by re-running the
+ * analyzer so it stays correct if the constant changes.
+ */
+ public static List expectedUtf8Terms() throws Exception {
+ TreeSet terms = new TreeSet<>();
+ try (Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_48);
+ TokenStream ts = analyzer.tokenStream("utf8", new StringReader(UTF8_VALUE))) {
+ CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
+ while (ts.incrementToken()) {
+ terms.add(termAttr.toString());
+ }
+ ts.end();
+ }
+ return new ArrayList<>(terms);
+ }
+}
diff --git a/src/java/index-compat/src/main/java/org/apache/lucenenet/compat/GenerateIndex.java b/src/java/index-compat/src/main/java/org/apache/lucenenet/compat/GenerateIndex.java
new file mode 100644
index 0000000000..86da5443d1
--- /dev/null
+++ b/src/java/index-compat/src/main/java/org/apache/lucenenet/compat/GenerateIndex.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucenenet.compat;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.SimpleFSDirectory;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Writes the deterministic compatibility index (both compound-file and
+ * non-compound-file variants) with Apache Lucene 4.8.1, for Lucene.NET to read
+ * back. This is the Java side of the "Java -> .NET" direction of issue #270.
+ *
+ * Usage (from the {@code src/java/index-compat} directory):
+ *
+ * ./mvnw -q compile exec:java
+ * ./mvnw -q compile exec:java -Dexec.args="/path/to/output"
+ *
+ *
+ * The output is written under a temporary, gitignored {@code work/java}
+ * folder by default (or the directory named by the {@code lucenenet.work.dir}
+ * system property, or the first command-line argument). Two subdirectories are
+ * created: {@code index.481.cfs} and {@code index.481.nocfs}.
+ */
+public final class GenerateIndex {
+
+ private GenerateIndex() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ Path baseDir;
+ if (args.length > 0 && args[0] != null && !args[0].isEmpty()) {
+ baseDir = Paths.get(args[0]);
+ } else {
+ String prop = System.getProperty("lucenenet.work.dir");
+ baseDir = (prop != null && !prop.isEmpty())
+ ? Paths.get(prop)
+ : Paths.get("work", "java");
+ }
+ Files.createDirectories(baseDir);
+
+ write(baseDir.resolve("index.481.cfs"), true);
+ write(baseDir.resolve("index.481.nocfs"), false);
+
+ System.out.println("Wrote Java 4.8.1 compatibility indexes under: " + baseDir.toAbsolutePath());
+ }
+
+ private static void write(Path indexPath, boolean useCompoundFile) throws Exception {
+ File dirFile = indexPath.toFile();
+ if (dirFile.exists()) {
+ File[] files = dirFile.listFiles();
+ if (files == null) {
+ throw new IllegalStateException("Index path exists but is not a directory: " + dirFile.getAbsolutePath());
+ }
+ for (File f : files) {
+ if (!f.delete()) {
+ throw new IllegalStateException("Failed to delete existing file: " + f.getAbsolutePath());
+ }
+ }
+ } else {
+ Files.createDirectories(indexPath);
+ }
+ try (Directory dir = new SimpleFSDirectory(dirFile)) {
+ CompatDocs.writeIndex(dir, useCompoundFile);
+ CompatDocs.checkIndex(dir, System.out);
+ }
+ System.out.println(" " + (useCompoundFile ? "cfs " : "nocfs ") + "-> " + indexPath.toAbsolutePath());
+ }
+}
diff --git a/src/java/index-compat/src/test/java/org/apache/lucenenet/compat/TestDotNetCompatibility.java b/src/java/index-compat/src/test/java/org/apache/lucenenet/compat/TestDotNetCompatibility.java
new file mode 100644
index 0000000000..671a962cfd
--- /dev/null
+++ b/src/java/index-compat/src/test/java/org/apache/lucenenet/compat/TestDotNetCompatibility.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucenenet.compat;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.junit.Test;
+
+import java.io.File;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+/**
+ * Verifies that Apache Lucene 4.8.1 can open an index that was written by
+ * Lucene.NET, that {@code CheckIndex} passes, and that the contents match the
+ * shared {@link CompatDocs} contract. This is the ".NET -> Java" direction of
+ * issue #270.
+ *
+ *
The index location is provided via {@code -Dlucenenet.index.dir=...}. Per
+ * the harness contract, this test fails (it does not skip) if no path is
+ * supplied or the index is missing: when Java is asked to read a .NET index, the
+ * absence of that index is a failure of the pipeline that was supposed to
+ * produce it.
+ */
+public class TestDotNetCompatibility {
+
+ @Test
+ public void dotNetIndexReadsCleanlyInJava() throws Exception {
+ String path = System.getProperty("lucenenet.index.dir");
+ if (path == null || path.trim().isEmpty()) {
+ fail("System property 'lucenenet.index.dir' is not set. Generate the "
+ + ".NET index first and pass -Dlucenenet.index.dir=.");
+ }
+
+ File dirFile = new File(path);
+ if (!dirFile.isDirectory() || !new File(dirFile, "segments.gen").exists()
+ && segmentsFile(dirFile) == null) {
+ fail("No Lucene index found at '" + dirFile.getAbsolutePath() + "'. "
+ + "Generate the .NET index before running this test.");
+ }
+
+ try (Directory dir = new SimpleFSDirectory(dirFile)) {
+ // Codec integrity gate.
+ CompatDocs.checkIndex(dir, System.out);
+
+ // Semantic read back.
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ assertContents(reader);
+ }
+ }
+ }
+
+ private static File segmentsFile(File dir) {
+ File[] files = dir.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ if (f.getName().startsWith("segments_")) {
+ return f;
+ }
+ }
+ }
+ return null;
+ }
+
+ private static void assertContents(IndexReader reader) throws Exception {
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ Bits liveDocs = MultiFields.getLiveDocs(reader);
+
+ for (int i = 0; i < CompatDocs.DOC_COUNT; i++) {
+ boolean live = liveDocs == null || liveDocs.get(i);
+ if (!live) {
+ assertEquals("only id 7 should be deleted", CompatDocs.DELETED_ID, i);
+ continue;
+ }
+
+ Document d = reader.document(i);
+ assertEquals("id", Integer.toString(i), d.get("id"));
+ assertEquals("utf8", CompatDocs.UTF8_VALUE, d.get("utf8"));
+ assertEquals("autf8", CompatDocs.UTF8_VALUE, d.get("autf8"));
+ assertEquals("content2", CompatDocs.CONTENT2_VALUE, d.get("content2"));
+ assertEquals(CompatDocs.NON_ASCII_FIELD_NAME,
+ CompatDocs.NON_ASCII_FIELD_VALUE, d.get(CompatDocs.NON_ASCII_FIELD_NAME));
+
+ Fields tvFields = reader.getTermVectors(i);
+ assertNotNull("term vectors missing for doc " + i, tvFields);
+ assertNotNull("utf8 term vector missing for doc " + i, tvFields.terms("utf8"));
+ }
+
+ assertDocValues(reader, liveDocs);
+
+ // content term should match every live doc (34 of 35).
+ ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;
+ assertEquals(CompatDocs.DOC_COUNT - 1, hits.length);
+ assertEquals("first hit should be id 0", "0",
+ searcher.getIndexReader().document(hits[0].doc).get("id"));
+
+ // offsets/positions-bearing fields.
+ assertEquals(CompatDocs.DOC_COUNT - 1,
+ searcher.search(new TermQuery(new Term("content5", "aaa")), 1000).scoreDocs.length);
+ assertEquals(CompatDocs.DOC_COUNT - 1,
+ searcher.search(new TermQuery(new Term("content6", "aaa")), 1000).scoreDocs.length);
+
+ // The utf8 field's term dictionary must be identical to what the writing
+ // runtime produced. Both sides analyze the same string with the same
+ // StandardAnalyzer, so the produced term set is the cross-runtime contract
+ // for the UTF-8 edge cases (astral planes, the skull, etc.).
+ assertEquals("utf8 term set", CompatDocs.expectedUtf8Terms(),
+ collectTerms(reader, "utf8"));
+
+ // sanity: the content terms enum has exactly the single term "aaa".
+ Terms contentTerms = MultiFields.getTerms(reader, "content");
+ assertNotNull(contentTerms);
+ }
+
+
+ private static java.util.List collectTerms(IndexReader reader, String field) throws Exception {
+ java.util.List result = new java.util.ArrayList<>();
+ Terms terms = MultiFields.getTerms(reader, field);
+ if (terms == null) {
+ return result;
+ }
+ org.apache.lucene.index.TermsEnum te = terms.iterator(null);
+ BytesRef term;
+ while ((term = te.next()) != null) {
+ result.add(term.utf8ToString());
+ }
+ java.util.Collections.sort(result);
+ return result;
+ }
+
+ private static void assertDocValues(IndexReader reader, Bits liveDocs) throws Exception {
+ NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");
+ BinaryDocValues dvBytesDerefFixed = MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");
+ BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");
+ SortedDocValues dvBytesSortedFixed = MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");
+ SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");
+ BinaryDocValues dvBytesStraightFixed = MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");
+ BinaryDocValues dvBytesStraightVar = MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");
+ NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");
+ NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");
+ NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");
+ NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");
+ NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");
+ NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");
+ SortedSetDocValues dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");
+
+ assertNotNull("dvByte", dvByte);
+ assertNotNull("dvSortedSet", dvSortedSet);
+
+ for (int i = 0; i < CompatDocs.DOC_COUNT; i++) {
+ boolean live = liveDocs == null || liveDocs.get(i);
+ if (!live) {
+ continue;
+ }
+ int id = Integer.parseInt(reader.document(i).get("id"));
+ assertEquals("dvByte", id, dvByte.get(i));
+
+ byte[] bytes = new byte[] {
+ (byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id
+ };
+ BytesRef expected = new BytesRef(bytes);
+ BytesRef scratch = new BytesRef();
+
+ dvBytesDerefFixed.get(i, scratch);
+ assertEquals("dvBytesDerefFixed", expected, scratch);
+ dvBytesDerefVar.get(i, scratch);
+ assertEquals("dvBytesDerefVar", expected, scratch);
+ dvBytesSortedFixed.get(i, scratch);
+ assertEquals("dvBytesSortedFixed", expected, scratch);
+ dvBytesSortedVar.get(i, scratch);
+ assertEquals("dvBytesSortedVar", expected, scratch);
+ dvBytesStraightFixed.get(i, scratch);
+ assertEquals("dvBytesStraightFixed", expected, scratch);
+ dvBytesStraightVar.get(i, scratch);
+ assertEquals("dvBytesStraightVar", expected, scratch);
+
+ assertEquals("dvDouble", (double) id, Double.longBitsToDouble(dvDouble.get(i)), 0D);
+ assertEquals("dvFloat", (float) id, Float.intBitsToFloat((int) dvFloat.get(i)), 0F);
+ assertEquals("dvInt", id, dvInt.get(i));
+ assertEquals("dvLong", id, dvLong.get(i));
+ assertEquals("dvPacked", id, dvPacked.get(i));
+ assertEquals("dvShort", id, dvShort.get(i));
+
+ dvSortedSet.setDocument(i);
+ long ord = dvSortedSet.nextOrd();
+ assertEquals("dvSortedSet single ord",
+ SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());
+ dvSortedSet.lookupOrd(ord, scratch);
+ assertEquals("dvSortedSet value", expected, scratch);
+ }
+ }
+}