Skip to content
Draft
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions java/mlt-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -67,19 +67,24 @@ spotless {
}
}

tasks.register('compileWrapper', Exec) {
workingDir = project.projectDir
doFirst {
if (System.properties['os.name'].toLowerCase().contains('windows')) {
executable "../resources/compile-windows.bat"
} else {
executable "./compile-wrapper.sh"
tasks.register('compileWrapper') {
description = 'Builds the native FSST wrapper'
group = 'build'
outputs.files layout.projectDirectory.dir('../resources/build/libfsstwrapper.so')

doLast {
def baseDir = file(layout.projectDirectory.dir('../resources'))
exec {
workingDir baseDir
if (System.properties['os.name'].toLowerCase().contains('windows')) {
commandLine "compile-windows.bat"
} else {
commandLine './compile'
}
}
}
}

// compileJava.dependsOn compileWrapper // Disabled due to CMake compatibility issues

gradle.projectsEvaluated {
tasks.withType(JavaCompile).tap {
configureEach {
Expand All @@ -92,6 +97,7 @@ gradle.projectsEvaluated {

tasks.withType(Test).configureEach {
useJUnitPlatform()
dependsOn(tasks.named('compileWrapper'))
}

tasks.register('validateSemver') {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,18 @@ class FsstDebug implements Fsst {

@Override
public SymbolTable encode(byte[] data) {
new LongSummaryStatistics();

long a = System.currentTimeMillis();
var fromJni = jni.encode(data);
long b = System.currentTimeMillis();
var fromJava = java.encode(data);
long c = System.currentTimeMillis();
jniTime.addAndGet(b - a);
javaTime.addAndGet(c - b);
jniSize.addAndGet(fromJni.weight());
javaSize.addAndGet(fromJava.weight());
(fromJava.weight() <= fromJni.weight() ? javaSmaller : jniSmaller).incrementAndGet();
final long a = System.currentTimeMillis();
final var fromJni = FsstJni.isLoaded() ? jni.encode(data) : null;
final long b = System.currentTimeMillis();
final var fromJava = java.encode(data);
final long c = System.currentTimeMillis();
if (fromJni != null) {
jniTime.addAndGet(b - a);
javaTime.addAndGet(c - b);
jniSize.addAndGet(fromJni.weight());
javaSize.addAndGet(fromJava.weight());
(fromJava.weight() <= fromJni.weight() ? javaSmaller : jniSmaller).incrementAndGet();
}
return fromJava;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
import java.nio.file.FileSystems;

class FsstJni implements Fsst {
private static boolean isLoaded = false;

static {
String os = System.getProperty("os.name").toLowerCase();
boolean isWindows = os.contains("win");
String moduleDir = "build/FsstWrapper.so";
final String os = System.getProperty("os.name").toLowerCase();
final boolean isWindows = os.contains("win");
String moduleDir = "../build/FsstWrapper.so";
if (isWindows) {
// TODO: figure out how to get cmake to put in common directory
moduleDir = "build/Release/FsstWrapper.so";
Expand All @@ -16,6 +17,7 @@ class FsstJni implements Fsst {
FileSystems.getDefault().getPath(moduleDir).normalize().toAbsolutePath().toString();
try {
System.load(modulePath);
isLoaded = true;
} catch (UnsatisfiedLinkError e) {
System.out.println("Error: " + e.getMessage() + " - " + modulePath);
}
Expand All @@ -25,5 +27,9 @@ public SymbolTable encode(byte[] data) {
return FsstJni.compress(data);
}

public static boolean isLoaded() {
return isLoaded;
}

public static native SymbolTable compress(byte[] inputBytes);
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
Expand All @@ -25,33 +24,28 @@ static void printStats() {
}

@Test
@Disabled
void decode_simpleString_ValidEncodedAndDecoded() {
test("AAAAAAABBBAAACCdddddEEEEEEfffEEEEAAAAAddddCC");
}

@Test
@Disabled
void decodeLongRepeated() {
test("AAAAAAABBBAAACCdddddEEEEEEfffEEEEAAAAAddddCC".repeat(100));
}

@Test
@Disabled
void empty() {
test("");
}

@Test
@Disabled
void repeatedStrings() {
for (int i = 1; i < 1000; i++) {
test("a".repeat(i));
}
}

@Test
@Disabled
void allBytes() {
byte[] toEncode = new byte[1000];
for (int i = 0; i < toEncode.length; i++) {
Expand All @@ -61,7 +55,7 @@ void allBytes() {
}

static List<Path> tiles() throws IOException {
try (var stream = Files.walk(Path.of("..", "test"))) {
try (var stream = Files.walk(Path.of("../..", "test"))) {
return stream
.filter(file -> Files.isRegularFile(file) && file.toString().endsWith(".mvt"))
.toList();
Expand All @@ -70,7 +64,6 @@ static List<Path> tiles() throws IOException {

@ParameterizedTest
@MethodSource("tiles")
@Disabled
void fsstEncodeTile(Path path) throws IOException {
// stress-test FSST encoding by using it to encode raw tiles
// ideally this would encode just the dictionaries, but it's close enough for now
Expand Down Expand Up @@ -108,29 +101,34 @@ private static void assertSymbolSortOrder(SymbolTable table) {

@SuppressWarnings("deprecation")
private static void test(byte[] input) {
var encodedJava = JAVA.encode(input);
var encodedJni = JNI.encode(input);
int maxAllowed = Math.max((int) (encodedJni.weight() * 1.02), encodedJni.weight() + 2);
assertTrue(
encodedJava.weight() <= maxAllowed,
() ->
"""
Input: byte[%d]
Encoded java >5%% larger than JNI
Java: %s
JNI: %s
"""
.formatted(input.length, encodedJava, encodedJni));
assertSymbolSortOrder(encodedJni);
final var encodedJava = JAVA.encode(input);
final var encodedJni = FsstJni.isLoaded() ? JNI.encode(input) : null;
assertSymbolSortOrder(encodedJava);
assertArrayEquals(input, JAVA.decode(encodedJava));
assertArrayEquals(input, JAVA.decode(encodedJni));
assertArrayEquals(input, JNI.decode(encodedJava));
assertArrayEquals(input, JNI.decode(encodedJni));
assertArrayEquals(
input,
JNI.decode(
encodedJava.symbols(), encodedJava.symbolLengths(), encodedJava.compressedData()));

if (encodedJni != null) {
assertArrayEquals(input, JAVA.decode(encodedJni));

final int maxAllowed = Math.max((int) (encodedJni.weight() * 1.02), encodedJni.weight() + 2);
assertTrue(
encodedJava.weight() <= maxAllowed,
() ->
"""
Input: byte[%d]
Encoded java >5%% larger than JNI
Java: %s
JNI: %s
"""
.formatted(input.length, encodedJava, encodedJni));

assertSymbolSortOrder(encodedJni);
assertArrayEquals(input, JNI.decode(encodedJava));
assertArrayEquals(input, JNI.decode(encodedJni));
assertArrayEquals(
input,
JNI.decode(
encodedJava.symbols(), encodedJava.symbolLengths(), encodedJava.compressedData()));
}
DEBUG.encode(input);
}
}
2 changes: 1 addition & 1 deletion java/resources/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ ExternalProject_Add(FsstDownload
GIT_TAG bd766bc # https://github.com/springmeyer/fsst/tree/dane/cross-platform-build-fixes
INSTALL_COMMAND ""
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libfsst-prefix
CMAKE_ARGS "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}"
CMAKE_ARGS "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}" "-DCMAKE_POLICY_VERSION_MINIMUM=3.5"
)
SET(LIBFSST_LIB_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/libfsst-prefix/src/FsstDownload-build)
SET(LIBFSST_INC_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/libfsst-prefix/src/FsstDownload)
Expand Down
6 changes: 3 additions & 3 deletions java/resources/FsstWrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ SymbolTableStruct fsstCompress(std::vector<unsigned char> inputBytes) {
return symbolTableStruct;
}

JNIEXPORT jobject JNICALL Java_com_mlt_converter_encodings_fsst_FsstJni_compress(JNIEnv* env,
jclass cls,
jbyteArray inputBytes) {
JNIEXPORT jobject JNICALL Java_org_maplibre_mlt_converter_encodings_fsst_FsstJni_compress(JNIEnv* env,
jclass cls,
jbyteArray inputBytes) {
jbyte* bytes = env->GetByteArrayElements(inputBytes, NULL);
jsize length = env->GetArrayLength(inputBytes);

Expand Down
9 changes: 5 additions & 4 deletions java/resources/compile
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ if [[ -f build/FsstWrapper.so ]]; then
echo " Remove ./build to reconfigure & compiled"
else
echo "FsstWrapper.so does not exist, building now"
mkdir -p build
cd build
cmake ../resources
cmake --build . --config Release
mkdir -p resources/build
cd resources/build
cmake .. -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=Release
cmake --build .
cp -f FsstWrapper.so ../../build/FsstWrapper.so
fi
echo "Compilation successful"
9 changes: 5 additions & 4 deletions java/resources/compile-windows.bat
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ IF EXIST build\Release\FsstWrapper.so (
echo " Remove ./build to reconfigure & compiled"
) ELSE (
echo "FsstWrapper.so does not exist, building now"
mkdir build
cd build
cmake ../Resources
cmake --build . --config Release
mkdir resources\build
cd resources\build
cmake .. -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=Release
cmake --build .
copy FsstWrapper.so ..\..\build\FsstWrapper.so
)
IF %ERRORLEVEL% EQU 0 (
echo "Compilation successful"
Expand Down
Loading