Feature/separate search include paths

src/main/java/com/vizor/unreal/Main.java

@@ -26,6 +26,9 @@
 
 import java.nio.file.Path;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static com.vizor.unreal.util.Misc.findFilesRecursively;
 import static com.vizor.unreal.util.Misc.stringIsNullOrEmpty;
@@ -73,13 +76,21 @@ private static void launch(String[] args)
             log.debug("Globally changed {} log level from {} to {}", packageName, previousLevel.name(), logLevel.name());
         }
 
-        final Path srcPath = get(config.getSrcPath());
+        final List<Path> searchPaths = config.getProtoSearchPaths().stream().map(pathString->get(pathString)).collect(Collectors.toList());
+        final List<Path> includePaths = config.getProtoIncludePaths().stream().map(pathString->get(pathString)).collect(Collectors.toList());
         final DestinationConfig dstPath = config.getDstPath();
 
         final Converter converter = new Converter(config.getModuleName());
 
-        if (!srcPath.toFile().isDirectory())
-            throw new IllegalArgumentException("Source folder '" + srcPath + "' does not exist, or isn't a directory");
+        searchPaths.forEach(searchPath->{
+            if (!searchPath.toFile().isDirectory())
+                throw new IllegalArgumentException("Search path folder '" + searchPath + "' does not exist, or isn't a directory");
+        });
+
+        includePaths.forEach(includePath -> {
+            if (!includePath.toFile().isDirectory())
+                throw new IllegalArgumentException("Include path folder '" + includePath + "' does not exist, or isn't a directory");
+        });
 
         if (!dstPath.pathPublic.toFile().isDirectory())
             throw new IllegalArgumentException("Destination Public folder '" + dstPath.pathPublic + "' does not exist, or isn't a directory");
@@ -89,8 +100,8 @@ private static void launch(String[] args)
 
         log.info("Running cornerstone...");
         log.info("Logging level: {}", log.getLevel().toString());
-        log.info("Source path: '{}'", srcPath);
-        log.info("Destination path: '{}'", dstPath);
+
+        log.info("Destination paths: '{}'", dstPath);
 
         if (!stringIsNullOrEmpty(config.getModuleName()))
         {
@@ -105,18 +116,42 @@ private static void launch(String[] args)
         log.info("Company name: {}", config.getCompanyName());
         log.info("Wrappers path: %INCLUDE_DIR%/{}", config.getWrappersPath());
 
-        launchSingle(srcPath, dstPath, converter);
+        launchSingle(searchPaths, includePaths, dstPath, converter);
     }
 
-    private static void launchSingle(final Path srcPath, final DestinationConfig dstPath, final Converter converter)
+    private static Path getIncludeRelativePath(final Path absolutePath, final List<Path> includePaths)
     {
-        final long start = nanoTime();
+        for (final Path possibleBasePath : includePaths) 
+        {
+            if (absolutePath.startsWith(possibleBasePath))
+            {
+                return possibleBasePath;
+            }
+        }
 
-        final List<Tuple<Path, DestinationConfig>> paths = findFilesRecursively(srcPath, dstPath, "proto");
+        throw new RuntimeException("Couldn't find a base path for " + absolutePath);
+    }
+
+    private static void launchSingle(final List<Path> searchPaths, final List<Path> includePaths, final DestinationConfig dstPath, final Converter converter)
+    {
+        final long start = nanoTime();
 
-        // Display how many proto file(s) pending processed
-        log.info("Running converter, {} proto-files pending processed.", paths.size());
-        converter.convert(srcPath, paths);
+        final Map<Path, List<Tuple<Path, DestinationConfig>>> paths = 
+            searchPaths.stream()
+                .flatMap(
+                    searchPath -> findFilesRecursively(searchPath, dstPath, "proto").stream()
+                )
+                .collect(Collectors.groupingBy(pathTuple -> getIncludeRelativePath(pathTuple.first(), includePaths)));
+
+        final Map<Path, List<Tuple<Path, DestinationConfig>>> visiblePaths = includePaths.stream()
+            .flatMap(
+                includePath -> findFilesRecursively(includePath, dstPath, "proto")
+                    .stream()
+                    .map(pathTuple -> Tuple.of(includePath, pathTuple))
+                )
+            .collect(Collectors.groupingBy(tuple -> tuple.first(), Collectors.mapping(tuple -> tuple.second(), Collectors.toList())));
+
+        converter.convert(visiblePaths, paths);
 
         final float elapsed = (float) round((double) (nanoTime() - start) / 1000000.0) / 1000.0f;
         log.info("All done in {} seconds. Shutting converter down...", elapsed);

src/main/java/com/vizor/unreal/config/Config.java

@@ -70,7 +70,10 @@ public final class Config
     private static Config config = null;
 
     @ConfigField
-    private String srcPath;
+    private List<String> protoSearchPaths;
+
+    @ConfigField
+    private List<String> protoIncludePaths;
 
     @ConfigField
     private String dstPublicPath;
@@ -96,15 +99,24 @@ public final class Config
     @ConfigField
     private boolean noFork;
 
+    public final List<String> getProtoSearchPaths()
+    {
+        return protoSearchPaths;
+    }
 
-    public final String getSrcPath()
+    public void setProtoSearchPaths(List<String> protoSearchPaths)
     {
-        return srcPath;
+        this.protoSearchPaths = protoSearchPaths;
     }
 
-    public void setSrcPath(String srcPath)
+    public final List<String> getProtoIncludePaths()
     {
-        this.srcPath = srcPath;
+        return protoIncludePaths;
+    }
+
+    public void setProtoIncludePaths(List<String> protoIncludePaths)
+    {
+        this.protoIncludePaths = protoIncludePaths;
     }
 
     public final String getDstPublicPath()
@@ -291,9 +303,16 @@ private void checkString(final String string, final String failMessage)
             throw new RuntimeException(failMessage);
     }
 
+    private void checkStringList(final List<String> strings, final String failMessage)
+    {
+        if (strings == null || strings.isEmpty() || strings.stream().anyMatch(string->stringIsNullOrEmpty(string)))
+            throw new RuntimeException(failMessage);
+    }
+
     public final void validate()
     {
-        checkString(srcPath, "srcPath must not be null or empty");
+        checkStringList(protoSearchPaths, "protoSearchPaths must not be null or empty");
+        checkStringList(protoIncludePaths, "protoIncludePaths must not be null or empty");
         checkString(dstPublicPath, "dstPublicPath must not be null or empty");
         checkString(dstPrivatePath, "dstPrivatePath must not be null or empty");
 

src/main/java/com/vizor/unreal/config/DestinationConfig.java

@@ -24,4 +24,10 @@ public DestinationConfig append(String childPathString) {
 
     public Path pathPublic;
     public Path pathPrivate;
+
+    @Override
+    public String toString()
+    {
+        return "Public: " + pathPublic + " / Private: " + pathPrivate;
+    }
 }

src/main/java/com/vizor/unreal/convert/Converter.java

@@ -25,9 +25,13 @@
 import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -58,6 +62,143 @@ public Converter(final String moduleName)
         this.moduleName = moduleName;
     }
 
+    private static void gatherImportedPathsInternal(final Map<Path, List<Tuple<Path, DestinationConfig>>> visiblePaths,
+        final Map<Path, List<Tuple<Path, DestinationConfig>>> requestedPaths)
+    {
+
+    }
+
+    private List<ProtoProcessorArgs> createArgs(final Path basePath, final Path pathToProto, final DestinationConfig destinationConfig)
+    {
+        String fileContent = null;
+
+        try 
+        {
+            fileContent = join(lineSeparator(), readAllLines(pathToProto));
+        }
+        catch (IOException ex)
+        {
+            throw new RuntimeException(ex);
+        }
+
+        final List<ProtoProcessorArgs> protoArgs = preProcess(parse(get(pathToProto.toString()), fileContent))
+        .stream()
+        .map(
+            protoFile -> 
+            {
+                final Path relativePath = basePath.relativize(pathToProto); 
+                return new ProtoProcessorArgs(protoFile, relativePath, destinationConfig, moduleName);
+            })
+        .collect(Collectors.toList());
+
+        return protoArgs;
+    }
+
+	public void convert(final Map<Path, List<Tuple<Path, DestinationConfig>>> visiblePaths,
+			final Map<Path, List<Tuple<Path, DestinationConfig>>> requestedPaths) {
+
+                // include-relative path to base path and full path and destination
+                Map<Path, Tuple<Path, Tuple<Path, DestinationConfig>>> importPathToPathSetupLookup = visiblePaths.entrySet().stream().flatMap((protoBasePathTuple)->{
+                    final Path basePath = protoBasePathTuple.getKey();
+                    final List<Tuple<Path, DestinationConfig>> protoPathTuples = protoBasePathTuple.getValue();
+                    return protoPathTuples.stream().map(protoPathTuple->{
+                        return Tuple.of(basePath, protoPathTuple);
+                    });
+                })
+                .collect(Collectors.toMap(tuple->tuple.first().relativize(tuple.second().first()), tuple->tuple));
+
+        // full path to base path and destination
+        final Map<Path, Tuple<Path, DestinationConfig>> pathsToActuallyLoad = new HashMap<>();
+
+        requestedPaths.forEach((basePath, pathTuples) -> {
+            pathTuples.forEach(pathTuple->{
+                pathsToActuallyLoad.put(pathTuple.first(), Tuple.of(basePath, pathTuple.second()));
+            });
+        });
+
+        while (true)
+{
+        final Map<Path, Tuple<Path, DestinationConfig>> pathsToActuallyLoadIter = new HashMap<>(pathsToActuallyLoad);
+        requestedPaths.forEach((basePath, protoPathTuples)-> {
+            protoPathTuples.forEach(protoPathTuple-> {
+                final List<ProtoProcessorArgs> args = createArgs(basePath, protoPathTuple.first(), protoPathTuple.second());
+
+                args
+                    .stream()
+                    .map(arg1->{return arg1.parse;})
+                    .flatMap(parse1->{return parse1.imports().stream();})
+                    .forEach(
+                        importPath->
+                        {
+                            final Tuple<Path, Tuple<Path, DestinationConfig>> importedTuple = importPathToPathSetupLookup.get(FileSystems.getDefault().getPath(importPath));
+
+if (!pathsToActuallyLoadIter.containsKey(importedTuple.first()))
+{
+pathsToActuallyLoadIter.put(importedTuple.second().first(), Tuple.of(importedTuple.first(), importedTuple.second().second()));
+}
+
+                        });
+            });
+        });
+
+        if (pathsToActuallyLoadIter.equals(pathsToActuallyLoad))
+        {
+            break;
+        }
+
+        pathsToActuallyLoad.putAll(pathsToActuallyLoadIter);
+    }
+
+        convert(pathsToActuallyLoad);
+	}
+
+    // full path to base path to destination
+	private void convert(final Map<Path, Tuple<Path, DestinationConfig>> pathsToActuallyLoad) {
+
+        final List<ProtoProcessorArgs> protosToGenerate = new ArrayList<>();
+
+        pathsToActuallyLoad.forEach((pathToProto, basePathDestinationTuple) -> {
+            final Path basePath = basePathDestinationTuple.first();
+            final DestinationConfig pathToConverted = basePathDestinationTuple.second();
+
+            String fileContent = null;
+
+            try 
+            {
+                fileContent = join(lineSeparator(), readAllLines(pathToProto));
+            }
+            catch (IOException ex)
+            {
+                throw new RuntimeException(ex);
+            }
+
+
+            final List<ProtoProcessorArgs> protoArgs = preProcess(parse(get(pathToProto.toString()), fileContent))
+            .stream()
+            .map(
+                protoFile -> 
+                {
+                    final Path relativePath = basePath.relativize(pathToProto); 
+                    return new ProtoProcessorArgs(protoFile, relativePath, pathToConverted, moduleName);
+                })
+            .collect(Collectors.toList());
+
+            protosToGenerate.addAll(protoArgs);
+        });
+
+        Stream<ProtoProcessorArgs> argsStream = protosToGenerate.stream();
+
+        if (!Config.get().isNoFork())
+        {
+            argsStream = argsStream.parallel();
+        }
+
+        argsStream.forEach(arg -> {
+            log.info("Converting {}", arg.pathToProto);
+            new ProtoProcessor(arg, protosToGenerate).run();
+        });
+    }
+
     public void convert(final Path srcPath, final List<Tuple<Path, DestinationConfig>> paths)
     {
         List<ProtoProcessorArgs> args = new ArrayList<>();
@@ -67,7 +208,7 @@ public void convert(final Path srcPath, final List<Tuple<Path, DestinationConfig
         pathsStream.forEach(pathPair -> {
             final Path pathToProto = pathPair.first();
             final DestinationConfig pathToConverted = pathPair.second();
-            
+
             String fileContent = null;
 
             try 

src/main/java/com/vizor/unreal/convert/ProtoProcessor.java

@@ -96,15 +96,15 @@ class ProtoProcessorArgs
         this.packageNamespace = new CppNamespace(parse.packageName());
     }
 
-    final ProtoFileElement parse;
-    final Path pathToProto;
-    final DestinationConfig pathToConverted;
-    final String moduleName;
+    public final ProtoFileElement parse;
+    public final Path pathToProto;
+    public final DestinationConfig pathToConverted;
+    public final String moduleName;
 
-    final String wrapperName;
+    public final String wrapperName;
 
-    final String className;
-    final CppNamespace packageNamespace;
+    public final String className;
+    public final CppNamespace packageNamespace;
 }
 
 class ProtoProcessor implements Runnable
@@ -402,7 +402,13 @@ private CppStruct extractStruct(final TypesProvider provider, final MessageEleme
             if (!sourceDoc.isEmpty())
                 field.javaDoc.set(sourceDoc);
 
-            field.addAnnotation(fieldAnnotations);
+field.addAnnotation(fieldAnnotations);
+
+if (ueType == type)
+{
+    // disable UPROPERTY on self-references
+    field.enableAnnotations(false);
+}
             fields.add(field);
         }
 

src/main/java/com/vizor/unreal/provider/ProtoTypesProvider.java

@@ -73,6 +73,7 @@ protected final void init()
         register("int32", plainNs("int32", Primitive, protobufNamespace), int.class);
         register("int64", plainNs("int64", Primitive, protobufNamespace), long.class);
         register("float", plain("float", Primitive), float.class);
+        register("double", plain("double", Primitive), double.class);
         register("bool", plain("bool", Primitive), boolean.class);
         register("void", plain("void", Primitive), void.class);
 

src/main/java/com/vizor/unreal/provider/TypesProvider.java

@@ -148,8 +148,11 @@ final void register(final String protoType, final CppType cppType, final Class<?
     {
         final CppType previous = types.put(protoType, cppType);
         if (nonNull(previous))
-            throw new RuntimeException("Type association '" + protoType + "' -> '" + previous.getName() +
-                    "' is already defined");
+            {
+                return;
+                // throw new RuntimeException("Type association '" + protoType + "' -> '" + previous.getName() +
+                //     "' is already defined");
+    }
 
         if (nonNull(nativeType))
             cppType.markAsNative(nativeType);