Skip to content

Commit da13f01

Browse files
FrottyFrotty
authored
Preserve JassDoc database during updates (#1281)
1 parent 7f25b41 commit da13f01

2 files changed

Lines changed: 406 additions & 11 deletions

File tree

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/JassDocService.java‎

Lines changed: 225 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,12 @@
1313
import java.io.IOException;
1414
import java.io.InputStream;
1515
import java.net.HttpURLConnection;
16+
import java.net.InetSocketAddress;
17+
import java.net.Proxy;
18+
import java.net.URI;
1619
import java.net.URL;
20+
import java.nio.charset.StandardCharsets;
21+
import java.nio.file.AtomicMoveNotSupportedException;
1722
import java.nio.file.Files;
1823
import java.nio.file.Path;
1924
import java.nio.file.StandardCopyOption;
@@ -24,22 +29,30 @@
2429
import java.sql.PreparedStatement;
2530
import java.sql.ResultSet;
2631
import java.sql.SQLException;
32+
import java.sql.Statement;
2733
import java.time.Duration;
2834
import java.time.Instant;
2935
import java.util.ArrayList;
36+
import java.util.Base64;
3037
import java.util.Comparator;
38+
import java.util.HashSet;
3139
import java.util.LinkedHashMap;
3240
import java.util.List;
3341
import java.util.Locale;
3442
import java.util.Map;
3543
import java.util.Optional;
44+
import java.util.Set;
3645
import java.util.concurrent.ConcurrentHashMap;
3746
import java.util.concurrent.atomic.AtomicBoolean;
3847
import java.util.function.Function;
3948
import java.util.stream.Stream;
4049

4150
public final class JassDocService {
4251

52+
// Optional JassDoc configuration (environment variable or system property):
53+
// WURST_JASSDOC_DB_AUTO_UPDATE=false keeps an existing latest DB indefinitely.
54+
// WURST_JASSDOC_DB_PROXY overrides HTTPS_PROXY/HTTP_PROXY for JassDoc requests.
55+
4356
public enum SymbolKind {
4457
FUNCTION, VARIABLE
4558
}
@@ -247,7 +260,7 @@ private Optional<String> lookupDocumentation(LookupKey key) {
247260
}
248261
try (Connection conn = open(dbPath.get())) {
249262
List<TableSchema> schemas = discoverSchemas(conn);
250-
boolean hasLegacySchema = hasLegacyJassdocSchema(conn);
263+
boolean hasLegacySchema = hasCompatibleLegacyJassdocSchema(conn);
251264
if (schemas.isEmpty() && !hasLegacySchema) {
252265
WLogger.warning("JassDoc DB found, but no compatible documentation tables were detected.");
253266
initFailed = true;
@@ -355,7 +368,7 @@ private void triggerAsyncInit() {
355368
}
356369

357370
private @Nullable String lookupFromLegacyJassdocTables(Connection conn, LookupKey key) throws SQLException {
358-
if (!tableExists(conn, "parameters")) {
371+
if (!hasCompatibleLegacyJassdocSchema(conn)) {
359372
return null;
360373
}
361374
Map<String, String> params = readKeyValueRows(conn, "parameters", "fnname", "param", "value", key.symbolName());
@@ -527,7 +540,7 @@ private Optional<Path> ensureDbAvailable() throws IOException {
527540
}
528541

529542
boolean needsDownload = !Files.exists(dbPath);
530-
if (!needsDownload && "latest".equals(revision)) {
543+
if (!needsDownload && "latest".equals(revision) && autoUpdateEnabled()) {
531544
needsDownload = isStaleLatest(dbPath);
532545
}
533546

@@ -590,7 +603,7 @@ private List<String> resolveLatestReleaseAssetUrls() {
590603
private List<String> readNewestReleaseAssetUrlsFromList() {
591604
List<String> urls = new ArrayList<>();
592605
try {
593-
HttpURLConnection con = (HttpURLConnection) new URL(RELEASES_API).openConnection();
606+
HttpURLConnection con = openHttpConnection(new URL(RELEASES_API));
594607
con.setConnectTimeout(10_000);
595608
con.setReadTimeout(20_000);
596609
con.setRequestMethod("GET");
@@ -625,7 +638,7 @@ private List<String> readNewestReleaseAssetUrlsFromList() {
625638
private List<String> readReleaseAssetUrls(String apiUrl) {
626639
List<String> urls = new ArrayList<>();
627640
try {
628-
HttpURLConnection con = (HttpURLConnection) new URL(apiUrl).openConnection();
641+
HttpURLConnection con = openHttpConnection(new URL(apiUrl));
629642
con.setConnectTimeout(10_000);
630643
con.setReadTimeout(20_000);
631644
con.setRequestMethod("GET");
@@ -710,6 +723,14 @@ private boolean isStaleLatest(Path dbPath) throws IOException {
710723
return modified.toInstant().isBefore(cutoff);
711724
}
712725

726+
boolean autoUpdateEnabled() {
727+
return Utils.getEnvOrConfig("WURST_JASSDOC_DB_AUTO_UPDATE")
728+
.map(value -> !value.equalsIgnoreCase("false")
729+
&& !value.equalsIgnoreCase("no")
730+
&& !value.equals("0"))
731+
.orElse(true);
732+
}
733+
713734
private Duration parseDurationOrDefault(String text) {
714735
try {
715736
return Duration.parse(text);
@@ -720,22 +741,191 @@ private Duration parseDurationOrDefault(String text) {
720741

721742
private void download(String urlString, Path target) throws IOException {
722743
URL url = new URL(urlString);
723-
HttpURLConnection con = (HttpURLConnection) url.openConnection();
744+
HttpURLConnection con = openHttpConnection(url);
724745
con.setConnectTimeout(10_000);
725746
con.setReadTimeout(20_000);
726747
con.setInstanceFollowRedirects(true);
727748
con.setRequestMethod("GET");
749+
con.setRequestProperty("User-Agent", "WurstScript-LSP");
728750
int code = con.getResponseCode();
729751
if (code < 200 || code >= 300) {
730752
throw new IOException("HTTP " + code + " for " + urlString);
731753
}
732754
Path tmp = Files.createTempFile(target.getParent(), "jassdoc-", ".tmp");
733-
try (InputStream in = con.getInputStream()) {
734-
Files.copy(in, tmp, StandardCopyOption.REPLACE_EXISTING);
755+
try {
756+
try (InputStream in = con.getInputStream()) {
757+
Files.copy(in, tmp, StandardCopyOption.REPLACE_EXISTING);
758+
}
759+
installDownloadedDatabase(tmp, target);
735760
} finally {
736761
con.disconnect();
762+
Files.deleteIfExists(tmp);
737763
}
738-
Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
764+
}
765+
766+
void installDownloadedDatabase(Path downloaded, Path target) throws IOException {
767+
validateDownloadedDatabase(downloaded);
768+
769+
Path backup = target.resolveSibling(target.getFileName() + ".bak");
770+
if (Files.exists(target)) {
771+
Path backupTmp = Files.createTempFile(target.getParent(), "jassdoc-backup-", ".tmp");
772+
try {
773+
Files.copy(target, backupTmp, StandardCopyOption.REPLACE_EXISTING,
774+
StandardCopyOption.COPY_ATTRIBUTES);
775+
replaceAtomically(backupTmp, backup);
776+
} finally {
777+
Files.deleteIfExists(backupTmp);
778+
}
779+
}
780+
781+
try {
782+
replaceAtomically(downloaded, target);
783+
} catch (IOException installFailure) {
784+
if (Files.exists(backup)) {
785+
restoreBackup(backup, target, installFailure);
786+
}
787+
throw installFailure;
788+
}
789+
}
790+
791+
void restoreBackup(Path backup, Path target, IOException installFailure) {
792+
Path restoreTmp = null;
793+
try {
794+
restoreTmp = Files.createTempFile(target.getParent(), "jassdoc-restore-", ".tmp");
795+
Files.copy(backup, restoreTmp, StandardCopyOption.REPLACE_EXISTING,
796+
StandardCopyOption.COPY_ATTRIBUTES);
797+
replaceAtomically(restoreTmp, target);
798+
} catch (IOException restoreFailure) {
799+
installFailure.addSuppressed(restoreFailure);
800+
} finally {
801+
if (restoreTmp != null) {
802+
try {
803+
Files.deleteIfExists(restoreTmp);
804+
} catch (IOException cleanupFailure) {
805+
installFailure.addSuppressed(cleanupFailure);
806+
}
807+
}
808+
}
809+
}
810+
811+
private void validateDownloadedDatabase(Path downloaded) throws IOException {
812+
try (Connection conn = open(downloaded)) {
813+
if (!passesIntegrityCheck(conn)) {
814+
throw new IOException("Downloaded JassDoc database failed SQLite integrity check");
815+
}
816+
if (discoverSchemas(conn).isEmpty() && !hasCompatibleLegacyJassdocSchema(conn)) {
817+
throw new IOException("Downloaded file is not a compatible JassDoc database");
818+
}
819+
} catch (SQLException e) {
820+
throw new IOException("Downloaded file is not a valid JassDoc database", e);
821+
}
822+
}
823+
824+
private boolean passesIntegrityCheck(Connection conn) throws SQLException {
825+
try (Statement statement = conn.createStatement();
826+
ResultSet result = statement.executeQuery("PRAGMA integrity_check")) {
827+
return result.next()
828+
&& "ok".equalsIgnoreCase(result.getString(1))
829+
&& !result.next();
830+
}
831+
}
832+
833+
private void replaceAtomically(Path source, Path target) throws IOException {
834+
try {
835+
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING,
836+
StandardCopyOption.ATOMIC_MOVE);
837+
} catch (AtomicMoveNotSupportedException e) {
838+
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING);
839+
}
840+
}
841+
842+
private HttpURLConnection openHttpConnection(URL url) throws IOException {
843+
Optional<String> proxySetting = selectProxySetting(url, Utils::getEnvOrConfig);
844+
if (proxySetting.isEmpty()) {
845+
return (HttpURLConnection) url.openConnection();
846+
}
847+
848+
URI proxyUri = parseHttpProxyUri(proxySetting.get());
849+
int port = proxyUri.getPort() >= 0 ? proxyUri.getPort() : 80;
850+
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), port));
851+
HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
852+
if (proxyUri.getUserInfo() != null) {
853+
String credentials = Base64.getEncoder().encodeToString(
854+
proxyUri.getUserInfo().getBytes(StandardCharsets.UTF_8));
855+
connection.setRequestProperty("Proxy-Authorization", "Basic " + credentials);
856+
}
857+
return connection;
858+
}
859+
860+
static Optional<String> selectProxySetting(URL url,
861+
Function<String, Optional<String>> lookup) {
862+
Optional<String> proxySetting = lookup.apply("WURST_JASSDOC_DB_PROXY");
863+
if (proxySetting.isPresent()) {
864+
return proxySetting;
865+
}
866+
Optional<String> noProxy = firstConfigured(lookup, "NO_PROXY", "no_proxy");
867+
if (noProxy.isPresent() && shouldBypassProxy(url.getHost(), noProxy.get())) {
868+
return Optional.empty();
869+
}
870+
if ("https".equalsIgnoreCase(url.getProtocol())) {
871+
return firstConfigured(lookup, "HTTPS_PROXY", "https_proxy");
872+
}
873+
if ("http".equalsIgnoreCase(url.getProtocol())) {
874+
return firstConfigured(lookup, "HTTP_PROXY", "http_proxy");
875+
}
876+
return Optional.empty();
877+
}
878+
879+
static URI parseHttpProxyUri(String value) throws IOException {
880+
URI proxyUri;
881+
try {
882+
proxyUri = URI.create(value.contains("://") ? value : "http://" + value);
883+
} catch (IllegalArgumentException e) {
884+
throw new IOException("Invalid JassDoc proxy URL", e);
885+
}
886+
if (proxyUri.getHost() == null) {
887+
throw new IOException("Invalid JassDoc proxy URL: missing host");
888+
}
889+
if (!"http".equalsIgnoreCase(proxyUri.getScheme())) {
890+
throw new IOException("Unsupported JassDoc proxy scheme '" + proxyUri.getScheme()
891+
+ "'; use an http:// proxy URL");
892+
}
893+
return proxyUri;
894+
}
895+
896+
static boolean shouldBypassProxy(String host, String noProxySetting) {
897+
String normalizedHost = host.toLowerCase(Locale.ROOT);
898+
for (String rawEntry : noProxySetting.split(",")) {
899+
String entry = rawEntry.trim().toLowerCase(Locale.ROOT);
900+
if (entry.equals("*")) {
901+
return true;
902+
}
903+
int portSeparator = entry.lastIndexOf(':');
904+
if (portSeparator > 0 && entry.indexOf(':') == portSeparator) {
905+
entry = entry.substring(0, portSeparator);
906+
}
907+
if (entry.startsWith("*.")) {
908+
entry = entry.substring(1);
909+
}
910+
if (entry.startsWith(".")) {
911+
if (normalizedHost.endsWith(entry)
912+
|| normalizedHost.equals(entry.substring(1))) {
913+
return true;
914+
}
915+
} else if (!entry.isEmpty() && (normalizedHost.equals(entry)
916+
|| normalizedHost.endsWith("." + entry))) {
917+
return true;
918+
}
919+
}
920+
return false;
921+
}
922+
923+
private static Optional<String> firstConfigured(
924+
Function<String, Optional<String>> lookup, String... names) {
925+
return Stream.of(names)
926+
.map(lookup)
927+
.flatMap(Optional::stream)
928+
.findFirst();
739929
}
740930

741931
private Connection open(Path dbPath) throws SQLException {
@@ -778,8 +968,32 @@ private List<TableSchema> discoverSchemas(Connection conn) throws SQLException {
778968
return result;
779969
}
780970

781-
private boolean hasLegacyJassdocSchema(Connection conn) throws SQLException {
782-
return tableExists(conn, "parameters");
971+
private boolean hasCompatibleLegacyJassdocSchema(Connection conn) throws SQLException {
972+
return tableHasColumns(conn, "parameters", "fnname", "param", "value")
973+
&& (!tableExists(conn, "annotations")
974+
|| tableHasColumns(conn, "annotations", "fnname", "anname", "value"))
975+
&& (!tableExists(conn, "params_extra")
976+
|| tableHasColumns(conn, "params_extra", "fnname", "param", "anname", "value"));
977+
}
978+
979+
private boolean tableHasColumns(Connection conn, String tableName, String... requiredColumns)
980+
throws SQLException {
981+
Set<String> columns = new HashSet<>();
982+
DatabaseMetaData md = conn.getMetaData();
983+
try (ResultSet result = md.getColumns(null, null, tableName, "%")) {
984+
while (result.next()) {
985+
String name = result.getString("COLUMN_NAME");
986+
if (name != null) {
987+
columns.add(name.toLowerCase(Locale.ROOT));
988+
}
989+
}
990+
}
991+
for (String required : requiredColumns) {
992+
if (!columns.contains(required)) {
993+
return false;
994+
}
995+
}
996+
return true;
783997
}
784998

785999
private boolean tableExists(Connection conn, String tableName) throws SQLException {

0 commit comments

Comments
 (0)