From b555d00eff367a151733439f601966c09f6b5291 Mon Sep 17 00:00:00 2001 From: Carpe-Wang Date: Sun, 18 Aug 2024 17:36:24 -0400 Subject: [PATCH 1/4] support remote files, --- .../main/java/org/pkl/core/util/IoUtils.java | 43 ++++++++++++++++++- .../kotlin/org/pkl/core/util/IoUtilsTest.kt | 14 ++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index 96ee80b76..2c687196d 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -16,10 +16,13 @@ package org.pkl.core.util; import com.oracle.truffle.api.TruffleOptions; +import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -55,6 +58,13 @@ private IoUtils() {} public static URL toUrl(URI uri) throws IOException { try { + if ("file".equalsIgnoreCase(uri.getScheme()) && uri.getHost() != null) { + // Dynamically determine the protocol, defaulting to http + String protocol = "http"; // Default protocol + URL remoteFileUrl = new URL(protocol, uri.getHost(), uri.getPort(), uri.getPath()); + File localFile = downloadFile(remoteFileUrl); + return localFile.toURI().toURL(); + } return uri.toURL(); } catch (Error e) { // best we can do for now @@ -62,11 +72,42 @@ public static URL toUrl(URI uri) throws IOException { if (e.getClass().getName().equals("com.oracle.svm.core.jdk.UnsupportedFeatureError")) { throw new IOException("Unsupported protocol: " + uri.getScheme()); } - throw e; } } + private static File downloadFile(URL url) throws IOException { + File tempFile = File.createTempFile("pkl_", ".tmp"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setInstanceFollowRedirects(true); // Enable automatic redirection + // connection.setRequestProperty("User-Agent", "Mozilla/5.0"); // Some servers might reject requests without a User-Agent header + + int responseCode = connection.getResponseCode(); + + // Handle redirection manually + if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || responseCode == HttpURLConnection.HTTP_MOVED_TEMP) { + String newUrl = connection.getHeaderField("Location"); + connection = (HttpURLConnection) new URL(newUrl).openConnection(); + // connection.setRequestProperty("User-Agent", "Mozilla/5.0"); + responseCode = connection.getResponseCode(); + } + + if (responseCode != HttpURLConnection.HTTP_OK) { + throw new IOException("Failed to download file: HTTP response code " + responseCode); + } + + try (InputStream in = connection.getInputStream(); + FileOutputStream out = new FileOutputStream(tempFile)) { + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + return tempFile; + } + + /** Checks whether the given string is "URI-like", i.e. matches a pattern like {@code foo:bar}. */ public static boolean isUriLike(String str) { return uriLike.matcher(str).matches(); diff --git a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt index 618afe58c..d412b9936 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt @@ -31,6 +31,7 @@ import org.pkl.core.SecurityManager import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.module.ModuleKeys import org.pkl.core.runtime.ModuleResolver +import java.io.File class IoUtilsTest { object FakeSecurityManager : SecurityManager { @@ -400,6 +401,19 @@ class IoUtilsTest { assertThrows { IoUtils.readString(URI("http://example.com").toURL()) } } + @Test + fun `toUrl() supports remote file URIs`() { + // Use a publicly available text file on GitHub as the test URI + val uri = URI("file://raw.githubusercontent.com/octocat/Hello-World/master/README") + val resultUrl = IoUtils.toUrl(uri) + + val tempFile = File(resultUrl.toURI()) + assertThat(tempFile.exists()).isTrue + + val downloadedContent = IoUtils.readString(resultUrl.openStream()) + assertThat(downloadedContent).contains("Hello World!") + } + @Test fun `encodePath encodes characters reserved on windows`() { assertThat(IoUtils.encodePath("foo:bar")).isEqualTo("foo(3a)bar") From c795f4e94a02dd8ae961cde2f0bffce5f36af0a0 Mon Sep 17 00:00:00 2001 From: Carpe-Wang Date: Sun, 18 Aug 2024 17:55:42 -0400 Subject: [PATCH 2/4] support remote files, after ./gradlew :pkl-core:spotlessApply --- pkl-core/src/main/java/org/pkl/core/util/IoUtils.java | 11 ++++++----- .../src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index 2c687196d..d65761f0b 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -80,15 +80,17 @@ private static File downloadFile(URL url) throws IOException { File tempFile = File.createTempFile("pkl_", ".tmp"); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setInstanceFollowRedirects(true); // Enable automatic redirection - // connection.setRequestProperty("User-Agent", "Mozilla/5.0"); // Some servers might reject requests without a User-Agent header + // connection.setRequestProperty("User-Agent", "Mozilla/5.0"); // Some servers might reject + // requests without a User-Agent header int responseCode = connection.getResponseCode(); // Handle redirection manually - if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || responseCode == HttpURLConnection.HTTP_MOVED_TEMP) { + if (responseCode == HttpURLConnection.HTTP_MOVED_PERM + || responseCode == HttpURLConnection.HTTP_MOVED_TEMP) { String newUrl = connection.getHeaderField("Location"); connection = (HttpURLConnection) new URL(newUrl).openConnection(); - // connection.setRequestProperty("User-Agent", "Mozilla/5.0"); + // connection.setRequestProperty("User-Agent", "Mozilla/5.0"); responseCode = connection.getResponseCode(); } @@ -97,7 +99,7 @@ private static File downloadFile(URL url) throws IOException { } try (InputStream in = connection.getInputStream(); - FileOutputStream out = new FileOutputStream(tempFile)) { + FileOutputStream out = new FileOutputStream(tempFile)) { byte[] buffer = new byte[1024]; int bytesRead; while ((bytesRead = in.read(buffer)) != -1) { @@ -107,7 +109,6 @@ private static File downloadFile(URL url) throws IOException { return tempFile; } - /** Checks whether the given string is "URI-like", i.e. matches a pattern like {@code foo:bar}. */ public static boolean isUriLike(String str) { return uriLike.matcher(str).matches(); diff --git a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt index d412b9936..8f8bbc37d 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt @@ -15,6 +15,7 @@ */ package org.pkl.core.util +import java.io.File import java.io.FileNotFoundException import java.net.URI import java.net.URISyntaxException @@ -31,7 +32,6 @@ import org.pkl.core.SecurityManager import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.module.ModuleKeys import org.pkl.core.runtime.ModuleResolver -import java.io.File class IoUtilsTest { object FakeSecurityManager : SecurityManager { @@ -413,7 +413,7 @@ class IoUtilsTest { val downloadedContent = IoUtils.readString(resultUrl.openStream()) assertThat(downloadedContent).contains("Hello World!") } - + @Test fun `encodePath encodes characters reserved on windows`() { assertThat(IoUtils.encodePath("foo:bar")).isEqualTo("foo(3a)bar") From 00bbb38206c402b32f855c812f4a53570aca2b25 Mon Sep 17 00:00:00 2001 From: Carpe-Wang Date: Tue, 20 Aug 2024 09:19:44 -0400 Subject: [PATCH 3/4] refactor: will throw exception if Remote file URIs --- .../main/java/org/pkl/core/util/IoUtils.java | 48 ++----------------- .../kotlin/org/pkl/core/util/IoUtilsTest.kt | 15 ++---- 2 files changed, 9 insertions(+), 54 deletions(-) diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index d65761f0b..1a97eccd5 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -16,13 +16,10 @@ package org.pkl.core.util; import com.oracle.truffle.api.TruffleOptions; -import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; -import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -59,16 +56,10 @@ private IoUtils() {} public static URL toUrl(URI uri) throws IOException { try { if ("file".equalsIgnoreCase(uri.getScheme()) && uri.getHost() != null) { - // Dynamically determine the protocol, defaulting to http - String protocol = "http"; // Default protocol - URL remoteFileUrl = new URL(protocol, uri.getHost(), uri.getPort(), uri.getPath()); - File localFile = downloadFile(remoteFileUrl); - return localFile.toURI().toURL(); + throw new UnsupportedOperationException("Remote file URIs are not supported: " + uri); } return uri.toURL(); } catch (Error e) { - // best we can do for now - // rely on caller to provide context, e.g., the requested module URI if (e.getClass().getName().equals("com.oracle.svm.core.jdk.UnsupportedFeatureError")) { throw new IOException("Unsupported protocol: " + uri.getScheme()); } @@ -76,39 +67,6 @@ public static URL toUrl(URI uri) throws IOException { } } - private static File downloadFile(URL url) throws IOException { - File tempFile = File.createTempFile("pkl_", ".tmp"); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setInstanceFollowRedirects(true); // Enable automatic redirection - // connection.setRequestProperty("User-Agent", "Mozilla/5.0"); // Some servers might reject - // requests without a User-Agent header - - int responseCode = connection.getResponseCode(); - - // Handle redirection manually - if (responseCode == HttpURLConnection.HTTP_MOVED_PERM - || responseCode == HttpURLConnection.HTTP_MOVED_TEMP) { - String newUrl = connection.getHeaderField("Location"); - connection = (HttpURLConnection) new URL(newUrl).openConnection(); - // connection.setRequestProperty("User-Agent", "Mozilla/5.0"); - responseCode = connection.getResponseCode(); - } - - if (responseCode != HttpURLConnection.HTTP_OK) { - throw new IOException("Failed to download file: HTTP response code " + responseCode); - } - - try (InputStream in = connection.getInputStream(); - FileOutputStream out = new FileOutputStream(tempFile)) { - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = in.read(buffer)) != -1) { - out.write(buffer, 0, bytesRead); - } - } - return tempFile; - } - /** Checks whether the given string is "URI-like", i.e. matches a pattern like {@code foo:bar}. */ public static boolean isUriLike(String str) { return uriLike.matcher(str).matches(); @@ -168,6 +126,10 @@ public static String readString(InputStream inputStream) throws IOException { } public static byte[] readBytes(URI uri) throws IOException { + if ("file".equalsIgnoreCase(uri.getScheme()) && uri.getHost() != null) { + throw new UnsupportedOperationException("Remote file URIs are not supported: " + uri); + } + if (HttpUtils.isHttpUrl(uri)) { throw new IllegalArgumentException("Should use HTTP client to GET " + uri); } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt index 8f8bbc37d..70b383eb2 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt @@ -15,7 +15,6 @@ */ package org.pkl.core.util -import java.io.File import java.io.FileNotFoundException import java.net.URI import java.net.URISyntaxException @@ -402,16 +401,10 @@ class IoUtilsTest { } @Test - fun `toUrl() supports remote file URIs`() { - // Use a publicly available text file on GitHub as the test URI - val uri = URI("file://raw.githubusercontent.com/octocat/Hello-World/master/README") - val resultUrl = IoUtils.toUrl(uri) - - val tempFile = File(resultUrl.toURI()) - assertThat(tempFile.exists()).isTrue - - val downloadedContent = IoUtils.readString(resultUrl.openStream()) - assertThat(downloadedContent).contains("Hello World!") + fun `toUrl() throws UnsupportedOperationException for file URIs with remote hostnames`() { + val uri = URI("file://example.com/bar/baz.pkl") + val exception = assertThrows { IoUtils.toUrl(uri) } + assertThat(exception.message).contains("Remote file URIs are not supported") } @Test From ec39c8c72e70da757730e6531c405430b791d03c Mon Sep 17 00:00:00 2001 From: Carpe-Wang Date: Tue, 20 Aug 2024 20:18:56 -0400 Subject: [PATCH 4/4] refactor: will throw exception if Remote file URIs --- pkl-core/src/main/java/org/pkl/core/util/IoUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index 1a97eccd5..a79e0f169 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -60,6 +60,8 @@ public static URL toUrl(URI uri) throws IOException { } return uri.toURL(); } catch (Error e) { + // best we can do for now + // rely on caller to provide context, e.g., the requested module URI if (e.getClass().getName().equals("com.oracle.svm.core.jdk.UnsupportedFeatureError")) { throw new IOException("Unsupported protocol: " + uri.getScheme()); }