diff --git a/.gitignore b/.gitignore index 171de2920..59234fe4d 100755 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,4 @@ jacoco/ wheelhouse/ vc*.pdb *.class +/project/jpype_java_old/nbproject/private/ diff --git a/doc/CHANGELOG.rst b/doc/CHANGELOG.rst index 9ea7594c1..be81528ed 100644 --- a/doc/CHANGELOG.rst +++ b/doc/CHANGELOG.rst @@ -5,6 +5,11 @@ This changelog *only* contains changes from the *first* pypi release (0.5.4.3) o Latest Changes: - **1.5.2.dev0 - 2024-11-18** + + - Roll back agent change due to misbehaving JVM installs. + + - Correct issues with non-ascii path for jdbc connections and forName. + - **1.5.1 - 2024-11-09** - Future proofing for Python 3.14 diff --git a/jpype/_classpath.py b/jpype/_classpath.py index 2d8c919eb..160017005 100644 --- a/jpype/_classpath.py +++ b/jpype/_classpath.py @@ -52,7 +52,7 @@ def addClassPath(path1: typing.Union[str, _os.PathLike]) -> None: path1 = path2.joinpath(path1) # If the JVM is already started then we will have to load the paths - # immediately into the DynamicClassLoader + # immediately into the JPypeClassLoader if _jpype.isStarted(): Paths = _jpype.JClass('java.nio.file.Paths') JContext = _jpype.JClass('org.jpype.JPypeContext') @@ -62,9 +62,9 @@ def addClassPath(path1: typing.Union[str, _os.PathLike]) -> None: if len(paths) == 0: return for path in paths: - classLoader.addFile(Paths.get(str(path))) + classLoader.addPath(Paths.get(str(path))) else: - classLoader.addFile(Paths.get(str(path1))) + classLoader.addPath(Paths.get(str(path1))) _CLASSPATHS.append(path1) diff --git a/jpype/_core.py b/jpype/_core.py index 096c7a461..fc727645d 100644 --- a/jpype/_core.py +++ b/jpype/_core.py @@ -109,30 +109,29 @@ def isJVMStarted(): return _jpype.isStarted() -def _getOldClassPath(args) -> list[str]: - for i in args: - if not isinstance(i, str): +def _getOption(args, var, sep=None, keep=False): + """ Get an option and remove it from the current jvm arguments list """ + for i,v in enumerate(args): + if not isinstance(v, str): continue - _, _, classpath = i.partition('-Djava.class.path=') - if classpath: - return classpath.split(_classpath._SEP) + _, _, value = v.partition('%s='%var) + if value: + if not keep: + del args[i] + if sep is not None: + return value.split(sep) + return value return [] - -def _hasSystemClassLoader(args) -> bool: - for i in args: - if isinstance(i, str) and i.startswith('-Djava.system.class.loader'): - return True - return False - - -def _handleClassPath( +def _expandClassPath( classpath: typing.Union[typing.Sequence[_PathOrStr], _PathOrStr, None] = None -) -> typing.Sequence[str]: +) -> typing.List[str]: """ Return a classpath which represents the given tuple of classpath specifications """ out: list[str] = [] + if classpath is None: + return out if isinstance(classpath, (str, os.PathLike)): classpath = (classpath,) try: @@ -160,21 +159,45 @@ def _handleClassPath( return out -def _removeClassPath(args) -> tuple[str]: - return tuple(arg for arg in args if not str(arg).startswith("-Djava.class.path")) - - _JVM_started = False - def interactive(): return bool(getattr(sys, 'ps1', sys.flags.interactive)) +def _findTemp(): + dirlist = [] + # Mirror Python tempfile with a check for ascii + for envname in 'TMPDIR', 'TEMP', 'TMP': + dirname = os.getenv(envname) + if dirname and dirname.isascii(): + dirlist.append(dirname) + if os.name == 'nt': + for dirname in [ os.path.expanduser(r'~\AppData\Local\Temp'), + os.path.expandvars(r'%SYSTEMROOT%\Temp'), + r'c:\temp', r'c:\tmp', r'\temp', r'\tmp' ]: + if dirname and dirname.isascii(): + dirlist.append(dirname) + else: + dirlist.extend([ '/tmp', '/var/tmp', '/usr/tmp' ]) + + name = str(os.getpid()) + for d in dirlist: + p = Path("%s/%s"%(d,name)) + try: + p.touch() + p.unlink() + except Exception as ex: + continue + return d + raise SystemError("Unable to find non-ansii path") + def startJVM( *jvmargs: str, jvmpath: typing.Optional[_PathOrStr] = None, classpath: typing.Union[typing.Sequence[_PathOrStr], _PathOrStr, None] = None, + modulepath: typing.Union[typing.Sequence[_PathOrStr], _PathOrStr, None] = None, + modules: typing.Union[typing.Sequence[str], str, None] = None, ignoreUnrecognized: bool = False, convertStrings: bool = False, interrupt: bool = not interactive(), @@ -197,6 +220,9 @@ def startJVM( classpath (str, PathLike, [str, PathLike]): Set the classpath for the JVM. This will override any classpath supplied in the arguments list. A value of None will give no classpath to JVM. + modulepath (str, PathLike, [str, PathLike]): Set the modulepath for the JVM. + This will override any modulepath supplied in the arguments + list. A value of None will give no classpath to JVM. ignoreUnrecognized (bool): Option to ignore invalid JVM arguments. Default is False. convertStrings (bool): Option to force Java strings to @@ -216,23 +242,27 @@ def startJVM( TypeError: if a keyword argument conflicts with the positional arguments. """ + +# Code for 1.6 +# modulepath: typing.Union[typing.Sequence[_PathOrStr], _PathOrStr, None] = None, + if _jpype.isStarted(): raise OSError('JVM is already started') global _JVM_started if _JVM_started: raise OSError('JVM cannot be restarted') - has_classloader = _hasSystemClassLoader(jvmargs) - + # Convert to list + jvm_args: list[str] = list(jvmargs) # JVM path - if jvmargs: + if jvm_args: # jvm is the first argument the first argument is a path or None - if jvmargs[0] is None or (isinstance(jvmargs[0], str) and not jvmargs[0].startswith('-')): + if jvm_args[0] is None or (isinstance(jvm_args[0], str) and not jvm_args[0].startswith('-')): if jvmpath: raise TypeError('jvmpath specified twice') - jvmpath = jvmargs[0] - jvmargs = jvmargs[1:] + jvmpath = jvm_args[0] + jvm_args = jvm_args[1:] if not jvmpath: jvmpath = getDefaultJVMPath() @@ -240,8 +270,29 @@ def startJVM( # Allow the path to be a PathLike. jvmpath = os.fspath(jvmpath) + # Handle strings and list of strings. + extra_jvm_args: list[str] = [] + + # Get the support library + support_lib = os.path.join(os.path.dirname(os.path.dirname(__file__)), "org.jpype.jar") + if not os.path.exists(support_lib): + raise RuntimeError("Unable to find org.jpype.jar support library at " + support_lib) + + # If we are not installed on an ascii path then we will need to copy the jar to a new location + if not support_lib.isascii(): + import tempfile + import shutil + fd, path = tempfile.mkstemp(dir = _findTemp()) + if not path.isascii(): + raise ValueError("Unable to find ascii temp directory.") + shutil.copyfile(support_lib, path) + support_lib = path + tmp = path + os.close(fd) + # Don't remove + # Classpath handling - old_classpath = _getOldClassPath(jvmargs) + old_classpath = _getOption(jvm_args, "-Djava.class.path", _classpath._SEP) if old_classpath: # Old style, specified in the arguments if classpath is not None: @@ -252,41 +303,58 @@ def startJVM( # Not specified at all, use the default classpath. classpath = _classpath.getClassPath() - # Handle strings and list of strings. - extra_jvm_args: list[str] = [] + # Modulepath handling + old_modulepath = _getOption(jvm_args, "--module-path", _classpath._SEP) + if old_modulepath: + # Old style, specified in the arguments + if modulepath is not None: + # Cannot apply both styles, conflict + raise TypeError('modulepath specified twice') + modulepath = old_modulepath - supportLib = os.path.join(os.path.dirname(os.path.dirname(__file__)), "org.jpype.jar") - if not os.path.exists(supportLib): - raise RuntimeError("Unable to find org.jpype.jar support library at " + supportLib) - - late_load = not has_classloader - if classpath: - cp = _classpath._SEP.join(_handleClassPath(classpath)) - if cp.isascii(): - # no problems - extra_jvm_args += ['-Djava.class.path=%s'%cp ] - jvmargs = _removeClassPath(jvmargs) - late_load = False - elif has_classloader: + # Modules + old_modules = _getOption(jvm_args, "--add-modules", ',') + if old_modules: + # Old style, specified in the arguments + if modules is not None: + # Cannot apply both styles, conflict + raise TypeError('modules specified twice') + modules = old_modules + if modules is None: + modules = [] + modules.append("org.jpype") + + system_class_loader = _getOption(jvm_args, "-Djava.system.class.loader", keep=True) + + java_class_path = _expandClassPath(classpath) + java_class_path = list(filter(len, java_class_path)) + classpath = _classpath._SEP.join(java_class_path) + tmp = None + + # Make sure our module is always on the classpath + if not classpath.isascii(): + if system_class_loader: # https://bugs.openjdk.org/browse/JDK-8079633?jql=text%20~%20%22ParseUtil%22 raise ValueError("system classloader cannot be specified with non ascii characters in the classpath") - elif supportLib.isascii(): - # ok, setup the jpype system classloader and add to the path after startup - # this guarentees all classes have the same permissions as they did in the past - extra_jvm_args += [ - '-Djava.system.class.loader=org.jpype.classloader.JpypeSystemClassLoader', - '-Djava.class.path=%s'%supportLib - ] - jvmargs = _removeClassPath(jvmargs) - else: - # We are screwed no matter what we try or do. - # Unfortunately the jdk maintainers don't seem to care either. - # This bug is almost 10 years old and spans 16 jdk versions and counting. - # https://bugs.openjdk.org/browse/JDK-8079633?jql=text%20~%20%22ParseUtil%22 - raise ValueError("jpype jar must be ascii to add to the system class path") + # ok, setup the jpype system classloader and add to the path after startup + # this guarentees all classes have the same permissions as they did in the past + from urllib.parse import quote + extra_jvm_args += [ + '-Djava.system.class.loader=org.jpype.JPypeClassLoader', + '-Djava.class.path=%s'%support_lib, + '-Djpype.class.path=%s'%quote(classpath), + '-Xshare:off' + ] + else: + # no problems + if classpath: + extra_jvm_args += ['-Djava.class.path=%s'%classpath ] - extra_jvm_args += ['-javaagent:' + supportLib] + mp =_expandClassPath(modulepath) + mp.append(support_lib) + extra_jvm_args += ['--module-path=%s'%_classpath._SEP.join(mp) ] + extra_jvm_args += ['--add-modules=%s'%(",".join(modules))] try: import locale @@ -295,8 +363,8 @@ def startJVM( # Keep the current locale settings, else Java will replace them. prior = [locale.getlocale(i) for i in categories] # Start the JVM - _jpype.startup(jvmpath, jvmargs + tuple(extra_jvm_args), - ignoreUnrecognized, convertStrings, interrupt) + _jpype.startup(jvmpath, tuple(jvm_args + extra_jvm_args), + ignoreUnrecognized, convertStrings, interrupt, tmp) # Collect required resources for operation initializeResources() # Restore locale @@ -315,23 +383,6 @@ def startJVM( raise RuntimeError(f"{jvmpath} is older than required Java version{version}") from ex raise - """Prior versions of JPype used the jvmargs to setup the class paths via - JNI (Java Native Interface) option strings: - i.e -Djava.class.path=... - See: https://docs.oracle.com/javase/7/docs/technotes/guides/jni/spec/invocation.html - - Unfortunately, only ascii paths work because url encoding is not handled correctly - see: https://bugs.openjdk.org/browse/JDK-8079633?jql=text%20~%20%22ParseUtil%22 - - To resolve this issue, we add the classpath after initialization since Java has - had the utilities to correctly encode it since 1.0 - """ - if late_load and classpath: - # now we can add to the system classpath - cl = _jpype.JClass("java.lang.ClassLoader").getSystemClassLoader() - for cp in _handleClassPath(classpath): - cl.addPath(_jpype._java_lang_String(cp)) - def initializeResources(): global _JVM_started diff --git a/jpype/dbapi2.py b/jpype/dbapi2.py index e391e3af8..0b39ad97c 100644 --- a/jpype/dbapi2.py +++ b/jpype/dbapi2.py @@ -401,8 +401,9 @@ def connect(dsn, *, driver=None, driver_args=None, """ Properties = _jpype.JClass("java.util.Properties") if driver: - _jpype.JClass('java.lang.Class').forName(driver).newInstance() + _jpype.JClass('java.lang.Class').forName(driver,True,_jpype.JPypeClassLoader).newInstance() DM = _jpype.JClass('java.sql.DriverManager') + #DM.setLogStream(_jpype.JClass("java.lang.System").out) # User is supplying Java properties if isinstance(driver_args, Properties): diff --git a/native/common/include/jp_context.h b/native/common/include/jp_context.h index 0b357f786..bc3a2242b 100644 --- a/native/common/include/jp_context.h +++ b/native/common/include/jp_context.h @@ -223,6 +223,7 @@ class JPContext private: JPClassRef m_Array; + JPObjectRef m_Reflector; // Java Functions jmethodID m_Object_ToStringID{}; diff --git a/native/common/include/jp_gc.h b/native/common/include/jp_gc.h index cad182122..a50c83189 100644 --- a/native/common/include/jp_gc.h +++ b/native/common/include/jp_gc.h @@ -63,7 +63,6 @@ class JPGarbageCollection jmethodID _freeMemoryID; jmethodID _maxMemoryID; jmethodID _usedMemoryID; - jmethodID _heapMemoryID; size_t last_python; size_t last_java; diff --git a/native/common/jp_classloader.cpp b/native/common/jp_classloader.cpp index 0dbfa750d..cac86282a 100644 --- a/native/common/jp_classloader.cpp +++ b/native/common/jp_classloader.cpp @@ -38,9 +38,16 @@ JPClassLoader::JPClassLoader(JPJavaFrame& frame) m_SystemClassLoader = JPObjectRef(frame, frame.CallStaticObjectMethodA(classLoaderClass, getSystemClassLoader, nullptr)); - jclass dynamicLoaderClass = frame.getEnv()->FindClass("org/jpype/classloader/DynamicClassLoader"); + jclass dynamicLoaderClass = frame.getEnv()->FindClass("org/jpype/JPypeClassLoader"); if (dynamicLoaderClass != nullptr) { + // Use the one in place already + if (frame.IsInstanceOf(m_SystemClassLoader.get(), dynamicLoaderClass)) + { + m_BootLoader = m_SystemClassLoader; + return; + } + // Easy the Dynamic loader is already in the path, so just use it as the bootloader jmethodID newDyLoader = frame.GetMethodID(dynamicLoaderClass, "", "(Ljava/lang/ClassLoader;)V"); diff --git a/native/common/jp_context.cpp b/native/common/jp_context.cpp index d6e25cb2f..eaf6e44c6 100644 --- a/native/common/jp_context.cpp +++ b/native/common/jp_context.cpp @@ -22,6 +22,18 @@ #include "jp_platform.h" #include "jp_gc.h" +#ifdef WIN32 +#include +#else +#if defined(_HPUX) && !defined(_IA64) +#include +#else +#include +#endif // HPUX +#include +#endif + + JPResource::~JPResource() = default; @@ -159,6 +171,35 @@ void JPContext::attachJVM(JNIEnv* env) initializeResources(env, false); } +std::string getShared() +{ +#ifdef WIN32 + // Windows specific + char path[MAX_PATH]; + HMODULE hm = NULL; + if (GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCSTR) &getShared, &hm) != 0 && + GetModuleFileName(hm, path, sizeof(path)) != 0) + { + // This is needed when there is no-ascii characters in path + char shortPathBuffer[MAX_PATH]; + GetShortPathName(path, shortPathBuffer, MAX_PATH); + return shortPathBuffer; + } +#else + // Linux specific + Dl_info info; + if (dladdr((void*)getShared, &info)) + return info.dli_fname; +#endif + // Generic + JPPyObject import = JPPyObject::use(PyImport_AddModule("importlib.util")); + JPPyObject jpype = JPPyObject::call(PyObject_CallMethod(import.get(), "find_spec", "s", "_jpype")); + JPPyObject origin = JPPyObject::call(PyObject_GetAttrString(jpype.get(), "origin")); + return JPPyString::asStringUTF8(origin.get()); +} + void JPContext::initializeResources(JNIEnv* env, bool interrupt) { JPJavaFrame frame = JPJavaFrame::external(this, env); @@ -215,10 +256,8 @@ void JPContext::initializeResources(JNIEnv* env, bool interrupt) if (!m_Embedded) { - JPPyObject import = JPPyObject::use(PyImport_AddModule("importlib.util")); - JPPyObject jpype = JPPyObject::call(PyObject_CallMethod(import.get(), "find_spec", "s", "_jpype")); - JPPyObject origin = JPPyObject::call(PyObject_GetAttrString(jpype.get(), "origin")); - val[2].l = frame.fromStringUTF8(JPPyString::asStringUTF8(origin.get())); + std::string shared = getShared(); + val[2].l = frame.fromStringUTF8(shared); } // Required before launch @@ -238,7 +277,10 @@ void JPContext::initializeResources(JNIEnv* env, bool interrupt) // Set up methods after everything is start so we get better error // messages - m_CallMethodID = frame.GetMethodID(contextClass, "callMethod", + jclass reflectorClass = frame.FindClass("org/jpype/JPypeReflector"); + jfieldID reflectorField = frame.GetFieldID(contextClass, "reflector", "Lorg/jpype/JPypeReflector;"); + m_Reflector = JPObjectRef(frame, frame.GetObjectField(m_JavaContext.get(), reflectorField)); + m_CallMethodID = frame.GetMethodID(reflectorClass, "callMethod", "(Ljava/lang/reflect/Method;Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;"); m_Context_collectRectangularID = frame.GetMethodID(contextClass, "collectRectangular", diff --git a/native/common/jp_gc.cpp b/native/common/jp_gc.cpp index 6d2a47ce0..d2b6e04d0 100644 --- a/native/common/jp_gc.cpp +++ b/native/common/jp_gc.cpp @@ -160,7 +160,6 @@ void JPGarbageCollection::init(JPJavaFrame& frame) _freeMemoryID = frame.GetStaticMethodID(ctxt, "getFreeMemory", "()J"); _maxMemoryID = frame.GetStaticMethodID(ctxt, "getMaxMemory", "()J"); _usedMemoryID = frame.GetStaticMethodID(ctxt, "getUsedMemory", "()J"); - _heapMemoryID = frame.GetStaticMethodID(ctxt, "getHeapMemory", "()J"); running = true; high_water = getWorkingSize(); @@ -257,7 +256,6 @@ void JPGarbageCollection::onEnd() jlong freeMemory = frame.CallStaticLongMethodA(_ContextClass, _freeMemoryID, nullptr); jlong maxMemory = frame.CallStaticLongMethodA(_ContextClass, _maxMemoryID, nullptr); jlong usedMemory = frame.CallStaticLongMethodA(_ContextClass, _usedMemoryID, nullptr); - jlong heapMemory = frame.CallStaticLongMethodA(_ContextClass, _heapMemoryID, nullptr); printf("consider gc run=%d (current=%ld, low=%ld, high=%ld, limit=%ld) %ld\n", run_gc, current, low_water, high_water, limit, limit - pred); printf(" java total=%ld free=%ld max=%ld used=%ld heap=%ld\n", totalMemory, freeMemory, maxMemory, usedMemory, heapMemory); diff --git a/native/common/jp_javaframe.cpp b/native/common/jp_javaframe.cpp index 62bfd2626..136b69ccc 100644 --- a/native/common/jp_javaframe.cpp +++ b/native/common/jp_javaframe.cpp @@ -1162,7 +1162,7 @@ jobject JPJavaFrame::callMethod(jobject method, jobject obj, jobject args) v[0].l = method; v[1].l = obj; v[2].l = args; - return frame.keep(frame.CallObjectMethodA(m_Context->m_JavaContext.get(), m_Context->m_CallMethodID, v)); + return frame.keep(frame.CallObjectMethodA(m_Context->m_Reflector.get(), m_Context->m_CallMethodID, v)); JP_TRACE_OUT; } diff --git a/native/java/manifest.txt b/native/java/manifest.txt index 0b556c29d..fa49425d5 100644 --- a/native/java/manifest.txt +++ b/native/java/manifest.txt @@ -1,3 +1,4 @@ Manifest-Version: 1.0 -Premain-Class: org.jpype.agent.JPypeAgent - +Specification-Title: org.jpype +Implementation-Title: org.jpype +Implementation-Version: 1.5.1 diff --git a/native/java/org/jpype/classloader/JpypeSystemClassLoader.java b/native/java/module-info.java similarity index 50% rename from native/java/org/jpype/classloader/JpypeSystemClassLoader.java rename to native/java/module-info.java index efa1853ff..a8e2535f0 100644 --- a/native/java/org/jpype/classloader/JpypeSystemClassLoader.java +++ b/native/java/module-info.java @@ -13,31 +13,16 @@ See NOTICE file for details. **************************************************************************** */ -package org.jpype.classloader; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.Paths; - -public final class JpypeSystemClassLoader extends URLClassLoader { - - public JpypeSystemClassLoader(ClassLoader parent) throws Throwable { - super(new URL[0], parent); - } - - public void addPath(String path) throws Throwable { - addURL(Paths.get(path).toAbsolutePath().toUri().toURL()); - } - - public void addPaths(String[] paths) throws Throwable { - for (String path : paths) { - addPath(path); - } - } - - // this is required to add a Java agent even if it is already in the path - @SuppressWarnings("unused") - private void appendToClassPathForInstrumentation(String path) throws Throwable { - addPath(path); - } +module org.jpype { + requires java.sql; + + exports org.jpype; + exports org.jpype.manager; + exports org.jpype.html; + exports org.jpype.javadoc; + exports org.jpype.pickle; + exports org.jpype.pkg; + exports org.jpype.proxy; + exports org.jpype.ref; } diff --git a/native/java/org/jpype/classloader/DynamicClassLoader.java b/native/java/org/jpype/JPypeClassLoader.java similarity index 52% rename from native/java/org/jpype/classloader/DynamicClassLoader.java rename to native/java/org/jpype/JPypeClassLoader.java index 2192391e9..9e0eb6645 100644 --- a/native/java/org/jpype/classloader/DynamicClassLoader.java +++ b/native/java/org/jpype/JPypeClassLoader.java @@ -1,11 +1,29 @@ -package org.jpype.classloader; +/* **************************************************************************** + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + See NOTICE file for details. +**************************************************************************** */ +package org.jpype; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; import java.net.URLConnection; @@ -14,32 +32,104 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; +import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarFile; +import java.net.URLDecoder; -public class DynamicClassLoader extends ClassLoader +/** + * Class loader for JPype. + * + * This is augmented to manage directory resources, allow for late loading, + * and handling of resources on non-ASCII paths. + */ +public class JPypeClassLoader extends URLClassLoader { - List loaders = new LinkedList<>(); HashMap> map = new HashMap<>(); + int code = 0; - public DynamicClassLoader(ClassLoader parent) + public JPypeClassLoader(ClassLoader parent) { - super(parent); + super(initial(), parent); } + /** + * Used to keep the cache up to date. + * + * @return + */ public int getCode() { - return loaders.hashCode(); + return code; + } + + /** + * Special routine for handling non-ascii paths. + * + * If we are loaded as the system ClassLoader, then we will use + * "jpype.class.path" rather than "java.class.path" during the load process. + * We will move it into the expected place after so no one is the wiser. + * + * @return + */ + private static URL[] initial() + { + // Check to see if we have a late loaded path + String cp = System.getProperty("jpype.class.path"); + if (cp == null) + return new URL[0]; + + try + { + cp = URLDecoder.decode(cp, "UTF-8"); + } catch (UnsupportedEncodingException ex) + { + // ignored + } + + ArrayList path = new ArrayList<>(); + int last = 0; + int next = 0; + + while (next != -1) + { + // Find the parts + next = cp.indexOf(File.pathSeparator, last); + String element = (next == -1) ? cp.substring(last) : cp.substring(last, next); + if (!element.isEmpty()) + { + try + { + URL url = Paths.get(element).toUri().toURL(); + if (url != null) + path.add(url); + } catch (MalformedURLException ex) + { + System.err.println("Malformed url in classpath skipped " + element); + } + } + last = next + 1; + } + + // Replace the path + System.clearProperty("jpype.class.path"); + System.setProperty("java.class.path", cp); + return path.toArray(new URL[0]); + } + + // This is required to add a Java agent even if it is already in the path + @SuppressWarnings("unused") + private void appendToClassPathForInstrumentation(String path) throws Throwable + { + addURL(Paths.get(path).toAbsolutePath().toUri().toURL()); } /** @@ -49,11 +139,10 @@ public int getCode() * @param glob * @throws IOException */ - public void addFiles(Path root, String glob) throws IOException + public void addPaths(Path root, String glob) throws IOException { final PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher(glob); - List urls = new LinkedList<>(); Files.walkFileTree(root, new SimpleFileVisitor() { @@ -64,7 +153,7 @@ public FileVisitResult visitFile(Path path, if (pathMatcher.matches(root.relativize(path))) { URL url = path.toUri().toURL(); - urls.add(url); + addURL(url); } return FileVisitResult.CONTINUE; } @@ -77,23 +166,21 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) } }); - loaders.add(new URLClassLoader(urls.toArray(new URL[urls.size()]))); } - public void addFile(Path path) throws FileNotFoundException + /** + * Add a path to the loader after the JVM is started. + * + * @param path + * @throws FileNotFoundException + */ + public void addPath(Path path) throws FileNotFoundException { try { if (!Files.exists(path)) throw new FileNotFoundException(path.toString()); - URL[] urls = new URL[] - { - path.toUri().toURL() - }; - loaders.add(new URLClassLoader(urls)); - - // Scan the file for directory entries - this.scanJar(path); + this.addURL(path.toUri().toURL()); } catch (MalformedURLException ex) { // This should never happen @@ -120,7 +207,7 @@ public Class findClass(String name) throws ClassNotFoundException, ClassFormatEr try { URLConnection connection = url.openConnection(); - try ( InputStream is = connection.getInputStream()) + try (InputStream is = connection.getInputStream()) { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); int bytes; @@ -132,7 +219,7 @@ public Class findClass(String name) throws ClassNotFoundException, ClassFormatEr buffer.flush(); byte[] data = buffer.toByteArray(); - return defineClass(name, data, 0, data.length); + return defineClass(null, data, 0, data.length); } } catch (IOException ex) { @@ -141,36 +228,30 @@ public Class findClass(String name) throws ClassNotFoundException, ClassFormatEr } @Override - public URL getResource(String name) + public URL findResource(String name) { - URL url = this.getParent().getResource(name); + // Check local first + URL url = super.findResource(name); if (url != null) return url; - for (ClassLoader cl : this.loaders) - { - url = cl.getResource(name); - if (url != null) - return url; - } + // Both with and without / should generate the same result if (name.endsWith("/")) name = name.substring(0, name.length() - 1); if (map.containsKey(name)) return map.get(name).get(0); + + // We have some resource which must be sourced to a particular class loader + if (name.startsWith("org/jpype/")) + return getResource("META-INF/versions/0/" + name); return null; } @Override - public Enumeration getResources(String name) throws IOException + public Enumeration findResources(String name) throws IOException { ArrayList out = new ArrayList<>(); - Enumeration urls = getParent().getResources(name); - out.addAll(Collections.list(urls)); - for (URLClassLoader cl : this.loaders) - { - urls = cl.findResources(name); - out.addAll(Collections.list(urls)); - } + out.addAll(Collections.list(super.findResources(name))); // Both with and without / should generate the same result if (name.endsWith("/")) name = name.substring(0, name.length() - 1); @@ -179,6 +260,15 @@ public Enumeration getResources(String name) throws IOException return Collections.enumeration(out); } + /** + * Add a resource to the search. + * + * Many jar files lack directory support which is needed for the packaging + * import. + * + * @param name + * @param url + */ public void addResource(String name, URL url) { if (!this.map.containsKey(name)) @@ -186,6 +276,29 @@ public void addResource(String name, URL url) this.map.get(name).add(url); } + @Override + public void addURL(URL url) + { + // Mark our cache as dirty + code = code * 98745623 + url.hashCode(); + + // add to the search tree + super.addURL(url); + + // See if it is a path + Path path; + try + { + path = Paths.get(url.toURI()); + } catch (URISyntaxException ex) + { + return; + } + + // Scan for missing resources + scanJar(path); + } + /** * Recreate missing directory entries for Jars that lack indexing. * @@ -193,18 +306,19 @@ public void addResource(String name, URL url) * properly importing their contents. This procedure scans a jar file when * loaded to build missing directories. * - * @param p1 + * @param path */ - public void scanJar(Path p1) + void scanJar(Path path) { - if (!Files.exists(p1)) + if (!Files.exists(path)) return; - if (Files.isDirectory(p1)) + if (Files.isDirectory(path)) return; - try ( JarFile jf = new JarFile(p1.toFile())) + + try (JarFile jf = new JarFile(path.toFile())) { Enumeration entries = jf.entries(); - URI abs = p1.toAbsolutePath().toUri(); + URI abs = path.toAbsolutePath().toUri(); Set urls = new java.util.HashSet(); while (entries.hasMoreElements()) { diff --git a/native/java/org/jpype/JPypeContext.java b/native/java/org/jpype/JPypeContext.java index c997d3759..878b4957d 100644 --- a/native/java/org/jpype/JPypeContext.java +++ b/native/java/org/jpype/JPypeContext.java @@ -19,18 +19,14 @@ import java.lang.reflect.Array; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.lang.reflect.Modifier; import java.nio.Buffer; import java.nio.ByteOrder; -import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import org.jpype.classloader.DynamicClassLoader; import org.jpype.manager.TypeFactory; import org.jpype.manager.TypeFactoryNative; import org.jpype.manager.TypeManager; @@ -75,16 +71,17 @@ public class JPypeContext public final String VERSION = "1.5.2.dev0"; - private static JPypeContext INSTANCE = new JPypeContext(); + private static final JPypeContext INSTANCE = new JPypeContext(); // This is the C++ portion of the context. private long context; private TypeFactory typeFactory; private TypeManager typeManager; - private DynamicClassLoader classLoader; + private JPypeClassLoader classLoader; private final AtomicInteger shutdownFlag = new AtomicInteger(); private final List shutdownHooks = new ArrayList<>(); private final List postHooks = new ArrayList<>(); public static boolean freeResources = true; + public JPypeReflector reflector = null; static public JPypeContext getInstance() { @@ -95,30 +92,49 @@ static public JPypeContext getInstance() * Start the JPype system. * * @param context is the C++ portion of the context. - * @param bootLoader is the classloader holding JPype resources. + * @param loader is the classloader holding JPype resources. * @return the created context. */ - static JPypeContext createContext(long context, ClassLoader bootLoader, String nativeLib, boolean interrupt) + private static JPypeContext createContext(long context, ClassLoader loader, String nativeLib, boolean interrupt) throws Throwable { - if (nativeLib != null) + try { - System.load(nativeLib); - } - INSTANCE.context = context; - INSTANCE.classLoader = (DynamicClassLoader) bootLoader; - INSTANCE.typeFactory = new TypeFactoryNative(); - INSTANCE.typeManager = new TypeManager(context, INSTANCE.typeFactory); - INSTANCE.initialize(interrupt); + if (nativeLib != null) + { + System.load(nativeLib); + } + INSTANCE.context = context; + INSTANCE.classLoader = (JPypeClassLoader) loader; + INSTANCE.typeFactory = new TypeFactoryNative(); + INSTANCE.typeManager = new TypeManager(context, INSTANCE.typeFactory); + INSTANCE.initialize(interrupt); - scanExistingJars(); - return INSTANCE; + try + { + INSTANCE.reflector = (JPypeReflector) Class.forName("org.jpype.Reflector0", true, loader) + .getConstructor() + .newInstance(); + } catch (ClassNotFoundException | NoSuchMethodException | SecurityException + | InstantiationException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException ex) + { + throw new RuntimeException("Unable to create reflector "+ ex.getMessage(), ex); + } + + scanExistingJars(); + return INSTANCE; + } catch (Throwable ex) + { + ex.printStackTrace(System.err); + throw ex; + } } private JPypeContext() { } - void initialize(boolean interrupt) + private void initialize(boolean interrupt) { // Okay everything is setup so lets give it a go. this.typeManager.init(); @@ -244,21 +260,21 @@ private void shutdown() if (freeResources) { - // Release all Python references - try - { - JPypeReferenceQueue.getInstance().stop(); - } catch (Throwable th) - { - } - - // Release any C++ resources - try - { - this.typeManager.shutdown(); - } catch (Throwable th) - { - } + // Release all Python references + try + { + JPypeReferenceQueue.getInstance().stop(); + } catch (Throwable th) + { + } + + // Release any C++ resources + try + { + this.typeManager.shutdown(); + } catch (Throwable th) + { + } } // Execute post hooks @@ -269,7 +285,7 @@ private void shutdown() } - static native void onShutdown(long ctxt); + private static native void onShutdown(long ctxt); public void addShutdownHook(Thread th) { @@ -323,31 +339,6 @@ public void _addPost(Runnable run) this.postHooks.add(run); } - /** - * Call a method using reflection.This method creates a stackframe so that - * caller sensitive methods will execute properly. - * - * - * @param method is the method to call. - * @param obj is the object to operate on, it will be null if the method is - * static. - * @param args the arguments to method. - * @return the object that results form the invocation. - * @throws java.lang.Throwable throws whatever type the called method - * produces. - */ - public Object callMethod(Method method, Object obj, Object[] args) - throws Throwable - { - try - { - return method.invoke(obj, args); - } catch (InvocationTargetException ex) - { - throw ex.getCause(); - } - } - /** * Helper function for collect rectangular, */ @@ -442,7 +433,7 @@ private Object unpack(int size, Object parts) return a1; } - public Object assemble(int[] dims, Object parts) + private Object assemble(int[] dims, Object parts) { int n = dims.length; if (n == 1) @@ -461,15 +452,6 @@ public boolean isShutdown() return shutdownFlag.get() > 0; } -// public void incrementProxy() -// { -// proxyCount.incrementAndGet(); -// } -// -// public void decrementProxy() -// { -// proxyCount.decrementAndGet(); -// } /** * Clear the current interrupt. * @@ -522,12 +504,12 @@ public long getExcValue(Throwable th) return 0; } - public Exception createException(long l0, long l1) + private Exception createException(long l0, long l1) { return new PyExceptionProxy(l0, l1); } - public boolean order(Buffer b) + private boolean order(Buffer b) { if (b instanceof java.nio.ByteBuffer) return ((java.nio.ByteBuffer) b).order() == ByteOrder.LITTLE_ENDIAN; @@ -637,29 +619,25 @@ private static void scanExistingJars() } } - private static long getTotalMemory() + private static long getTotalMemory() { return Runtime.getRuntime().totalMemory(); } - private static long getFreeMemory() + private static long getFreeMemory() { return Runtime.getRuntime().freeMemory(); } - private static long getMaxMemory() + private static long getMaxMemory() { return Runtime.getRuntime().maxMemory(); } - private static long getUsedMemory() + private static long getUsedMemory() { return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); } - private static long getHeapMemory() - { - java.lang.management.MemoryMXBean memoryBean = java.lang.management.ManagementFactory.getMemoryMXBean(); - return memoryBean.getHeapMemoryUsage().getUsed(); - } + } diff --git a/native/java/org/jpype/JPypeReflector.java b/native/java/org/jpype/JPypeReflector.java new file mode 100644 index 000000000..f92d6de0e --- /dev/null +++ b/native/java/org/jpype/JPypeReflector.java @@ -0,0 +1,27 @@ +package org.jpype; + +import java.lang.reflect.Method; + +/** + * + * @author nelson85 + */ +public interface JPypeReflector +{ + /** + * Call a method using reflection. + * + * This method creates a stackframe so that caller sensitive methods will + * execute properly. + * + * @param method is the method to call. + * @param obj is the object to operate on, it will be null if the method is + * static. + * @param args the arguments to method. + * @return the object that results form the invocation. + * @throws java.lang.Throwable throws whatever type the called method + * produces. + */ + public Object callMethod(Method method, Object obj, Object[] args) + throws Throwable; +} diff --git a/native/java/org/jpype/agent/JPypeAgent.java b/native/java/org/jpype/agent/JPypeAgent.java deleted file mode 100644 index c15ac7334..000000000 --- a/native/java/org/jpype/agent/JPypeAgent.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.jpype.agent; - -import java.lang.instrument.Instrumentation; - -public class JPypeAgent -{ - public static void premain(String agentArgs, Instrumentation inst) { - // This doesn't have to do anything. - // We just need to be an agent to load elevated privileges - } -} diff --git a/native/java/org/jpype/classloader/JPypeClassLoader.java b/native/java/org/jpype/classloader/JPypeClassLoader.java deleted file mode 100644 index 21e5268bb..000000000 --- a/native/java/org/jpype/classloader/JPypeClassLoader.java +++ /dev/null @@ -1,149 +0,0 @@ -/* **************************************************************************** - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - See NOTICE file for details. -**************************************************************************** */ -package org.jpype.classloader; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.TreeMap; -import java.util.jar.JarEntry; -import java.util.jar.JarInputStream; - -/** - * Specialized class loader for JPype resources. - *

- * Loader to convert the internally stored resources into java classes. This - * prevents class load order problems when there are class dependencies. - *

- */ -public class JPypeClassLoader extends ClassLoader -{ - - static private JPypeClassLoader instance; - private TreeMap map = new TreeMap<>(); - - /** - * Get the class loader. - * - * @return the singleton class loader. - */ - public static JPypeClassLoader getInstance() - { - if (instance == null) - { - JPypeClassLoader.instance = new JPypeClassLoader(getSystemClassLoader()); - } - return instance; - } - - private JPypeClassLoader(ClassLoader parent) - { - super(parent); - } - - /** - * Add a class to the class loader. - *

- * This can be called from within python to add a class to the Java JVM. - * - * @param name is the name of the class. - * @param code is the byte code. - */ - public void importClass(String name, byte[] code) - { - map.put(name, code); - } - - /** - * Import a jar from memory into the class loader. - *

- * Does not handle unknown jar entry lengths. - * - * @param bytes - */ - public void importJar(byte[] bytes) - { - try (JarInputStream is = new JarInputStream(new ByteArrayInputStream(bytes))) - { - while (true) - { - JarEntry nextEntry = is.getNextJarEntry(); - if (nextEntry == null) - break; - - // Skip directories and other non-class resources - long size = nextEntry.getSize(); - if (size == 0) - continue; - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int q; - while ((q = is.read()) != -1) - baos.write(q); - byte[] data = baos.toByteArray(); - - // Store all classes we find - String name = nextEntry.getName(); - importClass(name, data); - } - } catch (IOException ex) - { - throw new RuntimeException(ex); - } - } - - /** - * Loads a class from the class loader. - * - * @param name is the name of the class with java class notation (using dots). - * @return the class - * @throws ClassNotFoundException was not found by the class loader. - * @throws ClassFormatError if the class byte code was invalid. - */ - @Override - public Class findClass(String name) throws ClassNotFoundException, ClassFormatError - { - String mname = name.replace('.', '/') + ".class"; - byte[] data = map.get(mname); - if (data == null) - { - // Call the default implementation, throws ClassNotFoundException - return super.findClass(name); - } - - Class cls = defineClass(name, data, 0, data.length); - if (cls == null) - throw new ClassFormatError("Class load was null"); - return cls; - } - - /** - * Overload for thunk resources. - * - * @param s - * @return - */ - @Override - public InputStream getResourceAsStream(String s) - { - if (this.map.containsKey(s)) - { - return new ByteArrayInputStream(this.map.get(s)); - } - return super.getResourceAsStream(s); - } -} diff --git a/native/java/org/jpype/html/Html.java b/native/java/org/jpype/html/Html.java index 35739ee48..ad48fba95 100644 --- a/native/java/org/jpype/html/Html.java +++ b/native/java/org/jpype/html/Html.java @@ -25,7 +25,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import org.jpype.JPypeContext; import org.w3c.dom.Attr; import org.w3c.dom.Document; @@ -72,8 +71,8 @@ public static List parseAttributes(Document doc, String str) static { - try (InputStream is = JPypeContext.getInstance().getClass().getClassLoader() - .getResourceAsStream("org/jpype/html/entities.txt"); + ClassLoader cl = ClassLoader.getSystemClassLoader(); + try (InputStream is = cl.getResourceAsStream("org/jpype/html/entities.txt"); InputStreamReader isr = new InputStreamReader(is); BufferedReader rd = new BufferedReader(isr)) { diff --git a/native/java/org/jpype/pkg/JPypePackage.java b/native/java/org/jpype/pkg/JPypePackage.java index 1541ccea4..1321b40d9 100644 --- a/native/java/org/jpype/pkg/JPypePackage.java +++ b/native/java/org/jpype/pkg/JPypePackage.java @@ -25,9 +25,9 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Map; +import org.jpype.JPypeClassLoader; import org.jpype.JPypeContext; import org.jpype.JPypeKeywords; -import org.jpype.classloader.DynamicClassLoader; /** * Representation of a JPackage in Java. @@ -45,13 +45,13 @@ public class JPypePackage // A mapping from Python names into Paths into the module/jar file system. Map contents; int code; - private final DynamicClassLoader classLoader; + private final JPypeClassLoader classLoader; public JPypePackage(String pkg) { this.pkg = pkg; this.contents = JPypePackageManager.getContentMap(pkg); - this.classLoader = ((DynamicClassLoader)(JPypeContext.getInstance().getClassLoader())); + this.classLoader = ((JPypeClassLoader)(JPypeContext.getInstance().getClassLoader())); this.code = classLoader.getCode(); } diff --git a/native/java0/org/jpype/Reflector0.java b/native/java0/org/jpype/Reflector0.java new file mode 100644 index 000000000..f4ff1891f --- /dev/null +++ b/native/java0/org/jpype/Reflector0.java @@ -0,0 +1,38 @@ +package org.jpype; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +public class Reflector0 implements JPypeReflector +{ + + public Reflector0() + {} + + /** + * Call a method using reflection. + * + * This method creates a stackframe so that caller sensitive methods will + * execute properly. + * + * @param method is the method to call. + * @param obj is the object to operate on, it will be null if the method is + * static. + * @param args the arguments to method. + * @return the object that results form the invocation. + * @throws java.lang.Throwable throws whatever type the called method + * produces. + */ + public Object callMethod(Method method, Object obj, Object[] args) + throws Throwable + { + try + { + return method.invoke(obj, args); + } catch (InvocationTargetException ex) + { +// ex.printStackTrace(); + throw ex.getCause(); + } + } +} diff --git a/native/python/pyjp_module.cpp b/native/python/pyjp_module.cpp index 338e92d24..2999689c1 100644 --- a/native/python/pyjp_module.cpp +++ b/native/python/pyjp_module.cpp @@ -20,6 +20,9 @@ #include "jp_gc.h" #include "jp_stringtype.h" #include "jp_classloader.h" +#ifdef WIN32 +#include +#endif void PyJPModule_installGC(PyObject* module); @@ -229,6 +232,7 @@ int PyJP_IsInstanceSingle(PyObject* obj, PyTypeObject* type) #ifndef ANDROID extern JNIEnv *Android_JNI_GetEnv(); +static string jarTmpPath; static PyObject* PyJPModule_startup(PyObject* module, PyObject* pyargs) { JP_PY_TRY("PyJPModule_startup"); @@ -238,11 +242,23 @@ static PyObject* PyJPModule_startup(PyObject* module, PyObject* pyargs) char ignoreUnrecognized = true; char convertStrings = false; char interrupt = false; + PyObject* tmp; - if (!PyArg_ParseTuple(pyargs, "OO!bbb", &vmPath, &PyTuple_Type, &vmOpt, - &ignoreUnrecognized, &convertStrings, &interrupt)) + if (!PyArg_ParseTuple(pyargs, "OO!bbbO", &vmPath, &PyTuple_Type, &vmOpt, + &ignoreUnrecognized, &convertStrings, &interrupt, &tmp)) return nullptr; + if (tmp != Py_None) + { + if (!(JPPyString::check(tmp))) + { + PyErr_SetString(PyExc_TypeError, "Java jar path must be a string"); + return nullptr; + } + jarTmpPath = JPPyString::asStringUTF8(tmp); + } + + if (!(JPPyString::check(vmPath))) { PyErr_SetString(PyExc_TypeError, "Java JVM path must be a string"); @@ -298,6 +314,18 @@ static PyObject* PyJPModule_shutdown(PyObject* obj, PyObject* pyargs, PyObject* return nullptr; JPContext_global->shutdownJVM(destroyJVM, freeJVM); + +#ifdef WIN32 + // Thus far this doesn't work on WINDOWS. The issue is a bug in the JVM + // is holding the file open and there is no apparent method to close it + // so that this can succeed + if (jarTmpPath != "") + remove(jarTmpPath.c_str()); +#else + if (jarTmpPath != "") + unlink(jarTmpPath.c_str()); +#endif + Py_RETURN_NONE; JP_PY_CATCH(nullptr); } diff --git a/project/jpype_java/nb-configuration.xml b/project/jpype_java/nb-configuration.xml new file mode 100644 index 000000000..c4b9dd492 --- /dev/null +++ b/project/jpype_java/nb-configuration.xml @@ -0,0 +1,39 @@ + + + + + + true + none + 2 + 2 + 8 + 80 + true + project + none + 2 + 2 + 8 + 80 + true + NEW_LINE + NEW_LINE + LEAVE_ALONE + LEAVE_ALONE + NEW_LINE + LEAVE_ALONE + NEW_LINE + true + + diff --git a/project/jpype_java/pom.xml b/project/jpype_java/pom.xml new file mode 100644 index 000000000..3dbe9b759 --- /dev/null +++ b/project/jpype_java/pom.xml @@ -0,0 +1,126 @@ + + + 4.0.0 + org.jpype + org.jpype + 1.5.1 + jar + + UTF-8 + 9 + 9 + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.2.0 + + + add-source + generate-sources + + add-source + + + + ../../native/java + + + + + + + + maven-resources-plugin + 3.3.1 + + + copy-resources + validate + + copy-resources + + + ${basedir}/target/classes/org/jpype/html + true + + + ../../native/java/org/jpype/html + true + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + compile-java-9 + + compile + + + 9 + 9 + + ../../native/java + + + + + + compile-java-0 + compile + + compile + + + 9 + + ../../native/java0 + + ${project.build.outputDirectory}/META-INF/versions/0 + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + true + + + true + lib/ + + true + + true + + + + + + + + + + \ No newline at end of file diff --git a/project/jpype_java/build.xml b/project/jpype_java_old/build.xml similarity index 100% rename from project/jpype_java/build.xml rename to project/jpype_java_old/build.xml diff --git a/project/jpype_java_old/nbproject/build-impl.xml b/project/jpype_java_old/nbproject/build-impl.xml new file mode 100644 index 000000000..f0ef9b9ba --- /dev/null +++ b/project/jpype_java_old/nbproject/build-impl.xml @@ -0,0 +1,1781 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set src.java.dir + Must set test.src.dir + Must set test.harness.dir + Must set build.dir + Must set dist.dir + Must set build.classes.dir + Must set dist.javadoc.dir + Must set build.test.classes.dir + Must set build.test.results.dir + Must set build.classes.excludes + Must set dist.jar + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No tests executed. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set JVM to use for profiling in profiler.info.jvm + Must set profiler agent JVM arguments in profiler.info.jvmargs.agent + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + To run this application from the command line without Ant, try: + + java -jar "${dist.jar.resolved}" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + Must select one file in the IDE or set run.class + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set debug.class + + + + + Must select one file in the IDE or set debug.class + + + + + Must set fix.includes + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + Must select one file in the IDE or set profile.class + This target only works when run from inside the NetBeans IDE. + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + + + Must select some files in the IDE or set test.includes + + + + + Must select one file in the IDE or set run.class + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + Some tests failed; see details above. + + + + + + + + + Must select some files in the IDE or set test.includes + + + + Some tests failed; see details above. + + + + Must select some files in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + Some tests failed; see details above. + + + + + Must select one file in the IDE or set test.class + + + + Must select one file in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + + + + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/project/jpype_java_old/nbproject/genfiles.properties b/project/jpype_java_old/nbproject/genfiles.properties new file mode 100644 index 000000000..de59bcefd --- /dev/null +++ b/project/jpype_java_old/nbproject/genfiles.properties @@ -0,0 +1,5 @@ +# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml. +# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you. +nbproject/build-impl.xml.data.CRC32=07f287a1 +nbproject/build-impl.xml.script.CRC32=dd1dbfbf +nbproject/build-impl.xml.stylesheet.CRC32=12e0a6c2@1.109.0.48 diff --git a/project/jpype_java/nbproject/license.txt b/project/jpype_java_old/nbproject/license.txt similarity index 100% rename from project/jpype_java/nbproject/license.txt rename to project/jpype_java_old/nbproject/license.txt diff --git a/project/jpype_java/nbproject/project.properties b/project/jpype_java_old/nbproject/project.properties similarity index 97% rename from project/jpype_java/nbproject/project.properties rename to project/jpype_java_old/nbproject/project.properties index 3ec6b403a..bb5767b6e 100755 --- a/project/jpype_java/nbproject/project.properties +++ b/project/jpype_java_old/nbproject/project.properties @@ -71,10 +71,10 @@ javac.source=1.8 javac.target=1.8 javac.test.classpath=\ ${javac.classpath}:\ - ${build.classes.dir}:\ ${libs.testng.classpath} javac.test.modulepath=\ - ${javac.modulepath} + ${javac.modulepath}:\ + ${build.classes.dir} javac.test.processorpath=\ ${javac.test.classpath} javadoc.additionalparam= @@ -105,8 +105,7 @@ run.jvmargs= run.modulepath=\ ${javac.modulepath} run.test.classpath=\ - ${javac.test.classpath}:\ - ${build.test.classes.dir} + ${javac.test.classpath} run.test.modulepath=\ ${javac.test.modulepath} source.encoding=UTF-8 diff --git a/project/jpype_java/nbproject/project.xml b/project/jpype_java_old/nbproject/project.xml similarity index 100% rename from project/jpype_java/nbproject/project.xml rename to project/jpype_java_old/nbproject/project.xml diff --git a/project/jpype_java/test/org/jpype/manager/TestDefault.java b/project/jpype_java_old/test/org/jpype/manager/TestDefault.java similarity index 100% rename from project/jpype_java/test/org/jpype/manager/TestDefault.java rename to project/jpype_java_old/test/org/jpype/manager/TestDefault.java diff --git a/project/jpype_java/test/org/jpype/manager/TestMethodResolution.java b/project/jpype_java_old/test/org/jpype/manager/TestMethodResolution.java similarity index 100% rename from project/jpype_java/test/org/jpype/manager/TestMethodResolution.java rename to project/jpype_java_old/test/org/jpype/manager/TestMethodResolution.java diff --git a/project/jpype_java/test/org/jpype/manager/TestTypeManager.java b/project/jpype_java_old/test/org/jpype/manager/TestTypeManager.java similarity index 100% rename from project/jpype_java/test/org/jpype/manager/TestTypeManager.java rename to project/jpype_java_old/test/org/jpype/manager/TestTypeManager.java diff --git a/project/jpype_java/test/org/jpype/manager/TypeFactoryHarness.java b/project/jpype_java_old/test/org/jpype/manager/TypeFactoryHarness.java similarity index 100% rename from project/jpype_java/test/org/jpype/manager/TypeFactoryHarness.java rename to project/jpype_java_old/test/org/jpype/manager/TypeFactoryHarness.java diff --git a/project/jpype_java/test/org/jpype/pkg/JPypePackageNGTest.java b/project/jpype_java_old/test/org/jpype/pkg/JPypePackageNGTest.java similarity index 100% rename from project/jpype_java/test/org/jpype/pkg/JPypePackageNGTest.java rename to project/jpype_java_old/test/org/jpype/pkg/JPypePackageNGTest.java diff --git a/project/jpype_java/test/org/jpype/pkg/ListPackage.java b/project/jpype_java_old/test/org/jpype/pkg/ListPackage.java similarity index 100% rename from project/jpype_java/test/org/jpype/pkg/ListPackage.java rename to project/jpype_java_old/test/org/jpype/pkg/ListPackage.java diff --git a/setupext/build_ext.py b/setupext/build_ext.py index 00cc0a72d..09e6fe51c 100644 --- a/setupext/build_ext.py +++ b/setupext/build_ext.py @@ -300,7 +300,7 @@ def build_java_ext(self, ext): distutils.log.info( "Jar cache is missing, using --enable-build-jar to recreate it.") - target_version = "1.8" + target_version = "9" # build the jar try: dirname = os.path.dirname(self.get_ext_fullpath("JAVA")) @@ -311,10 +311,16 @@ def build_java_ext(self, ext): cmd1 = shlex.split('%s -cp "%s" -d "%s" -g:none -source %s -target %s -encoding UTF-8' % (javac, classpath, build_dir, target_version, target_version)) cmd1.extend(ext.sources) + self.announce(" %s" % " ".join(cmd1), level=distutils.log.INFO) + subprocess.check_call(cmd1) - os.makedirs("build/classes", exist_ok=True) + cmd1 = shlex.split('%s -cp "%s" -d "%s" -g:none -source %s -target %s -encoding UTF-8' % + (javac, build_dir, os.path.join(build_dir,"META-INF","versions","0"), target_version, target_version)) + cmd1.extend(glob.glob(os.path.join("native","java0","**","*.java"), recursive=True)) + os.makedirs(os.path.join(build_dir,"META-INF","versions","0"), exist_ok=True) self.announce(" %s" % " ".join(cmd1), level=distutils.log.INFO) subprocess.check_call(cmd1) + manifest = None try: for file in glob.iglob("native/java/**/*.*", recursive=True): diff --git a/test/jpypetest/common.py b/test/jpypetest/common.py index 10e62e60c..8494ff568 100644 --- a/test/jpypetest/common.py +++ b/test/jpypetest/common.py @@ -67,6 +67,15 @@ def f(self): raise unittest.SkipTest("numpy required") return f +def requireAscii(func): + def f(self): + try: + root = path.dirname(path.abspath(path.dirname(__file__))) + if root.isascii(): + return func(self) + except ImportError: + raise unittest.SkipTest("Ascii root directory required") + return f class UseFunc(object): def __init__(self, obj, func, attr): @@ -117,9 +126,8 @@ def setUp(self): args.append( "-javaagent:lib/org.jacoco.agent-0.8.5-runtime.jar=destfile=build/coverage/jacoco.exec,includes=org.jpype.*") warnings.warn("using JaCoCo") - import pathlib - jpype.addClassPath(pathlib.Path("lib/*").absolute()) - jpype.addClassPath(pathlib.Path("test/jar/*").absolute()) + jpype.addClassPath(path.join(root, "../lib/*")) + jpype.addClassPath(path.join(root, "jar/*")) classpath_arg %= jpype.getClassPath() args.append(classpath_arg) _jpype.enableStacktraces(True) diff --git a/test/jpypetest/subrun.py b/test/jpypetest/subrun.py index f0a24f787..41a51f181 100644 --- a/test/jpypetest/subrun.py +++ b/test/jpypetest/subrun.py @@ -23,6 +23,8 @@ import queue import unittest import common +from contextlib import redirect_stdout +import io _modules = {} # type: ignore[var-annotated] @@ -51,14 +53,17 @@ def _execute(inQueue, outQueue): ret = None (func_name, func_file, args, kwargs) = datum try: - module = _import(func_file) - func = getattr(module, func_name) - ret = func(*args, **kwargs) + f = io.StringIO() + with redirect_stdout(f): + module = _import(func_file) + func = getattr(module, func_name) + ret = func(*args, **kwargs) except Exception as ex1: traceback.print_exc() ex = ex1 + out = f.getvalue() # This may fail if we get a Java exception so timeout is used - outQueue.put([ret, ex]) + outQueue.put([ret, out, ex]) class Client(object): @@ -77,7 +82,7 @@ def execute(self, function, *args, **kwargs): self.inQueue.put([function.__name__, os.path.abspath( inspect.getfile(function)), args, kwargs]) try: - (ret, ex) = self.outQueue.get(True, self.timeout) + (ret, out, ex) = self.outQueue.get(True, self.timeout) except queue.Empty: raise AssertionError("function {func} FAILED with args: {args} and kwargs: {kwargs}" .format(func=function, args=args, kwargs=kwargs)) diff --git a/test/jpypetest/test_caller_sensitive.py b/test/jpypetest/test_caller_sensitive.py index 1c2c733fc..ca97a6af8 100644 --- a/test/jpypetest/test_caller_sensitive.py +++ b/test/jpypetest/test_caller_sensitive.py @@ -109,4 +109,4 @@ def testStackWalker1(self): def testStackWalker2(self): self.assertEqual(self.obj.callStackWalker2(), jpype.JClass( - jpype.java.lang.Class.forName("org.jpype.JPypeContext")).class_) + jpype.java.lang.Class.forName("org.jpype.Reflector0")).class_) diff --git a/test/jpypetest/test_coverage.py b/test/jpypetest/test_coverage.py index f981caa95..6921f80ee 100644 --- a/test/jpypetest/test_coverage.py +++ b/test/jpypetest/test_coverage.py @@ -49,8 +49,8 @@ def testWin32(self): def testHandleClassPath(self): with self.assertRaises(TypeError): - jpype._core._handleClassPath([1]) - jpype._core._handleClassPath(["./*.jar"]) + jpype._core._expandClassPath([1]) + jpype._core._expandClassPath(["./*.jar"]) def testRestart(self): with self.assertRaises(OSError): diff --git a/test/jpypetest/test_fault.py b/test/jpypetest/test_fault.py index c84ab3e30..7c0853def 100644 --- a/test/jpypetest/test_fault.py +++ b/test/jpypetest/test_fault.py @@ -1083,13 +1083,15 @@ def testStartupBadArg(self): with self.assertRaisesRegex(TypeError, "takes exactly"): _jpype.startup() with self.assertRaisesRegex(TypeError, "must be tuple"): - _jpype.startup(object(), object(), True, True, True) + _jpype.startup(object(), object(), True, True, True, None) with self.assertRaisesRegex(TypeError, "must be strings"): - _jpype.startup("", (object(),), True, True, True) + _jpype.startup("", (object(),), True, True, True, None) with self.assertRaisesRegex(TypeError, "must be a string"): - _jpype.startup(object(), tuple(), True, True, True) + _jpype.startup(object(), tuple(), True, True, True, None) + with self.assertRaisesRegex(TypeError, "must be a string"): + _jpype.startup("", tuple(), True, True, True, object()) with self.assertRaisesRegex(OSError, "started"): - _jpype.startup("", tuple(), True, True, True) + _jpype.startup("", tuple(), True, True, True, None) def testGetClass(self): with self.assertRaisesRegex(TypeError, "not found"): diff --git a/test/jpypetest/test_startup.py b/test/jpypetest/test_startup.py index 8dff5d4e1..ce6156bcf 100644 --- a/test/jpypetest/test_startup.py +++ b/test/jpypetest/test_startup.py @@ -20,6 +20,7 @@ import os from pathlib import Path import unittest +import common root = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) cp = os.path.join(root, 'classes').replace('\\', '/') @@ -152,7 +153,7 @@ def testNonASCIIPath(self): """ jpype.startJVM(jvmpath=Path(self.jvmpath), classpath="test/jar/unicode_à😎/sample_package.jar") cl = jpype.JClass("java.lang.ClassLoader").getSystemClassLoader() - self.assertEqual(type(cl), jpype.JClass("org.jpype.classloader.JpypeSystemClassLoader")) + self.assertEqual(type(cl), jpype.JClass("org.jpype.JPypeClassLoader")) assert dir(jpype.JPackage('org.jpype.sample_package')) == ['A', 'B'] @@ -162,7 +163,7 @@ def testOldStyleNonASCIIPath(self): """ jpype.startJVM("-Djava.class.path=test/jar/unicode_à😎/sample_package.jar", jvmpath=Path(self.jvmpath)) cl = jpype.JClass("java.lang.ClassLoader").getSystemClassLoader() - self.assertEqual(type(cl), jpype.JClass("org.jpype.classloader.JpypeSystemClassLoader")) + self.assertEqual(type(cl), jpype.JClass("org.jpype.JPypeClassLoader")) assert dir(jpype.JPackage('org.jpype.sample_package')) == ['A', 'B'] def testNonASCIIPathWithSystemClassLoader(self): @@ -181,6 +182,7 @@ def testOldStyleNonASCIIPathWithSystemClassLoader(self): "-Djava.class.path=test/jar/unicode_à😎/sample_package.jar" ) + @common.requireAscii def testASCIIPathWithSystemClassLoader(self): jpype.startJVM( "-Djava.system.class.loader=jpype.startup.TestSystemClassLoader", @@ -192,6 +194,7 @@ def testASCIIPathWithSystemClassLoader(self): self.assertEqual(type(classloader), test_classLoader) assert dir(jpype.JPackage('jpype.startup')) == ['TestSystemClassLoader'] + @common.requireAscii def testOldStyleASCIIPathWithSystemClassLoader(self): jpype.startJVM( self.jvmpath, @@ -203,11 +206,12 @@ def testOldStyleASCIIPathWithSystemClassLoader(self): self.assertEqual(type(classloader), test_classLoader) assert dir(jpype.JPackage('jpype.startup')) == ['TestSystemClassLoader'] + @common.requireAscii def testDefaultSystemClassLoader(self): # we introduce no behavior change unless absolutely necessary jpype.startJVM(jvmpath=Path(self.jvmpath)) cl = jpype.JClass("java.lang.ClassLoader").getSystemClassLoader() - self.assertNotEqual(type(cl), jpype.JClass("org.jpype.classloader.JpypeSystemClassLoader")) + self.assertNotEqual(type(cl), jpype.JClass("org.jpype.JPypeClassLoader")) def testServiceWithNonASCIIPath(self): jpype.startJVM(