8277489: Rewrite JAAS UnixLoginModule with FFM

make/modules/jdk.security.auth/Lib.gmk

@@ -31,13 +31,14 @@ include LibCommon.gmk
 ## Build libjaas
 ################################################################################
 
-$(eval $(call SetupJdkLibrary, BUILD_LIBJAAS, \
-    NAME := jaas, \
-    OPTIMIZATION := LOW, \
-    EXTRA_HEADER_DIRS := java.base:libjava, \
-    LIBS_windows := advapi32.lib mpr.lib netapi32.lib user32.lib, \
-))
-
-TARGETS += $(BUILD_LIBJAAS)
+ifeq ($(call isTargetOs, windows), true)
+  $(eval $(call SetupJdkLibrary, BUILD_LIBJAAS, \
+      NAME := jaas, \
+      OPTIMIZATION := LOW, \
+      EXTRA_HEADER_DIRS := java.base:libjava, \
+      LIBS_windows := advapi32.lib mpr.lib netapi32.lib user32.lib, \
+  ))
 
+  TARGETS += $(BUILD_LIBJAAS)
+endif
 ################################################################################

src/jdk.security.auth/share/classes/com/sun/security/auth/module/UnixLoginModule.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,6 @@
 package com.sun.security.auth.module;
 
 import java.util.*;
-import java.io.IOException;
 import javax.security.auth.*;
 import javax.security.auth.callback.*;
 import javax.security.auth.login.*;
@@ -121,20 +120,33 @@ public void initialize(Subject subject, CallbackHandler callbackHandler,
      */
     public boolean login() throws LoginException {
 
-        long[] unixGroups = null;
-
         try {
+            // TODO: Shall we fail early for unsupported systems?
+            // 1. If it's Windows, we should fail immediately to avoid
+            //    cygwin-like functions being loaded, which are not supported
+            //    and could have a different `long`.
+            // 2. The FFM code has only been tested on macOS and Linux and
+            //    might fail on other *nix systems. Especially, the `passwd`
+            //    struct could be defined differently, although I've checked
+            //    several and an extra 100 chars at the end seems enough.
             ss = new UnixSystem();
-        } catch (UnsatisfiedLinkError ule) {
+        } catch (ExceptionInInitializerError | UnsatisfiedLinkError ule) {
+            // Errors could happen in either static blocks or the constructor,
+            // both have a cause.
             succeeded = false;
-            throw new FailedLoginException
+            var error = new FailedLoginException
                                 ("Failed in attempt to import " +
                                 "the underlying system identity information" +
                                 " on " + System.getProperty("os.name"));
+            if (ule.getCause() != null) {
+                error.initCause(ule.getCause());
+            }
+            throw error;
         }
         userPrincipal = new UnixPrincipal(ss.getUsername());
         UIDPrincipal = new UnixNumericUserPrincipal(ss.getUid());
         GIDPrincipal = new UnixNumericGroupPrincipal(ss.getGid(), true);
+        long[] unixGroups = null;
         if (ss.getGroups() != null && ss.getGroups().length > 0) {
             unixGroups = ss.getGroups();
             for (int i = 0; i < unixGroups.length; i++) {
@@ -150,9 +162,10 @@ public boolean login() throws LoginException {
                     "succeeded importing info: ");
             System.out.println("\t\t\tuid = " + ss.getUid());
             System.out.println("\t\t\tgid = " + ss.getGid());
-            unixGroups = ss.getGroups();
-            for (int i = 0; i < unixGroups.length; i++) {
-                System.out.println("\t\t\tsupp gid = " + unixGroups[i]);
+            if (unixGroups != null) {
+                for (int i = 0; i < unixGroups.length; i++) {
+                    System.out.println("\t\t\tsupp gid = " + unixGroups[i]);
+                }
             }
         }
         succeeded = true;

src/jdk.security.auth/share/classes/com/sun/security/auth/module/UnixSystem.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,38 +25,144 @@
 
 package com.sun.security.auth.module;
 
+import java.lang.foreign.AddressLayout;
+import java.lang.foreign.Arena;
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.GroupLayout;
+import java.lang.foreign.Linker;
+import java.lang.foreign.MemoryLayout;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.SymbolLookup;
+import java.lang.foreign.ValueLayout;
+import java.lang.invoke.MethodHandle;
+
+import static java.lang.foreign.MemoryLayout.PathElement.groupElement;
+
 /**
  * This class implementation retrieves and makes available Unix
  * UID/GID/groups information for the current user.
  *
  * @since 1.4
  */
+@SuppressWarnings("restricted")
 public class UnixSystem {
 
-    private native void getUnixInfo();
-
-    // Warning: the following 4 fields are used by Unix.c
-
-    /** The current username. */
+    /**
+     * The current username.
+     */
     protected String username;
 
-    /** The current user ID. */
+    /**
+     * The current user ID.
+     */
     protected long uid;
 
-    /** The current group ID. */
+    /**
+     * The current group ID.
+     */
     protected long gid;
 
-    /** The current list of groups. */
+    /**
+     * The current list of groups.
+     */
     protected long[] groups;
 
+    private static final Linker LINKER = Linker.nativeLinker();
+    private static final SymbolLookup SYMBOL_LOOKUP = SymbolLookup.loaderLookup()
+            .or(LINKER.defaultLookup());
+
+    private static final ValueLayout.OfByte C_CHAR
+            = (ValueLayout.OfByte) LINKER.canonicalLayouts().get("char");
+    private static final ValueLayout.OfInt C_INT
+            = (ValueLayout.OfInt) LINKER.canonicalLayouts().get("int");
+    private static final AddressLayout C_POINTER
+            = ((AddressLayout) LINKER.canonicalLayouts().get("void*"))
+            .withTargetLayout(MemoryLayout.sequenceLayout(java.lang.Long.MAX_VALUE, C_CHAR));
+    private static final ValueLayout.OfLong C_LONG
+            = (ValueLayout.OfLong) LINKER.canonicalLayouts().get("long");
+
+    private static final MethodHandle getgroups = LINKER
+            .downcallHandle(SYMBOL_LOOKUP.findOrThrow("getgroups"),
+                    FunctionDescriptor.of(C_INT, C_INT, C_POINTER));
+    private static final MethodHandle getuid = LINKER
+            .downcallHandle(SYMBOL_LOOKUP.findOrThrow("getuid"),
+                    FunctionDescriptor.of(C_INT));
+    private static final MethodHandle getgid = LINKER
+            .downcallHandle(SYMBOL_LOOKUP.findOrThrow("getgid"),
+                    FunctionDescriptor.of(C_INT));
+    private static final MethodHandle getpwuid_r = LINKER
+            .downcallHandle(SYMBOL_LOOKUP.findOrThrow("getpwuid_r"),
+                    FunctionDescriptor.of(C_INT, C_INT, C_POINTER, C_POINTER,
+                            C_LONG, C_POINTER));
+
+    private static final GroupLayout passwd_layout = MemoryLayout.structLayout(
+            C_POINTER.withName("pw_name"),
+            C_POINTER.withName("pw_passwd"),
+            C_INT.withName("pw_uid"),
+            C_INT.withName("pw_gid"),
+            // Different platforms have different fields in `struct passwd`.
+            // While we don't need those fields here, the struct needs to be
+            // big enough to avoid buffer overflow when `getpwuid_r` is called.
+            MemoryLayout.sequenceLayout(100, C_CHAR).withName("dummy"));
+
+    private static final ValueLayout.OfInt pw_uid_layout
+            = (ValueLayout.OfInt) passwd_layout.select(groupElement("pw_uid"));
+    private static final long pw_uid_offset
+            = passwd_layout.byteOffset(groupElement("pw_uid"));
+    private static final ValueLayout.OfInt pw_gid_layout
+            = (ValueLayout.OfInt) passwd_layout.select(groupElement("pw_gid"));
+    private static final long pw_gid_offset
+            = passwd_layout.byteOffset(groupElement("pw_gid"));
+    private static final AddressLayout pw_name_layout
+            = (AddressLayout) passwd_layout.select(groupElement("pw_name"));
+    private static final long pw_name_offset
+            = passwd_layout.byteOffset(groupElement("pw_name"));
+
+    // sysconf(_SC_GETPW_R_SIZE_MAX) on macOS is 4096 and 1024 on Linux
+    private static final long GETPW_R_SIZE_MAX = 4096L;
+
     /**
      * Instantiate a {@code UnixSystem} and load
      * the native library to access the underlying system information.
      */
-    @SuppressWarnings("restricted")
     public UnixSystem() {
-        System.loadLibrary("jaas");
-        getUnixInfo();
+        try (Arena scope = Arena.ofConfined()) {
+            int groupnum = (int) getgroups.invokeExact(0, MemorySegment.NULL);
+            if (groupnum == -1) {
+                throw new RuntimeException("getgroups returns " + groupnum);
+            }
+
+            var gs = scope.allocate(C_INT, groupnum);
+            groupnum = (int) getgroups.invokeExact(groupnum, gs);
+            if (groupnum == -1) {
+                throw new RuntimeException("getgroups returns " + groupnum);
+            }
+
+            groups = new long[groupnum];
+            for (int i = 0; i < groupnum; i++) {
+                groups[i] = gs.getAtIndex(C_INT, i);
+            }
+
+            var resbuf = scope.allocate(passwd_layout);
+            var pwd = scope.allocate(C_POINTER);
+            var pwd_buf = scope.allocate(GETPW_R_SIZE_MAX);
+            var tmpUid = (int)getuid.invokeExact();
+            int out = (int) getpwuid_r.invokeExact(
+                    tmpUid, resbuf, pwd_buf, GETPW_R_SIZE_MAX, pwd);
+            if (out != 0 || pwd.get(ValueLayout.ADDRESS, 0).equals(MemorySegment.NULL)) {
+                uid = tmpUid;
+                gid = (int)getgid.invokeExact();
+                username = null;
+            } else {
+                uid = resbuf.get(pw_uid_layout, pw_uid_offset);
+                gid = resbuf.get(pw_gid_layout, pw_gid_offset);
+                username = resbuf.get(pw_name_layout, pw_name_offset).getString(0);
+            }
+        } catch (Throwable t) {
+            var error = new UnsatisfiedLinkError("FFM calls failed");
+            error.initCause(t);
+            throw error;
+        }
     }
 
     /**