tls: fix IPv6 hostname verification regression

The previous normalization logic applied domainToASCII() to all hostnames,
which returns an empty string for raw IPv6 addresses. This caused the IP SAN
check to be skipped and verification to incorrectly fall back to domain
matching. This patch moves the IP address check before any IDNA conversion,
ensuring IPv6 (and IPv4) addresses are compared directly against the
certificate's IP Subject Alternative Names.

Adds regression tests for IPv6 SAN verification.

Fixes: #64032
Signed-off-by: Paraspandey-debugs <p.pandey250806@gmail.com>

Author: Paraspandey-debugs

lib/tls.js

@@ -336,7 +336,7 @@ function check(hostParts, pattern, wildcards) {
   // RFC 6125 does not allow wildcard substitution for components
   // containing IDNA A-labels (Punycode) so match those verbatim.
   if (patternSubdomainParts.length === 1 ||
-      patternSubdomain.includes('xn--'))
+    patternSubdomain.includes('xn--'))
     return hostSubdomain === patternSubdomain;
 
   if (!wildcards)
@@ -409,11 +409,11 @@ exports.checkServerIdentity = function checkServerIdentity(hostname, cert) {
   const ips = [];
 
   hostname = '' + hostname;
-  const hostnameASCII = domainToASCII(hostname);
 
   // Remove trailing dots for error messages and matching.
   hostname = unfqdn(hostname);
-  const hostnameASCIIWithoutFQDN = unfqdn(hostnameASCII);
+  const hostnameASCIIWithoutFQDN = net.isIP(hostname) ?
+    hostname : domainToASCII(hostname);
 
   if (altNames) {
     const splitAltNames = altNames.includes('"') ?

test/parallel/test-tls-check-server-identity.js

@@ -404,6 +404,27 @@ const tests = [
     error: 'Host: xn--bcher-kva.example.com. is not cert\'s CN: ' +
             'xn--*.example.com',
   },
+  // IPv6 addresses
+  {
+    host: '::1', cert: {
+      subjectaltname: 'IP Address:0:0:0:0:0:0:0:1',
+      subject: {}
+    }
+  },
+  {
+    host: '::1', cert: {
+      subjectaltname: 'IP Address:127.0.0.1',
+      subject: {}
+    },
+    error: 'IP: ::1 is not in the cert\'s list: ' +
+           '127.0.0.1'
+  },
+  {
+    host: '2001:db8::1', cert: {
+      subjectaltname: 'IP Address:2001:0DB8:0000:0000:0000:0000:0000:0001',
+      subject: {}
+    }
+  },
 ];
 
 tests.forEach(function(test, i) {