Adding new Identity Map V3 and Refreshable UID fields to SaltEntry

src/main/java/com/uid2/shared/model/SaltEntry.java

@@ -1,31 +1,43 @@
 package com.uid2.shared.model;
 
-public class SaltEntry {
-    private final long id;
-    private final String hashedId;
-    private final long lastUpdated;
-    private final String salt;
+public record SaltEntry(
+        long id,
+        String hashedId,
+        long lastUpdated,
+        String currentSalt,
 
-    public SaltEntry(long id, String hashedId, long lastUpdated, String salt) {
-        this.id = id;
-        this.lastUpdated = lastUpdated;
-        this.hashedId = hashedId;
-        this.salt = salt;
-    }
-
-    public long getId() {
-        return id;
-    }
-
-    public String getHashedId() {
-        return hashedId;
-    }
+        Long refreshFrom, // needs to be nullable until V3 Identity Map is fully rolled out
+        String previousSalt,
 
-    public long getLastUpdated() {
-        return lastUpdated;
+        KeyMaterial currentKey,
+        KeyMaterial previousKey
+) {
+    @Override
+    public String toString() {
+        return "SaltEntry{" +
+                "id=" + id +
+                ", hashedId='" + hashedId + '\'' +
+                ", lastUpdated=" + lastUpdated +
+                ", currentSalt=<REDACTED>" +
+                ", refreshFrom=" + refreshFrom +
+                ", previousSalt=<REDACTED>" +
+                ", currentKey=" + currentKey +
+                ", previousKey=" + previousKey +
+                '}';
     }
 
-    public String getSalt() {
-        return salt;
+    public record KeyMaterial(
+            int id,
+            String key,
+            String salt
+    ) {
+        @Override
+        public String toString() {
+            return "KeyMaterial{" +
+                    "id=" + id +
+                    ", key=<REDACTED>" +
+                    ", currentSalt=<REDACTED>" +
+                    '}';
+        }
     }
 }

src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java → src/main/java/com/uid2/shared/store/salt/EncryptedRotatingSaltProvider.java

@@ -1,4 +1,4 @@
-package com.uid2.shared.store;
+package com.uid2.shared.store.salt;
 
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.model.SaltEntry;
@@ -19,15 +19,8 @@ public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, Ro
     }
 
     @Override
-    protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException {
+    protected SaltEntry[] readInputStream(InputStream inputStream, SaltFileParser saltFileParser, Integer size) throws IOException {
         String decrypted = decryptInputStream(inputStream, cloudEncryptionKeyProvider, "salts");
-        SaltEntry[] entries = new SaltEntry[size];
-        int idx = 0;
-        for (String line : decrypted.split("\n")) {
-            final SaltEntry entry = entryBuilder.toEntry(line);
-            entries[idx] = entry;
-            idx++;
-        }
-        return entries;
+        return saltFileParser.parseFile(decrypted, size);
     }
 }

src/main/java/com/uid2/shared/store/ISaltProvider.java → src/main/java/com/uid2/shared/store/salt/ISaltProvider.java

@@ -1,4 +1,4 @@
-package com.uid2.shared.store;
+package com.uid2.shared.store.salt;
 
 import com.uid2.shared.model.SaltEntry;
 import org.slf4j.Logger;

src/main/java/com/uid2/shared/store/salt/IdHashingScheme.java

@@ -0,0 +1,17 @@
+package com.uid2.shared.store.salt;
+
+import org.hashids.Hashids;
+
+public class IdHashingScheme {
+    private final String prefix;
+    private final Hashids hasher;
+
+    public IdHashingScheme(final String prefix, final String secret) {
+        this.prefix = prefix;
+        this.hasher = new Hashids(secret, 9);
+    }
+
+    public String encode(long id) {
+        return prefix + this.hasher.encode(id);
+    }
+}

src/main/java/com/uid2/shared/store/RotatingSaltProvider.java → src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java

@@ -1,4 +1,4 @@
-package com.uid2.shared.store;
+package com.uid2.shared.store.salt;
 
 import com.uid2.shared.Utils;
 import com.uid2.shared.attest.UidCoreClient;
@@ -10,7 +10,6 @@
 import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.hashids.Hashids;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -41,9 +40,9 @@
        ]
     }
 
-  2. salt file format
-        <id>,   <hash_id>,    <salt>
-        9000099,1614556800000,salt
+  2. currentSalt file format
+        <id>,   <hash_id>,    <currentSalt>
+        9000099,1614556800000,currentSalt
  */
 public class RotatingSaltProvider implements ISaltProvider, IMetadataVersionedStore {
     private static final Logger LOGGER = LoggerFactory.getLogger(RotatingSaltProvider.class);
@@ -80,14 +79,14 @@ public long getVersion(JsonObject metadata) {
     public long loadContent(JsonObject metadata) throws Exception {
         final JsonArray salts = metadata.getJsonArray("salts");
         final String firstLevelSalt = metadata.getString("first_level");
-        final SaltEntryBuilder entryBuilder = new SaltEntryBuilder(
+        final SaltFileParser saltFileParser = new SaltFileParser(
                 new IdHashingScheme(metadata.getString("id_prefix"), metadata.getString("id_secret")));
         final Instant now = Instant.now();
         final List<SaltSnapshot> snapshots = new ArrayList<>();
 
         int saltCount = 0;
         for (int i = 0; i < salts.size(); ++i) {
-            final SaltSnapshot snapshot = this.loadSnapshot(salts.getJsonObject(i), firstLevelSalt, entryBuilder, now);
+            final SaltSnapshot snapshot = this.loadSnapshot(salts.getJsonObject(i), firstLevelSalt, saltFileParser, now);
             if (snapshot == null) continue;
             snapshots.add(snapshot);
 
@@ -120,33 +119,26 @@ public ISaltSnapshot getSnapshot(Instant asOf) {
             if (!snapshot.isEffective(asOf)) break;
             current = snapshot;
         }
-        return current != null ? current : snapshots.get(snapshots.size() - 1);
+        return current != null ? current : snapshots.getLast();
     }
 
-    private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltEntryBuilder entryBuilder, Instant now) throws Exception {
+    private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltFileParser saltFileParser, Instant now) throws Exception {
         final Instant defaultExpires = now.plus(365, ChronoUnit.DAYS);
         final Instant effective = Instant.ofEpochMilli(spec.getLong("effective"));
         final Instant expires = Instant.ofEpochMilli(spec.getLong("expires", defaultExpires.toEpochMilli()));
 
         final String path = spec.getString("location");
         Integer size = spec.getInteger("size");
-        SaltEntry[] entries = readInputStream(this.contentStreamProvider.download(path), entryBuilder, size);
+        SaltEntry[] entries = readInputStream(this.contentStreamProvider.download(path), saltFileParser, size);
 
         LOGGER.info("Loaded {} salts", size);
         return new SaltSnapshot(effective, expires, entries, firstLevelSalt);
     }
 
-    protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException {
+    protected SaltEntry[] readInputStream(InputStream inputStream, SaltFileParser saltFileParser, Integer size) throws IOException {
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
-            String line;
-            SaltEntry[] entries = new SaltEntry[size];
-            int idx = 0;
-            while ((line = reader.readLine()) != null) {
-                final SaltEntry entry = entryBuilder.toEntry(line);
-                entries[idx] = entry;
-                idx++;
-            }
-            return entries;
+            String[] saltFileLines = reader.lines().toArray(String[]::new);
+            return saltFileParser.parseFileLines(saltFileLines, size);
         }
     }
 
@@ -167,10 +159,10 @@ public SaltSnapshot(Instant effective, Instant expires, SaltEntry[] entries, Str
             this.entries = entries;
             this.firstLevelSalt = firstLevelSalt;
             if (entries.length == 1_048_576) {
-                LOGGER.info("Total salt entries 1 million, {}, special production salt entry indexer", entries.length);
+                LOGGER.info("Total currentSalt entries 1 million, {}, special production currentSalt entry indexer", entries.length);
                 this.saltEntryIndexer = MILLION_ENTRY_INDEXER;
             } else {
-                LOGGER.warn("Total salt entries {}, using slower mod-based indexer", entries.length);
+                LOGGER.warn("Total currentSalt entries {}, using slower mod-based indexer", entries.length);
                 this.saltEntryIndexer = MOD_BASED_INDEXER;
             }
         }
@@ -203,42 +195,8 @@ public SaltEntry getRotatingSalt(byte[] identity) {
         @Override
         public List<SaltEntry> getModifiedSince(Instant timestamp) {
             final long timestampMillis = timestamp.toEpochMilli();
-            return Arrays.stream(this.entries).filter(e -> e.getLastUpdated() >= timestampMillis).collect(Collectors.toList());
-        }
-    }
-
-    protected static final class IdHashingScheme {
-        private final String prefix;
-        private final Hashids hasher;
-
-        public IdHashingScheme(final String prefix, final String secret) {
-            this.prefix = prefix;
-            this.hasher = new Hashids(secret, 9);
-        }
-
-        public String encode(long id) {
-            return prefix + this.hasher.encode(id);
+            return Arrays.stream(this.entries).filter(e -> e.lastUpdated() >= timestampMillis).collect(Collectors.toList());
         }
     }
 
-    protected static final class SaltEntryBuilder {
-        private final IdHashingScheme idHashingScheme;
-
-        public SaltEntryBuilder(IdHashingScheme idHashingScheme) {
-            this.idHashingScheme = idHashingScheme;
-        }
-
-        public SaltEntry toEntry(String line) {
-            try {
-                final String[] fields = line.split(",");
-                final long id = Integer.parseInt(fields[0]);
-                final String hashedId = this.idHashingScheme.encode(id);
-                final long lastUpdated = Long.parseLong(fields[1]);
-                final String salt = fields[2];
-                return new SaltEntry(id, hashedId, lastUpdated, salt);
-            } catch (Exception e) {
-                throw new RuntimeException("Trouble parsing Salt Entry " + line, e);
-            }
-        }
-    }
 }

src/main/java/com/uid2/shared/store/salt/SaltFileParser.java

@@ -0,0 +1,63 @@
+package com.uid2.shared.store.salt;
+
+import com.uid2.shared.model.SaltEntry;
+
+public class SaltFileParser {
+    private final IdHashingScheme idHashingScheme;
+
+    public SaltFileParser(IdHashingScheme idHashingScheme) {
+        this.idHashingScheme = idHashingScheme;
+    }
+
+    public SaltEntry[] parseFile(String saltFileContent, Integer size) {
+        var lines = saltFileContent.split("\n");
+        return parseFileLines(lines, size);
+    }
+
+    public SaltEntry[] parseFileLines(String[] saltFileLines, Integer size) {
+        var entries = new SaltEntry[size];
+        int lineNumber = 0;
+        for (String line : saltFileLines) {
+            final SaltEntry entry = parseLine(line, lineNumber);
+            entries[lineNumber] = entry;
+            lineNumber++;
+        }
+        return entries;
+    }
+
+    private SaltEntry parseLine(String line, int lineNumber) {
+        try {
+            final String[] fields = line.split(",");
+            final long id = Integer.parseInt(fields[0]);
+            final String hashedId = this.idHashingScheme.encode(id);
+            final long lastUpdated = Long.parseLong(fields[1]);
+            final String salt = fields[2];
+
+            Long refreshFrom = null;
+            String previousSalt = null;
+            SaltEntry.KeyMaterial currentKey = null;
+            SaltEntry.KeyMaterial previousKey = null;
+
+            // TODO: The fields below should stop being optional once the refresh from, previous salt
+            // and refreshable UIDs features get rolled out in production. We can remove them one by one as necessary.
+            // AU, 2025/04/28
+            if (fields.length > 3) {
+                refreshFrom = Long.parseLong(fields[3]);
+            }
+            if (fields.length > 4) {
+                previousSalt = fields[4];
+            }
+            if (fields.length > 7) {
+                currentKey = new SaltEntry.KeyMaterial(Integer.parseInt(fields[5]), fields[6], fields[7]);
+            }
+            if (fields.length > 10) {
+                previousKey = new SaltEntry.KeyMaterial(Integer.parseInt(fields[8]), fields[9], fields[10]);
+            }
+
+            return new SaltEntry(id, hashedId, lastUpdated, salt, refreshFrom, previousSalt, currentKey, previousKey);
+        } catch (Exception e) {
+            throw new RuntimeException("Trouble parsing Salt Entry, line number: " + lineNumber, e);
+        }
+    }
+
+}

src/test/java/com/uid2/shared/secret/KeyHasherTest.java

@@ -22,7 +22,7 @@ public void hashKey_returnsNewHashEverytime_withRandomSalt() {
         KeyHashResult result2 = hasher.hashKey("test-key");
 
         assertAll(
-                "hashKey returns new hash every time with random salt",
+                "hashKey returns new hash every time with random currentSalt",
                 () -> assertNotEquals(result1.getHash(), result2.getHash()),
                 () -> assertNotEquals(result1.getSalt(), result2.getSalt())
         );

src/test/java/com/uid2/shared/secure/AttestationTokenTest.java

@@ -18,7 +18,7 @@
 
 public class AttestationTokenTest {
     private static final String ENCRYPTION_KEY = "attestation-token-secret";
-    private static final String SALT = "attestation-token-salt";
+    private static final String SALT = "attestation-token-currentSalt";
 
     private final Clock clock = mock(Clock.class);
 

src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java

@@ -4,6 +4,8 @@
 import com.uid2.shared.encryption.AesGcm;
 import com.uid2.shared.model.CloudEncryptionKey;
 import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.salt.EncryptedRotatingSaltProvider;
+import com.uid2.shared.store.salt.ISaltProvider;
 import com.uid2.shared.store.scope.EncryptedScope;
 import io.vertx.core.json.JsonArray;
 import io.vertx.core.json.JsonObject;
@@ -157,7 +159,7 @@ public void loadSaltSingleVersion1mil() throws Exception {
         final String effectiveTimeString = String.valueOf(generatedTime.getEpochSecond() * 1000L);
         StringBuilder salts = new StringBuilder();
         for (int i = 0; i < 1000000; i++) {
-            salts.append(i).append(",").append(effectiveTimeString).append(",").append("salt-string").append("\n");
+            salts.append(i).append(",").append(effectiveTimeString).append(",").append("currentSalt-string").append("\n");
         }
 
         when(cloudStorage.download("sites/encrypted/1_public/metadata.json"))
@@ -256,7 +258,7 @@ public void loadSaltMultipleVersions() throws Exception {
         assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt());
         assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty());
         assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).size());
-        assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).getFirst().getId());
+        assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).getFirst().id());
     }
 
     @Test
@@ -338,6 +340,6 @@ public void loadSaltMultipleVersionsExpired() throws Exception {
         assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt());
         assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty());
         assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).size());
-        assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).getFirst().getId());
+        assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).getFirst().id());
     }
 }

src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java

@@ -1,6 +1,8 @@
 package com.uid2.shared.store;
 
 import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.store.salt.ISaltProvider;
+import com.uid2.shared.store.salt.RotatingSaltProvider;
 import io.vertx.core.json.JsonArray;
 import io.vertx.core.json.JsonObject;
 import org.junit.jupiter.api.Test;
@@ -157,7 +159,7 @@ public void loadSaltMultipleVersions() throws Exception {
         assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt());
         assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty());
         assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).size());
-        assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).get(0).getId());
+        assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).get(0).id());
     }
 
     @Test
@@ -239,7 +241,7 @@ public void loadSaltMultipleVersionsExpired() throws Exception {
         assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt());
         assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty());
         assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).size());
-        assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).get(0).getId());
+        assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).get(0).id());
     }