From d3cb879e08791bc1ec3068bf3bf60774bc067fda Mon Sep 17 00:00:00 2001 From: Aleksandrs Ulme Date: Fri, 25 Apr 2025 16:09:46 +0800 Subject: [PATCH 1/7] Adding new Identity Map V3 and Refreshable UID fields to SaltEntry --- .../java/com/uid2/shared/model/SaltEntry.java | 52 +++--- .../shared/store/RotatingSaltProvider.java | 4 +- .../EncryptedRotatingSaltProviderTest.java | 4 +- .../store/RotatingSaltProviderTest.java | 4 +- .../shared/store/parser/ClientParserTest.java | 174 ++++++++++++++++++ 5 files changed, 209 insertions(+), 29 deletions(-) create mode 100644 src/test/java/com/uid2/shared/store/parser/ClientParserTest.java diff --git a/src/main/java/com/uid2/shared/model/SaltEntry.java b/src/main/java/com/uid2/shared/model/SaltEntry.java index d481a508..2a658cdc 100644 --- a/src/main/java/com/uid2/shared/model/SaltEntry.java +++ b/src/main/java/com/uid2/shared/model/SaltEntry.java @@ -1,31 +1,37 @@ package com.uid2.shared.model; -public class SaltEntry { - private final long id; - private final String hashedId; - private final long lastUpdated; - private final String salt; +public record SaltEntry( + long id, + String hashedId, + long lastUpdated, + String salt, - public SaltEntry(long id, String hashedId, long lastUpdated, String salt) { - this.id = id; - this.lastUpdated = lastUpdated; - this.hashedId = hashedId; - this.salt = salt; - } - - public long getId() { - return id; - } - - public String getHashedId() { - return hashedId; - } + Long refreshFrom, // needs to be nullable until V3 Identity Map is fully rolled out + String previousSalt, - public long getLastUpdated() { - return lastUpdated; + KeyMaterial currentKey, + KeyMaterial previousKey +) { + @Override + public String toString() { + return "SaltEntry{" + + "id=" + id + + ", hashedId='" + hashedId + '\'' + + ", lastUpdated=" + lastUpdated + + ", refreshFrom=" + refreshFrom + + ", currentKey=" + currentKey + + ", previousKey=" + previousKey + + '}'; } - public String getSalt() { - return salt; + public record KeyMaterial( + int id, + String key, + String salt + ){ + @Override + public String toString() { + return "KeyMaterial{id=%d}".formatted(id); + } } } diff --git a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java index c2efa8b7..699d0773 100644 --- a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java @@ -203,7 +203,7 @@ public SaltEntry getRotatingSalt(byte[] identity) { @Override public List getModifiedSince(Instant timestamp) { final long timestampMillis = timestamp.toEpochMilli(); - return Arrays.stream(this.entries).filter(e -> e.getLastUpdated() >= timestampMillis).collect(Collectors.toList()); + return Arrays.stream(this.entries).filter(e -> e.lastUpdated() >= timestampMillis).collect(Collectors.toList()); } } @@ -235,7 +235,7 @@ public SaltEntry toEntry(String line) { final String hashedId = this.idHashingScheme.encode(id); final long lastUpdated = Long.parseLong(fields[1]); final String salt = fields[2]; - return new SaltEntry(id, hashedId, lastUpdated, salt); + return new SaltEntry(id, hashedId, lastUpdated, salt, null, null, null, null); } catch (Exception e) { throw new RuntimeException("Trouble parsing Salt Entry " + line, e); } diff --git a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java index 9b26e092..4aa470fe 100644 --- a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java +++ b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java @@ -256,7 +256,7 @@ public void loadSaltMultipleVersions() throws Exception { assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).size()); - assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).getFirst().getId()); + assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).getFirst().id()); } @Test @@ -338,6 +338,6 @@ public void loadSaltMultipleVersionsExpired() throws Exception { assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).size()); - assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).getFirst().getId()); + assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).getFirst().id()); } } diff --git a/src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java index a909c6cf..9d47a563 100644 --- a/src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java +++ b/src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java @@ -157,7 +157,7 @@ public void loadSaltMultipleVersions() throws Exception { assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).size()); - assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).get(0).getId()); + assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).get(0).id()); } @Test @@ -239,7 +239,7 @@ public void loadSaltMultipleVersionsExpired() throws Exception { assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).size()); - assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).get(0).getId()); + assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).get(0).id()); } diff --git a/src/test/java/com/uid2/shared/store/parser/ClientParserTest.java b/src/test/java/com/uid2/shared/store/parser/ClientParserTest.java new file mode 100644 index 00000000..50fe995d --- /dev/null +++ b/src/test/java/com/uid2/shared/store/parser/ClientParserTest.java @@ -0,0 +1,174 @@ +package com.uid2.shared.store.parser; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.uid2.shared.auth.ClientKey; +import com.uid2.shared.util.Mapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.*; + +class ClientParserTest { + + private ClientParser parser; + + @BeforeEach + void setUp() { + parser = new ClientParser(); + } + + @Test + void testDeserialize() throws IOException { + String json = "[\n" + + " {\n" + + " \"key\": \"UID2-C-L-999-fCXrMM.fsR3mDqAXELtWWMS+xG1s7RdgRTMqdOH2qaAo=\",\n" + + " \"secret\": \"DzBzbjTJcYL0swDtFs2krRNu+g1Eokm2tBU4dEuD0Wk=\",\n" + + " \"name\": \"Special\",\n" + + " \"contact\": \"Special\",\n" + + " \"created\": 1701210253,\n" + + " \"roles\": [\n" + + " \"MAPPER\",\n" + + " \"GENERATOR\",\n" + + " \"ID_READER\",\n" + + " \"SHARER\",\n" + + " \"OPTOUT\"\n" + + " ],\n" + + " \"disabled\": false,\n" + + " \"site_id\": 999,\n" + + " \"key_hash\": \"fsSGnDxa/V9eJZ9Tas+dowwyO/X1UsC68RN9qM2xUu9ZOaKEOv9EVd7pkt3As/nE5B6TRu0PzK+IDzSQhD1+rw==\",\n" + + " \"key_salt\": \"jySwjjqo9O6OU01OWujBWC6xZNpBqRTk5H7K2borcFA=\",\n" + + " \"key_id\": \"UID2-C-L-999-fCXrM\"\n" + + " },\n" + + " {\n" + + " \"key\": \"LOCALbGlvbnVuZGVybGluZXdpbmRzY2FyZWRzb2Z0ZGVzZXI=\",\n" + + " \"secret\": \"c3RlZXBzcGVuZHNsb3BlZnJlcXVlbnRseWRvd2lkZWM=\",\n" + + " \"name\": \"Special (Legacy Key)\",\n" + + " \"contact\": \"Special (Legacy Key)\",\n" + + " \"created\": 1701210253,\n" + + " \"roles\": [\n" + + " \"MAPPER\",\n" + + " \"GENERATOR\",\n" + + " \"ID_READER\",\n" + + " \"SHARER\",\n" + + " \"OPTOUT\"\n" + + " ],\n" + + " \"disabled\": false,\n" + + " \"site_id\": 999,\n" + + " \"key_hash\": \"OPIi+MWKNz41wzu+atsBLAtXDTLFhLWPq5mCxA3L8anX+fjKaVDAf55D98BSLAh/EFQE/xTQyo/YK5snPS8ivA==\",\n" + + " \"key_salt\": \"FpgbvHGqpVhi3I/b8/9HnguiycUzb2y+KsdicPpNLJI=\",\n" + + " \"key_id\": \"LOCALbGlvb\"\n" + + " },\n" + + " {\n" + + " \"key\": \"UID2-C-L-123-t32pCM.5NCX1E94UgOd2f8zhsKmxzCoyhXohHYSSWR8U=\",\n" + + " \"secret\": \"FsD4bvtjMkeTonx6HvQp6u0EiI1ApGH4pIZzZ5P7UcQ=\",\n" + + " \"name\": \"DSP\",\n" + + " \"contact\": \"DSP\",\n" + + " \"created\": 1609459200,\n" + + " \"roles\": [\n" + + " \"ID_READER\"\n" + + " ],\n" + + " \"disabled\": false,\n" + + " \"site_id\": 123,\n" + + " \"key_hash\": \"vVb/MjymmYAE3L6as5t1DCjbts4bT2wZh4V4iAagOAe97jthFmT4YAb6gGVfEs4Pq+CqNPgz+X338RNRa8NOlA==\",\n" + + " \"key_salt\": \"G36g+KxlS+z5NwSXUOnBtc9yJKHECvXgjbS13X5A7rw=\",\n" + + " \"key_id\": \"UID2-C-L-123-t32pC\"\n" + + " },\n" + + " {\n" + + " \"key\": \"UID2-C-L-124-H8VwqX.l2G4TCuUWYAqdqkeG/UqtFoPEoXirKn4kHWxc=\",\n" + + " \"secret\": \"NcMgi6Y8C80SlxvV7pYlfcvEIo+2b0508tYQ3pKK8HM=\",\n" + + " \"name\": \"Publisher\",\n" + + " \"contact\": \"Publisher\",\n" + + " \"created\": 1609459200,\n" + + " \"roles\": [\n" + + " \"GENERATOR\",\n" + + " \"ID_READER\"\n" + + " ],\n" + + " \"disabled\": false,\n" + + " \"site_id\": 124,\n" + + " \"key_hash\": \"uA1aRDR9owk53W47zZpI6cS/bRVgKm4ggRvr9m0pz+I/5IzQcIQqfurm1Ors96r8Q2xC8GZVG3spwR/H89rQmA==\",\n" + + " \"key_salt\": \"rSwnZ5aKauMLPLMHvvH25C1LU5MdJv5+fjQ5/Yy5hP0=\",\n" + + " \"key_id\": \"UID2-C-L-124-H8Vwq\"\n" + + " },\n" + + " {\n" + + " \"key\": \"UID2-C-L-125-E5w9L8.T5og45yFqQeoj4ubh9IVqXcaSVwk7A5XyG958=\",\n" + + " \"secret\": \"3YAgjckHGQyBgSFj64ZsLf8WlUnvrQhLKuG7rljp6W4=\",\n" + + " \"name\": \"Data Provider\",\n" + + " \"contact\": \"Data Provider\",\n" + + " \"created\": 1609459200,\n" + + " \"roles\": [\n" + + " \"MAPPER\",\n" + + " \"SHARER\"\n" + + " ],\n" + + " \"disabled\": false,\n" + + " \"site_id\": 125,\n" + + " \"key_hash\": \"0GFyfie9Vz7INDxG5gT3MeHOnrXdIy/H9I5OrTZHx/cX5zToF8BngbseREbeEG7xH3KFs5TfSdwI5N/OWzYrGQ==\",\n" + + " \"key_salt\": \"taG4CBJ1F4aWwL8XwyilKl9WzYSWoG9RjvB4BGqf0/w=\",\n" + + " \"key_id\": \"UID2-C-L-125-E5w9L\"\n" + + " },\n" + + " {\n" + + " \"key\": \"UID2-C-L-126-GMP9tD.jn2o3vmXzn7vmKRlTT6BEUrPUaPJuQmDBdq38=\",\n" + + " \"secret\": \"1ydXM0rEj+ROUazVpZjNZOGu2T5+f/BIiBfnK8xGh/A=\",\n" + + " \"name\": \"Advertiser\",\n" + + " \"contact\": \"Advertiser\",\n" + + " \"created\": 1609459200,\n" + + " \"roles\": [\n" + + " \"MAPPER\",\n" + + " \"SHARER\"\n" + + " ],\n" + + " \"disabled\": false,\n" + + " \"site_id\": 126,\n" + + " \"key_hash\": \"lDl6HiO7hVdXmHm+gogCZmiCzhWcDLVIxBItR+0GMBWpRxleIr2HQG2oAHVKYd63AKeMZGwh5svbbJ6Gu0RUMQ==\",\n" + + " \"key_salt\": \"FH6UNMUCJKday6FWTLUtmg9Hwh4Rd/HhenfjtRyaAEI=\",\n" + + " \"key_id\": \"UID2-C-L-126-GMP9t\"\n" + + " },\n" + + " {\n" + + " \"key\": \"UID2-C-L-127-aHVydH.JlZnVzZWRmYXN0ZW5lbXliYWdpZGVudGl0eWU=\",\n" + + " \"secret\": \"c3VpdG9wcG9zaXRlaW1hZ2Vsb29rc2ltcGxlc3RmaXI=\",\n" + + " \"name\": \"OptOut\",\n" + + " \"contact\": \"OptOut\",\n" + + " \"created\": 1609459200,\n" + + " \"roles\": [\n" + + " \"OPTOUT\"\n" + + " ],\n" + + " \"disabled\": false,\n" + + " \"site_id\": 127,\n" + + " \"key_hash\": \"BEEnHVPHwbMYUtZk/N6jjnN04U7xpu6hV5yF4Nn2Zw9pigD43JLZdEleRW/Mz7LAQfYtLTJk768J8WK6F4Ku/Q==\",\n" + + " \"key_salt\": \"cw9wfsevy1xRiPys5JkBSTPHmTickWkFWQ0zrIF2C60=\",\n" + + " \"key_id\": \"UID2-C-L-127-aHVyd\"\n" + + " },\n" + + " {\n" + + " \"key\": \"UID2-C-L-1000-qxpBsF.ibeCDBpD2bq4Zm7inDacGioUk1aaLeNJrabow=\",\n" + + " \"secret\": \"VT7+t0G/RVueMuVZAL56I2c3JJFSYQfhbu8yo0V/Tds=\",\n" + + " \"name\": \"Legacy Site Client\",\n" + + " \"contact\": \"Legacy Site Client\",\n" + + " \"created\": 1609459200,\n" + + " \"roles\": [\n" + + " \"MAPPER\",\n" + + " \"GENERATOR\",\n" + + " \"ID_READER\",\n" + + " \"SHARER\",\n" + + " \"OPTOUT\"\n" + + " ],\n" + + " \"disabled\": false,\n" + + " \"site_id\": 1000,\n" + + " \"key_hash\": \"654FIeR8DFtLi5AC8RXvwfBQ1b9J8L+dVyJUxoTSCpMBQ3z937CxQ1fp40fHIs9SbQPnivBMV5s+TdDMZXZqgQ==\",\n" + + " \"key_salt\": \"huTnT+HyINotMK0W00Gy7VGaQT9XR0KaxZTBvVuTCF0=\",\n" + + " \"key_id\": \"UID2-C-L-1000-qxpBs\"\n" + + " }\n" + + "]"; + ObjectMapper OBJECT_MAPPER = new ObjectMapper() +// .configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, true) + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);//Mapper.getInstance(); + ClientKey[] clientKeys = OBJECT_MAPPER.readValue(json, ClientKey[].class); + + assertNotNull(clientKeys); + } +} From 73c14f2fcce2af97cd3e864d6bcd47367d756245 Mon Sep 17 00:00:00 2001 From: Aleksandrs Ulme Date: Fri, 25 Apr 2025 17:26:52 +0800 Subject: [PATCH 2/7] Refactoring --- .../java/com/uid2/shared/model/SaltEntry.java | 8 +- .../EncryptedRotatingSaltProvider.java | 13 +-- .../store/{ => salt}/ISaltProvider.java | 2 +- .../shared/store/salt/IdHashingScheme.java | 17 ++++ .../{ => salt}/RotatingSaltProvider.java | 60 ++----------- .../shared/store/salt/SaltFileParser.java | 55 ++++++++++++ .../EncryptedRotatingSaltProviderTest.java | 2 + .../store/RotatingSaltProviderTest.java | 2 + .../shared/store/salt/SaltFileParserTest.java | 89 +++++++++++++++++++ 9 files changed, 185 insertions(+), 63 deletions(-) rename src/main/java/com/uid2/shared/store/{ => salt}/EncryptedRotatingSaltProvider.java (73%) rename src/main/java/com/uid2/shared/store/{ => salt}/ISaltProvider.java (97%) create mode 100644 src/main/java/com/uid2/shared/store/salt/IdHashingScheme.java rename src/main/java/com/uid2/shared/store/{ => salt}/RotatingSaltProvider.java (77%) create mode 100644 src/main/java/com/uid2/shared/store/salt/SaltFileParser.java create mode 100644 src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java diff --git a/src/main/java/com/uid2/shared/model/SaltEntry.java b/src/main/java/com/uid2/shared/model/SaltEntry.java index 2a658cdc..24cf7c0e 100644 --- a/src/main/java/com/uid2/shared/model/SaltEntry.java +++ b/src/main/java/com/uid2/shared/model/SaltEntry.java @@ -18,7 +18,9 @@ public String toString() { "id=" + id + ", hashedId='" + hashedId + '\'' + ", lastUpdated=" + lastUpdated + + ", salt=" + ", refreshFrom=" + refreshFrom + + ", previousSalt=" + ", currentKey=" + currentKey + ", previousKey=" + previousKey + '}'; @@ -31,7 +33,11 @@ public record KeyMaterial( ){ @Override public String toString() { - return "KeyMaterial{id=%d}".formatted(id); + return "KeyMaterial{" + + "id=" + id + + ", key=" + + ", salt=" + + '}'; } } } diff --git a/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/salt/EncryptedRotatingSaltProvider.java similarity index 73% rename from src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java rename to src/main/java/com/uid2/shared/store/salt/EncryptedRotatingSaltProvider.java index 787828be..b0322852 100644 --- a/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/salt/EncryptedRotatingSaltProvider.java @@ -1,4 +1,4 @@ -package com.uid2.shared.store; +package com.uid2.shared.store.salt; import com.uid2.shared.cloud.DownloadCloudStorage; import com.uid2.shared.model.SaltEntry; @@ -19,15 +19,8 @@ public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, Ro } @Override - protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException { + protected SaltEntry[] readInputStream(InputStream inputStream, SaltFileParser saltFileParser, Integer size) throws IOException { String decrypted = decryptInputStream(inputStream, cloudEncryptionKeyProvider, "salts"); - SaltEntry[] entries = new SaltEntry[size]; - int idx = 0; - for (String line : decrypted.split("\n")) { - final SaltEntry entry = entryBuilder.toEntry(line); - entries[idx] = entry; - idx++; - } - return entries; + return saltFileParser.parseFile(decrypted, size); } } diff --git a/src/main/java/com/uid2/shared/store/ISaltProvider.java b/src/main/java/com/uid2/shared/store/salt/ISaltProvider.java similarity index 97% rename from src/main/java/com/uid2/shared/store/ISaltProvider.java rename to src/main/java/com/uid2/shared/store/salt/ISaltProvider.java index c2a193ee..9d92e972 100644 --- a/src/main/java/com/uid2/shared/store/ISaltProvider.java +++ b/src/main/java/com/uid2/shared/store/salt/ISaltProvider.java @@ -1,4 +1,4 @@ -package com.uid2.shared.store; +package com.uid2.shared.store.salt; import com.uid2.shared.model.SaltEntry; import org.slf4j.Logger; diff --git a/src/main/java/com/uid2/shared/store/salt/IdHashingScheme.java b/src/main/java/com/uid2/shared/store/salt/IdHashingScheme.java new file mode 100644 index 00000000..d0afd2dc --- /dev/null +++ b/src/main/java/com/uid2/shared/store/salt/IdHashingScheme.java @@ -0,0 +1,17 @@ +package com.uid2.shared.store.salt; + +import org.hashids.Hashids; + +public class IdHashingScheme { + private final String prefix; + private final Hashids hasher; + + public IdHashingScheme(final String prefix, final String secret) { + this.prefix = prefix; + this.hasher = new Hashids(secret, 9); + } + + public String encode(long id) { + return prefix + this.hasher.encode(id); + } +} diff --git a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java similarity index 77% rename from src/main/java/com/uid2/shared/store/RotatingSaltProvider.java rename to src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java index 699d0773..4930d144 100644 --- a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java @@ -1,4 +1,4 @@ -package com.uid2.shared.store; +package com.uid2.shared.store.salt; import com.uid2.shared.Utils; import com.uid2.shared.attest.UidCoreClient; @@ -10,7 +10,6 @@ import lombok.Getter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.hashids.Hashids; import java.io.BufferedReader; import java.io.IOException; @@ -80,14 +79,14 @@ public long getVersion(JsonObject metadata) { public long loadContent(JsonObject metadata) throws Exception { final JsonArray salts = metadata.getJsonArray("salts"); final String firstLevelSalt = metadata.getString("first_level"); - final SaltEntryBuilder entryBuilder = new SaltEntryBuilder( + final SaltFileParser saltFileParser = new SaltFileParser( new IdHashingScheme(metadata.getString("id_prefix"), metadata.getString("id_secret"))); final Instant now = Instant.now(); final List snapshots = new ArrayList<>(); int saltCount = 0; for (int i = 0; i < salts.size(); ++i) { - final SaltSnapshot snapshot = this.loadSnapshot(salts.getJsonObject(i), firstLevelSalt, entryBuilder, now); + final SaltSnapshot snapshot = this.loadSnapshot(salts.getJsonObject(i), firstLevelSalt, saltFileParser, now); if (snapshot == null) continue; snapshots.add(snapshot); @@ -120,33 +119,26 @@ public ISaltSnapshot getSnapshot(Instant asOf) { if (!snapshot.isEffective(asOf)) break; current = snapshot; } - return current != null ? current : snapshots.get(snapshots.size() - 1); + return current != null ? current : snapshots.getLast(); } - private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltEntryBuilder entryBuilder, Instant now) throws Exception { + private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltFileParser saltFileParser, Instant now) throws Exception { final Instant defaultExpires = now.plus(365, ChronoUnit.DAYS); final Instant effective = Instant.ofEpochMilli(spec.getLong("effective")); final Instant expires = Instant.ofEpochMilli(spec.getLong("expires", defaultExpires.toEpochMilli())); final String path = spec.getString("location"); Integer size = spec.getInteger("size"); - SaltEntry[] entries = readInputStream(this.contentStreamProvider.download(path), entryBuilder, size); + SaltEntry[] entries = readInputStream(this.contentStreamProvider.download(path), saltFileParser, size); LOGGER.info("Loaded {} salts", size); return new SaltSnapshot(effective, expires, entries, firstLevelSalt); } - protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException { + protected SaltEntry[] readInputStream(InputStream inputStream, SaltFileParser saltFileParser, Integer size) throws IOException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - String line; - SaltEntry[] entries = new SaltEntry[size]; - int idx = 0; - while ((line = reader.readLine()) != null) { - final SaltEntry entry = entryBuilder.toEntry(line); - entries[idx] = entry; - idx++; - } - return entries; + var saltFileContent = reader.lines().collect(Collectors.joining(System.lineSeparator())); + return saltFileParser.parseFile(saltFileContent, size); } } @@ -207,38 +199,4 @@ public List getModifiedSince(Instant timestamp) { } } - protected static final class IdHashingScheme { - private final String prefix; - private final Hashids hasher; - - public IdHashingScheme(final String prefix, final String secret) { - this.prefix = prefix; - this.hasher = new Hashids(secret, 9); - } - - public String encode(long id) { - return prefix + this.hasher.encode(id); - } - } - - protected static final class SaltEntryBuilder { - private final IdHashingScheme idHashingScheme; - - public SaltEntryBuilder(IdHashingScheme idHashingScheme) { - this.idHashingScheme = idHashingScheme; - } - - public SaltEntry toEntry(String line) { - try { - final String[] fields = line.split(","); - final long id = Integer.parseInt(fields[0]); - final String hashedId = this.idHashingScheme.encode(id); - final long lastUpdated = Long.parseLong(fields[1]); - final String salt = fields[2]; - return new SaltEntry(id, hashedId, lastUpdated, salt, null, null, null, null); - } catch (Exception e) { - throw new RuntimeException("Trouble parsing Salt Entry " + line, e); - } - } - } } diff --git a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java new file mode 100644 index 00000000..80be214d --- /dev/null +++ b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java @@ -0,0 +1,55 @@ +package com.uid2.shared.store.salt; + +import com.uid2.shared.model.SaltEntry; + +public class SaltFileParser { + private final IdHashingScheme idHashingScheme; + + public SaltFileParser(IdHashingScheme idHashingScheme) { + this.idHashingScheme = idHashingScheme; + } + + public SaltEntry[] parseFile(String saltFileContent, Integer size) { + var entries = new SaltEntry[size]; + int idx = 0; + for (String line : saltFileContent.split("\n")) { + final SaltEntry entry = parseLine(line); + entries[idx] = entry; + idx++; + } + return entries; + } + + private SaltEntry parseLine(String line) { + try { + final String[] fields = line.split(","); + final long id = Integer.parseInt(fields[0]); + final String hashedId = this.idHashingScheme.encode(id); + final long lastUpdated = Long.parseLong(fields[1]); + final String salt = fields[2]; + + Long refreshFrom = null; + String previousSalt = null; + SaltEntry.KeyMaterial currentKey = null; + SaltEntry.KeyMaterial previousKey = null; + + if (fields.length > 3) { + refreshFrom = Long.parseLong(fields[3]); + } + if (fields.length > 4) { + previousSalt = fields[4]; + } + if (fields.length > 7) { + currentKey = new SaltEntry.KeyMaterial(Integer.parseInt(fields[5]), fields[6], fields[7]); + } + if (fields.length > 10) { + previousKey = new SaltEntry.KeyMaterial(Integer.parseInt(fields[8]), fields[9], fields[10]); + } + + return new SaltEntry(id, hashedId, lastUpdated, salt, refreshFrom, previousSalt, currentKey, previousKey); + } catch (Exception e) { + throw new RuntimeException("Trouble parsing Salt Entry " + line, e); + } + } + +} diff --git a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java index 4aa470fe..bc05ccba 100644 --- a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java +++ b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java @@ -4,6 +4,8 @@ import com.uid2.shared.encryption.AesGcm; import com.uid2.shared.model.CloudEncryptionKey; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.salt.EncryptedRotatingSaltProvider; +import com.uid2.shared.store.salt.ISaltProvider; import com.uid2.shared.store.scope.EncryptedScope; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; diff --git a/src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java index 9d47a563..20ccde31 100644 --- a/src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java +++ b/src/test/java/com/uid2/shared/store/RotatingSaltProviderTest.java @@ -1,6 +1,8 @@ package com.uid2.shared.store; import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.store.salt.ISaltProvider; +import com.uid2.shared.store.salt.RotatingSaltProvider; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.Test; diff --git a/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java b/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java new file mode 100644 index 00000000..91c7ba2f --- /dev/null +++ b/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java @@ -0,0 +1,89 @@ +package com.uid2.shared.store.salt; + +import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.model.SaltEntry.KeyMaterial; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class SaltFileParserTest { + + private final IdHashingScheme hashingScheme = new IdHashingScheme("id-prefix", "id secret"); + private final SaltFileParser parser = new SaltFileParser(hashingScheme); + + private final String hashed1 = hashingScheme.encode(1); + private final String hashed2 = hashingScheme.encode(2); + private final String hashed3 = hashingScheme.encode(3); + + @Test + void parsesSaltFileWithMinimalFields() { + var file = """ +1,100,salt1 +2,200,salt2 +"""; + SaltEntry[] actual = parser.parseFile(file, 2); + + SaltEntry[] expected = new SaltEntry[]{ + new SaltEntry(1, hashed1, 100, "salt1", null, null, null, null), + new SaltEntry(2, hashed2, 200, "salt2", null, null, null, null) + }; + + assertThat(actual).isEqualTo(expected); + } + + @Test + void parsesSaltFileWithAllFields() { + var file = """ +1,100,salt1,1000,old_salt1,10,key_1,key_salt_1,100,old_key_1,old_key_1_salt +2,200,salt2,2000,old_salt2,20,key_2,key_salt_2,200,old_key_2,old_key_2_salt +"""; + SaltEntry[] actual = parser.parseFile(file, 2); + + SaltEntry[] expected = new SaltEntry[]{ + new SaltEntry(1, hashed1, 100, "salt1", 1000L, "old_salt1", + new KeyMaterial(10, "key_1", "key_salt_1"), + new KeyMaterial(100, "old_key_1", "old_key_1_salt") + ), + new SaltEntry(2, hashed2, 200, "salt2", 2000L, "old_salt2", + new KeyMaterial(20, "key_2", "key_salt_2"), + new KeyMaterial(200, "old_key_2", "old_key_2_salt") + ) + }; + assertThat(actual).isEqualTo(expected); + } + + @Test + void parsesSaltFileWithoutEncryptionKeyFields() { + var file = """ +1,100,salt1,1000,old_salt1,10 +2,200,salt2,2000,old_salt2,20 +"""; + SaltEntry[] actual = parser.parseFile(file, 2); + + SaltEntry[] expected = new SaltEntry[]{ + new SaltEntry(1, hashed1, 100, "salt1", 1000L, "old_salt1", null, null), + new SaltEntry(2, hashed2, 200, "salt2", 2000L, "old_salt2",null, null) + }; + assertThat(actual).isEqualTo(expected); + } + + @Test + void canMixDifferentFieldsPresenceInSameFile() { + var file = """ +1,100,salt1,1000,old_salt1,10,key_1,key_salt_1,100,old_key_1,old_key_1_salt +2,200,salt2,2000,old_salt2,20 +3,300,salt3,3000 +"""; + SaltEntry[] actual = parser.parseFile(file, 3); + + SaltEntry[] expected = new SaltEntry[]{ + new SaltEntry(1, hashed1, 100, "salt1", 1000L, "old_salt1", + new KeyMaterial(10, "key_1", "key_salt_1"), + new KeyMaterial(100, "old_key_1", "old_key_1_salt") + ), + new SaltEntry(2, hashed2, 200, "salt2", 2000L, "old_salt2",null, null), + new SaltEntry(3, hashed3, 300, "salt3", null, null,null, null) + }; + + } +} From ea5da3a82caeb095faed293527efb397c749b4f1 Mon Sep 17 00:00:00 2001 From: Aleksandrs Ulme Date: Fri, 25 Apr 2025 17:43:57 +0800 Subject: [PATCH 3/7] Refactoring so we don't need to join and split the salt files into lines again --- .../com/uid2/shared/store/salt/RotatingSaltProvider.java | 4 ++-- .../java/com/uid2/shared/store/salt/SaltFileParser.java | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java index 4930d144..edf07ec6 100644 --- a/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java @@ -137,8 +137,8 @@ private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltFi protected SaltEntry[] readInputStream(InputStream inputStream, SaltFileParser saltFileParser, Integer size) throws IOException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - var saltFileContent = reader.lines().collect(Collectors.joining(System.lineSeparator())); - return saltFileParser.parseFile(saltFileContent, size); + String[] saltFileLines = reader.lines().toArray(String[]::new); + return saltFileParser.parseFile(saltFileLines, size); } } diff --git a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java index 80be214d..2e67a7c3 100644 --- a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java +++ b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java @@ -10,9 +10,14 @@ public SaltFileParser(IdHashingScheme idHashingScheme) { } public SaltEntry[] parseFile(String saltFileContent, Integer size) { + var lines = saltFileContent.split("\n"); + return parseFile(lines, size); + } + + public SaltEntry[] parseFile(String[] saltFileLines, Integer size) { var entries = new SaltEntry[size]; int idx = 0; - for (String line : saltFileContent.split("\n")) { + for (String line : saltFileLines) { final SaltEntry entry = parseLine(line); entries[idx] = entry; idx++; From f300d8038a3313e850ec6158cbf6c7ef6c9e402f Mon Sep 17 00:00:00 2001 From: Aleksandrs Ulme Date: Fri, 25 Apr 2025 17:47:14 +0800 Subject: [PATCH 4/7] Remove unnecessary file --- .../shared/store/parser/ClientParserTest.java | 174 ------------------ 1 file changed, 174 deletions(-) delete mode 100644 src/test/java/com/uid2/shared/store/parser/ClientParserTest.java diff --git a/src/test/java/com/uid2/shared/store/parser/ClientParserTest.java b/src/test/java/com/uid2/shared/store/parser/ClientParserTest.java deleted file mode 100644 index 50fe995d..00000000 --- a/src/test/java/com/uid2/shared/store/parser/ClientParserTest.java +++ /dev/null @@ -1,174 +0,0 @@ -package com.uid2.shared.store.parser; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.uid2.shared.auth.ClientKey; -import com.uid2.shared.util.Mapper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import static org.junit.jupiter.api.Assertions.*; - -class ClientParserTest { - - private ClientParser parser; - - @BeforeEach - void setUp() { - parser = new ClientParser(); - } - - @Test - void testDeserialize() throws IOException { - String json = "[\n" + - " {\n" + - " \"key\": \"UID2-C-L-999-fCXrMM.fsR3mDqAXELtWWMS+xG1s7RdgRTMqdOH2qaAo=\",\n" + - " \"secret\": \"DzBzbjTJcYL0swDtFs2krRNu+g1Eokm2tBU4dEuD0Wk=\",\n" + - " \"name\": \"Special\",\n" + - " \"contact\": \"Special\",\n" + - " \"created\": 1701210253,\n" + - " \"roles\": [\n" + - " \"MAPPER\",\n" + - " \"GENERATOR\",\n" + - " \"ID_READER\",\n" + - " \"SHARER\",\n" + - " \"OPTOUT\"\n" + - " ],\n" + - " \"disabled\": false,\n" + - " \"site_id\": 999,\n" + - " \"key_hash\": \"fsSGnDxa/V9eJZ9Tas+dowwyO/X1UsC68RN9qM2xUu9ZOaKEOv9EVd7pkt3As/nE5B6TRu0PzK+IDzSQhD1+rw==\",\n" + - " \"key_salt\": \"jySwjjqo9O6OU01OWujBWC6xZNpBqRTk5H7K2borcFA=\",\n" + - " \"key_id\": \"UID2-C-L-999-fCXrM\"\n" + - " },\n" + - " {\n" + - " \"key\": \"LOCALbGlvbnVuZGVybGluZXdpbmRzY2FyZWRzb2Z0ZGVzZXI=\",\n" + - " \"secret\": \"c3RlZXBzcGVuZHNsb3BlZnJlcXVlbnRseWRvd2lkZWM=\",\n" + - " \"name\": \"Special (Legacy Key)\",\n" + - " \"contact\": \"Special (Legacy Key)\",\n" + - " \"created\": 1701210253,\n" + - " \"roles\": [\n" + - " \"MAPPER\",\n" + - " \"GENERATOR\",\n" + - " \"ID_READER\",\n" + - " \"SHARER\",\n" + - " \"OPTOUT\"\n" + - " ],\n" + - " \"disabled\": false,\n" + - " \"site_id\": 999,\n" + - " \"key_hash\": \"OPIi+MWKNz41wzu+atsBLAtXDTLFhLWPq5mCxA3L8anX+fjKaVDAf55D98BSLAh/EFQE/xTQyo/YK5snPS8ivA==\",\n" + - " \"key_salt\": \"FpgbvHGqpVhi3I/b8/9HnguiycUzb2y+KsdicPpNLJI=\",\n" + - " \"key_id\": \"LOCALbGlvb\"\n" + - " },\n" + - " {\n" + - " \"key\": \"UID2-C-L-123-t32pCM.5NCX1E94UgOd2f8zhsKmxzCoyhXohHYSSWR8U=\",\n" + - " \"secret\": \"FsD4bvtjMkeTonx6HvQp6u0EiI1ApGH4pIZzZ5P7UcQ=\",\n" + - " \"name\": \"DSP\",\n" + - " \"contact\": \"DSP\",\n" + - " \"created\": 1609459200,\n" + - " \"roles\": [\n" + - " \"ID_READER\"\n" + - " ],\n" + - " \"disabled\": false,\n" + - " \"site_id\": 123,\n" + - " \"key_hash\": \"vVb/MjymmYAE3L6as5t1DCjbts4bT2wZh4V4iAagOAe97jthFmT4YAb6gGVfEs4Pq+CqNPgz+X338RNRa8NOlA==\",\n" + - " \"key_salt\": \"G36g+KxlS+z5NwSXUOnBtc9yJKHECvXgjbS13X5A7rw=\",\n" + - " \"key_id\": \"UID2-C-L-123-t32pC\"\n" + - " },\n" + - " {\n" + - " \"key\": \"UID2-C-L-124-H8VwqX.l2G4TCuUWYAqdqkeG/UqtFoPEoXirKn4kHWxc=\",\n" + - " \"secret\": \"NcMgi6Y8C80SlxvV7pYlfcvEIo+2b0508tYQ3pKK8HM=\",\n" + - " \"name\": \"Publisher\",\n" + - " \"contact\": \"Publisher\",\n" + - " \"created\": 1609459200,\n" + - " \"roles\": [\n" + - " \"GENERATOR\",\n" + - " \"ID_READER\"\n" + - " ],\n" + - " \"disabled\": false,\n" + - " \"site_id\": 124,\n" + - " \"key_hash\": \"uA1aRDR9owk53W47zZpI6cS/bRVgKm4ggRvr9m0pz+I/5IzQcIQqfurm1Ors96r8Q2xC8GZVG3spwR/H89rQmA==\",\n" + - " \"key_salt\": \"rSwnZ5aKauMLPLMHvvH25C1LU5MdJv5+fjQ5/Yy5hP0=\",\n" + - " \"key_id\": \"UID2-C-L-124-H8Vwq\"\n" + - " },\n" + - " {\n" + - " \"key\": \"UID2-C-L-125-E5w9L8.T5og45yFqQeoj4ubh9IVqXcaSVwk7A5XyG958=\",\n" + - " \"secret\": \"3YAgjckHGQyBgSFj64ZsLf8WlUnvrQhLKuG7rljp6W4=\",\n" + - " \"name\": \"Data Provider\",\n" + - " \"contact\": \"Data Provider\",\n" + - " \"created\": 1609459200,\n" + - " \"roles\": [\n" + - " \"MAPPER\",\n" + - " \"SHARER\"\n" + - " ],\n" + - " \"disabled\": false,\n" + - " \"site_id\": 125,\n" + - " \"key_hash\": \"0GFyfie9Vz7INDxG5gT3MeHOnrXdIy/H9I5OrTZHx/cX5zToF8BngbseREbeEG7xH3KFs5TfSdwI5N/OWzYrGQ==\",\n" + - " \"key_salt\": \"taG4CBJ1F4aWwL8XwyilKl9WzYSWoG9RjvB4BGqf0/w=\",\n" + - " \"key_id\": \"UID2-C-L-125-E5w9L\"\n" + - " },\n" + - " {\n" + - " \"key\": \"UID2-C-L-126-GMP9tD.jn2o3vmXzn7vmKRlTT6BEUrPUaPJuQmDBdq38=\",\n" + - " \"secret\": \"1ydXM0rEj+ROUazVpZjNZOGu2T5+f/BIiBfnK8xGh/A=\",\n" + - " \"name\": \"Advertiser\",\n" + - " \"contact\": \"Advertiser\",\n" + - " \"created\": 1609459200,\n" + - " \"roles\": [\n" + - " \"MAPPER\",\n" + - " \"SHARER\"\n" + - " ],\n" + - " \"disabled\": false,\n" + - " \"site_id\": 126,\n" + - " \"key_hash\": \"lDl6HiO7hVdXmHm+gogCZmiCzhWcDLVIxBItR+0GMBWpRxleIr2HQG2oAHVKYd63AKeMZGwh5svbbJ6Gu0RUMQ==\",\n" + - " \"key_salt\": \"FH6UNMUCJKday6FWTLUtmg9Hwh4Rd/HhenfjtRyaAEI=\",\n" + - " \"key_id\": \"UID2-C-L-126-GMP9t\"\n" + - " },\n" + - " {\n" + - " \"key\": \"UID2-C-L-127-aHVydH.JlZnVzZWRmYXN0ZW5lbXliYWdpZGVudGl0eWU=\",\n" + - " \"secret\": \"c3VpdG9wcG9zaXRlaW1hZ2Vsb29rc2ltcGxlc3RmaXI=\",\n" + - " \"name\": \"OptOut\",\n" + - " \"contact\": \"OptOut\",\n" + - " \"created\": 1609459200,\n" + - " \"roles\": [\n" + - " \"OPTOUT\"\n" + - " ],\n" + - " \"disabled\": false,\n" + - " \"site_id\": 127,\n" + - " \"key_hash\": \"BEEnHVPHwbMYUtZk/N6jjnN04U7xpu6hV5yF4Nn2Zw9pigD43JLZdEleRW/Mz7LAQfYtLTJk768J8WK6F4Ku/Q==\",\n" + - " \"key_salt\": \"cw9wfsevy1xRiPys5JkBSTPHmTickWkFWQ0zrIF2C60=\",\n" + - " \"key_id\": \"UID2-C-L-127-aHVyd\"\n" + - " },\n" + - " {\n" + - " \"key\": \"UID2-C-L-1000-qxpBsF.ibeCDBpD2bq4Zm7inDacGioUk1aaLeNJrabow=\",\n" + - " \"secret\": \"VT7+t0G/RVueMuVZAL56I2c3JJFSYQfhbu8yo0V/Tds=\",\n" + - " \"name\": \"Legacy Site Client\",\n" + - " \"contact\": \"Legacy Site Client\",\n" + - " \"created\": 1609459200,\n" + - " \"roles\": [\n" + - " \"MAPPER\",\n" + - " \"GENERATOR\",\n" + - " \"ID_READER\",\n" + - " \"SHARER\",\n" + - " \"OPTOUT\"\n" + - " ],\n" + - " \"disabled\": false,\n" + - " \"site_id\": 1000,\n" + - " \"key_hash\": \"654FIeR8DFtLi5AC8RXvwfBQ1b9J8L+dVyJUxoTSCpMBQ3z937CxQ1fp40fHIs9SbQPnivBMV5s+TdDMZXZqgQ==\",\n" + - " \"key_salt\": \"huTnT+HyINotMK0W00Gy7VGaQT9XR0KaxZTBvVuTCF0=\",\n" + - " \"key_id\": \"UID2-C-L-1000-qxpBs\"\n" + - " }\n" + - "]"; - ObjectMapper OBJECT_MAPPER = new ObjectMapper() -// .configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, true) - .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);//Mapper.getInstance(); - ClientKey[] clientKeys = OBJECT_MAPPER.readValue(json, ClientKey[].class); - - assertNotNull(clientKeys); - } -} From 04843118131ae84d0c13f659b65c3dc3b685ced6 Mon Sep 17 00:00:00 2001 From: Aleksandrs Ulme Date: Fri, 25 Apr 2025 17:57:12 +0800 Subject: [PATCH 5/7] Extra test and refactoring --- .../store/salt/RotatingSaltProvider.java | 2 +- .../shared/store/salt/SaltFileParser.java | 4 +-- .../shared/store/salt/SaltFileParserTest.java | 30 +++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java index edf07ec6..d9a0d2e9 100644 --- a/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java @@ -138,7 +138,7 @@ private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltFi protected SaltEntry[] readInputStream(InputStream inputStream, SaltFileParser saltFileParser, Integer size) throws IOException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { String[] saltFileLines = reader.lines().toArray(String[]::new); - return saltFileParser.parseFile(saltFileLines, size); + return saltFileParser.parseFileLines(saltFileLines, size); } } diff --git a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java index 2e67a7c3..56ecd1f3 100644 --- a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java +++ b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java @@ -11,10 +11,10 @@ public SaltFileParser(IdHashingScheme idHashingScheme) { public SaltEntry[] parseFile(String saltFileContent, Integer size) { var lines = saltFileContent.split("\n"); - return parseFile(lines, size); + return parseFileLines(lines, size); } - public SaltEntry[] parseFile(String[] saltFileLines, Integer size) { + public SaltEntry[] parseFileLines(String[] saltFileLines, Integer size) { var entries = new SaltEntry[size]; int idx = 0; for (String line : saltFileLines) { diff --git a/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java b/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java index 91c7ba2f..9d4975b1 100644 --- a/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java +++ b/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java @@ -52,6 +52,36 @@ void parsesSaltFileWithAllFields() { assertThat(actual).isEqualTo(expected); } + @Test + void parsesSaltFileWithNullValuesForNewFields() { + var file = """ +1,100,salt1,,,,,,,, +2,200,salt2,,,,,,,, +"""; + SaltEntry[] actual = parser.parseFile(file, 2); + + SaltEntry[] expected = new SaltEntry[]{ + new SaltEntry(1, hashed1, 100, "salt1", null, null,null, null), + new SaltEntry(2, hashed2, 200, "salt2", null, null,null, null) + }; + assertThat(actual).isEqualTo(expected); + } + + @Test + void parsesSaltFileWithNullValuesForKeyFields() { + var file = """ +1,100,salt1,1000,old_salt1,,,,,, +2,200,salt2,2000,old_salt2,,,,,, +"""; + SaltEntry[] actual = parser.parseFile(file, 2); + + SaltEntry[] expected = new SaltEntry[]{ + new SaltEntry(1, hashed1, 100, "salt1", 1000L, "old_salt1",null, null), + new SaltEntry(2, hashed2, 200, "salt2", 2000L, "old_salt2",null, null) + }; + assertThat(actual).isEqualTo(expected); + } + @Test void parsesSaltFileWithoutEncryptionKeyFields() { var file = """ From b03b9482d261d6cfeda85227335c3d4da9529ed7 Mon Sep 17 00:00:00 2001 From: Aleksandrs Ulme Date: Mon, 28 Apr 2025 11:17:07 +0800 Subject: [PATCH 6/7] Address some feedback --- src/main/java/com/uid2/shared/model/SaltEntry.java | 8 ++++---- .../uid2/shared/store/salt/RotatingSaltProvider.java | 10 +++++----- .../com/uid2/shared/store/salt/SaltFileParser.java | 3 +++ .../java/com/uid2/shared/secret/KeyHasherTest.java | 4 ++-- .../com/uid2/shared/secure/AttestationTokenTest.java | 2 +- .../store/EncryptedRotatingSaltProviderTest.java | 2 +- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/uid2/shared/model/SaltEntry.java b/src/main/java/com/uid2/shared/model/SaltEntry.java index 24cf7c0e..9405e8fb 100644 --- a/src/main/java/com/uid2/shared/model/SaltEntry.java +++ b/src/main/java/com/uid2/shared/model/SaltEntry.java @@ -4,7 +4,7 @@ public record SaltEntry( long id, String hashedId, long lastUpdated, - String salt, + String currentSalt, Long refreshFrom, // needs to be nullable until V3 Identity Map is fully rolled out String previousSalt, @@ -18,7 +18,7 @@ public String toString() { "id=" + id + ", hashedId='" + hashedId + '\'' + ", lastUpdated=" + lastUpdated + - ", salt=" + + ", currentSalt=" + ", refreshFrom=" + refreshFrom + ", previousSalt=" + ", currentKey=" + currentKey + @@ -30,13 +30,13 @@ public record KeyMaterial( int id, String key, String salt - ){ + ) { @Override public String toString() { return "KeyMaterial{" + "id=" + id + ", key=" + - ", salt=" + + ", currentSalt=" + '}'; } } diff --git a/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java index d9a0d2e9..84f90f8e 100644 --- a/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java @@ -40,9 +40,9 @@ ] } - 2. salt file format - , , - 9000099,1614556800000,salt + 2. currentSalt file format + , , + 9000099,1614556800000,currentSalt */ public class RotatingSaltProvider implements ISaltProvider, IMetadataVersionedStore { private static final Logger LOGGER = LoggerFactory.getLogger(RotatingSaltProvider.class); @@ -159,10 +159,10 @@ public SaltSnapshot(Instant effective, Instant expires, SaltEntry[] entries, Str this.entries = entries; this.firstLevelSalt = firstLevelSalt; if (entries.length == 1_048_576) { - LOGGER.info("Total salt entries 1 million, {}, special production salt entry indexer", entries.length); + LOGGER.info("Total currentSalt entries 1 million, {}, special production currentSalt entry indexer", entries.length); this.saltEntryIndexer = MILLION_ENTRY_INDEXER; } else { - LOGGER.warn("Total salt entries {}, using slower mod-based indexer", entries.length); + LOGGER.warn("Total currentSalt entries {}, using slower mod-based indexer", entries.length); this.saltEntryIndexer = MOD_BASED_INDEXER; } } diff --git a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java index 56ecd1f3..ae2b8876 100644 --- a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java +++ b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java @@ -38,6 +38,9 @@ private SaltEntry parseLine(String line) { SaltEntry.KeyMaterial currentKey = null; SaltEntry.KeyMaterial previousKey = null; + // TODO: The fields below should stop being optional once the refresh from, previous salt + // and refreshable UIDs features get rolled out in production. We can remove them one by one as necessary. + // AU, 2025/04/28 if (fields.length > 3) { refreshFrom = Long.parseLong(fields[3]); } diff --git a/src/test/java/com/uid2/shared/secret/KeyHasherTest.java b/src/test/java/com/uid2/shared/secret/KeyHasherTest.java index 4fadc11b..61cc3610 100644 --- a/src/test/java/com/uid2/shared/secret/KeyHasherTest.java +++ b/src/test/java/com/uid2/shared/secret/KeyHasherTest.java @@ -11,7 +11,7 @@ public class KeyHasherTest { @Test public void hashKey_returnsKnownHash_withGivenSalt() { KeyHasher hasher = new KeyHasher(); - byte[] hashedBytes = hasher.hashKey("test-key", "test-salt".getBytes(StandardCharsets.UTF_8)); + byte[] hashedBytes = hasher.hashKey("test-key", "test-currentSalt".getBytes(StandardCharsets.UTF_8)); assertEquals("hzXFALLdI9ji4ajnzhWdbEQNci+kAoA40Ie6X7bEyjIvMFbhQfYZC1sTPeK+14QM+Ox2a6wJ0U2fLzqnoUgCbQ==", Base64.getEncoder().encodeToString(hashedBytes)); } @@ -22,7 +22,7 @@ public void hashKey_returnsNewHashEverytime_withRandomSalt() { KeyHashResult result2 = hasher.hashKey("test-key"); assertAll( - "hashKey returns new hash every time with random salt", + "hashKey returns new hash every time with random currentSalt", () -> assertNotEquals(result1.getHash(), result2.getHash()), () -> assertNotEquals(result1.getSalt(), result2.getSalt()) ); diff --git a/src/test/java/com/uid2/shared/secure/AttestationTokenTest.java b/src/test/java/com/uid2/shared/secure/AttestationTokenTest.java index 039351f2..8930ea64 100644 --- a/src/test/java/com/uid2/shared/secure/AttestationTokenTest.java +++ b/src/test/java/com/uid2/shared/secure/AttestationTokenTest.java @@ -18,7 +18,7 @@ public class AttestationTokenTest { private static final String ENCRYPTION_KEY = "attestation-token-secret"; - private static final String SALT = "attestation-token-salt"; + private static final String SALT = "attestation-token-currentSalt"; private final Clock clock = mock(Clock.class); diff --git a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java index bc05ccba..e25b6025 100644 --- a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java +++ b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java @@ -159,7 +159,7 @@ public void loadSaltSingleVersion1mil() throws Exception { final String effectiveTimeString = String.valueOf(generatedTime.getEpochSecond() * 1000L); StringBuilder salts = new StringBuilder(); for (int i = 0; i < 1000000; i++) { - salts.append(i).append(",").append(effectiveTimeString).append(",").append("salt-string").append("\n"); + salts.append(i).append(",").append(effectiveTimeString).append(",").append("currentSalt-string").append("\n"); } when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) From c745404c5203e3328f67b9db98fdeec5b2c8a171 Mon Sep 17 00:00:00 2001 From: Aleksandrs Ulme Date: Mon, 28 Apr 2025 12:21:17 +0800 Subject: [PATCH 7/7] Fix test --- .../com/uid2/shared/store/salt/SaltFileParser.java | 12 ++++++------ .../java/com/uid2/shared/secret/KeyHasherTest.java | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java index ae2b8876..20ce9b78 100644 --- a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java +++ b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java @@ -16,16 +16,16 @@ public SaltEntry[] parseFile(String saltFileContent, Integer size) { public SaltEntry[] parseFileLines(String[] saltFileLines, Integer size) { var entries = new SaltEntry[size]; - int idx = 0; + int lineNumber = 0; for (String line : saltFileLines) { - final SaltEntry entry = parseLine(line); - entries[idx] = entry; - idx++; + final SaltEntry entry = parseLine(line, lineNumber); + entries[lineNumber] = entry; + lineNumber++; } return entries; } - private SaltEntry parseLine(String line) { + private SaltEntry parseLine(String line, int lineNumber) { try { final String[] fields = line.split(","); final long id = Integer.parseInt(fields[0]); @@ -56,7 +56,7 @@ private SaltEntry parseLine(String line) { return new SaltEntry(id, hashedId, lastUpdated, salt, refreshFrom, previousSalt, currentKey, previousKey); } catch (Exception e) { - throw new RuntimeException("Trouble parsing Salt Entry " + line, e); + throw new RuntimeException("Trouble parsing Salt Entry, line number: " + lineNumber, e); } } diff --git a/src/test/java/com/uid2/shared/secret/KeyHasherTest.java b/src/test/java/com/uid2/shared/secret/KeyHasherTest.java index 61cc3610..a47b7026 100644 --- a/src/test/java/com/uid2/shared/secret/KeyHasherTest.java +++ b/src/test/java/com/uid2/shared/secret/KeyHasherTest.java @@ -11,7 +11,7 @@ public class KeyHasherTest { @Test public void hashKey_returnsKnownHash_withGivenSalt() { KeyHasher hasher = new KeyHasher(); - byte[] hashedBytes = hasher.hashKey("test-key", "test-currentSalt".getBytes(StandardCharsets.UTF_8)); + byte[] hashedBytes = hasher.hashKey("test-key", "test-salt".getBytes(StandardCharsets.UTF_8)); assertEquals("hzXFALLdI9ji4ajnzhWdbEQNci+kAoA40Ie6X7bEyjIvMFbhQfYZC1sTPeK+14QM+Ox2a6wJ0U2fLzqnoUgCbQ==", Base64.getEncoder().encodeToString(hashedBytes)); }