From a041612f81a118bcee77709ec7b4259c47dd3369 Mon Sep 17 00:00:00 2001 From: Will Date: Wed, 8 Jul 2026 14:32:35 -0400 Subject: [PATCH] Matchmaking quality gate (dark release) --- .../java/ti4/settings/GlobalSettings.java | 2 + .../matchmaking/queue/MatchDescriber.java | 12 +- .../queue/MatchQualityCalculator.java | 33 ++++- .../matchmaking/queue/MatchQualityGate.java | 44 +++++++ .../matchmaking/queue/MatchmakingGrouper.java | 13 ++ .../queue/MatchQualityCalculatorTest.java | 117 ++++++++++++++++++ .../queue/MatchQualityGateTest.java | 61 +++++++++ 7 files changed, 278 insertions(+), 4 deletions(-) create mode 100644 src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityGate.java create mode 100644 src/test/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityCalculatorTest.java create mode 100644 src/test/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityGateTest.java diff --git a/src/main/java/ti4/settings/GlobalSettings.java b/src/main/java/ti4/settings/GlobalSettings.java index 5c7f760cac..c8247b81b6 100644 --- a/src/main/java/ti4/settings/GlobalSettings.java +++ b/src/main/java/ti4/settings/GlobalSettings.java @@ -30,6 +30,8 @@ public enum ImplementedSettings { ALLOW_GAME_CREATION, SQLITE_PERSISTENCE_DISABLED, // Temporarily no-op auxiliary SQLite/JDBC reads and writes during migration READY_TO_RECEIVE_COMMANDS, // Whether the bot is ready to receive commands + MATCHMAKING_QUALITY_GATE_ENFORCED, // When true, block matches whose normalized quality is below the + // wait-relaxed threshold (default false = shadow/log-only) BOT_LOG_WEBHOOK_URL; // Webhook URL to send rogue bot log messages to @Override diff --git a/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchDescriber.java b/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchDescriber.java index 1891176745..eefce94672 100644 --- a/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchDescriber.java +++ b/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchDescriber.java @@ -43,11 +43,19 @@ static String setupBody(MatchedGame game) { static void logFormedMatch( MatchedGame game, Map playerMatchmakingData) { - double matchQuality = MatchQualityCalculator.matchQuality(game.members(), playerMatchmakingData); + MatchQualityCalculator.Result quality = MatchQualityCalculator.calculate(game.members(), playerMatchmakingData); + Duration maxWait = MatchQualityGate.maxQueueWait(game.members(), playerMatchmakingData); + double threshold = MatchQualityGate.currentThreshold(maxWait); + boolean wouldBlock = MatchQualityGate.wouldBlock(quality.normalized(), maxWait); StringBuilder log = new StringBuilder(" Matchmade: ") .append("\n``` • ") .append(threadTitle(game)) - .append(" — match quality %.1f%%".formatted(matchQuality * 100)); + .append(" — match quality %.1f%% · balance %.0f%% (gate %.0f%%%s)" + .formatted( + quality.raw() * 100, + quality.normalized() * 100, + threshold * 100, + wouldBlock ? " — WOULD BLOCK" : "")); for (MatchmakingQueueMember member : game.members()) { log.append("\n • ").append(describePlayer(playerMatchmakingData.get(member))); } diff --git a/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityCalculator.java b/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityCalculator.java index 15a841d12a..4cdaba2021 100644 --- a/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityCalculator.java +++ b/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityCalculator.java @@ -3,6 +3,7 @@ import de.gesundkrank.jskills.GameInfo; import de.gesundkrank.jskills.ITeam; import de.gesundkrank.jskills.Player; +import de.gesundkrank.jskills.Rating; import de.gesundkrank.jskills.Team; import de.gesundkrank.jskills.trueskill.FactorGraphTrueSkillCalculator; import java.util.ArrayList; @@ -10,19 +11,47 @@ import java.util.Map; import lombok.experimental.UtilityClass; +/** + * TrueSkill's raw match quality (draw probability) penalizes rating uncertainty: a perfectly balanced + * table of new players (high sigma) scores a few percent while the same table of calibrated players + * scores far higher. To judge balance independently of uncertainty, we also compute the ideal quality + * for the same players with all means equalized (keeping their real sigmas) — the mean-imbalance term + * becomes 1 while the mean-independent uncertainty term is unchanged. The ratio of actual to ideal is a + * pure balance score in (0, 1], equal to 1.0 for perfectly balanced groups regardless of sigma or + * player count. + */ @UtilityClass class MatchQualityCalculator { private static final GameInfo GAME_INFO = GameInfo.getDefaultGameInfo(); private static final FactorGraphTrueSkillCalculator CALCULATOR = new FactorGraphTrueSkillCalculator(); + // Guards the normalization against ideal-quality underflow for very large groups. + private static final double MIN_IDEAL_QUALITY = 1.0e-9; - static double matchQuality( + record Result(double raw, double normalized) {} + + static Result calculate( List members, Map matchmakingData) { + double raw = quality(members, matchmakingData, false); + double ideal = quality(members, matchmakingData, true); + double normalized = Math.min(1.0, raw / Math.max(ideal, MIN_IDEAL_QUALITY)); + return new Result(raw, normalized); + } + + private static double quality( + List members, + Map matchmakingData, + boolean equalizeMeans) { List teams = new ArrayList<>(); for (MatchmakingQueueMember member : members) { PlayerMatchmakingData data = matchmakingData.get(member); + Rating rating = equalizeMeans + ? new Rating( + GAME_INFO.getDefaultRating().getMean(), + data.rating().getStandardDeviation()) + : data.rating(); Team team = new Team(); - team.addPlayer(new Player<>(data.userId()), data.rating()); + team.addPlayer(new Player<>(data.userId()), rating); teams.add(team); } return CALCULATOR.calculateMatchQuality(GAME_INFO, teams); diff --git a/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityGate.java b/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityGate.java new file mode 100644 index 0000000000..64b0c1b418 --- /dev/null +++ b/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityGate.java @@ -0,0 +1,44 @@ +package ti4.spring.service.statistics.matchmaking.queue; + +import java.time.Duration; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import lombok.experimental.UtilityClass; +import ti4.settings.GlobalSettings; + +/** + * Gates match formation on normalized match quality (see {@link MatchQualityCalculator}). The threshold + * relaxes as the group's longest-waiting member waits, so no one is starved: at the default constants the + * gate is fully open after 20 hours, before the 24-hour near-match trigger. Shadow mode by default: + * quality is logged by MatchDescriber but never blocks formation until the global setting is enabled. + */ +@UtilityClass +class MatchQualityGate { + + private static final double NORMALIZED_QUALITY_STARTING_THRESHOLD = 0.5; + private static final double NORMALIZED_QUALITY_RELAXATION_PER_WINDOW = 0.05; + private static final Duration QUALITY_THRESHOLD_DECAY_INTERVAL = Duration.ofHours(2); + + static boolean isEnforced() { + return GlobalSettings.ImplementedSettings.MATCHMAKING_QUALITY_GATE_ENFORCED.getAsBoolean(false); + } + + static double currentThreshold(Duration maxWait) { + long intervalsElapsed = maxWait.toMinutes() / QUALITY_THRESHOLD_DECAY_INTERVAL.toMinutes(); + return Math.max( + 0, NORMALIZED_QUALITY_STARTING_THRESHOLD - intervalsElapsed * NORMALIZED_QUALITY_RELAXATION_PER_WINDOW); + } + + static boolean wouldBlock(double normalizedQuality, Duration maxWait) { + return normalizedQuality < currentThreshold(maxWait); + } + + static Duration maxQueueWait( + List members, Map matchmakingData) { + return members.stream() + .map(member -> matchmakingData.get(member).queueWait()) + .max(Comparator.naturalOrder()) + .orElse(Duration.ZERO); + } +} diff --git a/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchmakingGrouper.java b/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchmakingGrouper.java index 3a89ed5cf4..0eaa7592d7 100644 --- a/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchmakingGrouper.java +++ b/src/main/java/ti4/spring/service/statistics/matchmaking/queue/MatchmakingGrouper.java @@ -150,12 +150,25 @@ private static List> formGroupsOfSize( searching.remove(seed); continue; } + if (MatchQualityGate.isEnforced() && isBlockedByQualityGate(members, playerMatchmakingData)) { + // Group balance is too poor for its current wait; drop the seed like an incomplete group. + searching.remove(seed); + continue; + } groups.add(group); searching.removeAll(group); } return groups; } + private static boolean isBlockedByQualityGate( + List members, + Map playerMatchmakingData) { + double normalized = + MatchQualityCalculator.calculate(members, playerMatchmakingData).normalized(); + return MatchQualityGate.wouldBlock(normalized, MatchQualityGate.maxQueueWait(members, playerMatchmakingData)); + } + private static void collectGames( List groupParties, GameConfig config, diff --git a/src/test/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityCalculatorTest.java b/src/test/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityCalculatorTest.java new file mode 100644 index 0000000000..742f8d5dbd --- /dev/null +++ b/src/test/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityCalculatorTest.java @@ -0,0 +1,117 @@ +package ti4.spring.service.statistics.matchmaking.queue; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +import de.gesundkrank.jskills.GameInfo; +import de.gesundkrank.jskills.Rating; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class MatchQualityCalculatorTest { + + private static final Rating NEW_PLAYER_RATING = + GameInfo.getDefaultGameInfo().getDefaultRating(); + // Confident sigma, as for calibrated players. + private static final double CONFIDENT_SIGMA = 1.5; + + @Test + void balancedAllNewPlayersScoreNearOne() { + // The motivating case: six brand-new players are perfectly balanced, yet raw quality is tiny + // because TrueSkill penalizes their uncertainty. + Fixture fixture = fixture( + NEW_PLAYER_RATING, + NEW_PLAYER_RATING, + NEW_PLAYER_RATING, + NEW_PLAYER_RATING, + NEW_PLAYER_RATING, + NEW_PLAYER_RATING); + + MatchQualityCalculator.Result result = MatchQualityCalculator.calculate(fixture.members(), fixture.data()); + + assertThat(result.normalized()).isCloseTo(1.0, within(1e-6)); + assertThat(result.raw()).isLessThan(0.2); + } + + @Test + void balancedCalibratedPlayersScoreNearOne() { + Fixture fixture = fixture( + new Rating(30, CONFIDENT_SIGMA), + new Rating(30, CONFIDENT_SIGMA), + new Rating(30, CONFIDENT_SIGMA), + new Rating(30, CONFIDENT_SIGMA)); + + MatchQualityCalculator.Result result = MatchQualityCalculator.calculate(fixture.members(), fixture.data()); + + assertThat(result.normalized()).isCloseTo(1.0, within(1e-6)); + } + + @Test + void equalMeansMixedSigmasScoreNearOne() { + Fixture fixture = fixture(new Rating(25, 25.0 / 3), new Rating(25, 4.0), new Rating(25, CONFIDENT_SIGMA)); + + MatchQualityCalculator.Result result = MatchQualityCalculator.calculate(fixture.members(), fixture.data()); + + assertThat(result.normalized()).isCloseTo(1.0, within(1e-6)); + } + + @Test + void lopsidedGroupScoresLow() { + Fixture fixture = fixture( + new Rating(20, CONFIDENT_SIGMA), new Rating(25, CONFIDENT_SIGMA), new Rating(40, CONFIDENT_SIGMA)); + + MatchQualityCalculator.Result result = MatchQualityCalculator.calculate(fixture.members(), fixture.data()); + + assertThat(result.normalized()).isLessThan(0.3); + } + + @Test + void normalizedQualityIsInvariantAcrossPlayerCounts() { + Rating rating = new Rating(30, CONFIDENT_SIGMA); + Fixture threePlayers = fixture(rating, rating, rating); + Fixture sixPlayers = fixture(rating, rating, rating, rating, rating, rating); + + MatchQualityCalculator.Result threePlayerResult = + MatchQualityCalculator.calculate(threePlayers.members(), threePlayers.data()); + MatchQualityCalculator.Result sixPlayerResult = + MatchQualityCalculator.calculate(sixPlayers.members(), sixPlayers.data()); + + assertThat(threePlayerResult.normalized()).isCloseTo(1.0, within(1e-6)); + assertThat(sixPlayerResult.normalized()).isCloseTo(1.0, within(1e-6)); + // Raw quality degrades with player count even for identical players; normalized does not. + assertThat(sixPlayerResult.raw()).isLessThan(threePlayerResult.raw()); + } + + private static Fixture fixture(Rating... ratings) { + List members = new ArrayList<>(); + Map data = new HashMap<>(); + for (int i = 0; i < ratings.length; i++) { + String userId = "user" + i; + MatchmakingQueueMember member = new MatchmakingQueueMember(); + member.setUserId(userId); + members.add(member); + data.put( + member, + new PlayerMatchmakingData( + userId, + List.of(), + List.of(), + ratings[i], + Set.of(), + 0, + Set.of(), + Duration.ZERO, + false, + List.of())); + } + return new Fixture(members, data); + } + + private record Fixture( + List members, Map data) {} +} diff --git a/src/test/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityGateTest.java b/src/test/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityGateTest.java new file mode 100644 index 0000000000..3e552cc946 --- /dev/null +++ b/src/test/java/ti4/spring/service/statistics/matchmaking/queue/MatchQualityGateTest.java @@ -0,0 +1,61 @@ +package ti4.spring.service.statistics.matchmaking.queue; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +import de.gesundkrank.jskills.Rating; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class MatchQualityGateTest { + + @Test + void thresholdStartsAtBase() { + assertThat(MatchQualityGate.currentThreshold(Duration.ZERO)).isCloseTo(0.5, within(1e-9)); + } + + @Test + void thresholdRelaxesPerTwoHourWindow() { + assertThat(MatchQualityGate.currentThreshold(Duration.ofHours(2))).isCloseTo(0.45, within(1e-9)); + assertThat(MatchQualityGate.currentThreshold(Duration.ofHours(3))).isCloseTo(0.45, within(1e-9)); + assertThat(MatchQualityGate.currentThreshold(Duration.ofHours(4))).isCloseTo(0.40, within(1e-9)); + } + + @Test + void thresholdNeverGoesNegative() { + assertThat(MatchQualityGate.currentThreshold(Duration.ofHours(100))).isEqualTo(0.0); + } + + @Test + void wouldBlockComparesAgainstRelaxedThreshold() { + assertThat(MatchQualityGate.wouldBlock(0.48, Duration.ZERO)).isTrue(); + assertThat(MatchQualityGate.wouldBlock(0.48, Duration.ofHours(2))).isFalse(); + } + + @Test + void maxQueueWaitPicksLongestMemberWait() { + MatchmakingQueueMember shortWaiter = member("a"); + MatchmakingQueueMember longWaiter = member("b"); + Map data = Map.of( + shortWaiter, playerData("a", Duration.ofHours(1)), + longWaiter, playerData("b", Duration.ofHours(3))); + + Duration maxWait = MatchQualityGate.maxQueueWait(List.of(shortWaiter, longWaiter), data); + + assertThat(maxWait).isEqualTo(Duration.ofHours(3)); + } + + private static MatchmakingQueueMember member(String userId) { + MatchmakingQueueMember member = new MatchmakingQueueMember(); + member.setUserId(userId); + return member; + } + + private static PlayerMatchmakingData playerData(String userId, Duration queueWait) { + return new PlayerMatchmakingData( + userId, List.of(), List.of(), new Rating(25, 1.5), Set.of(), 0, Set.of(), queueWait, false, List.of()); + } +}