use phoneme matching

Author: Alex-Wengg

Sources/FluidAudio/ASR/AsrManager.swift

@@ -33,6 +33,9 @@ public final class AsrManager {
     }
     #endif
 
+    /// Custom vocabulary context for context biasing
+    private var customVocabulary: CustomVocabularyContext?
+
     // TODO:: the decoder state should be moved higher up in the API interface
     internal var microphoneDecoderState: TdtDecoderState
     internal var systemDecoderState: TdtDecoderState
@@ -92,6 +95,25 @@ public final class AsrManager {
         logger.info("AsrManager initialized successfully with provided models")
     }
 
+    /// Update custom vocabulary for context biasing without reinitializing ASR
+    /// - Parameter vocabulary: New custom vocabulary context, or nil to disable context biasing
+    public func setCustomVocabulary(_ vocabulary: CustomVocabularyContext?) {
+        self.customVocabulary = vocabulary
+        if let vocab = vocabulary {
+            logger.info(
+                "Custom vocabulary updated: \(vocab.terms.count) terms, "
+                    + "thresholds: similarity=\(String(format: "%.2f", vocab.minSimilarity)), "
+                    + "combined=\(String(format: "%.2f", vocab.minCombinedConfidence))")
+        } else {
+            logger.info("Custom vocabulary disabled")
+        }
+    }
+
+    /// Get current custom vocabulary
+    public func getCustomVocabulary() -> CustomVocabularyContext? {
+        return customVocabulary
+    }
+
     private func createFeatureProvider(
         features: [(name: String, array: MLMultiArray)]
     ) throws

Sources/FluidAudio/ASR/ContextBiasing/CtcKeywordSpotter.swift

@@ -141,6 +141,15 @@ public struct CtcKeywordSpotter {
                             }
                         }
                         print("      frame[\(frameIdx)]: \(tokenLogProbs.joined(separator: ", "))", to: &standardError)
+
+                        // Show top 10 most likely tokens at this frame
+                        let topK = 10
+                        let sortedIndices = frame.enumerated()
+                            .sorted { $0.element > $1.element }
+                            .prefix(topK)
+                        print("      top-\(topK) tokens: ", terminator: "", to: &standardError)
+                        let topTokens = sortedIndices.map { "id\($0.offset)=\(String(format: "%.4f", $0.element))" }
+                        print(topTokens.joined(separator: ", "), to: &standardError)
                     }
                 }
             }
@@ -451,6 +460,9 @@ public struct CtcKeywordSpotter {
 
     /// Dynamic programming keyword alignment, ported from
     /// `NeMo/scripts/asr_context_biasing/ctc_word_spotter.py:ctc_word_spot`.
+    // Wildcard token ID: -1 represents "*" that matches anything at zero cost
+    private static let WILDCARD_TOKEN_ID = -1
+
     func ctcWordSpot(
         logProbs: [[Float]],
         keywordTokens: [Int]
@@ -483,6 +495,18 @@ public struct CtcKeywordSpotter {
             for n in 1...N {
                 let tokenId = keywordTokens[n - 1]
 
+                // Wildcard token: matches any symbol (including blank) at zero cost
+                if tokenId == Self.WILDCARD_TOKEN_ID {
+                    // Wildcard can skip this frame at zero cost
+                    let wildcardSkip = dp[t - 1][n - 1]  // Move to next token
+                    let wildcardStay = dp[t - 1][n]  // Stay on wildcard
+
+                    let wildcardScore = max(wildcardSkip, wildcardStay)
+                    dp[t][n] = wildcardScore
+                    backtrackTime[t][n] = wildcardScore == wildcardSkip ? t - 1 : backtrackTime[t - 1][n]
+                    continue
+                }
+
                 if tokenId < 0 || tokenId >= frame.count {
                     continue
                 }
@@ -522,7 +546,10 @@ public struct CtcKeywordSpotter {
         }
 
         let bestStart = backtrackTime[bestEnd][N]
-        let normalizedScore = bestScore / Float(N)
+
+        // Normalize score only by non-wildcard tokens
+        let nonWildcardCount = keywordTokens.filter { $0 != Self.WILDCARD_TOKEN_ID }.count
+        let normalizedScore = nonWildcardCount > 0 ? bestScore / Float(nonWildcardCount) : bestScore
 
         return (normalizedScore, bestStart, bestEnd)
     }

Sources/FluidAudio/ASR/ContextBiasing/CustomVocabularyContext.swift

@@ -1,4 +1,5 @@
 import Foundation
+import OSLog
 
 /// A single custom vocabulary entry.
 public struct CustomVocabularyTerm: Codable, Sendable {
@@ -21,6 +22,11 @@ public struct CustomVocabularyConfig: Codable, Sendable {
     public let depthScaling: Float?
     public let scorePerPhrase: Float?
     public let terms: [CustomVocabularyTerm]
+
+    // CTC keyword boosting confidence thresholds
+    public let minCtcScore: Float?
+    public let minSimilarity: Float?
+    public let minCombinedConfidence: Float?
 }
 
 /// Runtime context used by the decoder biasing system.
@@ -31,35 +37,105 @@ public struct CustomVocabularyContext: Sendable {
     public let depthScaling: Float
     public let scorePerPhrase: Float
 
+    // CTC keyword boosting confidence thresholds
+    public let minCtcScore: Float
+    public let minSimilarity: Float
+    public let minCombinedConfidence: Float
+
     public init(
         terms: [CustomVocabularyTerm],
         alpha: Float = 0.5,
         contextScore: Float = 1.2,
         depthScaling: Float = 2.0,
-        scorePerPhrase: Float = 0.0
+        scorePerPhrase: Float = 0.0,
+        minCtcScore: Float = -10.0,
+        minSimilarity: Float = 0.50,
+        minCombinedConfidence: Float = 0.54
     ) {
         self.terms = terms
         self.alpha = alpha
         self.contextScore = contextScore
         self.depthScaling = depthScaling
         self.scorePerPhrase = scorePerPhrase
+        self.minCtcScore = minCtcScore
+        self.minSimilarity = minSimilarity
+        self.minCombinedConfidence = minCombinedConfidence
     }
 
     /// Load a custom vocabulary JSON file produced by the analysis tooling.
     public static func load(from url: URL) throws -> CustomVocabularyContext {
+        let logger = Logger(subsystem: "com.fluidaudio", category: "CustomVocabulary")
         let data = try Data(contentsOf: url)
         let config = try JSONDecoder().decode(CustomVocabularyConfig.self, from: data)
 
         let alpha = config.alpha ?? 0.5
         let contextScore = config.contextScore ?? 1.2
         let depthScaling = config.depthScaling ?? 2.0
         let scorePerPhrase = config.scorePerPhrase ?? 0.0
+        let minCtcScore = config.minCtcScore ?? -10.0
+        let minSimilarity = config.minSimilarity ?? 0.50
+        let minCombinedConfidence = config.minCombinedConfidence ?? 0.54
+
+        // Validate and normalize vocabulary terms
+        var validatedTerms: [CustomVocabularyTerm] = []
+        for term in config.terms {
+            let (sanitized, warnings) = sanitizeVocabularyTerm(term.text)
+
+            if !warnings.isEmpty {
+                logger.warning("Term '\(term.text)': \(warnings.joined(separator: ", "))")
+            }
+
+            // Skip empty terms after sanitization
+            guard !sanitized.isEmpty else {
+                logger.warning("Term '\(term.text)' is empty after sanitization, skipping")
+                continue
+            }
+
+            validatedTerms.append(term)
+        }
+
         return CustomVocabularyContext(
-            terms: config.terms,
+            terms: validatedTerms,
             alpha: alpha,
             contextScore: contextScore,
             depthScaling: depthScaling,
-            scorePerPhrase: scorePerPhrase
+            scorePerPhrase: scorePerPhrase,
+            minCtcScore: minCtcScore,
+            minSimilarity: minSimilarity,
+            minCombinedConfidence: minCombinedConfidence
         )
     }
+
+    /// Sanitize a vocabulary term and return warnings about potential issues.
+    private static func sanitizeVocabularyTerm(_ text: String) -> (sanitized: String, warnings: [String]) {
+        var warnings: [String] = []
+        var result = text
+
+        // 1. Check for control characters
+        if result.rangeOfCharacter(from: .controlCharacters) != nil {
+            warnings.append("contains control characters")
+            result = result.filter { !$0.isNewline && !$0.isWhitespace || $0 == " " }
+        }
+
+        // 2. Check for diacritics (informational, not blocking)
+        if result.folding(options: .diacriticInsensitive, locale: nil) != result {
+            warnings.append("contains diacritics - consider adding ASCII alias")
+        }
+
+        // 3. Check for numbers (informational)
+        if result.rangeOfCharacter(from: .decimalDigits) != nil {
+            warnings.append("contains numbers")
+        }
+
+        // 4. Check for unusual characters (not letters, spaces, hyphens, apostrophes)
+        let allowedChars = CharacterSet.letters
+            .union(.whitespaces)
+            .union(CharacterSet(charactersIn: "-'"))
+
+        if result.rangeOfCharacter(from: allowedChars.inverted) != nil {
+            warnings.append("contains unusual characters")
+        }
+
+        return (result, warnings)
+    }
 }