Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,13 @@ import Foundation
/// citation form (`tˈO`) that over-stresses them (issue #691)
/// 6. strict ASCII all-caps initialisms (`FBI`, `ATP`) spelled as
/// letter names after a full lexicon miss (issue #710)
/// 7. BART G2P CoreML fallback for OOV words (injected by the caller)
/// 7. whole-compound possessive stem lookup, using lexicons only
/// (`C-section's` → lexicon `C-section` + /z/)
/// 8. hyphenated-compound split after a whole-stem miss
/// (`land-use's` → `land` + lexicon `use's`) (issue #775)
/// 9. `-'s` stem + clitic for other known stems (`today's` → `today` + /z/),
/// including letter-name initialisms (`FBI's`)
/// 10. BART G2P CoreML fallback for OOV words (injected by the caller)
///
/// Punctuation supported by the chain's `vocab.json` (`, . ! ? ; …` etc.)
/// is preserved and attached to the preceding word — Kokoro treats those
Expand Down Expand Up @@ -111,8 +117,14 @@ struct KokoroAneEnglishPhonemizer: Sendable {

// MARK: - Word resolution

/// - Parameter allowFallback: when `false`, the BART G2P fallback is
/// skipped and an OOV word resolves to `nil`. Used by the possessive
/// rule, which — like Misaki's `stem_s` — only fires when the stem is a
/// *known* word; an OOV stem must leave the whole token on its original
/// path instead of quietly re-shaping it.
private func resolveWord(
_ word: String,
allowFallback: Bool = true,
fallback: (String) async throws -> [String]?
) async throws -> String? {
let normalized = Self.normalizeKey(word)
Expand Down Expand Up @@ -144,13 +156,8 @@ struct KokoroAneEnglishPhonemizer: Sendable {
+ "falling back to the bundled pronunciation")
}

if let phonemes = caseSensitiveWordToPhonemes[word]
?? caseSensitiveWordToPhonemes[normalized]
?? wordToPhonemes[lowered]
?? wordToPhonemes[normalized],
!phonemes.isEmpty
{
return phonemes.joined()
if let phonemes = lookupMisakiWord(word) {
return phonemes
}

// After a full lexicon miss, read strict ASCII all-caps tokens of a
Expand All @@ -162,18 +169,41 @@ struct KokoroAneEnglishPhonemizer: Sendable {
return spelled
}

// A hyphenated compound that missed every lexicon as a whole
// (`tales-to-amaze`) — resolve each part and join, so it reads as
// `tales to amaze` instead of BART G2P on the glued `talestoamaze`
// (issue #775). Real lexicon compounds (`twenty-one`) already returned
// above, so only genuine misses reach here.
// A known whole-compound stem carries stress and reduced vowels
// that splitting would lose (`mother-in-law's`, `C-section's`). This
// probe must be lexicon-only: recursively resolving `land-use` would
// derive from the verb `use` before its noun-possessive entry `use's`
// gets a chance to match in the component path below.
if let possessive = resolveWholeCompoundPossessive(word, lowered: lowered) {
return possessive
}

// Whole token and whole possessive stem both missed: resolve parts
// independently, preserving any explicit possessive entry on a part
// (`land-use's` → `land` + `use's`). Ordinary compounds retain #775's
// behavior, including per-part G2P when needed.
if word.contains("-"),
let compound = try await resolveHyphenatedCompound(word, fallback: fallback)
let compound = try await resolveHyphenatedCompound(
word, allowFallback: allowFallback, fallback: fallback)
{
return compound
}

guard !normalized.isEmpty else { return nil }
// A possessive / `-'s` clitic whose stem is a known word (`today's`,
// `someone's`, `the boss's`). The lexicon stores the clitic `'s` as its
// own entry and has no glued key for ordinary words like `today's`, so
// these miss above and the whole inflected token goes to BART G2P,
// which mangles it (`someone's` → "Samian's"). Resolve the stem and
// append the clitic by rule instead — same shape as Misaki's
// `Lexicon.stem_s`. Glued entries that *do* exist won the lexicon
// lookups above, so this only fires on genuine misses.
if let possessive = try await resolvePossessive(
word, lowered: lowered, fallback: fallback)
{
return possessive
}

guard allowFallback, !normalized.isEmpty else { return nil }
do {
if let phonemes = try await fallback(normalized), !phonemes.isEmpty {
return phonemes.joined()
Expand All @@ -186,6 +216,40 @@ struct KokoroAneEnglishPhonemizer: Sendable {
}
}

/// Direct bundled lookup, shared by ordinary words and the whole-stem
/// probe. No initialism spelling, compound splitting, stemming, or G2P.
private func lookupMisakiWord(_ word: String) -> String? {
let normalized = Self.normalizeKey(word)
guard
let phonemes = caseSensitiveWordToPhonemes[word]
?? caseSensitiveWordToPhonemes[normalized]
?? wordToPhonemes[word.lowercased()]
?? wordToPhonemes[normalized],
!phonemes.isEmpty
else {
return nil
}
return phonemes.joined()
}

/// Try only a whole hyphenated stem's lexicon entries. Non-compound
/// stems keep the existing resolution path (notably `AI`/`US` letter-name
/// overrides), and explicit entries for the inflected token already won.
private func resolveWholeCompoundPossessive(_ word: String, lowered: String) -> String? {
guard word.contains("-"), lowered.hasSuffix("'s") else { return nil }
let stem = String(word.dropLast(2))
guard !stem.isEmpty, !stem.hasSuffix("'") else { return nil }
guard
let stemIPA = customLexicon[stem]
?? customLexicon[Self.normalizeKey(stem)]
?? lookupMisakiWord(stem),
!stemIPA.isEmpty
else {
return nil
}
return stemIPA + Self.clitic(after: stemIPA)
}

/// Resolve a hyphenated compound that missed the lexicon by splitting on
/// hyphens and resolving each part, joining the phoneme strings with a
/// space (word boundary). Returns `nil` if the token isn't a multi-part
Expand All @@ -194,6 +258,7 @@ struct KokoroAneEnglishPhonemizer: Sendable {
/// recurse back into itself.
private func resolveHyphenatedCompound(
_ word: String,
allowFallback: Bool = true,
fallback: (String) async throws -> [String]?
) async throws -> String? {
let parts = word.split(separator: "-", omittingEmptySubsequences: true).map(String.init)
Expand All @@ -202,14 +267,77 @@ struct KokoroAneEnglishPhonemizer: Sendable {
var resolved: [String] = []
resolved.reserveCapacity(parts.count)
for part in parts {
guard let ipa = try await resolveWord(part, fallback: fallback), !ipa.isEmpty else {
guard
let ipa = try await resolveWord(
part, allowFallback: allowFallback, fallback: fallback),
!ipa.isEmpty
else {
return nil
}
resolved.append(ipa)
}
return resolved.joined(separator: " ")
}

// MARK: - Possessive / `-'s` clitic

/// Resolve a lower-cased token ending in `'s` as stem + `-s` clitic.
///
/// Mirrors Misaki's `Lexicon.stem_s`, which only accepts the split when
/// the stem is a known word — so an OOV stem returns `nil` here and the
/// caller falls through to whole-token G2P exactly as before. The stem is
/// resolved through the normal chain minus the G2P fallback, which keeps
/// custom-lexicon overrides and letter-name spelling working.
///
/// Known whole-compound stems have already returned through the direct
/// lexicon probe. Otherwise the hyphen split gives each part its own
/// lexicon lookup before this derivation (`land-use's` → `use's`).
///
/// - Parameters:
/// - word: the token as written (apostrophes already folded to ASCII by
/// ``normalizeApostrophes``). Original case is preserved so the stem
/// can still reach case-sensitive entries (`NASA's`, `iPhone's`).
/// - lowered: `word.lowercased()`, so `TODAY'S` matches too.
private func resolvePossessive(
_ word: String,
lowered: String,
fallback: (String) async throws -> [String]?
) async throws -> String? {
// `len(word) < 3` in Misaki: a bare `'s` (and anything shorter than
// three characters) never stems.
guard lowered.count >= 3, lowered.hasSuffix("'s") else { return nil }
let stem = String(word.dropLast(2))
guard !stem.isEmpty, !stem.hasSuffix("'") else { return nil }

guard
let stemIPA = try await resolveWord(stem, allowFallback: false, fallback: fallback),
!stemIPA.isEmpty
else {
return nil
}
return stemIPA + Self.clitic(after: stemIPA)
}

/// Voiceless non-sibilant obstruents — the `-s` clitic devoices after
/// these (`cat's` → `kˈæts`).
private static let voicelessNonSibilants: Set<Character> = ["p", "t", "k", "f", "θ"]

/// Sibilants — the clitic takes an epenthetic vowel after these
/// (`boss's` → `bˈɑsᵻz`). Note the Misaki lexicon spells the affricates
/// with the single-scalar ligatures `ʧ` / `ʤ`, not `tʃ` / `dʒ`.
private static let sibilants: Set<Character> = ["s", "z", "ʃ", "ʒ", "ʧ", "ʤ"]

/// The `-s` clitic phoneme for a stem, by English phonology — a direct
/// port of Misaki's `Lexicon._s`. The US form of the epenthetic vowel is
/// `ᵻ` (Misaki uses `ɪ` only when `british`); this frontend loads the US
/// lexicon, and `ᵻ` is in the chain's `vocab.json`.
static func clitic(after stemIPA: String) -> String {
guard let last = stemIPA.last else { return "z" }
if voicelessNonSibilants.contains(last) { return "s" }
if sibilants.contains(last) { return "ᵻz" }
return "z"
}

// MARK: - Letter-name initialisms (issue #710)

/// Spell a token as a sequence of letter names using the per-letter
Expand Down
Loading
Loading