444 lines
21 KiB
Python
444 lines
21 KiB
Python
from __future__ import annotations
|
||
|
||
import importlib.util
|
||
import json
|
||
import re
|
||
import unicodedata
|
||
from pathlib import Path
|
||
from typing import Any, Callable, Iterable
|
||
|
||
import numpy as np
|
||
import torch
|
||
from torch import Tensor
|
||
|
||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||
|
||
# tiny-voca, https://huggingface.co/n4ze3m/tiny-voca
|
||
#
|
||
# text model: phoneme encoder (4 blocks), the input rows are built by the audio pipeline (token + stress + voice)
|
||
# mmproj: input tables, voices, duration predictor, prosody and acoustic stacks, source-filter heads
|
||
#
|
||
# the text to phonemes step (dizione) is the "tts" chat template, see models/templates/n4ze3m-tiny-voca-tts.jinja
|
||
# its lexicon comes from the phonemizer/ directory of the model, the other tables from the dizione python package
|
||
#
|
||
# the prosody and acoustic stacks have the same blocks as the encoder, they are stored with the libllama names
|
||
# of the tiny-voca arch under a.gen.prosody. and a.gen.acoustic., mtmd runs them on llama contexts of its own
|
||
|
||
CHECKPOINT = "tiny-voca.pt"
|
||
DEFAULT_VOICE = "M2"
|
||
|
||
# phone classes of the reference (text.py): the phones get at least one frame, the vowels are always voiced
|
||
VOWELS = ("ə", "ɐ", "ɚ", "ᵻ", "ɪ", "i", "iː", "ɛ", "æ", "ʌ", "ɑː", "ɔ", "ɔː", "ʊ", "uː", "ɜː",
|
||
"eɪ", "aɪ", "ɔɪ", "oʊ", "aʊ", "iə", "oː",
|
||
"ɑːɹ", "ɔːɹ", "oːɹ", "ɪɹ", "ɛɹ", "ʊɹ", "aɪɚ", "aɪə", "əl", "n̩")
|
||
CONSONANTS = ("p", "b", "t", "d", "k", "ɡ", "f", "v", "θ", "ð", "s", "z", "ʃ", "ʒ", "h",
|
||
"m", "n", "ŋ", "l", "ɹ", "j", "w", "tʃ", "dʒ", "ɾ", "ʔ")
|
||
|
||
# input letters and output phonemes of the unknown-word model, fixed in mtmd (see mtmd_graph_tiny_voca)
|
||
G2P_LETTERS = "abcdefghijklmnopqrstuvwxyz'"
|
||
G2P_SYMBOLS = [v + d for v in ("AA", "AE", "AH", "AO", "AW", "AY") for d in "012"] + ["B", "CH", "D", "DH"] + \
|
||
[v + d for v in ("EH", "ER", "EY") for d in "012"] + ["F", "G", "HH"] + [v + d for v in ("IH", "IY") for d in "012"] + \
|
||
["JH", "K", "L", "M", "N", "NG"] + [v + d for v in ("OW", "OY") for d in "012"] + ["P", "R", "S", "SH", "T", "TH"] + \
|
||
[v + d for v in ("UH", "UW") for d in "012"] + ["V", "W", "Y", "Z", "ZH"]
|
||
|
||
# block tensors: reference name -> libllama name
|
||
BLOCK_MAP = {
|
||
"norm1": "attn_norm",
|
||
"attention.qkv": "attn_qkv",
|
||
"attention.out": "attn_output",
|
||
"norm2": "ffn_norm",
|
||
"depthwise": "ssm_conv1d",
|
||
"expand": "ffn_up",
|
||
"project": "ffn_down",
|
||
}
|
||
|
||
# mmproj stacks: reference module -> prefix
|
||
STACK_PREFIX = {
|
||
"prosody_stack": "a.gen.prosody.",
|
||
"acoustic_stack": "a.gen.acoustic.",
|
||
}
|
||
|
||
# other mmproj tensors: reference module -> name
|
||
MMPROJ_MAP = {
|
||
"tokens": "a.gen.tok_embd",
|
||
"stress": "a.gen.stress_embd",
|
||
"duration.conv1": "a.gen.dur.conv1",
|
||
"duration.conv2": "a.gen.dur.conv2",
|
||
"duration.norm1": "a.gen.dur.norm1",
|
||
"duration.norm2": "a.gen.dur.norm2",
|
||
"duration.out": "a.gen.dur.out",
|
||
"frame_position": "a.gen.frame_pos",
|
||
"prosody_head": "a.gen.prosody_head",
|
||
"prosody_projection": "a.gen.prosody_proj",
|
||
"vocoder.harmonic": "a.gen.voc.harmonic",
|
||
"vocoder.noise": "a.gen.voc.noise",
|
||
}
|
||
|
||
|
||
def _is_tiny_voca_checkpoint(dir_model: Path) -> bool:
|
||
if not (dir_model / CHECKPOINT).is_file() or not (dir_model / "config.json").is_file():
|
||
return False
|
||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||
config = json.load(f)
|
||
return config.get("phonemizer", {}).get("name") == "dizione" and "vocab" in config and "model" in config
|
||
|
||
|
||
@ModelBase.register_hparams_loader(_is_tiny_voca_checkpoint)
|
||
def _load_tiny_voca_hparams(dir_model: Path) -> dict[str, Any]:
|
||
logger.info("gguf: detected tiny-voca checkpoint")
|
||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||
config = json.load(f)
|
||
c = config["model"]
|
||
return {
|
||
"architectures": ["TinyVocaModel"],
|
||
"model_type": "tiny-voca",
|
||
"num_hidden_layers": c["encoder_layers"],
|
||
"hidden_size": c["channels"],
|
||
"intermediate_size": c["feedforward"],
|
||
"num_attention_heads": c["heads"],
|
||
"num_key_value_heads": c["heads"],
|
||
"max_position_embeddings": c["max_tokens"],
|
||
"vocab_size": c["vocab_size"],
|
||
"layer_norm_eps": 1e-5, # torch LayerNorm default
|
||
"rope_theta": 10000.0,
|
||
"tiny_voca": config,
|
||
# the mmproj tensors are named explicitly, they do not go through the block tensor map
|
||
"audio_config": {"num_hidden_layers": 0},
|
||
}
|
||
|
||
|
||
def _table_rows(path: Path) -> Iterable[list[str]]:
|
||
# dizione tables: columns separated by tabs or two or more spaces, "#" comments
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
line = line.split("#", 1)[0].rstrip() if line.lstrip().startswith("#") else line.rstrip()
|
||
if line.strip():
|
||
yield [c.strip() for c in re.split(r"\t+| {2,}", line.strip())]
|
||
|
||
|
||
def _g2p_dicts(dir_model: Path) -> dict[str, dict[str, Any]] | None:
|
||
lexicon_path = dir_model / "phonemizer" / "lexicon.tsv"
|
||
spec = importlib.util.find_spec("dizione")
|
||
if not lexicon_path.is_file() or spec is None or spec.origin is None:
|
||
logger.warning("dizione or phonemizer/lexicon.tsv not found, the model will take phonemes as prompt "
|
||
"(pip install git+https://github.com/n4ze3m/dizione)")
|
||
return None
|
||
data = Path(spec.origin).parent / "data"
|
||
lexicon: dict[str, str] = {}
|
||
for line in lexicon_path.read_text(encoding="utf-8").splitlines():
|
||
if "\t" in line:
|
||
word, phones = line.split("\t", 1)
|
||
lexicon[word] = phones.strip()
|
||
|
||
# word -> [stress class, form, final form or ""], see function_words.txt
|
||
functions = {r[0]: [r[1], r[2], r[3] if len(r) > 3 else ""] for r in _table_rows(data / "function_words.txt")}
|
||
# words -> [class, forms (one, or one per word), final form or ""], see phrases.txt
|
||
phrases = {r[0]: [r[1], [f.strip() for f in r[2].split("|")], r[3] if len(r) > 3 else ""] for r in _table_rows(data / "phrases.txt")}
|
||
letters = {r[0]: r[1] for r in _table_rows(data / "letters.txt")}
|
||
# a value is phones, or CMUdict notation (converted by the template, like the output of the unknown-word model)
|
||
exceptions = {r[0].lower(): r[1] for r in _table_rows(data / "exceptions.txt") if " " not in r[0]}
|
||
# a name of several words: "w1 w2 ..." -> one pronunciation per word; first word -> the longest name it starts
|
||
names: dict[str, Any] = {}
|
||
name_starts: dict[str, int] = {}
|
||
for r in _table_rows(data / "exceptions.txt"):
|
||
words = r[0].lower().split()
|
||
if len(words) > 1:
|
||
names[" ".join(words)] = [part.strip() for part in r[1].split("|")]
|
||
name_starts[words[0]] = max(name_starts.get(words[0], 0), len(words))
|
||
# phoneme (CMUdict, no stress) -> {letters: log probability}: aligns the letters of a word to its phonemes
|
||
with open(dir_model / "phonemizer" / "align.json", encoding="utf-8") as f:
|
||
align = json.load(f)
|
||
# abbreviation -> [when, words]
|
||
abbreviations = {r[0]: [r[1], r[2]] for r in _table_rows(data / "abbreviations.txt")}
|
||
# chat spelling -> [words to say ("*LOL": letter by letter, "-": nothing), only as a reply by itself], see chat.txt
|
||
chat = {r[0].lower(): [r[1] if len(r) > 1 else "-", len(r) > 2] for r in _table_rows(data / "chat.txt")}
|
||
# accented latin letter -> plain letter(s): "café" is in the dictionary as "cafe" (dizione: _plain_letter)
|
||
accents = {"ß": "ss", "æ": "ae", "Æ": "Ae", "œ": "oe", "Œ": "Oe", "ø": "o", "Ø": "O", "đ": "d", "ł": "l", "Ł": "L"}
|
||
for cp in range(0x80, 0x2000):
|
||
char = chr(cp)
|
||
if char.isalpha() and char not in accents:
|
||
plain = "".join(c for c in unicodedata.normalize("NFKD", char) if not unicodedata.combining(c))
|
||
if plain.isascii() and plain.isalpha():
|
||
accents[char] = plain
|
||
# compatibility forms of the text (NFKC), a character at a time: "…" is "...", "fi" is "fi", "№" is "No", full
|
||
# width letters. Sequences that NFKC would compose are left as they are
|
||
nfkc: dict[str, str] = {}
|
||
for cp in list(range(0x80, 0x3000)) + list(range(0xFB00, 0xFB07)) + list(range(0xFF01, 0xFF5F)):
|
||
if unicodedata.normalize("NFKC", chr(cp)) != chr(cp):
|
||
nfkc[chr(cp)] = unicodedata.normalize("NFKC", chr(cp))
|
||
# units after a number: name -> [one, many]; "exact" ones are written in one letter case only ("m", not "M")
|
||
units: dict[str, Any] = {}
|
||
exact_units: dict[str, Any] = {}
|
||
for r in _table_rows(data / "units.txt"):
|
||
if len(r) > 3 and r[3] == "exact":
|
||
exact_units[r[0]] = [r[1], r[2]]
|
||
else:
|
||
units[r[0].lower()] = [r[1], r[2]]
|
||
# the unit names as a pattern of tts_norm_scan(), the longest first; a unit does not stop inside a word
|
||
unit_names = sorted(set(units) | set(exact_units), key=lambda name: (-len(name), name))
|
||
# (the pattern ignores the case of ASCII letters only, the greek letter is written in both)
|
||
unit_pattern = "|".join("".join("[μΜ]" if c == "μ" else c if c.isalnum() else "%" + c for c in name) + "(?![A-Za-z])" for name in unit_names)
|
||
# endings of file names and addresses: "word" (dot com) or "letters" (dot P D F)
|
||
endings = {r[0]: (r[1] if len(r) > 1 else "") for r in _table_rows(data / "endings.txt")}
|
||
out = {
|
||
"lexicon": lexicon,
|
||
"functions": functions,
|
||
"phrases": phrases,
|
||
"letters": letters,
|
||
"exceptions": exceptions,
|
||
"abbreviations": abbreviations,
|
||
"align": align,
|
||
"names": names,
|
||
"chat": chat,
|
||
"accents": accents,
|
||
"nfkc": nfkc,
|
||
"units": units,
|
||
"exact_units": exact_units,
|
||
"endings": endings,
|
||
"patterns": {"units": unit_pattern},
|
||
"name_starts": name_starts,
|
||
}
|
||
out.update(_homograph_dicts(dir_model, data))
|
||
return out
|
||
|
||
|
||
def _homograph_dicts(dir_model: Path, data: Path) -> dict[str, dict[str, Any]]:
|
||
# words spelled alike and pronounced differently (dizione homographs.py): a linear classifier on the neighbouring
|
||
# words, and hand-written rules that are tried first. Weights are stored x10000 (they have four decimals), so the
|
||
# template adds integers.
|
||
path = dir_model / "phonemizer" / "homographs.json"
|
||
if not path.is_file():
|
||
return {"homographs": {}, "homograph_shared": {}, "homograph_plurals": {}, "homograph_rules": {}}
|
||
with open(path, encoding="utf-8") as f:
|
||
source = json.load(f)
|
||
|
||
def fixed(x: float) -> int:
|
||
value = round(x * 10000)
|
||
if abs(value - x * 10000) > 1e-6:
|
||
raise ValueError(f"homographs.json: weight {x} has more than four decimals")
|
||
return value
|
||
|
||
words: dict[str, Any] = {}
|
||
for word, entry in source["words"].items():
|
||
model = entry["model"]
|
||
noun_verb = model["kind"] == "noun-verb"
|
||
if not noun_verb and model["kind"] != "senses":
|
||
raise ValueError(f"homographs.json: unknown model kind {model['kind']!r}")
|
||
words[word] = {
|
||
# sense: [id, role, phones]
|
||
"senses": [[s["id"], s["role"], s["phones"]] for s in entry["senses"]],
|
||
# noun-verb: one score for the verb reading; else one score per sense
|
||
"noun_verb": noun_verb,
|
||
"bias": [fixed(model["bias"])] if noun_verb else [fixed(b) for b in model["bias"]],
|
||
"weights": [{k: fixed(v) for k, v in w.items()} for w in ([model["weights"]] if noun_verb else model["weights"])],
|
||
}
|
||
# the weights all noun-or-verb homographs share; "" is the bias
|
||
shared = {k: fixed(v) for k, v in source["pooled"]["weights"].items()}
|
||
shared[""] = fixed(source["pooled"]["bias"])
|
||
# word -> [[sense id, [[condition, [words]], ...]], ...], see homograph_rules.txt
|
||
rules: dict[str, Any] = {}
|
||
for r in _table_rows(data / "homograph_rules.txt"):
|
||
conditions = []
|
||
for part in r[2].split(";"):
|
||
name, _, listed = part.strip().partition(":")
|
||
conditions.append([name, listed.split()])
|
||
rules.setdefault(r[0], []).append([r[1], conditions])
|
||
return {
|
||
"homographs": words,
|
||
"homograph_shared": shared,
|
||
"homograph_plurals": {base: 1 for base in source.get("plural_bases", ())},
|
||
"homograph_rules": rules,
|
||
}
|
||
|
||
|
||
def _load_state(dir_model: Path) -> dict[str, Tensor]:
|
||
ckpt = torch.load(dir_model / CHECKPOINT, map_location="cpu", weights_only=True)
|
||
if ckpt.get("format_version") != 4:
|
||
raise ValueError(f"unsupported tiny-voca checkpoint format {ckpt.get('format_version')}")
|
||
return ckpt["model"]
|
||
|
||
|
||
def _block_name(name: str) -> tuple[int, str] | None:
|
||
# "blocks.<i>.<module>.<weight|bias>" -> (i, "blk.<i>.<libllama name>.<weight|bias>")
|
||
parts = name.split(".")
|
||
if len(parts) < 4 or parts[0] != "blocks":
|
||
return None
|
||
bid = int(parts[1])
|
||
module, suffix = ".".join(parts[2:-1]), parts[-1]
|
||
return bid, f"blk.{bid}.{BLOCK_MAP[module]}.{suffix}"
|
||
|
||
|
||
def _block_tensors(new_name: str, t: Tensor) -> Iterable[tuple[str, Tensor]]:
|
||
# depthwise conv [C, 1, K] -> [C, K], the layout of ggml_ssm_conv
|
||
if ".ssm_conv1d.weight" in new_name:
|
||
yield (new_name, t.squeeze(1))
|
||
return
|
||
# separate q, k, v: a model that libllama builds from metadata (the mmproj stacks) has no fused qkv
|
||
if ".attn_qkv." in new_name:
|
||
for part, chunk in zip(("attn_q", "attn_k", "attn_v"), t.chunk(3, dim=0)):
|
||
yield (new_name.replace("attn_qkv", part), chunk.contiguous())
|
||
return
|
||
yield (new_name, t)
|
||
|
||
|
||
@ModelBase.register("TinyVocaModel")
|
||
@ModelBase.example("n4ze3m/tiny-voca")
|
||
class TinyVocaTextModel(TextModel):
|
||
model_arch = gguf.MODEL_ARCH.TINY_VOCA
|
||
|
||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||
del remote_hf_model_id
|
||
state = _load_state(self.dir_model)
|
||
out: dict[str, Callable[[], Tensor]] = {}
|
||
for name, t in state.items():
|
||
if name == "tokens.weight" or name.startswith("encoder."):
|
||
out[name] = (lambda t=t: t)
|
||
return out
|
||
|
||
def set_vocab(self):
|
||
config = self.hparams["tiny_voca"]
|
||
n_vocab = config["model"]["vocab_size"]
|
||
tokens: list[str] = [""] * n_vocab
|
||
toktypes = [gguf.TokenType.NORMAL] * n_vocab
|
||
for text, i in config["vocab"].items():
|
||
tokens[i] = text
|
||
if text.startswith("<") and text.endswith(">"):
|
||
toktypes[i] = gguf.TokenType.CONTROL
|
||
assert all(tokens)
|
||
|
||
# the prompt is split into phonemes by the audio pipeline, see mtmd-helper-gen.cpp
|
||
self.gguf_writer.add_tokenizer_model("llama")
|
||
self.gguf_writer.add_tokenizer_pre("default")
|
||
self.gguf_writer.add_token_list(tokens)
|
||
self.gguf_writer.add_token_scores([0.0] * n_vocab)
|
||
self.gguf_writer.add_token_types(toktypes)
|
||
self.gguf_writer.add_bos_token_id(config["vocab"]["<bos>"])
|
||
self.gguf_writer.add_eos_token_id(config["vocab"]["<eos>"])
|
||
self.gguf_writer.add_pad_token_id(config["vocab"]["<pad>"])
|
||
self.gguf_writer.add_unk_token_id(config["vocab"]["<unk>"])
|
||
self.gguf_writer.add_add_bos_token(False)
|
||
self.gguf_writer.add_add_eos_token(False)
|
||
|
||
dicts = _g2p_dicts(self.dir_model)
|
||
if dicts is not None:
|
||
template_path = Path(__file__).parent.parent / "models" / "templates" / "n4ze3m-tiny-voca-tts.jinja"
|
||
with open(template_path, encoding="utf-8") as f:
|
||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||
self.gguf_writer.add_tts_dicts(dicts)
|
||
|
||
def set_gguf_parameters(self):
|
||
super().set_gguf_parameters()
|
||
self.gguf_writer.add_causal_attention(False)
|
||
self.gguf_writer.add_pooling_type(gguf.PoolingType.NONE)
|
||
|
||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||
del bid
|
||
if name == "tokens.weight":
|
||
yield ("token_embd.weight", data_torch)
|
||
return
|
||
if name.startswith("encoder.norm."):
|
||
yield ("output_norm." + name.rsplit(".", 1)[1], data_torch)
|
||
return
|
||
res = _block_name(name.removeprefix("encoder."))
|
||
if res is None:
|
||
raise ValueError(f"unexpected tensor {name}")
|
||
yield from _block_tensors(res[1], data_torch)
|
||
|
||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||
del name, bid, n_dims
|
||
# ggml_ssm_conv takes F32 weights; token_embd is also read row by row on the host side
|
||
if ".ssm_conv1d." in new_name or new_name == "token_embd.weight":
|
||
return gguf.GGMLQuantizationType.F32
|
||
return False
|
||
|
||
|
||
@ModelBase.register("TinyVocaModel")
|
||
@ModelBase.example("n4ze3m/tiny-voca")
|
||
class TinyVocaMmprojModel(MmprojModel):
|
||
has_vision_encoder = False
|
||
has_audio_encoder = False
|
||
|
||
def get_audio_config(self) -> dict[str, Any] | None:
|
||
return self.global_config.get("audio_config")
|
||
|
||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||
del remote_hf_model_id
|
||
state = _load_state(self.dir_model)
|
||
out: dict[str, Callable[[], Tensor]] = {}
|
||
for name, t in state.items():
|
||
if name.startswith("encoder.") or name in ("voices.weight", "frame_voices.weight"):
|
||
continue
|
||
out[name] = (lambda t=t: t)
|
||
return out
|
||
|
||
@classmethod
|
||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||
return item
|
||
|
||
def set_gguf_parameters(self):
|
||
c = self.global_config["tiny_voca"]["model"]
|
||
self.gguf_writer.add_file_type(self.ftype)
|
||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.TINY_VOCA)
|
||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||
self.gguf_writer.add_gen_audio_embedding_length(c["channels"])
|
||
self.gguf_writer.add_gen_audio_feed_forward_length(c["feedforward"])
|
||
self.gguf_writer.add_gen_audio_block_count(c["acoustic_layers"])
|
||
self.gguf_writer.add_gen_audio_head_count(c["heads"])
|
||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
|
||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||
|
||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||
del bid
|
||
# from generate_extra_tensors(), named already
|
||
if name.startswith(("voice.", "a.gen.")):
|
||
yield (name, data_torch)
|
||
return
|
||
stack, _, rest = name.partition(".")
|
||
if stack in STACK_PREFIX:
|
||
prefix = STACK_PREFIX[stack]
|
||
if rest.startswith("norm."):
|
||
yield (prefix + "output_norm." + rest.rsplit(".", 1)[1], data_torch)
|
||
return
|
||
res = _block_name(rest)
|
||
if res is None:
|
||
raise ValueError(f"unexpected tensor {name}")
|
||
yield from _block_tensors(prefix + res[1], data_torch)
|
||
return
|
||
module, suffix = name.rsplit(".", 1)
|
||
if module not in MMPROJ_MAP:
|
||
raise ValueError(f"unexpected tensor {name}")
|
||
yield (MMPROJ_MAP[module] + "." + suffix, data_torch)
|
||
|
||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||
del name, bid
|
||
# the blocks of the stacks are loaded by libllama, like the text model
|
||
if new_name.startswith(tuple(STACK_PREFIX.values())) and ".ssm_conv1d." not in new_name and n_dims >= 2:
|
||
return False
|
||
# the rest is small and partly read on the host
|
||
return gguf.GGMLQuantizationType.F32
|
||
|
||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||
# the unknown-word model of dizione (letters -> CMUdict phonemes), run on the host by mtmd_gen_audio_g2p()
|
||
g2p_dir = self.dir_model / "phonemizer"
|
||
if (g2p_dir / "g2p.npz").is_file():
|
||
with open(g2p_dir / "g2p.json", encoding="utf-8") as f:
|
||
settings = json.load(f)
|
||
if settings["letters"] != G2P_LETTERS or settings["symbols"] != G2P_SYMBOLS or settings["convs"] != 0 or settings["slots"] != 2:
|
||
raise ValueError("unsupported g2p model in phonemizer/g2p.json")
|
||
with np.load(g2p_dir / "g2p.npz") as arrays:
|
||
for name in arrays.files:
|
||
yield ("a.gen.g2p." + name.replace("__", "."), torch.from_numpy(arrays[name].astype(np.float32)))
|
||
config = self.global_config["tiny_voca"]
|
||
state = _load_state(self.dir_model)
|
||
# a voice row is the embedding added to the encoder input, then the one added to the frames
|
||
for i, voice in enumerate(config["voices"]):
|
||
yield (f"voice.{voice}", torch.cat([state["voices.weight"][i], state["frame_voices.weight"][i]]).float())
|
||
# class of each token: 0 = pause (special or punctuation), 1 = consonant or <unk>, 2 = vowel
|
||
vocab = config["vocab"]
|
||
cls = torch.zeros(len(vocab), dtype=torch.float32)
|
||
for p in CONSONANTS + ("<unk>",):
|
||
cls[vocab[p]] = 1.0
|
||
for p in VOWELS:
|
||
cls[vocab[p]] = 2.0
|
||
yield ("a.gen.token_class", cls)
|