WindyMadman/voice-changer
1
0
Fork 0
mirror of https://github.com/w-okada/voice-changer.git synced 2025-01-07 03:16:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

WIP: docker support v1.5.x trial 5

Browse Source
This commit is contained in:
wataru 2023-02-11 00:59:44 +09:00
parent 954a26b0c6
commit 3ec1902045
11 changed files with 946 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
client/demo/dist/index.html vendored
View File

@ -1 +1,10 @@
<!doctype html><html style="width:100%;height:100%;overflow:hidden"><head><meta charset="utf-8"/><title>Voice Changer Client Demo</title><script defer="defer" src="index.js"></script></head><body style="width:100%;height:100%;margin:0"><div id="app" style="width:100%;height:100%"></div></body></html>
<!DOCTYPE html>
<html style="width: 100%; height: 100%; overflow: hidden">
<head>
<meta charset="utf-8" />
<title>Voice Changer Client Demo</title>
<script defer src="index.js"></script></head>
<body style="width: 100%; height: 100%; margin: 0px">
<div id="app" style="width: 100%; height: 100%"></div>
</body>
</html>

597
client/demo/dist/index.js vendored
View File

File diff suppressed because one or more lines are too long

31
client/demo/dist/index.js.LICENSE.txt vendored
View File

@ -1,31 +0,0 @@
/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
/**
* @license React
* react-dom.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/**
* @license React
* react.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/**
* @license React
* scheduler.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

20
client/demo/src/103_speaker_setting.tsx
View File

@ -94,6 +94,23 @@ export const useSpeakerSetting = (props: UseSpeakerSettingProps) => {
}, [props.clientState.clientSetting.setting.speakers, editSpeakerTargetId, editSpeakerTargetName])
const f0FactorRow = useMemo(() => {
return (
<div className="body-row split-3-2-1-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">F0 Factor</div>
<div className="body-input-container">
<input type="range" className="body-item-input" min="0.1" max="5.0" step="0.1" value={props.clientState.serverSetting.setting.f0Factor} onChange={(e) => {
props.clientState.serverSetting.setF0Factor(Number(e.target.value))
}}></input>
</div>
<div className="body-item-text">
<div>{props.clientState.serverSetting.setting.f0Factor}</div>
</div>
<div className="body-item-text"></div>
</div>
)
}, [props.clientState.serverSetting.setting.f0Factor, props.clientState.serverSetting.setF0Factor])
const speakerSetting = useMemo(() => {
return (
<>
@ -105,9 +122,10 @@ export const useSpeakerSetting = (props: UseSpeakerSettingProps) => {
{srcIdRow}
{dstIdRow}
{editSpeakerIdMappingRow}
{f0FactorRow}
</>
)
}, [srcIdRow, dstIdRow, editSpeakerIdMappingRow])
}, [srcIdRow, dstIdRow, editSpeakerIdMappingRow, f0FactorRow])
return {
speakerSetting,

22
client/demo/src/css/App.css
View File

@ -147,6 +147,28 @@ body {
width: 40%;
}
}
.split-3-2-1-4 {
display: flex;
width: 100%;
justify-content: center;
margin: 1px 0px 1px 0px;
& > div:nth-child(1) {
left: 0px;
width: 30%;
}
& > div:nth-child(2) {
left: 30%;
width: 20%;
}
& > div:nth-child(3) {
left: 50%;
width: 10%;
}
& > div:nth-child(4) {
left: 60%;
width: 40%;
}
}
.split-3-2-2-2-1 {
display: flex;
width: 100%;

7
client/lib/src/const.ts
View File

@ -20,6 +20,8 @@ export type VoiceChangerServerSetting = {
framework: Framework
onnxExecutionProvider: OnnxExecutionProvider,
f0Factor: number
}
export type VoiceChangerClientSetting = {
@ -61,6 +63,7 @@ export type ServerInfo = {
dstId: number,
framework: Framework,
onnxExecutionProvider: string[]
f0Factor: number
}
@ -120,7 +123,8 @@ export const ServerSettingKey = {
"crossFadeEndRate": "crossFadeEndRate",
"crossFadeOverlapRate": "crossFadeOverlapRate",
"framework": "framework",
"onnxExecutionProvider": "onnxExecutionProvider"
"onnxExecutionProvider": "onnxExecutionProvider",
"f0Factor": "f0Factor"
} as const
export type ServerSettingKey = typeof ServerSettingKey[keyof typeof ServerSettingKey]
@ -136,6 +140,7 @@ export const DefaultVoiceChangerServerSetting: VoiceChangerServerSetting = {
crossFadeEndRate: 0.9,
crossFadeOverlapRate: 0.5,
framework: "ONNX",
f0Factor: 1.0,
onnxExecutionProvider: "CPUExecutionProvider"
}

14
client/lib/src/hooks/useServerSetting.ts
View File

@ -48,6 +48,7 @@ export type ServerSettingState = {
setCrossFadeOffsetRate: (num: number) => Promise<boolean>;
setCrossFadeEndRate: (num: number) => Promise<boolean>;
setCrossFadeOverlapRate: (num: number) => Promise<boolean>;
setF0Factor: (num: number) => Promise<boolean>;
reloadServerInfo: () => Promise<void>;
setFileUploadSetting: (val: FileUploadSetting) => void
loadModel: () => Promise<void>
@ -95,6 +96,7 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
props.voiceChangerClient.updateServerSettings(ServerSettingKey.crossFadeOffsetRate, "" + setting.crossFadeOffsetRate)
props.voiceChangerClient.updateServerSettings(ServerSettingKey.crossFadeEndRate, "" + setting.crossFadeEndRate)
props.voiceChangerClient.updateServerSettings(ServerSettingKey.crossFadeOverlapRate, "" + setting.crossFadeOverlapRate)
props.voiceChangerClient.updateServerSettings(ServerSettingKey.f0Factor, "" + setting.f0Factor)
}, [props.voiceChangerClient])
@ -120,7 +122,8 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
crossFadeEndRate: res.crossFadeEndRate,
crossFadeOverlapRate: res.crossFadeOverlapRate,
framework: res.framework,
onnxExecutionProvider: (!!res.onnxExecutionProvider && res.onnxExecutionProvider.length > 0) ? res.onnxExecutionProvider[0] as OnnxExecutionProvider : DefaultVoiceChangerServerSetting.onnxExecutionProvider
onnxExecutionProvider: (!!res.onnxExecutionProvider && res.onnxExecutionProvider.length > 0) ? res.onnxExecutionProvider[0] as OnnxExecutionProvider : DefaultVoiceChangerServerSetting.onnxExecutionProvider,
f0Factor: res.f0Factor
}
_setSetting(newSetting)
setItem(INDEXEDDB_KEY_SERVER, newSetting)
@ -191,6 +194,11 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
}
}, [props.voiceChangerClient])
const setF0Factor = useMemo(() => {
return async (num: number) => {
return await _set_and_store(ServerSettingKey.f0Factor, "" + num)
}
}, [props.voiceChangerClient])
//////////////
// 操作
/////////////
@ -328,7 +336,8 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
crossFadeEndRate: res.crossFadeEndRate,
crossFadeOverlapRate: res.crossFadeOverlapRate,
framework: res.framework,
onnxExecutionProvider: (!!res.onnxExecutionProvider && res.onnxExecutionProvider.length > 0) ? res.onnxExecutionProvider[0] as OnnxExecutionProvider : DefaultVoiceChangerServerSetting.onnxExecutionProvider
onnxExecutionProvider: (!!res.onnxExecutionProvider && res.onnxExecutionProvider.length > 0) ? res.onnxExecutionProvider[0] as OnnxExecutionProvider : DefaultVoiceChangerServerSetting.onnxExecutionProvider,
f0Factor: res.f0Factor
})
}
}, [props.voiceChangerClient])
@ -354,6 +363,7 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
setCrossFadeOffsetRate,
setCrossFadeEndRate,
setCrossFadeOverlapRate,
setF0Factor,
reloadServerInfo,
setFileUploadSetting,
loadModel,

2
docker/Dockerfile
View File

@ -23,7 +23,7 @@ RUN cd MMVC_Client && git checkout 04f3fec4fd82dea6657026ec4e1cd80fb29a415c && c
WORKDIR /
ADD dummy /
RUN git clone --depth 1 https://github.com/w-okada/voice-changer.git
RUN git clone --depth 1 https://github.com/w-okada/voice-changer.git
#########

6
server/README.md
View File

@ -8,7 +8,11 @@ $ conda activate mmvc-server
$ pip install -r requirements.txt
$ git clone https://github.com/isletennos/MMVC_Client.git
$ cd MMVC_Client && git checkout 04f3fec4fd82dea6657026ec4e1cd80fb29a415c && cd -
$ cd MMVC_Client && git checkout 3374a1177b73e3f6d600e5dbe93af033c36ee120 && cd -
$ git clone https://github.com/isletennos/MMVC_Trainer.git
$ cd MMVC_Trainer && git checkout c242d3d1cf7f768af70d9735082ca2bdd90c45f3 && cd -
$ python3 MMVCServerSIO.py -p 18888 --https true
```

84
server/voice_changer/VoiceChanger.py
View File

@ -10,7 +10,12 @@ import onnxruntime
from symbols import symbols
from models import SynthesizerTrn
from voice_changer.TrainerFunctions import TextAudioSpeakerCollate, spectrogram_torch, load_checkpoint, get_hparams_from_file
import pyworld as pw
# from voice_changer.TrainerFunctions import TextAudioSpeakerCollate, spectrogram_torch, load_checkpoint, get_hparams_from_file
from voice_changer.client_modules import convert_continuos_f0, spectrogram_torch, TextAudioSpeakerCollate, get_hparams_from_file, load_checkpoint
providers = ['OpenVINOExecutionProvider', "CUDAExecutionProvider", "DmlExecutionProvider", "CPUExecutionProvider"]
@ -26,12 +31,15 @@ class VocieChangerSettings():
convertChunkNum: int = 32
minConvertSize: int = 0
framework: str = "ONNX" # PyTorch or ONNX
f0Factor: float = 1.0
pyTorchModelFile: str = ""
onnxModelFile: str = ""
configFile: str = ""
# ↓mutableな物だけ列挙
intData = ["gpu", "srcId", "dstId", "convertChunkNum", "minConvertSize"]
floatData = ["crossFadeOffsetRate", "crossFadeEndRate", "crossFadeOverlapRate"]
floatData = ["crossFadeOffsetRate", "crossFadeEndRate", "crossFadeOverlapRate", "f0Factor"]
strData = ["framework"]
@ -66,11 +74,23 @@ class VoiceChanger():
# PyTorchモデル生成
if pyTorch_model_file != None:
self.net_g = SynthesizerTrn(
len(symbols),
self.hps.data.filter_length // 2 + 1,
self.hps.train.segment_size // self.hps.data.hop_length,
spec_channels=self.hps.data.filter_length // 2 + 1,
segment_size=self.hps.train.segment_size // self.hps.data.hop_length,
inter_channels=self.hps.model.inter_channels,
hidden_channels=self.hps.model.hidden_channels,
upsample_rates=self.hps.model.upsample_rates,
upsample_initial_channel=self.hps.model.upsample_initial_channel,
upsample_kernel_sizes=self.hps.model.upsample_kernel_sizes,
n_flow=self.hps.model.n_flow,
dec_out_channels=1,
dec_kernel_size=7,
n_speakers=self.hps.data.n_speakers,
**self.hps.model)
gin_channels=self.hps.model.gin_channels,
requires_grad_pe=self.hps.requires_grad.pe,
requires_grad_flow=self.hps.requires_grad.flow,
requires_grad_text_enc=self.hps.requires_grad.text_enc,
requires_grad_dec=self.hps.requires_grad.dec
)
self.net_g.eval()
load_checkpoint(pyTorch_model_file, self.net_g, None)
# utils.load_checkpoint(pyTorch_model_file, self.net_g, None)
@ -174,14 +194,31 @@ class VoiceChanger():
audio_norm = self.audio_buffer[:, -convertSize:] # 変換対象の部分だけ抽出
self.audio_buffer = audio_norm
# TBD: numpy <--> pytorch変換が行ったり来たりしているが、まずは動かすことを最優先。
audio_norm_np = audio_norm.squeeze().numpy().astype(np.double)
_f0, _time = pw.dio(audio_norm_np, self.hps.data.sampling_rate, frame_period=5.5)
f0 = pw.stonemask(audio_norm_np, _f0, _time, self.hps.data.sampling_rate)
f0 = convert_continuos_f0(f0, int(audio_norm_np.shape[0] / self.hps.data.hop_length))
f0 = torch.from_numpy(f0.astype(np.float32))
spec = spectrogram_torch(audio_norm, self.hps.data.filter_length,
self.hps.data.sampling_rate, self.hps.data.hop_length, self.hps.data.win_length,
center=False)
# dispose_stft_specs = 2
# spec = spec[:, dispose_stft_specs:-dispose_stft_specs]
# f0 = f0[dispose_stft_specs:-dispose_stft_specs]
spec = torch.squeeze(spec, 0)
sid = torch.LongTensor([int(self.settings.srcId)])
data = (self.text_norm, spec, audio_norm, sid)
data = TextAudioSpeakerCollate()([data])
# data = (self.text_norm, spec, audio_norm, sid)
# data = TextAudioSpeakerCollate()([data])
data = TextAudioSpeakerCollate(
sample_rate=self.hps.data.sampling_rate,
hop_size=self.hps.data.hop_length,
f0_factor=self.settings.f0Factor # TBD: parameter
# f0_factor=2.4 # TBD: parameter
)([(spec, sid, f0)])
return data
def _onnx_inference(self, data, inputSize):
@ -224,10 +261,15 @@ class VoiceChanger():
if self.settings.gpu < 0 or self.gpu_num == 0:
with torch.no_grad():
x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [x.cpu() for x in data]
sid_tgt1 = torch.LongTensor([self.settings.dstId]).cpu()
audio1 = (self.net_g.cpu().voice_conversion(spec, spec_lengths, sid_src=sid_src,
sid_tgt=sid_tgt1)[0, 0].data * self.hps.data.max_wav_value)
spec, spec_lengths, sid_src, sin, d = data
spec = spec.cpu()
spec_lengths = spec_lengths.cpu()
sid_src = sid_src.cpu()
sin = sin.cpu()
d = tuple([d[:1].cpu() for d in d])
sid_target = torch.LongTensor([self.settings.dstId]).cpu()
audio1 = self.net_g.cpu().voice_conversion(spec, spec_lengths, sin, d, sid_src, sid_target)[0, 0].data * self.hps.data.max_wav_value
if self.prev_strength.device != torch.device('cpu'):
print(f"prev_strength move from {self.prev_strength.device} to cpu")
@ -256,11 +298,21 @@ class VoiceChanger():
else:
with torch.no_grad():
x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [x.cuda(self.settings.gpu) for x in data]
sid_tgt1 = torch.LongTensor([self.settings.dstId]).cuda(self.settings.gpu)
audio1 = self.net_g.cuda(self.settings.gpu).voice_conversion(spec, spec_lengths, sid_src=sid_src,
sid_tgt=sid_tgt1)[0, 0].data * self.hps.data.max_wav_value
spec, spec_lengths, sid_src, sin, d = data
spec = spec.cuda(self.settings.gpu)
spec_lengths = spec_lengths.cuda(self.settings.gpu)
sid_src = sid_src.cuda(self.settings.gpu)
sin = sin.cuda(self.settings.gpu)
d = tuple([d[:1].cuda(self.settings.gpu) for d in d])
sid_target = torch.LongTensor([self.settings.dstId]).cuda(self.settings.gpu)
# audio1 = self.net_g.cuda(self.settings.gpu).voice_conversion(spec, spec_lengths, sid_src=sid_src,
# sid_tgt=sid_tgt1)[0, 0].data * self.hps.data.max_wav_value
audio1 = self.net_g.cuda(self.settings.gpu).voice_conversion(spec, spec_lengths, sin, d,
sid_src, sid_target)[0, 0].data * self.hps.data.max_wav_value
# audio1 = audio1[10:-10]
if self.prev_strength.device != torch.device('cuda', self.settings.gpu):
print(f"prev_strength move from {self.prev_strength.device} to gpu{self.settings.gpu}")
self.prev_strength = self.prev_strength.cuda(self.settings.gpu)

208
server/voice_changer/client_modules.py Normal file
View File

@ -0,0 +1,208 @@
from features import SignalGenerator, dilated_factor
from scipy.interpolate import interp1d
import torch
import numpy as np
import json
import os
hann_window = {}
class TextAudioSpeakerCollate():
""" Zero-pads model inputs and targets
"""
def __init__(
self,
sample_rate,
hop_size,
f0_factor=1.0,
dense_factors=[0.5, 1, 4, 8],
upsample_scales=[8, 4, 2, 2],
sine_amp=0.1,
noise_amp=0.003,
signal_types=["sine"],
):
self.dense_factors = dense_factors
self.prod_upsample_scales = np.cumprod(upsample_scales)
self.sample_rate = sample_rate
self.signal_generator = SignalGenerator(
sample_rate=sample_rate,
hop_size=hop_size,
sine_amp=sine_amp,
noise_amp=noise_amp,
signal_types=signal_types,
)
self.f0_factor = f0_factor
def __call__(self, batch):
"""Collate's training batch from normalized text, audio and speaker identities
PARAMS
------
batch: [text_normalized, spec_normalized, wav_normalized, sid, note]
"""
spec_lengths = torch.LongTensor(len(batch))
sid = torch.LongTensor(len(batch))
spec_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), batch[0][0].size(1))
f0_padded = torch.FloatTensor(len(batch), 1, batch[0][2].size(0))
# 返り値の初期化
spec_padded.zero_()
f0_padded.zero_()
# dfs
dfs_batch = [[] for _ in range(len(self.dense_factors))]
# row spec, sid, f0
for i in range(len(batch)):
row = batch[i]
spec = row[0]
spec_padded[i, :, :spec.size(1)] = spec
spec_lengths[i] = spec.size(1)
sid[i] = row[1]
# 推論時 f0/cf0にf0の倍率を乗算してf0/cf0を求める
f0 = row[2] * self.f0_factor
f0_padded[i, :, :f0.size(0)] = f0
# dfs
dfs = []
# dilated_factor の入力はnumpy!!
for df, us in zip(self.dense_factors, self.prod_upsample_scales):
dfs += [
np.repeat(dilated_factor(torch.unsqueeze(f0, dim=1).to('cpu').detach().numpy(), self.sample_rate, df), us)
]
# よくわからないけど、後で論文ちゃんと読む
for i in range(len(self.dense_factors)):
dfs_batch[i] += [
dfs[i].astype(np.float32).reshape(-1, 1)
] # [(T', 1), ...]
# よくわからないdfsを転置
for i in range(len(self.dense_factors)):
dfs_batch[i] = torch.FloatTensor(np.array(dfs_batch[i])).transpose(
2, 1
) # (B, 1, T')
# f0/cf0を実際に使うSignalに変換する
in_batch = self.signal_generator(f0_padded)
return spec_padded, spec_lengths, sid, in_batch, dfs_batch
def convert_continuos_f0(f0, f0_size):
# get start and end of f0
if (f0 == 0).all():
return np.zeros((f0_size,))
start_f0 = f0[f0 != 0][0]
end_f0 = f0[f0 != 0][-1]
# padding start and end of f0 sequence
cf0 = f0
start_idx = np.where(cf0 == start_f0)[0][0]
end_idx = np.where(cf0 == end_f0)[0][-1]
cf0[:start_idx] = start_f0
cf0[end_idx:] = end_f0
# get non-zero frame index
nz_frames = np.where(cf0 != 0)[0]
# perform linear interpolation
f = interp1d(nz_frames, cf0[nz_frames], bounds_error=False, fill_value=0.0)
cf0_ = f(np.arange(0, f0_size))
# print(cf0.shape, cf0_.shape, f0.shape, f0_size)
# print(cf0_)
return f(np.arange(0, f0_size))
def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
if torch.min(y) < -1.:
print('min value is ', torch.min(y))
if torch.max(y) > 1.:
print('max value is ', torch.max(y))
dtype_device = str(y.dtype) + '_' + str(y.device)
wnsize_dtype_device = str(win_size) + '_' + dtype_device
if wnsize_dtype_device not in hann_window:
hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), mode='reflect')
y = y.squeeze(1)
spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=True)
spec = torch.view_as_real(spec)
spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
return spec
def get_hparams_from_file(config_path):
with open(config_path, "r", encoding="utf-8") as f:
data = f.read()
config = json.loads(data)
hparams = HParams(**config)
return hparams
class HParams():
def __init__(self, **kwargs):
for k, v in kwargs.items():
if type(v) == dict:
v = HParams(**v)
self[k] = v
def keys(self):
return self.__dict__.keys()
def items(self):
return self.__dict__.items()
def values(self):
return self.__dict__.values()
def __len__(self):
return len(self.__dict__)
def __getitem__(self, key):
return getattr(self, key)
def __setitem__(self, key, value):
return setattr(self, key, value)
def __contains__(self, key):
return key in self.__dict__
def __repr__(self):
return self.__dict__.__repr__()
def load_checkpoint(checkpoint_path, model, optimizer=None):
assert os.path.isfile(checkpoint_path), f"No such file or directory: {checkpoint_path}"
checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
iteration = checkpoint_dict['iteration']
learning_rate = checkpoint_dict['learning_rate']
if optimizer is not None:
optimizer.load_state_dict(checkpoint_dict['optimizer'])
saved_state_dict = {
**checkpoint_dict['pe'],
**checkpoint_dict['flow'],
**checkpoint_dict['text_enc'],
**checkpoint_dict['dec'],
**checkpoint_dict['emb_g']
}
if hasattr(model, 'module'):
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
new_state_dict = {}
for k, v in state_dict.items():
try:
new_state_dict[k] = saved_state_dict[k]
except:
new_state_dict[k] = v
if hasattr(model, 'module'):
model.module.load_state_dict(new_state_dict)
else:
model.load_state_dict(new_state_dict)
return model, optimizer, learning_rate, iteration