{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "fb38c6f2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "808ab61e", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ['CUDA_VISIBLE_DEVICES'] = ''" ] }, { "cell_type": "code", "execution_count": 3, "id": "6f6fa2f9", "metadata": {}, "outputs": [], "source": [ "# cd /home/georg/code/NeMo\n", "# git checkout univnet_test\n", "# ./reinstall" ] }, { "cell_type": "code", "execution_count": 4, "id": "ad30d77c", "metadata": {}, "outputs": [], "source": [ "import re\n", "import pandas as pd\n", "import numpy as np\n", "import librosa\n", "from scipy.io import wavfile\n", "import librosa.display\n", "from pydub import AudioSegment\n", "from IPython.display import display_html\n", "import torch\n", "\n", "\n", "def _array_to_numpy(arr):\n", " # convert to np if torch\n", " if isinstance(arr, torch.Tensor):\n", " arr = arr.detach().cpu().numpy()\n", " # squash extra dimensions\n", " arr = arr.squeeze()\n", " # TODO: deal with 2 channels\n", " if 2 in arr.shape:\n", " raise NotImplementedError(\"stereo detected\")\n", " return arr\n", " \n", " \n", "def _array_to_float(arr):\n", " arr = _array_to_numpy(arr)\n", " arr = arr = arr.copy().astype(np.float32) / 32768\n", " return arr\n", "\n", "\n", "def _array_to_int(arr):\n", " arr = _array_to_numpy(arr)\n", " # enforce int16\n", " if arr.dtype == np.int16:\n", " pass\n", " elif arr.dtype in (np.float32, np.float64):\n", " # alert if signal too high\n", " if np.abs(arr).max() > 1:\n", " raise ValueError(\"signal overflow\")\n", " arr = (arr * 32768).clip(-32768, 32767).astype(np.int16)\n", " else:\n", " # TODO: convert other formats\n", " raise NotImplementedError(\"unknown format\")\n", " # alert if audio too loud\n", " if np.abs(arr).mean() / 32768 >= 0.1:\n", " raise ValueError(\"audio seems too loud\")\n", " return arr\n", "\n", "\n", "def plot_ft(arr, window_size=1024, hop_length=512, n_mels=128, sr=22050):\n", " arr = _array_to_float(arr)\n", " window = np.hanning(window_size)\n", " stft= librosa.core.spectrum.stft(arr, n_fft=window_size, hop_length=hop_length, window=window)\n", " out = 2 * np.abs(stft) / np.sum(window)\n", " data = librosa.amplitude_to_db(out, ref=np.max)\n", " plt.figure(figsize=(16, 10))\n", " librosa.display.specshow(data, x_axis='time', y_axis='mel', sr=sr)\n", " plt.set_cmap('afmhot')\n", " \n", "\n", "def play_array(arr, sr=22050): \n", " arr = _array_to_int(arr)\n", " display(AudioSegment(\n", " arr.tobytes(), \n", " frame_rate=sr,\n", " sample_width=2, \n", " channels=1\n", " ))" ] }, { "cell_type": "code", "execution_count": 5, "id": "e9306a8b", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-01-20 21:00:13 optimizers:52] Apex was not found. Using the lamb or fused_adam optimizer will error out.\n", "################################################################################\n", "### WARNING, path does not exist: KALDI_ROOT=/mnt/matylda5/iveselyk/Tools/kaldi-trunk\n", "### (please add 'export KALDI_ROOT=' in your $HOME/.profile)\n", "### (or run as: KALDI_ROOT= python .py)\n", "################################################################################\n", "\n", "[NeMo W 2022-01-20 21:00:13 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", "[NeMo W 2022-01-20 21:00:13 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", "[NeMo W 2022-01-20 21:00:13 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", "[NeMo W 2022-01-20 21:00:13 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", "[NeMo W 2022-01-20 21:00:13 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", "[NeMo W 2022-01-20 21:00:13 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", "[NeMo W 2022-01-20 21:00:13 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[NeMo I 2022-01-20 21:00:13 cloud:56] Found existing object /home/georg/.cache/torch/NeMo/NeMo_1.6.0rc0/tts_en_fastpitch_align/b50e16c5d695b00855ae53d6ba4e4f7f/tts_en_fastpitch_align.nemo.\n", "[NeMo I 2022-01-20 21:00:13 cloud:62] Re-using file from: /home/georg/.cache/torch/NeMo/NeMo_1.6.0rc0/tts_en_fastpitch_align/b50e16c5d695b00855ae53d6ba4e4f7f/tts_en_fastpitch_align.nemo\n", "[NeMo I 2022-01-20 21:00:13 common:729] Instantiating model from pre-trained checkpoint\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo E 2022-01-20 21:00:14 vocabs:323] Torch distributed needs to be initialized before you initialized . This class is prone to data access race conditions. Now downloading corpora from global rank 0. If other ranks pass this before rank 0, errors might result.\n", "[NeMo W 2022-01-20 21:00:15 modelPT:135] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.\n", " Train config : \n", " dataset:\n", " _target_: nemo.collections.asr.data.audio_to_text.AudioToCharWithPriorAndPitchDataset\n", " manifest_filepath: /raid/LJSpeech/nvidia_ljspeech_train.json\n", " max_duration: null\n", " min_duration: 0.1\n", " int_values: false\n", " normalize: true\n", " sample_rate: 22050\n", " trim: false\n", " sup_data_path: /raid/LJSpeech/prior\n", " n_window_stride: 256\n", " n_window_size: 1024\n", " pitch_fmin: 80\n", " pitch_fmax: 640\n", " pitch_avg: 211.27540199742586\n", " pitch_std: 52.1851002822779\n", " vocab:\n", " notation: phonemes\n", " punct: true\n", " spaces: true\n", " stresses: true\n", " add_blank_at: None\n", " pad_with_space: true\n", " chars: true\n", " improved_version_g2p: true\n", " dataloader_params:\n", " drop_last: false\n", " shuffle: true\n", " batch_size: 32\n", " num_workers: 12\n", " \n", "[NeMo W 2022-01-20 21:00:15 modelPT:142] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \n", " Validation config : \n", " dataset:\n", " _target_: nemo.collections.asr.data.audio_to_text.AudioToCharWithPriorAndPitchDataset\n", " manifest_filepath: /raid/LJSpeech/nvidia_ljspeech_val.json\n", " max_duration: null\n", " min_duration: null\n", " int_values: false\n", " normalize: true\n", " sample_rate: 22050\n", " trim: false\n", " sup_data_path: /raid/LJSpeech/prior\n", " n_window_stride: 256\n", " n_window_size: 1024\n", " pitch_fmin: 80\n", " pitch_fmax: 640\n", " pitch_avg: 211.27540199742586\n", " pitch_std: 52.1851002822779\n", " vocab:\n", " notation: phonemes\n", " punct: true\n", " spaces: true\n", " stresses: true\n", " add_blank_at: None\n", " pad_with_space: true\n", " chars: true\n", " improved_version_g2p: true\n", " dataloader_params:\n", " drop_last: false\n", " shuffle: false\n", " batch_size: 32\n", " num_workers: 8\n", " \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[NeMo I 2022-01-20 21:00:15 features:264] PADDING: 1\n", "[NeMo I 2022-01-20 21:00:15 features:281] STFT using torch\n", "[NeMo I 2022-01-20 21:00:16 save_restore_connector:149] Model FastPitchModel was successfully restored from /home/georg/.cache/torch/NeMo/NeMo_1.6.0rc0/tts_en_fastpitch_align/b50e16c5d695b00855ae53d6ba4e4f7f/tts_en_fastpitch_align.nemo.\n" ] } ], "source": [ "# Load PastPitch\n", "from nemo.collections.tts.models import FastPitchModel\n", "spec_generator = FastPitchModel.from_pretrained(\"tts_en_fastpitch\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "29316557", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-01-20 21:00:36 modelPT:135] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.\n", " Train config : \n", " dataset:\n", " _target_: nemo.collections.tts.data.datalayers.AudioDataset\n", " manifest_filepath: /home/tkdrlf9202/Datasets/LibriTTS-22k/train-all.json\n", " max_duration: null\n", " min_duration: 0.75\n", " n_segments: 16384\n", " trim: false\n", " dataloader_params:\n", " drop_last: false\n", " shuffle: true\n", " batch_size: 8\n", " num_workers: 4\n", " \n", "[NeMo W 2022-01-20 21:00:36 modelPT:142] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \n", " Validation config : \n", " dataset:\n", " _target_: nemo.collections.tts.data.datalayers.AudioDataset\n", " manifest_filepath: /home/tkdrlf9202/Datasets/LibriTTS-22k/dev-clean-mini.json\n", " max_duration: null\n", " min_duration: null\n", " n_segments: -1\n", " trim: false\n", " dataloader_params:\n", " drop_last: false\n", " shuffle: false\n", " batch_size: 16\n", " num_workers: 1\n", " \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[NeMo I 2022-01-20 21:00:36 features:264] PADDING: 0\n", "[NeMo I 2022-01-20 21:00:36 features:281] STFT using torch\n", "[NeMo I 2022-01-20 21:00:36 features:283] STFT using exact pad\n", "[NeMo I 2022-01-20 21:00:36 features:264] PADDING: 0\n", "[NeMo I 2022-01-20 21:00:36 features:281] STFT using torch\n", "[NeMo I 2022-01-20 21:00:36 features:283] STFT using exact pad\n", "[NeMo I 2022-01-20 21:00:36 save_restore_connector:149] Model UnivNetModel was successfully restored from /home/georg/models/tts_en_libritts_multispeaker_univnet.nemo.\n" ] } ], "source": [ "# Load UnivNet\n", "from nemo.collections.tts.models import UnivNetModel\n", "model = UnivNetModel.restore_from(\"/home/georg/models/tts_en_lj_univnet.nemo\")\n", "# model = UnivNetModel.restore_from(\"/home/georg/models/tts_en_libritts_multispeaker_univnet.nemo\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "e1b07a6f", "metadata": {}, "outputs": [], "source": [ "# Generate audio\n", "import soundfile as sf\n", "parsed = spec_generator.parse(\"You can type your sentence here to get nemo to produce speech.\")\n", "spectrogram = spec_generator.generate_spectrogram(tokens=parsed)#, speaker=5)\n", "audio = model.convert_spectrogram_to_audio(spec=spectrogram)" ] }, { "cell_type": "code", "execution_count": 13, "id": "16ac21f3", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "play_array(audio)" ] }, { "cell_type": "code", "execution_count": null, "id": "78c7597a", "metadata": {}, "outputs": [], "source": [ "# ### Save the audio to disk in a file called speech.wav\n", "# sf.write(\"speech.wav\", audio.to('cpu').numpy(), 22050)" ] }, { "cell_type": "code", "execution_count": null, "id": "49acccfe", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "383e9c06", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "27eb4c9b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "69dd213b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e744b44f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c105d405", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }