{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "coated-usage",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "short-numbers",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ['CUDA_VISIBLE_DEVICES'] = ''"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "inclusive-hybrid",
"metadata": {},
"outputs": [],
"source": [
"import re\n",
"import pandas as pd\n",
"import numpy as np\n",
"import librosa\n",
"from scipy.io import wavfile\n",
"import librosa.display\n",
"from pydub import AudioSegment\n",
"from IPython.display import display_html\n",
"import torch\n",
"import tqdm\n",
"\n",
"\n",
"def _convert_array(arr, force_int=False, force_float=False):\n",
" if isinstance(arr, torch.Tensor):\n",
" arr = arr.cpu().detach().numpy()\n",
" assert(len(arr.shape) in (1, 2))\n",
" if len(arr.shape) == 2:\n",
" arr = arr.mean(0)\n",
" if force_int and \"float\" in str(arr.dtype):\n",
" arr = np.floor(arr.copy() * 32768).astype(np.int16)\n",
" if force_float and \"int\" in str(arr.dtype):\n",
" arr = arr.copy().astype(np.float32) / 32768\n",
" return arr\n",
"\n",
"\n",
"def plot_ft(arr, window_size=1024, hop_length=512, n_mels=128):\n",
" arr = _convert_array(arr, force_float=True)\n",
" window = np.hanning(window_size)\n",
" stft= librosa.core.spectrum.stft(arr, n_fft=window_size, hop_length=hop_length, window=window)\n",
" out = 2 * np.abs(stft) / np.sum(window)\n",
" data = librosa.amplitude_to_db(out, ref=np.max)\n",
"\n",
" plt.figure(figsize=(16, 10))\n",
" librosa.display.specshow(data, x_axis='time', y_axis='mel', sr=16000)\n",
" plt.set_cmap('afmhot')\n",
"\n",
"\n",
"def play_array(arr, sr=16000, nc=1):\n",
" # peak normalize to avoid clipping\n",
" if np.abs(arr).max() >= 1:\n",
" arr = arr / (0.1 + np.abs(arr).max())\n",
" arr = _convert_array(arr, force_int=True)\n",
" display_html(AudioSegment(\n",
" arr.tobytes(), \n",
" frame_rate=sr,\n",
" sample_width=2, \n",
" channels=nc\n",
" ))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "sudden-short",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "exact-invitation",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "optical-bidding",
"metadata": {},
"source": [
"## Noise suppression & echo removal"
]
},
{
"cell_type": "code",
"execution_count": 218,
"id": "presidential-listening",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.insert(0, '/data-ssd-3/georg/data/DNS-Challenge/NSNet2-baseline')\n",
"sys.path.insert(0, '/data-ssd-3/georg/code/bk/denoiser')\n",
"\n",
"# baseline model\n",
"from enhance_onnx import NSnet2Enhancer\n",
"fp = \"/data-ssd-3/georg/data/DNS-Challenge/NSNet2-baseline/nsnet2-20ms-baseline.onnx\"\n",
"base_model = NSnet2Enhancer(modelfile=fp)\n",
"\n",
"# facebook models\n",
"from denoiser import pretrained\n",
"class Args():\n",
" def __init__(self, model_path=None, dns48=None, dns64=None, master64=None, valentini_nc=None):\n",
" self.model_path = model_path\n",
" self.dns48 = dns48\n",
" self.dns64 = dns64\n",
" self.master64 = master64\n",
" self.valentini_nc = valentini_nc\n",
"fb_models = {\n",
" \"dns48\": pretrained.get_model(Args(dns48=True)),\n",
" \"dns64\": pretrained.get_model(Args(dns64=True)),\n",
" \"master64\": pretrained.get_model(Args(master64=True)),\n",
" \"valentini_nc\": pretrained.get_model(Args(valentini_nc=True)),\n",
"}\n",
"\n",
"def remove_noise_0(arr, dry=0.0):\n",
" \"\"\"using baseline model\"\"\"\n",
" arr = _convert_array(arr, force_float=True)\n",
" arr_enh = base_model(arr, 16000)[:len(arr)]\n",
" arr_enh = dry * arr + (1 - dry) * arr_enh\n",
" return arr_enh\n",
"\n",
"def remove_noise_1(arr, model_key=\"master64\", dry=0.0):\n",
" \"\"\"using facebook model\"\"\"\n",
" arr = _convert_array(arr, force_float=True)\n",
" arr_enh = fb_models[model_key](torch.from_numpy(arr).view(1, -1))\n",
" arr_enh = (arr_enh / max(arr_enh.abs().max().item(), 1)).cpu().detach().numpy()[0]\n",
" arr_enh = dry * arr + (1 - dry) * arr_enh\n",
" return arr_enh"
]
},
{
"cell_type": "code",
"execution_count": 238,
"id": "simplified-cuisine",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"dog.wav libri.wav street.wav\r\n"
]
}
],
"source": [
"!ls tmp/noise_removal"
]
},
{
"cell_type": "code",
"execution_count": 239,
"id": "induced-amsterdam",
"metadata": {},
"outputs": [],
"source": [
"fp = \"tmp/noise_removal/libri.wav\"\n",
"_, noisy = wavfile.read(fp)"
]
},
{
"cell_type": "code",
"execution_count": 242,
"id": "neural-bennett",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"play_array(noisy)\n",
"sim_clean = remove_noise_1(noisy, model_key=\"master64\", dry=0.0)\n",
"sim_noise = noisy - sim_clean\n",
"play_array(sim_clean)\n",
"# play_array(sim_noise)"
]
},
{
"cell_type": "code",
"execution_count": 236,
"id": "ignored-thermal",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"_, noisy = wavfile.read(\"tmp/noise_removal/street.wav\")\n",
"play_array(remove_noise_1(noisy))"
]
},
{
"cell_type": "code",
"execution_count": 237,
"id": "threatened-shore",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"_, noisy = wavfile.read(\"tmp/noise_removal/dog.wav\")\n",
"play_array(remove_noise_1(noisy))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "noble-lender",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "split-quest",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "excess-video",
"metadata": {},
"source": [
"## Speech enhancement"
]
},
{
"cell_type": "code",
"execution_count": 243,
"id": "entertaining-mediterranean",
"metadata": {},
"outputs": [],
"source": [
"from denoiser.demucs import Demucs\n",
"enh_model = Demucs(hidden=48, causal=False)\n",
"fp = \"/data-ssd-3/georg/code/denoiser/outputs/exp_dummy=new_data/best.th\"\n",
"state_dict = torch.load(fp, map_location='cpu')[\"state\"]\n",
"enh_model.load_state_dict(state_dict);"
]
},
{
"cell_type": "code",
"execution_count": 244,
"id": "adjacent-victor",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"arr_noisy, _ = torchaudio.load(\"tmp/amr_noisy.wav\")\n",
"play_array(arr_noisy)"
]
},
{
"cell_type": "code",
"execution_count": 245,
"id": "chicken-johnston",
"metadata": {},
"outputs": [],
"source": [
"out = enh_model(arr_noisy)\n",
"arr_enh = (out / max(out.abs().max().item(), 1)).cpu().detach()[0]"
]
},
{
"cell_type": "code",
"execution_count": 246,
"id": "mighty-institute",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"play_array(arr_enh)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "responsible-basic",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "welcome-brisbane",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "owned-stamp",
"metadata": {},
"source": [
"## Speech Synthesis"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "declared-wings",
"metadata": {},
"outputs": [],
"source": [
"# # Using NeMo\n",
"\n",
"# import soundfile as sf\n",
"# from nemo.collections.tts.models.base import SpectrogramGenerator, Vocoder\n",
"\n",
"# # tts_en_glowtts\n",
"# # tts_en_tacotron2 - best quality\n",
"# # tts_en_fastspeech2\n",
"# # tts_en_fastpitch - default\n",
"# # tts_en_talknet\n",
"# spec_generator = SpectrogramGenerator.from_pretrained(model_name=\"tts_en_tacotron2\")\n",
"\n",
"# # tts_hifigan - best\n",
"# # tts_melgan\n",
"# # tts_squeezewave\n",
"# # tts_uniglow\n",
"# # tts_waveglow_268m\n",
"# # tts_waveglow_88m\n",
"# vocoder = Vocoder.from_pretrained(model_name=\"tts_hifigan\")\n",
"\n",
"# def generate_speech(text):\n",
"# parsed = spec_generator.parse(text)\n",
"# spectrogram = spec_generator.generate_spectrogram(tokens=parsed)\n",
"# # [1, 80, 393] at 22.05 khz\n",
"# audio = vocoder.convert_spectrogram_to_audio(spec=spectrogram)\n",
"# return audio\n",
"\n",
"# # audio = generate_speech(\"And finally, this is it. Am I right?\")\n",
"# # play_array(audio, sr=22050)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "warming-scheme",
"metadata": {},
"outputs": [],
"source": [
"# # Using realtime VC toolkit\n",
"# import sys\n",
"# sys.path.insert(0, '/data-ssd-3/georg/code/Real-Time-Voice-Cloning')\n",
"# import pathlib\n",
"# from demo_cli import check_model_paths, Synthesizer, encoder, vocoder\n",
"\n",
"# d = \"/data-ssd-3/georg/models/realtime_vc/\"\n",
"\n",
"# encoder.load_model(pathlib.Path(d + \"encoder/saved_models/pretrained.pt\"))\n",
"# synthesizer = Synthesizer(pathlib.Path(d + \"synthesizer/saved_models/pretrained/pretrained.pt\"))\n",
"# vocoder.load_model(pathlib.Path(d + \"vocoder/saved_models/pretrained/pretrained.pt\"))\n",
"\n",
"# def generate_speech(text, style_fp=\"tmp/obama_16.wav\"):\n",
"# original_wav, sampling_rate = librosa.load(style_fp)\n",
"# preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate)\n",
"# embed = encoder.embed_utterance(preprocessed_wav)\n",
"# texts = [text]\n",
"# embeds = [embed]\n",
"# specs = synthesizer.synthesize_spectrograms(texts, embeds)\n",
"# spec = specs[0]\n",
"# generated_wav = vocoder.infer_waveform(spec)\n",
"# # There's a bug with sounddevice that makes the audio cut one second earlier, so we pad it.\n",
"# generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode=\"constant\")\n",
"# # Trim excess silences to compensate for gaps in spectrograms (issue #53)\n",
"# generated_wav = encoder.preprocess_wav(generated_wav)\n",
"# return generated_wav\n",
"\n",
"# text = \"Hey, my beautiful rabbit. Let's go for a little walk outside.\"\n",
"# out = generate_speech(text, style_fp=\"tmp/obama_16.wav\")\n",
"# play_array(out)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "immune-airplane",
"metadata": {},
"outputs": [],
"source": [
"# # MAKE X-VECTORS for espnet\n",
"# docker run -it kaldiasr/kaldi:latest bash\n",
"# docker cp /home/georg/notebooks/s2t/stuff/tmp/obama_16.wav d28acf07fd1:/tmp\n",
"# docker cp /data-ssd-3/georg/mfcc.conf d28acf07fd1:/tmp\n",
"# docker cp /data-ssd-3/georg/vad.conf d28acf07fd1:/tmp\n",
"# docker cp /data-ssd-3/georg/models/0008_sitw_v2_1a/exp/xvector_nnet_1a d28acf07fd1:/tmp\n",
"# mkdir /tmp/data\n",
"# echo \"001 /tmp/obama_16.wav\" > /tmp/data/wav.scp\n",
"# echo \"001 001\" > /tmp/data/spk2utt\n",
"# echo \"001 001\" > /tmp/data/utt2spk\n",
"\n",
"# cd /opt/kaldi/egs/wsj/s5\n",
"\n",
"# steps/make_mfcc.sh \\\n",
"# --write-utt2num-frames true \\\n",
"# --mfcc-config \"/tmp/mfcc.conf\" \\\n",
"# --nj 1 --cmd \"run.pl\" \\\n",
"# \"/tmp/data\" \"/tmp/log\" \"/tmp/mfcc\"\n",
"\n",
"# cd /opt/kaldi/egs/sre08/v1\n",
"\n",
"# sid/compute_vad_decision.sh --nj 1 --cmd \"run.pl\" \\\n",
"# --vad_config \"/tmp/vad.conf\" \\\n",
"# \"/tmp/data\" \"/tmp/log\" \"/tmp/mfcc\"\n",
"\n",
"# sid/nnet3/xvector/extract_xvectors.sh --cmd \"run.pl --mem 4G\" --nj 1 \\\n",
"# \"/tmp/xvector_nnet_1a\" \\\n",
"# \"/tmp/data\" \\\n",
"# \"/tmp/xvectors\"\n",
"\n",
"# docker cp d28acf07fd1:/tmp/xvectors/spk_xvector.ark /tmp/spk_xvector.ark"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "extended-antigua",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"https://zenodo.org/record/5521416/files/tts_train_xvector_vits_raw_phn_tacotron_g2p_en_no_space_train.total_count.ave.zip?download=1: 100%|██████████| 375M/375M [01:33<00:00, 4.20MB/s] \n"
]
}
],
"source": [
"model = Text2Speech.from_pretrained(\n",
" model_tag=\"kan-bayashi/libritts_xvector_vits\",\n",
" vocoder_tag=None,\n",
" device=\"cpu\",\n",
" speed_control_alpha=0.8,\n",
" noise_scale=0.667,\n",
" noise_scale_dur=0.8,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "studied-register",
"metadata": {},
"outputs": [],
"source": [
"# Using ESPNET\n",
"\n",
"import torch\n",
"from espnet2.bin.tts_inference import Text2Speech\n",
"import torchaudio\n",
"import kaldiio\n",
"from espnet2.gan_tts.vits.vits import VITS\n",
"from espnet_model_zoo.downloader import ModelDownloader\n",
"\n",
"# https://github.com/espnet/espnet_model_zoo/blob/master/espnet_model_zoo/table.csv\n",
"espnet_model_dict = {\n",
" \"libri\": \"kan-bayashi/libritts_gst+xvector_conformer_fastspeech2\",\n",
" \"vctk\": \"kan-bayashi/vctk_gst+xvector_tacotron2\",\n",
" \"vits\": \"kan-bayashi/vctk_full_band_multi_spk_vits\",\n",
"}\n",
"\n",
"vocoder_model_dict = {\n",
" \"libri\": \"parallel_wavegan/libritts_hifigan.v1\",\n",
" \"vctk\": \"parallel_wavegan/vctk_hifigan.v1\",\n",
" \"vits\": None,\n",
"}\n",
"\n",
"model_container = {\n",
" # best for specific speaker (bad oov)\n",
" \"libri\": Text2Speech.from_pretrained(\n",
" model_tag=espnet_model_dict[\"libri\"],\n",
" vocoder_tag=vocoder_model_dict[\"libri\"],\n",
" device=\"cpu\",\n",
" threshold=0.5,\n",
" speed_control_alpha=1.0,\n",
" ),\n",
" # best overall\n",
" \"vctk\": Text2Speech.from_pretrained(\n",
" model_tag=espnet_model_dict[\"vctk\"],\n",
" vocoder_tag=vocoder_model_dict[\"vctk\"],\n",
" device=\"cpu\",\n",
" threshold=0.5,\n",
" minlenratio=0.0,\n",
" maxlenratio=10.0,\n",
" use_att_constraint=False,\n",
" backward_window=1,\n",
" forward_window=3,\n",
" ),\n",
" # most natural\n",
" \"vits\": Text2Speech.from_pretrained(\n",
" model_tag=espnet_model_dict[\"vits\"],\n",
" vocoder_tag=vocoder_model_dict[\"vits\"],\n",
" device=\"cpu\",\n",
" speed_control_alpha=0.8,\n",
" noise_scale=0.667,\n",
" noise_scale_dur=0.8,\n",
" ),\n",
"}\n",
"\n",
"d = ModelDownloader()\n",
"\n",
"model_dir = os.path.dirname(d.download_and_unpack(espnet_model_dict[\"libri\"])[\"train_config\"])\n",
"xvector_fp = model_dir + \"/../../dump/xvector/tr_no_dev/spk_xvector.ark\"\n",
"libri_speaker_list = [v for k, v in kaldiio.load_ark(xvector_fp)]\n",
"\n",
"model_dir = os.path.dirname(d.download_and_unpack(espnet_model_dict[\"vctk\"])[\"train_config\"])\n",
"xvector_fp = model_dir + \"/../../dump/xvector/tr_no_dev/spk_xvector.ark\"\n",
"vctk_speaker_list = [v for k, v in kaldiio.load_ark(xvector_fp)]\n",
"\n",
"speaker_embed_container = {\n",
" \"libri\": libri_speaker_list,\n",
" \"vctk\": vctk_speaker_list,\n",
"}\n",
"\n",
"def _get_speaker_embed(model_type, spk_id=0):\n",
" model = model_container[model_type]\n",
" model_tag = espnet_model_dict[model_type]\n",
" voice_ref = torch.randn(50000,) * 0.01\n",
" if isinstance(model.tts, VITS) or model.tts.spk_embed_dim is None:\n",
" return voice_ref, None\n",
" if spk_id == -1:\n",
" spk_vectors = {k: v for k, v in kaldiio.load_ark(\"/tmp/spk_xvector.ark\")}\n",
" spk_vector = spk_vectors[\"001\"]\n",
" voice_ref, sr = torchaudio.load(\"tmp/obama_24.wav\")\n",
" voice_ref = voice_ref[0]\n",
" assert(sr == model.fs)\n",
" return voice_ref, spk_vector\n",
" return voice_ref, speaker_embed_container[model_type][spk_id]\n",
"\n",
"\n",
"def generate_speech(text, model_type=\"libri\", spk_id=0):\n",
" model = model_container[model_type]\n",
" if model_type == \"libri\":\n",
" voice_ref, spk_emb = _get_speaker_embed(model_type, spk_id=spk_id)\n",
" sids = None\n",
" elif model_type == \"vctk\":\n",
" voice_ref, spk_emb = _get_speaker_embed(model_type, spk_id=spk_id)\n",
" sids = None\n",
" elif model_type == \"vits\":\n",
" voice_ref = None\n",
" spk_emb = None\n",
" sids = torch.from_numpy(np.array([[spk_id]], dtype=np.int32))\n",
" else:\n",
" raise NotImplementedError()\n",
" model = model_container[model_type]\n",
" with torch.no_grad():\n",
" speech = model(text, sids=sids, speech=voice_ref, spembs=spk_emb)[\"wav\"]\n",
" return model.fs, speech\n",
"\n",
"# text = \"And finally, this is it.\" #\" Am I right? I don't think so.\"\n",
"text = (\n",
" \"And in particular, since 2019, it's worth noting that we've grown the size of the combined \"\n",
" \"New Zealand Meridian and Powershop customer bases by 20 percent and the total volume of energy sold \"\n",
" \"through our retail channels by 14 percent while spending less than $13.4.\"\n",
")\n",
"\n",
"# sr, speech = generate_speech(text, model_type=\"libri\", spk_id=5)\n",
"# sr, speech = generate_speech(text, model_type=\"vctk\", spk_id=1)\n",
"sr, speech = generate_speech(text, model_type=\"vits\", spk_id=3)\n",
"\n",
"play_array(speech, sr=sr)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "suitable-creature",
"metadata": {},
"outputs": [],
"source": [
"# model_container[\"vits\"].decode_conf[\"alpha\"] = 1.0"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "fitting-mozambique",
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'NoneType' object has no attribute 'unsqueeze'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mspeech\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspeech\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspembs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"wav\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/venvs/py37/lib/python3.7/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/py37/lib/python3.7/site-packages/espnet2/bin/tts_inference.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, text, speech, durations, spembs, sids, lids, decode_conf)\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malways_fix_seed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[0mset_all_random_seed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m \u001b[0moutput_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minference\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[0;31m# calculate additional metrics\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/py37/lib/python3.7/site-packages/espnet2/tts/espnet_model.py\u001b[0m in \u001b[0;36minference\u001b[0;34m(self, text, speech, spembs, sids, lids, durations, pitch, energy, **decode_config)\u001b[0m\n\u001b[1;32m 297\u001b[0m \u001b[0minput_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 298\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 299\u001b[0;31m \u001b[0moutput_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minference\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0minput_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mdecode_config\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 300\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 301\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0moutput_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"feat_gen\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/py37/lib/python3.7/site-packages/espnet2/gan_tts/vits/vits.py\u001b[0m in \u001b[0;36minference\u001b[0;34m(self, text, feats, sids, spembs, lids, durations, noise_scale, noise_scale_dur, alpha, max_len, use_teacher_forcing)\u001b[0m\n\u001b[1;32m 602\u001b[0m \u001b[0mnoise_scale_dur\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnoise_scale_dur\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 603\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 604\u001b[0;31m \u001b[0mmax_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 605\u001b[0m )\n\u001b[1;32m 606\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwav\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwav\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0matt_w\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0matt_w\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mduration\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdur\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/py37/lib/python3.7/site-packages/espnet2/gan_tts/vits/generator.py\u001b[0m in \u001b[0;36minference\u001b[0;34m(self, text, text_lengths, feats, feats_lengths, sids, spembs, lids, dur, noise_scale, noise_scale_dur, alpha, max_len, use_teacher_forcing)\u001b[0m\n\u001b[1;32m 453\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspk_embed_dim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 454\u001b[0m \u001b[0;31m# (B, global_channels, 1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 455\u001b[0;31m \u001b[0mg_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspemb_proj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspembs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 456\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mg\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 457\u001b[0m \u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mg_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'unsqueeze'"
]
}
],
"source": [
"text = \"Type a sentence here.\"\n",
"# text = \"And finally, this is it. Am I right? I don't think so.\"\n",
"\n",
"sids = torch.from_numpy(np.array([[1]], dtype=np.int32))\n",
"\n",
"with torch.no_grad():\n",
" speech = model(text, sids=sids, speech=None, spembs=None)[\"wav\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "weird-measure",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "liked-relationship",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 12,
"id": "protective-point",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# text = \"And finally, this is it.\" #\" Am I right? I don't think so.\"\n",
"text = (\n",
" \"And in particular, since 2019, it's worth noting that we've grown the size of the combined \"\n",
" \"New Zealand Meridian and Powershop customer bases by 20 percent and the total volume of energy sold \"\n",
" \"through our retail channels by 14 percent while spending less than $13.4.\"\n",
")\n",
"text = \"Type a sentence here.\"\n",
"\n",
"# sr, speech = generate_speech(text, model_type=\"libri\", spk_id=5)\n",
"# sr, speech = generate_speech(text, model_type=\"vctk\", spk_id=1)\n",
"# sr, speech = generate_speech(text, model_type=\"vits\", spk_id=3)\n",
"sr, speech = generate_speech(text, model_type=\"vits\", spk_id=0)\n",
"\n",
"play_array(speech, sr=sr)"
]
},
{
"cell_type": "code",
"execution_count": 142,
"id": "vocal-october",
"metadata": {},
"outputs": [],
"source": [
"text = \"Hey! Thanks for checking in Lock-E, things are going well.\"\n",
"model_container[\"vits\"].decode_conf[\"alpha\"] = 0.7\n",
"sr, speech1 = generate_speech(text, model_type=\"vits\", spk_id=19)\n",
"\n",
"text = \"I'm learning.\"\n",
"model_container[\"vits\"].decode_conf[\"alpha\"] = 1.1\n",
"sr, speech3 = generate_speech(text, model_type=\"vits\", spk_id=19)\n",
"\n",
"speech = np.hstack([\n",
" speech1,\n",
"# np.zeros((8000,), dtype=np.float32),\n",
" np.zeros((50000,), dtype=np.float32),\n",
" speech3,\n",
"])\n",
"play_array(speech, sr=sr)\n",
"\n",
"# torchaudio.save(\"lachy.wav\", torch.from_numpy(speech).view(1, -1), sr)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "super-macintosh",
"metadata": {},
"outputs": [],
"source": [
"# # Example\n",
"# conversation = [\n",
"# (\n",
"# 0,\n",
"# \"Hello?\",\n",
"# ),\n",
"# (\n",
"# 1,\n",
"# \"Hello! This is Jennifer Lange-Rebner.\",\n",
"# ),\n",
"# (\n",
"# 0,\n",
"# \"Hey, Jennifer, great to meet you. This is Vivek. I think Jack should be joining us shortly from our side. I don't know if he's on.\",\n",
"# ),\n",
"# (\n",
"# 1,\n",
"# \"Okay, great!\",\n",
"# ),\n",
"# (\n",
"# 0,\n",
"# \"Vivek, did you give Jennifer context on who we are and what we do and what we're looking for?\",\n",
"# ),\n",
"# (\n",
"# 2,\n",
"# \"Yes, let me give some brief context first. So yes, we're an investment fund, and we're looking into Notable Health, and we're primarily looking into the process automation space in health care.\",\n",
"# ),\n",
"# ]\n",
"\n",
"# # using vctk\n",
"# vctk_speaker_ids = [2, 3, 7]\n",
"# silence_segment = np.zeros((4000,), dtype=np.float32)\n",
"# speech_segments = []\n",
"# sr = 0\n",
"# for spk_idx, text in tqdm.tqdm(conversation):\n",
"# speech_segments.append(silence_segment)\n",
"# for sent in re.split(r\"(?<=[\\.\\?\\!])\\s+\", text):\n",
"# sr, speech = generate_speech(sent, model_type=\"vctk\", spk_id=vctk_speaker_ids[spk_idx])\n",
"# speech_segments.append(speech)\n",
"# speech_segments.append(silence_segment)\n",
"# # clean up some artifacts\n",
"# a = []\n",
"# for e in speech_segments:\n",
"# if len(e) == 4000:\n",
"# a.append(e)\n",
"# elif len(e) <= 50000:\n",
"# a.append(e[2000:])\n",
"# else:\n",
"# a.append(e[2000:-8000])\n",
"# speech = np.hstack(a)\n",
"\n",
"# # using vits\n",
"# vits_speaker_ids = [3, 11, 13]\n",
"# silence_segment = np.zeros((8000,), dtype=np.float32)\n",
"# speech_segments = []\n",
"# sr = 0\n",
"# for spk_idx, text in tqdm.tqdm(conversation):\n",
"# speech_segments.append(silence_segment)\n",
"# sr, speech = generate_speech(text, model_type=\"vits\", spk_id=vits_speaker_ids[spk_idx])\n",
"# speech_segments.append(speech)\n",
"# speech_segments.append(silence_segment)\n",
"# speech = np.hstack(speech_segments)\n",
"\n",
"# play_array(speech, sr=sr)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "future-aaron",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "intermediate-apollo",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "spiritual-original",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "regulated-commissioner",
"metadata": {},
"source": [
"## Speech Separation"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "powered-teens",
"metadata": {},
"outputs": [],
"source": [
"# https://github.com/facebookresearch/svoice\n",
"# facebook model can be trained with custom but does not (yet) have pretrained"
]
},
{
"cell_type": "code",
"execution_count": 71,
"id": "bearing-nylon",
"metadata": {},
"outputs": [],
"source": [
"# # speechbrain sepformer\n",
"# from speechbrain.pretrained import SepformerSeparation as separator\n",
"# import torchaudio\n",
"\n",
"# model = separator.from_hparams(source=\"speechbrain/sepformer-whamr16k\", savedir='pretrained_models/sepformer-whamr16k')\n",
"\n",
"# # for custom file, change path\n",
"# est_sources = model.separate_file(path='speechbrain/sepformer-whamr16k/test_mixture16k.wav', savedir=\"tmp\") \n",
"\n",
"# AudioSegment.from_wav(\"tmp/test_mixture16k.wav\")\n",
"# play_array(est_sources[:, :, 0])\n",
"# play_array(est_sources[:, :, 1])"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "16e15215",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using cache found in /home/georg/.cache/torch/hub/JorisCos_asteroid_master\n",
"/home/georg/venvs/ml/lib/python3.8/site-packages/huggingface_hub/file_download.py:588: FutureWarning: `cached_download` is the legacy way to download files from the HF hub, please consider upgrading to `hf_hub_download`\n",
" warnings.warn(\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# using asteroid\n",
"from torch import hub\n",
"import torchaudio\n",
"\n",
"# models here: https://huggingface.co/models?filter=asteroid\n",
"model = hub.load('JorisCos/asteroid', 'conv_tasnet', 'JorisCos/ConvTasNet_Libri2Mix_sepnoisy_16k')\n",
"\n",
"arr, _ = torchaudio.load(\"mix.mp3\")\n",
"out = model.separate(arr)[0].numpy()\n",
"\n",
"play_array(arr)\n",
"play_array(out[0, :])\n",
"play_array(out[1, :])\n",
"play_array(arr - out.sum(axis=0))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "false-ready",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using cache found in /home/georg/.cache/torch/hub/JorisCos_asteroid_master\n",
"/home/georg/venvs/ml/lib/python3.8/site-packages/huggingface_hub/file_download.py:588: FutureWarning: `cached_download` is the legacy way to download files from the HF hub, please consider upgrading to `hf_hub_download`\n",
" warnings.warn(\n"
]
},
{
"ename": "FileNotFoundError",
"evalue": "[Errno 2] No such file or directory: '/data-ssd-3/georg/data/LibriMix/data/Libri2Mix/wav16k/min/metadata/metrics_test_mix_both.csv'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/tmp/ipykernel_2927413/2534557302.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0mbase_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"/data-ssd-3/georg/data/LibriMix/data/\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbase_dir\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"Libri2Mix/wav16k/min/metadata/metrics_test_mix_both.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;31m# mixture_test_mix_both\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m )\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m 584\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 585\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 587\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 588\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 481\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 484\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 809\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 811\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 812\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 813\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1038\u001b[0m )\n\u001b[1;32m 1039\u001b[0m \u001b[0;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1040\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1041\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1042\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/pandas/io/parsers/c_parser_wrapper.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0;31m# open handles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 52\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/pandas/io/parsers/base_parser.py\u001b[0m in \u001b[0;36m_open_handles\u001b[0;34m(self, src, kwds)\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 221\u001b[0m \"\"\"\n\u001b[0;32m--> 222\u001b[0;31m self.handles = get_handle(\n\u001b[0m\u001b[1;32m 223\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 701\u001b[0m \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m handle = open(\n\u001b[0m\u001b[1;32m 703\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/data-ssd-3/georg/data/LibriMix/data/Libri2Mix/wav16k/min/metadata/metrics_test_mix_both.csv'"
]
}
],
"source": [
"# using asteroid\n",
"from torch import hub\n",
"import torchaudio\n",
"\n",
"# models here: https://huggingface.co/models?filter=asteroid\n",
"model = hub.load('JorisCos/asteroid', 'conv_tasnet', 'JorisCos/ConvTasNet_Libri2Mix_sepnoisy_16k')\n",
"\n",
"base_dir = \"/data-ssd-3/georg/data/LibriMix/data/\"\n",
"df = pd.read_csv(base_dir + \"Libri2Mix/wav16k/min/metadata/metrics_test_mix_both.csv\")\n",
"# mixture_test_mix_both\n",
"\n",
"row = df.iloc[2998]\n",
"fp = base_dir + \"Libri2Mix/wav16k/min/test/mix_both/{}.wav\".format(row['mixture_ID'])\n",
"arr, _ = torchaudio.load(fp)\n",
"out = model.separate(arr)[0].numpy()\n",
"\n",
"play_array(arr)\n",
"play_array(out[0, :])\n",
"play_array(out[1, :])\n",
"play_array(arr - out.sum(axis=0))"
]
},
{
"cell_type": "code",
"execution_count": 181,
"id": "cordless-ending",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 0.00912476, 0.01550293, 0.01513672, ..., -0.03311157,\n",
" -0.01278687, 0.00805664]], dtype=float32)"
]
},
"execution_count": 181,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"arr.numpy()"
]
},
{
"cell_type": "code",
"execution_count": 182,
"id": "thorough-serve",
"metadata": {},
"outputs": [],
"source": [
"out = model.separate(arr.numpy())[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "toxic-marker",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cultural-malawi",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "floating-steam",
"metadata": {},
"source": [
"## Music source separation"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "pediatric-stationery",
"metadata": {},
"outputs": [],
"source": [
"import torchaudio\n",
"from demucs.separate import load_pretrained, load_track, apply_model"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "private-worst",
"metadata": {},
"outputs": [],
"source": [
"sep_model = load_pretrained(\"demucs_quantized\")\n",
"\n",
"def separate(mix):\n",
" ref = mix.mean(0)\n",
" wav = (mix - ref.mean()) / ref.std()\n",
" sources = apply_model(sep_model, wav, shifts=0, split=True, overlap=0.25)\n",
" sources = sources * ref.std() + ref.mean()\n",
" sources = [s / max(1.01 * s.abs().max(), 1) for s in sources]\n",
" sources = [s.mean(axis=0) for s in sources]\n",
" return sources"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "ahead-conservation",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"mix = load_track(\"tmp/music_mix/sample_1.mp3\", \"cpu\", 2, 44100)\n",
"play_array(mix, sr=44100)"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "backed-indication",
"metadata": {},
"outputs": [],
"source": [
"sources = separate(mix)"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "popular-jordan",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"drums\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"bass\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"vocals\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"other\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"print(\"drums\")\n",
"play_array(sources[0], sr=44100)\n",
"print(\"bass\")\n",
"play_array(sources[1], sr=44100)\n",
"print(\"vocals\")\n",
"play_array(sources[3], sr=44100)\n",
"print(\"other\")\n",
"play_array(sources[2], sr=44100)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "generic-consolidation",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "civic-middle",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "fifth-sleeping",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "political-large",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "shaped-reply",
"metadata": {},
"source": [
"## Scene Annotation?"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "elementary-invention",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "rapid-easter",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "grateful-contamination",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "working-counter",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}