{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "8fae8f9c", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:33:31.780214Z", "start_time": "2023-12-22T14:33:28.671259Z" } }, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n", "\n", "import numpy as np\n", "from suno_utils.audio import Audio\n", "from suno_utils.gpt.chirp_v2 import (\n", " generate_audio,\n", " generate_audio_stream,\n", " preload_models,\n", " codec_decode,\n", " GenerationConfig,\n", " semantic_encode,\n", " codec_encode,\n", ")\n", "import torch" ] }, { "cell_type": "code", "execution_count": null, "id": "d57160f2", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:33:31.784351Z", "start_time": "2023-12-22T14:33:31.782495Z" } }, "outputs": [], "source": [ "#2023-11-29_23-55-25" ] }, { "cell_type": "code", "execution_count": null, "id": "01a25410", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:33:31.980703Z", "start_time": "2023-12-22T14:33:31.785979Z" } }, "outputs": [], "source": [ "!ls /app/suno/checkpoints/2023-12-17_19-09-54" ] }, { "cell_type": "code", "execution_count": null, "id": "d7778d50", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:33:31.985406Z", "start_time": "2023-12-22T14:33:31.983444Z" } }, "outputs": [], "source": [ "#2023-12-06_02-54-39 v5 # 2023-12-02_23-30-13 fine tune v4 # 2023-12-01_17-14-49 fine tune v3 # 2023-10-17_16-01-32 is the default one " ] }, { "cell_type": "code", "execution_count": null, "id": "9108708b", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:33:55.132034Z", "start_time": "2023-12-22T14:33:31.987293Z" } }, "outputs": [], "source": [ "_ = preload_models(\n", " gpt_ckpt_path=\"/app/suno/checkpoints/2023-12-17_19-09-54/last_ckpt_infer.pt\",\n", " tokenizer_path=\"/home/victor/data/models/chirp_v2/tokenizer_60k.json\",\n", " semantic_ckpt_path=\"/home/victor/data/models/chirp_v2/mert_25.pt\",\n", " semantic_centroids_path=\"/home/victor/data/models/chirp_v2/mert_25_2x4k.npy\",\n", " codec_ckpt_path=\"/home/victor/data/models/chirp_v2/dac_2c_25x8.pt\",\n", " fasttext_ckpt_path=\"/home/victor/data/models/lid.176.bin\",\n", " load_whisper=True,\n", " use_gpu=True,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "072adcda", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:33:55.137450Z", "start_time": "2023-12-22T14:33:55.134198Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "[Verse 2]\n", "一壶漂泊浪迹天涯难入喉\n", "你走之后酒暖回忆思念瘦\n", "水向东流时间怎么偷\n", "花开就一次成熟我却错过\n", "[Chorus]\n", "谁在用琵琶弹奏一曲东风破\n", "岁月在墙上剥落看见小时候\n", "犹记得那年我们都还很年幼\n", "而如今琴声幽幽我的等候你没听过\n", "[end]\n", "\"\"\"\n", "# [Verse 1]\n", "# 一盏离愁孤灯伫立在窗口\n", "# 我在门后假装你人还没走\n", "# 旧地如重游月圆更寂寞\n", "# 夜半清醒的烛火不忍苛责我\n", "# [Chorus][Chorus][Chorus]\n", "# 谁在用琵琶弹奏一曲东风破\n", "# 枫叶将故事染色结局我看透\n", "# 篱笆外的古道我牵着你走过\n", "# 荒烟蔓草的年头就连分手都很沉默" ] }, { "cell_type": "code", "execution_count": null, "id": "3f5a71c6", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:34:47.425004Z", "start_time": "2023-12-22T14:33:55.139467Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=text,\n", " text_tags=\"pop woman\",\n", " n_batch=4,\n", " max_gen_duration_s=90,\n", " # cfg_coef=1\n", " # stream=False,\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "bf3538f1", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:34:51.363512Z", "start_time": "2023-12-22T14:34:47.427224Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play(compress=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "7d6c3514", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:36:09.108598Z", "start_time": "2023-12-22T14:34:51.365010Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=text.replace(\"[end]\", \"\"),\n", " text_tags=\"pop woman\",\n", " n_batch=4,\n", " max_gen_duration_s=90,\n", " # cfg_coef=1\n", " # stream=False,\n", " )\n", ")\n", "for e in out_audio:\n", " e.play(compress=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "bee63248", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:36:09.138376Z", "start_time": "2023-12-22T14:36:09.110845Z" } }, "outputs": [], "source": [ "torch.cuda.empty_cache()" ] }, { "cell_type": "code", "execution_count": null, "id": "5fa595a6", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:36:10.204759Z", "start_time": "2023-12-22T14:36:09.139603Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "[verse]\n", "Walking down the street, feeling so alive\n", "Got my head in the clouds, got a gleam in my eye\n", "Every step I take, it's like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "(oh-oh-oh)\n", "\n", "[chorus]\n", "Life is like a high-wire act, we're dancing in the sky\n", "No need to worry, no need to ask why\n", "With a little bit of courage, we can chase our dreams\n", "No matter what comes our way, we'll always be a team\n", "(we're unstoppable, yeah)\n", "[end]\n", "\"\"\"\n", "# Almost heaven, West Virginia\n", "# Blue Ridge Mountains, Shenandoah River\n", "# Life is old there, older than the trees\n", "# Younger than the mountains, growin' like a breeze\n", "\n", "# Country roads, take me home\n", "# To the place I belong\n", "# West Virginia, mountain mama\n", "# Take me home, country roads\n", "\n", "# text = \"\"\"\n", "# Beaux comme l'Eden, pays auvergnat,\n", "# Ou de Marseille/\n", "# Rivières de Savoie\n", "# La vie y est / plus vieille\n", "# Que Moīse\n", "# Plus jeune que les montagnes,\n", "# Forte comme la brise.\n", " \n", "# Petite route,/ramène-moi\n", "# Au pays, / de mon âme\n", "# Jura, le Doubs, /montagne aimée,\n", "# Ramène-moi, petite route.\n", " \n", "# Tous mes souvenirs/ forment (comme) un choeur\n", "# Peu de loisirs, /pour les femmes de mineurs,\n", "# Sombre et vieillie, une eau-forte, peinte au ciel,\n", "# Lune en forme de bouteille\n", "# Me met la larme à l' œil\n", "# [end]\n", "# \"\"\"\n", " \n", "# Petite route,/ramène-moi\n", "# Au pays, / de mon âme\n", "# Jura, le Doubs, /montagne aimée,\n", "# Ramène-moi, petite route.\n", " \n", "# J'entends sa voix, / quand, le matin, elle m'appelle\n", "# La radio me redit / de revoir mon pays\n", "# Et sur ces petites routes /, d'un coup, je sens\n", "# Qu' j'aurais dû / être chez moi\n", "# Hier, oui, hier, oui.\n", " \n", "# Petite route,/ramène-moi\n", "# Au pays, / de mon âme\n", "# Jura, le Doubs, /montagne aimée,\n", "# Ramène-moi, petite route." ] }, { "cell_type": "code", "execution_count": null, "id": "0e279fcc", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:36:10.254517Z", "start_time": "2023-12-22T14:36:10.207665Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "[Verse]\n", "ah ah ah ah ah ah ah\n", "Do re mi fa so la ti do one\n", "ah ah ah ah ah ah ah\n", "Do re mi fa so la ti do two\n", "[Chorus]\n", "ah ah ah ah ah ah ah\n", "Do re mi fa so la ti do three\n", "ah ah ah ah ah ah ah\n", "Do re mi fa so la ti do four\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "a7a281ab", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:37:12.962529Z", "start_time": "2023-12-22T14:36:10.255882Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=text,\n", " text_tags=\"gregorian chant\",\n", " text_neg_control_tags=\"instrumental noise\",\n", " n_batch=4,\n", " max_gen_duration_s=80,\n", " # cfg_coef=1.25,\n", " # temp_semantic=0.90,\n", " # cfg_coef_tags=2.0,\n", " # rep_penality=0.125\n", " top_p_semantic=None,\n", " top_p_coarse=None,\n", " # stream=False,\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "88cf31a0", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:37:19.237935Z", "start_time": "2023-12-22T14:37:12.964308Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play(compress=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "08d15aa1", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:37:19.241682Z", "start_time": "2023-12-22T14:37:19.239843Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "[Verse]\n", "Ni MaBi, Ni MaBi, can't get you out my mind\n", "Ni MaBi, Ni MaBi, catchin' feelings all the time\n", "Ni MaBi, Ni MaBi, got me dancin' in the light\n", "Ni MaBi, Ni MaBi, you're the rhythm in my life (ooh-yeah)\n", "Every day, every night, it's you that I crave\n", "Ni MaBi, Ni MaBi, gotta find a way to say\n", "Ni MaBi, Ni MaBi, you're the beat that makes me move\n", "Ni MaBi, Ni MaBi, can't resist your groove (ooh-yeah)\n", "\n", "[Chorus]\n", "Ni MaBi, Ni MaBi, in my heart and on my lips (on my lips)\n", "Ni MaBi, Ni MaBi, with you the melody never slips (never slips)\n", "Ni MaBi, Ni MaBi, I'm addicted to your vibe (to your vibe)\n", "Ni MaBi, Ni MaBi, you're the rhythm in my life (ooh-yeah)\n", "\n", "[outro]\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "ef2a197a", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:40:19.446503Z", "start_time": "2023-12-22T14:40:19.443848Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "[Verse]\n", "I used to think it's all a game\n", "That somethin' more was to blame\n", "But now I see, it's in the details, baby\n", "Just a little tweak here and there\n", "Can make our model beyond compare\n", "Yeah, it's the small things that drive it crazy (ooh-yeah)\n", "\n", "[Chorus]\n", "Fine tuning, baby, it's the key\n", "Turnin' up the signal for you and me\n", "When we're off, we find our way back again (oh-oh)\n", "Fine tuning, baby, it's the truth\n", "We make it work, we're bulletproof\n", "Together we train a model that's the best (oh-oh-oh-oh)\n", "[End]\n", "\"\"\"\n", "\n", "# [Verse]\n", "# In a world of endless possibilities\n", "# There's a rhythm that keeps callin' to me\n", "# I'm searching for that perfect harmony\n", "# Fine tuning infinity, yeah (Baby)\n", "# I'm on a quest, gotta find the key\n", "# To unlock the secrets of eternity\n", "# I'm chasing dreams, I won't let 'em slip away\n", "# Gonna make the most of every single day\n", "\n", "# [Verse]\n", "# Ni MaBi, Ni MaBi, can't get you out my mind\n", "# Ni MaBi, Ni MaBi, catchin' feelings all the time\n", "# Ni MaBi, Ni MaBi, got me dancin' in the light\n", "# Ni MaBi, Ni MaBi, you're the rhythm in my life (ooh-yeah)\n", "# Every day, every night, it's you that I crave\n", "# Ni MaBi, Ni MaBi, gotta find a way to say\n", "# Ni MaBi, Ni MaBi, you're the beat that makes me move\n", "# Ni MaBi, Ni MaBi, can't resist your groove (ooh-yeah)\n", "\n", "# [Chorus]\n", "# Ni MaBi, Ni MaBi, in my heart and on my lips (on my lips)\n", "# Ni MaBi, Ni MaBi, with you the melody never slips (never slips)\n", "# Ni MaBi, Ni MaBi, I'm addicted to your vibe (to your vibe)\n", "# Ni MaBi, Ni MaBi, you're the rhythm in my life (ooh-yeah)" ] }, { "cell_type": "code", "execution_count": null, "id": "5865edaf", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:40:20.339871Z", "start_time": "2023-12-22T14:40:20.312379Z" } }, "outputs": [], "source": [ "torch.cuda.empty_cache()" ] }, { "cell_type": "code", "execution_count": null, "id": "36589135", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:41:09.596551Z", "start_time": "2023-12-22T14:40:21.099560Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=text,\n", " text_tags=\"pop\",\n", " n_batch=4,\n", " max_gen_duration_s=90,\n", " # cfg_coef=1.6,\n", " temp_semantic=0.95,\n", " # cfg_coef_tags=2.5,\n", " # cfg_coef=1\n", " # stream=False,\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "d3005f6a", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:41:13.634812Z", "start_time": "2023-12-22T14:41:09.598174Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play(compress=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "9592f21d", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:09.251994Z", "start_time": "2023-12-22T14:38:09.250144Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "I am fine-tuning version 13.\n", "Hello Suno.\n", "[Outro]\n", "\"\"\"\n", "# text = \"\"\"\n", "# Nants ingonyama bagithi baba\n", "# Sithi uhm ingonyama\n", "# Nants ingonyama bagithi baba\n", "# Sithi uhhmm ingonyama\n", "# Ingonyama\n", "# Siyo Nqoba\n", "# Ingonyama\n", "# Ingonyama nengw' enamabala\n", "# Ingonyama nengw' enamabala\n", "# Ingonyama nengw' enamabala\n", "# Ingonyama nengw' enamabala\n", "# Ingonyama nengw' enamabala\n", "# Ingonyama nengw' enamabala (se-to-kwa!)\n", "# Ingonyama nengw' enamabala (asana)\n", "# Ingonyama nengw' enamabala (se-to-kwa!)\n", "# Ingonyama nengw' enamabala (se-to-kwa!)\n", "# \"\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "282c044f", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:54.157749Z", "start_time": "2023-12-22T14:38:09.253467Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=text,\n", " text_tags=\"pop\",\n", " n_batch=4,\n", " max_gen_duration_s=60,\n", " # cfg_coef=1\n", " # stream=False,\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "08069fc1", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.609025Z", "start_time": "2023-12-22T14:38:54.159231Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play(compress=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "34624075", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.931456Z", "start_time": "2023-12-22T14:38:57.610746Z" } }, "outputs": [], "source": [ "BREAK" ] }, { "cell_type": "code", "execution_count": null, "id": "c3372767", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.933406Z", "start_time": "2023-12-22T14:38:57.933397Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "[Verse 1]\n", "笑你我枉花光心計\n", "愛競逐鏡花那美麗\n", "怕幸運會轉眼遠逝\n", "為貪嗔喜惡怒著迷\n", "[Verse 2]\n", "責你我太貪功戀勢\n", "怪大地眾生太美麗\n", "悔舊日太執信約誓\n", "為悲歡哀怨妒著迷\n", "[Verse 3]\n", "啊 捨不得璀璨俗世\n", "啊 躲不開癡戀的欣慰\n", "啊 找不到色相代替\n", "啊 參一生參不透這條難題\n", "[Chorus]\n", "吞風吻雨葬落日未曾徬徨\n", "欺山趕海踐雪徑也未絕望\n", "拈花把酒偏折煞世人情狂\n", "憑這兩眼與百臂或千手不能防\n", "[Chorus]\n", "天闊闊雪漫漫共誰同航\n", "這沙滾滾水皺皺笑著浪蕩\n", "貪歡一餉偏教那女兒情長埋葬\n", "\"\"\" * 10" ] }, { "cell_type": "code", "execution_count": null, "id": "c12aceec", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.934389Z", "start_time": "2023-12-22T14:38:57.934379Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=text,\n", " text_tags=\"chinese pop\",\n", " n_batch=2,\n", " max_gen_duration_s=120,\n", " # stream=False,\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "73dd0216", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.935199Z", "start_time": "2023-12-22T14:38:57.935188Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "918ae244", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.935832Z", "start_time": "2023-12-22T14:38:57.935824Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=\"[christmas]\",\n", " text_tags=\"christmas happy music\",\n", " n_batch=2,\n", " max_gen_duration_s=120,\n", " # stream=False,\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "9d8bed82", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.936678Z", "start_time": "2023-12-22T14:38:57.936670Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play()" ] }, { "cell_type": "markdown", "id": "33cd6562", "metadata": {}, "source": [ "# continue canon" ] }, { "cell_type": "code", "execution_count": null, "id": "8f9e0366", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.937441Z", "start_time": "2023-12-22T14:38:57.937433Z" } }, "outputs": [], "source": [ "input_audio = Audio.from_file(\"audios/Pachelbel - Canon In D Major. Best version. [NlprozGcs80].wav\").get_segment(from_s=60, to_s=80)" ] }, { "cell_type": "code", "execution_count": null, "id": "37d50b41", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.938161Z", "start_time": "2023-12-22T14:38:57.938153Z" } }, "outputs": [], "source": [ "input_audio = Audio.from_file(\"audios/Pachelbel - Canon In D Major. Best version. [NlprozGcs80].wav\")" ] }, { "cell_type": "code", "execution_count": null, "id": "2f9d3323", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.938947Z", "start_time": "2023-12-22T14:38:57.938939Z" } }, "outputs": [], "source": [ "input_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "001ec800", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.939803Z", "start_time": "2023-12-22T14:38:57.939793Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=\"\",\n", " text_tags=\"canon\",\n", " n_batch=4,\n", " max_gen_duration_s=30,\n", " # stream=False,\n", " ),\n", " history_audio=input_audio\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "6309815d", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.940381Z", "start_time": "2023-12-22T14:38:57.940372Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "3d1a8965", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.941151Z", "start_time": "2023-12-22T14:38:57.941144Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=\"[start] \\n\" + \"I love you \\n\" * 5 + \"[end]\",\n", " text_tags=\"\",\n", " n_batch=4,\n", " max_gen_duration_s=30,\n", " cfg_coef=1.8,\n", " # stream=False,\n", " ),\n", " history_audio=input_audio\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "c36e347c", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.942108Z", "start_time": "2023-12-22T14:38:57.942100Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play()" ] }, { "cell_type": "markdown", "id": "2270b189", "metadata": {}, "source": [ "# Oracle" ] }, { "cell_type": "code", "execution_count": null, "id": "772a5d20", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.942795Z", "start_time": "2023-12-22T14:38:57.942787Z" } }, "outputs": [], "source": [ "input_audio = Audio.from_file(\"audios/test_swift.mp3\")\n", "\n", "test_input_audio = input_audio.get_segment(from_s=180, to_s=200)\n", "test_input_audio.play(compress=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "d4b5d48c", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.943507Z", "start_time": "2023-12-22T14:38:57.943499Z" } }, "outputs": [], "source": [ "oracle_semantic_arr = semantic_encode(test_input_audio, n_codebooks=1)\n", "oracle_coarse_arr = codec_encode(test_input_audio)[:oracle_semantic_arr.shape[0], :]\n", "print(oracle_semantic_arr.shape, oracle_coarse_arr.shape)\n", "oracle_arr = np.concatenate(\n", " [oracle_semantic_arr, oracle_coarse_arr],\n", " axis=-1,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "808f5130", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.944231Z", "start_time": "2023-12-22T14:38:57.944223Z" } }, "outputs": [], "source": [ "input_oracle_arr = oracle_arr[:, :1]\n", "print(input_oracle_arr.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "752ba22f", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.945074Z", "start_time": "2023-12-22T14:38:57.945067Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=\" \" * 20,\n", " text_tags=\"\",\n", " n_batch=4,\n", " max_gen_duration_s=30,\n", " oracle_arr=input_oracle_arr,\n", " ),\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "bba0d8c9", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.945992Z", "start_time": "2023-12-22T14:38:57.945984Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play()\n", " print(e.array_float[-100:].mean(axis=1))" ] }, { "cell_type": "code", "execution_count": null, "id": "ad979c8b", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.946559Z", "start_time": "2023-12-22T14:38:57.946551Z" } }, "outputs": [], "source": [ "input_pavarotti = Audio.from_file(\"audios/test_pavarotti.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "id": "6f0f8785", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.947370Z", "start_time": "2023-12-22T14:38:57.947362Z" } }, "outputs": [], "source": [ "input_pavarotti_seg = input_pavarotti.get_segment(from_s=152, to_s=172)\n", "input_pavarotti_seg.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "4e91f1d1", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.948191Z", "start_time": "2023-12-22T14:38:57.948181Z" } }, "outputs": [], "source": [ "pavarotti_oracle_semantic_arr = semantic_encode(input_pavarotti_seg, n_codebooks=1)\n", "pavarotti_oracle_coarse_arr = codec_encode(input_pavarotti_seg)[\n", " : pavarotti_oracle_semantic_arr.shape[0], :\n", "]\n", "print(pavarotti_oracle_semantic_arr.shape, pavarotti_oracle_coarse_arr.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "6502b74a", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.948955Z", "start_time": "2023-12-22T14:38:57.948945Z" } }, "outputs": [], "source": [ "masked_oracle_semantic_arr = oracle_semantic_arr.copy().astype(np.int16)\n", "# masked_oracle_semantic_arr[0::5, :] = -1\n", "masked_oracle_coarse_arr = pavarotti_oracle_coarse_arr.copy().astype(np.int16)\n", "# masked_oracle_coarse_arr[20:, :] = -1" ] }, { "cell_type": "code", "execution_count": null, "id": "b97481d6", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.949729Z", "start_time": "2023-12-22T14:38:57.949721Z" } }, "outputs": [], "source": [ "mixed_oracle_arr = np.concatenate(\n", " [masked_oracle_semantic_arr], #, masked_oracle_coarse_arr[:, 0:1]],\n", " axis=-1,\n", ")\n", "print(mixed_oracle_arr.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "76dcc73a", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.950465Z", "start_time": "2023-12-22T14:38:57.950455Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=\" \" * 20,\n", " text_tags=\"\",\n", " n_batch=4,\n", " max_gen_duration_s=30,\n", " oracle_arr=mixed_oracle_arr,\n", " ),\n", " # history_audio=input_pavarotti_seg\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "84e04fd4", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.951235Z", "start_time": "2023-12-22T14:38:57.951227Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play()\n", " print(e.array_float[-100:].mean(axis=1))" ] }, { "cell_type": "code", "execution_count": null, "id": "b79d93dc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "edc87efa", "metadata": {}, "source": [ "# continuation" ] }, { "cell_type": "code", "execution_count": null, "id": "8b6938c4", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.951990Z", "start_time": "2023-12-22T14:38:57.951980Z" } }, "outputs": [], "source": [ "input_audio = Audio.from_file(\"audios/test_swift.mp3\")\n", "test_input_audio = input_audio.get_segment(from_s=180, to_s=200)\n", "test_input_audio.play(compress=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "d4cbb76b", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.952848Z", "start_time": "2023-12-22T14:38:57.952839Z" } }, "outputs": [], "source": [ "hoot_encode([test_input_audio])" ] }, { "cell_type": "code", "execution_count": null, "id": "984e8c9b", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.953544Z", "start_time": "2023-12-22T14:38:57.953535Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=\"[chorus] purple rain\" * 20,\n", " text_tags=\"\",\n", " n_batch=4,\n", " max_gen_duration_s=30,\n", " # stream=False,\n", " ),\n", " history_audio=test_input_audio,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "296b34eb", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.954289Z", "start_time": "2023-12-22T14:38:57.954280Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "df83ed93", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.955036Z", "start_time": "2023-12-22T14:38:57.955026Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "On a vibrant summer morning, curiosity drew me to the innovative realm of Suno AI. Their latest\n", "masterpiece, Chirp, known for its unique ability to generate music from words, promised a bold\n", "step forward in music creation. I dived into their platform, typing in familiar lyrics. As I clicked\n", "'Create,' the ensuing symphony seamlessly blended human emotion and machine precision,\n", "heralding a new era in the convergence of music, words, and machine learning\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "1b5a2270", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.955814Z", "start_time": "2023-12-22T14:38:57.955804Z" } }, "outputs": [], "source": [ "out_audio = generate_audio(\n", " GenerationConfig(\n", " text=text,\n", " text_tags=\"hifi rap\",\n", " n_batch=2,\n", " max_gen_duration_s=120,\n", " # stream=False,\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "7b5656bb", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.956570Z", "start_time": "2023-12-22T14:38:57.956560Z" } }, "outputs": [], "source": [ "for e in out_audio:\n", " e.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "84046437", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.957163Z", "start_time": "2023-12-22T14:38:57.957155Z" } }, "outputs": [], "source": [ "[1, 1, 1].any()" ] }, { "cell_type": "code", "execution_count": null, "id": "12dd84ce", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.957855Z", "start_time": "2023-12-22T14:38:57.957847Z" } }, "outputs": [], "source": [ "import torch" ] }, { "cell_type": "code", "execution_count": null, "id": "12cee8f4", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.958574Z", "start_time": "2023-12-22T14:38:57.958567Z" } }, "outputs": [], "source": [ "a = torch.tensor([[1.,0, -1.]])" ] }, { "cell_type": "code", "execution_count": null, "id": "ae48a13e", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.959459Z", "start_time": "2023-12-22T14:38:57.959451Z" } }, "outputs": [], "source": [ "a.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "387a0670", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.960333Z", "start_time": "2023-12-22T14:38:57.960323Z" } }, "outputs": [], "source": [ "a.unsqueeze(-1).shape" ] }, { "cell_type": "code", "execution_count": null, "id": "cb1992fb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3e115ea2", "metadata": { "ExecuteTime": { "end_time": "2023-12-06T03:57:06.155345Z", "start_time": "2023-12-06T03:57:05.137102Z" } }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "50a3e18e", "metadata": { "ExecuteTime": { "end_time": "2023-12-22T14:38:57.961080Z", "start_time": "2023-12-22T14:38:57.961070Z" } }, "outputs": [], "source": [ "aligned_lyrics = hoot_encode_and_align(test_input_audio, prior_texts=\"\"\"\n", "Can't you see that I'm the one\n", "Who understands you?\n", "Been here all along\n", "So, why can't you see?\n", "You belong with me\n", "Standing by and waiting at your backdoor\n", "All this time how could you not know, baby?\n", "You belong with me\n", "You belong with me\n", "\"\"\", align_word_p=0.1)" ] }, { "cell_type": "code", "execution_count": null, "id": "83fec26f", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }