{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"5\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import numpy as np\n", "from suno_utils.audio import Audio\n", "from suno_utils.gpt.chirp_v2_5 import (\n", " _get_model_if_needed,\n", " GenerationConfig,\n", ")\n", "from suno_utils.gpt.generation_engine import (\n", " make_prompt,\n", " align_codes,\n", " make_request,\n", ")\n", "from suno_utils.gpt.engine import Engine, Request, Job\n", "from suno_utils.gpt.prompt import Prompt\n", "from suno_utils.tasks.dac_2c_12cb import (\n", " preload_models as preload_codec_models,\n", " decode_stream_to_full_audio as codec_decode_stream_to_full_audio,\n", " encode as codec_encode,\n", " decode as codec_decode,\n", ")\n", "from suno_utils.tasks.mert_25 import (\n", " preload_models as preload_semantic_models,\n", " encode as semantic_encode,\n", ")\n", "\n", "\n", "def _interleave(semantic_arr, n_factor=1):\n", " new_semantic_arr = (\n", " np.zeros(\n", " (semantic_arr.shape[0] * n_factor, semantic_arr.shape[-1]),\n", " dtype=semantic_arr.dtype,\n", " )\n", " + cfg.semantic_vocab_size\n", " - 1\n", " )\n", " new_semantic_arr[::n_factor] = semantic_arr\n", " return new_semantic_arr\n", "\n", "\n", "def process_audio(audio, cfg, n_factor=1):\n", " audio = audio.normalize_volume(-16)\n", " sem_arr = semantic_encode(audio, device=\"cpu\")\n", " if n_factor > 1:\n", " sem_arr = _interleave(sem_arr, n_factor=n_factor)\n", " coarse_arr = codec_encode(audio)\n", " n_frames = min(sem_arr.shape[0], coarse_arr.shape[0])\n", " sem_arr = sem_arr[:n_frames, : cfg.semantic_n_codebooks]\n", " coarse_arr = coarse_arr[:n_frames, : cfg.coarse_n_codebooks]\n", "\n", " a_arr = np.concatenate([sem_arr, coarse_arr], axis=-1)\n", " return a_arr\n", "\n", "\n", "def load_audio(fp):\n", " return Audio.from_file(fp, n_channels=2, sample_rate=48_000, byte_width=2)\n", "\n", "\n", "N_BATCH = 2\n", "MAX_STREAMS = N_BATCH * 4\n", "\n", "# preload codec\n", "_ = preload_codec_models(\"/app/suno/models/chirp_v2/dac_2c_25x12.pt\")\n", "\n", "# preload mert\n", "_ = preload_semantic_models(\n", " checkpoint_filepath=\"/app/suno/models/chirp_v2/mert_25.pt\",\n", " centroids_filepath=\"/app/suno/models/chirp_v2/mert_25_2x4k.npy\",\n", " device=\"cpu\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# import torch\n", "# sd1 = torch.load(\"/app/suno/data/dpo/models/model_30b_ft_t3.pt\", map_location=\"cpu\")\n", "# sd2 = torch.load(\"/app/suno/checkpoints/2024-09-14_17-41-31/last_ckpt_infer.pt\", map_location=\"cpu\")\n", "# for k, v in sd1[\"model\"].items():\n", "# sd1[\"model\"][k] = (sd1[\"model\"][k] + sd2[\"model\"][k]) / 2\n", "# torch.save(sd1, \"/app/suno/tmp/guetta_mix_ckpt_infer.pt\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "USE_COMPILE = False\n", "gpt_ckpt_path = _get_model_if_needed(\n", " # \"/app/suno/checkpoints/2024-09-26_03-57-23/last_ckpt_infer.pt\", # 13b chef+\n", " # \"/app/suno/checkpoints/2024-09-27_20-28-55/500_ckpt_infer.pt\", # 13b rym\n", " # \"/app/suno/models/model_30b_fix_ft2_20k.pt\", # 30b raw\n", " \"/app/suno/data/dpo/models/model_30b_ft_t3.pt\", # 30b dpo (used for guetta ft)\n", " # \"/app/suno/checkpoints/2024-09-14_17-41-31/last_ckpt_infer.pt\", # guetta ft\n", " # \"/app/suno/checkpoints/2024-09-19_15-32-34/last_ckpt_infer.pt\", # guetta ft+dpo\n", " # \"/app/suno/tmp/guetta_mix_ckpt_infer.pt\", # 50/50 dpo/guetta\n", ")\n", "engine = Engine(\n", " gpt_ckpt_path,\n", " \"/app/suno/models/chirp_v2/tokenizer_60k.json\",\n", " max_sequences=MAX_STREAMS,\n", " compile=USE_COMPILE,\n", ")\n", "model = engine.model\n", "cfg = model.config\n", "tokenizer = engine.tokenizer" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Basic preds" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "Of mist and moss,\n", "Through haze and heather,\n", "A field of folk,\n", "Stood, fought together;\n", "In wind and weather.\n", "\n", "With gleam of gold\n", "And glint of feather,\n", "Buckler and spear,\n", "A medley of mirth,\n", "They came\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # text=\"[instrumental]\",\n", " # text_tags=\"dark epic trailer music, violin, vocalizations, female\",\n", " # text_tags=\"pop, power ballad, female, violin\",\n", " # text_tags=\"bluegrass, female\",\n", " # text_tags=\"gospel\",\n", " text_tags=\"bluegrass, female\",\n", " # text_tags=\"upbeat\",\n", " # text_tags=\"sea shanty\",\n", " cfg_coef=1.1,\n", " cfg_coef_tags=2,\n", " n_repeat_tags=3,\n", " text_start_control_tags=\"{start:0}\",\n", " # text_start_control_tags=\"{start:0;vocals:intro}\",\n", " # text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", " # text_end_control_tags=\"{end_s:55;original_duration_s:55}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " # temp_semantic=0.85,\n", " # temp_coarse=0.9,\n", " # top_k_semantic=None,\n", " # top_k_coarse=None,\n", " # top_p_semantic=0.95,\n", " # top_p_coarse=None,\n", " n_batch=1,\n", " min_eos_p=0.1,\n", " # allow_eos=False,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "audios = []\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "[Intro]\n", "I can feel the heat risin'\n", "Everything is on fire\n", "Today's a painful re-\n", "Minder of why\n", "It can only get brighter\n", "The further you put it behind ya\n", "But right now I'm on the\n", "Inside lookin' out, ‘cause—\n", "\n", "[Chorus]\n", "I'm standin' in the flames\n", "And it’s a beautiful kind of pain\n", "Settin' fire to yesterday\n", "To find the light, find the light, find the light\n", "Standin' in the flames\n", "And it’s a beautiful kind of pain\n", "Settin' fire to yesterday\n", "Find the light, find the light, find the light\n", "\n", "[Verse 1]\n", "Yesterday was the tornado warning, today's like the morning after\n", "Your world is torn in half, you wake in its wake\n", "To start the mourning process and rebuilding, you're still a work in progress\n", "Today's a whole new chapter, it's like an enormous asthma\n", "Thunderstorm has passed ya, you weathered it and poked its\n", "Eye out with the thornbush that ya used to smell the roses\n", "Stopped to inhale, can't even tell your nose is stuffed\n", "So focused on the bright side then you floor the gas pedal\n", "And hit the corner faster, more assertive, never looking back\n", "May hit the curb, but every day's a new learning curve as ya\n", "Steer through life, sometimes you might not wanna swerve but you have to\n", "To avert a disaster, lucky no permanent damage\n", "‘Cause they hurt you so ba\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " text_tags=\"pop, hip-hop\",\n", " # text_tags=\"gospel\",\n", " # text_tags=\"Rap, In English, USA, Midwest Rap, Detroit Rap, Alternative, Hip-Hop\",\n", " cfg_coef=1.1,\n", " cfg_coef_tags=2,\n", " n_repeat_tags=3,\n", " text_start_control_tags=\"{start:0}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " n_batch=1,\n", " min_eos_p=0.5,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60 * 2,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "audios = []\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# text = \"\"\"\n", "# [Opening Shout]\n", "# Leader: Steel City, are you ready to praise?\n", "# All: GLORY HALLELUJAH!\n", "\n", "# [Call and Response Intro]\n", "# Leader: Who's the cornerstone of Pittsburgh?\n", "# All: JESUS!\n", "# Leader: Who's the fire in our furnace?\n", "# All: JESUS!\n", "# Leader: Who's forging our salvation?\n", "# All: JESUS! JESUS! JESUS!\n", "\n", "# [Verse 1]\n", "# In this valley of steel, once shadowed by sin (Oh yes!)\n", "# The Lord's mighty hammer came crashing in (Praise Him!)\n", "# He melted our hearts, poured us in His mold (Hallelujah!)\n", "# Now Pittsburgh's got a testimony to be told! (Tell it!)\n", "\n", "# [Chorus]\n", "# We're the Steel City Choir, singing for the Lord! (Yes we are!)\n", "# Our praise rises higher than smoke ever soared! (Rise up!)\n", "# From the ashes of sin to glory divine (Oh the glory!)\n", "# Pittsburgh's revival is right on time! (It's revival time!)\n", "\n", "# [Call and Response]\n", "# Leader: Can these dry bones live?\n", "# All: Yes they can!\n", "# Leader: Can this steel praise Him?\n", "# All: Yes it can!\n", "# Leader: Will Pittsburgh be saved?\n", "# All: Yes we will! Saved by grace!\n", "\n", "# [Verse 2]\n", "# The enemy thought he had us bound in chains (But he didn't!)\n", "# But God's holy fire melted those refrains (Freedom!)\n", "# Now every steel mill is an altar of praise (Lift it up!)\n", "# And every worker a minister of grace! (Preach it!)\n", "\n", "# [Chorus]\n", "# We're the Steel City Choir, singing for the Lord! (Yes we are!)\n", "# Our praise rises higher than smoke ever soared! (Rise up!)\n", "# From the ashes of sin to glory divine (Oh the glory!)\n", "# Pittsburgh's revival is right on time! (It's revival time!)\n", "\n", "# [Bridge]\n", "# (Spoken) Brothers and sisters, the Lord is moving in Pittsburgh!\n", "# Can you feel the Holy Ghost fire in the furnace of your soul?\n", "# Let me hear you say:\n", "\n", "# [Call and Response Bridge]\n", "# Leader: I'm tried!\n", "# All: In the fire!\n", "# Leader: I'm purified!\n", "# All: By the Holy Ghost!\n", "# Leader: I'm molded!\n", "# All: In His image!\n", "# Leader: I'm Pittsburgh!\n", "# All: SAVED BY GRACE!\n", "\n", "# [Final Verse]\n", "# The rivers of Jordan flow through our streets (Cleanse us!)\n", "# Every knee bowing, every tongue speaks (Praise Him!)\n", "# From the slag heaps of sin to mountains of faith (Hallelujah!)\n", "# Pittsburgh's transformed by amazing grace! (Thank you Jesus!)\n", "\n", "# [Final Chorus]\n", "# We're the Steel City Choir, singing for the Lord! (Yes we are!)\n", "# Our praise rises higher than smoke ever soared! (Rise up!)\n", "# From the ashes of sin to glory divine (Oh the glory!)\n", "# Pittsburgh's revival is right on time! (It's revival time!)\n", "\n", "# [Closing Call and Response]\n", "# Leader: Steel City!\n", "# All: Redeemed!\n", "# Leader: Steel City!\n", "# All: Revived!\n", "# Leader: Steel City!\n", "# All: GLORIFIED!\n", "\n", "# [Final Shout]\n", "# All: PITTSBURGH FOR JESUS! AMEN AND AMEN!\n", "# \"\"\"\n", "\n", "text = \"\"\"\n", "[Verse]\n", "In the city of bridges and steel\n", "We forge our pride on the anvil of will\n", "Metal sparks ignite the night\n", "In Pittsburgh we shine so bright\n", "\n", "[Verse 2]\n", "Iron veins run through our streets\n", "Every heartbeat hammer beats\n", "Steelworkers raise their hands\n", "In this town we make our stand\n", "\n", "[Chorus]\n", "Steel city strong forever we'll be\n", "Blazing a trail in the land of the free\n", "From the mills to the sky so tall\n", "Pittsburgh's power standing strong for all\n", "\n", "[Verse 3]\n", "Smoke stacks kiss the morning air\n", "Hard work lingers everywhere\n", "Men and women side by side\n", "In steel we share our pride\n", "\n", "[Chorus]\n", "Steel city strong forever we'll be\n", "Blazing a trail in the land of the free\n", "From the mills to the sky so tall\n", "Pittsburgh's power standing strong for all\n", "\n", "[Bridge]\n", "Rivers flow with history\n", "Past to future endlessly\n", "Here we rise and here we fall\n", "Steel unites us Pittsburgh calls\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # text_tags=\"edm, female vocals\",\n", " # text_tags=\"Gospel\",\n", " text_tags=\"Gospel, Soulful, Uplifting, Female Vocals, Choir Backing, Evocative, Organ Accompaniment, Spiritual, Joyful, Emotional Harmonies\",\n", " # text_tags=\"Rap, In English, USA, Midwest Rap, Detroit Rap, Alternative, Hip-Hop\",\n", " # text_tags=\"Dance-Pop, Electro House, Electropop, Festival Progressive House, Future Rave, Tech House\",\n", " cfg_coef=1.1,\n", " cfg_coef_tags=2,\n", " n_repeat_tags=3,\n", " # cfg_coef_tags_max_steps=None,\n", " text_start_control_tags=\"{start:0}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " n_batch=1,\n", " min_eos_p=0.5,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60 * 2,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "audios = []\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "gconf = GenerationConfig(\n", " text=\"[instrumental]\",\n", " # text_tags=\"instrumental, Melodic Techno, Future Rave, Melodic House, 2023\",\n", " text_tags=\"instrumental, Film Score, Cinematic Classical, Scottish Folk Music, Celtic New Age, war, spring, melancholic, epic, sombre, orchestral\",\n", " cfg_coef=1,\n", " cfg_coef_tags=3,\n", " n_repeat_tags=3,\n", " # cfg_coef_tags_max_steps=None,\n", " # text_start_control_tags=\"{start:60;remaining:60;duration:120}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " n_batch=1,\n", " min_eos_p=0.5,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "audios = []\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Continue" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = load_audio(\"tmp/50cent.wav\").get_segment(from_s=24.99, to_s=45)\n", "# history_text = \"\"\"\n", "# Welcome to the candy shop\n", "\n", "# Yeah, uh-huh\n", "# So seductive\n", "\n", "# I take you to the candy shop\n", "# I let you lick the lollipop\n", "# Go 'head, girl, don't you stop\n", "# \"\"\"\n", "# in_history_arr = process_audio(audio, cfg)\n", "# audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "audio = load_audio(\"../../samples/glitch_gen.mp3\").get_segment(to_s=60)\n", "history_text = None\n", "in_history_arr = process_audio(audio, cfg)\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# # Audio.from_youtube(\"x_bkzkO6n28\", n_channels=2, sample_rate=48_000).to_wav(\"../samples/gospel.wav\")\n", "# audio = Audio.from_file(\"../samples/gospel.wav\", n_channels=2, sample_rate=48_000, byte_width=2)\n", "# audio = audio.get_segment(from_s=95, to_s=115.01)\n", "# history_text = \"\"\"\n", "# hold me close\n", "# (Rain down) rain down on me, (oh, come on here, somebody) rain down on me\n", "# (Fill me) fill me with Your precious Holy Ghost\n", "# (Rain down) rain down on me, (rain down, here we go) rain down on me\n", "# \"\"\"\n", "# in_history_arr = process_audio(audio, cfg)\n", "# audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = load_audio(\"../samples/rihanna.wav\").get_segment(40, 60.01)\n", "# history_text = \"\"\"\n", "# Even though you've lost your mind\n", "\n", "# Just gonna stand there and watch me burn\n", "# Well, that's alright because I like the way it hurts\n", "# Just gonna stand there and\n", "# \"\"\"\n", "# in_history_arr = process_audio(audio, cfg, n_factor=1)\n", "# audio.normalize_volume().play()\n", "# # audio.normalize_volume().to_hq_mp3(\"beautiful_pain.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# # Audio.from_youtube(\"EnkUHXYWiOg\", n_channels=2, sample_rate=48_000).to_wav(\"../samples/radetzky.wav\")\n", "# audio = load_audio(\"../samples/radetzky.wav\").get_segment(from_s=99.99, to_s=120)\n", "# history_text = \"\"\n", "# in_history_arr = process_audio(audio, cfg)\n", "# audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = load_audio(\"../samples/walking_down.mp3\").get_segment(from_s=0, to_s=9.99)\n", "# history_text = \"\"\n", "# in_history_arr = process_audio(audio, cfg)\n", "# audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Audio.from_youtube(\"9bFHsd3o1w0\", n_channels=2, sample_rate=48_000).to_wav(\"tmp/titanic.wav\")\n", "# audio = load_audio(\"../samples/titanic.wav\").get_segment(from_s=19.49, to_s=39.5)\n", "# history_text = \"\"\"\n", "# Every night in my dreams\n", "# I see you, I feel you\n", "# That is how I know you go on\n", "# \"\"\"\n", "# in_history_arr = process_audio(audio, cfg)\n", "# audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=49.99, to_s=70)\n", "# history_text = \"\"\"\n", "# It's like I've been awakened\n", "# Every rule, I had you breakin'\n", "# The risk that I'm takin'\n", "# I'm never gonna shut you out\n", "\n", "# Everywhere I'm lookin' now\n", "# I'm surrounded by your embrace\n", "# Baby, I can see your halo\n", "# \"\"\"\n", "# in_history_arr = process_audio(audio, cfg)\n", "# audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "codec_decode(codec_encode(audio)).play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "Almost Heaven, West Virginia\n", "Blue Ridge Mountains, Shenandoah River\n", "Life is old there, older than the trees\n", "Younger than the mountains, growing like a breeze\n", "\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "\"\"\"\n", "text = \"\"\"\n", "[Verse]\n", "Walking down the street feeling so alive\n", "Got my head in the clouds with a gleam in my eyes\n", "Every step I take is like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "\n", "[Chorus]\n", "oooo\n", "Life is like a high-wire act\n", "We're dancing in the sky\n", "It's a worry no need to ask why\n", "With a little bit of courage\n", "\"\"\"\n", "\n", "# text = \"\"\"\n", "# And I don't know if I was awake or asleep when I wrote this\n", "# All I know is, you came to me when I was at my lowest\n", "# You picked me up, breathed new life in me, I owe my life to you\n", "# But for the life of me, I don't see why you don't see like I do\n", "# But it just dawned on me you lost a son\n", "# Demons fightin' you, it's dark\n", "# Let me turn on the lights and brighten me and enlighten you\n", "# I don't think you realize what you mean to me\n", "# Not the slightest clue\n", "# 'Cause me and you were like a crew, I was like your sidekick\n", "# You gon' either wanna fight when I get off this fuckin' mic\n", "# Or you gon' hug me, but I'm outta options\n", "# There's nothin' else I can do 'cause—\n", "\n", "# [Chorus]\n", "# I'm about to lose my mind\n", "# You've been gone for so long\n", "# I'm runnin' out of time\n", "# I need a doctor, call me a doctor\n", "# I need a doctor, doctor\n", "# To bring me back to life\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# Rain down on me (rain) (ah yeah)\n", "# L-let it fall on me (rain) (come on)\n", "# Rain down on me (rain)\n", "# L-let it fall on me (rain) (come on, everybody do like this, do like this)\n", "# Rain down on me (rain) (here we go, here we go, here we go)\n", "# \"\"\"\n", "\n", "# in_oracle_array = np.zeros((25*max_gen_duration_s, 1), dtype=np.int32)\n", "# in_oracle_array[:] = cfg.semantic_mask_token\n", "\n", "gconf = GenerationConfig(\n", " # oracle_arr=in_oracle_arr[:,:1],\n", " text=text,\n", " # text=\"\",\n", " # text_tags=\"female vocals\",\n", " history_arr=in_history_arr,\n", " history_text=history_text,\n", " cfg_coef=1.0,\n", " cfg_coef_tags=0,\n", " cfg_coef_neg_tags=0,\n", " n_repeat_tags=1,\n", " # temp_semantic=0.8,\n", " # temp_coarse=0.8,\n", " max_gen_duration_s=80,\n", " n_batch=1,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "audios = []\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audios[0].to_hq_mp3(\"out/base_continue_3.mp3\")\n", "# audios[1].to_hq_mp3(\"out/base_continue_4.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Oracle" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = load_audio(\"../samples/rihanna.wav\").get_segment(40, 60.01)\n", "# text = \"\"\"\n", "# Even though you've lost your mind\n", "\n", "# Just gonna stand there and watch me burn\n", "# Well, that's alright because I like the way it hurts\n", "# Just gonna stand there and\n", "# \"\"\"\n", "# in_oracle_arr = process_audio(audio, cfg, n_factor=1)\n", "# # in_oracle_arr[:,:1] = cfg.semantic_mask_token\n", "# audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "audio = load_audio(\"../../samples/glitch_gen.mp3\").get_segment(from_s=0, to_s=39.99)\n", "text = \"\"\"\n", "Walking down the street feeling so alive\n", "Got my head in the clouds with a gleam in my eyes\n", "Every step I take is like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "\n", "Life is like a high-wire act\n", "We're dancing in the sky\n", "It's a worry no need to ask why\n", "With a little bit of courage\n", "\"\"\"\n", "in_oracle_arr = process_audio(audio, cfg, n_factor=1)\n", "in_oracle_arr[:, :1] = cfg.semantic_mask_token\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=49.99, to_s=70)\n", "# text = \"\"\"\n", "# It's like I've been awakened\n", "# Every rule, I had you breakin'\n", "# The risk that I'm takin'\n", "# I'm never gonna shut you out\n", "\n", "# Everywhere I'm lookin' now\n", "# I'm surrounded by your embrace\n", "# Baby, I can see your halo\n", "# \"\"\"\n", "# in_oracle_arr = process_audio(audio, cfg, n_factor=1)\n", "# # in_oracle_arr[:,:1] = cfg.semantic_mask_token\n", "# audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = Audio.from_file(\"../samples/eminem_doctor.wav\", n_channels=2, sample_rate=48_000, byte_width=2)\n", "# audio = audio.get_segment(from_s=0, to_s=50.01)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = load_audio(\"../samples/leander2.mp3\").get_segment(from_s=0, to_s=40.01)\n", "# text = None\n", "# in_oracle_arr = process_audio(audio, cfg, n_factor=1)\n", "# # in_oracle_arr[:,:1] = cfg.semantic_mask_token\n", "# audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# codec_decode(codec_encode(audio)[:,:4]).normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "gconf = GenerationConfig(\n", " text=text,\n", " text_tags=\"ballad, female\",\n", " oracle_arr=in_oracle_arr[:, :1],\n", " n_batch=1,\n", " cfg_coef=1.2, # .3,\n", " cfg_coef_tags=2,\n", " cfg_coef_neg_tags=0,\n", " max_gen_duration_s=40,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " # temp_semantic=0.9,\n", " # temp_coarse=0.7,\n", " # top_k_semantic=None,\n", " # top_k_coarse=None,\n", " # top_p_semantic=None,\n", " # top_p_coarse=None,\n", " allow_eos=False,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "audios = []\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audios[0].to_hq_mp3(\"out/base_oracle_3.mp3\")\n", "# audios[1].to_hq_mp3(\"out/base_oracle_4.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Covers" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Audio.from_youtube(\"czdfqv-O9bU\", n_channels=2, sample_rate=48_000).to_wav(\"../samples/star_wars2.wav\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = Audio.from_file(\"../samples/georg_country.mp3\", n_channels=2)#.get_segment(to_s=60.01)\n", "# audio = Audio.from_file(\"../samples/georg_bday.mp3\", n_channels=2)#.get_segment(to_s=60.01)\n", "# audio = Audio.from_file(\"../samples/booboo.m4a\", n_channels=2).get_segment(from_s=32, to_s=85.01)\n", "# audio = Audio.from_file(\"../samples/martin.m4a\", n_channels=2)\n", "# audio = Audio.from_file(\"../samples/martin_piano.m4a\", n_channels=2)\n", "# audio = Audio.from_file(\"../samples/martin2.m4a\", n_channels=2)\n", "# audio = Audio.from_file(\"../samples/maid_of_honor.m4a\", n_channels=2)\n", "# audio = load_audio(\"../samples/tetris.wav\").get_segment(from_s=0, to_s=80.01)\n", "# audio = load_audio(\"../samples/mario.wav\").get_segment(from_s=0, to_s=80.01)\n", "# audio = load_audio(\"../samples/star_wars.wav\").get_segment(from_s=0, to_s=80.01)\n", "# audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=49.99, to_s=70)\n", "# audio = load_audio(\"../samples/leander2.mp3\").get_segment(from_s=0, to_s=40.01)\n", "audio = load_audio(\"../../samples/glitch_gen.mp3\").get_segment(from_s=0, to_s=120.01)\n", "# audio = Audio.from_youtube(\"1WaV2x8GXj0\", n_channels=2, sample_rate=48_000).get_segment(from_s=0, to_s=120.01)\n", "# audio = load_audio(\"../samples/leander1.mp3\")\n", "# audio = load_audio(\"../samples/leander2.mp3\")\n", "# audio = Audio.from_youtube(\"-rh8gMvzPw0\", n_channels=2, sample_rate=48_000).get_segment(from_s=0, to_s=60.01)\n", "# audio = load_audio(\"../samples/mikey_stone.mp3\")\n", "# audio = load_audio(\"../samples/doc_baby.wav\")\n", "# audio = load_audio(\"../samples/stone.mp3\")\n", "audio.play()\n", "in_cover_arr = process_audio(audio, cfg)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# text = \"\"\"\n", "# [Verse]\n", "# Raindrops falling away\n", "# Dancing in the grey\n", "# Clouds are here to stay\n", "# Love on a rainy day\n", "\n", "# [Verse 2]\n", "# Puddles on the ground\n", "# Feet splashing 'round\n", "# You and me we found\n", "# Paradise in the sound\n", "\n", "# [Chorus]\n", "# Love on a rainy day\n", "# Washing doubts away\n", "# Hold me and let's sway\n", "# It's our perfect cliche\n", "\n", "# Love on a rainy day\n", "# Washing doubts away\n", "# Hold me and let's sway\n", "# It's our perfect cliche\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [Verse 1]\n", "# I know your eyes in the mornin' sun\n", "# I feel you touch me in the pourin' rain\n", "# And the moment that you wander far from me\n", "# I wanna feel you in my arms again\n", "\n", "# [Pre-Chorus]\n", "# And you come to me on a summer breeze\n", "# Keep me warm in your love, then you softly leave\n", "# And it's me you need to show\n", "# How deep is your love?\n", "\n", "# [Chorus]\n", "# How deep is your love? How deep is your love?\n", "# I really mean to learn\n", "# 'Cause we're livin' in a world of fools\n", "# Breakin' us down\n", "# When they all should let us be\n", "# We belong to you and me\n", "\n", "# [Verse 2]\n", "# I believe in you\n", "# You know the door to my very soul\n", "# You're the light in my deepest, darkest hour\n", "# You're my saviour when I fall\n", "# See upcoming rock shows\n", "# Get tickets for your favorite artists\n", "# You might also like\n", "# “Slut!” (Taylor’s Version) [From The Vault]\n", "# Taylor Swift\n", "# Say Don’t Go (Taylor’s Version) [From The Vault]\n", "# Taylor Swift\n", "# Alam Mo Ba Girl\n", "# Hev Abi\n", "# [Pre-Chorus]\n", "# And you may not think I care for you\n", "# When you know down inside that I really do\n", "# And it's me you need to show\n", "# How deep is your love?\n", "\n", "# [Chorus]\n", "# How deep is your love? How deep is your love?\n", "# I really mean to learn\n", "# 'Cause we're livin' in a world of fools\n", "# Breakin' us down\n", "# When they all should let us be\n", "# We belong to you and me\n", "\n", "# [Bridge]\n", "# Na-na-na-na-na\n", "# Na-na-na-na, na-na-na-na-na\n", "# Na-na-na, na-na-na-na-na-na-na\n", "# Na-na-na, na-na-na-na\n", "\n", "# [Pre-Chorus]\n", "# And you come to me on a summer breeze\n", "# Keep me warm in your love, then you softly leave\n", "# And it's me you need to show\n", "# How deep is your love?\n", "# [Chorus]\n", "# How deep is your love? How deep is your love?\n", "# I really mean to learn (I really mean to learn)\n", "# 'Cause we're livin' in a world of fools\n", "# Breakin' us down\n", "# When they all should let us be\n", "# We belong to you and me\n", "# (Na-na-na-na-na)\n", "# How deep is your love? How deep is your love?\n", "# I really mean to learn\n", "# 'Cause we're livin' in a world of fools\n", "# Breakin' us down\n", "# When they all should let us be\n", "# We belong to you and me\n", "# (Na-na-na-na-na)\n", "# How deep is your love? How deep is your love?\n", "# I really mean to learn\n", "# 'Cause we're livin' in a world of fools\n", "# Breakin' us down\n", "# When they all should let us be\n", "# We belong to you and me\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# Heer Heer na akho adiyo\n", "# Main te Sahiban hoi\n", "# Ghodi leke aave le jaaye\n", "# Ghodi leke aave le jaaye\n", "\n", "# Ho mainu le jaaye Mirza koi\n", "# Le jaaye Mirza koi\n", "# Le jaaye Mirza koi...\n", "\n", "# Heer Heer na aakho adiyo\n", "# Main te Sahiban hoi\n", "# Ghodi leke aave le jaaye\n", "# Ghodi leke aave le jaaye\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Margu\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Keenan\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Mikey\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Suno\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# I've run out of things to say\n", "# I don't know what I'm gonna do next\n", "# Happy birthday to you\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [Verse 1]\n", "# Almost Heaven, West Virginia\n", "# Blue Ridge Mountains, Shenandoah River\n", "# Life is old there, older than the trees\n", "# Younger than the mountains, growing like a breeze\n", "\n", "# [Chorus]\n", "# Country roads, take me home\n", "# To the place I belong\n", "# West Virginia, mountain mama\n", "# Take me home, country roads\n", "\n", "# [Verse 2]\n", "# All my memories gather 'round her\n", "# Miner's lady, stranger to blue water\n", "# Dark and dusty, painted on the sky\n", "# Misty taste of moonshine, teardrop in my eye\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# been awakened\n", "# Every rule, I had you breakin'\n", "# The risk that I'm takin'\n", "# I'm never gonna shut you out\n", "\n", "# [Chorus]\n", "# Everywhere I'm lookin' now\n", "# I'm surrounded by your embrace\n", "# Baby, I can see your halo\n", "# \"\"\"\n", "\n", "text = \"\"\"\n", "[Verse 1]\n", "Remember those walls I built?\n", "Well, baby, they're tumblin' down\n", "And they didn't even put up a fight\n", "They didn't even make a sound\n", "I found a way to let you in\n", "But I never really had a doubt\n", "Standin' in the light of your halo\n", "I got my angel now\n", "\n", "[Pre-Chorus]\n", "It's like I've been awakened\n", "Every rule, I had you breakin'\n", "It's the risk that I'm takin'\n", "I ain't never gonna shut you out\n", "\n", "[Chorus]\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "You know you're my savin' grace\n", "You're everything I need and more\n", "It's written all over your face\n", "Baby, I can feel your halo\n", "Pray it won't fade away\n", "\n", "[Post-Chorus]\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "\n", "[Verse 2]\n", "Hit me like a ray of sun\n", "Burnin' through my darkest night\n", "You're the only one that I want\n", "Think I'm addicted to your light\n", "I swore I'd never fall again\n", "But this don't even feel like fallin'\n", "Gravity can't begin\n", "To pull me back to\n", "\"\"\"\n", "\n", "# text = \"\"\"\n", "# [Verse]\n", "# Es ist Zeit, wild zu sein,\n", "# wir sind bereit\n", "# Mit dem Panda Sonnenschutz\n", "# wir sind bereit\n", "# Die Sonne scheint stark,\n", "# aber wir sind stärker\n", "\n", "# [Chorus]\n", "# Panda Sonnenschutz\n", "# Panda Sonnenschutz\n", "# schützt uns vor der Sonne (ooooo)\n", "# Keine Sorge um einen Sonnenbrand\n", "# Sonnenbrand\n", "\n", "# [instrumental]\n", "# ooooooo\n", "# Sonnenschutz für Pandas\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [Verse]\n", "# In the midst of bustling streets and crowded sounds,\n", "# A little bonsai tree stands its ground,\n", "# Fragile leaves and delicate roots,\n", "# Trying to grow in a world so brute.\n", "\n", "# [Chorus]\n", "# Oh, little bonsai, reaching for the sky,\n", "# Your spirit so strong, you'll never die,\n", "# Through the cracks in the concrete, you'll find your way,\n", "# Unyielding, unbreakable, come what may.\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [verse]\n", "# oh, my love\n", "# My friend you know\n", "# it's been a while\n", "# Without thinking of you\n", "# but the thought makes me smile\n", "\n", "# [chorus]\n", "# I'm so tired of wanting\n", "# wanting more than this\n", "# i know it but what am i to do\n", "# i need some space to breathe,\n", "# so give me some room\n", "# \"\"\"\n", "\n", "# text = None\n", "\n", "\n", "# text = \"\"\"\n", "# [Verse 1]\n", "# Love is a burning thing\n", "# And it makes a fiery ring\n", "# Bound by wild desire\n", "# I fell into a ring of fire\n", "\n", "# [Chorus]\n", "# I fell into a burning ring of fire\n", "# I went down, down, down\n", "# And the flames went higher\n", "# And it burns, burns, burns\n", "# The ring of fire\n", "# The ring of fire\n", "\n", "# [Instrumental Break]\n", "\n", "# [Chorus]\n", "# I fell into a burning ring of fire\n", "# I went down, down, down\n", "# And the flames went higher\n", "# And it burns, burns, burns\n", "# The ring of fire\n", "# The ring of fire\n", "\n", "# [Verse 2]\n", "# The taste of love is sweet\n", "# When hearts like ours meet\n", "# I fell for you like a child\n", "# Oh, but the fire went wild\n", "\n", "# [Chorus]\n", "# I fell into a burning ring of fire\n", "# I went down, down, down\n", "# And the flames went higher\n", "# And it burns, burns, burns\n", "# The ring of fire\n", "# The ring of fire\n", "# I fell into a burning ring of fire\n", "# I went down, down, down\n", "# And the flames went higher\n", "# And it burns, burns, burns\n", "# The ring of fire\n", "# The ring of fire\n", "\n", "# [Outro]\n", "# And it burns, burns, burns\n", "# The ring of fire\n", "# The ring of fire\n", "# The ring of fire\n", "# The ring of fire\n", "# The ring of fire\n", "# The ring...\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# Dear honored guests and family and friends it’s a real honor\n", "# To be the maid of honor\n", "# To be\n", "# Able to raise a glass and speak of moments passed that brought these 2 lovebirds together forever\n", "\n", "# Luck\n", "# is not a term I’d use to describe it\n", "# Luck\n", "# is too fickle a being to decide this\n", "# Nah these two found each other through deliberate conspiracies of\n", "# The universe\n", "# Now I’ll summarize in just a verse\n", "# Or two\n", "# These two\n", "\n", "# So they say love is like a Nelly rap\n", "# Through ups and downs the paths are found\n", "# On their way back to each other\n", "# Watch the fireworks explode like the Fourth of July you know why?\n", "\n", "# Because this love is like a Nelly rap\n", "\n", "# We all know Michele\n", "# Bougie gorgeous fly as hell\n", "# A genius\n", "# But you know what y’all she’ll never tell\n", "# We ate strawberry cannolis we fixed smoke alarms\n", "# We saved lives for a living\n", "# She saved mine all along\n", "# Then along came Bretty\n", "# This barefoot animal\n", "# Smart sure assertive\n", "# Aye-yo!\n", "# Is this kid Hamilton?\n", "\n", "# He’ll go out of his way any day to make a stranger’s day in a way that’ll stay\n", "# Yet his heart beats true to my boo day after day\n", "# Hey\n", "\n", "# May you live and love and learn earn time and respect\n", "# Every struggle bring you closer love leftover never left\n", "# Kept it honest kept it effortless yess you kept it powerful\n", "# Kept rapping that real love song just like Nelly do\n", "# It’s true!\n", "# It’s true this love is like a Nelly rap\n", "\n", "# I think you know where this is headed\n", "# If love was an arena these two have been getting shredded\n", "# For years\n", "# Now let’s move on without much ado\n", "# Brett Sternfield Michele Esposito do you two Take each other whoa I should hand this over now\n", "# Mark take it away\n", "# We can’t wait to hear the vows\n", "# Yea\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# Yeah, this little piggy so sweet and pretty\n", "# On the grind in this city like daddy I'm gritty\n", "# If they fucking with you mommy they fucking with kitty\n", "# And they worse than Diddy\n", "# Cuz they think and they litty\n", "# And I'm worse than any bad boy\n", "# I'm mad boy\n", "# Cuz you fucking with my dad boy\n", "# I'm max joy\n", "# Something like Max Julian and my dad's a pimp\n", "# And so nine months later on the side of the blimp\n", "# And say I don't give a fuck cuz I'm having a baby\n", "# baby baby, the baby\n", "# I don't give a fuck cuz I'm having a baby\n", "# Baby baby my baby\n", "# I don't give a fuck cuz I'm having a baby\n", "# Baby baby my baby\n", "# one two\n", "# fuck you\n", "# cuz I don't give a fuck cuz I'm having a baby\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [verse]\n", "# oh, my love\n", "# My friend you know\n", "# it's been a while\n", "# Without thinking of you\n", "# but the thought makes me smile\n", "\n", "# [chorus]\n", "# I'm so tired of wanting\n", "# wanting more than this\n", "# i know it but what am i to do\n", "# i need some space to breathe,\n", "# so give me some room\n", "\n", "# [verse]\n", "# oh, my love\n", "# you have a heart of stone\n", "# cause since i've come home\n", "# i've never felt so alone\n", "# but the thought makes me smile\n", "\n", "# [chorus]\n", "# I'm so tired of wanting\n", "# wanting more than this\n", "# i know it but what am i to do\n", "# i need some space to breathe,\n", "# so give me some room\n", "\n", "# [verse]\n", "# oh, my love\n", "# you're coming with me\n", "# I know you'll leave me one day\n", "# can you just stay till monday\n", "# the thought makes me smile\n", "\n", "# [chorus]\n", "# I'm so tired of wanting\n", "# wanting more than this\n", "# i know it but what am i to do\n", "# i need some space to breathe,\n", "# so give me some room\n", "# so give me some room\n", "# i need room\n", "\n", "# [outro]\n", "# \"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # text=\"[instrumental]\",\n", " # text_tags=\"instrumental, bluegrass\",\n", " # text_tags=\"instrumental, piano\",\n", " text_tags=\"country, male\",\n", " # text_tags=\"pop\",\n", " # text_tags=\"pop, female vocals, upbeat, catchy, vibrant, danceable, synthesizers, electric guitar, infectious chorus, modern production, youthful energy, urban life, confident, empowering, love, self-discovery\",\n", " # text_tags=\"melodic techno, Dance-Pop, Electro House, Electropop, Festival Progressive House, Future Rave, Tech House\",\n", " # text_tags=\"Electro House, Dance-Pop, Electropop, Festival Progressive House, Hip House, Pop Rap, party, female vocalist, summer, male vocalist, rhythmic, energetic, hedonistic, sexual, uplifting\",\n", " # text_tags=\"Melodic Techno, Future Rave, Melodic House, 2023\",\n", " # text_tags=\"violin, female\",\n", " # text_tags=\"country, female vocals, melancholic, nostalgic, slow, ballad, acoustic guitar, pedal steel, harmonica, heartbreak, love lost, rural life, raspy, soulful\",\n", " cover_arr=in_cover_arr, # [:25*60*2],\n", " n_batch=1,\n", " # cfg_coef=0.95,\n", " cfg_coef=1.0,\n", " cfg_coef_tags=4,\n", " # cfg_coef_tags_max_steps=None,\n", " max_gen_duration_s=2 * 60,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " min_eos_p=0.05,\n", " # cfg_coef_neg_tags=-2,\n", " # text_neg_tags=\"piano\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert N_BATCH * 4 <= 16\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "audios = []\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert N_BATCH * 4 <= 16\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "audios = []\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audios[-1].get_segment(from_s=2).to_hq_mp3(\"out/covers_bday_country.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Artist" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Audio.from_youtube(\"O6M0j2Zzl4Y\", n_channels=2, sample_rate=48_000).to_wav(\"../samples/sbob2.wav\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = load_audio(\"../samples/mistletoe.mp3\").get_segment(from_s=0, to_s=60.01)\n", "# audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=20, to_s=60.01)\n", "# audio = load_audio(\"../samples/martin2.m4a\")\n", "# audio = load_audio(\"../samples/doc_better.wav\").get_segment(from_s=20, to_s=60.01)\n", "# audio = load_audio(\"../samples/georg_country.mp3\").get_segment(to_s=40.01)\n", "# audio = load_audio(\"../samples/mikey_stone.mp3\").get_segment(from_s=0, to_s=20.01)\n", "# audio = load_audio(\"../samples/sbob.wav\").get_segment(to_s=60.01)\n", "# audio = load_audio(\"test2.wav\").get_segment(to_s=60.01)\n", "audio = load_audio(\"../samples/anyma.mp3\").get_segment(to_s=60.01)\n", "audio.play()\n", "in_artist_arr = process_audio(audio, cfg)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# text = \"\"\"\n", "# [Verse 1]\n", "# Almost Heaven, West Virginia\n", "# Blue Ridge Mountains, Shenandoah River\n", "# Life is old there, older than the trees\n", "# Younger than the mountains, growing like a breeze\n", "\n", "# [Chorus]\n", "# Country roads, take me home\n", "# To the place I belong\n", "# West Virginia, mountain mama\n", "# Take me home, country roads\n", "\n", "# [Verse 2]\n", "# All my memories gather 'round her\n", "# Miner's lady, stranger to blue water\n", "# Dark and dusty, painted on the sky\n", "# Misty taste of moonshine, teardrop in my eye\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [Intro]\n", "# Ask for money and get advice, huh\n", "# Ask for advice, get money twice, huh\n", "# I'm from the dirty, huh, but that chico nice, ha\n", "# Y'all call it a moment, I call it a life\n", "\n", "# [Pre-Chorus]\n", "# One day while my light is glowin'\n", "# I'll be in my castle golden\n", "# But until the gates are open\n", "# I just wanna feel this moment\n", "\n", "# [Chorus]\n", "# Woah-oh-oh-oh\n", "# I just wanna feel this moment\n", "# Woah-oh-oh-oh\n", "# I just wanna feel this moment\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [Verse 1]\n", "# Remember those walls I built?\n", "# Well, baby, they're tumblin' down\n", "# And they didn't even put up a fight\n", "# They didn't even make a sound\n", "\n", "# [Chorus]\n", "# Everywhere I'm lookin' now\n", "# I'm surrounded by your embrace\n", "# Baby, I can see your halo\n", "# You know you're my savin' grace\n", "\n", "# [Verse 2]\n", "# Hit me like a ray of sun\n", "# Burnin' through my darkest night\n", "# You're the only one that I want\n", "# Think I'm addicted to your light\n", "# \"\"\"\n", "\n", "text = \"\"\"\n", "Yeah, this little piggy so sweet and pretty \n", "On the grind in this city like daddy I'm gritty \n", "If they fucking with you mommy they fucking with kitty \n", "And they worse than Diddy \n", "Cuz they think and they litty\n", "And I'm worse than any bad boy \n", "I'm mad boy \n", "Cuz you fucking with my dad boy \n", "I'm max joy \n", "Something like Max Julian and my dad's a pimp \n", "And so nine months later on the side of the blimp \n", "And say I don't give a fuck cuz I'm having a baby \n", "baby baby, the baby\n", "I don't give a fuck cuz I'm having a baby \n", "Baby baby my baby \n", "I don't give a fuck cuz I'm having a baby \n", "Baby baby my baby \n", "one two \n", "fuck you \n", "cuz I don't give a fuck cuz I'm having a baby\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " # text=\"[instrumental]\",\n", " # text=text,\n", " # text_tags=\"pop\",\n", " text_tags=\"Melodic Techno, Future Rave, Melodic House, 2023\",\n", " # text_tags=\"country, female vocals, melancholic, nostalgic, slow, ballad, acoustic guitar, pedal steel, harmonica, heartbreak, love lost, rural life, raspy, soulful\",\n", " artist_arr=in_artist_arr,\n", " n_batch=1,\n", " cfg_coef=1.0,\n", " cfg_coef_tags=3,\n", " max_gen_duration_s=3 * 60,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert N_BATCH * 4 <= 16" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "audios = []\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()\n", "# not using instrumental" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# using instrumental" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audios[-1].to_hq_mp3(\"out/artist_b_pitbull.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Infill" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = Audio.from_file(\"../samples/georg_country.mp3\", n_channels=2).get_segment(to_s=5.01)\n", "# audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=49.99, to_s=70)\n", "# audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=9.99, to_s=70)\n", "audio = load_audio(\"../samples/glitch_gen.mp3\").get_segment(to_s=22)\n", "audio.play()\n", "in_infill_arr = process_audio(audio, cfg)\n", "# in_history_arr = in_infill_arr[:25*5,:].copy()\n", "# in_future_arr = in_infill_arr[-25*5:,:].copy()\n", "in_history_arr = None\n", "in_future_arr = in_infill_arr[25 * 10 :, :].copy()\n", "# in_history_arr[:,:1] = cfg.semantic_mask_token\n", "# in_future_arr[:,:1] = cfg.semantic_mask_token" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# text = \"\"\"\n", "# been awakened\n", "# Every rule, I had you breakin'\n", "# The risk that I'm faking\n", "# I'm never gonna mow your lawn\n", "\n", "# Everywhere I'm trolling now\n", "# I'm surrounded by your embrace\n", "# Baby, I can see your halo\n", "# \"\"\"\n", "\n", "text = \"\"\"\n", "[Verse]\n", "I made a Suno song\n", "to show Franklin I was cool\n", "I used the new infill \n", "To edit the intro and break all of the rules\n", "\n", "[Verse 2]\n", "Sky wide open space\n", "No limits here to bind\n", "With every fold in place\n", "Freedom in my mind\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=\"[instrumental]\",\n", " text_tags=\"steady beat,techno\",\n", " history_arr=in_history_arr,\n", " future_arr=in_future_arr,\n", " n_batch=1,\n", " cfg_coef=2,\n", " cfg_coef_tags=0,\n", " max_gen_duration_s=60,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " min_eos_p=0.05,\n", " # text_start_control_tags=\"{start}\",\n", " # text_start_control_tags=\"{start:0}\",\n", " text_start_control_tags=\"{start:0;duration:22}\",\n", " # text_start_control_tags=\"{remaining:120}\",\n", " # text_start_control_tags=\"{start:0;remaining:120}\",\n", " # text_start_control_tags=\"{start:0;vocals:start}\",\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert N_BATCH * 4 <= 16\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", " v = np.concatenate(\n", " [\n", " torch.stack(list(align_codes(stream, cfg))).detach().cpu().numpy()[:, -12:],\n", " in_future_arr[:, -12:],\n", " ],\n", " axis=0,\n", " )\n", " if in_history_arr is not None:\n", " v = np.concatenate(\n", " [\n", " in_history_arr[:, -12:],\n", " v,\n", " ],\n", " axis=0,\n", " )\n", " audio = codec_decode(v)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Prefill" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# TODO maybe needs audio-only cfg (line 138 in generation_engine.py):\n", "# if gconf.future_arr is not None:\n", "# text_arr[1] = text_arr[0]\n", "# print(\"replaced text for audio cfg\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = Audio.from_file(\"../samples/georg_country.mp3\", n_channels=2).get_segment(to_s=5.01)\n", "# audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=49.99, to_s=70)\n", "audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=9.99, to_s=70)\n", "audio.play()\n", "in_infill_arr = process_audio(audio, cfg)\n", "in_history_arr = in_infill_arr[: 25 * 5, :].copy()\n", "in_future_arr = in_infill_arr[-25 * 5 :, :].copy()\n", "# in_history_arr[:,:1] = cfg.semantic_mask_token\n", "# in_future_arr[:,:1] = cfg.semantic_mask_token" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "been awakened\n", "Every rule, I had you breakin'\n", "The risk that I'm faking\n", "I'm never gonna mow your lawn\n", "\n", "Everywhere I'm trolling now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # history_arr=in_history_arr,\n", " future_arr=in_future_arr,\n", " n_batch=1,\n", " cfg_coef=2,\n", " cfg_coef_tags=0,\n", " max_gen_duration_s=60,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " min_eos_p=0.05,\n", " # text_start_control_tags=\"{start}\",\n", " text_start_control_tags=\"{start:0;vocals:intro}\",\n", " # text_start_control_tags=\"{start:0;duration:40}\",\n", " # text_start_control_tags=\"{remaining:120}\",\n", " # text_start_control_tags=\"{start:0;remaining:120}\",\n", " # text_start_control_tags=\"{start:0;vocals:start}\",\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert N_BATCH * 4 <= 16\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", " audio = codec_decode(\n", " np.concatenate(\n", " [\n", " # in_history_arr[:,-12:],\n", " torch.stack(list(align_codes(stream, cfg)))\n", " .detach()\n", " .cpu()\n", " .numpy()[:, -12:],\n", " in_future_arr[:, -12:],\n", " ],\n", " axis=0,\n", " )\n", " )\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Artist & Cover" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Audio.from_youtube(\"JHimGT3il8M\", n_channels=2, sample_rate=48_000).to_wav(\"../samples/doc_better.wav\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# artist_audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=20, to_s=80.01)\n", "# artist_audio = Audio.from_file(\"../samples/martin2.m4a\", n_channels=2)\n", "# artist_audio = Audio.from_file(\"../samples/doc_vocals.wav\", n_channels=2)\n", "artist_audio = load_audio(\"../samples/doc_better.wav\").get_segment(\n", " from_s=20, to_s=80.01\n", ")\n", "\n", "artist_audio = artist_audio.normalize_volume()\n", "artist_audio.play()\n", "in_artist_arr = process_audio(artist_audio, cfg)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# cover_audio = load_audio(\"../samples/mikey_stone.mp3\")\n", "# cover_audio = Audio.from_file(\"../samples/georg_country.mp3\", n_channels=2).get_segment(to_s=65.01)\n", "# cover_audio = Audio.from_file(\"../samples/georg_bday.mp3\", n_channels=2).get_segment(to_s=60.01)\n", "# cover_audio = Audio.from_file(\"../samples/martin2.m4a\", n_channels=2)\n", "# cover_audio = Audio.from_file(\"../samples/maid_of_honor.m4a\", n_channels=2)\n", "cover_audio = load_audio(\"../samples/doc_baby.wav\")\n", "\n", "cover_audio = cover_audio.normalize_volume()\n", "cover_audio.play()\n", "in_cover_arr = process_audio(cover_audio, cfg)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# text = \"\"\"\n", "# [Verse 1]\n", "# Almost Heaven, West Virginia\n", "# Blue Ridge Mountains, Shenandoah River\n", "# Life is old there, older than the trees\n", "# Younger than the mountains, growing like a breeze\n", "\n", "# [Chorus]\n", "# Country roads, take me home\n", "# To the place I belong\n", "# West Virginia, mountain mama\n", "# Take me home, country roads\n", "\n", "# [Verse 2]\n", "# All my memories gather 'round her\n", "# Miner's lady, stranger to blue water\n", "# Dark and dusty, painted on the sky\n", "# Misty taste of moonshine, teardrop in my eyes\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Margu\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Keenan\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Mikey\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Suno\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# I've run out of things to say\n", "# I don't know what I'm gonna do next\n", "# Happy birthday to you\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [Verse]\n", "# Raindrops falling away\n", "# Dancing in the grey\n", "# Clouds are here to stay\n", "# Love on a rainy days\n", "\n", "# [Verse 2]\n", "# Puddles on the ground\n", "# Feet splashing 'round\n", "# You and me we found\n", "# Paradise in the sound\n", "\n", "# [Chorus]\n", "# Love on a rainy day\n", "# Washing doubts away\n", "# Hold me and let's sway\n", "# It's our perfect cliche\n", "\n", "# Love on a rainy day\n", "# Washing doubts away\n", "# Hold me and let's sway\n", "# It's our perfect cliche\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# Dear honored guests and family and friends it’s a real honor\n", "# To be the maid of honor\n", "# To be\n", "# Able to raise a glass and speak of moments passed that brought these 2 lovebirds together forever\n", "\n", "# Luck\n", "# is not a term I’d use to describe it\n", "# Luck\n", "# is too fickle a being to decide this\n", "# Nah these two found each other through deliberate conspiracies of\n", "# The universe\n", "# Now I’ll summarize in just a verse\n", "# Or two\n", "# These two\n", "\n", "# So they say love is like a Nelly rap\n", "# Through ups and downs the paths are found\n", "# On their way back to each other\n", "# Watch the fireworks explode like the Fourth of July you know why?\n", "\n", "# Because this love is like a Nelly rap\n", "\n", "# We all know Michele\n", "# Bougie gorgeous fly as hell\n", "# A genius\n", "# But you know what y’all she’ll never tell\n", "# We ate strawberry cannolis we fixed smoke alarms\n", "# We saved lives for a living\n", "# She saved mine all along\n", "# Then along came Bretty\n", "# This barefoot animal\n", "# Smart sure assertive\n", "# Aye-yo!\n", "# Is this kid Hamilton?\n", "\n", "# He’ll go out of his way any day to make a stranger’s day in a way that’ll stay\n", "# Yet his heart beats true to my boo day after day\n", "# Hey\n", "\n", "# May you live and love and learn earn time and respect\n", "# Every struggle bring you closer love leftover never left\n", "# Kept it honest kept it effortless yess you kept it powerful\n", "# Kept rapping that real love song just like Nelly do\n", "# It’s true!\n", "# It’s true this love is like a Nelly rap\n", "\n", "# I think you know where this is headed\n", "# If love was an arena these two have been getting shredded\n", "# For years\n", "# Now let’s move on without much ado\n", "# Brett Sternfield Michele Esposito do you two Take each other whoa I should hand this over now\n", "# Mark take it away\n", "# We can’t wait to hear the vows\n", "# Yea\n", "# \"\"\"\n", "\n", "\n", "text = \"\"\"\n", "Yeah, this little piggy so sweet and pretty \n", "On the grind in this city like daddy I'm gritty \n", "If they fucking with you mommy they fucking with kitty \n", "And they worse than Diddy \n", "Cuz they think and they litty\n", "And I'm worse than any bad boy \n", "I'm mad boy \n", "Cuz you fucking with my dad boy \n", "I'm max joy \n", "Something like Max Julian and my dad's a pimp \n", "And so nine months later on the side of the blimp \n", "And say I don't give a fuck cuz I'm having a baby \n", "baby baby, the baby\n", "I don't give a fuck cuz I'm having a baby \n", "Baby baby my baby \n", "I don't give a fuck cuz I'm having a baby \n", "Baby baby my baby \n", "one two \n", "fuck you \n", "cuz I don't give a fuck cuz I'm having a baby\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # text_tags=\"pop\",\n", " artist_arr=in_artist_arr,\n", " cover_arr=in_cover_arr,\n", " n_batch=1,\n", " cfg_coef=1.3, # .05,\n", " cfg_coef_tags=2,\n", " max_gen_duration_s=60,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "\n", "assert N_BATCH * 4 <= 16" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "audios = []\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# requests = [\n", "# make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", "# for i in range(N_BATCH)\n", "# ]\n", "# for job in engine.run_request(requests, tqdm_enabled=True):\n", "# stream = engine.token_generator(job)\n", "\n", "# audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", "# audios.append(audio)\n", "# audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# in_arr = request.streams[0].prompt\n", "# prompt = Prompt(gconf, cfg, tokenizer)\n", "# prompt.visualize(in_arr, compress=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# import numpy as np\n", "# in_arr = requests[0].streams[0].prompt\n", "# generated = np.stack([t.cpu().numpy() for t in job.generated_tokens]).T\n", "# print(generated.shape)\n", "# arr = np.concatenate([in_arr, generated], axis=1)\n", "# prompt.visualize(arr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "## different sampling\n", "# if temps_coarse.max() < 0.5:\n", "# temperatures = ((torch.linspace(1, 0, 12)**4)*0.8+0.1)[None, :, None].repeat(cfg_coarse_logits.shape[0], 1, 1).type(torch.float32).to(cfg_coarse_logits.device)\n", "# cfg_coarse_probs = cfg_coarse_logits / temperatures\n", "# cfg_coarse_probs = F.softmax(cfg_coarse_probs, dim=-1)\n", "# else:\n", "# cfg_coarse_probs = compute_softmax_batch(cfg_coarse_logits, temps_coarse)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "[Chorus: Bobby Vinton]\n", "Lonely, I'm Mr. Lonely\n", "I have nobody for my own\n", "I'm so lonely, I'm Mr. Lonely\n", "I have nobody for my own\n", "I'm so lonely\n", "\n", "[Interlude: Akon]\n", "Yo, this one here\n", "Goes out to all my players out there, man, you know\n", "That got that one good girl, dawg, that's always been there, man\n", "Like, took all the bullshit\n", "But then one day she can't take it no more and decides to leave\n", "\n", "[Verse 1: Akon]\n", "Yeah, I woke up in the middle of the night\n", "And I noticed my girl wasn't by my side\n", "Could've sworn I was dreamin' for her\n", "I was feenin' so I had to take a little ride\n", "Backtrackin' on these few years\n", "Tryin' to figure out what I do to make it go bad\n", "'Cause ever since my girl left me\n", "My whole life came crashin', and I'm so\n", "\n", "[Chorus: Bobby Vinton & Akon]\n", "Lonely (So lonely)\n", "I'm Mr. Lonely (Mr. Lonely)\n", "I have nobody (I have nobody)\n", "For my own (To call my own, girl)\n", "I'm so lonely (So lonely)\n", "I'm Mr. Lonely (Mr. Lonely)\n", "I have nobody (I have nobody)\n", "For my own (To call my own, girl)\n", "I'm so lonely\n", "\n", "[Verse 2: Akon]\n", "Can't believe I had a girl like you\n", "And I just let you walk right out of my life\n", "After all I put you through\n", "You still stuck around and stayed by my side\n", "What really hurt me is I broke your heart\n", "Baby, you a good girl and I had no right\n", "I really wanna make things right\n", "'Cause without you in my life, girl, I'm so\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # text=\"[instrumental]\",\n", " # text_tags=\"bag pipes, funeral song, scottish\",\n", " # text_tags=\"pop, power ballad, female, violin\",\n", " text_tags=\"pop\", # , r&b, ballad, female\",\n", " # text_tags=\"Gospel\",\n", " # text_tags=\"bluegrass, female\",\n", " # text_tags=\"upbeat\",\n", " # text_tags=\"sea shanty, sea shanty, sea shanty\",\n", " cfg_coef=1.1,\n", " cfg_coef_tags=2,\n", " text_start_control_tags=\"{start_s:0;vocals:intro}\",\n", " # text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", " # text_end_control_tags=\"{end_s:55;original_duration_s:55}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " n_batch=1,\n", " min_eos_p=0.1,\n", " # allow_eos=False,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "audios = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "nn = 0\n", "for nnn in range(1):\n", " print(nnn)\n", " requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", " ]\n", " # print(requests[0])\n", " jobs = engine.run_request(requests, tqdm_enabled=True)\n", "\n", " for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()\n", " # audio.convert(44_100, 2, 2).to_hq_mp3(f\"clips/gen{nn}.mp3\")\n", " nn += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "nn = 0\n", "for nnn in range(1):\n", " print(nnn)\n", " requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", " ]\n", " # print(requests[0])\n", " jobs = engine.run_request(requests, tqdm_enabled=True)\n", "\n", " for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()\n", " # audio.convert(44_100, 2, 2).to_hq_mp3(f\"clips/gen{nn}.mp3\")\n", " nn += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "audios[-1].convert(44_100, 2, 2).to_hq_mp3(f\"clips/gen_cr_pop.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "[Verse 1]\n", "Every night in my dreams\n", "I see you, I feel you\n", "That is how I know you go on\n", "Far across the distance\n", "And spaces between us\n", "You have come to show you go on\n", "\n", "[Chorus]\n", "Near, far, wherever you are\n", "I believe that the heart does go on\n", "Once more, you open the door\n", "And you're here in my heart\n", "And my heart will go on and on\n", "\n", "[Verse 2]\n", "Love can touch us one time\n", "And last for a lifetime\n", "And never let go 'til we're gone\n", "Love was when I loved you\n", "One true time I'd hold to\n", "In my life, we'll always go on\n", "\n", "[Chorus]\n", "Near, far, wherever you are\n", "I believe that the heart does go on\n", "(Why does the heart go on?)\n", "Once more, you open the door\n", "And you're here in my heart\n", "And my heart will go on and on\n", "\n", "[Instrumental Bridge]\n", "\n", "[Chorus]\n", "You're here, there's nothing I fear\n", "And I know that my heart will go on\n", "We'll stay forever this way\n", "You are safe in my heart\n", "And my heart will go on and on\n", "\n", "[Outro]\n", "Mm, mm-mm\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # text=\"[instrumental]\",\n", " # text_tags=\"dark epic trailer music, violin, vocalizations, female\",\n", " text_tags=\"bluegrass, female vocals\",\n", " # text_tags=\"playful energetic synthwave\",\n", " # text_tags=\"bluegrass, female\",\n", " # text_tags=\"upbeat\",\n", " # text_tags=\"sea shanty, sea shanty, sea shanty\",\n", " cfg_coef=1.1,\n", " cfg_coef_tags=2,\n", " # n_repeat_tags=3,\n", " text_start_control_tags=\"{start_s:0}\",\n", " # text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", " # text_end_control_tags=\"{end_s:55;original_duration_s:55}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " n_batch=1,\n", " min_eos_p=0.1,\n", " # allow_eos=False,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=2 * 60,\n", ")\n", "\n", "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "# print(requests[0])\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "\n", "audios = []\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audios[1].convert(44_100, 2, 2).to_hq_mp3(f\"clips/tmp2.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "max_gen_duration_s = 60\n", "\n", "text = \"\"\"\n", "[Verse 1]\n", "Almost Heaven, West Virginia\n", "Blue Ridge Mountains, Shenandoah River\n", "Life is old there, older than the trees\n", "Younger than the mountains, growing like a breeze\n", "\n", "[Chorus]\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "\n", "[Verse 2]\n", "All my memories gather 'round her\n", "Miner's lady, stranger to blue water\n", "Dark and dusty, painted on the sky\n", "Misty taste of moonshine, teardrop in my eye\n", "\n", "[Chorus]\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "\n", "[Bridge]\n", "I hear her voice in the morning hour, she calls me\n", "The radio reminds me of my home far away\n", "Driving down the road, I get a feeling\n", "That I should have been home yesterday, yesterday\n", "\n", "[Chorus]\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "\n", "[Outro]\n", "Take me home, (Down) country roads\n", "Take me home, (Down) country roads\n", "\"\"\"\n", "\n", "in_oracle_array = np.zeros((25 * max_gen_duration_s, 1), dtype=np.int32)\n", "in_oracle_array[:] = cfg.semantic_mask_token\n", "\n", "gconf = GenerationConfig(\n", " oracle_arr=in_oracle_arr[:, :1],\n", " text=text,\n", " # text=\"[instrumental]\",\n", " # text_tags=\"dark epic trailer music, violin, vocalizations, female\",\n", " text_tags=\"pop, power ballad, female, violin\",\n", " # text_tags=\"pop\",\n", " # text_tags=\"gospel\",\n", " # text_tags=\"bluegrass, female\",\n", " # text_tags=\"upbeat\",\n", " # text_tags=\"sea shanty, sea shanty, sea shanty\",\n", " cfg_coef=1.1,\n", " cfg_coef_tags=2,\n", " # n_repeat_tags=3,\n", " text_start_control_tags=\"{start_s:0}\",\n", " # text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", " # text_end_control_tags=\"{end_s:55;original_duration_s:55}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " # temp_semantic=0.85,\n", " # temp_coarse=0.9,\n", " # top_k_semantic=None,\n", " # top_k_coarse=None,\n", " # top_p_semantic=0.95,\n", " # top_p_coarse=None,\n", " n_batch=1,\n", " min_eos_p=0.1,\n", " # allow_eos=False,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=max_gen_duration_s,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "audios = []\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "max_gen_duration_s = 60\n", "\n", "text = \"\"\"\n", "[Intro]\n", "I can feel the heat risin'\n", "Everything is on fire\n", "Today's a painful re-\n", "Minder of why\n", "It can only get brighter\n", "The further you put it behind ya\n", "But right now I'm on the\n", "Inside lookin' out, ‘cause—\n", "\n", "[Chorus]\n", "I'm standin' in the flames\n", "And it’s a beautiful kind of pain\n", "Settin' fire to yesterday\n", "To find the light, find the light, find the light\n", "Standin' in the flames\n", "And it’s a beautiful kind of pain\n", "Settin' fire to yesterday\n", "Find the light, find the light, find the light\n", "\n", "[Verse 1]\n", "Yesterday was the tornado warning, today's like the morning after\n", "Your world is torn in half, you wake in its wake\n", "To start the mourning process and rebuilding, you're still a work in progress\n", "Today's a whole new chapter, it's like an enormous asthma\n", "Thunderstorm has passed ya, you weathered it and poked its\n", "Eye out with the thornbush that ya used to smell the roses\n", "Stopped to inhale, can't even tell your nose is stuffed\n", "So focused on the bright side then you floor the gas pedal\n", "And hit the corner faster, more assertive, never looking back\n", "May hit the curb, but every day's a new learning curve as ya\n", "Steer through life, sometimes you might not wanna swerve but you have to\n", "To avert a disaster, lucky no permanent damage\n", "‘Cause they hurt you so ba\n", "\"\"\"\n", "\n", "in_oracle_array = np.zeros((25 * max_gen_duration_s, 1), dtype=np.int32)\n", "in_oracle_array[:] = cfg.semantic_mask_token\n", "\n", "gconf = GenerationConfig(\n", " oracle_arr=in_oracle_arr[:, :1],\n", " text=text,\n", " # text=\"[instrumental]\",\n", " # text_tags=\"dark epic trailer music, violin, vocalizations, female\",\n", " # text_tags=\"pop, power ballad, female, violin\",\n", " text_tags=\"Rap, In English, USA, Midwest Rap, Detroit Rap, Alternative, Hip-Hop\",\n", " # text_tags=\"pop\",\n", " # text_tags=\"gospel\",\n", " # text_tags=\"bluegrass, female\",\n", " # text_tags=\"upbeat\",\n", " # text_tags=\"sea shanty, sea shanty, sea shanty\",\n", " cfg_coef=1.1,\n", " cfg_coef_tags=2,\n", " # n_repeat_tags=3,\n", " text_start_control_tags=\"{start_s:0}\",\n", " # text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", " # text_end_control_tags=\"{end_s:55;original_duration_s:55}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " # temp_semantic=0.85,\n", " # temp_coarse=0.9,\n", " # top_k_semantic=None,\n", " # top_k_coarse=None,\n", " # top_p_semantic=0.95,\n", " # top_p_coarse=None,\n", " n_batch=1,\n", " min_eos_p=0.1,\n", " # allow_eos=False,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=max_gen_duration_s,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "audios = []\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# general_config = dict(\n", "# cfg_coef=1.0, # no text cfg for dpo stream\n", "# min_eos_p=0.1,\n", "# eos_pad_duration_s=0,\n", "# # cfg_coef_tags=0.0,\n", "# # cfg_coef_tags_max_steps=None, # collect the data for now\n", "# cfg_coef_tags=2,\n", "# cfg_coef_neg_tags=-1,\n", "# text_neg_tags=\"repetitive, loop, noisy, distorted\",\n", "# n_repeat_tags=1,\n", "# use_whisper=False,\n", "# text_start_control_tags=\"{start:0}\",\n", "# text_end_control_tags=\"{end}\",\n", "# random_seed=42,\n", "# n_batch=1,\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "gconf = GenerationConfig(\n", " text=\"[instrumental, instrumental, instrumental]\",\n", " text_tags=\"upbeat\",\n", " cfg_coef=1.0,\n", " cfg_coef_tags=2,\n", " # n_repeat_tags=3,\n", " text_start_control_tags=\"{start_s:0}\",\n", " # text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", " # text_end_control_tags=\"{end_s:55;original_duration_s:55}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " # cfg_coef_tags_max_steps=99999,\n", " # temp_semantic=0.85,\n", " # temp_coarse=0.9,\n", " # top_k_semantic=None,\n", " # top_k_coarse=None,\n", " # top_p_semantic=0.95,\n", " # top_p_coarse=None,\n", " n_batch=1,\n", " min_eos_p=0.1,\n", " # allow_eos=False,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60,\n", ")\n", "\n", "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "# print(requests[0])\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "\n", "audios = []\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Prefill" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audio = Audio.from_file(\"../samples/georg_country.mp3\", n_channels=2).get_segment(to_s=5.01)\n", "audio = load_audio(\"../samples/halo.wav\").get_segment(from_s=49.99, to_s=70)\n", "# audio.play()\n", "in_infill_arr = process_audio(audio, cfg)\n", "# in_history_arr = in_infill_arr[:25*5,:].copy()\n", "# in_future_arr = in_infill_arr[-25*5:,:].copy()\n", "# in_future_arr = in_infill_arr.copy()\n", "in_future_arr = in_infill_arr[25 * 5 :, :].copy()\n", "codec_decode(in_future_arr[:, -12:]).play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "Like I've been awakened\n", "Every rule, I had you breakin'\n", "Is the risk that I'm taking\n", "I am never gonna shut you out\n", "\n", "Everywhere I'm looking now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # history_arr=in_history_arr,\n", " future_arr=in_future_arr,\n", " n_batch=1,\n", " cfg_coef=4,\n", " cfg_coef_tags=2,\n", " max_gen_duration_s=40,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " # text_start_control_tags=\"{start}\",\n", " # text_start_control_tags=\"{start:0}\",\n", " text_start_control_tags=\"{start:0;duration:40}\",\n", " # text_start_control_tags=\"{start:0;remaining:120}\",\n", " # text_start_control_tags=\"{start:0;vocals:start}\",\n", " min_eos_p=0.01,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert N_BATCH * 4 <= 16\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "out = []\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", " out.append(align_codes(stream, cfg))\n", "for e in out:\n", " audio = codec_decode(\n", " np.concatenate(\n", " [\n", " # in_history_arr[:,-12:],\n", " torch.stack(list(e)).detach().cpu().numpy()[:, -12:],\n", " # in_future_arr[:,-12:],\n", " ],\n", " axis=0,\n", " )\n", " )\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "audios = []\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# prefill\n", "prompt.visualize(in_arr, compress=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# infill\n", "prompt.visualize(in_arr, compress=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "[Intro]\n", "Nants ingonyama bagithi Baba\n", "Sithi uhm ingonyama\n", "Nants ingonyama bagithi baba\n", "Sithi uhhmm ingonyama\n", "Ingonyama\n", "Siyo Nqoba\n", "Ingonyama\n", "\n", "[Refrain]\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "\n", "[Verse]\n", "From the day we arrive on the planet\n", "And blinking, step into the sun\n", "There's more to see than can ever be seen\n", "More to do than can ever be done\n", "There's far too much to take in here\n", "More to find than can ever be found\n", "But the sun rolling high through the sapphire sky\n", "Keeps great and small on the endless round\n", "\n", "[Chorus]\n", "It's the circle of life\n", "And it moves us all\n", "Through despair and hope\n", "Through faith and love\n", "'Til we find our place\n", "On the path unwinding\n", "In the circle\n", "The circle of life\n", "\n", "[Refrain]\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "Ingonyama nengw' enamabala\n", "\n", "[Chorus]\n", "It's the circle of life\n", "And it moves us all\n", "Through despair and hope\n", "Through faith and love\n", "'Til we find our place\n", "On the path unwinding\n", "In the circle\n", "The circle of life\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " # text=\"[instrumental]\",\n", " # text_tags=\"dark epic trailer music, violin, vocalizations, female\",\n", " # text_tags=\"pop, power ballad, female, violin\",\n", " # text_tags=\"bluegrass, female\",\n", " # text_tags=\"gospel\",\n", " # text_tags=\"bluegrass, female\",\n", " # text_tags=\"upbeat\",\n", " # text_tags=\"sea shanty, sea shanty, sea shanty\",\n", " cfg_coef=1.1,\n", " cfg_coef_tags=2,\n", " n_repeat_tags=3,\n", " text_start_control_tags=\"{start:0}\",\n", " # text_start_control_tags=\"{start:0;vocals:intro}\",\n", " # text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", " # text_end_control_tags=\"{end_s:55;original_duration_s:55}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " # temp_semantic=0.85,\n", " # temp_coarse=0.9,\n", " # top_k_semantic=None,\n", " # top_k_coarse=None,\n", " # top_p_semantic=0.95,\n", " # top_p_coarse=None,\n", " n_batch=1,\n", " min_eos_p=0.1,\n", " # allow_eos=False,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "audios = []\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audios.append(audio)\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_utils", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 2 }