{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "0", "metadata": {}, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"" ] }, { "cell_type": "code", "execution_count": null, "id": "1", "metadata": {}, "outputs": [], "source": [ "from suno_utils.diffusion import generation as diffusion_gen\n", "from suno_utils.tasks.upsample_engine import UpsampleEngine, Request\n", "from suno_utils.tasks.dac_vae_fixed_25hz import decode_stream_to_full_audio\n", "import suno_utils.tasks.audio_features.beat_this_downbeat\n", "from suno_utils.audio import Audio\n", "import torch\n", "import matplotlib.pyplot as plt\n", "from enum import Enum" ] }, { "cell_type": "code", "execution_count": null, "id": "2", "metadata": {}, "outputs": [], "source": [ "diffusion_gen.preload_models(\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-07-30_17-01-42_s4725/last_ckpt_infer.pt\", # ft model\n", " # dit_model_filepath=\"/app/suno/checkpoints/2025-07-30_16-16-20_s6206/last_ckpt_infer.pt\", # pretraining\n", " #dit_model_filepath = \"/app/suno/checkpoints/2025-08-04_21-42-57_s7854/last_ckpt_infer.pt\", # up 7.5 by 1.5 ft\n", " # dit_model_filepath = \"/app/suno/checkpoints/2025-08-04_22-24-40_s611/last_ckpt_infer.pt\", # up 8 by 3 ft\n", " #dit_model_filepath = \"/app/suno/checkpoints/2025-08-05_01-49-36_s1309/last_ckpt_infer.pt\", # up 8 by 3 pretrain\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-08-06_15-00-45_s9040/last_ckpt_infer.pt\", # up 8.1 by 2 ft\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-08-14_14-08-33_s8943/last_ckpt_infer.pt\", # up 8.1 by 2 filter v1 keybpm\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-08-17_20-46-13_s1637/last_ckpt_infer.pt\", # 8n_25hz_v45_sfx_ft_filter_v1_e6_up2_by_2_keybpm\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-08-18_14-30-48_s6487/last_ckpt_infer.pt\", #8n_25hz_v45_sfx_ft_filter_v1_5e5_up2_by_2_keybpm\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-08-18_14-58-19_s2236/last_ckpt_infer.pt\", #8n_25hz_v45_sfx_ft_filter_v1_2e5_up2_by_2_keybpm\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-08-20_15-03-40_s6578/last_ckpt_infer.pt\", # best so far\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-08-28_17-59-30_s8523/last_ckpt_infer.pt\", # up 1.5 by 1.6 key bpm\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-08-28_18-05-19_s9420/last_ckpt_infer.pt\", # up 8.1 by 1.5 key bpm\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-09-02_17-16-01_s755/last_ckpt_infer.pt\", # ft model filter v2 with key bpm\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-09-06_15-03-24_s5551/last_ckpt_infer.pt\", # 8n_25hz_v45_sfx_ft_filter_v3_5e5_keybpm_es\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-11-07_14-51-51_s6298/step_250000_infer.pt\", # 8n_25hz_v45_sfx_ft_filter_v3_5e5_keybpm_es_pad_aligned2\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-11-11_16-15-46_s2839/last_ckpt_infer.pt\", # 8n_25hz_v45_sfx_ft_filter_v3_5e5_keybpm_es_pad_aligned_100k\n", " #dit_model_filepath=\"/app/suno/checkpoints/2025-10-14_15-08-40_s8969/last_ckpt_infer.pt\", # 8n_25hz_v45_sfx_ft_filter_v3_5e5_keybpm_es_aligned_info_v1\n", " dit_model_filepath=\"/app/suno/checkpoints/2025-11-12_15-42-00_s2799/last_ckpt_infer.pt\", # 8n_25hz_v45_sfx_ft_filter_v3_5e5_keybpm_es_pad_aligned_400k_up4by1_5\n", " codec_filepath=\"s3://suno-data/minz/models/dac_vae_tuned_25hz.pth\",\n", ")\n", "\n", "diffusion_engine = UpsampleEngine(min_chunk_size=25 * 15)" ] }, { "cell_type": "code", "execution_count": null, "id": "3", "metadata": {}, "outputs": [], "source": [ "semantic_codes = torch.ones((375,), dtype=torch.int32)" ] }, { "cell_type": "code", "execution_count": null, "id": "4", "metadata": {}, "outputs": [], "source": [ "extractor = suno_utils.tasks.audio_features.beat_this_downbeat.BeatThisDownbeatExtractor(device=\"cuda\", model_path=\"s3://suno-data/m4burns/beat_this_rc_12l.pt\", resample_on_device=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "5", "metadata": {}, "outputs": [], "source": [ "class BeatIssueType(Enum):\n", " \"\"\"Categories for beat pattern issues\"\"\"\n", " NO_START_DOWNBEAT = \"Loop doesn't start on downbeat\"\n", " LARGE_FIRST_BAR_DIFF = \"First bar length difference is large (>100ms)\"\n", " MEDIUM_FIRST_BAR_DIFF = \"First bar length difference ~40ms (30-100ms)\"\n", " SMALL_FIRST_BAR_DIFF = \"First bar length difference is very small\"\n", " NORMAL = \"Normal beat pattern\"\n", "\n", "def detect_silence_boundary(audio_data, sample_rate, silence_threshold=0.01, window_size=2048, min_silent_windows=10):\n", " \"\"\"Detect where silence begins in audio data.\"\"\"\n", " num_windows = len(audio_data) // window_size\n", " consecutive_silent = 0\n", " \n", " for i in range(num_windows):\n", " window = audio_data[i * window_size:(i + 1) * window_size]\n", " rms = np.sqrt(np.mean(window ** 2))\n", " \n", " if rms < silence_threshold:\n", " consecutive_silent += 1\n", " if consecutive_silent >= min_silent_windows:\n", " return i * window_size\n", " else:\n", " consecutive_silent = 0\n", " \n", " return len(audio_data)\n", "\n", "def classify_interval_diff(diff, large_threshold, medium_low, medium_high, small_threshold):\n", " \"\"\"Classify a time interval difference into BeatIssueType categories.\"\"\"\n", " if diff >= large_threshold:\n", " return BeatIssueType.LARGE_FIRST_BAR_DIFF\n", " elif medium_low <= diff <= medium_high:\n", " return BeatIssueType.MEDIUM_FIRST_BAR_DIFF\n", " elif diff <= small_threshold:\n", " return BeatIssueType.SMALL_FIRST_BAR_DIFF\n", " else:\n", " return BeatIssueType.NORMAL\n", "\n", "def analyze_intervals(intervals, name, title_suffix, large_threshold, medium_low, medium_high, small_threshold, verbose=True):\n", " \"\"\"Analyze and print beat/bar interval statistics.\"\"\"\n", " if len(intervals) < 2:\n", " return None\n", " \n", " first, second = intervals[0], intervals[1]\n", " diff = abs(second - first)\n", " issue_type = classify_interval_diff(diff, large_threshold, medium_low, medium_high, small_threshold)\n", " \n", " if verbose:\n", " print(f\"{name}{title_suffix}: {issue_type.value}\")\n", " print(f\" First: {first:.3f}s, Second: {second:.3f}s, Diff: {diff*1000:.1f}ms\")\n", " print(f\" All: {intervals}\")\n", " print(f\" Mean: {intervals.mean():.3f}s, Std: {intervals.std():.3f}s\\n\")\n", " \n", " return issue_type\n", "\n", "def plot_audio_with_beats(audio_mono, beats_refined, title_suffix=\"\", silence_threshold=0.01, window_size=2048, \n", " only_downbeats=False, start_threshold=0.05, large_diff_threshold=0.100, \n", " medium_diff_low=0.030, medium_diff_high=0.100, small_diff_threshold=0.010, verbose=False):\n", " \"\"\"Plot audio waveform with beat markers, truncating at silence. Classifies beat patterns.\"\"\"\n", " # Get audio data and detect silence boundary\n", " audio_data = audio_mono.array_float\n", " sample_rate = audio_mono.sample_rate\n", " silence_idx = detect_silence_boundary(audio_data, sample_rate, silence_threshold, window_size)\n", " \n", " # Truncate at silence and create time axis\n", " audio_data = audio_data[:silence_idx]\n", " duration = len(audio_data) / sample_rate\n", " time_axis = np.linspace(0, duration, len(audio_data))\n", " \n", " # Analyze downbeats (bar lengths)\n", " downbeats = beats_refined[beats_refined[:, 1] == 1.0]\n", " issue_type = BeatIssueType.NORMAL\n", " \n", " if len(downbeats) >= 2:\n", " downbeat_times = downbeats[:, 0]\n", " \n", " # Check if loop starts on downbeat\n", " if downbeat_times[0] > start_threshold:\n", " issue_type = BeatIssueType.NO_START_DOWNBEAT\n", " if verbose:\n", " print(f\"\\n{'='*60}\")\n", " print(f\"Beat pattern{title_suffix}: {issue_type.value}\")\n", " print(f\" First downbeat at {downbeat_times[0]:.3f}s (threshold: {start_threshold}s)\")\n", " print(f\"{'='*60}\\n\")\n", " return issue_type, BeatIssueType.NORMAL\n", " \n", " # Analyze downbeat intervals (bar lengths)\n", " downbeat_distances = np.diff(downbeat_times)\n", " if len(downbeat_distances) >= 2:\n", " if verbose:\n", " print(f\"{'='*60}\")\n", " issue_type = analyze_intervals(downbeat_distances, \"Bar pattern\", title_suffix,\n", " large_diff_threshold, medium_diff_low, medium_diff_high, small_diff_threshold, verbose)\n", " if verbose:\n", " print(f\"{'='*60}\")\n", " \n", " # Analyze all beat intervals\n", " beat_issue_type = BeatIssueType.NORMAL\n", " all_beat_distances = np.diff(beats_refined[:, 0])\n", " if len(all_beat_distances) >= 2:\n", " beat_issue_type = analyze_intervals(all_beat_distances, \"All beats analysis\", title_suffix,\n", " large_diff_threshold, medium_diff_low, medium_diff_high, small_diff_threshold, verbose)\n", " \n", " # Create plot if verbose\n", " if verbose:\n", " plt.figure(figsize=(16, 6))\n", " plt.plot(time_axis, audio_data, linewidth=0.5, alpha=0.7, color='gray')\n", " \n", " # Plot beat markers\n", " plotted = {'downbeat': False, 'beat': False}\n", " for beat_time, beat_position in beats_refined:\n", " if beat_time > duration:\n", " break\n", " if only_downbeats and beat_position != 1.0:\n", " continue\n", " \n", " is_downbeat = beat_position == 1.0\n", " color, linewidth, beat_type = ('red', 1.5, 'downbeat') if is_downbeat else ('blue', 1, 'beat')\n", " label = beat_type.capitalize() if not plotted[beat_type] else ''\n", " \n", " plt.axvline(x=beat_time, color=color, linestyle='-', linewidth=linewidth, alpha=0.8 if is_downbeat else 0.6, label=label)\n", " plotted[beat_type] = True\n", " \n", " plt.xlabel('Time (seconds)', fontsize=12)\n", " plt.ylabel('Amplitude', fontsize=12)\n", " plt.title(f'Audio Waveform with Beat Markers{title_suffix}', fontsize=14)\n", " plt.legend(loc='upper right')\n", " plt.grid(True, alpha=0.3)\n", " plt.tight_layout()\n", " plt.show()\n", " \n", " return issue_type, beat_issue_type\n" ] }, { "cell_type": "code", "execution_count": null, "id": "6", "metadata": {}, "outputs": [], "source": [ "# Generation parameters\n", "diffusion_seeds = np.random.randint(0, 1000000, size=3)\n", "verbose = True # Set to True to show plots and detailed output for each generation\n", "\n", "gen_params = {\n", " 'steps': 32,\n", " 'lyrics': \"\",\n", " 'tags': \"four on the floor techno drum loop, duration_sec: 8.0\",\n", " 'text_cfg_coef': 4.0,\n", " 'ctx_cfg_coef': 1.0,\n", " 'codec_scale_factor': 0.4,\n", " 'scale_ctx_vector': True,\n", " 'noise_ctx_level': 0.5,\n", " 'noise_ctx_pad_len': 0,\n", " 'drop_semantic_tokens': True,\n", " 'rho': 1.0,\n", " 'sigma_min': 0.5,\n", " 'sigma_max': 50.0,\n", " 'objective': \"rectified_flow\",\n", "}\n", "\n", "upsampled_audios = []\n", "bar_issues = []\n", "beat_issues = []\n", "\n", "for idx, seed in enumerate(diffusion_seeds):\n", " # Generate audio\n", " gen_cfg = diffusion_gen.DiffusionGenerationConfig(**gen_params, seed=int(seed))\n", " request = Request(id=\"dummy\", generation_config=gen_cfg, tokens=semantic_codes[:375], input_tokens_finished=True)\n", " result = diffusion_engine.run_request(request)\n", " \n", " # Decode VAE latents\n", " vae_latents = torch.concat([vae_latent for vae_latent in result.vae_latents])\n", " upsampled_audio = decode_stream_to_full_audio(vae_latents)\n", " \n", " # Extract and analyze beats\n", " audio_mono = Audio.convert(upsampled_audio, n_channels=1, sample_rate=upsampled_audio.sample_rate, byte_width=upsampled_audio.byte_width)\n", " beats_refined = np.array(extractor.extract(audio_mono)[\"downbeats\"])\n", " bar_issue, beat_issue = plot_audio_with_beats(audio_mono, beats_refined, title_suffix=f\" - Audio {idx}\", verbose=verbose)\n", " \n", " # Track issues\n", " bar_issues.append(bar_issue)\n", " beat_issues.append(beat_issue)\n", " \n", " # Save and play\n", " if verbose:\n", " upsampled_audio.write_opus(f\"upsampled_audio_{idx}.opus\")\n", " upsampled_audio.play()\n", " upsampled_audios.append(upsampled_audio)" ] }, { "cell_type": "code", "execution_count": null, "id": "7", "metadata": {}, "outputs": [], "source": [ "# Summary statistics\n", "total = len(bar_issues)\n", "bar_medium = sum(1 for issue in bar_issues if issue == BeatIssueType.MEDIUM_FIRST_BAR_DIFF)\n", "beat_medium = sum(1 for issue in beat_issues if issue == BeatIssueType.MEDIUM_FIRST_BAR_DIFF)\n", "either_medium = sum(1 for b1, b2 in zip(bar_issues, beat_issues) \n", " if b1 == BeatIssueType.MEDIUM_FIRST_BAR_DIFF or b2 == BeatIssueType.MEDIUM_FIRST_BAR_DIFF)\n", "\n", "print(f\"\\n{'='*60}\")\n", "print(f\"SUMMARY: Medium (~40ms) First Interval Difference\")\n", "print(f\"{'='*60}\")\n", "print(f\"Total generations: {total}\")\n", "print(f\"Bar pattern with medium diff: {bar_medium}/{total} ({bar_medium/total*100:.1f}%)\")\n", "print(f\"Beat pattern with medium diff: {beat_medium}/{total} ({beat_medium/total*100:.1f}%)\")\n", "print(f\"Either bar or beat with medium diff: {either_medium}/{total} ({either_medium/total*100:.1f}%)\")\n", "print(f\"{'='*60}\\n\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.text import read_jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "9", "metadata": {}, "outputs": [], "source": [ "sfx_data = read_jsonl(\"/app2/suno/data/sara/sfx_get_beats/combined_v3_w_extreme_metas_v0_aligned_filter_column.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "10", "metadata": {}, "outputs": [], "source": [ "aligned_data = []\n", "for x in sfx_data:\n", " if \"aligned_audio\" in x[\"s3_filepath\"] and x[\"is_reliable\"] and \"drums\" in x[\"tags\"] and x[\"duration_s\"] > 4.0:\n", " aligned_data.append(x)\n", "print(len(aligned_data))" ] }, { "cell_type": "code", "execution_count": null, "id": "11", "metadata": {}, "outputs": [], "source": [ "from suno_utils.tasks.dac_vae_fixed_25hz import encode_overlap, preload_models, decode\n", "\n", "model_filepath = \"s3://suno-data/minz/models/dac_vae_tuned_25hz.pth\"\n", "_ = preload_models(\n", " checkpoint_filepath=model_filepath\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "12", "metadata": {}, "outputs": [], "source": [ "# Analyze training examples for 40ms delay issue\n", "num_training_samples = 100 # Analyze first 100 training examples\n", "training_bar_issues = []\n", "training_beat_issues = []\n", "\n", "for idx in range(min(num_training_samples, len(aligned_data))):\n", " s3_filepath = aligned_data[idx][\"s3_filepath\"]\n", " audio = Audio.from_s3(s3_filepath)\n", " #output = encode_overlap([audio], normalize_volume=False)[0]\n", " #audio = decode(output)\n", " \n", " # Convert to mono for beat detection\n", " audio_mono = Audio.convert(audio, n_channels=1, sample_rate=audio.sample_rate, byte_width=audio.byte_width)\n", " \n", " # Extract beats - reinitialize extractor if it fails\n", " try:\n", " beats_refined = np.array(extractor.extract(audio_mono)[\"downbeats\"])\n", " except Exception as beat_error:\n", " # Reinitialize extractor if process pool fails\n", " print(f\"Reinitializing extractor at example {idx} due to: {beat_error}\")\n", " extractor = suno_utils.tasks.audio_features.beat_this_downbeat.BeatThisDownbeatExtractor(\n", " device=\"cuda\", \n", " model_path=\"s3://suno-data/m4burns/beat_this_rc_12l.pt\", \n", " resample_on_device=True\n", " )\n", " beats_refined = np.array(extractor.extract(audio_mono)[\"downbeats\"])\n", " \n", " # Analyze for beat issues (verbose=False to avoid plotting each one)\n", " bar_issue, beat_issue = plot_audio_with_beats(audio_mono, beats_refined, \n", " title_suffix=f\" - Training {idx}\", \n", " verbose=False)\n", " \n", " training_bar_issues.append(bar_issue)\n", " training_beat_issues.append(beat_issue)\n", " \n", " if (idx + 1) % 10 == 0:\n", " print(f\"Processed {idx + 1}/{num_training_samples} training examples...\")\n", "\n", "print(f\"Completed analysis of {len(training_bar_issues)} training examples\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "13", "metadata": {}, "outputs": [], "source": [ "# Summary statistics for training examples\n", "total_training = len(training_bar_issues)\n", "training_bar_medium = sum(1 for issue in training_bar_issues if issue == BeatIssueType.MEDIUM_FIRST_BAR_DIFF)\n", "training_beat_medium = sum(1 for issue in training_beat_issues if issue == BeatIssueType.MEDIUM_FIRST_BAR_DIFF)\n", "training_either_medium = sum(1 for b1, b2 in zip(training_bar_issues, training_beat_issues) \n", " if b1 == BeatIssueType.MEDIUM_FIRST_BAR_DIFF or b2 == BeatIssueType.MEDIUM_FIRST_BAR_DIFF)\n", "\n", "print(f\"\\n{'='*60}\")\n", "print(f\"SUMMARY: Medium (~40ms) First Interval Difference - TRAINING DATA\")\n", "print(f\"{'='*60}\")\n", "print(f\"Total training examples: {total_training}\")\n", "print(f\"Bar pattern with medium diff: {training_bar_medium}/{total_training} ({training_bar_medium/total_training*100:.1f}%)\")\n", "print(f\"Beat pattern with medium diff: {training_beat_medium}/{total_training} ({training_beat_medium/total_training*100:.1f}%)\")\n", "print(f\"Either bar or beat with medium diff: {training_either_medium}/{total_training} ({training_either_medium/total_training*100:.1f}%)\")\n", "print(f\"{'='*60}\\n\")\n", "\n", "# Also print comparison with generation results\n", "print(f\"{'='*60}\")\n", "print(f\"COMPARISON: Training Data vs Generations\")\n", "print(f\"{'='*60}\")\n", "print(f\"Training - Either bar or beat: {training_either_medium}/{total_training} ({training_either_medium/total_training*100:.1f}%)\")\n", "print(f\"Generation - Either bar or beat: {either_medium}/{total} ({either_medium/total*100:.1f}%)\")\n", "print(f\"{'='*60}\\n\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "14", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_clean", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 5 }