{
"cells": [
{
"cell_type": "code",
"execution_count": 434,
"id": "4a2f4e76",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import re\n",
"import time\n",
"import random\n",
"import json\n",
"import tqdm\n",
"import uuid\n",
"from bs4 import BeautifulSoup\n",
"\n",
"import cloudscraper\n",
"import youtube_dl\n",
"\n",
"from suno_utils.audio import Audio\n",
"\n",
"\n",
"def _courtesy_sleep(avg_sleep_dur_s=1.0):\n",
" time.sleep((0.5 + random.random() / 2) * avg_sleep_dur_s)\n",
"\n",
" \n",
"def get_cloudscraper(test_url=\"https://genius.com/songs/all\"):\n",
" for _ in range(10):\n",
" scraper = cloudscraper.create_scraper(disableCloudflareV1=True)\n",
" out = scraper.get(test_url)\n",
" if out.ok:\n",
" return scraper\n",
" _courtesy_sleep()\n",
" raise ValueError(\"could not initiate cloudscraper\")\n",
"\n",
"\n",
"def _parse_songs(soup):\n",
" song_data = []\n",
" for e in soup.find_all(\"ul\", attrs={\"class\": \"song_list\"}):\n",
" for ee in e.find_all(\"li\"):\n",
" data_id = ee.get_attribute_list(\"data-id\")[0]\n",
" url = ee.find_all(\"a\")[0].get_attribute_list(\"href\")[0]\n",
" title = ee.find_all(\"a\")[0].get_attribute_list(\"title\")[0]\n",
" song_data.append({\n",
" \"url\": url,\n",
" \"title\": title,\n",
" \"data_id\": data_id,\n",
" \"uid\": str(uuid.uuid4()),\n",
" })\n",
" return song_data\n",
"\n",
"\n",
"def get_genius_songs(scraper, from_page=1, to_page=3):\n",
" all_songs_data = []\n",
" for n in tqdm.tqdm(range(from_page, to_page + 1)):\n",
" out = scraper.get(f\"https://genius.com/songs/all?page={n}\")\n",
" assert(out.ok)\n",
" soup = BeautifulSoup(out.text)\n",
" all_songs_data.extend(_parse_songs(soup))\n",
" _courtesy_sleep()\n",
" return all_songs_data\n",
"\n",
"\n",
"def _parse_genius_info(soup):\n",
" title = None\n",
" artist = None\n",
" for e in soup.find_all(\"h1\"):\n",
" l = [ee for ee in e.get_attribute_list(\"class\") if ee is not None]\n",
" if any([re.search(r\"SongHeader[a-z]+\\_\\_Title\", ee) for ee in l]):\n",
" title = e.get_text()\n",
" for e in soup.find_all(\"a\"):\n",
" l = [ee for ee in e.get_attribute_list(\"class\") if ee is not None]\n",
" if any([re.search(r\"SongHeader[a-z]+\\_\\_Artist\", ee) for ee in l]):\n",
" artist = e.get_text()\n",
" lyrics_containers = soup.find_all(attrs={\"data-lyrics-container\": \"true\"})\n",
" tmp_container = []\n",
" for lyrics_container in lyrics_containers:\n",
" tmp_container.append(lyrics_container.get_text(separator=\" \\n \"))\n",
" lyrics = \" \\n\\n \".join(tmp_container)\n",
" return title, artist, lyrics\n",
"\n",
"\n",
"def get_genius_info(scraper, url):\n",
" out = scraper.get(url)\n",
" assert(out.ok)\n",
" soup = BeautifulSoup(out.text)\n",
" title, artist, lyrics = _parse_genius_info(soup)\n",
" youtube_url = re.findall(\n",
" r\"\\\\\\\"url\\\\\\\"\\:\\\\\\\"(https?\\:\\/\\/www\\.youtube\\.com\\/watch\\?v\\=[a-zA-Z0-9\\_]+)\", \n",
" out.text, \n",
" flags=re.DOTALL,\n",
" )[0]\n",
" song_info = {\n",
" \"title\": title,\n",
" \"artist\": artist,\n",
" \"lyrics\": lyrics,\n",
" \"youtube_url\": youtube_url,\n",
" }\n",
" return out.text, song_info"
]
},
{
"cell_type": "code",
"execution_count": 390,
"id": "3166f3d7",
"metadata": {},
"outputs": [],
"source": [
"GENIUS_DIR = \"/mnt/data-ssd-1/data/custom/genius\"\n",
"GENIUS_HTML_DIR = os.path.join(GENIUS_DIR, \"raw_html\")\n",
"GENIUS_AUDIO_DIR = os.path.join(GENIUS_DIR, \"audio\")\n",
"\n",
"os.makedirs(GENIUS_DIR, exist_ok=True)\n",
"os.makedirs(GENIUS_HTML_DIR, exist_ok=True)\n",
"os.makedirs(GENIUS_AUDIO_DIR, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 438,
"id": "2d516aaa",
"metadata": {},
"outputs": [],
"source": [
"scraper = get_cloudscraper()"
]
},
{
"cell_type": "markdown",
"id": "58580725",
"metadata": {},
"source": [
"## Get song list"
]
},
{
"cell_type": "code",
"execution_count": 323,
"id": "8df88bc6",
"metadata": {},
"outputs": [],
"source": [
"song_metas = get_genius_songs(scraper, from_page=1, to_page=100)"
]
},
{
"cell_type": "code",
"execution_count": 391,
"id": "ce73a6a6",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(GENIUS_DIR, \"song_metas.json\"), \"w\") as f:\n",
" json.dump(song_metas, f)"
]
},
{
"cell_type": "markdown",
"id": "38c121c7",
"metadata": {},
"source": [
"## Get song details"
]
},
{
"cell_type": "code",
"execution_count": 444,
"id": "e4a7eb39",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 88.1 ms, sys: 0 ns, total: 88.1 ms\n",
"Wall time: 397 ms\n"
]
}
],
"source": [
"# ~250 ms per page\n",
"\n",
"# get_genius_info(\"https://genius.com/Johnny-cash-hurt-lyrics\")\n",
"# raw_html, song_info = get_genius_info(scraper, \"https://genius.com/Cro-bye-bye-lyrics\")\n",
"\n",
"song_detail_metas = []\n",
"for song_meta in song_metas[:2]:\n",
" page_html, parsed_info = get_genius_info(scraper, song_meta[\"url\"])\n",
" with open(os.path.join(GENIUS_HTML_DIR, f\"{song_meta['uid']}.html\"), \"w\") as f:\n",
" f.write(page_html)\n",
" song_detail_metas.append({\n",
" \"uid\": song_meta[\"uid\"],\n",
" \"parsed_info\": parsed_info,\n",
" })\n",
" _courtesy_sleep(avg_sleep_dur_s=0.2)"
]
},
{
"cell_type": "code",
"execution_count": 436,
"id": "5088d594",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(GENIUS_DIR, \"song_detail_metas.json\"), \"w\") as f:\n",
" json.dump(song_detail_metas, f)"
]
},
{
"cell_type": "markdown",
"id": "dc450088",
"metadata": {},
"source": [
"## Download youtubes"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cfcbe400",
"metadata": {},
"outputs": [],
"source": [
"# TODO: add time logging"
]
},
{
"cell_type": "code",
"execution_count": 413,
"id": "c9fe73b9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[youtube] 72UO0v5ESUo: Downloading webpage\n",
"[download] Destination: /mnt/data-ssd-1/data/custom/genius/audio/8ed46560-5e75-484f-8841-8f96fcaee698.webm\n",
"[download] 100% of 1.45MiB in 00:2399KiB/s ETA 00:00\n",
"[youtube] XbGs_qK2PQA: Downloading webpage\n",
"[download] Destination: /mnt/data-ssd-1/data/custom/genius/audio/8bdad336-0a29-485f-ae4c-6dcbd31f72d4.webm\n",
"[download] 100% of 2.21MiB in 00:5085KiB/s ETA 00:00\n"
]
}
],
"source": [
"for song_detail_meta in tqdm.tqdm(song_detail_metas):\n",
" ydl_opts = {\n",
" \"format\": \"worstaudio/worst\",\n",
" \"outtmpl\": os.path.join(GENIUS_AUDIO_DIR, f\"{song_detail_meta['uid']}.webm\"),\n",
" \"verbose\": False,\n",
" \"quiet\": True,\n",
" }\n",
" with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n",
" out = ydl.download([song_detail_meta[\"parsed_info\"][\"youtube_url\"]])\n",
" assert(out == 0)\n",
" _courtesy_sleep()"
]
},
{
"cell_type": "code",
"execution_count": 446,
"id": "ae80834d",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"Audio.from_file(os.path.join(GENIUS_AUDIO_DIR, \"8bdad336-0a29-485f-ae4c-6dcbd31f72d4.webm\")).play()"
]
},
{
"cell_type": "code",
"execution_count": 450,
"id": "ad17fce6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"-rw-rw-r-- 1 georg georg 2.3M Mar 26 2022 /mnt/data-ssd-1/data/custom/genius/audio/8bdad336-0a29-485f-ae4c-6dcbd31f72d4.webm\r\n",
"-rw-rw-r-- 1 georg georg 1.1M Dec 8 2015 test.webm\r\n"
]
}
],
"source": [
"!ls -lah /mnt/data-ssd-1/data/custom/genius/audio/8bdad336-0a29-485f-ae4c-6dcbd31f72d4.webm test.webm"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ebc78c7c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "97069573",
"metadata": {},
"source": [
"## estimated"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8d6c542e",
"metadata": {},
"outputs": [],
"source": [
"250ms per lyrics page\n",
"10s per youtube download\n",
"25Mb per hour of music\n",
"\n",
"aim ~10k hours: 250Gb, 10h download lyrics, ?? download music"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "38c98df8",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "c579e3da",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "48a39fd3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "be2f7993",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "0dfae2d6",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": 168,
"id": "cdfa4c91",
"metadata": {},
"outputs": [],
"source": [
"from youtube_search import YoutubeSearch\n",
"\n",
"videos = YoutubeSearch('Hurt Johnny Cash', max_results=10).to_dict()\n",
"video_url = \"https://www.youtube.com/\" + videos[0][\"url_suffix\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4dedc1e9",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 13,
"id": "4ef860a3",
"metadata": {},
"outputs": [],
"source": [
"import youtube_dl\n",
"from IPython.utils.io import capture_output"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1fb3ff12",
"metadata": {},
"outputs": [],
"source": [
"from IPython.utils import io\n",
"\n",
"with io.capture_output() as captured:"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "c86c89ff",
"metadata": {},
"outputs": [
{
"ename": "ImportError",
"evalue": "cannot import name 'suppress_output' from 'IPython.display' (/home/georg/venvs/ml/lib/python3.8/site-packages/IPython/display.py)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/tmp/ipykernel_3822810/1974657839.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msuppress_output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mImportError\u001b[0m: cannot import name 'suppress_output' from 'IPython.display' (/home/georg/venvs/ml/lib/python3.8/site-packages/IPython/display.py)"
]
}
],
"source": [
"from IPython.display import %%capture"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "a5abf5fa",
"metadata": {},
"outputs": [],
"source": [
"for n in range(2):\n",
" try:\n",
" with youtube_dl.YoutubeDL({\"logger\": True}) as ydl:\n",
" with redirect_stderr(None):\n",
" out = ydl.download([\"https://www.youtube.com/watch?v=_Z3QKkl\"])\n",
" except:\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c0209625",
"metadata": {},
"outputs": [],
"source": [
"youtube_dl.YoutubeDL()"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "50aff41e",
"metadata": {},
"outputs": [],
"source": [
"from contextlib import redirect_stderr"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f706594a",
"metadata": {},
"outputs": [],
"source": [
"from contextlib import redirect_stdout\n",
"\n",
"with redirect_stdout(None):"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e5a21041",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}