{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b304dfff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f955e852",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e9716000",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import os\n",
    "import tqdm\n",
    "import shutil\n",
    "import random\n",
    "import funcy\n",
    "import json\n",
    "import numpy as np\n",
    "import multiprocessing\n",
    "import pandas as pd\n",
    "from suno_utils.audio import Audio\n",
    "from suno_utils.audio.conversion import play_audio, get_audio_properties\n",
    "from suno_utils.utils.slicer import get_vad_gaps, get_intervals_from_gaps\n",
    "\n",
    "\n",
    "DATE_DIR = \"2022_08_11\"\n",
    "\n",
    "BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/\"\n",
    "TO_REV_DIR = os.path.join(BASE_DIR, \"to_rev\", DATE_DIR)\n",
    "FROM_REV_DIR = os.path.join(BASE_DIR, \"from_rev\", DATE_DIR)\n",
    "TO_REV_AUDIO_DIR = os.path.join(TO_REV_DIR, \"audio\")\n",
    "REV_MANIFEST_FILEPATH = os.path.join(TO_REV_DIR, \"rev_manifest.jsonl\")\n",
    "FAILED_MANIFEST_FILEPATH = os.path.join(TO_REV_DIR, \"failed_manifest.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f4938567",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 27.5 hours we sent in first batch\n",
    "# 0.954 - fraction we retain roughly after discard silences\n",
    "# -> we want 180 hours to beable to deliver 172.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "143da265",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4002 files total\n",
      "814.1h total\n",
      "199.9h rev total\n",
      "179.7h segment total\n"
     ]
    }
   ],
   "source": [
    "metadata = []\n",
    "with open(REV_MANIFEST_FILEPATH) as f:\n",
    "    for l in f.read().strip().split(\"\\n\"):\n",
    "        metadata.append(json.loads(l))\n",
    "        \n",
    "FIRST_N = 4360\n",
    "# TODO: filter stuff shorter than 30 seconds?\n",
    "metadata = [m for m in metadata[:FIRST_N] if m[\"audio_duration_s\"] >= 30]\n",
    "        \n",
    "tot_duration_s = [get_audio_properties(m[\"original_audio_filepath\"])[\"duration_s\"] for m in metadata]\n",
    "tot_rev_duration_s = [m[\"audio_duration_s\"] for m in metadata]\n",
    "tot_seg_duration_s = 0\n",
    "for m in metadata:\n",
    "    for e in m[\"segments_meta\"]:\n",
    "        if e[\"type\"] == \"speech\":\n",
    "            tot_seg_duration_s += e[\"duration_s\"]\n",
    "print(len(metadata), \"files total\")\n",
    "print(\"{}h total\".format(round(sum(tot_duration_s) / 60 / 60, 1)))\n",
    "print(\"{}h rev total\".format(round(sum(tot_rev_duration_s) / 60 / 60, 1)))\n",
    "print(\"{}h segment total\".format(round(tot_seg_duration_s / 60 / 60, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f378e839",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbfb03be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c9b03909",
   "metadata": {},
   "source": [
    "## Investigate for mistakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3bf60d34",
   "metadata": {},
   "outputs": [],
   "source": [
    "# l = metadata[:]\n",
    "# random.shuffle(l)\n",
    "# for m in l[:10]:\n",
    "#     print(m[\"uid\"])\n",
    "#     Audio.from_file(m[\"audio_filepath\"]).play()\n",
    "#     print(\"-\"*10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e6d1a1bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for meta in metadata:\n",
    "#     if meta[\"uid\"] == \"29c7be40-46ef-4000-a8e4-e0a4e4a6b828\":\n",
    "#         print(\"found\")\n",
    "#         break\n",
    "# #     print(meta[\"misc_meta\"][\"tot_silence_frac\"] + meta[\"misc_meta\"][\"segment_cov_frac\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b3ca219",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c91afe9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "0868d256",
   "metadata": {},
   "source": [
    "## (Optional) Check price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "37decb32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4002 files total\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "suno     16779.6\n",
       "sanas    29109.5\n",
       "dtype: float64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suno_price_per_hour = 72\n",
    "suno_price_per_min = suno_price_per_hour / 60\n",
    "\n",
    "sanas_price = 162 # dollars_per_hour\n",
    "sanas_price_per_s = sanas_price / 3600  # dollars per s\n",
    "\n",
    "\n",
    "costs = []\n",
    "for rm in metadata:\n",
    "    sanas_duration_s = np.sum([sm[\"duration_s\"] for sm in rm[\"segments_meta\"] if sm[\"type\"] == \"speech\"])\n",
    "    sanas_cost = sanas_price_per_s * sanas_duration_s\n",
    "    \n",
    "    suno_cost = np.ceil(rm[\"audio_duration_s\"] / 60.) * suno_price_per_min\n",
    "    costs.append({\"suno\": suno_cost, \"sanas\": sanas_cost})\n",
    "\n",
    "print(len(metadata), \"files total\")\n",
    "costs = pd.DataFrame(costs)\n",
    "# suno     2770.8\n",
    "# sanas    4715.9\n",
    "costs.sum().round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "def684e9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcf743da",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a31702b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "88a9e9cc",
   "metadata": {},
   "source": [
    "## Send stuff to rev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "613bd82e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ORDER_REF_STR = \"sns_180\"\n",
    "GLOBAL_HOTWORDS = [\"WinID\", \"Continuum Service Desk\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1a576034",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4002 files total\n",
      "199.9h total\n"
     ]
    }
   ],
   "source": [
    "tot_duration_s = [m[\"audio_duration_s\"] for m in metadata]\n",
    "print(len(metadata), \"files total\")\n",
    "print(\"{}h total\".format(round(sum(tot_duration_s) / 60 / 60, 1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b16a0dd0",
   "metadata": {},
   "source": [
    "### 01 - upload items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2453089e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from suno_utils.web.rev import get_auth_string, upload_file\n",
    "\n",
    "with open(\"/home/georg/.secrets/secrets.json\") as f:\n",
    "    secrets = json.load(f)\n",
    "    \n",
    "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n",
    "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n",
    "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8c66f7f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:02<00:00,  1.43s/it]\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "\n",
    "order_items = []\n",
    "for audio_meta in tqdm.tqdm(metadata):\n",
    "    media_loc = upload_file(AUTH_STR, audio_meta[\"audio_filepath\"], file_ref_str=audio_meta[\"uid\"])\n",
    "    order_items.append({\n",
    "        \"media_loc\": media_loc,\n",
    "        \"hotwords\": audio_meta[\"hotwords\"] + GLOBAL_HOTWORDS,\n",
    "#         \"accents\": [\"Australian\"],\n",
    "        \"duration_s\": int(round(audio_meta[\"audio_duration_s\"])),\n",
    "        \"uuid\": audio_meta[\"uid\"],\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "a0ee5f4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"w\") as f:\n",
    "    json.dump(order_items, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9131358e",
   "metadata": {},
   "source": [
    "### 02 - make order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7ccf6e0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from suno_utils.web.rev import get_auth_string, make_order\n",
    "\n",
    "with open(\"/home/georg/.secrets/secrets.json\") as f:\n",
    "    secrets = json.load(f)\n",
    "    \n",
    "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n",
    "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n",
    "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0b300ef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"r\") as f:\n",
    "    order_items = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8318e46f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import funcy\n",
    "order_numbers = []\n",
    "for n, items_chunk in enumerate(funcy.chunks(500, order_items)):\n",
    "    order_number = make_order(AUTH_STR, items_chunk, order_ref_str=ORDER_REF_STR + f\"_{n}\", add_rush=False)\n",
    "    order_numbers.append(order_number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "27549d51",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\"), \"w\") as f:\n",
    "    f.write(\"\\n\".join(order_numbers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5531cb02",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c5f5bd2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f30acc60",
   "metadata": {},
   "source": [
    "### 04 - pull transcripts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "65101ca4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n",
    "\n",
    "with open(\"/home/georg/.secrets/secrets.json\") as f:\n",
    "    secrets = json.load(f)\n",
    "    \n",
    "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n",
    "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n",
    "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "de613d78",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\")) as f:\n",
    "    order_numbers = f.read().split(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "9319c7f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4000 transcripts\n"
     ]
    }
   ],
   "source": [
    "# get intermediate results\n",
    "from suno_utils.web.rev import _get_order_details, _parse_order_response\n",
    "transcript_metas = []\n",
    "for order_number in order_numbers:\n",
    "    order_details = _get_order_details(AUTH_STR, order_number)\n",
    "    order_status, transcript_metas_chunk = _parse_order_response(order_details)\n",
    "    transcript_metas.extend(transcript_metas_chunk)\n",
    "print(len(transcript_metas), \"transcripts\")\n",
    "# 3993"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "58ae9a27",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transcript_metas = []\n",
    "# for order_number in order_numbers:\n",
    "#     transcript_metas.extend(get_finished_order(AUTH_STR, order_number, blocking=True))\n",
    "# print(len(transcript_metas), \"transcripts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "71cd19bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4000/4000 [16:07<00:00,  4.13it/s]\n"
     ]
    }
   ],
   "source": [
    "for transcript_meta in tqdm.tqdm(transcript_metas):\n",
    "    transcript_filepath = os.path.join(FROM_REV_DIR, f\"{transcript_meta['uuid']}.json\")\n",
    "    if os.path.exists(transcript_filepath):\n",
    "        continue\n",
    "    transcript_json = get_transcript(AUTH_STR, transcript_meta[\"rev_id\"])\n",
    "    with open(transcript_filepath, \"w\") as f:\n",
    "        json.dump(transcript_json, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "2931df3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3083 transcripts\n"
     ]
    }
   ],
   "source": [
    "batch_1_fns = []\n",
    "for fn in os.listdir(FROM_REV_DIR):\n",
    "    if not fn.endswith(\".json\"):\n",
    "        continue\n",
    "    batch_1_fns.append(fn)\n",
    "print(len(batch_1_fns), \"transcripts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "91713713",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(TO_REV_DIR, \"batch_1_fns.txt\"), \"w\") as f:\n",
    "    f.write(\"\\n\".join(batch_1_fns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b34c3810",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "917 transcripts\n"
     ]
    }
   ],
   "source": [
    "batch_2_fns = []\n",
    "for fn in os.listdir(FROM_REV_DIR):\n",
    "    if not fn.endswith(\".json\"):\n",
    "        continue\n",
    "    if fn in set(batch_1_fns):\n",
    "        continue\n",
    "    batch_2_fns.append(fn)\n",
    "print(len(batch_2_fns), \"transcripts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "f7d843a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(TO_REV_DIR, \"batch_2_fns.txt\"), \"w\") as f:\n",
    "    f.write(\"\\n\".join(batch_2_fns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82cec0a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26291bd7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de0f162",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "40f74bcc",
   "metadata": {},
   "source": [
    "## Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a581abf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7073241f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0174211",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e8dbe8e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}