{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "ae928c71", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:45:40.346949Z", "start_time": "2024-03-17T02:45:40.345606Z" } }, "outputs": [], "source": [ "# https://gist.github.com/karpathy/00103b0037c5aaea32fe1da1af553355" ] }, { "cell_type": "code", "execution_count": 2, "id": "08218f19", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:45:43.747076Z", "start_time": "2024-03-17T02:45:40.347895Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/tony/anaconda3/envs/suno_env/lib/python3.10/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", " torch.utils._pytree._register_pytree_node(\n", "/home/tony/anaconda3/envs/suno_env/lib/python3.10/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", " torch.utils._pytree._register_pytree_node(\n" ] } ], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n", "import torch\n", "from diffusers import (\n", " StableDiffusionXLPipeline,\n", " UNet2DConditionModel,\n", " EulerDiscreteScheduler,\n", ")\n", "from huggingface_hub import hf_hub_download\n", "from safetensors.torch import load_file\n", "import tqdm\n", "\n", "# iterate the loop\n", "import numpy as np\n", "import inspect\n", "from diffusers.schedulers import LMSDiscreteScheduler\n", "from PIL import Image\n", "import cv2\n", "\n", "from diffusers import (\n", " StableDiffusionControlNetPipeline,\n", " ControlNetModel,\n", " UniPCMultistepScheduler,\n", " StableDiffusionImg2ImgPipeline,\n", ")\n", "from diffusers.utils import load_image" ] }, { "cell_type": "code", "execution_count": 3, "id": "6917f125", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:45:44.017085Z", "start_time": "2024-03-17T02:45:43.749178Z" } }, "outputs": [], "source": [ "# download an image\n", "image = load_image(\n", " \"https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png\"\n", ")\n", "image = np.array(image)\n", "\n", "# get canny image\n", "image = cv2.Canny(image, 100, 200)\n", "image = image[:, :, None]\n", "image = np.concatenate([image, image, image], axis=2)\n", "canny_image = Image.fromarray(cv2.resize(image, (1024, 1024)))" ] }, { "cell_type": "code", "execution_count": 4, "id": "b13cdc23", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:02.490986Z", "start_time": "2024-03-17T02:45:44.020190Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/tony/anaconda3/envs/suno_env/lib/python3.10/site-packages/diffusers/configuration_utils.py:244: FutureWarning: It is deprecated to pass a pretrained model name or path to `from_config`.If you were trying to load a model, please use .load_config(...) followed by .from_config(...) instead. Otherwise, please make sure to pass a configuration dictionary instead. This functionality will be removed in v1.0.0.\n", " deprecate(\"config-passed-as-path\", \"1.0.0\", deprecation_message, standard_warn=False)\n", "Cannot initialize model with low cpu memory usage because `accelerate` was not found in the environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install `accelerate` for faster and less memory-intense model loading. You can do so with: \n", "```\n", "pip install accelerate\n", "```\n", ".\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "9660c0bfae93486e8831b3ed51078707", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading pipeline components...: 0%| | 0/7 [00:00" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "image = pipe(\n", " prompt=\"a woman\",\n", " num_inference_steps=num_inference_steps,\n", " image=canny_image,\n", " guidance_scale=guidance_scale,\n", " height=height,\n", " width=width,\n", ").images[0]\n", "image" ] }, { "cell_type": "code", "execution_count": 6, "id": "53b112ad", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:04.150775Z", "start_time": "2024-03-17T02:46:04.147115Z" } }, "outputs": [], "source": [ "# base = \"stabilityai/stable-diffusion-xl-base-1.0\"\n", "# repo = \"ByteDance/SDXL-Lightning\"\n", "# ckpt = \"sdxl_lightning_1step_unet_x0.safetensors\" # Use the correct ckpt for your step setting!\n", "\n", "# # Load model.\n", "# unet = UNet2DConditionModel.from_config(base, subfolder=\"unet\").to(\n", "# \"cuda\", torch.float16\n", "# )\n", "# unet.load_state_dict(load_file(hf_hub_download(repo, ckpt), device=\"cuda\"))\n", "# pipe = StableDiffusionXLPipeline.from_pretrained(\n", "# base, unet=unet, torch_dtype=torch.float16, variant=\"fp16\"\n", "# ).to(\"cuda\")\n", "\n", "# # Ensure sampler uses \"trailing\" timesteps and \"sample\" prediction type.\n", "# pipe.scheduler = EulerDiscreteScheduler.from_config(\n", "# pipe.scheduler.config, timestep_spacing=\"trailing\", prediction_type=\"sample\"\n", "# )" ] }, { "cell_type": "code", "execution_count": 7, "id": "0c23378b", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:04.224888Z", "start_time": "2024-03-17T02:46:04.152217Z" } }, "outputs": [], "source": [ "def slerp(t, v0, v1, DOT_THRESHOLD=0.9995):\n", " \"\"\"helper function to spherically interpolate two arrays v1 v2\"\"\"\n", "\n", " if not isinstance(v0, np.ndarray):\n", " inputs_are_torch = True\n", " input_device = v0.device\n", " v0 = v0.cpu().numpy()\n", " v1 = v1.cpu().numpy()\n", "\n", " dot = np.sum(v0 * v1 / (np.linalg.norm(v0) * np.linalg.norm(v1)))\n", " if np.abs(dot) > DOT_THRESHOLD:\n", " v2 = (1 - t) * v0 + t * v1\n", " else:\n", " theta_0 = np.arccos(dot)\n", " sin_theta_0 = np.sin(theta_0)\n", " theta_t = theta_0 * t\n", " sin_theta_t = np.sin(theta_t)\n", " s0 = np.sin(theta_0 - theta_t) / sin_theta_0\n", " s1 = sin_theta_t / sin_theta_0\n", " v2 = s0 * v0 + s1 * v1\n", "\n", " if inputs_are_torch:\n", " v2 = torch.from_numpy(v2).to(input_device)\n", "\n", " return v2\n", "\n", "\n", "@torch.no_grad()\n", "def diffuse(\n", " pipe,\n", " cond_embeddings, # text conditioning, should be (1, 77, 768)\n", " cond_latents, # image conditioning, should be (1, 4, 64, 64)\n", " num_inference_steps,\n", " guidance_scale,\n", " eta,\n", "):\n", " torch_device = cond_latents.get_device()\n", "\n", " # classifier guidance: add the unconditional embedding\n", " max_length = cond_embeddings.shape[1] # 77\n", " uncond_input = pipe.tokenizer(\n", " [\"\"], padding=\"max_length\", max_length=max_length, return_tensors=\"pt\"\n", " )\n", " uncond_embeddings = pipe.text_encoder(uncond_input.input_ids.to(torch_device))[0]\n", " text_embeddings = torch.cat([uncond_embeddings, cond_embeddings])\n", "\n", " # init the scheduler\n", " accepts_offset = \"offset\" in set(\n", " inspect.signature(pipe.scheduler.set_timesteps).parameters.keys()\n", " )\n", " extra_set_kwargs = {}\n", " if accepts_offset:\n", " extra_set_kwargs[\"offset\"] = 1\n", " pipe.scheduler.set_timesteps(num_inference_steps, **extra_set_kwargs)\n", "\n", " # diffuse!\n", " for i, t in enumerate(pipe.scheduler.timesteps):\n", " # expand the latents for classifier free guidance\n", " latent_model_input = torch.cat([cond_latents] * 2)\n", "\n", " # predict the noise residual\n", " pipe.unet.config.addition_embed_type = None\n", " noise_pred = pipe.unet(\n", " latent_model_input,\n", " t,\n", " encoder_hidden_states=text_embeddings,\n", " added_cond_kwargs={},\n", " )[\"sample\"]\n", "\n", " # compute the previous noisy sample x_t -> x_t-1\n", " cond_latents = pipe.scheduler.step(\n", " noise_pred, t, cond_latents, **extra_step_kwargs\n", " )[\"prev_sample\"]\n", "\n", " # scale and decode the image latents with vae\n", " cond_latents = 1 / 0.18215 * cond_latents\n", " image = pipe.vae.decode(cond_latents)[\"sample\"]\n", "\n", " # generate output numpy image as uint8\n", " image = (image / 2 + 0.5).clamp(0, 1)\n", " image = image.cpu().permute(0, 2, 3, 1).numpy()\n", " image = (image[0] * 255).astype(np.uint8)\n", "\n", " return image" ] }, { "cell_type": "code", "execution_count": 8, "id": "3d9ec02f", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:04.293391Z", "start_time": "2024-03-17T02:46:04.226988Z" } }, "outputs": [], "source": [ "prompt = \"a beautiful woman\"" ] }, { "cell_type": "code", "execution_count": 9, "id": "34269806", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:04.624600Z", "start_time": "2024-03-17T02:46:04.295383Z" } }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "08598cd3cb3948b5ba5985c37f70cfd4", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/2 [00:00" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "image = pipe(\n", " prompt=prompt,\n", " num_inference_steps=num_inference_steps,\n", " latents=test_latents,\n", " guidance_scale=guidance_scale,\n", " height=height,\n", " width=width,\n", ").images[0]\n", "image" ] }, { "cell_type": "code", "execution_count": 12, "id": "6a665db7", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.622796Z", "start_time": "2024-03-17T02:46:05.672484Z" } }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c4eafb88059343d39b34b0a0d024f26a", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/2 [00:00" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "image = pipe(\n", " prompt=prompt,\n", " num_inference_steps=num_inference_steps,\n", " latents=test_latents_2,\n", " guidance_scale=guidance_scale,\n", " height=height,\n", " width=width,\n", ").images[0]\n", "image" ] }, { "cell_type": "code", "execution_count": 13, "id": "69bdd4fd", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.812464Z", "start_time": "2024-03-17T02:46:06.625793Z" } }, "outputs": [ { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[13], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "BREAK" ] }, { "cell_type": "markdown", "id": "7e99099e", "metadata": {}, "source": [ "# Make the frames" ] }, { "cell_type": "code", "execution_count": null, "id": "0291cab9", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.814069Z", "start_time": "2024-03-17T02:46:06.814060Z" } }, "outputs": [], "source": [ "outdir = \"/home/tony/Work/tony/videos/frames\"\n", "os.makedirs(outdir, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "a773cdd2", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.814939Z", "start_time": "2024-03-17T02:46:06.814930Z" } }, "outputs": [], "source": [ "prompt = \"pikachu walking down the street, feeling so alive\"" ] }, { "cell_type": "code", "execution_count": null, "id": "04b110dd", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.815716Z", "start_time": "2024-03-17T02:46:06.815708Z" } }, "outputs": [], "source": [ "pipe.set_progress_bar_config(disable=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "14a38ac4", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.816410Z", "start_time": "2024-03-17T02:46:06.816402Z" } }, "outputs": [], "source": [ "num_steps = 30 * 5\n", "n_images = 12\n", "max_frames = num_steps * n_images # 10000\n", "guidance_scale = 0\n", "init1 = torch.randn(my_latents.shape, device=\"cuda\").to(torch.float16)\n", "frame_index = 0\n", "for n_image in tqdm.tqdm(range(n_images)):\n", " # sample the destination\n", " init2 = torch.randn(my_latents.shape, device=\"cuda\").to(torch.float16)\n", "\n", " for t in np.linspace(0, 1, num_steps):\n", " init = slerp(float(t), init1, init2)\n", " # print(\"dreaming... \", frame_index)\n", " image = pipe(\n", " prompt=prompt,\n", " num_inference_steps=num_inference_steps,\n", " latents=init,\n", " guidance_scale=guidance_scale,\n", " height=height,\n", " width=width,\n", " ).images[0]\n", " outpath = os.path.join(outdir, \"frame%06d.jpg\" % frame_index)\n", " image.save(outpath, quality=75)\n", " frame_index += 1\n", " init1 = init2" ] }, { "cell_type": "code", "execution_count": null, "id": "ec67f2b8", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.817250Z", "start_time": "2024-03-17T02:46:06.817241Z" } }, "outputs": [], "source": [ "!ffmpeg -y -r 30 -f image2 -s 1024x1024 \\\n", " -i /home/tony/Work/tony/videos/frames/frame%06d.jpg \\\n", " -vcodec mpeg4 /home/tony/Work/tony/videos/test.avi" ] }, { "cell_type": "code", "execution_count": null, "id": "8e22081a", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.817978Z", "start_time": "2024-03-17T02:46:06.817970Z" } }, "outputs": [], "source": [ "!ffmpeg -y -i /home/tony/Work/tony/videos/test.avi \\\n", " -i /home/tony/Work/tony/audios/walking_down.mp3 \\\n", " -map 0:v -map 1:a -c:v copy -shortest /home/tony/Work/tony/videos/test2.avi" ] }, { "cell_type": "code", "execution_count": null, "id": "ad76be1c", "metadata": { "ExecuteTime": { "end_time": "2024-03-17T02:46:06.818760Z", "start_time": "2024-03-17T02:46:06.818753Z" } }, "outputs": [], "source": [ "!ffmpeg -y -i /home/tony/Work/tony/videos/test2.avi /home/tony/Work/tony/videos/test.mp4" ] }, { "cell_type": "code", "execution_count": null, "id": "574bca79", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1e0e467b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8afdefae", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8f36530a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5e8e5841", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e3563a02", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "12c0abf4", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }