TTS_voice_cloning_pub.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/olaviinha/NeuralTextToAudio/blob/main/TTS_voice_cloning_pub.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pVxFPluEoUka"
      },
      "source": [
        "#<font face=\"Trebuchet MS\" size=\"6\">Voice cloning TTS<font color=\"#999\" size=\"4\">&nbsp;&nbsp;&nbsp;|&nbsp;&nbsp;&nbsp;</font><font color=\"#999\" size=\"4\">Text-to-audio</font><font color=\"#999\" size=\"4\">&nbsp;&nbsp;&nbsp;|&nbsp;&nbsp;&nbsp;<a href=\"https://github.com/olaviinha/NeuralTextToAudio\" target=\"_blank\"><font color=\"#999\" size=\"4\">Github</font></a></font>\n",
        "\n",
        "Text-to-speech tool that takes a text input and and audio file of a voice, and produces a new audio file with input text spoken with the voice of the voice audio provided. Do not expect realistic state-of-the-art outputs. With a good speaker voice some results may be relatively good, but most will be poor.\n",
        "\n",
        "#### Tips:\n",
        "- `local_models_dir` is optional but recommended. It will store models in your Google Drive and/or use them from there if already available.\n",
        "- All directory paths should be relative to your Google Drive root, e.g. `output_dir` should be `music/ai-generated-sounds` if you have a directory called _music_ in your Drive, containing a subdirectory called _ai-generated-sounds_."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "4D3YFbKGoSje"
      },
      "outputs": [],
      "source": [
        "#@title #Setup\n",
        "#@markdown This cell needs to be run only once. It will mount your Google Drive and setup prerequisites.\n",
        "#@markdown <br><small>Mounting Google Drive is required by this notebook.</small>\n",
        "force_setup = False\n",
        "pip_packages = ''\n",
        "local_models_dir = \"\" #@param{type:\"string\"}\n",
        "# main_repository = 'https://github.com/CorentinJ/Real-Time-Voice-Cloning.git'\n",
        " \n",
        "import os\n",
        "from google.colab import output\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "%cd /content/\n",
        " \n",
        "# inhagcutils\n",
        "if not os.path.isfile('/content/inhagcutils.ipynb') and force_setup == False:\n",
        "  !pip -q install import-ipynb {pip_packages}\n",
        "  !curl -s -O https://raw.githubusercontent.com/olaviinha/inhagcutils/master/inhagcutils.ipynb\n",
        "import import_ipynb\n",
        "from inhagcutils import *\n",
        " \n",
        "# Mount Drive\n",
        "if not os.path.isdir('/content/drive') and force_setup == False:\n",
        "  from google.colab import drive\n",
        "  drive.mount('/content/drive')\n",
        " \n",
        "# Drive symlink\n",
        "if not os.path.isdir('/content/mydrive') and force_setup == False:\n",
        "  os.symlink('/content/drive/My Drive', '/content/mydrive')\n",
        "  drive_root_set = True\n",
        "drive_root = '/content/mydrive/'\n",
        " \n",
        "dir_tmp = '/content/tmp/'\n",
        "dir_source = '/content/tmp/source/'\n",
        "dir_chops = '/content/tmp/chops/'\n",
        "dir_clips = '/content/tmp/clips/'\n",
        "create_dirs([dir_tmp, dir_source, dir_chops, dir_clips])\n",
        " \n",
        "#----\n",
        "\n",
        "import os\n",
        "from os.path import exists, join, basename, splitext\n",
        " \n",
        "git_repo_url = 'https://github.com/CorentinJ/Real-Time-Voice-Cloning.git'\n",
        "project_name = splitext(basename(git_repo_url))[0]\n",
        "if not exists(project_name) or force_setup == True:\n",
        "  %cd /content/\n",
        "  # clone and install\n",
        "  !git clone -q --recursive {git_repo_url}\n",
        "  # install dependencies\n",
        "  !cd {project_name} && pip install -q -r requirements.txt\n",
        "  !pip install -q gdown\n",
        "  !apt-get install -qq libportaudio2\n",
        "  !apt install sox\n",
        " \n",
        "  # download pretrained model\n",
        "if local_models_dir == \"\":\n",
        "  dir_models = dir_tmp\n",
        "  # !cd {project_name} && wget https://github.com/blue-fish/Real-Time-Voice-Cloning/releases/download/v1.0/pretrained.zip && unzip -o pretrained.zip\n",
        "else:\n",
        "  dir_models = drive_root+fix_path(local_models_dir)\n",
        "  if not os.path.isdir(dir_models):\n",
        "    os.mkdir(dir_models)\n",
        " \n",
        "%cd /content/\n",
        "%cd {project_name}\n",
        "\n",
        "import sys\n",
        "sys.path.append(project_name)\n",
        " \n",
        "# from IPython.display import display, Audio, clear_output\n",
        "from IPython.display import Audio \n",
        "from IPython.core.display import display\n",
        "\n",
        "from IPython.utils import io\n",
        "import ipywidgets as widgets\n",
        "import numpy as np\n",
        " \n",
        "from synthesizer.inference import Synthesizer\n",
        "from encoder import inference as encoder\n",
        "from vocoder import inference as vocoder\n",
        "from pathlib import Path\n",
        "import soundfile as sf\n",
        "\n",
        "if not os.path.isfile(dir_models+\"encoder.pt\"):\n",
        "  %cd \"{dir_models}\"\n",
        "  !wget https://drive.google.com/uc?export=download&id=1q8mEGwCkFy23KZsinbuvdKAQLqNKbYf1\n",
        "if not os.path.isfile(dir_models+\"synthesizer.pt\"):\n",
        "  %cd \"{dir_models}\"\n",
        "  !wget https://drive.google.com/uc?export=download&id=1EqFMIbvxffxtjiVrtykroF6_mUh-5Z3s\n",
        "if not os.path.isfile(dir_models+\"vocoder.pt\"):\n",
        "  %cd \"{dir_models}\"\n",
        "  !wget https://drive.google.com/uc?export=download&id=1cf2NO6FtI0jDuy8AV3Xgn6leO6dHjIgu\n",
        "\n",
        "%cd /content/\n",
        "%cd {project_name}\n",
        "\n",
        "encoder.load_model(project_name / Path(dir_models+\"encoder.pt\"))\n",
        "synthesizer = Synthesizer(project_name / Path(dir_models+\"synthesizer.pt\"))\n",
        "vocoder.load_model(project_name / Path(dir_models+\"vocoder.pt\"))\n",
        " \n",
        "fade_ms = 3\n",
        "global_fade = fade_ms/1000\n",
        "\n",
        "#----\n",
        " \n",
        "def _compute_embedding(audio):\n",
        "  global embedding, sample_rate\n",
        "  # op(c.title, 'Voice source audio:')\n",
        "  # display(Audio(audio, rate=sample_rate, autoplay=False))\n",
        "  embedding = None\n",
        "  embedding = encoder.embed_utterance(encoder.preprocess_wav(audio,sample_rate))\n",
        " \n",
        "def synthesize(embed, text, save_as='', show_player=False):\n",
        "  #with io.capture_output() as captured:\n",
        "  specs = synthesizer.synthesize_spectrograms([text], [embed])\n",
        "  generated_wav = vocoder.infer_waveform(specs[0])\n",
        "  generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode=\"constant\")\n",
        "  if save_as != '':\n",
        "    sf.write(save_as, generated_wav, synthesizer.sample_rate, 'PCM_24')\n",
        "  if show_player:\n",
        "    audio_player(generated_wav)\n",
        " \n",
        "#----\n",
        "\n",
        "def normalize(audio):\n",
        "  return np.interp(audio, (audio.min(), audio.max()), (-1, 1))\n",
        "  \n",
        "def clip_audio(audio_data, start, duration, sr=44100):\n",
        "  print('clipping to', start, duration)\n",
        "  global global_fade\n",
        "  xstart = librosa.time_to_samples(start, sr=sr)\n",
        "  xduration = librosa.time_to_samples(start+duration, sr=sr)\n",
        "  audio_data = fade_audio(audio_data[:, xstart:xduration], sr=sr)\n",
        "  return audio_data\n",
        "\n",
        "def fade_audio(audio_data, fade_in=global_fade, fade_out=global_fade, sr=44100):\n",
        "  a_duration = librosa.get_duration(audio_data, sr=sr)\n",
        "  if fade_in > 0:\n",
        "    fade_in_to = librosa.time_to_samples(fade_in, sr=sr)\n",
        "    in_y = audio_data[:, 0:fade_in_to]\n",
        "    fade_ins = []\n",
        "    for channel in in_y:\n",
        "      fade = [ i/len(channel)*smp for i, smp in enumerate(channel) ]\n",
        "      fade_ins.append(fade)\n",
        "    fade_ins = np.array(fade_ins)\n",
        "    tail_start = fade_in_to+1  \n",
        "    tail = audio_data[:, tail_start:]\n",
        "    audio_data = np.concatenate([fade_ins, tail], axis=1)\n",
        "  if fade_out > 0:\n",
        "    fade_out_start = librosa.time_to_samples(a_duration-fade_out, sr=sr)\n",
        "    out_y = audio_data[:, fade_out_start:]\n",
        "    fade_outs = []\n",
        "    for channel in out_y:\n",
        "      fade = [ smp-(i/len(channel)*smp) for i, smp in enumerate(channel) ]\n",
        "      fade_outs.append(fade)\n",
        "    fade_outs = np.array(fade_outs)\n",
        "    head_start = fade_out_start-1\n",
        "    head = audio_data[:, :head_start]\n",
        "    audio_data = np.concatenate([head, fade_outs], axis=1)\n",
        "  return audio_data\n",
        "\n",
        "#---- \n",
        "\n",
        " \n",
        "output.clear()\n",
        "# !nvidia-smi\n",
        "op(c.ok, 'Setup finished.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "EBuZ6jhBo1ro"
      },
      "outputs": [],
      "source": [
        "#@title # Synthesize\n",
        "text = \"\" #@param {type:\"string\"}\n",
        "speaker_voice_wav = \"\" #@param {type:\"string\"}\n",
        "normalize_speaker_volume = True #@param {type:\"boolean\"}\n",
        "output_dir = \"\" #@param {type:\"string\"}\n",
        "\n",
        "sample_rate = 44100 #@param {type:\"slider\", min:22050, max:44100, step:22050}\n",
        "\n",
        "#@markdown <small>Processing a long text in chunks will produce much better results.</small>\n",
        "chunk_at = \"auto\" #@param [\"never\", \"auto\", \"chunk_division\"]\n",
        "chunk_division = 2 #@param {type:\"slider\", min:0, max:10, step:1}\n",
        "\n",
        "#@markdown <small>Save an accompanying txt file with the audio file, containing your `text` input.</small>\n",
        "save_txt = False #@param {type:\"boolean\"}\n",
        "\n",
        "#@markdown <small>Guarantees even more eerie output.</small>\n",
        "eerie = False #@param {type:\"boolean\"}\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "if chunk_at == 'auto': chunk_at = 'commas_and_dots'\n",
        "if chunk_at == 'commas_and_dots' or chunk_at == 'dots':\n",
        "  chunk_division = 0\n",
        "\n",
        " \n",
        "clip_audio_to = 99999999\n",
        "\n",
        "clean_dirs([dir_source, dir_chops, dir_clips])\n",
        "uniq_id = gen_id()\n",
        "\n",
        " \n",
        "SAMPLE_RATE = sample_rate\n",
        "embedding = None\n",
        "\n",
        "audio_input = drive_root+speaker_voice_wav\n",
        "output_dir = fix_path(drive_root+output_dir)\n",
        "\n",
        "if os.path.isdir(audio_input):\n",
        "  concat_file = dir_tmp+uniq_id+'_concat.wav'\n",
        "  wavs = list_audio(audio_input)\n",
        "  wavs_str = concat_list('-v 1', wavs)\n",
        "  !sox {sox_q} {wavs_str} \"{concat_file}\"\n",
        "  audio_input = concat_file\n",
        "\n",
        "if chunk_division == 0:\n",
        "  chunk_division = 999999\n",
        "\n",
        "if normalize_speaker_volume == True:\n",
        "  normalized_file = dir_tmp+uniq_id+'_normalized.wav'\n",
        "  y, sr = librosa.load(audio_input, sr=sample_rate, mono=True)\n",
        "  y = normalize(y)\n",
        "  sf.write(normalized_file, y.T, sr)\n",
        "  audio_input = normalized_file\n",
        "\n",
        "#--\n",
        " \n",
        "\n",
        "\n",
        "if eerie == True:\n",
        "  sample_rate = 22050\n",
        "  librosa_sr = 44100\n",
        "  chunk_division = 4\n",
        "else:\n",
        "  librosa_sr = sample_rate\n",
        "\n",
        "global_sr = sample_rate\n",
        "\n",
        "# title = ''.join(text.split(' ')[:5]).lower()\n",
        "title = ''.join(e for e in text.split(' ')[:10] if e.isalnum()).lower()\n",
        "\n",
        "\n",
        "\n",
        "op(c.title, 'Run ID:', uniq_id, time=True)\n",
        "print()\n",
        "\n",
        "\n",
        "\n",
        "# audio_clip = dir_source+uniq_id+'.wav'\n",
        "# tmp_y, tmp_sr = librosa.load(audio_input, sr=None, mono=True)\n",
        "audio, sr = librosa.load(audio_input, sr=librosa_sr)\n",
        "duration = librosa.get_duration(audio, sr=sr)\n",
        "if duration > clip_audio_to:\n",
        "  # print('clip audio')\n",
        "  # !sox {audio_input} {audio_clip} trim 0 180 {shell_silence}\n",
        "  last_y = librosa_sr * clip_audio_to\n",
        "  if audio.ndim > 1:\n",
        "    audio = audio[:last_y, :last_y]\n",
        "  else:\n",
        "    audio = audio[:last_y]\n",
        "\n",
        "_compute_embedding(audio)\n",
        "\n",
        "words = text.split(' ')\n",
        "def divide_chunks(l, n):\n",
        "  for i in range(0, len(l), n): \n",
        "    yield l[i:i + n]\n",
        "\n",
        "if chunk_at == 'chunk_division':\n",
        "  chunks = [' '.join(e) for e in list(divide_chunks(words, chunk_division))]\n",
        "elif chunk_at == 'dots' or chunk_at == 'commas_and_dots':\n",
        "  chunks = text.split('.')\n",
        "  if chunk_at is 'commas_and_dots':\n",
        "    new_chunks = []\n",
        "    for s in chunks:\n",
        "      if ',' in s:\n",
        "        sides = s.split(',')\n",
        "        new_chunks.extend(sides)\n",
        "      else:\n",
        "        new_chunks.append(s)\n",
        "    chunks = new_chunks\n",
        "else:\n",
        "  chunks = [text]\n",
        "total_chunks = len(chunks)\n",
        " \n",
        "if embedding is None:\n",
        "  print(\"Error.\")\n",
        "else:\n",
        "  save_as = output_dir+uniq_id+'_'+title+'.wav' #path_leaf(audio_file)\n",
        "  if save_txt == True:\n",
        "    txtfile = save_as.replace('.wav', '.txt')\n",
        "    append_txt(txtfile, text)\n",
        "  op(c.title, 'Syntheisizing:', str(total_chunks)+' chunks', time=True)\n",
        "  print()\n",
        "\n",
        "  op(c.title, 'Voice source')\n",
        "  print()\n",
        "  audio_player(audio, sr=sample_rate)\n",
        "  print()\n",
        "\n",
        "  if len(chunks) is 1:\n",
        "    synthesize(embedding, chunks[0], save_as)\n",
        "    op(c.ok, 'Synthesized audio saved as', save_as.replace(drive_root, ''), time=True)\n",
        "    print()\n",
        "    audio_player(save_as)\n",
        "    # display(Audio(save_as))\n",
        "  else:\n",
        "    for i, txt in enumerate(chunks):\n",
        "      tit = str(i)+'/'+str(len(chunks))\n",
        "      print('____________________________________________________________________')\n",
        "      print()\n",
        "      op(c.title, tit+' Sythesizing:', txt, time=True)\n",
        "      save_chop = dir_chops+str(i).zfill(4)+'.wav'\n",
        "      synthesize(embedding, txt, save_chop)\n",
        "      \n",
        "      chops = list_audio(dir_chops)\n",
        "      for chop in chops:\n",
        "        chop_data, clip_sr = librosa.load(chop, sr=sample_rate)\n",
        "        chop_data = np.array([chop_data, chop_data])\n",
        "        duration = librosa.get_duration(chop_data, sr=sample_rate)\n",
        "        shortened_duration = duration-1.1\n",
        "        # clipped = clip_audio(chop_data, 0.01, shortened_duration)\n",
        "        clipped, idx = librosa.effects.trim(chop_data)\n",
        "        save_clip = dir_clips + path_leaf(chop)\n",
        "        sf.write(save_clip, clipped.T, sample_rate)\n",
        "\n",
        "    clips = list_audio(dir_clips)\n",
        "    clip_list = concat_list('-v 1', clips)\n",
        "    !sox {sox_q} {clip_list} \"{save_as}\"\n",
        "\n",
        "    output.clear()\n",
        "    op(c.title, 'Run ID:', uniq_id)\n",
        "    print()\n",
        "    op(c.title, 'Voice source')\n",
        "    print()\n",
        "    audio_player(audio, sr=sample_rate)\n",
        "    print()\n",
        "\n",
        "    if os.path.isfile(save_as):\n",
        "      op(c.title, 'Synthesized result')\n",
        "      print()\n",
        "      audio_player(save_as, sr=sample_rate)\n",
        "      print()\n",
        "      op(c.ok, 'Synthesized audio saved as', save_as.replace(drive_root, ''), time=True)\n",
        "    else:\n",
        "      op(c.fail, 'Error saving', save_as.replace(drive_root, ''), time=True)\n",
        "    # display(Audio(save_as))\n",
        "    \n",
        "    print('\\n\\n')\n",
        "\n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        "# for i, filelist in enumerate(filelists):\n",
        "#   filelist_len = len(filelist)\n",
        "#   filelist = concat_list('-v '+str(vol), filelist)\n",
        "#   if number_files == True:\n",
        "#     output_file_f = path_dir(output_file)+basename(output_file)+'-'+str(i+1)+'.wav'\n",
        "#   !sox {filelist} \"{output_file_f}\"\n",
        "#   print(filelist_len, 'files concatenated to', output_file_f)\n",
        "# print('Done.')\n",
        "# if preview is True:\n",
        "#   Audio(output_file)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "mount_file_id": "1qhRa791cJcNfT6nUnhTyxI4FwGXFf_iH",
      "authorship_tag": "ABX9TyPS3s/9Fuj3tDKq8dTZaoj+",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}