prepare_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76e75727-b2dc-46bd-a017-fe5daa70fe2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://commonvoice.mozilla.org/en/datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2d8fb038-eafd-4cf4-9be7-65581c75e384",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import IPython\n",
    "\n",
    "import argparse\n",
    "import multiprocessing as mp\n",
    "import re\n",
    "from functools import partial\n",
    "from typing import Dict, List\n",
    "\n",
    "import pandas as pd\n",
    "from pythainlp.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e8c7055-a262-42e5-b578-f6a90e18bb3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load common voice validated data\n",
    "validated = pd.read_csv(\"/mnt/d/data/cv-corpus-15.0-2023-09-08/th/validated.tsv\", delimiter=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "04908296-adec-4846-868f-e8484b45de93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter only no down vote\n",
    "validated = validated[validated[\"down_votes\"] == 0].sort_values(\"client_id\").reset_index(drop=True)[[\"client_id\", \"path\", \"sentence\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bae7c27f-0b95-4f8c-9f59-8ef14b08d59e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_repeat(text: str) -> str:\n",
    "    \"\"\"Format Thai sentence containing ๆ\n",
    "    \n",
    "    Arguments\n",
    "    ---------\n",
    "    text: str\n",
    "        text to be processed\n",
    "\n",
    "    Return\n",
    "    ------\n",
    "    formatted_text: str\n",
    "        formatted text that got repeated by ๆ\n",
    "    \"\"\"\n",
    "    # check whether sentence start with ๆ\n",
    "    text = text.replace(\" \", \"\");\n",
    "    if text[0] == \"ๆ\":\n",
    "        raise ValueError(f\"ๆ must not be at the start of sentence: {text}\");\n",
    "\n",
    "    tokenized_text: List[str] = word_tokenize(text);\n",
    "    formatted_text: List[str] = [];\n",
    "\n",
    "    for i, word in enumerate(tokenized_text):\n",
    "        if \"ๆ\" in word:\n",
    "            splitted_word: List[str] = [x for x in re.split(\"(ๆ)\", word) if x != \"\"];\n",
    "            if splitted_word[0] == \"ๆ\":\n",
    "                # if splitted words are all ๆ\n",
    "                last_word: str = tokenized_text[i-1];\n",
    "                for c in word:\n",
    "                    if c == \"ๆ\":\n",
    "                        formatted_text.append(last_word);\n",
    "                formatted_text.append(word.replace(\"ๆ\", \"\"));\n",
    "            else:\n",
    "                current_word = splitted_word[0];\n",
    "                for w in splitted_word:\n",
    "                    if w != \"ๆ\":\n",
    "                        current_word: str = w;\n",
    "                        formatted_text.append(w);\n",
    "                    else:\n",
    "                        formatted_text.append(current_word);\n",
    "        else:\n",
    "            formatted_text.append(word)\n",
    "    return \"\".join(formatted_text);\n",
    "\n",
    "\n",
    "def correct_sentence(sentence: str, custom_dict: Dict[str, str] = {}) -> None:\n",
    "    \"\"\"Correct misspell sentence according to the following rule\n",
    "    1. check whether แ is spelled by เ + เ\n",
    "    2. check whether ำ is spelled by  ํ + า\n",
    "    3. check whether tonal mark ( ่,  ้,  ๊,  ๋ ) is followed after vowel ( ั, ำ, ุ, ู )\n",
    "    and save it in output file\n",
    "\n",
    "    Arguments\n",
    "    ---------\n",
    "    sentence: str\n",
    "        Sentence to be corrected\n",
    "    \"\"\"\n",
    "    tonal_marks: List[str] = [\"่\", \"้\", \"๊\", \"๋\"];\n",
    "    vowel: List[str] = [\"ั\", \"ุ\", \"ู\", \"ํ\"];\n",
    "\n",
    "    # replace custom dict\n",
    "    for word, replace_word in custom_dict.items():\n",
    "        if word in sentence:\n",
    "            sentence = sentence.replace(word, replace_word);\n",
    "            # print(f\"CUSTOM DICT: Replace `{word}` => `{replace_word}`\");\n",
    "\n",
    "    if \"เเ\" in sentence:\n",
    "        sentence = sentence.replace(\"เเ\", \"แ\");  # correct เ + เ -> แ\n",
    "        # print(f\"Correct เ + เ => แ\");\n",
    "    if \"ํา\" in sentence:\n",
    "        sentence = sentence.replace(\"ํา\", \"ำ\");  # correct ํ + า -> ำ\n",
    "        # print(f\"Correct ํ + า => ำ\");\n",
    "    if \"ๆ\" in sentence:\n",
    "        sentence = format_repeat(sentence);\n",
    "        # print(\"ๆ Replaced\")\n",
    "    # correct #3\n",
    "    corrected_sentence: str = sentence;\n",
    "    for i in range(len(sentence) - 1):\n",
    "        char: str = sentence[i];\n",
    "        next_char: str = sentence[i+1];\n",
    "        if char in tonal_marks and next_char in vowel:\n",
    "                corrected_sentence: List[str] = list(corrected_sentence);\n",
    "                corrected_sentence[i] = next_char;\n",
    "                corrected_sentence[i+1] = char;\n",
    "                corrected_sentence: str = str(corrected_sentence);\n",
    "                # print(f\"Corrected `{char}` + `{next_char}` => `{next_char}` + `{char}`\");\n",
    "        if char == \"ํ\" and next_char in tonal_marks and sentence[i+2] == \"า\":\n",
    "            corrected_sentence = corrected_sentence.replace(f\"ํ{next_char}า\", f\"{next_char}ำ\");\n",
    "            # print(f\"Corrected `ํ` + `{next_char}` + `า` => `{next_char}` + `ำ`\");\n",
    "    \n",
    "    return corrected_sentence\n",
    "\n",
    "def remove_special_char(text: str) -> str:\n",
    "    text = text.replace(\"!\", \"\")\n",
    "    text = text.replace('\"', \"\")\n",
    "    text = text.replace(\"'\", \"\")\n",
    "    text = text.replace(',', \"\")\n",
    "    text = text.replace('-', \"\")\n",
    "    text = text.replace('.', \"\")\n",
    "    text = text.replace(':', \"\")\n",
    "    text = text.replace(';', \"\")\n",
    "    text = text.replace('?', \"\")\n",
    "    text = text.replace('[', \"\")\n",
    "    text = text.replace(']', \"\")\n",
    "    text = text.replace(':', \"\")\n",
    "    text = text.replace('_', \"\")\n",
    "    text = text.replace('|', \"\")\n",
    "    text = text.replace('~', \"\")\n",
    "    text = text.replace('—', \"\")\n",
    "    text = text.replace('‘', \"\")\n",
    "    text = text.replace('’', \"\")\n",
    "    text = text.replace('“', \"\")\n",
    "    text = text.replace('”', \"\")\n",
    "    text = text.replace('ฯ', \"\")\n",
    "    return text.lower()\n",
    "    \n",
    "    \n",
    "def get_char(texts: List[str]) -> List[str]:\n",
    "    \"\"\"Get unique char from list of documents\"\"\"\n",
    "    return sorted(set([char for sent in texts for char in sent]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a1414a5b-3796-4abc-b0a5-58195b118da6",
   "metadata": {},
   "outputs": [],
   "source": [
    "validated[\"sentence\"] = validated[\"sentence\"].map(correct_sentence)\n",
    "validated[\"sentence\"] = validated[\"sentence\"].map(format_repeat)\n",
    "validated[\"sentence\"] = validated[\"sentence\"].map(remove_special_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2995f8ef-8e60-4c4e-8e5b-99561d9d96cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print unique character\n",
    "get_char(validated[\"sentence\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "fb839c13-0605-4984-85e7-246a0c6e512d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>client_id</th>\n",
       "      <th>path</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00086f5dc46f9038f13bbd829c4118fab3ac28688d5ee8...</td>\n",
       "      <td>common_voice_th_25695281.mp3</td>\n",
       "      <td>ใครเป็นผู้รับ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>001373e40d26bbdd9c2416bd189496a707950947125144...</td>\n",
       "      <td>common_voice_th_28444375.mp3</td>\n",
       "      <td>พวกเราต้องทำให้แน่ใจว่ามันจะไม่เกิดขึ้นอีก</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001373e40d26bbdd9c2416bd189496a707950947125144...</td>\n",
       "      <td>common_voice_th_28444373.mp3</td>\n",
       "      <td>ทางฝ่ายพระเจ้ากรุงจีน</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001373e40d26bbdd9c2416bd189496a707950947125144...</td>\n",
       "      <td>common_voice_th_28443814.mp3</td>\n",
       "      <td>ที่ข้าพเจ้าได้เคยล่วงเกินท่านไว้</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001373e40d26bbdd9c2416bd189496a707950947125144...</td>\n",
       "      <td>common_voice_th_28443793.mp3</td>\n",
       "      <td>เสื้อกันลมทำให้ตัวเขาแห้งสนิทถึงแม้มันจะเปียกฝน</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129513</th>\n",
       "      <td>fffc52acec32b85f7fed484ce96120c469628b46f7d553...</td>\n",
       "      <td>common_voice_th_25678469.mp3</td>\n",
       "      <td>ฉันตื่นเต้นที่กำลังจะได้เริ่มงานใหม่</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129514</th>\n",
       "      <td>fffc52acec32b85f7fed484ce96120c469628b46f7d553...</td>\n",
       "      <td>common_voice_th_25678473.mp3</td>\n",
       "      <td>สำหรับฉันแล้วดูเหมือนว่าอุตสาหกรรมของเขาผิดไปห...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129515</th>\n",
       "      <td>fffc52acec32b85f7fed484ce96120c469628b46f7d553...</td>\n",
       "      <td>common_voice_th_25678477.mp3</td>\n",
       "      <td>ในขณะเดียวกันหนังสือเล่มนี้ได้บังคับให้เขาหวนร...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129516</th>\n",
       "      <td>fffc52acec32b85f7fed484ce96120c469628b46f7d553...</td>\n",
       "      <td>common_voice_th_25678087.mp3</td>\n",
       "      <td>ในวิดีโอมีชายคนหนึ่งหยุดให้แมวข้ามถนน</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129517</th>\n",
       "      <td>fffc52acec32b85f7fed484ce96120c469628b46f7d553...</td>\n",
       "      <td>common_voice_th_25678081.mp3</td>\n",
       "      <td>พายุในถ้วยน้ำชา</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>129518 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                client_id  \\\n",
       "0       00086f5dc46f9038f13bbd829c4118fab3ac28688d5ee8...   \n",
       "1       001373e40d26bbdd9c2416bd189496a707950947125144...   \n",
       "2       001373e40d26bbdd9c2416bd189496a707950947125144...   \n",
       "3       001373e40d26bbdd9c2416bd189496a707950947125144...   \n",
       "4       001373e40d26bbdd9c2416bd189496a707950947125144...   \n",
       "...                                                   ...   \n",
       "129513  fffc52acec32b85f7fed484ce96120c469628b46f7d553...   \n",
       "129514  fffc52acec32b85f7fed484ce96120c469628b46f7d553...   \n",
       "129515  fffc52acec32b85f7fed484ce96120c469628b46f7d553...   \n",
       "129516  fffc52acec32b85f7fed484ce96120c469628b46f7d553...   \n",
       "129517  fffc52acec32b85f7fed484ce96120c469628b46f7d553...   \n",
       "\n",
       "                                path  \\\n",
       "0       common_voice_th_25695281.mp3   \n",
       "1       common_voice_th_28444375.mp3   \n",
       "2       common_voice_th_28444373.mp3   \n",
       "3       common_voice_th_28443814.mp3   \n",
       "4       common_voice_th_28443793.mp3   \n",
       "...                              ...   \n",
       "129513  common_voice_th_25678469.mp3   \n",
       "129514  common_voice_th_25678473.mp3   \n",
       "129515  common_voice_th_25678477.mp3   \n",
       "129516  common_voice_th_25678087.mp3   \n",
       "129517  common_voice_th_25678081.mp3   \n",
       "\n",
       "                                                 sentence  \n",
       "0                                           ใครเป็นผู้รับ  \n",
       "1              พวกเราต้องทำให้แน่ใจว่ามันจะไม่เกิดขึ้นอีก  \n",
       "2                                   ทางฝ่ายพระเจ้ากรุงจีน  \n",
       "3                        ที่ข้าพเจ้าได้เคยล่วงเกินท่านไว้  \n",
       "4         เสื้อกันลมทำให้ตัวเขาแห้งสนิทถึงแม้มันจะเปียกฝน  \n",
       "...                                                   ...  \n",
       "129513               ฉันตื่นเต้นที่กำลังจะได้เริ่มงานใหม่  \n",
       "129514  สำหรับฉันแล้วดูเหมือนว่าอุตสาหกรรมของเขาผิดไปห...  \n",
       "129515  ในขณะเดียวกันหนังสือเล่มนี้ได้บังคับให้เขาหวนร...  \n",
       "129516              ในวิดีโอมีชายคนหนึ่งหยุดให้แมวข้ามถนน  \n",
       "129517                                    พายุในถ้วยน้ำชา  \n",
       "\n",
       "[129518 rows x 3 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dca310e2-86d0-4262-9c22-5f3da57e39ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def contains_letters(text):\n",
    "    pattern = re.compile('[a-zA-Z]')\n",
    "    return bool(re.search(pattern, text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "acc0ec1c-7db7-4921-bfc5-7b1903fd1875",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 129518/129518 [04:17<00:00, 503.69it/s]\n"
     ]
    }
   ],
   "source": [
    "# split train test validate\n",
    "train_ratio = 0.9\n",
    "dev_ratio = 0.03\n",
    "test_ratio = 0.07\n",
    "\n",
    "train_row = int(train_ratio*validated.shape[0])\n",
    "dev_row = int(dev_ratio*validated.shape[0])\n",
    "test_row = int(test_ratio*validated.shape[0])\n",
    "\n",
    "train_client_set = set()\n",
    "dev_client_set = set()\n",
    "test_client_set = set()\n",
    "\n",
    "train_data = pd.DataFrame([], columns=[\"path\", \"sentence\"])\n",
    "dev_data = pd.DataFrame([], columns=[\"path\", \"sentence\"])\n",
    "test_data = pd.DataFrame([], columns=[\"path\", \"sentence\"])\n",
    "\n",
    "for idx, row in tqdm(validated.iterrows(), total=validated.shape[0]):\n",
    "    client_id = row[\"client_id\"]\n",
    "    path = row[\"path\"]\n",
    "    sentence = row[\"sentence\"]\n",
    "\n",
    "    # skip english sentence\n",
    "    if contains_letters(sentence):\n",
    "        continue\n",
    "\n",
    "    # filter some wrong sentence\n",
    "    if sentence == \"เขาหยุดแล้วคว้ามือออกมาเฉียดตัวฉันไปเส้นยาแเงผ่าแปด\":\n",
    "        sentence = \"เขาหยุดแล้วคว้ามือออกมาเฉียดตัวฉันไปเส้นยาแดงผ่าแปด\"\n",
    "\n",
    "    # we use leave one out method for evaluate, so the test and validate dataset won't have the same person id as train dataset\n",
    "    if client_id in train_client_set:\n",
    "        train_data = pd.concat([train_data, pd.DataFrame([[path, sentence]], columns=[\"path\", \"sentence\"])])\n",
    "    elif client_id in dev_client_set:\n",
    "        dev_data = pd.concat([dev_data, pd.DataFrame([[path, sentence]], columns=[\"path\", \"sentence\"])])\n",
    "    elif client_id in test_client_set:\n",
    "        test_data = pd.concat([test_data, pd.DataFrame([[path, sentence]], columns=[\"path\", \"sentence\"])])\n",
    "    elif train_data.shape[0] < train_row:\n",
    "        train_data = pd.concat([train_data, pd.DataFrame([[path, sentence]], columns=[\"path\", \"sentence\"])])\n",
    "        train_client_set.add(client_id)\n",
    "    elif dev_data.shape[0] < dev_row:\n",
    "        dev_data = pd.concat([dev_data, pd.DataFrame([[path, sentence]], columns=[\"path\", \"sentence\"])])\n",
    "        dev_client_set.add(client_id)\n",
    "    elif test_data.shape[0] < test_row:\n",
    "        test_data = pd.concat([test_data, pd.DataFrame([[path, sentence]], columns=[\"path\", \"sentence\"])])\n",
    "        test_client_set.add(client_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7db6edcd-3096-4a42-b3da-ae72eb54d31f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export data\n",
    "train_data.to_csv(\"cv15_train.csv\", index=False)\n",
    "dev_data.to_csv(\"cv15_dev.csv\", index=False)\n",
    "test_data.to_csv(\"cv15_test.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}