new_process_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 从train里面分出来valid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.将name加入到文档中"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xlrd\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_excel(\"labels.xlsx\")\n",
    "df_ref = pd.read_excel(\"reference.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = pd.merge(left=df_train, right=df_ref, how='inner', left_on='image_name', right_on='image_name')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 处理taskA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对overall_sentiment进行文字替换，negative为2，neutral为0，positive为1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def taskA(df):\n",
    "    df_copy=df.copy(deep=True)\n",
    "    i=0\n",
    "    while i < len(df):\n",
    "        if df_copy['overall_sentiment'][i]=='very_positive' or df_copy['overall_sentiment'][i]=='positive':\n",
    "            df['overall_sentiment'][i]=1\n",
    "        elif df_copy['overall_sentiment'][i]=='very_negative' or df_copy['overall_sentiment'][i]=='negative':\n",
    "            df['overall_sentiment'][i]=2\n",
    "        elif df_copy['overall_sentiment'][i]=='neutral':\n",
    "            df['overall_sentiment'][i]=0\n",
    "        else:\n",
    "            print(i)\n",
    "        i=i+1\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = taskA(df_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#选出taskA需要的列\n",
    "dict_df_total = {'label':df_total['overall_sentiment'], 'Text':df_total['text_corrected'], 'image_name': df_total['image_name'],'original_name': df_total['original_name']}\n",
    "#pd.DataFrame(dict_df_total).to_csv(\"taskA/Task_A_train.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "挑选出valid\n",
    "\n",
    "total中三者情况：\n",
    "\n",
    "0：2201 占0.315\n",
    "\n",
    "1：4160 占0.595\n",
    "\n",
    "2：631 占0.090 \n",
    "\n",
    "挑出1800：\n",
    "\n",
    "0：567\n",
    "\n",
    "1：1071\n",
    "\n",
    "2：162"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "total = np.array((pd.DataFrame(dict_df_total)).sample(frac=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 0:\n",
    "        df_0.append(total[i])\n",
    "        \n",
    "df_1 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 1:\n",
    "        df_1.append(total[i])\n",
    "        \n",
    "df_2 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 2:\n",
    "        df_2.append(total[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.DataFrame(df_1)\n",
    "df_2 = pd.DataFrame(df_2)\n",
    "df_0 = pd.DataFrame(df_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = ((df_0[:567]).append(df_1[:1071])).append(df_2[:162])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.sample(frac=1).to_csv(\"taskA/Task_A_trial.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = (((df_0[567:]).append(df_1[1071:])).append(df_2[162:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(frac=1).to_csv(\"taskA/Task_A_train.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 处理taskB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "taskB1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#not_funny为0，其余为1\n",
    "def taskB_1(df):\n",
    "    df_copy=df.copy(deep=True)\n",
    "    i=0\n",
    "    while i < len(df):\n",
    "        if df_copy['humour'][i]=='not_funny':\n",
    "            df['humour'][i]=0\n",
    "        elif df_copy['humour'][i]=='funny' or df_copy['humour'][i]=='very_funny'or df_copy['humour'][i]=='hilarious':\n",
    "            df['humour'][i]=1\n",
    "        else:\n",
    "            print(i)\n",
    "        i=i+1\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = taskB_1(df_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_df_total = {'label':df_total['humour'], 'Text':df_total['text_corrected'], 'image_name': df_total['image_name'],'original_name': df_total['original_name']}\n",
    "#pd.DataFrame(dict_df_total).to_csv(\"taskB/Task_B_train_1.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "挑出valid\n",
    "\n",
    "total中二者情况：\n",
    "\n",
    "0：1649 占0.24\n",
    "\n",
    "1：5339 占0.76\n",
    "\n",
    "抽取1800条valid\n",
    "\n",
    "0：432\n",
    "\n",
    "1：1368\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "total = np.array((pd.DataFrame(dict_df_total)).sample(frac=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 0:\n",
    "        df_0.append(total[i])\n",
    "        \n",
    "df_1 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 1:\n",
    "        df_1.append(total[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.DataFrame(df_1)\n",
    "df_0 = pd.DataFrame(df_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = (df_0[:432]).append(df_1[:1368])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.sample(frac=1).to_csv(\"taskB/Task_B_trial_1.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = (df_0[432:]).append(df_1[1368:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(frac=1).to_csv(\"taskB/Task_B_train_1.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "taskB2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#not_sarcastic为0，其余为1\n",
    "def taskB_2(df):\n",
    "    df_copy=df.copy(deep=True)\n",
    "    i=0\n",
    "    while i < len(df):\n",
    "        if df_copy['sarcasm'][i]=='not_sarcastic':\n",
    "            df['sarcasm'][i]=0\n",
    "        elif df_copy['sarcasm'][i]=='general' or df_copy['sarcasm'][i]=='twisted_meaning'or df_copy['sarcasm'][i]=='very_twisted':\n",
    "            df['sarcasm'][i]=1\n",
    "        else:\n",
    "            print(i)\n",
    "        i=i+1\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = taskB_2(df_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_df_total = {'label':df_total['sarcasm'], 'Text':df_total['text_corrected'], 'image_name': df_total['image_name'],'original_name': df_total['original_name']}\n",
    "pd.DataFrame(dict_df_total).to_csv(\"taskB/Task_B_train_2.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "挑出valid\n",
    "\n",
    "total中二者情况：\n",
    "\n",
    "0：1543 占0.22\n",
    "\n",
    "1：5445 占0.78\n",
    "\n",
    "抽取1800条valid\n",
    "\n",
    "0：396\n",
    "\n",
    "1：1404"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "total = np.array((pd.DataFrame(dict_df_total)).sample(frac=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 0:\n",
    "        df_0.append(total[i])\n",
    "        \n",
    "df_1 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 1:\n",
    "        df_1.append(total[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.DataFrame(df_1)\n",
    "df_0 = pd.DataFrame(df_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = (df_0[:396]).append(df_1[:1404])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.sample(frac=1).to_csv(\"taskB/Task_B_trial_2.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = (df_0[396:]).append(df_1[1404:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(frac=1).to_csv(\"taskB/Task_B_train_2.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "taskB3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#not_offensive为0，其余为1\n",
    "def taskB_3(df):\n",
    "    df_copy=df.copy(deep=True)\n",
    "    i=0\n",
    "    while i < len(df):\n",
    "        if df_copy['offensive'][i]=='not_offensive':\n",
    "            df['offensive'][i]=0\n",
    "        elif df_copy['offensive'][i]=='slight' or df_copy['offensive'][i]=='very_offensive'or df_copy['offensive'][i]=='hateful_offensive':\n",
    "            df['offensive'][i]=1\n",
    "        else:\n",
    "            print(i)\n",
    "        i=i+1\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = taskB_3(df_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_df_total = {'label':df_total['offensive'], 'Text':df_total['text_corrected'], 'image_name': df_total['image_name'],'original_name': df_total['original_name']}\n",
    "#pd.DataFrame(dict_df_total).to_csv(\"taskB/Task_B_train_3.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "挑出valid\n",
    "\n",
    "total中二者情况：\n",
    "\n",
    "0：2710 占0.39\n",
    "\n",
    "1：4278 占0.61\n",
    "\n",
    "抽取1800条valid\n",
    "\n",
    "0：702\n",
    "\n",
    "1：1098"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "total = np.array((pd.DataFrame(dict_df_total)).sample(frac=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 0:\n",
    "        df_0.append(total[i])\n",
    "        \n",
    "df_1 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 1:\n",
    "        df_1.append(total[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.DataFrame(df_1)\n",
    "df_0 = pd.DataFrame(df_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = (df_0[:702]).append(df_1[:1098])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.sample(frac=1).to_csv(\"taskB/Task_B_trial_3.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = (df_0[702:]).append(df_1[1098:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(frac=1).to_csv(\"taskB/Task_B_train_3.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "taskB4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "#not_motivational为0，其余为1\n",
    "def taskB_4(df):\n",
    "    df_copy=df.copy(deep=True)\n",
    "    i=0\n",
    "    while i < len(df):\n",
    "        if df_copy['motivational'][i]=='not_motivational':\n",
    "            df['motivational'][i]=0\n",
    "        elif df_copy['motivational'][i]=='motivational':\n",
    "            df['motivational'][i]=1\n",
    "        else:\n",
    "            print(i)\n",
    "        i=i+1\n",
    "    return df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = taskB_4(df_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_df_total = {'label':df_total['motivational'], 'Text':df_total['text_corrected'], 'image_name': df_total['image_name'],'original_name': df_total['original_name']}\n",
    "#pd.DataFrame(dict_df_total).to_csv(\"taskB/Task_B_train_4.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "挑出valid\n",
    "\n",
    "total中二者情况：\n",
    "\n",
    "0：4521 占0.65\n",
    "\n",
    "1：2467 占0.35\n",
    "\n",
    "抽取1800条valid\n",
    "\n",
    "0：1170\n",
    "\n",
    "1：630"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "total = np.array((pd.DataFrame(dict_df_total)).sample(frac=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 0:\n",
    "        df_0.append(total[i])\n",
    "        \n",
    "df_1 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 1:\n",
    "        df_1.append(total[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.DataFrame(df_1)\n",
    "df_0 = pd.DataFrame(df_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = (df_0[:1170]).append(df_1[:630])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.sample(frac=1).to_csv(\"taskB/Task_B_trial_4.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = (df_0[1170:]).append(df_1[630:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(frac=1).to_csv(\"taskB/Task_B_train_4.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# taskC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "taskc1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "#not_funny为0，funny为1，very_funny为2，hilarious为3\n",
    "def taskC_1(df):\n",
    "    df_copy=df.copy(deep=True)\n",
    "    i=0\n",
    "    while i < len(df):\n",
    "        if df_copy['humour'][i]=='not_funny':\n",
    "            df['humour'][i]=0\n",
    "        elif df_copy['humour'][i]=='funny':\n",
    "            df['humour'][i]=1\n",
    "        elif df_copy['humour'][i]=='very_funny':\n",
    "            df['humour'][i]=2\n",
    "        elif df_copy['humour'][i]=='hilarious':\n",
    "            df['humour'][i]=3        \n",
    "        i=i+1\n",
    "    dict_df = {'label':df_total['humour'], 'Text':df_total['text_corrected'], 'image_name': df_total['image_name'],'original_name': df_total['original_name']}\n",
    "    return pd.DataFrame(dict_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total  = taskC_1(df_total)\n",
    "#df_total.to_csv(\"temprory.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "挑出valid\n",
    "\n",
    "total中四者情况：\n",
    "\n",
    "0：1649 占0.24\n",
    "\n",
    "1：2451 占0.35\n",
    "\n",
    "2: 2237 占0.32\n",
    "\n",
    "3: 651 占0.09\n",
    "\n",
    "抽取1800条valid：\n",
    "\n",
    "0：432\n",
    "\n",
    "1：630\n",
    "\n",
    "2：576\n",
    "\n",
    "3：162"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "total = np.array(df_total.sample(frac=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 0:\n",
    "        df_0.append(total[i])\n",
    "        \n",
    "df_1 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 1:\n",
    "        df_1.append(total[i])\n",
    "        \n",
    "df_2 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 2:\n",
    "        df_2.append(total[i])\n",
    "        \n",
    "df_3 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 3:\n",
    "        df_3.append(total[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.DataFrame(df_1)\n",
    "df_0 = pd.DataFrame(df_0)\n",
    "df_2 = pd.DataFrame(df_2)\n",
    "df_3 = pd.DataFrame(df_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = (((df_0[:432]).append(df_1[:630])).append(df_2[:576])).append(df_3[:162])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.sample(frac=1).to_csv(\"taskC/Task_C_trial_1.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = (((df_0[432:]).append(df_1[630:])).append(df_2[576:])).append(df_3[162:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(frac=1).to_csv(\"taskC/Task_C_train_1.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "taskc2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Not Sarcastic => 0\n",
    "# General => 1\n",
    "# Twisted Meaning => 2\n",
    "# Very Twisted => 3\n",
    "def taskC_2(df):\n",
    "    df_copy=df.copy(deep=True)\n",
    "    i=0\n",
    "    while i < len(df):\n",
    "        if df_copy['sarcasm'][i]=='not_sarcastic':\n",
    "            df['sarcasm'][i]=0\n",
    "        elif df_copy['sarcasm'][i]=='general':\n",
    "            df['sarcasm'][i]=1\n",
    "        elif df_copy['sarcasm'][i]=='twisted_meaning':\n",
    "            df['sarcasm'][i]=2\n",
    "        elif df_copy['sarcasm'][i]=='very_twisted':\n",
    "            df['sarcasm'][i]=3    \n",
    "        i=i+1\n",
    "    dict_df = {'label':df_total['sarcasm'], 'Text':df_total['text_corrected'], 'image_name': df_total['image_name'],'original_name': df_total['original_name']}\n",
    "    return pd.DataFrame(dict_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total  = taskC_2(df_total)\n",
    "#df_total.to_csv(\"temprory.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "挑出valid\n",
    "\n",
    "total中四者情况：\n",
    "\n",
    "0：1543 占0.22\n",
    "\n",
    "1：3505 占0.50\n",
    "\n",
    "2: 1546 占0.22\n",
    "    \n",
    "3: 394 占0.06\n",
    "    \n",
    "抽取1800条valid：\n",
    "\n",
    "0：396\n",
    "\n",
    "1：900\n",
    "\n",
    "2：396\n",
    "\n",
    "3：108"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "total = np.array(df_total.sample(frac=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 0:\n",
    "        df_0.append(total[i])\n",
    "        \n",
    "df_1 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 1:\n",
    "        df_1.append(total[i])\n",
    "        \n",
    "df_2 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 2:\n",
    "        df_2.append(total[i])\n",
    "        \n",
    "df_3 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 3:\n",
    "        df_3.append(total[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.DataFrame(df_1)\n",
    "df_0 = pd.DataFrame(df_0)\n",
    "df_2 = pd.DataFrame(df_2)\n",
    "df_3 = pd.DataFrame(df_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = (((df_0[:396]).append(df_1[:900])).append(df_2[:396])).append(df_3[:108])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.sample(frac=1).to_csv(\"taskC/Task_C_trial_2.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = (((df_0[396:]).append(df_1[900:])).append(df_2[396:])).append(df_3[108:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(frac=1).to_csv(\"taskC/Task_C_train_2.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "taskc3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not offensive => 0\n",
    "# Slight => 1\n",
    "# Very Offensive => 2\n",
    "# Hateful Offensive => 3\n",
    "def taskC_3(df):\n",
    "    df_copy=df.copy(deep=True)\n",
    "    i=0\n",
    "    while i < len(df):\n",
    "        if df_copy['offensive'][i]=='not_offensive':\n",
    "            df['offensive'][i]=0\n",
    "        elif df_copy['offensive'][i]=='slight':\n",
    "            df['offensive'][i]=1\n",
    "        elif df_copy['offensive'][i]=='hateful_offensive':\n",
    "            df['offensive'][i]=3\n",
    "        elif df_copy['offensive'][i]=='very_offensive':\n",
    "            df['offensive'][i]=2    \n",
    "        i=i+1\n",
    "    dict_df = {'label':df_total['offensive'], 'Text':df_total['text_corrected'], 'image_name': df_total['image_name'],'original_name': df_total['original_name']}\n",
    "    return pd.DataFrame(dict_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total  = taskC_3(df_total)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "挑出valid\n",
    "\n",
    "total中四者情况：\n",
    "\n",
    "0：2710 占0.39\n",
    "\n",
    "1：2591 占0.37\n",
    "\n",
    "2: 1466 占0.21\n",
    "    \n",
    "3: 221 占0.03\n",
    "    \n",
    "抽取1800条valid：\n",
    "\n",
    "0：702\n",
    "\n",
    "1：666\n",
    "\n",
    "2：378\n",
    "\n",
    "3：54"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "total = np.array(df_total.sample(frac=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 0:\n",
    "        df_0.append(total[i])\n",
    "        \n",
    "df_1 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 1:\n",
    "        df_1.append(total[i])\n",
    "        \n",
    "df_2 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 2:\n",
    "        df_2.append(total[i])\n",
    "        \n",
    "df_3 = []\n",
    "for i in range(len(total)):\n",
    "    if total[i][0] == 3:\n",
    "        df_3.append(total[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.DataFrame(df_1)\n",
    "df_0 = pd.DataFrame(df_0)\n",
    "df_2 = pd.DataFrame(df_2)\n",
    "df_3 = pd.DataFrame(df_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = (((df_0[:702]).append(df_1[:666])).append(df_2[:378])).append(df_3[:54])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.sample(frac=1).to_csv(\"taskC/Task_C_trial_3.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = (((df_0[702:]).append(df_1[666:])).append(df_2[378:])).append(df_3[54:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(frac=1).to_csv(\"taskC/Task_C_train_3.csv\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}