MultimodalDeepLearning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.layers import Input, Embedding, LSTM, Dense, Dropout\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1031 16:20:22.984114 140611263473472 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "W1031 16:20:22.994876 140611263473472 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "img_maxlen = 100\n",
    "img_input = Input(shape=(img_maxlen,))\n",
    "x1 = Dense(1000, activation='relu')(img_input)\n",
    "x1 = Dropout(0.5)(x1)\n",
    "x1 = Dense(600, activation='relu')(x1)\n",
    "img_output = Dense(10)(x1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_maxlen = 100\n",
    "text_input = Input(shape=(text_maxlen,),dtype='int32')\n",
    "x2 = Embedding(output_dim=512, input_dim=10000, input_length=text_maxlen)(text_input)\n",
    "x2 = LSTM(32)(x2)\n",
    "x2 = Dropout(0.5)(x2)\n",
    "text_output = Dense(10)(x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1031 16:21:17.584550 140611263473472 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4185: The name tf.truncated_normal is deprecated. Please use tf.random.truncated_normal instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#用BERT来做文本处理\n",
    "from keras_bert import load_trained_model_from_checkpoint, Tokenizer\n",
    "from keras.layers import *\n",
    "\n",
    "config_path = '/openbayes/input/input0/BERT/cased_L-12_H-768_A-12/bert_config.json'\n",
    "checkpoint_path = '/openbayes/input/input0/BERT/cased_L-12_H-768_A-12/bert_model.ckpt'\n",
    "dict_path = '/openbayes/input/input0/BERT/cased_L-12_H-768_A-12/vocab.txt'\n",
    "\n",
    "bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None)\n",
    "\n",
    "for l in bert_model.layers:\n",
    "    l.trainable = True\n",
    "\n",
    "x1_in = Input(shape=(None,))\n",
    "x2_in = Input(shape=(None,))\n",
    "\n",
    "x = bert_model([x1_in, x2_in])\n",
    "x = Lambda(lambda x: x[:, 0])(x) # 取出[CLS]对应的向量用来做分类\n",
    "\n",
    "x = keras.layers.concatenate([img_output, x])\n",
    "\n",
    "p = Dense(3, activation='sigmoid')(x)\n",
    "\n",
    "#重新定义一个具有BERT的模型，BERT本身有两个输入\n",
    "model = Model([x1_in, x2_in, img_input], p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LSTM版本\n",
    "x = keras.layers.concatenate([img_output, text_output])\n",
    "\n",
    "# 堆叠多个全连接网络层\n",
    "x = Dense(64, activation='relu')(x)\n",
    "x = Dense(64, activation='relu')(x)\n",
    "x = Dense(64, activation='relu')(x)\n",
    "\n",
    "# 最后添加主要的逻辑回归层\n",
    "output = Dense(3, activation='softmax', name='output')(x)\n",
    "\n",
    "#定义一个具有两个输入和一个输出的模型：\n",
    "model = Model(inputs=[img_input, text_input], outputs=[output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1031 16:21:54.668922 140611263473472 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from keras.optimizers import Adam\n",
    "\n",
    "model.compile(\n",
    "    optimizer=Adam(1e-5),\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    metrics=['sparse_categorical_accuracy'],\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#文本数据处理\n",
    "import codecs\n",
    "import pandas as pd\n",
    "\n",
    "token_dict = {}\n",
    "\n",
    "with codecs.open(dict_path, 'r', 'utf8') as reader:\n",
    "    for line in reader:\n",
    "        token = line.strip()\n",
    "        token_dict[token] = len(token_dict)\n",
    "\n",
    "\n",
    "class OurTokenizer(Tokenizer):\n",
    "    def _tokenize(self, text):\n",
    "        R = []\n",
    "        for c in text:\n",
    "            if c in self._token_dict:\n",
    "                R.append(c)\n",
    "            elif self._is_space(c):\n",
    "                R.append('[unused1]') # space类用未经训练的[unused1]表示\n",
    "            else:\n",
    "                R.append('[UNK]') # 剩余的字符是[UNK]\n",
    "        return R\n",
    "\n",
    "tokenizer = OurTokenizer(token_dict)\n",
    "\n",
    "neg = pd.read_excel('/openbayes/input/input1/neg.xls', header=None)\n",
    "pos = pd.read_excel('/openbayes/input/input1/pos.xls', header=None)\n",
    "neu = pd.read_excel('/openbayes/input/input1/neu.xls', header=None)\n",
    "\n",
    "data = []\n",
    "\n",
    "for d in neg[0]:\n",
    "    data.append((d, 0))\n",
    "\n",
    "for d in pos[0]:\n",
    "    data.append((d, 1))\n",
    "    \n",
    "for d in neu[0]:\n",
    "    data.append((d, 2))\n",
    "\n",
    "\n",
    "# 按照9:1的比例划分训练集和验证集\n",
    "random_order = list(range(len(data)))\n",
    "np.random.shuffle(random_order)\n",
    "train_data = [data[j] for i, j in enumerate(random_order) if i % 10 != 0]\n",
    "valid_data = [data[j] for i, j in enumerate(random_order) if i % 10 == 0]\n",
    "\n",
    "\n",
    "def seq_padding(X, padding=0):\n",
    "    L = [len(x) for x in X]\n",
    "    ML = max(L)\n",
    "    return np.array([\n",
    "        np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X\n",
    "    ])\n",
    "\n",
    "\n",
    "class data_generator:\n",
    "    def __init__(self, data, batch_size=32):\n",
    "        self.data = data\n",
    "        self.batch_size = batch_size\n",
    "        self.steps = len(self.data) // self.batch_size\n",
    "        if len(self.data) % self.batch_size != 0:\n",
    "            self.steps += 1\n",
    "    def __len__(self):\n",
    "        return self.steps\n",
    "    def __iter__(self):\n",
    "        while True:\n",
    "            idxs = list(range(len(self.data)))\n",
    "            np.random.shuffle(idxs)\n",
    "            X1, X2, Y = [], [], []\n",
    "            for i in idxs:\n",
    "                d = self.data[i]\n",
    "                text = d[0][:maxlen]\n",
    "                x1, x2 = tokenizer.encode(first=text)\n",
    "                y = d[1]\n",
    "                X1.append(x1)\n",
    "                X2.append(x2)\n",
    "                Y.append([y])\n",
    "                if len(X1) == self.batch_size or i == idxs[-1]:\n",
    "                    X1 = seq_padding(X1)\n",
    "                    X2 = seq_padding(X2)\n",
    "                    Y = seq_padding(Y)\n",
    "                    yield [X1, X2], Y\n",
    "                    [X1, X2, Y] = [], [], []\n",
    "\n",
    "                    \n",
    "train_D = data_generator(train_data)\n",
    "valid_D = data_generator(valid_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}