final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "import codecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_path = '/openbayes/input/input1/BERT/cased_L-12_H-768_A-12/bert_config.json'\n",
    "checkpoint_path = '/openbayes/input/input1/BERT/cased_L-12_H-768_A-12/bert_model.ckpt'\n",
    "dict_path = '/openbayes/input/input1/BERT/cased_L-12_H-768_A-12/vocab.txt'\n",
    "\n",
    "maxlen = 200\n",
    "CLASS_NUM = 3\n",
    "batch_size = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seq_padding(X, padding=0):\n",
    "    L = [len(x) for x in X]\n",
    "    ML = max(L)\n",
    "    return np.array([\n",
    "        np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "from keras_bert import load_trained_model_from_checkpoint, Tokenizer\n",
    "\n",
    "token_dict = {}\n",
    "\n",
    "with codecs.open(dict_path, 'r', 'utf8') as reader:\n",
    "    for line in reader:\n",
    "        token = line.strip()\n",
    "        token_dict[token] = len(token_dict)\n",
    "\n",
    "\n",
    "class OurTokenizer(Tokenizer):\n",
    "    def _tokenize(self, text):\n",
    "        R = []\n",
    "        for c in text:\n",
    "            if c in self._token_dict:\n",
    "                R.append(c)\n",
    "            elif self._is_space(c):\n",
    "                R.append('[unused1]') # space类用未经训练的[unused1]表示\n",
    "            else:\n",
    "                R.append('[UNK]') # 剩余的字符是[UNK]\n",
    "        return R\n",
    "\n",
    "tokenizer = OurTokenizer(token_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text_path = r'/openbayes/input/input0/taskA/Task_A_train.csv'\n",
    "trial_text_path = r'/openbayes/input/input0/taskA/Task_A_trial.csv'\n",
    "# train_image_path = r'/openbayes/input/input0/train'\n",
    "# trial_image_path = r'/openbayes/input/input0/trial'\n",
    "train_text_data = pd.read_csv(train_text_path)\n",
    "train_text_data = np.array(train_text_data)\n",
    "train_text_data = train_text_data.tolist()\n",
    "\n",
    "trial_text_data = pd.read_csv(trial_text_path)\n",
    "trial_text_data = np.array(trial_text_data)\n",
    "trial_text_data = trial_text_data.tolist()\n",
    "\n",
    "image_path = r'/openbayes/home/img'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class data_generator:\n",
    "    def __init__(self, data, batch_size=16):\n",
    "        self.data = data\n",
    "        self.batch_size = batch_size\n",
    "        self.steps = len(self.data) // self.batch_size\n",
    "        if len(self.data) % self.batch_size != 0:\n",
    "            self.steps += 1\n",
    "    def __len__(self):\n",
    "        return self.steps\n",
    "    def __iter__(self):\n",
    "        while True:\n",
    "            idxs = list(range(len(self.data)))\n",
    "            np.random.shuffle(idxs)\n",
    "            X1, X2, X3, Y = [], [], [], []\n",
    "            for i in idxs:\n",
    "                d = self.data[i]\n",
    "                text = str(d[1])[:maxlen]\n",
    "                x1, x2 = tokenizer.encode(first=text)\n",
    "                imagePath = os.path.join(image_path,d[2])\n",
    "                img = image.load_img(imagePath, target_size=(224, 224))\n",
    "                x3 = image.img_to_array(img)\n",
    "                y = d[0]\n",
    "                X1.append(x1)\n",
    "                X2.append(x2)\n",
    "                X3.append(x3)\n",
    "                Y.append([y])               \n",
    "                if len(X1) == self.batch_size or i == idxs[-1]:\n",
    "                    X1 = seq_padding(X1)\n",
    "                    X2 = seq_padding(X2)\n",
    "                    X3 = np.array(X3, dtype=\"float\") / 255.0\n",
    "                    Y = seq_padding(Y)\n",
    "#                     Y = np.array(Y)\n",
    "#                     Y = to_categorical(Y, num_classes=CLASS_NUM)\n",
    "                    yield [X1, X2, X3], Y\n",
    "                    [X1, X2, X3, Y] = [], [], [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_D = data_generator(train_text_data)\n",
    "valid_D = data_generator(trial_text_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import random\n",
    "import numpy as np\n",
    "import cv2\n",
    "import os\n",
    "\n",
    "from keras.applications.resnet50 import ResNet50\n",
    "from keras.preprocessing import image\n",
    "from keras.applications.resnet50 import preprocess_input, decode_predictions\n",
    "from imutils import paths\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "#如果没有这两行代码，可能会报错OSError: image file is truncated\n",
    "from PIL import ImageFile\n",
    "ImageFile.LOAD_TRUNCATED_IMAGES = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "norm_size = 32\n",
    "CLASS_NUM = 3\n",
    "EPOCHS = 35\n",
    "INIT_LR = 1e-3\n",
    "BS = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import *\n",
    "bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None)\n",
    "\n",
    "for l in bert_model.layers:\n",
    "    l.trainable = True\n",
    "\n",
    "bert_x1_in = Input(shape=(None,))\n",
    "bert_x2_in = Input(shape=(None,))\n",
    "\n",
    "x = bert_model([bert_x1_in, bert_x2_in])\n",
    "text_output = Lambda(lambda x: x[:, 0])(x) # 取出[CLS]对应的向量用来做分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#图像模型部分\n",
    "model = ResNet50()\n",
    "#加一个dense层吧\n",
    "img_in = Input(shape=(None,None,None,))\n",
    "x = model(inputs = [img_in])\n",
    "img_output = Dense(10)(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "\n",
    "#将文字处理部分与图像处理部分连接起来\n",
    "x = keras.layers.concatenate([img_output, text_output])\n",
    "\n",
    "# 堆叠多个全连接网络层\n",
    "# x = Dense(64, activation='relu')(x)\n",
    "# x = Dense(64, activation='relu')(x)\n",
    "# x = Dense(64, activation='relu')(x)\n",
    "\n",
    "\n",
    "#顶层套一个全连接层\n",
    "p = Dense(CLASS_NUM, activation='softmax')(x)\n",
    "\n",
    "#重新定义一个模型，BERT本身有两个输入\n",
    "model = Model([bert_x1_in, bert_x2_in, img_in], p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_5\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_4 (InputLayer)            (None, None, None, N 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "resnet50 (Model)                (None, 1000)         25636712    input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model_4 (Model)                 (None, None, 768)    107719680   input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           10010       resnet50[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 768)          0           model_4[1][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 778)          0           dense_1[0][0]                    \n",
      "                                                                 lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 3)            2337        concatenate_1[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 133,368,739\n",
      "Trainable params: 133,315,619\n",
      "Non-trainable params: 53,120\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.optimizers import Adam\n",
    "\n",
    "model.compile(\n",
    "    optimizer=Adam(1e-5),\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    metrics=['sparse_categorical_accuracy'],\n",
    "             )\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "429/429 [==============================] - 369s 861ms/step - loss: 0.8933 - sparse_categorical_accuracy: 0.5934 - val_loss: 0.6652 - val_sparse_categorical_accuracy: 0.6204\n",
      "Epoch 2/5\n",
      "323/429 [=====================>........] - ETA: 1:26 - loss: 0.8768 - sparse_categorical_accuracy: 0.6002"
     ]
    }
   ],
   "source": [
    "model.fit_generator(\n",
    "    train_D.__iter__(),\n",
    "    steps_per_epoch=len(train_D),\n",
    "    epochs=5,\n",
    "    validation_data=valid_D.__iter__(),\n",
    "    validation_steps=len(valid_D)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}