Generate_HDFS_Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1><center>Reduce HDFS</center></h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read original data (.csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdfs_csv = pd.read_csv('../Anomaly-Detection/Models/loglizer/data/HDFS/hdfs_structured.csv')\n",
    "\n",
    "events = set(hdfs_csv['EventId'])\n",
    "events_mapping = dict(zip(events, range(len(events))))\n",
    "\n",
    "hdfs_csv['EventId'] = hdfs_csv['EventId'].apply(lambda x : events_mapping.get(x))\n",
    "hdfs_grouped = hdfs_csv.groupby('blk_id').agg(lambda x : list(x))\n",
    "\n",
    "labels = pd.read_csv('../Anomaly-Detection/Data/HDFS_Drain_Parsed/anomaly_label.csv')\n",
    "labels['Label'] = labels['Label'].apply(lambda x : 0 if x=='Normal' else 1)\n",
    "block_labels = dict(labels.values)\n",
    "\n",
    "x = hdfs_grouped['EventId'].values\n",
    "y = list(map(lambda x : block_labels.get(x),list(hdfs_grouped.index)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez('HDFS.npz', x_data=x, y_data=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Work on HDFS.npz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load data\n",
    "hdfs = np.load('HDFS.npz', allow_pickle=True)\n",
    "\n",
    "#Split into anomalies and normal points\n",
    "anomalies = hdfs['x_data'][np.argwhere(hdfs['y_data'] == 1).flatten()]\n",
    "normal = hdfs['x_data'][np.argwhere(hdfs['y_data'] == 0).flatten()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split data train test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_ratios = [0.5, 0.75, 0.9]\n",
    "for train_ratio in train_ratios:\n",
    "\n",
    "    normal_train, normal_test = train_test_split(normal, train_size=train_ratio)\n",
    "    anomalies_train, anomalies_test = train_test_split(anomalies, train_size=train_ratio)\n",
    "\n",
    "    #Train part\n",
    "    x = np.concatenate((normal_train, anomalies_train))\n",
    "    \n",
    "    zer = np.zeros(len(normal_train))\n",
    "    ones = np.ones(len(anomalies_train))\n",
    "    \n",
    "    y = np.concatenate((zer, ones))\n",
    "    np.savez('HDFS_train_'+str(train_ratio), x_data=x, y_data=y)\n",
    "\n",
    "    reduction_percentages = [0.0, 0.25, 0.50, 0.75, 0.99, 0.999]\n",
    "    for reduct_percentage in reduction_percentages:\n",
    "        #Test part\n",
    "        num_abnormal = len(anomalies_test)\n",
    "        num_reduced = int(num_abnormal*(1-reduct_percentage))\n",
    "\n",
    "        abnormal = np.random.choice(anomalies_test, num_reduced)\n",
    "\n",
    "        x = np.concatenate((normal_test, abnormal))\n",
    "        zer = np.zeros(len(normal_test))\n",
    "        ones = np.ones(len(abnormal))\n",
    "        y = np.concatenate((zer, ones))\n",
    "\n",
    "        np.savez('HDFS_test_'+ str(train_ratio) + '_'+ str(reduct_percentage), x_data=x, y_data=y)\n",
    "        \n",
    "for train_ratio in train_ratios:\n",
    "    train_data = np.load('HDFS_train_'+str(train_ratio) + '.npz', allow_pickle=True)\n",
    "    \n",
    "    anomalies = train_data['x_data'][np.argwhere(train_data['y_data'] == 1).flatten()]\n",
    "    normal = train_data['x_data'][np.argwhere(train_data['y_data'] == 0).flatten()]\n",
    "    \n",
    "    for reduct_percentage in reduction_percentages:\n",
    "        num_abnormal = len(anomalies)\n",
    "        num_reduced = int(num_abnormal*(1-reduct_percentage))\n",
    "\n",
    "        abnormal = np.random.choice(anomalies, num_reduced)\n",
    "\n",
    "        x = np.concatenate((normal, abnormal))\n",
    "        \n",
    "        zer = np.zeros(len(normal))\n",
    "        ones = np.ones(len(abnormal))\n",
    "        \n",
    "        y = np.concatenate((zer, ones))\n",
    "\n",
    "        np.savez('HDFS_train_'+ str(train_ratio) + '_'+ str(reduct_percentage), x_data=x, y_data=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil, os \n",
    "\n",
    "hdfs_files = [x for x in os.listdir() if ('HDFS' in x) and (len(x) > 20) and ('.ipynb' not in x) and (x not in ['HDFS.npz', 'HDFS_Data'])] \n",
    "for file in hdfs_files:\n",
    "    print(file)\n",
    "    train_test = file.split('_')[1]\n",
    "    train_ratio = file.split('_')[2]\n",
    "    reduction = file.split('_')[-1].split('.npz')[0]\n",
    "    dest = os.replace(file, 'HDFS_Data/'+train_test+'/'+train_ratio+'/'+reduction+'/' + file)\n",
    "    print('------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data for DeepLog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stringify(l):\n",
    "    return ' '.join([str(x) for x in l])\n",
    "\n",
    "def prepare_deeplog(path):\n",
    "    data = np.load(path, allow_pickle=True)\n",
    "\n",
    "    data_normal = data['x_data'][data['y_data'] == 0]\n",
    "    data_abnormal = data['x_data'][data['y_data'] == 1]\n",
    "\n",
    "    data_normal = '\\n'.join([stringify(x) for x in data_normal])\n",
    "    data_abnormal = '\\n'.join([stringify(x) for x in data_abnormal])\n",
    "    \n",
    "    return data_normal, data_abnormal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [],
   "source": [
    "for train_ratio in train_ratios:\n",
    "    for reduction_perc in reduction_percentages:\n",
    "        if(str(train_ratio) not in os.listdir('DeepLog/data/')):\n",
    "            os.mkdir('DeepLog/data/' + str(train_ratio))\n",
    "        if(str(reduction_perc) not in os.listdir('DeepLog/data/' + str(train_ratio))):\n",
    "            os.mkdir('DeepLog/data/' + str(train_ratio) + '/' + str(reduction_perc))\n",
    "        \n",
    "        path_train = 'DeepLog/data/' + str(train_ratio) + '/' \n",
    "        path_test = 'DeepLog/data/' + str(train_ratio) + '/' + str(reduction_perc) + '/'\n",
    "        \n",
    "        path_file_train = ''\n",
    "        path_file_test = 'HDFS_Data/test/' + str(train_ratio) + '/' + str(reduction_perc) + '/'\n",
    "\n",
    "        data_train_normal, data_train_abnormal = prepare_deeplog(path_file_train + 'HDFS_train_' + str(train_ratio) + '.npz')\n",
    "        data_test_normal, data_test_abnormal = prepare_deeplog(path_file_test + 'HDFS_test_'+ str(train_ratio) + '_' + str(reduction_perc) + '.npz')\n",
    "        \n",
    "        with open(path_train + 'HDFS_train_normal_' + str(train_ratio) + '.txt', 'w') as _ :\n",
    "            _.write(data_train_normal)\n",
    "            \n",
    "        with open(path_train + 'HDFS_train_abnormal_' + str(train_ratio) + '.txt', 'w') as _ :\n",
    "            _.write(data_train_abnormal)\n",
    "            \n",
    "        with open(path_test + 'HDFS_test_normal_' + str(train_ratio) + '_' + str(reduction_perc) + '.txt', 'w') as _ :\n",
    "            _.write(data_test_normal)\n",
    "            \n",
    "        with open(path_test + 'HDFS_test_abnormal_' + str(train_ratio) + '_' + str(reduction_perc)  + '.txt', 'w') as _ :\n",
    "            _.write(data_test_abnormal)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Split data train test**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train_normal, data_train_abnormal = prepare_deeplog('HDFS_train_0.5.npz')\n",
    "data_test_normal, data_test_abnormal = prepare_deeplog('HDFS_test_0.5_0.25.npz')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}