task2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # Do not forget to check if your notebook log collection plugin is working\n",
    "# from mining_extension import check_logging\n",
    "# check_logging(\"PASTE FULL URL HERE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5833, 32)\n",
      "(1927, 32)\n",
      "(1918, 32)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "features_path = 'data/task2/'\n",
    "test_features = pd.read_pickle(features_path+'test_features.pkl')\n",
    "train_features = pd.read_pickle(features_path+'train_features.pkl')\n",
    "validation_features = pd.read_pickle(features_path+'validation_features.pkl')\n",
    "\n",
    "train_features.index = range(train_features.shape[0])\n",
    "validation_features.index = range(validation_features.shape[0])\n",
    "test_features.index = range(test_features.shape[0])\n",
    "\n",
    "train_features.fillna(0,inplace=True)\n",
    "test_features.fillna(0,inplace=True)\n",
    "validation_features.fillna(0,inplace=True)\n",
    "\n",
    "print(train_features.shape)\n",
    "print(validation_features.shape)\n",
    "print(test_features.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-26T16:12:14.634105Z",
     "end_time": "2023-04-26T16:12:14.666731Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "            filename cell_type  cell_number  execution_count  linesofcomment   \n0     nb_54880.ipynb      code            0                1             0.0  \\\n1     nb_54880.ipynb      code            1                2             1.0   \n2     nb_54880.ipynb      code            2                3             0.0   \n3     nb_54880.ipynb      code            3                4             0.0   \n4     nb_54880.ipynb      code            4                5             0.0   \n...              ...       ...          ...              ...             ...   \n5828  nb_95821.ipynb      code           27               -1             0.0   \n5829  nb_95821.ipynb      code           29               -1             0.0   \n5830  nb_95821.ipynb      code           31               -1             0.0   \n5831  nb_95821.ipynb      code           33               -1             5.0   \n5832  nb_95821.ipynb      code           35               -1             0.0   \n\n      linesofcode  variable_count  function_count  text/plain  image/png  ...   \n0             2.0             0.0             0.0           0          0  ...  \\\n1             4.0             4.0             0.0           0          0  ...   \n2             1.0             0.0             0.0           1          0  ...   \n3             1.0             0.0             0.0           1          0  ...   \n4             1.0             1.0             0.0           0          0  ...   \n...           ...             ...             ...         ...        ...  ...   \n5828          2.0             1.0             0.0           0          0  ...   \n5829          2.0             1.0             0.0           0          0  ...   \n5830          3.0             2.0             0.0           0          0  ...   \n5831         17.0            14.0             0.0           0          0  ...   \n5832          2.0             1.0             0.0           0          0  ...   \n\n      helper_functions  load_data  data_exploration  data_preprocessing   \n0                    1          0                 0                   0  \\\n1                    0          1                 0                   0   \n2                    0          0                 1                   0   \n3                    0          0                 1                   0   \n4                    0          0                 0                   1   \n...                ...        ...               ...                 ...   \n5828                 0          0                 0                   0   \n5829                 0          0                 0                   0   \n5830                 0          0                 1                   1   \n5831                 0          0                 0                   0   \n5832                 0          0                 0                   0   \n\n      evaluation modelling prediction result_visualization save_results   \n0              0         0          0                    0            0  \\\n1              0         0          0                    0            0   \n2              0         0          0                    0            0   \n3              0         0          0                    0            0   \n4              0         0          0                    0            0   \n...          ...       ...        ...                  ...          ...   \n5828           1         0          0                    0            0   \n5829           0         1          0                    0            0   \n5830           0         0          0                    0            0   \n5831           1         1          1                    0            0   \n5832           1         0          0                    0            0   \n\n     comment_only  \n0               0  \n1               0  \n2               0  \n3               0  \n4               0  \n...           ...  \n5828            0  \n5829            0  \n5830            0  \n5831            0  \n5832            0  \n\n[5833 rows x 32 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>filename</th>\n      <th>cell_type</th>\n      <th>cell_number</th>\n      <th>execution_count</th>\n      <th>linesofcomment</th>\n      <th>linesofcode</th>\n      <th>variable_count</th>\n      <th>function_count</th>\n      <th>text/plain</th>\n      <th>image/png</th>\n      <th>...</th>\n      <th>helper_functions</th>\n      <th>load_data</th>\n      <th>data_exploration</th>\n      <th>data_preprocessing</th>\n      <th>evaluation</th>\n      <th>modelling</th>\n      <th>prediction</th>\n      <th>result_visualization</th>\n      <th>save_results</th>\n      <th>comment_only</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>nb_54880.ipynb</td>\n      <td>code</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>2.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>nb_54880.ipynb</td>\n      <td>code</td>\n      <td>1</td>\n      <td>2</td>\n      <td>1.0</td>\n      <td>4.0</td>\n      <td>4.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>nb_54880.ipynb</td>\n      <td>code</td>\n      <td>2</td>\n      <td>3</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>nb_54880.ipynb</td>\n      <td>code</td>\n      <td>3</td>\n      <td>4</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>nb_54880.ipynb</td>\n      <td>code</td>\n      <td>4</td>\n      <td>5</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>5828</th>\n      <td>nb_95821.ipynb</td>\n      <td>code</td>\n      <td>27</td>\n      <td>-1</td>\n      <td>0.0</td>\n      <td>2.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5829</th>\n      <td>nb_95821.ipynb</td>\n      <td>code</td>\n      <td>29</td>\n      <td>-1</td>\n      <td>0.0</td>\n      <td>2.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5830</th>\n      <td>nb_95821.ipynb</td>\n      <td>code</td>\n      <td>31</td>\n      <td>-1</td>\n      <td>0.0</td>\n      <td>3.0</td>\n      <td>2.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5831</th>\n      <td>nb_95821.ipynb</td>\n      <td>code</td>\n      <td>33</td>\n      <td>-1</td>\n      <td>5.0</td>\n      <td>17.0</td>\n      <td>14.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5832</th>\n      <td>nb_95821.ipynb</td>\n      <td>code</td>\n      <td>35</td>\n      <td>-1</td>\n      <td>0.0</td>\n      <td>2.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5833 rows × 32 columns</p>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-26T16:12:35.376467Z",
     "end_time": "2023-04-26T16:12:35.388786Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "[\"l_cols = ['user_id','movie_id','rating']\",\n \"r_cols = ['movie_id','title']\",\n \"l = pd.read_csv('u.data', sep='\\\\t', names=l_cols, usecols=range(3))\",\n 'r = pd.read_csv(\\'u.item\\', sep=\\'|\\', names=r_cols, usecols=range(2), encoding = \"ISO-8859-1\") ## contains unicode characters']"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features.text[1]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-26T16:13:27.979105Z",
     "end_time": "2023-04-26T16:13:27.981328Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}