{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dataiku\n",
    "from dataiku import pandasutils as pdu\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the dataset as a Pandas dataframe in memory\n",
    "# Note: here, we only read the first 100K rows. Other sampling options are available\n",
    "dataset_customers_stacked_prepared = dataiku.Dataset(\"customers_stacked_prepared\")\n",
    "df = dataset_customers_stacked_prepared.get_dataframe(limit=100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <button style=\"display:none\" \n",
       "            class=\"btn btn-default ipython-export-btn\" \n",
       "            id=\"btn-df-09d58e9d-b8a0-40f0-95cb-6e1fd4e86201\" \n",
       "            onclick=\"_export_df('09d58e9d-b8a0-40f0-95cb-6e1fd4e86201')\">\n",
       "                Export dataframe\n",
       "            </button>\n",
       "            \n",
       "            <script>\n",
       "                \n",
       "                function _check_export_df_possible(dfid,yes_fn,no_fn) {\n",
       "                    console.log('Checking dataframe exportability...')\n",
       "                    if(!IPython || !IPython.notebook || !IPython.notebook.kernel || !IPython.notebook.kernel) {\n",
       "                        console.log('Export is not possible (IPython kernel is not available)')\n",
       "                        if(no_fn) {\n",
       "                            no_fn();\n",
       "                        }\n",
       "                    } else {\n",
       "                        var pythonCode = 'from dataiku.notebook.export import IPythonExporter;IPythonExporter._check_export_stdout(\"'+dfid+'\")';\n",
       "                        IPython.notebook.kernel.execute(pythonCode,{iopub: {output: function(resp) {\n",
       "                            console.info(\"Exportability response\", resp);\n",
       "                            var size = /^([0-9]+)x([0-9]+)$/.exec(resp.content.data || resp.content.text)\n",
       "                            if(!size) {\n",
       "                                console.log('Export is not possible (dataframe is not in-memory anymore)')\n",
       "                                if(no_fn) {\n",
       "                                    no_fn();\n",
       "                                }\n",
       "                            } else {\n",
       "                                console.log('Export is possible')\n",
       "                                if(yes_fn) {\n",
       "                                    yes_fn(1*size[1],1*size[2]);\n",
       "                                }\n",
       "                            }\n",
       "                        }}});\n",
       "                    }\n",
       "                }\n",
       "            \n",
       "                function _export_df(dfid) {\n",
       "                    \n",
       "                    var btn = $('#btn-df-'+dfid);\n",
       "                    var btns = $('.ipython-export-btn');\n",
       "                    \n",
       "                    _check_export_df_possible(dfid,function() {\n",
       "                        \n",
       "                        window.parent.openExportModalFromIPython('Pandas dataframe',function(data) {\n",
       "                            btns.prop('disabled',true);\n",
       "                            btn.text('Exporting...');\n",
       "                            var command = 'from dataiku.notebook.export import IPythonExporter;IPythonExporter._run_export(\"'+dfid+'\",\"'+data.exportId+'\")';\n",
       "                            var callback = {iopub:{output: function(resp) {\n",
       "                                console.info(\"CB resp:\", resp);\n",
       "                                _check_export_df_possible(dfid,function(rows, cols) {\n",
       "                                    $('#btn-df-'+dfid)\n",
       "                                        .css('display','inline-block')\n",
       "                                        .text('Export this dataframe ('+rows+' rows, '+cols+' cols)')\n",
       "                                        .prop('disabled',false);\n",
       "                                },function() {\n",
       "                                    $('#btn-df-'+dfid).css('display','none');\n",
       "                                });\n",
       "                            }}};\n",
       "                            IPython.notebook.kernel.execute(command,callback,{silent:false}); // yes, silent now defaults to true. figures.\n",
       "                        });\n",
       "                    \n",
       "                    }, function(){\n",
       "                            alert('Unable to export : the Dataframe object is not loaded in memory');\n",
       "                            btn.css('display','none');\n",
       "                    });\n",
       "                    \n",
       "                }\n",
       "                \n",
       "                (function(dfid) {\n",
       "                \n",
       "                    var retryCount = 10;\n",
       "                \n",
       "                    function is_valid_websock(s) {\n",
       "                        return s && s.readyState==1;\n",
       "                    }\n",
       "                \n",
       "                    function check_conn() {\n",
       "                        \n",
       "                        if(!IPython || !IPython.notebook) {\n",
       "                            // Don't even try to go further\n",
       "                            return;\n",
       "                        }\n",
       "                        \n",
       "                        // Check if IPython is ready\n",
       "                        console.info(\"Checking conn ...\")\n",
       "                        if(IPython.notebook.kernel\n",
       "                        && IPython.notebook.kernel\n",
       "                        && is_valid_websock(IPython.notebook.kernel.ws)\n",
       "                        ) {\n",
       "                            \n",
       "                            _check_export_df_possible(dfid,function(rows, cols) {\n",
       "                                $('#btn-df-'+dfid).css('display','inline-block');\n",
       "                                $('#btn-df-'+dfid).text('Export this dataframe ('+rows+' rows, '+cols+' cols)');\n",
       "                            });\n",
       "                            \n",
       "                        } else {\n",
       "                            console.info(\"Conditions are not ok\", IPython.notebook.kernel);\n",
       "                            \n",
       "                            // Retry later\n",
       "                            \n",
       "                            if(retryCount>0) {\n",
       "                                setTimeout(check_conn,500);\n",
       "                                retryCount--;\n",
       "                            }\n",
       "                            \n",
       "                        }\n",
       "                    };\n",
       "                    \n",
       "                    setTimeout(check_conn,100);\n",
       "                    \n",
       "                })(\"09d58e9d-b8a0-40f0-95cb-6e1fd4e86201\");\n",
       "                \n",
       "            </script>\n",
       "            \n",
       "        <div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_a_variable</th>\n",
       "      <th>_b_data_type</th>\n",
       "      <th>_c_cardinality</th>\n",
       "      <th>_d_missings</th>\n",
       "      <th>_e_sample_values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>customerID</td>\n",
       "      <td>object</td>\n",
       "      <td>21123</td>\n",
       "      <td>0</td>\n",
       "      <td>[886900, 801797]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gender</td>\n",
       "      <td>object</td>\n",
       "      <td>3</td>\n",
       "      <td>2157</td>\n",
       "      <td>[M, F]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>birthdate</td>\n",
       "      <td>object</td>\n",
       "      <td>14583</td>\n",
       "      <td>0</td>\n",
       "      <td>[1953-01-01T10:05:00.000Z, 1956-01-01T07:19:00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>user_agent_brand</td>\n",
       "      <td>object</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[Edge, Chrome]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>user_agent_os</td>\n",
       "      <td>object</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>[Windows, MacOS X]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ip_address</td>\n",
       "      <td>object</td>\n",
       "      <td>21198</td>\n",
       "      <td>0</td>\n",
       "      <td>[69.8.136.110, 137.155.219.158]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ip_address_country</td>\n",
       "      <td>object</td>\n",
       "      <td>158</td>\n",
       "      <td>453</td>\n",
       "      <td>[United States, Germany]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ip_address_geopoint</td>\n",
       "      <td>object</td>\n",
       "      <td>6247</td>\n",
       "      <td>391</td>\n",
       "      <td>[POINT(-89.8467 43.2626), POINT(-76.4936 37.07...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>campaign</td>\n",
       "      <td>bool</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[False, True]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>original_dataset</td>\n",
       "      <td>object</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[labeled, unlabeled]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           _a_variable _b_data_type  _c_cardinality  _d_missings                                   _e_sample_values\n",
       "0           customerID       object           21123            0                                   [886900, 801797]\n",
       "1               gender       object               3         2157                                             [M, F]\n",
       "2            birthdate       object           14583            0  [1953-01-01T10:05:00.000Z, 1956-01-01T07:19:00...\n",
       "3     user_agent_brand       object               5            0                                     [Edge, Chrome]\n",
       "4        user_agent_os       object               3            0                                 [Windows, MacOS X]\n",
       "5           ip_address       object           21198            0                    [69.8.136.110, 137.155.219.158]\n",
       "6   ip_address_country       object             158          453                           [United States, Germany]\n",
       "7  ip_address_geopoint       object            6247          391  [POINT(-89.8467 43.2626), POINT(-76.4936 37.07...\n",
       "8             campaign         bool               2            0                                      [False, True]\n",
       "9     original_dataset       object               2            0                               [labeled, unlabeled]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get some simple descriptive statistics\n",
    "pdu.audit(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <button style=\"display:none\" \n",
       "            class=\"btn btn-default ipython-export-btn\" \n",
       "            id=\"btn-df-004a740d-06f9-4e41-87c9-9bcbb11b70aa\" \n",
       "            onclick=\"_export_df('004a740d-06f9-4e41-87c9-9bcbb11b70aa')\">\n",
       "                Export dataframe\n",
       "            </button>\n",
       "            \n",
       "            <script>\n",
       "                \n",
       "                function _check_export_df_possible(dfid,yes_fn,no_fn) {\n",
       "                    console.log('Checking dataframe exportability...')\n",
       "                    if(!IPython || !IPython.notebook || !IPython.notebook.kernel || !IPython.notebook.kernel) {\n",
       "                        console.log('Export is not possible (IPython kernel is not available)')\n",
       "                        if(no_fn) {\n",
       "                            no_fn();\n",
       "                        }\n",
       "                    } else {\n",
       "                        var pythonCode = 'from dataiku.notebook.export import IPythonExporter;IPythonExporter._check_export_stdout(\"'+dfid+'\")';\n",
       "                        IPython.notebook.kernel.execute(pythonCode,{iopub: {output: function(resp) {\n",
       "                            console.info(\"Exportability response\", resp);\n",
       "                            var size = /^([0-9]+)x([0-9]+)$/.exec(resp.content.data || resp.content.text)\n",
       "                            if(!size) {\n",
       "                                console.log('Export is not possible (dataframe is not in-memory anymore)')\n",
       "                                if(no_fn) {\n",
       "                                    no_fn();\n",
       "                                }\n",
       "                            } else {\n",
       "                                console.log('Export is possible')\n",
       "                                if(yes_fn) {\n",
       "                                    yes_fn(1*size[1],1*size[2]);\n",
       "                                }\n",
       "                            }\n",
       "                        }}});\n",
       "                    }\n",
       "                }\n",
       "            \n",
       "                function _export_df(dfid) {\n",
       "                    \n",
       "                    var btn = $('#btn-df-'+dfid);\n",
       "                    var btns = $('.ipython-export-btn');\n",
       "                    \n",
       "                    _check_export_df_possible(dfid,function() {\n",
       "                        \n",
       "                        window.parent.openExportModalFromIPython('Pandas dataframe',function(data) {\n",
       "                            btns.prop('disabled',true);\n",
       "                            btn.text('Exporting...');\n",
       "                            var command = 'from dataiku.notebook.export import IPythonExporter;IPythonExporter._run_export(\"'+dfid+'\",\"'+data.exportId+'\")';\n",
       "                            var callback = {iopub:{output: function(resp) {\n",
       "                                console.info(\"CB resp:\", resp);\n",
       "                                _check_export_df_possible(dfid,function(rows, cols) {\n",
       "                                    $('#btn-df-'+dfid)\n",
       "                                        .css('display','inline-block')\n",
       "                                        .text('Export this dataframe ('+rows+' rows, '+cols+' cols)')\n",
       "                                        .prop('disabled',false);\n",
       "                                },function() {\n",
       "                                    $('#btn-df-'+dfid).css('display','none');\n",
       "                                });\n",
       "                            }}};\n",
       "                            IPython.notebook.kernel.execute(command,callback,{silent:false}); // yes, silent now defaults to true. figures.\n",
       "                        });\n",
       "                    \n",
       "                    }, function(){\n",
       "                            alert('Unable to export : the Dataframe object is not loaded in memory');\n",
       "                            btn.css('display','none');\n",
       "                    });\n",
       "                    \n",
       "                }\n",
       "                \n",
       "                (function(dfid) {\n",
       "                \n",
       "                    var retryCount = 10;\n",
       "                \n",
       "                    function is_valid_websock(s) {\n",
       "                        return s && s.readyState==1;\n",
       "                    }\n",
       "                \n",
       "                    function check_conn() {\n",
       "                        \n",
       "                        if(!IPython || !IPython.notebook) {\n",
       "                            // Don't even try to go further\n",
       "                            return;\n",
       "                        }\n",
       "                        \n",
       "                        // Check if IPython is ready\n",
       "                        console.info(\"Checking conn ...\")\n",
       "                        if(IPython.notebook.kernel\n",
       "                        && IPython.notebook.kernel\n",
       "                        && is_valid_websock(IPython.notebook.kernel.ws)\n",
       "                        ) {\n",
       "                            \n",
       "                            _check_export_df_possible(dfid,function(rows, cols) {\n",
       "                                $('#btn-df-'+dfid).css('display','inline-block');\n",
       "                                $('#btn-df-'+dfid).text('Export this dataframe ('+rows+' rows, '+cols+' cols)');\n",
       "                            });\n",
       "                            \n",
       "                        } else {\n",
       "                            console.info(\"Conditions are not ok\", IPython.notebook.kernel);\n",
       "                            \n",
       "                            // Retry later\n",
       "                            \n",
       "                            if(retryCount>0) {\n",
       "                                setTimeout(check_conn,500);\n",
       "                                retryCount--;\n",
       "                            }\n",
       "                            \n",
       "                        }\n",
       "                    };\n",
       "                    \n",
       "                    setTimeout(check_conn,100);\n",
       "                    \n",
       "                })(\"004a740d-06f9-4e41-87c9-9bcbb11b70aa\");\n",
       "                \n",
       "            </script>\n",
       "            \n",
       "        <div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerID</th>\n",
       "      <th>gender</th>\n",
       "      <th>birthdate</th>\n",
       "      <th>user_agent_brand</th>\n",
       "      <th>user_agent_os</th>\n",
       "      <th>ip_address</th>\n",
       "      <th>ip_address_country</th>\n",
       "      <th>ip_address_geopoint</th>\n",
       "      <th>campaign</th>\n",
       "      <th>original_dataset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>886900</td>\n",
       "      <td>M</td>\n",
       "      <td>1953-01-01T10:05:00.000Z</td>\n",
       "      <td>Edge</td>\n",
       "      <td>Windows</td>\n",
       "      <td>69.8.136.110</td>\n",
       "      <td>United States</td>\n",
       "      <td>POINT(-89.8467 43.2626)</td>\n",
       "      <td>False</td>\n",
       "      <td>labeled</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>801797</td>\n",
       "      <td>M</td>\n",
       "      <td>1956-01-01T07:19:00.000Z</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>Windows</td>\n",
       "      <td>137.155.219.158</td>\n",
       "      <td>United States</td>\n",
       "      <td>POINT(-76.4936 37.0736)</td>\n",
       "      <td>False</td>\n",
       "      <td>labeled</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>vft1eu</td>\n",
       "      <td>F</td>\n",
       "      <td>1969-01-01T06:27:00.000Z</td>\n",
       "      <td>Safari</td>\n",
       "      <td>MacOS X</td>\n",
       "      <td>53.115.6.106</td>\n",
       "      <td>Germany</td>\n",
       "      <td>POINT(9.491 51.2993)</td>\n",
       "      <td>True</td>\n",
       "      <td>labeled</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>932235</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1976-01-01T03:05:00.000Z</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>Windows</td>\n",
       "      <td>56.207.37.202</td>\n",
       "      <td>United States</td>\n",
       "      <td>POINT(-78.6253 35.7977)</td>\n",
       "      <td>False</td>\n",
       "      <td>labeled</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b5cqwi</td>\n",
       "      <td>M</td>\n",
       "      <td>1987-01-01T05:06:00.000Z</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>Windows</td>\n",
       "      <td>166.143.61.233</td>\n",
       "      <td>United States</td>\n",
       "      <td>POINT(-97.822 37.751)</td>\n",
       "      <td>False</td>\n",
       "      <td>labeled</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerID gender                 birthdate user_agent_brand user_agent_os       ip_address ip_address_country      ip_address_geopoint  campaign original_dataset\n",
       "0     886900      M  1953-01-01T10:05:00.000Z             Edge       Windows     69.8.136.110      United States  POINT(-89.8467 43.2626)     False          labeled\n",
       "1     801797      M  1956-01-01T07:19:00.000Z           Chrome       Windows  137.155.219.158      United States  POINT(-76.4936 37.0736)     False          labeled\n",
       "2     vft1eu      F  1969-01-01T06:27:00.000Z           Safari       MacOS X     53.115.6.106            Germany     POINT(9.491 51.2993)      True          labeled\n",
       "3     932235    NaN  1976-01-01T03:05:00.000Z           Chrome       Windows    56.207.37.202      United States  POINT(-78.6253 35.7977)     False          labeled\n",
       "4     b5cqwi      M  1987-01-01T05:06:00.000Z           Chrome       Windows   166.143.61.233      United States    POINT(-97.822 37.751)     False          labeled"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named ggplot2",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mImportError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-e3634b7449ac>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mggplot2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m: No module named ggplot2"
     ]
    }
   ],
   "source": [
    "import ggplot2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "analyzedDataset": "customers_stacked_prepared",
  "creator": "alex",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  },
  "tags": []
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
