Nuovi_articoli.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/alex/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from AC import preproc\n",
    "from AC import get_news\n",
    "import inspect\n",
    "import nltk\n",
    "nltk.download(\"stopwords\")\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2400/2400 [03:43<00:00, 10.75it/s]\n"
     ]
    }
   ],
   "source": [
    "economia = get_news(\"./articoli_economia/\")\n",
    "cultura = get_news(\"./articoli_cultura/\")\n",
    "tech = get_news(\"./articoli_tech/\")\n",
    "politica = get_news(\"./articoli_politica/\")\n",
    "sport = get_news(\"./articoli_sport/\")\n",
    "cronaca = get_news(\"./articoli_cronaca/\")\n",
    "\n",
    "for articolo in economia:\n",
    "    articolo['categoria'] = \"Economia\"\n",
    "for articolo in cultura:\n",
    "    articolo['categoria'] = \"Cultura\"\n",
    "for articolo in tech:\n",
    "    articolo['categoria'] = \"Tech\"\n",
    "for articolo in politica:\n",
    "    articolo['categoria'] = \"Politica\"\n",
    "for articolo in sport:\n",
    "    articolo['categoria'] = \"Sport\"\n",
    "for articolo in cronaca:\n",
    "    articolo['categoria'] = \"Cronaca\"\n",
    "dati_preprocessati =  preproc(tech + politica + cultura + economia + sport + cronaca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alex/anaconda3/envs/msbd/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "seed = 0\n",
    "train_documents,test_documents = train_test_split(dati_preprocessati,random_state=seed,train_size=0.5)\n",
    "test_documents, val_documents = train_test_split(test_documents,random_state=seed,train_size=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_texts = [' '.join([word for word in x['testo']] + x['sottotitolo'] + x['titolo_articolo']) for x in dati_preprocessati]\n",
    "train_texts = [' '.join([word for word in x['testo']] + x['sottotitolo'] + x['titolo_articolo']) for x in train_documents]\n",
    "test_texts = [' '.join([word for word in x['testo']] + x['sottotitolo'] + x['titolo_articolo']) for x in test_documents]\n",
    "val_texts = [' '.join([word for word in x['testo']] + x['sottotitolo'] + x['titolo_articolo']) for x in val_documents]\n",
    "\n",
    "docs_cats = [x[\"categoria\"] for x in dati_preprocessati]\n",
    "train_cats = [x[\"categoria\"] for x in train_documents]\n",
    "test_cats = [x[\"categoria\"] for x in test_documents]\n",
    "val_cats = [x[\"categoria\"] for x in val_documents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('count_mx', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=400, max_features=1000000, min_df=9,\n",
       "        ngram_range=(1, 3), preprocessor=None, stop_words=None,\n",
       "      ...         min_weight_fraction_leaf=0.0, presort=False, random_state=0,\n",
       "            splitter='best'))])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldac = Pipeline([\n",
    "    (\"count_mx\",CountVectorizer(max_features=1000000, lowercase=True)),\n",
    "    (\"lda\", LatentDirichletAllocation(max_iter=50, learning_method='online',random_state=0)),\n",
    "    (\"classifier\",DecisionTreeClassifier(random_state=0))\n",
    "])\n",
    "params = {\n",
    "    'lda__n_components': 12,\n",
    "    'lda__learning_decay': 0.7, \n",
    "    'count_mx__ngram_range': (1, 3), \n",
    "    'count_mx__min_df': 9,\n",
    "    'count_mx__max_df': 400\n",
    "}\n",
    "ldac.set_params(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('count_mx', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=400, max_features=1000000, min_df=9,\n",
       "        ngram_range=(1, 3), preprocessor=None, stop_words=None,\n",
       "      ...         min_weight_fraction_leaf=0.0, presort=False, random_state=0,\n",
       "            splitter='best'))])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldac.fit(train_texts, train_cats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuovi articoli (possibilmente non gia' presenti nel dataset) da classificare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 11.96it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Tech'], dtype='<U8')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from estrattore_dati.estrattore_articoli_ansa import estrai\n",
    "p = estrai(\"http://www.ansa.it/sito/notizie/tecnologia/hitech/2019/06/03/amazon-apre-negozio-a-manchester_24353478-6d22-43b4-89d6-32725e95bf09.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/politica/2019/06/09/bufera-procure-ora-lanm-rischia-la-crisi_03006b0c-effb-4b43-be1b-1cf40ab78c2b.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/sport/tennis/2019/06/10/woods-e-molinari-celebrano-nadal_e7e5fb39-7c09-45d1-be35-65691117c3d2.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/cultura/musica/2019/06/09/ligabue-niente-e-come-un-concerto_1948cb23-6b91-4eec-92eb-d5e255556de1.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/tecnologia/hitech/2019/06/13/uber-testa-drone-per-i-pasti-a-domicilio_24010da2-972b-4d3b-bfeb-ea62f584111d.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/politica/2019/06/11/governo-conte-di-maio-salvini-ue_0eeb81e1-d058-480b-8550-9b9524a35060.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/sport/calcio/2019/06/13/il-presidente-della-lega-pro-ghirelli-ospite-del-forum-ansa_1646e537-9880-4e92-b64e-f45281d7cf69.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/cronaca/2019/06/12/dl-sicurezza-bis-da-multe-a-navi-a-fondo-rimpatri-_e93de1be-fffb-4f92-a2b1-a52ed246a6fe.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/cronaca/2019/06/12/addio-ai-furbetti-del-cartellino-arrivano-i-controlli-biometrici_eb36c913-b809-4ae0-b59c-5087222d8c60.html\")\n",
    "#p = estrai(\"http://www.ansa.it/sito/notizie/tecnologia/software_app/2019/06/12/facebook-paga-utenti-per-dati-su-uso-app_fee73f87-6558-486f-9ec5-be509c3cacef.html\")\n",
    "x_1 = preproc([p])\n",
    "\n",
    "ldac.predict([' '.join(x_1[0]['testo'] +x_1[0]['titolo_articolo']+x_1[0]['sottotitolo'] + x_1[0]['tags'])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}