Topic Extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "07a9d29f",
   "metadata": {},
   "source": [
    "### Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bb1be03d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch and KeyBERT to extract keyword\n",
    "# if you found kernel issue after import the keybert, you have to import the torch before the keybert\n",
    "import torch\n",
    "from keybert import KeyBERT\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "string.punctuation\n",
    "import datetime\n",
    "\n",
    "# initialize now as filename\n",
    "now = datetime.datetime.today().strftime('%d_%m_%Y')\n",
    "\n",
    "# nltk to nlp preprocessing\n",
    "import nltk\n",
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "\n",
    "# initialize KeyBERT model\n",
    "kw_model = KeyBERT()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8068caa1",
   "metadata": {},
   "source": [
    "### Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1c7cdd7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>sinta_id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ENDI SUHENDI</td>\n",
       "      <td>5976806</td>\n",
       "      <td>Model of a tunneling current in a p-n junction...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ENDI SUHENDI</td>\n",
       "      <td>5976806</td>\n",
       "      <td>Improving Students' Science Process Skills thr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ENDI SUHENDI</td>\n",
       "      <td>5976806</td>\n",
       "      <td>Alleviating students’ misconceptions about new...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ENDI SUHENDI</td>\n",
       "      <td>5976806</td>\n",
       "      <td>Preface: MSCEIS 2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ENDI SUHENDI</td>\n",
       "      <td>5976806</td>\n",
       "      <td>The transformation of two-tier test into four-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           name  sinta_id                                              title\n",
       "0  ENDI SUHENDI   5976806  Model of a tunneling current in a p-n junction...\n",
       "1  ENDI SUHENDI   5976806  Improving Students' Science Process Skills thr...\n",
       "2  ENDI SUHENDI   5976806  Alleviating students’ misconceptions about new...\n",
       "3  ENDI SUHENDI   5976806                               Preface: MSCEIS 2016\n",
       "4  ENDI SUHENDI   5976806  The transformation of two-tier test into four-..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read data, drop duplicates, and get 'name', 'sinta_id', 'title' column\n",
    "df = pd.read_csv(\"article_09_08_2022.csv\", encoding='utf-8')\n",
    "df = df.drop_duplicates(subset=None, keep='first', inplace=False, ignore_index=False)\n",
    "df = df[['name', 'sinta_id', 'title']]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "576fd1c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 10122 entries, 0 to 11230\n",
      "Data columns (total 3 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   name      10122 non-null  object\n",
      " 1   sinta_id  10122 non-null  int64 \n",
      " 2   title     10122 non-null  object\n",
      "dtypes: int64(1), object(2)\n",
      "memory usage: 316.3+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "eeb56913",
   "metadata": {},
   "source": [
    "### NLP Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "22f0550b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define preprocessing function\n",
    "def preprocessing(text):\n",
    "    # join title, set title as lower, replace several string from title, and tokenize title\n",
    "    title = text\n",
    "    title = \"\".join([i for i in title if i not in string.punctuation])\n",
    "    title = title.lower()\n",
    "    title = title.replace(\"alqur’an\", \"alquran\")\n",
    "    title = title.replace(\"qur’an\", \"quran\")    \n",
    "    title = word_tokenize(title)\n",
    "    \n",
    "    # initialize stopwords and extend several word to the stop_list\n",
    "    stopwords = nltk.corpus.stopwords.words('english')\n",
    "    stop_list = ['database', '20112020']\n",
    "    stopwords.extend(stop_list)\n",
    "    \n",
    "    # remove stop words \n",
    "    title = [i for i in title if i not in stopwords]\n",
    "    \n",
    "    # join title into string\n",
    "    title = \" \".join([i for i in title if i not in string.punctuation])\n",
    "    return title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a92ff3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply function to data\n",
    "df['title']= df['title'].apply(lambda x:preprocessing(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "28a86893",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>sinta_id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LINDA SETIAWATI</td>\n",
       "      <td>5995101</td>\n",
       "      <td>development computer teaching materials educat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A SOBANDI</td>\n",
       "      <td>5994552</td>\n",
       "      <td>knowledge management process knowledge sharing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A. BUDHI SALIRA</td>\n",
       "      <td>6771601</td>\n",
       "      <td>readiness online learning covid19 pandemic voc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AAH AHMAD SYAHID</td>\n",
       "      <td>5993425</td>\n",
       "      <td>alsumud wa altakayyuf wa altathaquf tabir huwi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AAM ABDUSSALAM</td>\n",
       "      <td>5993560</td>\n",
       "      <td>decade value education model bibliometric stud...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name  sinta_id  \\\n",
       "0   LINDA SETIAWATI   5995101   \n",
       "1         A SOBANDI   5994552   \n",
       "2   A. BUDHI SALIRA   6771601   \n",
       "3  AAH AHMAD SYAHID   5993425   \n",
       "4    AAM ABDUSSALAM   5993560   \n",
       "\n",
       "                                               title  \n",
       "0  development computer teaching materials educat...  \n",
       "1  knowledge management process knowledge sharing...  \n",
       "2  readiness online learning covid19 pandemic voc...  \n",
       "3  alsumud wa altakayyuf wa altathaquf tabir huwi...  \n",
       "4  decade value education model bibliometric stud...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# join title into one row by grouping data by 'name' and 'sinta_id' then reset index\n",
    "df = df.groupby(['name','sinta_id'])['title'].apply('. '.join).reset_index()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ffa29920",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'development computer teaching materials education management applications elearning module. hard skills versus soft skills affect different job types japanese language graduates. development automatic system icmls 20 improving educational technology competences industrial revolution 40. developing integrated management information system research study institute research community services universitas pendidikan indonesia'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check title after preprocessing\n",
    "df['title'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "25cd5613",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>sinta_id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LINDA SETIAWATI</td>\n",
       "      <td>5995101</td>\n",
       "      <td>development computer teaching materials educat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A SOBANDI</td>\n",
       "      <td>5994552</td>\n",
       "      <td>knowledge management process knowledge sharing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A. BUDHI SALIRA</td>\n",
       "      <td>6771601</td>\n",
       "      <td>readiness online learning covid19 pandemic voc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AAH AHMAD SYAHID</td>\n",
       "      <td>5993425</td>\n",
       "      <td>alsumud wa altakayyuf wa altathaquf tabir huwi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AAM ABDUSSALAM</td>\n",
       "      <td>5993560</td>\n",
       "      <td>decade value education model bibliometric stud...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name  sinta_id  \\\n",
       "0   LINDA SETIAWATI   5995101   \n",
       "1         A SOBANDI   5994552   \n",
       "2   A. BUDHI SALIRA   6771601   \n",
       "3  AAH AHMAD SYAHID   5993425   \n",
       "4    AAM ABDUSSALAM   5993560   \n",
       "\n",
       "                                               title  \n",
       "0  development computer teaching materials educat...  \n",
       "1  knowledge management process knowledge sharing...  \n",
       "2  readiness online learning covid19 pandemic voc...  \n",
       "3  alsumud wa altakayyuf wa altathaquf tabir huwi...  \n",
       "4  decade value education model bibliometric stud...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f36866d",
   "metadata": {},
   "source": [
    "### Extract Keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fd886f47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>sinta_id</th>\n",
       "      <th>title</th>\n",
       "      <th>topics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LINDA SETIAWATI</td>\n",
       "      <td>5995101</td>\n",
       "      <td>development computer teaching materials educat...</td>\n",
       "      <td>educational technology competences, management...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A SOBANDI</td>\n",
       "      <td>5994552</td>\n",
       "      <td>knowledge management process knowledge sharing...</td>\n",
       "      <td>literacy skills vocational, knowledge manageme...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A. BUDHI SALIRA</td>\n",
       "      <td>6771601</td>\n",
       "      <td>readiness online learning covid19 pandemic voc...</td>\n",
       "      <td>pandemic vocational education, readiness onlin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AAH AHMAD SYAHID</td>\n",
       "      <td>5993425</td>\n",
       "      <td>alsumud wa altakayyuf wa altathaquf tabir huwi...</td>\n",
       "      <td>school teachers ict, neural network extreme, h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AAM ABDUSSALAM</td>\n",
       "      <td>5993560</td>\n",
       "      <td>decade value education model bibliometric stud...</td>\n",
       "      <td>alquran literacy strategy, decade value educat...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name  sinta_id  \\\n",
       "0   LINDA SETIAWATI   5995101   \n",
       "1         A SOBANDI   5994552   \n",
       "2   A. BUDHI SALIRA   6771601   \n",
       "3  AAH AHMAD SYAHID   5993425   \n",
       "4    AAM ABDUSSALAM   5993560   \n",
       "\n",
       "                                               title  \\\n",
       "0  development computer teaching materials educat...   \n",
       "1  knowledge management process knowledge sharing...   \n",
       "2  readiness online learning covid19 pandemic voc...   \n",
       "3  alsumud wa altakayyuf wa altathaquf tabir huwi...   \n",
       "4  decade value education model bibliometric stud...   \n",
       "\n",
       "                                              topics  \n",
       "0  educational technology competences, management...  \n",
       "1  literacy skills vocational, knowledge manageme...  \n",
       "2  pandemic vocational education, readiness onlin...  \n",
       "3  school teachers ict, neural network extreme, h...  \n",
       "4  alquran literacy strategy, decade value educat...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# initialize model and apply lambda function to join the keywords as a string\n",
    "# parameter descriptions\n",
    "\"\"\"\n",
    "    top_n: the number of keywords(phrases) extracted for each row\n",
    "    stop_words: type of language to remove stop words\n",
    "    n_gram_range:\n",
    "    length of the word or phrase to be extracted, \n",
    "    if the value is (3, 3) then it will extract 3 phrases with each phrase consisting of 3 words \n",
    "    diversity: parameter level of similarity between words or phrases\n",
    "\"\"\"\n",
    "extract_kw = lambda x: ', '.join(k[0] for k in kw_model.extract_keywords(x, keyphrase_ngram_range=(3, 3), stop_words='english',\n",
    "                              use_mmr=True, top_n=3, diversity=0.5))\n",
    "\n",
    "# apply function to the 'topics' column\n",
    "df['topics'] = df['title'].apply(extract_kw)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2deab5fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[['sinta_id', 'title', 'topics']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "031ab366",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sinta_id</th>\n",
       "      <th>title</th>\n",
       "      <th>topics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5995101</td>\n",
       "      <td>development computer teaching materials educat...</td>\n",
       "      <td>educational technology competences, management...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5994552</td>\n",
       "      <td>knowledge management process knowledge sharing...</td>\n",
       "      <td>literacy skills vocational, knowledge manageme...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6771601</td>\n",
       "      <td>readiness online learning covid19 pandemic voc...</td>\n",
       "      <td>pandemic vocational education, readiness onlin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5993425</td>\n",
       "      <td>alsumud wa altakayyuf wa altathaquf tabir huwi...</td>\n",
       "      <td>school teachers ict, neural network extreme, h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5993560</td>\n",
       "      <td>decade value education model bibliometric stud...</td>\n",
       "      <td>alquran literacy strategy, decade value educat...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sinta_id                                              title  \\\n",
       "0   5995101  development computer teaching materials educat...   \n",
       "1   5994552  knowledge management process knowledge sharing...   \n",
       "2   6771601  readiness online learning covid19 pandemic voc...   \n",
       "3   5993425  alsumud wa altakayyuf wa altathaquf tabir huwi...   \n",
       "4   5993560  decade value education model bibliometric stud...   \n",
       "\n",
       "                                              topics  \n",
       "0  educational technology competences, management...  \n",
       "1  literacy skills vocational, knowledge manageme...  \n",
       "2  pandemic vocational education, readiness onlin...  \n",
       "3  school teachers ict, neural network extreme, h...  \n",
       "4  alquran literacy strategy, decade value educat...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "91750395",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export data to csv\n",
    "df.to_csv(f'1topic_result_{now}.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}