{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "uXh1s3mErCra" }, "source": [ "# 授業評価アンケートのデータ収集用スクリプト\n", "[調査と解析班](https://r.st.ie.u-ryukyu.ac.jp/assessment/)で収集している、知能除法コース専門科目の2021年度前期科目を対象に自由記述欄のデータを収集。\n", "- 更新ログ\n", " - 2025年4月14日: データ保存先をカレントディレクトリに変更。" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "2l0y8darrCrc", "outputId": "53d5c552-33e0-43f7-e0cf-ab3a01dc9547" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Mon Apr 14 05:04:24 AM UTC 2025\n", " % Total % Received % Xferd Average Speed Time Time Time Current\n", " Dload Upload Total Spent Left Speed\n", "100 2455 100 2455 0 0 1043 0 0:00:02 0:00:02 --:--:-- 1042\n" ] } ], "source": [ "!date\n", "!curl -O https://ie.u-ryukyu.ac.jp/~tnal/2022/dm/static/r_assesment_list.csv" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "id": "wCkC4giyrCrd" }, "outputs": [], "source": [ "import requests\n", "import pandas as pd\n", "from bs4 import BeautifulSoup\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 201 }, "id": "qSJyICOWrCre", "outputId": "87d040bb-af39-487b-fa5f-d531054b3c49" }, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " title grade required year \\\n", "0 工業数学Ⅰ 1 True 2021 \n", "1 技術者の倫理 1 True 2021 \n", "2 工学基礎演習 1 True 2021 \n", "3 プログラミングⅠ 1 True 2021 \n", "4 基礎数学Ⅰ 1 False 2021 \n", "\n", " url \n", "0 https://r.st.ie.u-ryukyu.ac.jp/assessment/2021... \n", "1 https://r.st.ie.u-ryukyu.ac.jp/assessment/2021... \n", "2 https://r.st.ie.u-ryukyu.ac.jp/assessment/2021... \n", "3 https://r.st.ie.u-ryukyu.ac.jp/assessment/2021... \n", "4 https://r.st.ie.u-ryukyu.ac.jp/assessment/2021... " ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titlegraderequiredyearurl
0工業数学Ⅰ1True2021https://r.st.ie.u-ryukyu.ac.jp/assessment/2021...
1技術者の倫理1True2021https://r.st.ie.u-ryukyu.ac.jp/assessment/2021...
2工学基礎演習1True2021https://r.st.ie.u-ryukyu.ac.jp/assessment/2021...
3プログラミングⅠ1True2021https://r.st.ie.u-ryukyu.ac.jp/assessment/2021...
4基礎数学Ⅰ1False2021https://r.st.ie.u-ryukyu.ac.jp/assessment/2021...
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 21,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 21,\n \"samples\": [\n \"\\u5de5\\u696d\\u6570\\u5b66\\u2160\",\n \"\\u30c7\\u30fc\\u30bf\\u30de\\u30a4\\u30cb\\u30f3\\u30b0\",\n \"\\u4e26\\u5217\\u5206\\u6563\\u51e6\\u7406\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"grade\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0,\n \"min\": 1,\n \"max\": 3,\n \"num_unique_values\": 3,\n \"samples\": [\n 1,\n 2,\n 3\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"required\",\n \"properties\": {\n \"dtype\": \"boolean\",\n \"num_unique_values\": 2,\n \"samples\": [\n false,\n true\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"year\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0,\n \"min\": 2021,\n \"max\": 2021,\n \"num_unique_values\": 1,\n \"samples\": [\n 2021\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"url\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 21,\n \"samples\": [\n \"https://r.st.ie.u-ryukyu.ac.jp/assessment/2021a/result/makepage.php?kamoku=e111\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 3 } ], "source": [ "source_file = 'r_assesment_list.csv'\n", "assesment_columns = ['title', 'grade', 'required', 'year', 'url']\n", "df = pd.read_csv(source_file, names=assesment_columns)\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "z1WlfEHVrCre", "outputId": "61671516-3fd8-404e-edc2-0cdad0f23ec2" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "{'grade': 3, 'required': False, 'year': 2021, 'url': 'https://r.st.ie.u-ryukyu.ac.jp/assessment/2021a/result/makepage.php?kamoku=i334'}\n" ] } ], "source": [ "lectures = {}\n", "for items in df.itertuples():\n", " title = items[1]\n", " grade = items[2]\n", " required = items[3]\n", " year = items[4]\n", " url = items[5]\n", " lectures[title] = {'grade':grade, 'required':required, 'year':year, 'url':url}\n", "\n", "print(lectures['データマイニング'])" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "id": "ejjm4NzBrCrf" }, "outputs": [], "source": [ "# urlから '&ex=1'を削除した状態でのセレクタ\n", "# Q21-(1), (2), (3), (4), Q22\n", "selectors = {'Q21 (1)':'body > div:nth-child(44) > ul:nth-child(1)',\n", " 'Q21 (2)': 'body > div:nth-child(47) > ul:nth-child(1)',\n", " 'Q21 (3)': 'body > div:nth-child(50) > ul:nth-child(1)',\n", " 'Q21 (4)': 'body > div:nth-child(53) > ul:nth-child(1)',\n", " 'Q22': 'body > div:nth-child(56) > ul:nth-child(1)'}" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 284 }, "id": "g-xB-j3mrCrf", "outputId": "dcb0f332-f830-4acb-ffb7-7e37e631e5d1" }, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " title grade required q_id comment\n", "0 工業数学Ⅰ 1 True Q21 (1) 特になし\n", "1 工業数学Ⅰ 1 True Q21 (2) 正直わかりずらい。むだに間があるし。\n", "2 工業数学Ⅰ 1 True Q21 (2) 例題を取り入れて理解しやすくしてほしい。\n", "3 工業数学Ⅰ 1 True Q21 (2) 特になし\n", "4 工業数学Ⅰ 1 True Q21 (2) スライドに書く文字をもう少しわかりやすくして欲しいです。" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titlegraderequiredq_idcomment
0工業数学Ⅰ1TrueQ21 (1)特になし
1工業数学Ⅰ1TrueQ21 (2)正直わかりずらい。むだに間があるし。
2工業数学Ⅰ1TrueQ21 (2)例題を取り入れて理解しやすくしてほしい。
3工業数学Ⅰ1TrueQ21 (2)特になし
4工業数学Ⅰ1TrueQ21 (2)スライドに書く文字をもう少しわかりやすくして欲しいです。
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 170,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 16,\n \"samples\": [\n \"\\u5de5\\u696d\\u6570\\u5b66\\u2160\",\n \"\\u6280\\u8853\\u8005\\u306e\\u502b\\u7406\",\n \"\\u30a2\\u30eb\\u30b4\\u30ea\\u30ba\\u30e0\\u3068\\u30c7\\u30fc\\u30bf\\u69cb\\u9020\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"grade\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0,\n \"min\": 1,\n \"max\": 3,\n \"num_unique_values\": 3,\n \"samples\": [\n 1,\n 2,\n 3\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"required\",\n \"properties\": {\n \"dtype\": \"boolean\",\n \"num_unique_values\": 2,\n \"samples\": [\n false,\n true\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"q_id\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 5,\n \"samples\": [\n \"Q21 (2)\",\n \"Q22\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"comment\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 153,\n \"samples\": [\n \"\\u30fb\\u6559\\u79d1\\u66f8\\u304c\\u5fc5\\u8981\\u306a\\u306e\\u304b\\u5fc5\\u8981\\u3067\\u306a\\u3044\\u306e\\u304b\\u304c\\u66d6\\u6627\\u306a\\u307e\\u307e\\u6388\\u696d\\u304c\\u59cb\\u307e\\u308a\\u3001\\u975e\\u5e38\\u306b\\u4e0d\\u5b89\\u3060\\u3063\\u305f\\u305f\\u3081\\u3001\\u6559\\u79d1\\u66f8\\u304c\\u5fc5\\u9808\\u304b\\u305d\\u3046\\u3067\\u306a\\u3044\\u306e\\u304b\\u306f\\u6700\\u521d\\u306b\\u306f\\u3063\\u304d\\u308a\\u3057\\u3066\\u6b32\\u3057\\u3044\\u3002\\r\\n\\u30fb\\u8ab2\\u984c\\u3092\\u51fa\\u3059\\u3060\\u3051\\u51fa\\u3055\\u305b\\u3066\\u304a\\u3044\\u3066\\u3001\\u63a1\\u70b9\\u3082\\u305b\\u305a\\u3001\\u3069\\u3046\\u3044\\u3063\\u305f\\u89e3\\u7b54\\u304c\\u6b63\\u3057\\u3044\\u306e\\u304b\\u3068\\u3044\\u3063\\u305f\\u6307\\u91dd\\u3082\\u51fa\\u3059\\u306e\\u304c\\u3068\\u3066\\u3082\\u9045\\u3044\\u3002\\u8ab2\\u984c\\u306f\\u89e3\\u304f\\u3060\\u3051\\u3067\\u306f\\u77e5\\u8b58\\u306e\\u5b9a\\u7740\\u306b\\u3064\\u306a\\u304c\\u3089\\u306a\\u3044\\u3068\\u601d\\u3044\\u307e\\u3059\\u304c\\u3001\\u305d\\u3053\\u3089\\u3078\\u3093\\u306f\\u3069\\u3046\\u306a\\u3093\\u3067\\u3057\\u3087\\u3046\\u304b\\u3002\\r\\n\\u30fb\\u914d\\u5e03\\u8cc7\\u6599\\u3068\\u3057\\u3066\\u3001\\u904e\\u53bb\\u554f\\u3082\\u914d\\u5e03\\u3057\\u3066\\u304f\\u308c\\u308b\\u3068\\u3068\\u3066\\u3082\\u52a9\\u304b\\u308b\\u306a\\u3001\\u3068\\u601d\\u3044\\u307e\\u3059\\u3002\\u3054\\u691c\\u8a0e\\u304a\\u9858\\u3044\\u3057\\u307e\\u3059\\u3002\",\n \"\\u30fb\\u4e2d\\u9593\\u30c6\\u30b9\\u30c8\\u3092\\u5ef6\\u671f\\u3057\\u7d9a\\u3051\\u3001\\u6700\\u7d42\\u7684\\u306b\\u4e2d\\u9593\\u30fb\\u671f\\u672b\\u8a66\\u9a13\\u3092\\uff12\\u9031\\u7d9a\\u3051\\u3066\\u3084\\u308b\\u3053\\u3068\\u3068\\u306a\\u308a\\u3001\\u8a08\\u753b\\u6027\\u304c\\u6b20\\u3051\\u3066\\u3044\\u308b\\u3002\\r\\n\\u30fb\\u914d\\u5e03\\u8cc7\\u6599\\u306e\\u8aa4\\u5b57\\u8131\\u5b57\\u304c\\u591a\\u3059\\u304e\\u308b\\u3002\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 6 } ], "source": [ "def get_comments(lectures, selectors):\n", " \"\"\"授業コメントを収集\n", " Returns dict:\n", " {授業名: {'q_id': ['コメント1', 'コメント2']},\n", " 授業名: {'q_id': ['コメント1', 'コメント2']},,,}\n", " \"\"\"\n", " result = {}\n", " for lec_name, items in lectures.items():\n", " #print(lec_name, items['grade'])\n", " r = requests.get(items['url'])\n", " r.encoding = r.apparent_encoding\n", " soup = BeautifulSoup(r.text, 'html.parser')\n", "\n", " comments = {}\n", " for q_id, selector in selectors.items():\n", " elements = soup.select(selector)\n", " #print(elements, '<=', items['url'], q_id, selector)\n", " if len(elements) != 0:\n", " for li in elements[0].find_all('li'):\n", " if q_id in comments:\n", " comments[q_id].append(li.text.rstrip())\n", " else:\n", " comments[q_id] = [li.text.rstrip()]\n", " result[lec_name] = comments\n", " return result\n", "\n", "def comment_to_dataframe(lectures, all_comments):\n", " \"\"\"扱いやすいように変換\n", " Returns pd.DataFrame:\n", " columns = ['title', 'grade', 'required', 'q_id', 'comment']\n", " \"\"\"\n", " tables = []\n", " for title, items in lectures.items():\n", " grade = items['grade']\n", " required = items['required']\n", " for q_id, comments in all_comments[title].items():\n", " for comment in comments:\n", " if len(comment) != 0:\n", " tables.append([title, grade, required, q_id, comment])\n", "\n", " columns_name = ['title', 'grade', 'required', 'q_id', 'comment']\n", " df = pd.DataFrame(tables, columns=columns_name)\n", " return df\n", "\n", "\n", "all_comments = get_comments(lectures, selectors)\n", "df = comment_to_dataframe(lectures, all_comments)\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "id": "bgeZYyFMrCrf" }, "outputs": [], "source": [ "df.to_pickle('./r_assesment.pkl')\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "id": "IqAG27ubrCrf" }, "outputs": [], "source": [] } ], "metadata": { "interpreter": { "hash": "880b2a8c90f9e6beae80b56829e3f671fedd58b6d14887184ddce26124cedfbd" }, "kernelspec": { "display_name": "Python 3.8.9 64-bit", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.9" }, "orig_nbformat": 4, "colab": { "provenance": [] } }, "nbformat": 4, "nbformat_minor": 0 }