{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!date\n",
        "!python --version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H-OK4eyTyn71",
        "outputId": "a02fc22b-2031-4eba-a950-3b3711e6ced0"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Thu May 30 04:39:14 AM UTC 2024\n",
            "Python 3.10.12\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kbUXbd64ynZj"
      },
      "source": [
        "# テキストをトークン出現回数でベクトル化する例（Spacy版）\n",
        "基本的には「形態素解析して単語に分割し、その回数をカウントしたうえでベクトル化する」という手順を取る。形態素解析には様々なツールがあるが、ここでは ``spacy.load(\"ja_ginza\")`` を用いた例を示す。単語分割した文字列を作成したら、後は[CountVectorizer](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html)を使うのが楽だ。"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U spacy ja_ginza"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RV_v6QHZyq0H",
        "outputId": "019a44ac-ec82-4b7e-9e7f-b981a4f1c4ab"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: spacy in /usr/local/lib/python3.10/dist-packages (3.7.4)\n",
            "Collecting ja_ginza\n",
            "  Downloading ja_ginza-5.2.0-py3-none-any.whl (59.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.1/59.1 MB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.3.0,>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (8.2.3)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.0.10)\n",
            "Requirement already satisfied: weasel<0.4.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (0.3.4)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (0.9.4)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (4.66.4)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.7.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.1.4)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (24.0)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.4.0)\n",
            "Requirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.25.2)\n",
            "Collecting sudachipy<0.7.0,>=0.6.2 (from ja_ginza)\n",
            "  Downloading SudachiPy-0.6.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m51.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sudachidict-core>=20210802 (from ja_ginza)\n",
            "  Downloading SudachiDict_core-20240409-py3-none-any.whl (72.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.0/72.0 MB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ginza<5.3.0,>=5.2.0 (from ja_ginza)\n",
            "  Downloading ginza-5.2.0-py3-none-any.whl (21 kB)\n",
            "Collecting plac>=1.3.3 (from ginza<5.3.0,>=5.2.0->ja_ginza)\n",
            "  Downloading plac-1.4.3-py2.py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: language-data>=1.2 in /usr/local/lib/python3.10/dist-packages (from langcodes<4.0.0,>=3.2.0->spacy) (1.2.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (2.18.2)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy) (0.1.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy) (8.1.7)\n",
            "Requirement already satisfied: cloudpathlib<0.17.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<0.4.0,>=0.1.0->spacy) (0.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy) (2.1.5)\n",
            "Requirement already satisfied: marisa-trie>=0.7.7 in /usr/local/lib/python3.10/dist-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy) (1.1.1)\n",
            "Installing collected packages: sudachipy, plac, sudachidict-core, ginza, ja_ginza\n",
            "Successfully installed ginza-5.2.0 ja_ginza-5.2.0 plac-1.4.3 sudachidict-core-20240409 sudachipy-0.6.8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1IN265gHynZm"
      },
      "source": [
        "## spacyで単語分割\n",
        "- 単に分割するだけではなく、``token.lemma_`` により基本形に変換している。元のままが良ければ ``token.text`` にしよう。\n",
        "- タスクによっては不要な単語や品詞もあるだろう。その場合には不要なものを除外しよう。\n",
        "- タスクによっては集約（例えば数字を全て`<数字>`という単語に集約する。感情語を全て`<感情>`に集約する）を検討すると良いだろう。"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hCCN8R_kynZm",
        "outputId": "c1884d85-0a2b-4e33-f16e-4b9788dcb8cb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "token.i=0, token.text='正直', token.lemma_='正直'\n",
            "token.i=1, token.text='わかり', token.lemma_='わかる'\n",
            "token.i=2, token.text='ずらい', token.lemma_='ずらい'\n",
            "token.i=3, token.text='。', token.lemma_='。'\n"
          ]
        }
      ],
      "source": [
        "import spacy\n",
        "\n",
        "# テキスト例\n",
        "texts = ['特になし',\n",
        "        '正直わかりずらい。むだに間があるし。',\n",
        "        '例題を取り入れて理解しやすくしてほしい。']\n",
        "\n",
        "# 解析器を用意\n",
        "nlp = spacy.load(\"ja_ginza\")\n",
        "\n",
        "# 解析例\n",
        "doc = nlp(\"正直わかりずらい。\")\n",
        "for token in doc:\n",
        "    print(f\"{token.i=}, {token.text=}, {token.lemma_=}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vg45sZGVynZo",
        "outputId": "a5752658-0e56-42b7-fad8-f3e5444c416d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['特に なし', '正直 わかる ずらい 。 むだ だ 間 が ある し 。', '例題 を 取り入れる て 理解 する やすい する て ほしい 。']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "def text2tokens(nlp:spacy.language.Language, text:str, sep=' '):\n",
        "    \"\"\"テキストを単語に分割した文字列に変換。\n",
        "    args:\n",
        "      nlp: spacy.load()で用意した解析器。\n",
        "      text: テキスト。\n",
        "      sep: セパレータ。単語と単語の間を埋める記号。\n",
        "\n",
        "    >>> nlp = spacy.load(\"ja_ginza\")\n",
        "    >>> result = text2tokens(nlp, \"これはテストです\")\n",
        "    >>> result\n",
        "    'これ は テスト です'\n",
        "    \"\"\"\n",
        "    doc = nlp(text)\n",
        "    tokens = []\n",
        "    for token in doc:\n",
        "        tokens.append(token.lemma_)\n",
        "    result = sep.join(tokens)\n",
        "    return result\n",
        "\n",
        "# 実行例\n",
        "tokens = []\n",
        "for text in texts:\n",
        "    tokens.append(text2tokens(nlp, text))\n",
        "\n",
        "tokens"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BBZSPgvoynZp"
      },
      "source": [
        "## CountVectorizerでベクトル化\n",
        "デフォルトでは1-gramモデル（各単語の出現回数に基づいた特徴）によりベクトル化する。引数指定により以下のような設定も可能。[詳細はドキュメント](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html)参照。\n",
        "- `ngram_range`\n",
        "    - 2-gram, 3-gram,,,といった「連続した語＝フレーズ」に基づいたベクトル化を行う。\n",
        "- `stop_words`\n",
        "    - 無視したい単語（ストップワード）を指定することができる。標準で用意されているリストを利用することも可能。\n",
        "- `analyzer`\n",
        "    - デフォルトでは単語を特徴として捉えるが、この単語とは「スペースで区切られたもの」として解釈される。\n",
        "    - 'char' を指定すると「文字」を特徴として捉えるようになる。"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fPjGqZ5synZp",
        "outputId": "9a9d82f0-0cbd-4836-e89b-98a808e60bf8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "features=array(['ある', 'する', 'ずらい', 'なし', 'ほしい', 'むだ', 'やすい', 'わかる', '例題', '取り入れる',\n",
            "       '正直', '特に', '理解'], dtype=object)\n",
            "[[0 0 0 1 0 0 0 0 0 0 0 1 0]\n",
            " [1 0 1 0 0 1 0 1 0 0 1 0 0]\n",
            " [0 2 0 0 1 0 1 0 1 1 0 0 1]]\n"
          ]
        }
      ],
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "# 1-gramで特徴ベクトル作成\n",
        "vectorizer = CountVectorizer() # デフォルトでは単語出現回数でベクトル化\n",
        "X = vectorizer.fit_transform(tokens) # ベクトル構築\n",
        "features = vectorizer.get_feature_names_out() # ベクトル構築した際の単語一覧\n",
        "print(f\"{features=}\")\n",
        "print(X.toarray())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l7UWmF6VynZp",
        "outputId": "bf972646-24ad-4a09-da79-57603c8a8faa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "features=array(['する ほしい', 'する やすい', 'ずらい むだ', 'むだ ある', 'やすい する', 'わかる ずらい',\n",
            "       '例題 取り入れる', '取り入れる 理解', '正直 わかる', '特に なし', '理解 する'], dtype=object)\n",
            "[[0 0 0 0 0 0 0 0 0 1 0]\n",
            " [0 0 1 1 0 1 0 0 1 0 0]\n",
            " [1 1 0 0 1 0 1 1 0 0 1]]\n"
          ]
        }
      ],
      "source": [
        "# 2-gramで特徴ベクトル作成\n",
        "vectorizer = CountVectorizer(ngram_range=(2, 2))\n",
        "X = vectorizer.fit_transform(tokens)\n",
        "features = vectorizer.get_feature_names_out()\n",
        "print(f\"{features=}\")\n",
        "print(X.toarray())"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "dm",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}