{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!date\n",
        "!python --version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H-OK4eyTyn71",
        "outputId": "1f3a6164-4423-49d0-def7-c6c1af62a5d3"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Wed Jun  3 02:03:09 AM UTC 2026\n",
            "Python 3.12.13\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kbUXbd64ynZj"
      },
      "source": [
        "# テキストをトークン出現回数でベクトル化する例（Spacy版）\n",
        "基本的には「形態素解析して単語に分割し、その回数をカウントしたうえでベクトル化する」という手順を取る。形態素解析には様々なツールがあるが、ここでは ``spacy.load(\"ja_ginza\")`` を用いた例を示す。単語分割した文字列を作成したら、後は[CountVectorizer](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html)を使うのが楽だ。"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U spacy ja_ginza"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RV_v6QHZyq0H",
        "outputId": "9a3f6c83-a990-4c89-c8e4-2345916b6654"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: spacy in /usr/local/lib/python3.12/dist-packages (3.8.14)\n",
            "Collecting ja_ginza\n",
            "  Downloading ja_ginza-5.2.0-py3-none-any.whl.metadata (5.8 kB)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.12/dist-packages (from spacy) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (1.0.15)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.12/dist-packages (from spacy) (2.0.13)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.12/dist-packages (from spacy) (3.0.13)\n",
            "Requirement already satisfied: thinc<8.4.0,>=8.3.12 in /usr/local/lib/python3.12/dist-packages (from spacy) (8.3.13)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.12/dist-packages (from spacy) (1.1.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.5.3 in /usr/local/lib/python3.12/dist-packages (from spacy) (2.5.3)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.12/dist-packages (from spacy) (2.0.10)\n",
            "Requirement already satisfied: weasel<2.0.0,>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (1.0.0)\n",
            "Requirement already satisfied: confection<2.0.0,>=1.3.2 in /usr/local/lib/python3.12/dist-packages (from spacy) (1.3.3)\n",
            "Requirement already satisfied: typer<1.0.0,>=0.3.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (0.25.1)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (4.67.3)\n",
            "Requirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (2.0.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (2.32.4)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (2.12.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from spacy) (3.1.6)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from spacy) (75.2.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from spacy) (26.2)\n",
            "Collecting sudachipy<0.7.0,>=0.6.2 (from ja_ginza)\n",
            "  Downloading sudachipy-0.6.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (12 kB)\n",
            "Collecting sudachidict-core>=20210802 (from ja_ginza)\n",
            "  Downloading sudachidict_core-20260428-py3-none-any.whl.metadata (2.7 kB)\n",
            "Collecting ginza<5.3.0,>=5.2.0 (from ja_ginza)\n",
            "  Downloading ginza-5.2.0-py3-none-any.whl.metadata (448 bytes)\n",
            "Collecting plac>=1.3.3 (from ginza<5.3.0,>=5.2.0->ja_ginza)\n",
            "  Downloading plac-1.4.5-py2.py3-none-any.whl.metadata (5.9 kB)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.0.0->spacy) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.41.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.0.0->spacy) (2.41.4)\n",
            "Requirement already satisfied: typing-extensions>=4.14.1 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.0.0->spacy) (4.15.0)\n",
            "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.0.0->spacy) (0.4.2)\n",
            "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.4.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.15)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2026.5.20)\n",
            "Requirement already satisfied: blis<1.4.0,>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from thinc<8.4.0,>=8.3.12->spacy) (1.3.3)\n",
            "Requirement already satisfied: click>=8.2.1 in /usr/local/lib/python3.12/dist-packages (from typer<1.0.0,>=0.3.0->spacy) (8.4.0)\n",
            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from typer<1.0.0,>=0.3.0->spacy) (1.5.4)\n",
            "Requirement already satisfied: rich>=13.8.0 in /usr/local/lib/python3.12/dist-packages (from typer<1.0.0,>=0.3.0->spacy) (13.9.4)\n",
            "Requirement already satisfied: annotated-doc>=0.0.2 in /usr/local/lib/python3.12/dist-packages (from typer<1.0.0,>=0.3.0->spacy) (0.0.4)\n",
            "Requirement already satisfied: cloudpathlib>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from weasel<2.0.0,>=1.0.0->spacy) (0.24.0)\n",
            "Requirement already satisfied: smart-open>=5.2.1 in /usr/local/lib/python3.12/dist-packages (from weasel<2.0.0,>=1.0.0->spacy) (7.6.1)\n",
            "Requirement already satisfied: httpx>=0.24.0 in /usr/local/lib/python3.12/dist-packages (from weasel<2.0.0,>=1.0.0->spacy) (0.28.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->spacy) (3.0.3)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.12/dist-packages (from httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy) (4.13.0)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy) (1.0.9)\n",
            "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy) (0.16.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich>=13.8.0->typer<1.0.0,>=0.3.0->spacy) (4.2.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich>=13.8.0->typer<1.0.0,>=0.3.0->spacy) (2.20.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.12/dist-packages (from smart-open>=5.2.1->weasel<2.0.0,>=1.0.0->spacy) (2.2.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich>=13.8.0->typer<1.0.0,>=0.3.0->spacy) (0.1.2)\n",
            "Downloading ja_ginza-5.2.0-py3-none-any.whl (59.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.1/59.1 MB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading ginza-5.2.0-py3-none-any.whl (21 kB)\n",
            "Downloading sudachidict_core-20260428-py3-none-any.whl (72.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.2/72.2 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading sudachipy-0.6.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (1.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m24.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading plac-1.4.5-py2.py3-none-any.whl (22 kB)\n",
            "Installing collected packages: sudachipy, plac, sudachidict-core, ginza, ja_ginza\n",
            "Successfully installed ginza-5.2.0 ja_ginza-5.2.0 plac-1.4.5 sudachidict-core-20260428 sudachipy-0.6.11\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1IN265gHynZm"
      },
      "source": [
        "## spacyで単語分割\n",
        "- 単に分割するだけではなく、``token.lemma_`` により基本形に変換している。元のままが良ければ ``token.text`` にしよう。\n",
        "- タスクによっては不要な単語や品詞もあるだろう。その場合には不要なものを除外しよう。\n",
        "- タスクによっては集約（例えば数字を全て`<数字>`という単語に集約する。感情語を全て`<感情>`に集約する）を検討すると良いだろう。"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hCCN8R_kynZm",
        "outputId": "209e2e1b-7429-4757-ff9a-600cde389178"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "token.i=0, token.text='正直', token.lemma_='正直'\n",
            "token.i=1, token.text='わかり', token.lemma_='わかる'\n",
            "token.i=2, token.text='ずらい', token.lemma_='ずらい'\n",
            "token.i=3, token.text='。', token.lemma_='。'\n"
          ]
        }
      ],
      "source": [
        "import spacy\n",
        "\n",
        "# テキスト例\n",
        "texts = ['特になし',\n",
        "        '正直わかりずらい。むだに間があるし。',\n",
        "        '例題を取り入れて理解しやすくしてほしい。']\n",
        "\n",
        "# 解析器を用意\n",
        "# Python 3.12 + Spacy 3.8 + Ginza 5.2 の構成だとそのままでは動作しないため、\n",
        "# 以下の設定を追加指定\n",
        "config = {\n",
        "    \"components\": {\n",
        "        \"compound_splitter\": {\n",
        "            \"split_mode\": \"A\"\n",
        "        }\n",
        "    }\n",
        "}\n",
        "nlp = spacy.load(\"ja_ginza\", config=config)\n",
        "\n",
        "# 解析例\n",
        "doc = nlp(\"正直わかりずらい。\")\n",
        "for token in doc:\n",
        "    print(f\"{token.i=}, {token.text=}, {token.lemma_=}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vg45sZGVynZo",
        "outputId": "87ddd2d3-b1e4-44ac-fa7c-f90380e9751c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['特に なし', '正直 わかる ずらい 。 むだ だ 間 が ある し 。', '例題 を 取り入れる て 理解 する やすい する て ほしい 。']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "def text2tokens(nlp:spacy.language.Language, text:str, sep=' '):\n",
        "    \"\"\"テキストを単語に分割した文字列に変換。\n",
        "    args:\n",
        "      nlp: spacy.load()で用意した解析器。\n",
        "      text: テキスト。\n",
        "      sep: セパレータ。単語と単語の間を埋める記号。\n",
        "\n",
        "    >>> nlp = spacy.load(\"ja_ginza\")\n",
        "    >>> result = text2tokens(nlp, \"これはテストです\")\n",
        "    >>> result\n",
        "    'これ は テスト です'\n",
        "    \"\"\"\n",
        "    doc = nlp(text)\n",
        "    tokens = []\n",
        "    for token in doc:\n",
        "        tokens.append(token.lemma_)\n",
        "    result = sep.join(tokens)\n",
        "    return result\n",
        "\n",
        "# 実行例\n",
        "tokens = []\n",
        "for text in texts:\n",
        "    tokens.append(text2tokens(nlp, text))\n",
        "\n",
        "tokens"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BBZSPgvoynZp"
      },
      "source": [
        "## CountVectorizerでベクトル化\n",
        "デフォルトでは1-gramモデル（各単語の出現回数に基づいた特徴）によりベクトル化する。引数指定により以下のような設定も可能。[詳細はドキュメント](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html)参照。\n",
        "- `ngram_range`\n",
        "    - 2-gram, 3-gram,,,といった「連続した語＝フレーズ」に基づいたベクトル化を行う。\n",
        "- `stop_words`\n",
        "    - 無視したい単語（ストップワード）を指定することができる。標準で用意されているリストを利用することも可能。\n",
        "- `analyzer`\n",
        "    - デフォルトでは単語を特徴として捉えるが、この単語とは「スペースで区切られたもの」として解釈される。\n",
        "    - 'char' を指定すると「文字」を特徴として捉えるようになる。"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fPjGqZ5synZp",
        "outputId": "df038322-4ed2-4275-883c-e0bb49a925b1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "features=array(['ある', 'する', 'ずらい', 'なし', 'ほしい', 'むだ', 'やすい', 'わかる', '例題', '取り入れる',\n",
            "       '正直', '特に', '理解'], dtype=object)\n",
            "[[0 0 0 1 0 0 0 0 0 0 0 1 0]\n",
            " [1 0 1 0 0 1 0 1 0 0 1 0 0]\n",
            " [0 2 0 0 1 0 1 0 1 1 0 0 1]]\n"
          ]
        }
      ],
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "# 1-gramで特徴ベクトル作成\n",
        "vectorizer = CountVectorizer() # デフォルトでは単語出現回数でベクトル化\n",
        "X = vectorizer.fit_transform(tokens) # ベクトル構築\n",
        "features = vectorizer.get_feature_names_out() # ベクトル構築した際の単語一覧\n",
        "print(f\"{features=}\")\n",
        "print(X.toarray())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l7UWmF6VynZp",
        "outputId": "c64a7fbb-df51-4223-d1b6-4b75db0a0869"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "features=array(['する ほしい', 'する やすい', 'ずらい むだ', 'むだ ある', 'やすい する', 'わかる ずらい',\n",
            "       '例題 取り入れる', '取り入れる 理解', '正直 わかる', '特に なし', '理解 する'], dtype=object)\n",
            "[[0 0 0 0 0 0 0 0 0 1 0]\n",
            " [0 0 1 1 0 1 0 0 1 0 0]\n",
            " [1 1 0 0 1 0 1 1 0 0 1]]\n"
          ]
        }
      ],
      "source": [
        "# 2-gramで特徴ベクトル作成\n",
        "vectorizer = CountVectorizer(ngram_range=(2, 2))\n",
        "X = vectorizer.fit_transform(tokens)\n",
        "features = vectorizer.get_feature_names_out()\n",
        "print(f\"{features=}\")\n",
        "print(X.toarray())"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "dm",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}