Word Cloudを使って見る

WordCloudなるライブラリがあるので使ってみました。
自分の環境ではそのままではちょっとうまく動かなかったのでメモです。

こちらを参考にしました。

環境

– MacOS10.11
– python 2.7.12
– mecab 0.996

インストール

brew install python
brew install mecab
brew install mecab-ipadic


git clone https://github.com/amueller/word_cloud
cd word_cloud 
pip install -r requirements.txt
python setup.py install
pip install beautifulsoup4
pip install requests

brew install python

brew install mecab

brew install mecab-ipadic

git clone https://github.com/amueller/word_cloud

cd word_cloud

pip install -r requirements.txt

python setup.py install

pip install beautifulsoup4

pip install requests

エラー

こちらのサンプルをそのまま実行するとエラーが出ます

/usr/local/lib/python2.7/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system ("html.parser"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.

The code that caused this warning is on line 53 of the file word_cloud.py. To get rid of this warning, change code that looks like this:

 BeautifulSoup([your markup])

to this:

 BeautifulSoup([your markup], "html.parser")

  markup_type=markup_type))
Traceback (most recent call last):
  File "word_cloud.py", line 53, in <module>
    wordlist = get_wordlist_from_QiitaURL(url)
  File "word_cloud.py", line 30, in get_wordlist_from_QiitaURL
    return mecab_analysis(text)
  File "word_cloud.py", line 10, in mecab_analysis
    t = mc.Tagger('-Ochasen -d /usr/local/Cellar/mecab/0.996/lib/mecab/dic/mecab-ipadic-neologd/')
  File "/usr/local/lib/python2.7/site-packages/MeCab.py", line 307, in __init__
    this = _MeCab.new_Tagger(*args)
RuntimeError

/usr/local/lib/python2.7/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system ("html.parser"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.

The code that caused this warning is on line 53 of the file word_cloud.py. To get rid of this warning, change code that looks like this:

BeautifulSoup([your markup])

to this:

BeautifulSoup([your markup], "html.parser")

markup_type=markup_type))

Traceback (most recent call last):

File "word_cloud.py", line 53, in <module>

wordlist = get_wordlist_from_QiitaURL(url)

File "word_cloud.py", line 30, in get_wordlist_from_QiitaURL

return mecab_analysis(text)

File "word_cloud.py", line 10, in mecab_analysis

t = mc.Tagger('-Ochasen -d /usr/local/Cellar/mecab/0.996/lib/mecab/dic/mecab-ipadic-neologd/')

File "/usr/local/lib/python2.7/site-packages/MeCab.py", line 307, in __init__

this = _MeCab.new_Tagger(*args)

RuntimeError

HTMLパーサーを明示的に入れます

    soup = BeautifulSoup(res.text,"html.parser")

1	soup = BeautifulSoup(res.text,"html.parser")

そのまま実行するとまたまたエラー

Traceback (most recent call last):
  File "word_cloud.py", line 59, in <module>
    create_wordcloud(" ".join(wordlist).decode('utf-8'))
  File "word_cloud.py", line 50, in create_wordcloud
    stopwords=set(stop_words)).generate(text)
  File "/usr/local/lib/python2.7/site-packages/wordcloud-1.2.1-py2.7-macosx-10.11-x86_64.egg/wordcloud/wordcloud.py", line 463, in generate
    return self.generate_from_text(text)
  File "/usr/local/lib/python2.7/site-packages/wordcloud-1.2.1-py2.7-macosx-10.11-x86_64.egg/wordcloud/wordcloud.py", line 448, in generate_from_text
    words = self.process_text(text)
  File "/usr/local/lib/python2.7/site-packages/wordcloud-1.2.1-py2.7-macosx-10.11-x86_64.egg/wordcloud/wordcloud.py", line 391, in process_text
    self.stopwords_lower_ = set(map(str.lower, self.stopwords))
TypeError: descriptor 'lower' requires a 'str' object but received a 'unicode'

Traceback (most recent call last):

File "word_cloud.py", line 59, in <module>

create_wordcloud(" ".join(wordlist).decode('utf-8'))

File "word_cloud.py", line 50, in create_wordcloud

stopwords=set(stop_words)).generate(text)

File "/usr/local/lib/python2.7/site-packages/wordcloud-1.2.1-py2.7-macosx-10.11-x86_64.egg/wordcloud/wordcloud.py", line 463, in generate

return self.generate_from_text(text)

File "/usr/local/lib/python2.7/site-packages/wordcloud-1.2.1-py2.7-macosx-10.11-x86_64.egg/wordcloud/wordcloud.py", line 448, in generate_from_text

words = self.process_text(text)

File "/usr/local/lib/python2.7/site-packages/wordcloud-1.2.1-py2.7-macosx-10.11-x86_64.egg/wordcloud/wordcloud.py", line 391, in process_text

self.stopwords_lower_ = set(map(str.lower, self.stopwords))

TypeError: descriptor 'lower' requires a 'str' object but received a 'unicode'

どうもUnicodeがらみのエラーです。stop_wordsのUnicode変換がうまくいっていないようなので普通の文字列にします

コード修正

自分の環境に合わせていじります

#!/bin/env python
# coding:utf-8
#%matplotlib inline
import urllib2
from bs4 import BeautifulSoup

import matplotlib.pyplot as plt
from wordcloud import WordCloud
from bs4 import BeautifulSoup
import requests
import MeCab as mc



def mecab_analysis(text):
    t = mc.Tagger('-Ochasen -d /usr/local/Cellar/mecab/0.996/lib/mecab/dic/ipadic/')
    enc_text = text.encode('utf-8')
    node = t.parseToNode(enc_text)
    output = []
    while(node):
        if node.surface != "":  # ヘッダとフッタを除外
            word_type = node.feature.split(",")[0]
            if word_type in ["形容詞", "動詞","名詞", "副詞"]:
                output.append(node.surface)
        node = node.next
        if node is None:
            break
    return output


def get_wordlist_from_QiitaURL(url):
    res = requests.get(url)
    soup = BeautifulSoup(res.text,"html.parser")

    text = soup.body.section.get_text().replace('\n','').replace('\t','')
    return mecab_analysis(text)

def create_wordcloud(text):

    # 環境に合わせてフォントのパスを指定する。
    #fpath = "/System/Library/Fonts/HelveticaNeue-UltraLight.otf"
    #fpath = "/Library/Fonts/ヒラギノ角ゴ Pro W3.otf"
    fpath = "/Library/Fonts/Osaka.ttf"

    # ストップワードの設定
    #stop_words = [ u'てる', u'いる', u'なる', u'れる', u'する', u'ある', u'こと
', u'これ', u'さん', u'して', u'くれる', u'やる', u'くださる', u'そう', u'せる', u'した',  u'思う',  u'それ', u'ここ', u'ちゃん', u'くん', u'', u'て',u'に',u'を
',u'は',u'の', u'が', u'と', u'た', u'し', u'で', u'ない', u'も', u'な', u'い', u'か', u'ので', u'よう', u'']
    stop_words = [ 'てる', 'いる', 'なる', 'れる', 'する', 'ある', 'こと', 'これ
', 'さん', 'して', 'くれる', 'やる', 'くださる', 'そう', 'せる', 'した',  '思う',  'それ', 'ここ', 'ちゃん', 'くん', '', 'て','に','を','は','の', 'が', 'と', 'た', 'し', 'で', 'ない', 'も', 'な', 'い', 'か', 'ので', 'よう', '']

    wordcloud = WordCloud(background_color="white",font_path=fpath, width=900, height=500, \
                          stopwords=set(stop_words)).generate(text)

    plt.figure(figsize=(15,12))
    plt.imshow(wordcloud)
    plt.axis("off")
    plt.show()

url = "http://qiita.com/t_saeko/items/2b475b8657c826abc114"
wordlist = get_wordlist_from_QiitaURL(url)
create_wordcloud(" ".join(wordlist).decode('utf-8'))

#!/bin/env python

# coding:utf-8

#%matplotlib inline

import urllib2

from bs4 import BeautifulSoup

import matplotlib.pyplot as plt

from wordcloud import WordCloud

from bs4 import BeautifulSoup

import requests

import MeCab as mc

def mecab_analysis(text):

t = mc.Tagger('-Ochasen -d /usr/local/Cellar/mecab/0.996/lib/mecab/dic/ipadic/')

enc_text = text.encode('utf-8')

node = t.parseToNode(enc_text)

output = []

while(node):

if node.surface != "": # ヘッダとフッタを除外

word_type = node.feature.split(",")[0]

if word_type in ["形容詞", "動詞","名詞", "副詞"]:

output.append(node.surface)

node = node.next

if node is None:

break

return output

def get_wordlist_from_QiitaURL(url):

res = requests.get(url)

soup = BeautifulSoup(res.text,"html.parser")

text = soup.body.section.get_text().replace('\n','').replace('\t','')

return mecab_analysis(text)

def create_wordcloud(text):

# 環境に合わせてフォントのパスを指定する。

#fpath = "/System/Library/Fonts/HelveticaNeue-UltraLight.otf"

#fpath = "/Library/Fonts/ヒラギノ角ゴ Pro W3.otf"

fpath = "/Library/Fonts/Osaka.ttf"

# ストップワードの設定

#stop_words = [ u'てる', u'いる', u'なる', u'れる', u'する', u'ある', u'こと

', u'これ', u'さん', u'して', u'くれる', u'やる', u'くださる', u'そう', u'せる', u'した', u'思う', u'それ', u'ここ', u'ちゃん', u'くん', u'', u'て',u'に',u'を

',u'は',u'の', u'が', u'と', u'た', u'し', u'で', u'ない', u'も', u'な', u'い', u'か', u'ので', u'よう', u'']

stop_words = [ 'てる', 'いる', 'なる', 'れる', 'する', 'ある', 'こと', 'これ

', 'さん', 'して', 'くれる', 'やる', 'くださる', 'そう', 'せる', 'した', '思う', 'それ', 'ここ', 'ちゃん', 'くん', '', 'て','に','を','は','の', 'が', 'と', 'た', 'し', 'で', 'ない', 'も', 'な', 'い', 'か', 'ので', 'よう', '']

wordcloud = WordCloud(background_color="white",font_path=fpath, width=900, height=500, \

stopwords=set(stop_words)).generate(text)

plt.figure(figsize=(15,12))

plt.imshow(wordcloud)

plt.axis("off")

plt.show()

url = "http://qiita.com/t_saeko/items/2b475b8657c826abc114"

wordlist = get_wordlist_from_QiitaURL(url)

create_wordcloud(" ".join(wordlist).decode('utf-8'))

実行

python word_cloud.py

1	python word_cloud.py

これで画像が表示されます

python