import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
%matplotlib inline

# 한글 문제 
import matplotlib.font_manager as fm
# 폰트 적용
font_location = '/usr/share/fonts/truetype/nanum/NanumBarunGothic.ttf'
font_name = fm.FontProperties(fname=font_location).get_name()

from matplotlib import rc
rc('font', family=font_name)

from bs4 import BeautifulSoup
import requests
import time

url = 'http://kin.naver.com/search/list.nhn?query={key_word}&page={num}'
url = url.format(
    key_word='여친 선물',
    num=1
)
url

response = requests.get(
    url,
    headers={ 'referer' : 'http://www.naver.com/' }
)

soup = BeautifulSoup(
    response.text,
    'lxml'
)

query_links = soup.find_all('dt')
query_links[0]

query_links[0].a['href']

url_query = query_links[0].a['href']

response_query = requests.get( query_links[0].a['href'] )

soup_tmp = BeautifulSoup( response_query.text, 'lxml' )

soup_tmp.find_all('div', '_endContentsText')

# 답변 페이지 url 모으기

url = 'http://kin.naver.com/search/list.nhn?query={key_word}&page={num}'
query_links = []

for n in range(1,10):
    url = url.format( key_word='여친 선물', num=n )
    response_query = requests.get(url)
    soup = BeautifulSoup( response_query.text, 'lxml' )
    query_links.extend( soup.find_all('dt') )
    
query_links[0].a['href']

import time
# 글 모으기


girl_friend_text = []

for each_link in query_links:
    res = requests.get(each_link.a['href'])
    soup_tmp = BeautifulSoup( res.text, 'lxml' )
    
    query_result = soup_tmp.find_all( 'div',  '_endContentsText' )
    time.sleep(1)
    
    for each in query_result:
        girl_friend_text.append( each.get_text() )

girl_friend_text[0]

len( girl_friend_text )

# 리스트의 각 글을 하나의 텍스트로 만들기

present_text = ""
for each in girl_friend_text:
    present_text += each + '\n'

present_text[:500]

import nltk
from konlpy.tag import Twitter


t = Twitter()

tokens_ko = t.morphs(present_text)
ko = nltk.Text( 
                      tokens_ko,
                      name='여자 친구 선물'
              )

ko.vocab().most_common(10)

stop_words_list = []
for m, n in ko.vocab().most_common(500):
    stop_words_list.append(m)
    
print( stop_words_list[ :50 ])

stop_words = ['요', '이', '도', '\xa0', '을', '.', '에', ',', '은', '것', '좋', '가', '로', '..', '고', 
  '는', '면', '~', '거', '를', '다', '?', '괜찮', '주', '서', '니다', '에서', '수', '만', '지', '한', '있', 
 '시', '들', '있는', '나', '일', '는데', '되', '분', '저', '습니다', '같아', '라고', '제', '!', '까', '데요', 
 '했', '개', '인데', '아요', '의', '때', '랑', '어요', '입니', '있어', '전', '게', '많이', '합', '야', '같네',
 '아', '알', '아니', '줘', '속', '중', '다른', '다면', '같은', '건', '적', '드리', '세요', '같이', '라구요',
 '...', 'ㅎㅎ', '넣어', '안', '어서', '넣', '있구', '님', '1', '하고', '데', '거든요', '좀', '으로', '실', 
'라', '안녕하세', '이런', '어떨', '않', '사', '어떠', '해서', '드려', '까지', '보시', '던', '길', '날', 
'있으', '다고', '하는', '음', '대', '어떤', '정도', '이나', '면서', '않고', '었', '구요', '그냥', '등', 
'와', '에요', '네이버', '추천해', '하시', '인', '없고', '듯', '3', '이랑', '더라구요', '질문', '여', 
'글구', '등등', '보고', '2', '만든', '하세', '또는', '일단', '생각하시', '한번', '쯤', '될', '이에요', 
'넘', '해', '배송', '제품', '원', '챙겨', '잘', '줬', '~^^', '가지', '항상', '마다', '20', '하니', 
'없어', '스', '보다', '역시',  '또', '으', '두', '해보세', '뭘', '너무', '엄청', '니', ')', '예', '!!', '걸', 
'ㅠㅠ', '을까', '알려', '보니', '답변', '볼게', '곳', '예요', '혹시', 'D', '하면', '용', '으로도',
'자신', '들어가서', '엮어', '나가는', '한마디', '책속', '바뀌', '받고', '얼마나', '웃었', '는지', 
'맨', '앞장', '아직', '해보셨', '한데', '9',  '첨부', '하구', '얼마', 'ㅋㅋㅋ', '^^', '이번', '싶은', '라는',
'하루', '만에', '오더', '아이디', '참조','에게', '있고', '과', '을까요', '더', '주세', '[', '마', ']',   
'대비', '여성', '근데', '요즘', '없이', '굉장히', '볼때', '왜', '건성', '기를', '이었', '꾸게', '해준', '다는',             
'겠', '을지', '해주셔', '상품', '같', '하셨', '지만', '잖', '사용하는','좋아할', '일정', '10', '일도', '큰',
'끈', '물이', '밑', '되는', '감', '완전', '재료', '원료', '아마', '다음', '뿐', '답', '실용', '게요',
'작은', '부탁드립니', '좋아하고', '겨울', '이니까', '(', '생활', '세트', '셔', '털', '몇', '매일', '즐겁',
'봤', '세', '한테', '하신', '찾으', '신', '아주', '골고루', '많', '올려', '됩', '해보시', '된', '고요', 
'보이', '찾고', '군요', '‘', '시간', '사람', '"', '네', '케이스', '관심', '해줬', '할','비교', '하', '가요', 
'ㅋㅋ', '\xa0\xa0', '이제', '그', '따로', '맞추는', '무난', '히', '쓸','애인', '자', '보','더니', '기념할',    
'선택', '이미지', '원하시', '많은', '보다는', '+', '000', '이라', '서울', '계신', "'", '해주고', 
'기대', '치가', '높아져', '200', '엔', '겟', '부탁드려', '비싸지', '오래', '모든', '남자',
'분들', '아래', '보는', '에서도', '처음', '들어', '꺼', '만원', '게게', '겔', '...?', '도하', '수공', '_', 
'아이디어', '정말', '푸석푸석', '이용하여', '뿌려', '그러', '는걸', '가만히', '놔두면', '당김', 
'느껴진', '이러므로', '감소', '시킬', '그런', '나이', '초', '나요', '스무살', '보다도', '특정', 
'색조', '품은', '개인', '차마', '선', '선물', '여친', '생일', '여자친구', '여자', '주소', '크리스마스',
'화이트데이', '특별한', '줄', '모르', '추천'
             ]

tokens_ko = [ each_word for each_word in tokens_ko if each_word not in stop_words]

ko = nltk.Text( tokens_ko,  name='여자 친구 선물')

plt.figure( figsize=(15, 6) )

ko.plot(50)
plt.show()

from wordcloud import WordCloud, STOPWORDS
from PIL import Image

from wordcloud import ImageColorGenerator

mask = np.array( Image.open("./data_science/heart.jpg"))

image_colors = ImageColorGenerator(mask)

data = ko.vocab().most_common(200)

wordcloud = WordCloud(
    font_path= '/usr/share/fonts/truetype/nanum/NanumBarunGothic.ttf',
    relative_scaling=0.2,
    mask=mask,
    stopwords=stop_words,
    background_color='white',
    margin=10,
    random_state=1,
    min_font_size=1,
    max_font_size=100
).generate_from_frequencies( dict(data) )

default_colors = wordcloud.to_array()

plt.figure( figsize=(12, 12) )
plt.imshow(
    wordcloud, 
    interpolation='bilinear'
)
plt.axis("off")
plt.show()

import gensim
from gensim.models import word2vec

twitter=Twitter()
results = []
lines = present_text

for line in lines:
    malist = twitter.pos( line, norm=True, stem=True )
    r=[]
    for word in malist:
        if not word[1] in ["Josa", "Eomi", "Punctuation"]:
            r.append( word[0] )
    r1=("".join(r)).strip()
    results.append(r1)        

results[ 100 : 110]

data_file = 'present_girl_friend.data'
with open(data_file, 'w', encoding='utf-8') as fp:
    fp.write("\n".join(results))
    
data = word2vec.LineSentence( data_file )    
model = word2vec.Word2Vec(
    data,
    size=200,
    window=10,
    hs=1,
    min_count=2,
    sg=1
)

model.save('present_girl_friend.model')

model = word2vec.Word2Vec.load('present_girl_friend.model')

model.most_similar( positive=['선'] )