김승훈 / big-data2project

Go to a project
Toggle navigation Toggle navigation pinning
김승훈
09350002 1 parent c23dc53f

update readme and file

Inline Side-by-side
Showing 11 changed files with 735 additions and 271 deletions
1. cd server
2. python pr.py ${currentpath}
3. python pr-finance.py
4. python data-visualize.py
\ No newline at end of file
#analyze between search rate and stocks correlation
##1. cd server && ipython notebook
##2. data (this data is naver trend - search rate, company name sorted by date) acquisition
and save it to csv file
### RUN pr.ipynb
##3. data (this data is crawled in naver financial by company code, it is stock sorted by date )
crawling and save it to csv file
### RUN pr-finance.ipynb
##4. data visualize to graph (as of current, there is three company visualization. later, I will visualize all company )
### RUN data-visualize.py
\ No newline at end of file
......
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
" window._Plotly = Plotly;\n",
" });\n",
" }\n",
" </script>\n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"start read_csv\n"
]
}
],
"source": [
"\n",
"import numpy as np # linear algebra\n",
"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
"from bs4 import BeautifulSoup\n",
"import urllib\n",
"from urllib import request\n",
"import re\n",
"import json\n",
"from datetime import datetime\n",
"\n",
"import plotly.offline as py\n",
"py.init_notebook_mode(connected=True)\n",
"import plotly.graph_objs as go\n",
"import plotly.tools as tls\n",
"import matplotlib.pyplot as plt\n",
"from random import shuffle\n",
"import random\n",
"#Setup\n",
"#load dataset\n",
"print(\"start read_csv\")\n",
"df = pd.read_csv(\"trendresult.csv\")\n",
"\n",
"df = pd.DataFrame(df)\n",
"\n",
"fig, axes = plt.subplots(nrows=3, ncols=2)\n",
"\n",
"\n",
"#df.plot(title = \"Naver Trend - Stock-Daily\", figsize = (20, 10), legend = False)\n",
"\n",
"print(\"start read_csv_1\")\n",
"df2 = pd.read_csv(\"finace_285130.csv\")\n",
"df2 = pd.DataFrame(df2)\n",
"print(df2)\n",
"df[\"SK케미칼\"].plot(title = \"SK-chemical\", figsize = (20,10), legend = False,ax=axes[0,0])\n",
"print(df[\"SK케미칼\"])\n",
"df2.plot(title = \"SK-chemical - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[0,1])\n",
"\n",
"df3 = pd.read_csv(\"finace_307070.csv\")\n",
"df3 = pd.DataFrame(df3)\n",
"df[\"삼성머스트스팩3호\"].plot(title = \"삼성머스트스팩3호\", figsize = (20,10), legend = False,ax=axes[1,0])\n",
"print(df3)\n",
"print(df[\"삼성머스트스팩3호\"])\n",
"df3.plot(title = \"SK hos - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[1,1])\n",
"\n",
"df4 = pd.read_csv(\"finace_309930.csv\")\n",
"df4 = pd.DataFrame(df4)\n",
"df[\"SK4호스팩\"].plot(title = \"SK-호스팩\", figsize = (20,10), legend = False,ax=axes[2,0])\n",
"print(df4)\n",
"print(df[\"SK4호스팩\"])\n",
"df4.plot(title = \"samsung - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[2,1])\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import urllib\n",
"import time\n",
"\n",
"from urllib.request import urlopen\n",
"from bs4 import BeautifulSoup\n",
"from urllib import request\n",
"import re\n",
"import json\n",
"from datetime import datetime\n",
"import os\n",
"import boto3\n",
"import time\n",
"import sys\n",
"import plotly.offline as py\n",
"import plotly.graph_objs as go\n",
"import plotly.tools as tls\n",
"import matplotlib\n",
"from random import shuffle\n",
"import numpy as np\n",
"import pandas as pd\n",
"stockItem = ['285130','309930','307070' ]\n",
"for k in range(0,3):\n",
" url = 'http://finance.naver.com/item/sise_day.nhn?code='+ stockItem[k]\n",
" html = urlopen(url) \n",
" source = BeautifulSoup(html.read(), \"html.parser\")\n",
" maxPage=source.find_all(\"table\",align=\"center\")\n",
" mp = maxPage[0].find_all(\"td\",class_=\"pgRR\")\n",
"\n",
" mpNum = int(mp[0].a.get('href')[-2:])\n",
" date = []\n",
" finance = []\n",
" df = {}\n",
" for page in range(1, mpNum+1):\n",
" print (page )\n",
" print (mpNum+1)\n",
" url = 'http://finance.naver.com/item/sise_day.nhn?code=' + stockItem[k] +'&page='+ str(page)\n",
" html = urlopen(url)\n",
" source = BeautifulSoup(html.read(), \"html.parser\")\n",
" srlists=source.find_all(\"tr\") \n",
" isCheckNone = None\n",
" if((page % 1) == 0):\n",
" time.sleep(0.1)\n",
"\n",
" for i in range(1,len(srlists)-1): \n",
" \n",
" if(srlists[i].span != isCheckNone):\n",
" srlists[i].td.text\n",
" date.append(srlists[i].find_all(\"td\",align=\"center\")[0].text)\n",
" finance.append(int(srlists[i].find_all(\"td\",class_=\"num\")[0].text.replace(',','')))\n",
" \n",
" if(page == mpNum):\n",
" date = np.array(date)\n",
" finance = np.array(finance)\n",
" df = pd.DataFrame(df)\n",
" df[\"finance\"] = finance\n",
" df[\"date\"] = date\n",
" df.set_index(\"date\", inplace=True, drop=True)\n",
" print(df)\n",
" df.to_csv(\"finace_\"+stockItem[k]+\".csv\",encoding = \"utf-8\")\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from bs4 import BeautifulSoup\n",
"import urllib\n",
"from urllib import request\n",
"import re\n",
"import json\n",
"from datetime import datetime\n",
"import os\n",
"import boto3\n",
"import time\n",
"import sys\n",
"import plotly.offline as py\n",
"import plotly.graph_objs as go\n",
"import plotly.tools as tls\n",
"import matplotlib\n",
"from random import shuffle\n",
"\n",
"py.init_notebook_mode(connected=True)\n",
"\n",
"client_id = \"sYcWggwUdtmXwGqUrzzN\"\n",
"client_secret = \"oxUqDSa22I\"\n",
"\n",
"link = \"https://openapi.naver.com/v1/datalab/search\"\n",
"requested = request.Request(link)\n",
"requested.add_header(\"X-Naver-Client-Id\",client_id)\n",
"requested.add_header(\"X-Naver-Client-Secret\",client_secret)\n",
"requested.add_header(\"Content-Type\",\"application/json\")\n",
"\n",
"df = pd.read_excel(\"index.xls\")\n",
"names = df.회사명.values\n",
"\n",
"now = datetime.now().strftime(\"%Y-%m-%d\")\n",
"body_dict = {\"startDate\":\"2017-01-01\", \n",
" \"endDate\":\"2020-05-07\",\n",
" \"timeUnit\":\"date\"}\n",
"v_list = [{\"groupName\" : i, \"keywords\" : [i]} for i in names]\n",
"df[\"대표자명\"] = df[\"대표자명\"].apply(lambda x: re.sub(r'\\(.*\\)', '', x))\n",
"df[\"대표자명\"] = df[\"대표자명\"].apply(lambda x: [re.compile('[^ㄱ-ㅣ가-힣]+').sub(\"\",x)] if len(re.compile('[^ㄱ-ㅣ가-힣]+').sub(\"\",x)) < 5 else re.findall(r\"[\\w']+\", x))\n",
"\n",
"for i in range(0, df.shape[0]):\n",
" for j in df[\"대표자명\"].values[i]:\n",
" if \"대표\" not in j and j!= \"\":\n",
" v_list[i][\"keywords\"].append(j) \n",
" \n",
"shuffle(v_list)\n",
"standard = v_list[0]\n",
"standard_keyword = standard[\"groupName\"]\n",
"list_use = v_list[1:]\n",
"\n",
"split_list = [list_use[i:i+4] for i in range(0, len(list_use), 4)]\n",
"\n",
"for i in split_list:\n",
" i.append(standard)\n",
"\n",
"sample_body = body_dict\n",
"sample_body[\"keywordGroups\"] = split_list[0]\n",
"sample_body = json.dumps(sample_body, ensure_ascii=False)\n",
"sample_response = request.urlopen(requested, data=sample_body.encode(\"utf-8\"))\n",
"\n",
"code = sample_response.getcode() \n",
"if code == 200: \n",
" sample_response_body = sample_response.read()\n",
" sample_scraped = sample_response_body.decode(\"utf-8\")\n",
"else: \n",
" print (\"Error Code:\", code)\n",
"\n",
"sample_result = json.loads(sample_scraped)\n",
"\n",
"for i in sample_result[\"results\"] :\n",
" if i[\"title\"] == standard_keyword:\n",
" sample_standard = i[\"data\"]\n",
"scale = sample_standard[0][\"ratio\"]\n",
"\n",
"df = {}\n",
"\n",
"df[standard_keyword] = np.array([i[\"ratio\"] for i in sample_standard])\n",
"length = len(df[standard_keyword])\n",
"date = np.array([i[\"period\"] for i in sample_standard])\n",
"for i in split_list:\n",
"\n",
" body_dict[\"keywordGroups\"] = i\n",
" body = json.dumps(body_dict, ensure_ascii=False)\n",
" \n",
" print(i)\n",
" response = request.urlopen(requested, data=body.encode(\"utf-8\"))\n",
" \n",
" code = response.getcode() \n",
" if code == 200: \n",
" response_body = response.read()\n",
" scraped = response_body.decode(\"utf-8\")\n",
" else: \n",
" print (\"Error Code:\", code)\n",
" \n",
" \n",
" \n",
" result = json.loads(scraped)\n",
" \n",
" for i in result[\"results\"]:\n",
" if i[\"title\"] == standard_keyword:\n",
" compare = i[\"data\"]\n",
" compare = compare[0][\"ratio\"]\n",
" \n",
" scaling = scale/compare\n",
" \n",
" for i in result[\"results\"]:\n",
" if i[\"title\"]!=standard_keyword:\n",
" value = [j[\"ratio\"]*scaling for j in i[\"data\"]]\n",
" if len(value)!=length:\n",
" value+=np.abs(length-len(value)) * [value[-1]]\n",
" df[i[\"title\"]] = np.array(value)\n",
"\n",
"df = pd.DataFrame(df)\n",
"df[\"date\"] = date\n",
"df = df.set_index(\"date\")\n",
"df.to_csv(\"trend.xls\", encoding = \"utf-8\")\n",
"\n",
"df.plot(title = \"Naver Trend - Stock-Daily\", figsize = (20, 10), legend = False)\n",
"\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
" window._Plotly = Plotly;\n",
" });\n",
" }\n",
" </script>\n",
" "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"start read_csv\n"
]
}
],
"source": [
"\n",
"import numpy as np # linear algebra\n",
"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
"from bs4 import BeautifulSoup\n",
"import urllib\n",
"from urllib import request\n",
"import re\n",
"import json\n",
"from datetime import datetime\n",
"\n",
"import plotly.offline as py\n",
"py.init_notebook_mode(connected=True)\n",
"import plotly.graph_objs as go\n",
"import plotly.tools as tls\n",
"import matplotlib.pyplot as plt\n",
"from random import shuffle\n",
"import random\n",
"#Setup\n",
"#load dataset\n",
"print(\"start read_csv\")\n",
"df = pd.read_csv(\"trendresult.csv\")\n",
"\n",
"df = pd.DataFrame(df)\n",
"\n",
"fig, axes = plt.subplots(nrows=3, ncols=2)\n",
"\n",
"\n",
"#df.plot(title = \"Naver Trend - Stock-Daily\", figsize = (20, 10), legend = False)\n",
"\n",
"print(\"start read_csv_1\")\n",
"df2 = pd.read_csv(\"finace_285130.csv\")\n",
"df2 = pd.DataFrame(df2)\n",
"print(df2)\n",
"df[\"SK케미칼\"].plot(title = \"SK-chemical\", figsize = (20,10), legend = False,ax=axes[0,0])\n",
"print(df[\"SK케미칼\"])\n",
"df2.plot(title = \"SK-chemical - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[0,1])\n",
"\n",
"df3 = pd.read_csv(\"finace_307070.csv\")\n",
"df3 = pd.DataFrame(df3)\n",
"df[\"삼성머스트스팩3호\"].plot(title = \"삼성머스트스팩3호\", figsize = (20,10), legend = False,ax=axes[1,0])\n",
"print(df3)\n",
"print(df[\"삼성머스트스팩3호\"])\n",
"df3.plot(title = \"SK hos - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[1,1])\n",
"\n",
"df4 = pd.read_csv(\"finace_309930.csv\")\n",
"df4 = pd.DataFrame(df4)\n",
"df[\"SK4호스팩\"].plot(title = \"SK-호스팩\", figsize = (20,10), legend = False,ax=axes[2,0])\n",
"print(df4)\n",
"print(df[\"SK4호스팩\"])\n",
"df4.plot(title = \"samsung - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[2,1])\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from bs4 import BeautifulSoup
import urllib
from urllib import request
import re
import json
from datetime import datetime
import plotly.offline as py
py.init_notebook_mode(connected=True)
import plotly.graph_objs as go
import plotly.tools as tls
import matplotlib.pyplot as plt
from random import shuffle
import random
#Setup
#load dataset
print("start read_csv")
df = pd.read_csv("trendresult.csv")
df = pd.DataFrame(df)
fig, axes = plt.subplots(nrows=3, ncols=2)
#df.plot(title = "Naver Trend - Stock-Daily", figsize = (20, 10), legend = False)
print("start read_csv_1")
df2 = pd.read_csv("finace_285130.csv")
df2 = pd.DataFrame(df2)
print(df2)
df["SK케미칼"].plot(title = "SK-chemical", figsize = (20,10), legend = False,ax=axes[0,0])
print(df["SK케미칼"])
df2.plot(title = "SK-chemical - Stock-Daily", figsize = (20, 10), legend = False,ax=axes[0,1])
df3 = pd.read_csv("finace_307070.csv")
df3 = pd.DataFrame(df3)
df["삼성머스트스팩3호"].plot(title = "삼성머스트스팩3호", figsize = (20,10), legend = False,ax=axes[1,0])
print(df3)
print(df["삼성머스트스팩3호"])
df3.plot(title = "SK hos - Stock-Daily", figsize = (20, 10), legend = False,ax=axes[1,1])
df4 = pd.read_csv("finace_309930.csv")
df4 = pd.DataFrame(df4)
df["SK4호스팩"].plot(title = "SK-호스팩", figsize = (20,10), legend = False,ax=axes[2,0])
print(df4)
print(df["SK4호스팩"])
df4.plot(title = "samsung - Stock-Daily", figsize = (20, 10), legend = False,ax=axes[2,1])
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import urllib\n",
"import time\n",
"\n",
"from urllib.request import urlopen\n",
"from bs4 import BeautifulSoup\n",
"from urllib import request\n",
"import re\n",
"import json\n",
"from datetime import datetime\n",
"import os\n",
"import boto3\n",
"import time\n",
"import sys\n",
"import plotly.offline as py\n",
"import plotly.graph_objs as go\n",
"import plotly.tools as tls\n",
"import matplotlib\n",
"from random import shuffle\n",
"import numpy as np\n",
"import pandas as pd\n",
"stockItem = ['285130','309930','307070' ]\n",
"for k in range(0,3):\n",
" url = 'http://finance.naver.com/item/sise_day.nhn?code='+ stockItem[k]\n",
" html = urlopen(url) \n",
" source = BeautifulSoup(html.read(), \"html.parser\")\n",
" maxPage=source.find_all(\"table\",align=\"center\")\n",
" mp = maxPage[0].find_all(\"td\",class_=\"pgRR\")\n",
"\n",
" mpNum = int(mp[0].a.get('href')[-2:])\n",
" date = []\n",
" finance = []\n",
" df = {}\n",
" for page in range(1, mpNum+1):\n",
" print (page )\n",
" print (mpNum+1)\n",
" url = 'http://finance.naver.com/item/sise_day.nhn?code=' + stockItem[k] +'&page='+ str(page)\n",
" html = urlopen(url)\n",
" source = BeautifulSoup(html.read(), \"html.parser\")\n",
" srlists=source.find_all(\"tr\") \n",
" isCheckNone = None\n",
" if((page % 1) == 0):\n",
" time.sleep(0.1)\n",
"\n",
" for i in range(1,len(srlists)-1): \n",
" \n",
" if(srlists[i].span != isCheckNone):\n",
" srlists[i].td.text\n",
" date.append(srlists[i].find_all(\"td\",align=\"center\")[0].text)\n",
" finance.append(int(srlists[i].find_all(\"td\",class_=\"num\")[0].text.replace(',','')))\n",
" \n",
" if(page == mpNum):\n",
" date = np.array(date)\n",
" finance = np.array(finance)\n",
" df = pd.DataFrame(df)\n",
" df[\"finance\"] = finance\n",
" df[\"date\"] = date\n",
" df.set_index(\"date\", inplace=True, drop=True)\n",
" print(df)\n",
" df.to_csv(\"finace_\"+stockItem[k]+\".csv\",encoding = \"utf-8\")\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
import urllib
import time
from urllib.request import urlopen
from bs4 import BeautifulSoup
from urllib import request
import re
import json
from datetime import datetime
import os
import boto3
import time
import sys
import plotly.offline as py
import plotly.graph_objs as go
import plotly.tools as tls
import matplotlib
from random import shuffle
import numpy as np
import pandas as pd
stockItem = ['285130','309930','307070' ]
for k in range(0,3):
url = 'http://finance.naver.com/item/sise_day.nhn?code='+ stockItem[k]
html = urlopen(url)
source = BeautifulSoup(html.read(), "html.parser")
maxPage=source.find_all("table",align="center")
mp = maxPage[0].find_all("td",class_="pgRR")
mpNum = int(mp[0].a.get('href')[-2:])
date = []
finance = []
df = {}
for page in range(1, mpNum+1):
print (page )
print (mpNum+1)
url = 'http://finance.naver.com/item/sise_day.nhn?code=' + stockItem[k] +'&page='+ str(page)
html = urlopen(url)
source = BeautifulSoup(html.read(), "html.parser")
srlists=source.find_all("tr")
isCheckNone = None
if((page % 1) == 0):
time.sleep(0.1)
for i in range(1,len(srlists)-1):
if(srlists[i].span != isCheckNone):
srlists[i].td.text
date.append(srlists[i].find_all("td",align="center")[0].text)
finance.append(int(srlists[i].find_all("td",class_="num")[0].text.replace(',','')))
if(page == mpNum):
date = np.array(date)
finance = np.array(finance)
df = pd.DataFrame(df)
df["finance"] = finance
df["date"] = date
df.set_index("date", inplace=True, drop=True)
print(df)
df.to_csv("finace_"+stockItem[k]+".csv",encoding = "utf-8")
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from bs4 import BeautifulSoup\n",
"import urllib\n",
"from urllib import request\n",
"import re\n",
"import json\n",
"from datetime import datetime\n",
"import os\n",
"import boto3\n",
"import time\n",
"import sys\n",
"import plotly.offline as py\n",
"import plotly.graph_objs as go\n",
"import plotly.tools as tls\n",
"import matplotlib\n",
"from random import shuffle\n",
"\n",
"py.init_notebook_mode(connected=True)\n",
"\n",
"client_id = \"sYcWggwUdtmXwGqUrzzN\"\n",
"client_secret = \"oxUqDSa22I\"\n",
"\n",
"link = \"https://openapi.naver.com/v1/datalab/search\"\n",
"requested = request.Request(link)\n",
"requested.add_header(\"X-Naver-Client-Id\",client_id)\n",
"requested.add_header(\"X-Naver-Client-Secret\",client_secret)\n",
"requested.add_header(\"Content-Type\",\"application/json\")\n",
"\n",
"df = pd.read_excel(\"index.xls\")\n",
"names = df.회사명.values\n",
"\n",
"now = datetime.now().strftime(\"%Y-%m-%d\")\n",
"body_dict = {\"startDate\":\"2017-01-01\", \n",
" \"endDate\":\"2020-05-07\",\n",
" \"timeUnit\":\"date\"}\n",
"v_list = [{\"groupName\" : i, \"keywords\" : [i]} for i in names]\n",
"df[\"대표자명\"] = df[\"대표자명\"].apply(lambda x: re.sub(r'\\(.*\\)', '', x))\n",
"df[\"대표자명\"] = df[\"대표자명\"].apply(lambda x: [re.compile('[^ㄱ-ㅣ가-힣]+').sub(\"\",x)] if len(re.compile('[^ㄱ-ㅣ가-힣]+').sub(\"\",x)) < 5 else re.findall(r\"[\\w']+\", x))\n",
"\n",
"for i in range(0, df.shape[0]):\n",
" for j in df[\"대표자명\"].values[i]:\n",
" if \"대표\" not in j and j!= \"\":\n",
" v_list[i][\"keywords\"].append(j) \n",
" \n",
"shuffle(v_list)\n",
"standard = v_list[0]\n",
"standard_keyword = standard[\"groupName\"]\n",
"list_use = v_list[1:]\n",
"\n",
"split_list = [list_use[i:i+4] for i in range(0, len(list_use), 4)]\n",
"\n",
"for i in split_list:\n",
" i.append(standard)\n",
"\n",
"sample_body = body_dict\n",
"sample_body[\"keywordGroups\"] = split_list[0]\n",
"sample_body = json.dumps(sample_body, ensure_ascii=False)\n",
"sample_response = request.urlopen(requested, data=sample_body.encode(\"utf-8\"))\n",
"\n",
"code = sample_response.getcode() \n",
"if code == 200: \n",
" sample_response_body = sample_response.read()\n",
" sample_scraped = sample_response_body.decode(\"utf-8\")\n",
"else: \n",
" print (\"Error Code:\", code)\n",
"\n",
"sample_result = json.loads(sample_scraped)\n",
"\n",
"for i in sample_result[\"results\"] :\n",
" if i[\"title\"] == standard_keyword:\n",
" sample_standard = i[\"data\"]\n",
"scale = sample_standard[0][\"ratio\"]\n",
"\n",
"df = {}\n",
"\n",
"df[standard_keyword] = np.array([i[\"ratio\"] for i in sample_standard])\n",
"length = len(df[standard_keyword])\n",
"date = np.array([i[\"period\"] for i in sample_standard])\n",
"for i in split_list:\n",
"\n",
" body_dict[\"keywordGroups\"] = i\n",
" body = json.dumps(body_dict, ensure_ascii=False)\n",
" \n",
" print(i)\n",
" response = request.urlopen(requested, data=body.encode(\"utf-8\"))\n",
" \n",
" code = response.getcode() \n",
" if code == 200: \n",
" response_body = response.read()\n",
" scraped = response_body.decode(\"utf-8\")\n",
" else: \n",
" print (\"Error Code:\", code)\n",
" \n",
" \n",
" \n",
" result = json.loads(scraped)\n",
" \n",
" for i in result[\"results\"]:\n",
" if i[\"title\"] == standard_keyword:\n",
" compare = i[\"data\"]\n",
" compare = compare[0][\"ratio\"]\n",
" \n",
" scaling = scale/compare\n",
" \n",
" for i in result[\"results\"]:\n",
" if i[\"title\"]!=standard_keyword:\n",
" value = [j[\"ratio\"]*scaling for j in i[\"data\"]]\n",
" if len(value)!=length:\n",
" value+=np.abs(length-len(value)) * [value[-1]]\n",
" df[i[\"title\"]] = np.array(value)\n",
"\n",
"df = pd.DataFrame(df)\n",
"df[\"date\"] = date\n",
"df = df.set_index(\"date\")\n",
"df.to_csv(\"trend.xls\", encoding = \"utf-8\")\n",
"\n",
"df.plot(title = \"Naver Trend - Stock-Daily\", figsize = (20, 10), legend = False)\n",
"\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
import urllib
from urllib import request
import re
import json
from datetime import datetime
import os
import boto3
import time
import sys
import plotly.offline as py
import plotly.graph_objs as go
import plotly.tools as tls
import matplotlib
from random import shuffle
py.init_notebook_mode(connected=True)
def upload(s3, local_file_path, bucket, obj):
s3.upload_file(local_file_path,bucket,obj)
def make_public_read(s3,bucket,key):
s3.put_object_acl(ACL="public-read",Bucket=bucket, Key=key)
def download(s3,bucket, obj, local_file_path):
s3.download_file(bucket,obj, local_file_path)
client_id = "sYcWggwUdtmXwGqUrzzN"
client_secret = "oxUqDSa22I"
link = "https://openapi.naver.com/v1/datalab/search"
requested = request.Request(link)
requested.add_header("X-Naver-Client-Id",client_id)
requested.add_header("X-Naver-Client-Secret",client_secret)
requested.add_header("Content-Type","application/json")
df = pd.read_excel("index.xls")
names = df.회사명.values
now = datetime.now().strftime("%Y-%m-%d")
body_dict = {"startDate":"2017-01-01",
"endDate":"2020-05-07",
"timeUnit":"date"}
v_list = [{"groupName" : i, "keywords" : [i]} for i in names]
df["대표자명"] = df["대표자명"].apply(lambda x: re.sub(r'\(.*\)', '', x))
df["대표자명"] = df["대표자명"].apply(lambda x: [re.compile('[^ㄱ-ㅣ가-힣]+').sub("",x)] if len(re.compile('[^ㄱ-ㅣ가-힣]+').sub("",x)) < 5 else re.findall(r"[\w']+", x))
for i in range(0, df.shape[0]):
for j in df["대표자명"].values[i]:
if "대표" not in j and j!= "":
v_list[i]["keywords"].append(j)
shuffle(v_list)
standard = v_list[0]
standard_keyword = standard["groupName"]
list_use = v_list[1:]
split_list = [list_use[i:i+4] for i in range(0, len(list_use), 4)]
for i in split_list:
i.append(standard)
sample_body = body_dict
sample_body["keywordGroups"] = split_list[0]
sample_body = json.dumps(sample_body, ensure_ascii=False)
sample_response = request.urlopen(requested, data=sample_body.encode("utf-8"))
code = sample_response.getcode()
if code == 200:
sample_response_body = sample_response.read()
sample_scraped = sample_response_body.decode("utf-8")
else:
print ("Error Code:", code)
sample_result = json.loads(sample_scraped)
for i in sample_result["results"] :
if i["title"] == standard_keyword:
sample_standard = i["data"]
scale = sample_standard[0]["ratio"]
df = {}
df[standard_keyword] = np.array([i["ratio"] for i in sample_standard])
length = len(df[standard_keyword])
date = np.array([i["period"] for i in sample_standard])
for i in split_list:
body_dict["keywordGroups"] = i
body = json.dumps(body_dict, ensure_ascii=False)
print(i)
response = request.urlopen(requested, data=body.encode("utf-8"))
code = response.getcode()
if code == 200:
response_body = response.read()
scraped = response_body.decode("utf-8")
else:
print ("Error Code:", code)
result = json.loads(scraped)
for i in result["results"]:
if i["title"] == standard_keyword:
compare = i["data"]
compare = compare[0]["ratio"]
scaling = scale/compare
for i in result["results"]:
if i["title"]!=standard_keyword:
value = [j["ratio"]*scaling for j in i["data"]]
if len(value)!=length:
value+=np.abs(length-len(value)) * [value[-1]]
df[i["title"]] = np.array(value)
df = pd.DataFrame(df)
df["date"] = date
df = df.set_index("date")
df.to_csv("trend.xls", encoding = "utf-8")
session = boto3.Session(profile_name = "class")
s3 = session.client("s3")
bucket = 'khu-big-data-ksh'
path = sys.argv[1].replace('\\','/')
filelist = os.listdir(path)
date = time.localtime()
if(path[len(path)-1] != '/'):
path +='/'
year = str(date.tm_year)
month = str(date.tm_mon)
day = str(date.tm_mday)
if(len(month) ==1):
month = '0'+month
if(len(day) ==1):
day = '0'+day
current_date = year+"/"+month+"-"+day
try:
if(sys.argv[3] =='1'):
os.mkdir(path+'downloads/')
except:
print("")
for element in filelist:
if(os.path.isdir(path+element)):
continue
upload(s3, path+element, bucket, current_date+"/"+element)
make_public_read(s3,bucket,current_date+"/"+element)
df.plot(title = "Naver Trend - Stock-Daily", figsize = (20, 10), legend = False)
No preview for this file type