김승훈 / big-data2project

Go to a project
Toggle navigation Toggle navigation pinning
김승훈
09350002 1 parent c23dc53f

update readme and file

Inline Side-by-side
Showing 11 changed files with 735 additions and 271 deletions
1 -1. cd server
2 -2. python pr.py ${currentpath}
3 -3. python pr-finance.py
4 -4. python data-visualize.py
...\ No newline at end of file ...\ No newline at end of file
1 +#analyze between search rate and stocks correlation
2 +
3 +
4 +##1. cd server && ipython notebook
5 +
6 +##2. data (this data is naver trend - search rate, company name sorted by date) acquisition
7 +and save it to csv file
8 +### RUN pr.ipynb
9 +
10 +
11 +##3. data (this data is crawled in naver financial by company code, it is stock sorted by date )
12 +crawling and save it to csv file
13 +### RUN pr-finance.ipynb
14 +
15 +
16 +##4. data visualize to graph (as of current, there is three company visualization. later, I will visualize all company )
17 +### RUN data-visualize.py
...\ No newline at end of file ...\ No newline at end of file
......
1 +{
2 + "cells": [
3 + {
4 + "cell_type": "code",
5 + "execution_count": null,
6 + "metadata": {},
7 + "outputs": [
8 + {
9 + "data": {
10 + "text/html": [
11 + " <script type=\"text/javascript\">\n",
12 + " window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
13 + " if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
14 + " if (typeof require !== 'undefined') {\n",
15 + " require.undef(\"plotly\");\n",
16 + " requirejs.config({\n",
17 + " paths: {\n",
18 + " 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
19 + " }\n",
20 + " });\n",
21 + " require(['plotly'], function(Plotly) {\n",
22 + " window._Plotly = Plotly;\n",
23 + " });\n",
24 + " }\n",
25 + " </script>\n",
26 + " "
27 + ]
28 + },
29 + "metadata": {},
30 + "output_type": "display_data"
31 + },
32 + {
33 + "name": "stdout",
34 + "output_type": "stream",
35 + "text": [
36 + "start read_csv\n"
37 + ]
38 + }
39 + ],
40 + "source": [
41 + "\n",
42 + "import numpy as np # linear algebra\n",
43 + "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
44 + "from bs4 import BeautifulSoup\n",
45 + "import urllib\n",
46 + "from urllib import request\n",
47 + "import re\n",
48 + "import json\n",
49 + "from datetime import datetime\n",
50 + "\n",
51 + "import plotly.offline as py\n",
52 + "py.init_notebook_mode(connected=True)\n",
53 + "import plotly.graph_objs as go\n",
54 + "import plotly.tools as tls\n",
55 + "import matplotlib.pyplot as plt\n",
56 + "from random import shuffle\n",
57 + "import random\n",
58 + "#Setup\n",
59 + "#load dataset\n",
60 + "print(\"start read_csv\")\n",
61 + "df = pd.read_csv(\"trendresult.csv\")\n",
62 + "\n",
63 + "df = pd.DataFrame(df)\n",
64 + "\n",
65 + "fig, axes = plt.subplots(nrows=3, ncols=2)\n",
66 + "\n",
67 + "\n",
68 + "#df.plot(title = \"Naver Trend - Stock-Daily\", figsize = (20, 10), legend = False)\n",
69 + "\n",
70 + "print(\"start read_csv_1\")\n",
71 + "df2 = pd.read_csv(\"finace_285130.csv\")\n",
72 + "df2 = pd.DataFrame(df2)\n",
73 + "print(df2)\n",
74 + "df[\"SK케미칼\"].plot(title = \"SK-chemical\", figsize = (20,10), legend = False,ax=axes[0,0])\n",
75 + "print(df[\"SK케미칼\"])\n",
76 + "df2.plot(title = \"SK-chemical - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[0,1])\n",
77 + "\n",
78 + "df3 = pd.read_csv(\"finace_307070.csv\")\n",
79 + "df3 = pd.DataFrame(df3)\n",
80 + "df[\"삼성머스트스팩3호\"].plot(title = \"삼성머스트스팩3호\", figsize = (20,10), legend = False,ax=axes[1,0])\n",
81 + "print(df3)\n",
82 + "print(df[\"삼성머스트스팩3호\"])\n",
83 + "df3.plot(title = \"SK hos - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[1,1])\n",
84 + "\n",
85 + "df4 = pd.read_csv(\"finace_309930.csv\")\n",
86 + "df4 = pd.DataFrame(df4)\n",
87 + "df[\"SK4호스팩\"].plot(title = \"SK-호스팩\", figsize = (20,10), legend = False,ax=axes[2,0])\n",
88 + "print(df4)\n",
89 + "print(df[\"SK4호스팩\"])\n",
90 + "df4.plot(title = \"samsung - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[2,1])\n"
91 + ]
92 + }
93 + ],
94 + "metadata": {
95 + "kernelspec": {
96 + "display_name": "Python 3",
97 + "language": "python",
98 + "name": "python3"
99 + },
100 + "language_info": {
101 + "codemirror_mode": {
102 + "name": "ipython",
103 + "version": 3
104 + },
105 + "file_extension": ".py",
106 + "mimetype": "text/x-python",
107 + "name": "python",
108 + "nbconvert_exporter": "python",
109 + "pygments_lexer": "ipython3",
110 + "version": "3.6.10"
111 + }
112 + },
113 + "nbformat": 4,
114 + "nbformat_minor": 4
115 +}
1 +{
2 + "cells": [
3 + {
4 + "cell_type": "code",
5 + "execution_count": null,
6 + "metadata": {},
7 + "outputs": [],
8 + "source": [
9 + "import urllib\n",
10 + "import time\n",
11 + "\n",
12 + "from urllib.request import urlopen\n",
13 + "from bs4 import BeautifulSoup\n",
14 + "from urllib import request\n",
15 + "import re\n",
16 + "import json\n",
17 + "from datetime import datetime\n",
18 + "import os\n",
19 + "import boto3\n",
20 + "import time\n",
21 + "import sys\n",
22 + "import plotly.offline as py\n",
23 + "import plotly.graph_objs as go\n",
24 + "import plotly.tools as tls\n",
25 + "import matplotlib\n",
26 + "from random import shuffle\n",
27 + "import numpy as np\n",
28 + "import pandas as pd\n",
29 + "stockItem = ['285130','309930','307070' ]\n",
30 + "for k in range(0,3):\n",
31 + " url = 'http://finance.naver.com/item/sise_day.nhn?code='+ stockItem[k]\n",
32 + " html = urlopen(url) \n",
33 + " source = BeautifulSoup(html.read(), \"html.parser\")\n",
34 + " maxPage=source.find_all(\"table\",align=\"center\")\n",
35 + " mp = maxPage[0].find_all(\"td\",class_=\"pgRR\")\n",
36 + "\n",
37 + " mpNum = int(mp[0].a.get('href')[-2:])\n",
38 + " date = []\n",
39 + " finance = []\n",
40 + " df = {}\n",
41 + " for page in range(1, mpNum+1):\n",
42 + " print (page )\n",
43 + " print (mpNum+1)\n",
44 + " url = 'http://finance.naver.com/item/sise_day.nhn?code=' + stockItem[k] +'&page='+ str(page)\n",
45 + " html = urlopen(url)\n",
46 + " source = BeautifulSoup(html.read(), \"html.parser\")\n",
47 + " srlists=source.find_all(\"tr\") \n",
48 + " isCheckNone = None\n",
49 + " if((page % 1) == 0):\n",
50 + " time.sleep(0.1)\n",
51 + "\n",
52 + " for i in range(1,len(srlists)-1): \n",
53 + " \n",
54 + " if(srlists[i].span != isCheckNone):\n",
55 + " srlists[i].td.text\n",
56 + " date.append(srlists[i].find_all(\"td\",align=\"center\")[0].text)\n",
57 + " finance.append(int(srlists[i].find_all(\"td\",class_=\"num\")[0].text.replace(',','')))\n",
58 + " \n",
59 + " if(page == mpNum):\n",
60 + " date = np.array(date)\n",
61 + " finance = np.array(finance)\n",
62 + " df = pd.DataFrame(df)\n",
63 + " df[\"finance\"] = finance\n",
64 + " df[\"date\"] = date\n",
65 + " df.set_index(\"date\", inplace=True, drop=True)\n",
66 + " print(df)\n",
67 + " df.to_csv(\"finace_\"+stockItem[k]+\".csv\",encoding = \"utf-8\")\n"
68 + ]
69 + }
70 + ],
71 + "metadata": {
72 + "kernelspec": {
73 + "display_name": "Python 3",
74 + "language": "python",
75 + "name": "python3"
76 + },
77 + "language_info": {
78 + "codemirror_mode": {
79 + "name": "ipython",
80 + "version": 3
81 + },
82 + "file_extension": ".py",
83 + "mimetype": "text/x-python",
84 + "name": "python",
85 + "nbconvert_exporter": "python",
86 + "pygments_lexer": "ipython3",
87 + "version": "3.6.10"
88 + }
89 + },
90 + "nbformat": 4,
91 + "nbformat_minor": 4
92 +}
1 +{
2 + "cells": [
3 + {
4 + "cell_type": "code",
5 + "execution_count": null,
6 + "metadata": {},
7 + "outputs": [],
8 + "source": [
9 + "import numpy as np\n",
10 + "import pandas as pd\n",
11 + "from bs4 import BeautifulSoup\n",
12 + "import urllib\n",
13 + "from urllib import request\n",
14 + "import re\n",
15 + "import json\n",
16 + "from datetime import datetime\n",
17 + "import os\n",
18 + "import boto3\n",
19 + "import time\n",
20 + "import sys\n",
21 + "import plotly.offline as py\n",
22 + "import plotly.graph_objs as go\n",
23 + "import plotly.tools as tls\n",
24 + "import matplotlib\n",
25 + "from random import shuffle\n",
26 + "\n",
27 + "py.init_notebook_mode(connected=True)\n",
28 + "\n",
29 + "client_id = \"sYcWggwUdtmXwGqUrzzN\"\n",
30 + "client_secret = \"oxUqDSa22I\"\n",
31 + "\n",
32 + "link = \"https://openapi.naver.com/v1/datalab/search\"\n",
33 + "requested = request.Request(link)\n",
34 + "requested.add_header(\"X-Naver-Client-Id\",client_id)\n",
35 + "requested.add_header(\"X-Naver-Client-Secret\",client_secret)\n",
36 + "requested.add_header(\"Content-Type\",\"application/json\")\n",
37 + "\n",
38 + "df = pd.read_excel(\"index.xls\")\n",
39 + "names = df.회사명.values\n",
40 + "\n",
41 + "now = datetime.now().strftime(\"%Y-%m-%d\")\n",
42 + "body_dict = {\"startDate\":\"2017-01-01\", \n",
43 + " \"endDate\":\"2020-05-07\",\n",
44 + " \"timeUnit\":\"date\"}\n",
45 + "v_list = [{\"groupName\" : i, \"keywords\" : [i]} for i in names]\n",
46 + "df[\"대표자명\"] = df[\"대표자명\"].apply(lambda x: re.sub(r'\\(.*\\)', '', x))\n",
47 + "df[\"대표자명\"] = df[\"대표자명\"].apply(lambda x: [re.compile('[^ㄱ-ㅣ가-힣]+').sub(\"\",x)] if len(re.compile('[^ㄱ-ㅣ가-힣]+').sub(\"\",x)) < 5 else re.findall(r\"[\\w']+\", x))\n",
48 + "\n",
49 + "for i in range(0, df.shape[0]):\n",
50 + " for j in df[\"대표자명\"].values[i]:\n",
51 + " if \"대표\" not in j and j!= \"\":\n",
52 + " v_list[i][\"keywords\"].append(j) \n",
53 + " \n",
54 + "shuffle(v_list)\n",
55 + "standard = v_list[0]\n",
56 + "standard_keyword = standard[\"groupName\"]\n",
57 + "list_use = v_list[1:]\n",
58 + "\n",
59 + "split_list = [list_use[i:i+4] for i in range(0, len(list_use), 4)]\n",
60 + "\n",
61 + "for i in split_list:\n",
62 + " i.append(standard)\n",
63 + "\n",
64 + "sample_body = body_dict\n",
65 + "sample_body[\"keywordGroups\"] = split_list[0]\n",
66 + "sample_body = json.dumps(sample_body, ensure_ascii=False)\n",
67 + "sample_response = request.urlopen(requested, data=sample_body.encode(\"utf-8\"))\n",
68 + "\n",
69 + "code = sample_response.getcode() \n",
70 + "if code == 200: \n",
71 + " sample_response_body = sample_response.read()\n",
72 + " sample_scraped = sample_response_body.decode(\"utf-8\")\n",
73 + "else: \n",
74 + " print (\"Error Code:\", code)\n",
75 + "\n",
76 + "sample_result = json.loads(sample_scraped)\n",
77 + "\n",
78 + "for i in sample_result[\"results\"] :\n",
79 + " if i[\"title\"] == standard_keyword:\n",
80 + " sample_standard = i[\"data\"]\n",
81 + "scale = sample_standard[0][\"ratio\"]\n",
82 + "\n",
83 + "df = {}\n",
84 + "\n",
85 + "df[standard_keyword] = np.array([i[\"ratio\"] for i in sample_standard])\n",
86 + "length = len(df[standard_keyword])\n",
87 + "date = np.array([i[\"period\"] for i in sample_standard])\n",
88 + "for i in split_list:\n",
89 + "\n",
90 + " body_dict[\"keywordGroups\"] = i\n",
91 + " body = json.dumps(body_dict, ensure_ascii=False)\n",
92 + " \n",
93 + " print(i)\n",
94 + " response = request.urlopen(requested, data=body.encode(\"utf-8\"))\n",
95 + " \n",
96 + " code = response.getcode() \n",
97 + " if code == 200: \n",
98 + " response_body = response.read()\n",
99 + " scraped = response_body.decode(\"utf-8\")\n",
100 + " else: \n",
101 + " print (\"Error Code:\", code)\n",
102 + " \n",
103 + " \n",
104 + " \n",
105 + " result = json.loads(scraped)\n",
106 + " \n",
107 + " for i in result[\"results\"]:\n",
108 + " if i[\"title\"] == standard_keyword:\n",
109 + " compare = i[\"data\"]\n",
110 + " compare = compare[0][\"ratio\"]\n",
111 + " \n",
112 + " scaling = scale/compare\n",
113 + " \n",
114 + " for i in result[\"results\"]:\n",
115 + " if i[\"title\"]!=standard_keyword:\n",
116 + " value = [j[\"ratio\"]*scaling for j in i[\"data\"]]\n",
117 + " if len(value)!=length:\n",
118 + " value+=np.abs(length-len(value)) * [value[-1]]\n",
119 + " df[i[\"title\"]] = np.array(value)\n",
120 + "\n",
121 + "df = pd.DataFrame(df)\n",
122 + "df[\"date\"] = date\n",
123 + "df = df.set_index(\"date\")\n",
124 + "df.to_csv(\"trend.xls\", encoding = \"utf-8\")\n",
125 + "\n",
126 + "df.plot(title = \"Naver Trend - Stock-Daily\", figsize = (20, 10), legend = False)\n",
127 + "\n"
128 + ]
129 + }
130 + ],
131 + "metadata": {
132 + "kernelspec": {
133 + "display_name": "Python 3",
134 + "language": "python",
135 + "name": "python3"
136 + },
137 + "language_info": {
138 + "codemirror_mode": {
139 + "name": "ipython",
140 + "version": 3
141 + },
142 + "file_extension": ".py",
143 + "mimetype": "text/x-python",
144 + "name": "python",
145 + "nbconvert_exporter": "python",
146 + "pygments_lexer": "ipython3",
147 + "version": "3.6.10"
148 + }
149 + },
150 + "nbformat": 4,
151 + "nbformat_minor": 4
152 +}
1 +{
2 + "cells": [
3 + {
4 + "cell_type": "code",
5 + "execution_count": null,
6 + "metadata": {},
7 + "outputs": [
8 + {
9 + "data": {
10 + "text/html": [
11 + " <script type=\"text/javascript\">\n",
12 + " window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
13 + " if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
14 + " if (typeof require !== 'undefined') {\n",
15 + " require.undef(\"plotly\");\n",
16 + " requirejs.config({\n",
17 + " paths: {\n",
18 + " 'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
19 + " }\n",
20 + " });\n",
21 + " require(['plotly'], function(Plotly) {\n",
22 + " window._Plotly = Plotly;\n",
23 + " });\n",
24 + " }\n",
25 + " </script>\n",
26 + " "
27 + ]
28 + },
29 + "metadata": {},
30 + "output_type": "display_data"
31 + },
32 + {
33 + "name": "stdout",
34 + "output_type": "stream",
35 + "text": [
36 + "start read_csv\n"
37 + ]
38 + }
39 + ],
40 + "source": [
41 + "\n",
42 + "import numpy as np # linear algebra\n",
43 + "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
44 + "from bs4 import BeautifulSoup\n",
45 + "import urllib\n",
46 + "from urllib import request\n",
47 + "import re\n",
48 + "import json\n",
49 + "from datetime import datetime\n",
50 + "\n",
51 + "import plotly.offline as py\n",
52 + "py.init_notebook_mode(connected=True)\n",
53 + "import plotly.graph_objs as go\n",
54 + "import plotly.tools as tls\n",
55 + "import matplotlib.pyplot as plt\n",
56 + "from random import shuffle\n",
57 + "import random\n",
58 + "#Setup\n",
59 + "#load dataset\n",
60 + "print(\"start read_csv\")\n",
61 + "df = pd.read_csv(\"trendresult.csv\")\n",
62 + "\n",
63 + "df = pd.DataFrame(df)\n",
64 + "\n",
65 + "fig, axes = plt.subplots(nrows=3, ncols=2)\n",
66 + "\n",
67 + "\n",
68 + "#df.plot(title = \"Naver Trend - Stock-Daily\", figsize = (20, 10), legend = False)\n",
69 + "\n",
70 + "print(\"start read_csv_1\")\n",
71 + "df2 = pd.read_csv(\"finace_285130.csv\")\n",
72 + "df2 = pd.DataFrame(df2)\n",
73 + "print(df2)\n",
74 + "df[\"SK케미칼\"].plot(title = \"SK-chemical\", figsize = (20,10), legend = False,ax=axes[0,0])\n",
75 + "print(df[\"SK케미칼\"])\n",
76 + "df2.plot(title = \"SK-chemical - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[0,1])\n",
77 + "\n",
78 + "df3 = pd.read_csv(\"finace_307070.csv\")\n",
79 + "df3 = pd.DataFrame(df3)\n",
80 + "df[\"삼성머스트스팩3호\"].plot(title = \"삼성머스트스팩3호\", figsize = (20,10), legend = False,ax=axes[1,0])\n",
81 + "print(df3)\n",
82 + "print(df[\"삼성머스트스팩3호\"])\n",
83 + "df3.plot(title = \"SK hos - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[1,1])\n",
84 + "\n",
85 + "df4 = pd.read_csv(\"finace_309930.csv\")\n",
86 + "df4 = pd.DataFrame(df4)\n",
87 + "df[\"SK4호스팩\"].plot(title = \"SK-호스팩\", figsize = (20,10), legend = False,ax=axes[2,0])\n",
88 + "print(df4)\n",
89 + "print(df[\"SK4호스팩\"])\n",
90 + "df4.plot(title = \"samsung - Stock-Daily\", figsize = (20, 10), legend = False,ax=axes[2,1])\n"
91 + ]
92 + }
93 + ],
94 + "metadata": {
95 + "kernelspec": {
96 + "display_name": "Python 3",
97 + "language": "python",
98 + "name": "python3"
99 + },
100 + "language_info": {
101 + "codemirror_mode": {
102 + "name": "ipython",
103 + "version": 3
104 + },
105 + "file_extension": ".py",
106 + "mimetype": "text/x-python",
107 + "name": "python",
108 + "nbconvert_exporter": "python",
109 + "pygments_lexer": "ipython3",
110 + "version": "3.6.10"
111 + }
112 + },
113 + "nbformat": 4,
114 + "nbformat_minor": 4
115 +}
1 -
2 -import numpy as np # linear algebra
3 -import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
4 -from bs4 import BeautifulSoup
5 -import urllib
6 -from urllib import request
7 -import re
8 -import json
9 -from datetime import datetime
10 -
11 -import plotly.offline as py
12 -py.init_notebook_mode(connected=True)
13 -import plotly.graph_objs as go
14 -import plotly.tools as tls
15 -import matplotlib.pyplot as plt
16 -from random import shuffle
17 -import random
18 -#Setup
19 -#load dataset
20 -print("start read_csv")
21 -df = pd.read_csv("trendresult.csv")
22 -
23 -df = pd.DataFrame(df)
24 -
25 -fig, axes = plt.subplots(nrows=3, ncols=2)
26 -
27 -
28 -#df.plot(title = "Naver Trend - Stock-Daily", figsize = (20, 10), legend = False)
29 -
30 -print("start read_csv_1")
31 -df2 = pd.read_csv("finace_285130.csv")
32 -df2 = pd.DataFrame(df2)
33 -print(df2)
34 -df["SK케미칼"].plot(title = "SK-chemical", figsize = (20,10), legend = False,ax=axes[0,0])
35 -print(df["SK케미칼"])
36 -df2.plot(title = "SK-chemical - Stock-Daily", figsize = (20, 10), legend = False,ax=axes[0,1])
37 -
38 -df3 = pd.read_csv("finace_307070.csv")
39 -df3 = pd.DataFrame(df3)
40 -df["삼성머스트스팩3호"].plot(title = "삼성머스트스팩3호", figsize = (20,10), legend = False,ax=axes[1,0])
41 -print(df3)
42 -print(df["삼성머스트스팩3호"])
43 -df3.plot(title = "SK hos - Stock-Daily", figsize = (20, 10), legend = False,ax=axes[1,1])
44 -
45 -df4 = pd.read_csv("finace_309930.csv")
46 -df4 = pd.DataFrame(df4)
47 -df["SK4호스팩"].plot(title = "SK-호스팩", figsize = (20,10), legend = False,ax=axes[2,0])
48 -print(df4)
49 -print(df["SK4호스팩"])
50 -df4.plot(title = "samsung - Stock-Daily", figsize = (20, 10), legend = False,ax=axes[2,1])
1 +{
2 + "cells": [
3 + {
4 + "cell_type": "code",
5 + "execution_count": null,
6 + "metadata": {},
7 + "outputs": [],
8 + "source": [
9 + "import urllib\n",
10 + "import time\n",
11 + "\n",
12 + "from urllib.request import urlopen\n",
13 + "from bs4 import BeautifulSoup\n",
14 + "from urllib import request\n",
15 + "import re\n",
16 + "import json\n",
17 + "from datetime import datetime\n",
18 + "import os\n",
19 + "import boto3\n",
20 + "import time\n",
21 + "import sys\n",
22 + "import plotly.offline as py\n",
23 + "import plotly.graph_objs as go\n",
24 + "import plotly.tools as tls\n",
25 + "import matplotlib\n",
26 + "from random import shuffle\n",
27 + "import numpy as np\n",
28 + "import pandas as pd\n",
29 + "stockItem = ['285130','309930','307070' ]\n",
30 + "for k in range(0,3):\n",
31 + " url = 'http://finance.naver.com/item/sise_day.nhn?code='+ stockItem[k]\n",
32 + " html = urlopen(url) \n",
33 + " source = BeautifulSoup(html.read(), \"html.parser\")\n",
34 + " maxPage=source.find_all(\"table\",align=\"center\")\n",
35 + " mp = maxPage[0].find_all(\"td\",class_=\"pgRR\")\n",
36 + "\n",
37 + " mpNum = int(mp[0].a.get('href')[-2:])\n",
38 + " date = []\n",
39 + " finance = []\n",
40 + " df = {}\n",
41 + " for page in range(1, mpNum+1):\n",
42 + " print (page )\n",
43 + " print (mpNum+1)\n",
44 + " url = 'http://finance.naver.com/item/sise_day.nhn?code=' + stockItem[k] +'&page='+ str(page)\n",
45 + " html = urlopen(url)\n",
46 + " source = BeautifulSoup(html.read(), \"html.parser\")\n",
47 + " srlists=source.find_all(\"tr\") \n",
48 + " isCheckNone = None\n",
49 + " if((page % 1) == 0):\n",
50 + " time.sleep(0.1)\n",
51 + "\n",
52 + " for i in range(1,len(srlists)-1): \n",
53 + " \n",
54 + " if(srlists[i].span != isCheckNone):\n",
55 + " srlists[i].td.text\n",
56 + " date.append(srlists[i].find_all(\"td\",align=\"center\")[0].text)\n",
57 + " finance.append(int(srlists[i].find_all(\"td\",class_=\"num\")[0].text.replace(',','')))\n",
58 + " \n",
59 + " if(page == mpNum):\n",
60 + " date = np.array(date)\n",
61 + " finance = np.array(finance)\n",
62 + " df = pd.DataFrame(df)\n",
63 + " df[\"finance\"] = finance\n",
64 + " df[\"date\"] = date\n",
65 + " df.set_index(\"date\", inplace=True, drop=True)\n",
66 + " print(df)\n",
67 + " df.to_csv(\"finace_\"+stockItem[k]+\".csv\",encoding = \"utf-8\")\n"
68 + ]
69 + }
70 + ],
71 + "metadata": {
72 + "kernelspec": {
73 + "display_name": "Python 3",
74 + "language": "python",
75 + "name": "python3"
76 + },
77 + "language_info": {
78 + "codemirror_mode": {
79 + "name": "ipython",
80 + "version": 3
81 + },
82 + "file_extension": ".py",
83 + "mimetype": "text/x-python",
84 + "name": "python",
85 + "nbconvert_exporter": "python",
86 + "pygments_lexer": "ipython3",
87 + "version": "3.6.10"
88 + }
89 + },
90 + "nbformat": 4,
91 + "nbformat_minor": 4
92 +}
1 -import urllib
2 -import time
3 -
4 -from urllib.request import urlopen
5 -from bs4 import BeautifulSoup
6 -from urllib import request
7 -import re
8 -import json
9 -from datetime import datetime
10 -import os
11 -import boto3
12 -import time
13 -import sys
14 -import plotly.offline as py
15 -import plotly.graph_objs as go
16 -import plotly.tools as tls
17 -import matplotlib
18 -from random import shuffle
19 -import numpy as np
20 -import pandas as pd
21 -stockItem = ['285130','309930','307070' ]
22 -for k in range(0,3):
23 - url = 'http://finance.naver.com/item/sise_day.nhn?code='+ stockItem[k]
24 - html = urlopen(url)
25 - source = BeautifulSoup(html.read(), "html.parser")
26 - maxPage=source.find_all("table",align="center")
27 - mp = maxPage[0].find_all("td",class_="pgRR")
28 -
29 - mpNum = int(mp[0].a.get('href')[-2:])
30 - date = []
31 - finance = []
32 - df = {}
33 - for page in range(1, mpNum+1):
34 - print (page )
35 - print (mpNum+1)
36 - url = 'http://finance.naver.com/item/sise_day.nhn?code=' + stockItem[k] +'&page='+ str(page)
37 - html = urlopen(url)
38 - source = BeautifulSoup(html.read(), "html.parser")
39 - srlists=source.find_all("tr")
40 - isCheckNone = None
41 - if((page % 1) == 0):
42 - time.sleep(0.1)
43 -
44 - for i in range(1,len(srlists)-1):
45 -
46 - if(srlists[i].span != isCheckNone):
47 - srlists[i].td.text
48 - date.append(srlists[i].find_all("td",align="center")[0].text)
49 - finance.append(int(srlists[i].find_all("td",class_="num")[0].text.replace(',','')))
50 -
51 - if(page == mpNum):
52 - date = np.array(date)
53 - finance = np.array(finance)
54 - df = pd.DataFrame(df)
55 - df["finance"] = finance
56 - df["date"] = date
57 - df.set_index("date", inplace=True, drop=True)
58 - print(df)
59 - df.to_csv("finace_"+stockItem[k]+".csv",encoding = "utf-8")
1 +{
2 + "cells": [
3 + {
4 + "cell_type": "code",
5 + "execution_count": null,
6 + "metadata": {},
7 + "outputs": [],
8 + "source": [
9 + "import numpy as np\n",
10 + "import pandas as pd\n",
11 + "from bs4 import BeautifulSoup\n",
12 + "import urllib\n",
13 + "from urllib import request\n",
14 + "import re\n",
15 + "import json\n",
16 + "from datetime import datetime\n",
17 + "import os\n",
18 + "import boto3\n",
19 + "import time\n",
20 + "import sys\n",
21 + "import plotly.offline as py\n",
22 + "import plotly.graph_objs as go\n",
23 + "import plotly.tools as tls\n",
24 + "import matplotlib\n",
25 + "from random import shuffle\n",
26 + "\n",
27 + "py.init_notebook_mode(connected=True)\n",
28 + "\n",
29 + "client_id = \"sYcWggwUdtmXwGqUrzzN\"\n",
30 + "client_secret = \"oxUqDSa22I\"\n",
31 + "\n",
32 + "link = \"https://openapi.naver.com/v1/datalab/search\"\n",
33 + "requested = request.Request(link)\n",
34 + "requested.add_header(\"X-Naver-Client-Id\",client_id)\n",
35 + "requested.add_header(\"X-Naver-Client-Secret\",client_secret)\n",
36 + "requested.add_header(\"Content-Type\",\"application/json\")\n",
37 + "\n",
38 + "df = pd.read_excel(\"index.xls\")\n",
39 + "names = df.회사명.values\n",
40 + "\n",
41 + "now = datetime.now().strftime(\"%Y-%m-%d\")\n",
42 + "body_dict = {\"startDate\":\"2017-01-01\", \n",
43 + " \"endDate\":\"2020-05-07\",\n",
44 + " \"timeUnit\":\"date\"}\n",
45 + "v_list = [{\"groupName\" : i, \"keywords\" : [i]} for i in names]\n",
46 + "df[\"대표자명\"] = df[\"대표자명\"].apply(lambda x: re.sub(r'\\(.*\\)', '', x))\n",
47 + "df[\"대표자명\"] = df[\"대표자명\"].apply(lambda x: [re.compile('[^ㄱ-ㅣ가-힣]+').sub(\"\",x)] if len(re.compile('[^ㄱ-ㅣ가-힣]+').sub(\"\",x)) < 5 else re.findall(r\"[\\w']+\", x))\n",
48 + "\n",
49 + "for i in range(0, df.shape[0]):\n",
50 + " for j in df[\"대표자명\"].values[i]:\n",
51 + " if \"대표\" not in j and j!= \"\":\n",
52 + " v_list[i][\"keywords\"].append(j) \n",
53 + " \n",
54 + "shuffle(v_list)\n",
55 + "standard = v_list[0]\n",
56 + "standard_keyword = standard[\"groupName\"]\n",
57 + "list_use = v_list[1:]\n",
58 + "\n",
59 + "split_list = [list_use[i:i+4] for i in range(0, len(list_use), 4)]\n",
60 + "\n",
61 + "for i in split_list:\n",
62 + " i.append(standard)\n",
63 + "\n",
64 + "sample_body = body_dict\n",
65 + "sample_body[\"keywordGroups\"] = split_list[0]\n",
66 + "sample_body = json.dumps(sample_body, ensure_ascii=False)\n",
67 + "sample_response = request.urlopen(requested, data=sample_body.encode(\"utf-8\"))\n",
68 + "\n",
69 + "code = sample_response.getcode() \n",
70 + "if code == 200: \n",
71 + " sample_response_body = sample_response.read()\n",
72 + " sample_scraped = sample_response_body.decode(\"utf-8\")\n",
73 + "else: \n",
74 + " print (\"Error Code:\", code)\n",
75 + "\n",
76 + "sample_result = json.loads(sample_scraped)\n",
77 + "\n",
78 + "for i in sample_result[\"results\"] :\n",
79 + " if i[\"title\"] == standard_keyword:\n",
80 + " sample_standard = i[\"data\"]\n",
81 + "scale = sample_standard[0][\"ratio\"]\n",
82 + "\n",
83 + "df = {}\n",
84 + "\n",
85 + "df[standard_keyword] = np.array([i[\"ratio\"] for i in sample_standard])\n",
86 + "length = len(df[standard_keyword])\n",
87 + "date = np.array([i[\"period\"] for i in sample_standard])\n",
88 + "for i in split_list:\n",
89 + "\n",
90 + " body_dict[\"keywordGroups\"] = i\n",
91 + " body = json.dumps(body_dict, ensure_ascii=False)\n",
92 + " \n",
93 + " print(i)\n",
94 + " response = request.urlopen(requested, data=body.encode(\"utf-8\"))\n",
95 + " \n",
96 + " code = response.getcode() \n",
97 + " if code == 200: \n",
98 + " response_body = response.read()\n",
99 + " scraped = response_body.decode(\"utf-8\")\n",
100 + " else: \n",
101 + " print (\"Error Code:\", code)\n",
102 + " \n",
103 + " \n",
104 + " \n",
105 + " result = json.loads(scraped)\n",
106 + " \n",
107 + " for i in result[\"results\"]:\n",
108 + " if i[\"title\"] == standard_keyword:\n",
109 + " compare = i[\"data\"]\n",
110 + " compare = compare[0][\"ratio\"]\n",
111 + " \n",
112 + " scaling = scale/compare\n",
113 + " \n",
114 + " for i in result[\"results\"]:\n",
115 + " if i[\"title\"]!=standard_keyword:\n",
116 + " value = [j[\"ratio\"]*scaling for j in i[\"data\"]]\n",
117 + " if len(value)!=length:\n",
118 + " value+=np.abs(length-len(value)) * [value[-1]]\n",
119 + " df[i[\"title\"]] = np.array(value)\n",
120 + "\n",
121 + "df = pd.DataFrame(df)\n",
122 + "df[\"date\"] = date\n",
123 + "df = df.set_index(\"date\")\n",
124 + "df.to_csv(\"trend.xls\", encoding = \"utf-8\")\n",
125 + "\n",
126 + "df.plot(title = \"Naver Trend - Stock-Daily\", figsize = (20, 10), legend = False)\n",
127 + "\n"
128 + ]
129 + }
130 + ],
131 + "metadata": {
132 + "kernelspec": {
133 + "display_name": "Python 3",
134 + "language": "python",
135 + "name": "python3"
136 + },
137 + "language_info": {
138 + "codemirror_mode": {
139 + "name": "ipython",
140 + "version": 3
141 + },
142 + "file_extension": ".py",
143 + "mimetype": "text/x-python",
144 + "name": "python",
145 + "nbconvert_exporter": "python",
146 + "pygments_lexer": "ipython3",
147 + "version": "3.6.10"
148 + }
149 + },
150 + "nbformat": 4,
151 + "nbformat_minor": 4
152 +}
1 -import numpy as np
2 -import pandas as pd
3 -from bs4 import BeautifulSoup
4 -import urllib
5 -from urllib import request
6 -import re
7 -import json
8 -from datetime import datetime
9 -import os
10 -import boto3
11 -import time
12 -import sys
13 -import plotly.offline as py
14 -import plotly.graph_objs as go
15 -import plotly.tools as tls
16 -import matplotlib
17 -from random import shuffle
18 -
19 -py.init_notebook_mode(connected=True)
20 -
21 -
22 -def upload(s3, local_file_path, bucket, obj):
23 - s3.upload_file(local_file_path,bucket,obj)
24 -
25 -def make_public_read(s3,bucket,key):
26 - s3.put_object_acl(ACL="public-read",Bucket=bucket, Key=key)
27 -
28 -def download(s3,bucket, obj, local_file_path):
29 - s3.download_file(bucket,obj, local_file_path)
30 -
31 -
32 -client_id = "sYcWggwUdtmXwGqUrzzN"
33 -client_secret = "oxUqDSa22I"
34 -
35 -link = "https://openapi.naver.com/v1/datalab/search"
36 -requested = request.Request(link)
37 -requested.add_header("X-Naver-Client-Id",client_id)
38 -requested.add_header("X-Naver-Client-Secret",client_secret)
39 -requested.add_header("Content-Type","application/json")
40 -
41 -df = pd.read_excel("index.xls")
42 -names = df.회사명.values
43 -
44 -now = datetime.now().strftime("%Y-%m-%d")
45 -body_dict = {"startDate":"2017-01-01",
46 - "endDate":"2020-05-07",
47 - "timeUnit":"date"}
48 -v_list = [{"groupName" : i, "keywords" : [i]} for i in names]
49 -df["대표자명"] = df["대표자명"].apply(lambda x: re.sub(r'\(.*\)', '', x))
50 -df["대표자명"] = df["대표자명"].apply(lambda x: [re.compile('[^ㄱ-ㅣ가-힣]+').sub("",x)] if len(re.compile('[^ㄱ-ㅣ가-힣]+').sub("",x)) < 5 else re.findall(r"[\w']+", x))
51 -
52 -for i in range(0, df.shape[0]):
53 - for j in df["대표자명"].values[i]:
54 - if "대표" not in j and j!= "":
55 - v_list[i]["keywords"].append(j)
56 -
57 -shuffle(v_list)
58 -standard = v_list[0]
59 -standard_keyword = standard["groupName"]
60 -list_use = v_list[1:]
61 -
62 -split_list = [list_use[i:i+4] for i in range(0, len(list_use), 4)]
63 -
64 -for i in split_list:
65 - i.append(standard)
66 -
67 -sample_body = body_dict
68 -sample_body["keywordGroups"] = split_list[0]
69 -sample_body = json.dumps(sample_body, ensure_ascii=False)
70 -sample_response = request.urlopen(requested, data=sample_body.encode("utf-8"))
71 -
72 -code = sample_response.getcode()
73 -if code == 200:
74 - sample_response_body = sample_response.read()
75 - sample_scraped = sample_response_body.decode("utf-8")
76 -else:
77 - print ("Error Code:", code)
78 -
79 -sample_result = json.loads(sample_scraped)
80 -
81 -for i in sample_result["results"] :
82 - if i["title"] == standard_keyword:
83 - sample_standard = i["data"]
84 -scale = sample_standard[0]["ratio"]
85 -
86 -df = {}
87 -
88 -df[standard_keyword] = np.array([i["ratio"] for i in sample_standard])
89 -length = len(df[standard_keyword])
90 -date = np.array([i["period"] for i in sample_standard])
91 -for i in split_list:
92 -
93 - body_dict["keywordGroups"] = i
94 - body = json.dumps(body_dict, ensure_ascii=False)
95 -
96 - print(i)
97 - response = request.urlopen(requested, data=body.encode("utf-8"))
98 -
99 - code = response.getcode()
100 - if code == 200:
101 - response_body = response.read()
102 - scraped = response_body.decode("utf-8")
103 - else:
104 - print ("Error Code:", code)
105 -
106 -
107 -
108 - result = json.loads(scraped)
109 -
110 - for i in result["results"]:
111 - if i["title"] == standard_keyword:
112 - compare = i["data"]
113 - compare = compare[0]["ratio"]
114 -
115 - scaling = scale/compare
116 -
117 - for i in result["results"]:
118 - if i["title"]!=standard_keyword:
119 - value = [j["ratio"]*scaling for j in i["data"]]
120 - if len(value)!=length:
121 - value+=np.abs(length-len(value)) * [value[-1]]
122 - df[i["title"]] = np.array(value)
123 -
124 -df = pd.DataFrame(df)
125 -df["date"] = date
126 -df = df.set_index("date")
127 -df.to_csv("trend.xls", encoding = "utf-8")
128 -
129 -session = boto3.Session(profile_name = "class")
130 -s3 = session.client("s3")
131 -bucket = 'khu-big-data-ksh'
132 -path = sys.argv[1].replace('\\','/')
133 -filelist = os.listdir(path)
134 -date = time.localtime()
135 -if(path[len(path)-1] != '/'):
136 - path +='/'
137 -year = str(date.tm_year)
138 -month = str(date.tm_mon)
139 -day = str(date.tm_mday)
140 -if(len(month) ==1):
141 - month = '0'+month
142 -if(len(day) ==1):
143 - day = '0'+day
144 -current_date = year+"/"+month+"-"+day
145 -try:
146 - if(sys.argv[3] =='1'):
147 - os.mkdir(path+'downloads/')
148 -except:
149 - print("")
150 -
151 -for element in filelist:
152 - if(os.path.isdir(path+element)):
153 - continue
154 - upload(s3, path+element, bucket, current_date+"/"+element)
155 - make_public_read(s3,bucket,current_date+"/"+element)
156 -
157 -df.plot(title = "Naver Trend - Stock-Daily", figsize = (20, 10), legend = False)
158 -
No preview for this file type