방승일 / mamuri-bot

Go to a project
Toggle navigation Toggle navigation pinning
윤준석
d5069fbd 1 parent c0502ab7

FIX: change crawling package (selenium -> go-rod)

Inline Side-by-side
Showing 3 changed files with 19 additions and 22 deletions
...@@ -9,6 +9,7 @@ require ( ...@@ -9,6 +9,7 @@ require (
9 github.com/bunsenapp/go-selenium v0.1.0 // indirect 9 github.com/bunsenapp/go-selenium v0.1.0 // indirect
10 github.com/caarlos0/env/v6 v6.9.1 // indirect 10 github.com/caarlos0/env/v6 v6.9.1 // indirect
11 github.com/fedesog/webdriver v0.0.0-20180606182539-99f36c92eaef // indirect 11 github.com/fedesog/webdriver v0.0.0-20180606182539-99f36c92eaef // indirect
12 + github.com/go-rod/rod v0.106.6 // indirect
12 github.com/joho/godotenv v1.4.0 // indirect 13 github.com/joho/godotenv v1.4.0 // indirect
13 github.com/labstack/echo/v4 v4.7.2 // indirect 14 github.com/labstack/echo/v4 v4.7.2 // indirect
14 github.com/labstack/gommon v0.3.1 // indirect 15 github.com/labstack/gommon v0.3.1 // indirect
...@@ -17,6 +18,9 @@ require ( ...@@ -17,6 +18,9 @@ require (
17 github.com/tebeka/selenium v0.9.9 // indirect 18 github.com/tebeka/selenium v0.9.9 // indirect
18 github.com/valyala/bytebufferpool v1.0.0 // indirect 19 github.com/valyala/bytebufferpool v1.0.0 // indirect
19 github.com/valyala/fasttemplate v1.2.1 // indirect 20 github.com/valyala/fasttemplate v1.2.1 // indirect
21 + github.com/ysmood/goob v0.4.0 // indirect
22 + github.com/ysmood/gson v0.7.1 // indirect
23 + github.com/ysmood/leakless v0.7.0 // indirect
20 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect 24 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
21 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect 25 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
22 golang.org/x/sys v0.0.0-20211103235746-7861aae1554b // indirect 26 golang.org/x/sys v0.0.0-20211103235746-7861aae1554b // indirect
......
...@@ -21,6 +21,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs ...@@ -21,6 +21,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
21 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 21 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
22 github.com/fedesog/webdriver v0.0.0-20180606182539-99f36c92eaef h1:0z8rB8nAGEso7PhKS21wBjjxTp2uGPyZ6STzRc7mnBY= 22 github.com/fedesog/webdriver v0.0.0-20180606182539-99f36c92eaef h1:0z8rB8nAGEso7PhKS21wBjjxTp2uGPyZ6STzRc7mnBY=
23 github.com/fedesog/webdriver v0.0.0-20180606182539-99f36c92eaef/go.mod h1:RUn/EmpfFIGHvmeXmh+hk1UaCbjOXa6vl7/kx1b6wxw= 23 github.com/fedesog/webdriver v0.0.0-20180606182539-99f36c92eaef/go.mod h1:RUn/EmpfFIGHvmeXmh+hk1UaCbjOXa6vl7/kx1b6wxw=
24 +github.com/go-rod/rod v0.106.6 h1:zJorVPG7s8Xgbh7PkSySP4FNoo0OiougKaMb3j6zT6w=
25 +github.com/go-rod/rod v0.106.6/go.mod h1:xkZOchuKqTOkMOBkrzb7uJpbKZRab1haPCWDvuZkS2U=
24 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 26 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
25 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 27 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
26 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 28 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
...@@ -60,6 +62,14 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw ...@@ -60,6 +62,14 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
60 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 62 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
61 github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= 63 github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
62 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 64 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
65 +github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
66 +github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
67 +github.com/ysmood/got v0.29.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
68 +github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
69 +github.com/ysmood/gson v0.7.1 h1:zKL2MTGtynxdBdlZjyGsvEOZ7dkxaY5TH6QhAbTgz0Q=
70 +github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
71 +github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
72 +github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
63 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 73 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
64 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 74 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
65 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 75 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
......
...@@ -15,7 +15,7 @@ import ( ...@@ -15,7 +15,7 @@ import (
15 "time" 15 "time"
16 16
17 "github.com/PuerkitoBio/goquery" 17 "github.com/PuerkitoBio/goquery"
18 - "github.com/fedesog/webdriver" 18 + "github.com/go-rod/rod"
19 ) 19 )
20 20
21 func GetItemByKeyword(keyword string) ([]model.Item, error) { 21 func GetItemByKeyword(keyword string) ([]model.Item, error) {
...@@ -79,29 +79,12 @@ func getItemsInfoByKeyword(keyword string) []model.ApiResponseItem { ...@@ -79,29 +79,12 @@ func getItemsInfoByKeyword(keyword string) []model.ApiResponseItem {
79 } 79 }
80 80
81 func crawlingNaverCafe(cafeUrl string) (string, int, string, string) { 81 func crawlingNaverCafe(cafeUrl string) (string, int, string, string) {
82 - driver := webdriver.NewChromeDriver("./chromedriver") 82 + page := rod.New().MustConnect().MustPage(cafeUrl)
83 - err := driver.Start() 83 +
84 - if err != nil {
85 - log.Println(err)
86 - }
87 - desired := webdriver.Capabilities{"Platform": "Linux"}
88 - required := webdriver.Capabilities{}
89 - session, err := driver.NewSession(desired, required)
90 - if err != nil {
91 - log.Println(err)
92 - }
93 - err = session.Url(cafeUrl)
94 - if err != nil {
95 - log.Println(err)
96 - }
97 time.Sleep(time.Second * 1) 84 time.Sleep(time.Second * 1)
98 - err = session.FocusOnFrame("cafe_main")
99 - if err != nil {
100 - log.Fatal(err)
101 - }
102 - resp, err := session.Source()
103 85
104 - html, err := goquery.NewDocumentFromReader(bytes.NewReader([]byte(resp))) 86 + source := page.MustElement("iframe#cafe_main").MustFrame().MustHTML()
87 + html, err := goquery.NewDocumentFromReader(bytes.NewReader([]byte(source)))
105 if err != nil { 88 if err != nil {
106 log.Fatal(err) 89 log.Fatal(err)
107 } 90 }
......