방승일 / mamuri-bot

Go to a project
Toggle navigation Toggle navigation pinning
윤준석
201b30e1 2 parents cbb69c89 ead0ff99

Merge branch 'feature/220523_joongna_api_fix' into 'main' 

Feature/220523 joongna api fix



See merge request !14
Inline Side-by-side
Showing 4 changed files with 49 additions and 69 deletions
......@@ -11,7 +11,5 @@ WORKDIR /src
COPY --from=builder /src/joongna_api_server /src/joongna_api_server
COPY --from=builder /src/config/.env /src/config/.env
RUN apk add chromium
EXPOSE 8080
CMD ["./joongna_api_server"]
......
......@@ -12,6 +12,11 @@ type Config struct {
CLIENTID string `env:"SECRET.CLIENTID"`
CLIENTSECRET string `env:"SECRET.CLIENTSECRET"`
}
Header struct {
Cookie string `env:"HEADER.COOKIE"`
UserAgent string `env:"HEADER.USERAGENT"`
}
}
var Cfg *Config
......
package model
type ApiResponse struct {
LastBuildDate string `json:"lastBuildDate"`
Total uint `json:"total"`
Start uint `json:"start"`
Display uint `json:"display"`
Items []ApiResponseItem `json:"items"`
CafeId int `json:"cafeId"`
ArticelCount int `json:"articleCount"`
Query string `json:"query"`
Items []ApiResponseItem `json:"articleList"`
}
type ApiResponseItem struct {
Title string `json:"title"`
Link string `json:"link"`
Description string `json:"description"`
CafeName string `json:"cafename"`
ArticleId int `json:"articleId"`
Title string `json:"subject"`
ExtraInfo string `json:"summary"`
ThumbnailUrl string `json:"thumbnailImageUrl"`
ProductSale ApiResponseItemSale `json:"productSale"`
}
type ApiResponseItemSale struct {
SaleStatus string `json:"saleStatue"`
Cost string `json:"cost"`
}
......
package service
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"joongna/config"
......@@ -13,36 +13,35 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
)
func GetItemByKeyword(keyword string) ([]model.Item, error) {
var items []model.Item
wg := sync.WaitGroup{}
itemsInfo, err := getItemsInfoByKeyword(keyword)
responseItems, err := getItemsInfoByKeyword(keyword)
if err != nil {
return nil, err
}
for _, itemInfo := range itemsInfo {
itemUrl := itemInfo.Link
if itemInfo.CafeName != "중고나라" {
continue
}
for _, responseItem := range responseItems {
wg.Add(1)
go func(itemUrl string) {
go func(responseItem model.ApiResponseItem) {
defer wg.Done()
item, err := crawlingNaverCafe(itemUrl)
if err != nil {
log.Fatal(err)
}
items = append(items, *item)
}(itemUrl)
item := model.Item{
Platform: "중고나라",
Name: responseItem.Title,
Price: priceStringToInt(responseItem.ProductSale.Cost),
ThumbnailUrl: responseItem.ThumbnailUrl,
ItemUrl: fmt.Sprintf("https://m.cafe.naver.com/ca-fe/web/cafes/10050146/articles/%d", responseItem.ArticleId),
ExtraInfo: responseItem.ExtraInfo,
}
items = append(items, item)
}(responseItem)
}
wg.Wait()
......@@ -50,8 +49,8 @@ func GetItemByKeyword(keyword string) ([]model.Item, error) {
}
func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) {
encText := url.QueryEscape("중고나라 " + keyword + " 판매중")
apiUrl := "https://openapi.naver.com/v1/search/cafearticle.json?query=" + encText + "&sort=sim"
encText := url.QueryEscape(keyword)
apiUrl := fmt.Sprintf("https://apis.naver.com/cafe-web/cafe-mobile/CafeMobileWebArticleSearchListV3?cafeId=10050146&query=%s&searchBy=0&sortBy=sim&page=1&perPage=10&adUnit=MW_CAFE_BOARD", encText)
req, err := http.NewRequest("GET", apiUrl, nil)
if err != nil {
......@@ -59,6 +58,8 @@ func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) {
}
req.Header.Add("X-Naver-Client-Id", config.Cfg.Secret.CLIENTID)
req.Header.Add("X-Naver-Client-Secret", config.Cfg.Secret.CLIENTSECRET)
req.Header.Add("Cookie", config.Cfg.Header.Cookie)
req.Header.Add("User-agent", config.Cfg.Header.UserAgent)
client := &http.Client{}
resp, err := client.Do(req)
......@@ -73,55 +74,26 @@ func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) {
}(resp.Body)
response, _ := ioutil.ReadAll(resp.Body)
var apiResponse model.ApiResponse
err = json.Unmarshal(response, &apiResponse)
var apiResult map[string]interface{}
err = json.Unmarshal(response, &apiResult)
if err != nil {
log.Fatal(err)
return nil, err
}
return apiResponse.Items, nil
}
func crawlingNaverCafe(cafeUrl string) (*model.Item, error) {
path, _ := launcher.LookPath()
u := launcher.New().Bin(path).MustLaunch()
browser := rod.New().ControlURL(u).MustConnect()
defer func(browser *rod.Browser) {
err := browser.Close()
if err != nil {
log.Fatal(err)
}
}(browser)
frame := browser.MustPage(cafeUrl).MustElement("iframe#cafe_main")
time.Sleep(time.Second * 2)
source := frame.MustFrame().MustHTML()
html, err := goquery.NewDocumentFromReader(bytes.NewReader([]byte(source)))
result := apiResult["message"].(map[string]interface{})["result"]
resultJson, err := json.Marshal(result)
if err != nil {
return nil, err
}
title := html.Find("h3.title_text").Text()
sold := html.Find("div.sold_area").Text()
price := priceStringToInt(html.Find(".ProductPrice").Text())
thumbnailUrl, _ := html.Find("div.product_thumb img").Attr("src")
extraInfo := html.Find(".se-module-text").Text()
title = strings.TrimSpace(title)
sold = strings.TrimSpace(sold)
thumbnailUrl = strings.TrimSpace(thumbnailUrl)
extraInfo = strings.TrimSpace(extraInfo)
item := model.Item{
Platform: "중고나라",
Name: title,
Price: price,
ThumbnailUrl: thumbnailUrl,
ItemUrl: cafeUrl,
ExtraInfo: extraInfo,
var apiResponse model.ApiResponse
err = json.Unmarshal(resultJson, &apiResponse)
if err != nil {
return nil, err
}
return &item, nil
return apiResponse.Items, nil
}
func priceStringToInt(priceString string) int {
......