add web crawling module

서민정
Commit c79f207bb5a069f0d6f0f2280c35819c2d119274 c79f207b 1 parent a0344e39
Showing 8 changed files with 85 additions and 31 deletions
client/package-lock.json
client/src/App.js
client/src/chatbot/chatbot.js
client/src/index.css
index.js
server/cheerio/video.js
server/routes/crawling.js
server/routes/dialogflow.js
--- a/client/package-lock.json
View file @c79f207
+++ b/client/package-lock.json
View file @c79f207
@@ -1323,9 +1323,9 @@
       },
       "dependencies": {
         "acorn": {
-          "version": "5.7.3",
+          "version": "5.7.4",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz",
-          "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw=="
+          "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg=="
         }
       }
     },
@@ -8448,9 +8448,9 @@
       },
       "dependencies": {
         "acorn": {
-          "version": "5.7.3",
+          "version": "5.7.4",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz",
-          "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw=="
+          "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg=="
         },
         "parse5": {
           "version": "4.0.0",
@@ -15942,9 +15942,9 @@
       },
       "dependencies": {
         "acorn": {
-          "version": "5.7.3",
+          "version": "5.7.4",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz",
-          "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw=="
+          "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg=="
         },
         "eslint-scope": {
           "version": "4.0.3",
--- a/client/src/App.js
View file @c79f207
+++ b/client/src/App.js
View file @c79f207
@@ -6,7 +6,7 @@ const { Title } = Typography;
 function App() {
   return (
     <div>
-      <div style={{ display: 'flex', justifyContent: 'center', marginTop: '2rem' }}>
+      <div style={{ display: 'flex', justifyContent: 'center', marginTop: '1rem' }}>
         <Title level={2} >CHATBOT&nbsp;<Icon type="robot" /></Title>
       </div>
       <div style={{ display: 'flex', justifyContent: 'center' }}>
--- a/client/src/chatbot/chatbot.js
View file @c79f207
+++ b/client/src/chatbot/chatbot.js
View file @c79f207
@@ -11,7 +11,8 @@ function Chatbot() {
     useEffect(() => {
-        eventQuery('welcomeToMyWebsite')
+        eventQuery('001_Welcome')
+        eventQuery('002_Intro')
     }, [])
@@ -39,7 +40,7 @@ function Chatbot() {
         }
         try {
             if(inputString[0] === '@'){
-                const response = await Axios.post('/api/crawling/textQuery', textQueryVariables)
+                await Axios.post('/api/crawling/textQuery', textQueryVariables)
             } else {
                 //I will send request to the textQuery ROUTE 
                 const response = await Axios.post('/api/dialogflow/textQuery', textQueryVariables)
@@ -47,7 +48,7 @@ function Chatbot() {
                 for (let content of response.data.fulfillmentMessages) {
                     conversation = {
-                        who: 'bot',
+                        who: '소통이',
                         content: content
                     }
@@ -57,7 +58,7 @@ function Chatbot() {
         } catch (error) {
             conversation = {
-                who: 'bot',
+                who: '소통이',
                 content: {
                     text: {
                         text: " Error just occured, please check the problem"
@@ -77,13 +78,16 @@ function Chatbot() {
         const eventQueryVariables = {
             event
         }
+
+
+
         try {
             //I will send request to the textQuery ROUTE 
             const response = await Axios.post('/api/dialogflow/eventQuery', eventQueryVariables)
             for (let content of response.data.fulfillmentMessages) {
                 let conversation = {
-                    who: 'bot',
+                    who: '소통이',
                     content: content
                 }
@@ -93,7 +97,7 @@ function Chatbot() {
         } catch (error) {
             let conversation = {
-                who: 'bot',
+                who: '소통이',
                 content: {
                     text: {
                         text: " Error just occured, please check the problem"
@@ -110,7 +114,7 @@ function Chatbot() {
         if (e.key === "Enter") {
             if (!e.target.value) {
-                return alert('you need to type somthing first')
+                return alert('you need to type something first')
             }
             //we will send request to text query route 
@@ -136,7 +140,7 @@ function Chatbot() {
             return <Message key={i} who={message.who} text={message.content.text.text} />
         } else if (message.content && message.content.payload.fields.card) {
-            const AvatarSrc = message.who === 'bot' ? <Icon type="robot" /> : <Icon type="smile" />
+            const AvatarSrc = message.who === '소통이' ? <Icon type="robot" /> : <Icon type="smile" />
             return <div>
                 <List.Item style={{ padding: '1rem' }}>
@@ -175,10 +179,10 @@ function Chatbot() {
     return (
         <div style={{
-            height: 500, width: 700,
+            height: 620, width: 700,
             border: '3px solid black', borderRadius: '7px'
         }}>
-            <div style={{ height: 444, width: '100%', overflow: 'auto' }}>
+            <div style={{ height: 564, width: '100%', overflow: 'auto' }}>
                 {renderMessage(messagesFromRedux)}
--- a/client/src/index.css
View file @c79f207
+++ b/client/src/index.css
View file @c79f207
+@import url('https://fonts.googleapis.com/css2?family=Nanum+Gothic&display=swap');
 * {
   word-break: keep-all;
   overflow-wrap: break-word;
@@ -12,7 +13,7 @@ body {
 body {
   margin: 0;
   padding: 0;
-  font-family: Lato, Helvetica Neue, Arial, Helvetica, sans-serif;
+  font-family: Nanum Gothic,Lato, Helvetica Neue, Arial, Helvetica, sans-serif;
   -webkit-font-smoothing: antialiased;
 }
--- a/index.js
View file @c79f207
+++ b/index.js
View file @c79f207
@@ -4,15 +4,11 @@ const bodyParser = require("body-parser");
 const app = express();
 const config = require("./server/config/keys");
-// const mongoose = require("mongoose");
-// mongoose.connect(config.mongoURI, { useNewUrlParser: true, useUnifiedTopology: true })
-//   .then(() => console.log('MongoDB Connected...'))
-//   .catch(err => console.log(err));
 app.use(bodyParser.urlencoded({ extended: true }));
 app.use(bodyParser.json());
-
+app.use('/api/crawling', require('./server/routes/crawling'));
 app.use('/api/dialogflow', require('./server/routes/dialogflow'));
 // Serve static assets if in production
--- a/server/cheerio/video.js 0 → 100644
View file @c79f207
+++ b/server/cheerio/video.js 0 → 100644
View file @c79f207
+const axios = require('axios');
+const cheerio = require('cheerio');
+
+/*
+한글 utf-8 인코딩해서 searchquery에 저장해야함.
+*/
+
+function video(name){
+    console.log("My favorite singer",name)
+    var url = 'https://tv.naver.com/search/clip?query=' //naverTV의 링크
+    var sort = '&sort=date'
+
+    url = url + name + sort
+    url = encodeURI(url)
+    console.log("url is ",url)
+    const getHtml = async() => {
+        try{
+            return await axios.get(url); //axios.get 함수를 이용해서 비동기로 네이버티비의 해당 가수의 최신 영상 html 파일을 가져온다. 
+        } catch(error){
+            console.log("error! check your code");
+        }
+    };
+
+    getHtml()
+    .then(html => {
+        let videoList = [];
+        const $ = cheerio.load(html.data);
+        const $bodyList = $("div.src_wrap div.thl ").children("div.thl_a");
+
+        $bodyList.each(function(i, elem){
+            videoList[i] = {
+                thumbnail : $(this).find('a.cds_thm').children('img').attr('src'), 
+                title : $(this).find('a.cds_thm').attr('title'),
+                link : "https://tv.naver.com/" + $(this).find('a.cds_thm').attr('href')
+            }
+        })
+
+        const data = videoList.filter(n => n.title);
+        return data;
+    })
+    .then(res=>console.log(res));
+}
+
+module.exports = video;
\ No newline at end of file
--- a/server/routes/crawling.js 0 → 100644
View file @c79f207
+++ b/server/routes/crawling.js 0 → 100644
View file @c79f207
+const express = require('express');
+const router = express.Router();
+const uuid = require('uuid'); //uuid 제대로 이해하고 다시 작성하기.
+const getvideo = require('../cheerio/video'); 
+//@가수명 으로 입력이 들어왔을 때, 가수명만 받아서 
+
+router.post('/textQuery',async(req,res)=>{
+    const result = req.body.text;
+    console.log(result.substring(1));
+    getvideo(result.substring(1));
+    res.send(result);
+})
+
+module.exports = router;
\ No newline at end of file
--- a/server/routes/dialogflow.js
View file @c79f207
+++ b/server/routes/dialogflow.js
View file @c79f207
@@ -15,14 +15,9 @@ const languageCode = config.dialogFlowSessionLanguageCode
 const sessionClient = new dialogflow.SessionsClient();
 const sessionPath = sessionClient.sessionPath(projectId, sessionId);
-// We will make two routes 
-
-
 // Text Query Route
 router.post('/textQuery', async (req, res) => {
-    //We need to send some information that comes from the client to Dialogflow API 
-    // The text query request.
     const request = {
         session: sessionPath,
         queryInput: {