Implement search and graph

Ma Suhyeon
Commit 5007d9154fdc1c6d2ccdf82453a47ff22d4d93d2 5007d915 1 parent 387d7f21
Showing 6 changed files with 118 additions and 9 deletions
src/server/app.go
src/server/data.go
src/server/extraction.go
src/server/go.mod
src/server/go.sum
src/server/graph.go
--- a/src/server/app.go
View file @5007d91
+++ b/src/server/app.go
View file @5007d91
@@ -2,9 +2,11 @@ package main
 import (
 	"fmt"
+	"net/http"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+	"github.com/olivere/elastic/v7"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/jmoiron/sqlx"
@@ -19,6 +21,7 @@ const (
 type App struct {
 	Config Config
 	db     *sqlx.DB
+	es     *elastic.Client
 	echo   *echo.Echo
 }
@@ -28,6 +31,11 @@ func NewApp(config Config) *App {
 	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true", config.Database.User, config.Database.Password, config.Database.Host, config.Database.Name)
 	app.db = sqlx.MustOpen("mysql", dsn)
+	es, err := elastic.NewClient()
+	if err != nil {
+		panic(err)
+	}
+	app.es = es
 	auth := middleware.JWTWithConfig(middleware.JWTConfig{
 		SigningKey: []byte(config.TokenSecret),
@@ -35,6 +43,20 @@ func NewApp(config Config) *App {
 	})
 	app.echo = echo.New()
+
+	app.echo.Use(middleware.CORSWithConfig(middleware.CORSConfig{
+		AllowMethods: []string{
+			http.MethodGet,
+			http.MethodPost,
+			http.MethodPut,
+			http.MethodDelete,
+			http.MethodPatch,
+			http.MethodOptions,
+			http.MethodHead,
+		},
+		AllowHeaders: []string{"*"},
+	}))
+
 	app.echo.POST("/users", app.PostUsers)
 	app.echo.POST("/users/tokens", app.PostTokens)
 	app.echo.POST("/extractions", app.PostExtractions, auth)
@@ -48,6 +70,8 @@ func NewApp(config Config) *App {
 	extraction.GET("/processes", app.GetProcesses)
 	extraction.GET("/alarms", app.GetAlarms)
+	app.echo.GET("/graph/:extractions", app.GetGraph)
+
 	return app
 }
--- a/src/server/data.go
View file @5007d91
+++ b/src/server/data.go
View file @5007d91
 package main
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -9,6 +10,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/labstack/echo/v4"
+	"github.com/olivere/elastic/v7"
 	_ "github.com/mattn/go-sqlite3"
 )
@@ -82,8 +84,32 @@ type Message struct {
 func (app *App) GetMessages(c echo.Context) error {
 	messages := []Message{}
-	query := `SELECT * FROM messages WHERE extraction_no=?`
+	no := c.Param("no")
-	app.db.Unsafe().Select(&messages, query, c.Param("no"))
+
+	q := c.QueryParam("q")
+	if len(q) > 0 {
+		res, err := app.es.Search("messages-" + no).
+			Query(elastic.NewMatchQuery("content", q)).
+			Do(context.Background())
+
+		if err != nil {
+			return err
+		}
+
+		for _, hit := range res.Hits.Hits {
+			message := Message{}
+
+			app.db.Unsafe().Get(
+				&message,
+				"SELECT * FROM messages WHERE extraction_no=? AND `id`=?",
+				no, hit.Id,
+			)
+
+			messages = append(messages, message)
+		}
+	} else {
+		app.db.Unsafe().Select(&messages, `SELECT * FROM messages WHERE extraction_no=?`, no)
+	}
 	return c.JSON(http.StatusOK, messages)
 }
--- a/src/server/extraction.go
View file @5007d91
+++ b/src/server/extraction.go
View file @5007d91
 package main
 import (
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -10,6 +11,8 @@ import (
 	"github.com/dgrijalva/jwt-go"
 	"github.com/jmoiron/sqlx"
 	"github.com/labstack/echo/v4"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 func (app *App) PostExtractions(c echo.Context) error {
@@ -52,13 +55,10 @@ func (app *App) PostExtractions(c echo.Context) error {
 	extNo, _ := res.LastInsertId()
 	rows, err := db.Queryx("SELECT * FROM calllog")
-	fmt.Println(err)
 	if err == nil {
 		for rows.Next() {
 			vals, _ := rows.SliceScan()
-			fmt.Println(vals)
 			_, err = tx.Exec("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
-			fmt.Println(err)
 		}
 	}
@@ -76,14 +76,30 @@ func (app *App) PostExtractions(c echo.Context) error {
 	rows, err = db.Queryx("SELECT mid, type, address, body, date FROM sms")
 	if err == nil {
+		idxName := fmt.Sprintf("messages-%d", extNo)
+		app.es.CreateIndex(idxName).Body(`{
+			"settings": {
+				"analysis": {
+					"analyzer": {
+						"default": {
+							"type": "custom",
+							"tokenizer": "nori_tokenizer"
+						}
+					}
+				}
+			}
+		}`).Do(context.Background())
+
 		for rows.Next() {
 			vals, _ := rows.SliceScan()
 			tx.Exec("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
+			app.es.Index().Index(idxName).Id(fmt.Sprint(vals[0])).BodyJson(echo.Map{"content": string(vals[3].([]byte))}).Do(context.Background())
 		}
 	}
 	rows, err = db.Queryx("SELECT PID, UID, PPID, STIME, TIME, CMD FROM process")
 	if err == nil {
+
 		for rows.Next() {
 			vals, _ := rows.SliceScan()
 			tx.Exec("INSERT INTO processes VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
--- a/src/server/go.mod
View file @5007d91
+++ b/src/server/go.mod
View file @5007d91
@@ -4,13 +4,11 @@ go 1.15
 require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/elastic/go-elasticsearch/v7 v7.12.0
 	github.com/go-sql-driver/mysql v1.5.0
-	github.com/google/uuid v1.1.2
-	github.com/gorilla/handlers v1.5.1
-	github.com/gorilla/mux v1.8.0
 	github.com/jmoiron/sqlx v1.2.0
-	github.com/labstack/echo v3.3.10+incompatible
 	github.com/labstack/echo/v4 v4.2.2
 	github.com/mattn/go-sqlite3 v1.9.0
+	github.com/olivere/elastic/v7 v7.0.24
 	golang.org/x/crypto v0.0.0-20201217014255-9d1352758620
 )
--- a/src/server/go.sum
View file @5007d91
+++ b/src/server/go.sum
View file @5007d91
--- a/src/server/graph.go 0 → 100644
View file @5007d91
+++ b/src/server/graph.go 0 → 100644
View file @5007d91
+package main
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/labstack/echo/v4"
+)
+
+func (app *App) GetGraph(c echo.Context) error {
+	sep := strings.Split(c.Param("extractions"), ",")
+	extractions := map[string]uint64{}
+	for _, s := range sep {
+		var phone string
+		app.db.Get(&phone, "SELECT phone FROM extractions WHERE `no`=?", s)
+		extractions[phone], _ = strconv.ParseUint(s, 10, 64)
+	}
+
+	calls := map[[2]string]int{}
+	for p, e := range extractions {
+		rows, _ := app.db.Query("SELECT `number`, COUNT(1) FROM calls WHERE extraction_no=? GROUP BY `number`", e)
+		for rows.Next() {
+			var number string
+			var count int
+			rows.Scan(&number, &count)
+
+			if _, ok := calls[[2]string{p, number}]; ok {
+				continue
+			}
+
+			calls[[2]string{p, number}] = count
+			calls[[2]string{number, p}] = count
+		}
+	}
+
+	result := []interface{}{}
+	for k, c := range calls {
+		if k[0] < k[1] && c >= 5 {
+			result = append(result, echo.Map{"numbers": k, "calls": c})
+		}
+	}
+
+	return c.JSON(http.StatusOK, result)
+}