20251209
This commit is contained in:
12600k-rog-d4
75
archive/bbs-go/server/internal/pkg/search/common.go
Normal file
75
archive/bbs-go/server/internal/pkg/search/common.go
Normal file
@@ -0,0 +1,75 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/mapping"
|
||||
"github.com/mlogclub/simple/common/jsons"
|
||||
)
|
||||
|
||||
type TopicDocument struct {
|
||||
Id int64 `json:"id"`
|
||||
NodeId int64 `json:"nodeId"`
|
||||
UserId int64 `json:"userId"`
|
||||
Nickname string `json:"nickname"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Tags []string `json:"tags"`
|
||||
Recommend bool `json:"recommend"`
|
||||
Status int `json:"status"`
|
||||
CreateTime int64 `json:"createTime"`
|
||||
}
|
||||
|
||||
func (t *TopicDocument) ToStr() string {
|
||||
str, err := jsons.ToStr(t)
|
||||
if err != nil {
|
||||
slog.Error(err.Error(), slog.Any("err", err))
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
func newIndex(indexPath string) bleve.Index {
|
||||
mapping := bleve.NewIndexMapping()
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("id", newNumField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("nodeId", newNumField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("userId", newNumField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("nickname", newTextField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("title", newTextField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("content", newTextField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("tags", newTextField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("recommend", newBoolField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("status", newNumField())
|
||||
mapping.DefaultMapping.AddFieldMappingsAt("createTime", newNumField())
|
||||
|
||||
index, err := bleve.New(indexPath, mapping)
|
||||
if err != nil {
|
||||
slog.Info("创建索引失败", slog.Any("err", err))
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
func newTextField() *mapping.FieldMapping {
|
||||
textField := bleve.NewTextFieldMapping()
|
||||
// textField.Store = true
|
||||
textField.Index = true
|
||||
textField.IncludeTermVectors = true
|
||||
textField.Analyzer = "en"
|
||||
return textField
|
||||
}
|
||||
|
||||
func newNumField() *mapping.FieldMapping {
|
||||
numField := bleve.NewNumericFieldMapping()
|
||||
// numField.Store = true
|
||||
numField.Index = true
|
||||
numField.DocValues = true
|
||||
return numField
|
||||
}
|
||||
|
||||
func newBoolField() *mapping.FieldMapping {
|
||||
boolField := bleve.NewBooleanFieldMapping()
|
||||
// boolField.Store = true
|
||||
boolField.Index = true
|
||||
boolField.DocValues = true
|
||||
return boolField
|
||||
}
|
||||
192
archive/bbs-go/server/internal/pkg/search/search.go
Normal file
192
archive/bbs-go/server/internal/pkg/search/search.go
Normal file
@@ -0,0 +1,192 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"bbs-go/internal/cache"
|
||||
"bbs-go/internal/models"
|
||||
html2 "bbs-go/internal/pkg/html"
|
||||
"bbs-go/internal/pkg/markdown"
|
||||
"bbs-go/internal/repositories"
|
||||
"html"
|
||||
"log/slog"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/mlogclub/simple/common/dates"
|
||||
"github.com/mlogclub/simple/common/strs"
|
||||
"github.com/mlogclub/simple/sqls"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
var index bleve.Index
|
||||
|
||||
func Init(indexPath string) {
|
||||
var err error
|
||||
if index, err = bleve.Open(indexPath); err != nil {
|
||||
if err == bleve.ErrorIndexPathDoesNotExist {
|
||||
index = newIndex(indexPath)
|
||||
} else {
|
||||
slog.Error(err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NewTopicDoc(topic *models.Topic) *TopicDocument {
|
||||
if topic == nil {
|
||||
return nil
|
||||
}
|
||||
doc := &TopicDocument{
|
||||
Id: topic.Id,
|
||||
NodeId: topic.NodeId,
|
||||
UserId: topic.UserId,
|
||||
Title: topic.Title,
|
||||
Status: topic.Status,
|
||||
Recommend: topic.Recommend,
|
||||
CreateTime: topic.CreateTime,
|
||||
}
|
||||
|
||||
// 处理内容
|
||||
content := markdown.ToHTML(topic.Content)
|
||||
content = html2.GetHtmlText(content)
|
||||
content = html.EscapeString(content)
|
||||
|
||||
doc.Content = content
|
||||
|
||||
// 处理用户
|
||||
user := cache.UserCache.Get(topic.UserId)
|
||||
if user != nil {
|
||||
doc.Nickname = user.Nickname
|
||||
}
|
||||
|
||||
// 处理标签
|
||||
tags := getTopicTags(topic.Id)
|
||||
var tagsArr []string
|
||||
for _, tag := range tags {
|
||||
tagsArr = append(tagsArr, tag.Name)
|
||||
}
|
||||
tagsArr = append(tagsArr, "hello")
|
||||
doc.Tags = tagsArr
|
||||
|
||||
return doc
|
||||
}
|
||||
|
||||
func getTopicTags(topicId int64) []models.Tag {
|
||||
topicTags := repositories.TopicTagRepository.Find(sqls.DB(), sqls.NewCnd().Where("topic_id = ?", topicId))
|
||||
|
||||
var tagIds []int64
|
||||
for _, topicTag := range topicTags {
|
||||
tagIds = append(tagIds, topicTag.TagId)
|
||||
}
|
||||
return cache.TagCache.GetList(tagIds)
|
||||
}
|
||||
|
||||
// IndexData 索引数据
|
||||
func UpdateTopicIndex(topic *models.Topic) {
|
||||
doc := NewTopicDoc(topic)
|
||||
if doc == nil {
|
||||
return
|
||||
}
|
||||
err := index.Index(cast.ToString(topic.Id), doc)
|
||||
if err != nil {
|
||||
slog.Error(err.Error())
|
||||
} else {
|
||||
slog.Info("add topic search index", slog.Any("id", topic.Id))
|
||||
}
|
||||
}
|
||||
|
||||
func DeleteTopicIndex(id int64) error {
|
||||
return index.Delete(cast.ToString(id))
|
||||
}
|
||||
|
||||
// 分页查询
|
||||
func SearchTopic(keyword string, nodeId int64, timeRange, page, limit int) (docs []TopicDocument, paging *sqls.Paging, err error) {
|
||||
paging = &sqls.Paging{Page: page, Limit: limit}
|
||||
|
||||
query := bleve.NewBooleanQuery()
|
||||
query.AddMust(bleve.NewMatchAllQuery())
|
||||
|
||||
if strs.IsNotBlank(keyword) {
|
||||
query.AddMust(bleve.NewMatchQuery(keyword))
|
||||
}
|
||||
|
||||
if nodeId != 0 {
|
||||
if nodeId == -1 { // 推荐
|
||||
boolFieldQuery := bleve.NewBoolFieldQuery(true)
|
||||
boolFieldQuery.SetField("recommend")
|
||||
query.AddMust(boolFieldQuery)
|
||||
} else {
|
||||
f := float64(nodeId)
|
||||
b := true
|
||||
nodeIdQuery := bleve.NewNumericRangeInclusiveQuery(&f, &f, &b, &b)
|
||||
nodeIdQuery.SetField("nodeId")
|
||||
query.AddMust(nodeIdQuery)
|
||||
}
|
||||
}
|
||||
if timeRange != 0 {
|
||||
var beginTime int64
|
||||
if timeRange == 1 { // 一天内
|
||||
beginTime = dates.Timestamp(time.Now().Add(-24 * time.Hour))
|
||||
} else if timeRange == 2 { // 一周内
|
||||
beginTime = dates.Timestamp(time.Now().Add(-7 * 24 * time.Hour))
|
||||
} else if timeRange == 3 { // 一月内
|
||||
beginTime = dates.Timestamp(time.Now().AddDate(0, -1, 0))
|
||||
} else if timeRange == 4 { // 一年内
|
||||
beginTime = dates.Timestamp(time.Now().AddDate(-1, 0, 0))
|
||||
}
|
||||
|
||||
min := float64(beginTime)
|
||||
max := float64(math.MaxInt64)
|
||||
createTimeQuery := bleve.NewNumericRangeQuery(&min, &max)
|
||||
createTimeQuery.SetField("createTime")
|
||||
query.AddMust(createTimeQuery)
|
||||
}
|
||||
|
||||
searchRequest := bleve.NewSearchRequest(query)
|
||||
searchRequest.From = paging.Offset()
|
||||
searchRequest.Size = paging.Limit
|
||||
searchRequest.Fields = []string{"*"}
|
||||
searchRequest.Highlight = bleve.NewHighlightWithStyle("html")
|
||||
searchRequest.Highlight.AddField("title")
|
||||
searchRequest.Highlight.AddField("content")
|
||||
|
||||
result, err := index.Search(searchRequest)
|
||||
if err != nil {
|
||||
slog.Error("搜索失败:", slog.Any("err", err))
|
||||
}
|
||||
|
||||
for _, hit := range result.Hits {
|
||||
|
||||
storedDoc := make(map[string]interface{})
|
||||
for key, field := range hit.Fields {
|
||||
storedDoc[key] = field
|
||||
}
|
||||
|
||||
for field, fragments := range hit.Fragments {
|
||||
if len(fragments) > 0 {
|
||||
storedDoc[field] = fragments[0]
|
||||
}
|
||||
}
|
||||
|
||||
if tagField, ok := storedDoc["tags"]; ok {
|
||||
switch v := tagField.(type) {
|
||||
case string:
|
||||
storedDoc["tags"] = []string{v}
|
||||
case []interface{}:
|
||||
var tags []string
|
||||
for _, tag := range v {
|
||||
tags = append(tags, tag.(string))
|
||||
}
|
||||
storedDoc["tags"] = tags
|
||||
}
|
||||
}
|
||||
|
||||
var doc TopicDocument
|
||||
if err := mapstructure.Decode(storedDoc, &doc); err != nil {
|
||||
slog.Error(err.Error())
|
||||
}
|
||||
docs = append(docs, doc)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
76
archive/bbs-go/server/internal/pkg/search/search_test.go
Normal file
76
archive/bbs-go/server/internal/pkg/search/search_test.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package search_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
)
|
||||
|
||||
type TopicDocument struct {
|
||||
Id int64 `json:"id"`
|
||||
NodeId int64 `json:"nodeId"`
|
||||
UserId int64 `json:"userId"`
|
||||
Nickname string `json:"nickname"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Tags []string `json:"tags"`
|
||||
Recommend bool `json:"recommend"`
|
||||
Status int `json:"status"`
|
||||
CreateTime int64 `json:"createTime"`
|
||||
}
|
||||
|
||||
func TestBleve(t *testing.T) {
|
||||
// 打开或创建索引
|
||||
index, err := bleve.Open("topic_index")
|
||||
if err == bleve.ErrorIndexPathDoesNotExist {
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
index, err = bleve.New("topic_index", indexMapping)
|
||||
// index, err = bleve.NewUsing("topic_index", indexMapping, scorch.Name, scorch.Name, nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
} else if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// 添加文档到索引
|
||||
doc := TopicDocument{
|
||||
Id: 1,
|
||||
NodeId: 1,
|
||||
UserId: 1,
|
||||
Nickname: "user1",
|
||||
Title: "Example Title",
|
||||
Content: "This is an example content.",
|
||||
Tags: []string{"example", "test"},
|
||||
Recommend: true,
|
||||
Status: 1,
|
||||
CreateTime: time.Now().Unix(),
|
||||
}
|
||||
err = index.Index(fmt.Sprintf("%d", doc.Id), doc)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// 执行查询
|
||||
query := bleve.NewMatchQuery("example") // 查询示例关键词 "example"
|
||||
searchRequest := bleve.NewSearchRequest(query)
|
||||
searchRequest.Fields = []string{"id", "title", "content", "createTime"}
|
||||
searchResult, err := index.Search(searchRequest)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// 输出查询结果
|
||||
for _, hit := range searchResult.Hits {
|
||||
var topicDoc TopicDocument
|
||||
err := json.Unmarshal(hit.Fields["title"].(json.RawMessage), &topicDoc)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("Title: %s, Content: %s\n", topicDoc.Title, topicDoc.Content)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user