lian/mchat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tokenizer

Browse Source
This commit is contained in:
1iaan
2026-04-03 10:29:38 +08:00
parent de99cb2806
commit c1a895258f
70 changed files with 22320 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@@ -17,4 +17,7 @@ HUSKY=0 pnpm bootstrap
pnpm dev
# tokenizer
docker build -t tokenizer:1.0.0 .
```

18
ai-chat-service/Dockerfile Normal file
View File

@@ -0,0 +1,18 @@
# 编译阶段
FROM quay.io/0voice/golang:1.20 as stage0
RUN go env -w GOPROXY=https://proxy.golang.com.cn,https://goproxy.cn,direct
ADD ./ /src/ai-chat-service
WORKDIR /src/ai-chat-service
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ai-chat-service ./chat-server
FROM quay.io/0voice/alpine:3.18 as stage1
ADD ./grpc_health_probe-linux-amd64 /usr/bin/grpc_health_probe
RUN chmod +x /usr/bin/grpc_health_probe
MAINTAINER nick
WORKDIR /app/
ADD ./dev.config.yaml /app/config.yaml
COPY --from=stage0 /src/ai-chat-service/ai-chat-service ./
# 指定入口程序
ENTRYPOINT ["./ai-chat-service"]
# 指定容器的启动命令或者入口程序的参数
CMD ["--config=config.yaml"]

20
ai-chat-service/chat-server/chat-context/chat_context.go Normal file
View File

@@ -0,0 +1,20 @@
package chat_context
import "github.com/sashabaranov/go-openai"
type ChatMessage struct {
//当前记录ID
ID string `json:"id,omitempty"`
//上一条记录ID
PID string `json:"pid,omitempty"`
//消息内容
Message openai.ChatCompletionMessage `json:"message"`
//该消息tokens数
Tokens int `json:"tokens,omitempty"`
}
type ContextCache interface {
Get(key string) (*ChatMessage, error)
Set(key string, value *ChatMessage, ttl int) error
Close()
}

50
ai-chat-service/chat-server/chat-context/redis.go Normal file
View File

@@ -0,0 +1,50 @@
package chat_context
import (
predis "ai-chat-service/pkg/db/redis"
"context"
"encoding/json"
"github.com/redis/go-redis/v9"
"time"
)
type redisCache struct {
redisClient *redis.Client
}
func NewRedisCache() ContextCache {
pool := predis.GetPool()
return &redisCache{
redisClient: pool.Get(),
}
}
func getRedisKey(key string) string {
return predis.GetKey(key)
}
func (c *redisCache) Get(key string) (*ChatMessage, error) {
key = getRedisKey(key)
str, err := c.redisClient.Get(context.Background(), key).Result()
if err == redis.Nil {
return nil, nil
}
if err != nil {
return nil, err
}
value := &ChatMessage{}
err = json.Unmarshal([]byte(str), value)
return value, err
}
func (c *redisCache) Set(key string, value *ChatMessage, ttl int) error {
key = getRedisKey(key)
bytes, err := json.Marshal(value)
if err != nil {
return err
}
str := string(bytes)
return c.redisClient.SetEx(context.Background(), key, str, time.Duration(ttl)*time.Second).Err()
}
func (c *redisCache) Close() {
pool := predis.GetPool()
pool.Put(c.redisClient)
}

54
ai-chat-service/chat-server/data/chat_records.go Normal file
View File

@@ -0,0 +1,54 @@
package data
import (
"database/sql"
"strings"
)
type IChatRecordsData interface {
Add(record *ChatRecord) error
GetById(id int64) (record *ChatRecord, err error)
}
type ChatRecord struct {
ID int64 `json:"id"`
UserMsg string `json:"user_msg"`
UserMsgTokens int `json:"user_msg_tokens"`
UserMsgKeywords []string `json:"user_msg_keywords"`
AIMsg string `json:"ai_msg"`
AIMsgTokens int `json:"ai_msg_tokens"`
ReqTokens int `json:"req_tokens"`
CreateAt int64 `json:"create_at"`
}
type chatRecordsData struct {
db *sql.DB
}
func NewChatRecordsData(db *sql.DB) IChatRecordsData {
return &chatRecordsData{
db: db,
}
}
func (data *chatRecordsData) Add(cr *ChatRecord) (err error) {
sqlStr := "insert into chat_records(user_msg,user_msg_tokens,user_msg_keywords,ai_msg,ai_msg_tokens,req_tokens,create_at)values(?,?,?,?,?,?,?)"
res, err := data.db.Exec(sqlStr, cr.UserMsg, cr.UserMsgTokens, strings.Join(cr.UserMsgKeywords, ","), cr.AIMsg, cr.AIMsgTokens, cr.ReqTokens, cr.CreateAt)
if err != nil {
return
}
cr.ID, _ = res.LastInsertId()
return
}
func (data *chatRecordsData) GetById(id int64) (cr *ChatRecord, err error) {
sqlStr := "select id,user_msg,user_msg_tokens,user_msg_keywords,ai_msg,ai_msg_tokens,req_tokens,create_at from chat_records where id = ?"
row := data.db.QueryRow(sqlStr, id)
cr = &ChatRecord{}
var keywords string
err = row.Scan(&cr.ID, &cr.UserMsg, &cr.UserMsgTokens, &keywords, &cr.AIMsg, &cr.AIMsgTokens, &cr.ReqTokens, &cr.CreateAt)
if err != nil {
return nil, err
}
cr.UserMsgKeywords = strings.Split(keywords, ",")
return cr, err
}

78
ai-chat-service/chat-server/main.go Normal file
View File

@@ -0,0 +1,78 @@
package main
import (
"ai-chat-service/chat-server/data"
metrics_app "ai-chat-service/chat-server/metrics-app"
metrics_bus "ai-chat-service/chat-server/metrics-bus"
"ai-chat-service/chat-server/server"
vector_data "ai-chat-service/chat-server/vector-data"
"ai-chat-service/interceptor"
"ai-chat-service/pkg/config"
"ai-chat-service/pkg/db/mysql"
"ai-chat-service/pkg/db/redis"
"ai-chat-service/pkg/db/vector"
"ai-chat-service/pkg/log"
"ai-chat-service/proto"
"flag"
"fmt"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
"google.golang.org/grpc/health/grpc_health_v1"
"net/http"
"net"
)
var (
configFile = flag.String("config", "dev.config.yaml", "")
)
func main() {
flag.Parse()
registry := prometheus.NewRegistry()
registry.MustRegister(collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
busMetrics := metrics_bus.NewBusMetrics(registry)
http.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
go http.ListenAndServe(":8080", nil)
//初始化配置文件
config.InitConfig(*configFile)
cnf := config.GetConfig()
//初始化日志
log.SetLevel(cnf.Log.Level)
log.SetOutput(log.GetRotateWriter(cnf.Log.LogPath))
log.SetPrintCaller(true)
logger := log.NewLogger()
logger.SetLevel(cnf.Log.Level)
logger.SetOutput(log.GetRotateWriter(cnf.Log.LogPath))
logger.SetPrintCaller(true)
// 初始化Mysql
mysql.InitMysql(cnf)
// 初始化redis
redis.InitRedisPool(cnf)
// 初始化向量数据库
vector.InitDB(cnf)
recordsData := data.NewChatRecordsData(mysql.GetDB())
lis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", cnf.Server.IP, cnf.Server.Port))
if err != nil {
log.Fatal(err)
}
s := grpc.NewServer(grpc.UnaryInterceptor(interceptor.UnaryAuthInterceptor), grpc.StreamInterceptor(metrics_app.NewStreamMiddleware(registry).WrapHandler()))
service := server.NewChatService(recordsData, vector_data.NewChatRecordsData(cnf, vector.GetVdb()), cnf, logger, busMetrics)
proto.RegisterChatServer(s, service)
healthCheckSrv := health.NewServer()
grpc_health_v1.RegisterHealthServer(s, healthCheckSrv)
if err = s.Serve(lis); err != nil {
log.Fatal(err)
}
}

82
ai-chat-service/chat-server/metrics-app/metrics_app.go Normal file
View File

@@ -0,0 +1,82 @@
package metrics_app
import (
"github.com/prometheus/client_golang/prometheus"
"google.golang.org/grpc"
"runtime"
"time"
)
type StreamMiddleware interface {
WrapHandler() grpc.StreamServerInterceptor
}
type streamMiddleware struct {
registry *prometheus.Registry
handlerCounter *prometheus.CounterVec
handlerDuration *prometheus.SummaryVec
handlerAtHour *prometheus.HistogramVec
}
const (
NAMESPACE = "ai_chat"
SUBSYSTEM = "chat_service"
)
func NewStreamMiddleware(registry *prometheus.Registry) StreamMiddleware {
counter := prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: NAMESPACE,
Subsystem: SUBSYSTEM,
Name: "requests_total",
ConstLabels: map[string]string{"app": "ai_chat"},
Help: "用于累计请求次数",
}, []string{"full_method"})
gauge := prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Namespace: NAMESPACE,
Subsystem: SUBSYSTEM,
Name: "curr_num_goroutine",
ConstLabels: map[string]string{"app": "ai_chat"},
Help: "当前存在的goroutine数量",
}, func() float64 {
return float64(runtime.NumGoroutine())
})
histogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: NAMESPACE,
Subsystem: SUBSYSTEM,
Name: "request_hour",
ConstLabels: map[string]string{"app": "ai_chat"},
Help: "http请求发生在一天之中的哪个小时",
Buckets: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23},
}, []string{"full_method"})
summary := prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: NAMESPACE,
Subsystem: SUBSYSTEM,
Name: "request_duration_ms",
ConstLabels: map[string]string{"app": "ai_chat"},
Help: "请求时长分布",
Objectives: map[float64]float64{0.1: 0.01, 0.5: 0.01, 0.9: 0.01, 0.99: 0.01},
}, []string{"full_method"})
registry.MustRegister(counter, gauge, histogram, summary)
return &streamMiddleware{
registry: registry,
handlerCounter: counter,
handlerDuration: summary,
handlerAtHour: histogram,
}
}
func (s *streamMiddleware) WrapHandler() grpc.StreamServerInterceptor {
return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
label := map[string]string{
"full_method": info.FullMethod,
}
s.handlerCounter.With(label).Inc()
hour := time.Now().Hour()
s.handlerAtHour.With(label).Observe(float64(hour))
start := time.Now()
defer func() {
s.handlerDuration.With(label).Observe(float64(time.Since(start).Milliseconds()))
}()
err := handler(srv, ss)
return err
}
}

53
ai-chat-service/chat-server/metrics-bus/metrics_bus.go Normal file
View File

@@ -0,0 +1,53 @@
package metrics_bus
import "github.com/prometheus/client_golang/prometheus"
type BusMetrics struct {
QuestionsTotalCounter prometheus.Counter
KeywordsQuestionsTotalCounter prometheus.Counter
SensitiveQuestionsTotalCounter prometheus.Counter
ErrQuestionsTotalCounter prometheus.Counter
}
const (
NAMESPACE = "ai_chat"
SUBSYSTEM = "chat_service"
)
func NewBusMetrics(registry *prometheus.Registry) *BusMetrics {
questionsTotalCounter := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: NAMESPACE,
Subsystem: SUBSYSTEM,
Name: "questions_total",
ConstLabels: map[string]string{"app": "ai_chat"},
Help: "记录用户提交问题的总数，仅包含记录到DB的问题数量",
})
keywordsQuestionsTotalCounter := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: NAMESPACE,
Subsystem: SUBSYSTEM,
Name: "keywords_questions_total",
ConstLabels: map[string]string{"app": "ai_chat"},
Help: "记录用户提交的包含关键词的问题总数",
})
sensitiveQuestionsTotalCounter := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: NAMESPACE,
Subsystem: SUBSYSTEM,
Name: "sensitive_questions_total",
ConstLabels: map[string]string{"app": "ai_chat"},
Help: "记录用户提交的触发敏感词的问题总数",
})
errQuestionsTotalCounter := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: NAMESPACE,
Subsystem: SUBSYSTEM,
Name: "err_questions_total",
ConstLabels: map[string]string{"app": "ai_chat"},
Help: "记录用户提交问题时报错的总数",
})
registry.MustRegister(questionsTotalCounter, keywordsQuestionsTotalCounter, sensitiveQuestionsTotalCounter, errQuestionsTotalCounter)
return &BusMetrics{
QuestionsTotalCounter: questionsTotalCounter,
KeywordsQuestionsTotalCounter: keywordsQuestionsTotalCounter,
SensitiveQuestionsTotalCounter: sensitiveQuestionsTotalCounter,
ErrQuestionsTotalCounter: errQuestionsTotalCounter,
}
}

293
ai-chat-service/chat-server/server/app.go Normal file
View File

@@ -0,0 +1,293 @@
package server
import (
chat_context "ai-chat-service/chat-server/chat-context"
"ai-chat-service/pkg/config"
"ai-chat-service/pkg/log"
"ai-chat-service/pkg/zerror"
"ai-chat-service/proto"
"ai-chat-service/services"
keywords_filter "ai-chat-service/services/keywords-filter"
keywords_proto "ai-chat-service/services/keywords-filter/proto"
"ai-chat-service/services/tokenizer"
"context"
"github.com/google/uuid"
"github.com/sashabaranov/go-openai"
"time"
)
const ChatPrimedTokens = 2
type openaiConf struct {
ApiKey string
BaseUrl string
Model string
MaxTokens int
Temperature float32
TopP float32
PresencePenalty float32
FrequencyPenalty float32
BotDesc string
ContextTTL int
ContextLen int
MinResponseTokens int
}
type app struct {
openaiConf *openaiConf
log log.ILogger
// TODO 内容上下文对象
contextCache chat_context.ContextCache
}
func (s *chatService) newApp(in *proto.ChatCompletionRequest, contextCache chat_context.ContextCache) *app {
conf := &openaiConf{
ApiKey: s.config.Chat.ApiKey,
BaseUrl: s.config.Chat.BaseUrl,
Model: s.config.Chat.Model,
MaxTokens: s.config.Chat.MaxTokens,
Temperature: s.config.Chat.Temperature,
TopP: s.config.Chat.TopP,
PresencePenalty: s.config.Chat.PresencePenalty,
FrequencyPenalty: s.config.Chat.FrequencyPenalty,
BotDesc: s.config.Chat.BotDesc,
ContextTTL: s.config.Chat.ContextTTL,
ContextLen: s.config.Chat.ContextLen,
MinResponseTokens: s.config.Chat.MinResponseTokens,
}
if in.ChatParam != nil {
if in.ChatParam.Model != "" {
conf.Model = in.ChatParam.Model
}
if in.ChatParam.TopP != 0 {
conf.TopP = in.ChatParam.TopP
}
if in.ChatParam.FrequencyPenalty != 0 {
conf.FrequencyPenalty = in.ChatParam.FrequencyPenalty
}
if in.ChatParam.PresencePenalty != 0 {
conf.PresencePenalty = in.ChatParam.PresencePenalty
}
if in.ChatParam.Temperature != 0 {
conf.Temperature = in.ChatParam.Temperature
}
if in.ChatParam.BotDesc != "" {
conf.BotDesc = in.ChatParam.BotDesc
}
if in.ChatParam.MaxTokens != 0 {
conf.MaxTokens = int(in.ChatParam.MaxTokens)
}
if in.ChatParam.ContextTTL != 0 {
conf.ContextTTL = int(in.ChatParam.ContextTTL)
}
if in.ChatParam.ContextLen != 0 {
conf.ContextLen = int(in.ChatParam.ContextLen)
}
if in.ChatParam.MinResponseTokens != 0 {
conf.MinResponseTokens = int(in.ChatParam.MinResponseTokens)
}
}
return &app{
openaiConf: conf,
log: s.log,
contextCache: contextCache,
}
}
func (a *app) getOpenaiClient() *openai.Client {
accessToken := a.openaiConf.ApiKey
config := openai.DefaultConfig(accessToken)
config.BaseURL = a.openaiConf.BaseUrl
client := openai.NewClientWithConfig(config)
return client
}
func (a *app) buildChatCompletionRequest(in *proto.ChatCompletionRequest, stream bool) (req openai.ChatCompletionRequest, tokens, currTokens int, currMessage openai.ChatCompletionMessage, err error) {
//当前消息
currMessage = openai.ChatCompletionMessage{
Role: openai.ChatMessageRoleUser,
Content: in.Message,
}
req = openai.ChatCompletionRequest{
Model: a.openaiConf.Model,
Messages: []openai.ChatCompletionMessage{
currMessage,
},
MaxTokens: a.openaiConf.MinResponseTokens,
Temperature: a.openaiConf.Temperature,
TopP: a.openaiConf.TopP,
PresencePenalty: a.openaiConf.PresencePenalty,
FrequencyPenalty: a.openaiConf.FrequencyPenalty,
Stream: stream,
}
contextList := make([]*chat_context.ChatMessage, 0)
if in.EnableContext {
//从缓存中获取上下文信息
contextList = a.getContext(in.Pid)
}
//重构req.Messages
tokens, currTokens, req.Messages, err = a.rebuildMessages(contextList, currMessage)
if err != nil {
a.log.Error(err)
return
}
req.MaxTokens = a.openaiConf.MaxTokens - tokens
return
}
func (a *app) rebuildMessages(contextList []*chat_context.ChatMessage, currMessage openai.ChatCompletionMessage) (tokens, currTokens int, messages []openai.ChatCompletionMessage, err error) {
var sysMessage openai.ChatCompletionMessage
botTokens := 0
if a.openaiConf.BotDesc != "" {
sysMessage = openai.ChatCompletionMessage{
Role: openai.ChatMessageRoleSystem,
Content: a.openaiConf.BotDesc,
}
botTokens, err = tokenizer.GetTokens(&sysMessage, a.openaiConf.Model)
if err != nil {
a.log.Error(err)
return
}
}
messages = []openai.ChatCompletionMessage{currMessage}
currTokens, err = tokenizer.GetTokens(&currMessage, a.openaiConf.Model)
if err != nil {
a.log.Error(err)
return
}
if currTokens > a.openaiConf.MaxTokens-a.openaiConf.MinResponseTokens-botTokens-ChatPrimedTokens {
err = zerror.NewByMsg("请求消息超限")
a.log.Error(err)
return
}
tokens = currTokens + botTokens + ChatPrimedTokens
if contextList != nil {
for _, item := range contextList {
if tokens+item.Tokens+ChatPrimedTokens > a.openaiConf.MaxTokens-a.openaiConf.MinResponseTokens {
break
}
messages = append(messages, item.Message)
tokens += item.Tokens + ChatPrimedTokens
}
}
for i, j := 0, len(messages)-1; i < j; i, j = i+1, j-1 {
messages[i], messages[j] = messages[j], messages[i]
}
if botTokens > 0 {
messages = append([]openai.ChatCompletionMessage{sysMessage}, messages...)
}
return
}
func (a *app) buildChatCompletionResponse(msg string) *proto.ChatCompletionResponse {
res := &proto.ChatCompletionResponse{
Id: uuid.New().String(),
Object: "chat.completion",
Created: time.Now().Unix(),
Model: a.openaiConf.Model,
Choices: []*proto.ChatCompletionChoice{
{
Message: &proto.ChatCompletionMessage{
Role: openai.ChatMessageRoleAssistant,
Content: msg,
},
FinishReason: "stop",
},
},
Usage: &proto.Usage{
PromptTokens: 0,
CompletionTokens: 0,
TotalTokens: 0,
},
}
return res
}
func (a *app) buildChatCompletionStreamResponse(id, delta, finishReason string) *proto.ChatCompletionStreamResponse {
res := &proto.ChatCompletionStreamResponse{
Id: id,
Object: "chat.completion.chunk",
Created: time.Now().Unix(),
Model: a.openaiConf.Model,
Choices: []*proto.ChatCompletionStreamChoice{
{
Index: 0,
Delta: &proto.ChatCompletionStreamChoiceDelta{
Content: delta,
Role: openai.ChatMessageRoleAssistant,
},
FinishReason: finishReason,
},
},
}
return res
}
func (a *app) buildChatCompletionStreamResponseList(id, msg string) []*proto.ChatCompletionStreamResponse {
list := make([]*proto.ChatCompletionStreamResponse, 0)
for _, delta := range msg {
list = append(list, a.buildChatCompletionStreamResponse(id, string(delta), ""))
}
return list
}
func (a *app) getContext(id string) []*chat_context.ChatMessage {
maxLen := a.openaiConf.ContextLen
list := make([]*chat_context.ChatMessage, 0, maxLen)
key := id
for i := 0; i < maxLen; i++ {
value, err := a.contextCache.Get(key)
if err != nil {
a.log.Error(err)
return nil
}
if value == nil {
break
}
list = append(list, value)
key = value.PID
}
return list
}
func (a *app) saveContext(value *chat_context.ChatMessage) error {
err := a.contextCache.Set(value.ID, value, a.openaiConf.ContextTTL)
if err != nil {
a.log.Error(err)
return err
}
return nil
}
func (a *app) keywords(in *proto.ChatCompletionRequest) []string {
pool := keywords_filter.GetKeywordsClientPool()
conn := pool.Get()
defer pool.Put(conn)
accessToken := config.GetConfig().DependOn.Keywords.AccessToken
client := keywords_proto.NewFilterClient(conn)
ctx := services.AppendBearerTokenToContext(context.Background(), accessToken)
req := &keywords_proto.FilterReq{
Text: in.Message,
}
res, err := client.FindAll(ctx, req)
if err != nil {
a.log.Error(err)
return []string{}
}
return res.Keywords
}
func (a *app) sensitive(in *proto.ChatCompletionRequest) (ok bool, msg string, err error) {
pool := keywords_filter.GetSensitiveClientPool()
conn := pool.Get()
defer pool.Put(conn)
accessToken := config.GetConfig().DependOn.Sensitive.AccessToken
client := keywords_proto.NewFilterClient(conn)
ctx := services.AppendBearerTokenToContext(context.Background(), accessToken)
req := &keywords_proto.FilterReq{
Text: in.Message,
}
res, err := client.Validate(ctx, req)
if err != nil {
a.log.Error(err)
return false, "", err
}
ok = res.Ok
if !ok {
msg = "触发到了知识盲区，请换个问题再问"
}
return
}

352
ai-chat-service/chat-server/server/server.go Normal file
View File

@@ -0,0 +1,352 @@
package server
import (
chat_context "ai-chat-service/chat-server/chat-context"
"ai-chat-service/chat-server/data"
metrics_bus "ai-chat-service/chat-server/metrics-bus"
vector_data "ai-chat-service/chat-server/vector-data"
"ai-chat-service/pkg/config"
"ai-chat-service/pkg/log"
"ai-chat-service/proto"
"ai-chat-service/services/tokenizer"
"context"
"encoding/json"
"github.com/golang/protobuf/jsonpb"
"github.com/google/uuid"
"github.com/sashabaranov/go-openai"
"io"
"strconv"
"strings"
"time"
)
type chatService struct {
proto.UnimplementedChatServer
config *config.Config
log log.ILogger
data data.IChatRecordsData
vectorData vector_data.IChatRecordsData
busMetrics *metrics_bus.BusMetrics
}
func NewChatService(data data.IChatRecordsData, vectorData vector_data.IChatRecordsData, config *config.Config, log log.ILogger, busMetrics *metrics_bus.BusMetrics) proto.ChatServer {
return &chatService{
config: config,
log: log,
data: data,
vectorData: vectorData,
busMetrics: busMetrics,
}
}
func (s *chatService) ChatCompletion(ctx context.Context, in *proto.ChatCompletionRequest) (*proto.ChatCompletionResponse, error) {
redisContextCache := chat_context.NewRedisCache()
defer redisContextCache.Close()
app := s.newApp(in, redisContextCache)
//敏感词过滤
ok, msg, err := app.sensitive(in)
if err != nil {
s.log.Error(err)
return nil, err
}
if !ok {
res := app.buildChatCompletionResponse(msg)
return res, nil
}
//关键词提取
keywords := app.keywords(in)
if len(keywords) > 0 {
idStr, score, err := s.vectorData.QueryData(context.Background(), map[string][]string{"keywords": {strings.Join(keywords, ",")}})
if err != nil {
s.log.Error(err)
} else if score > 0.99 {
id, err := strconv.ParseInt(idStr, 10, 64)
if err != nil {
s.log.Error(err)
} else {
record, err := s.data.GetById(id)
if err != nil {
s.log.Error(err)
} else {
res := app.buildChatCompletionResponse(record.AIMsg)
return res, nil
}
}
}
}
client := app.getOpenaiClient()
req, tokens, currTokens, currMessage, err := app.buildChatCompletionRequest(in, false)
resp, err := client.CreateChatCompletion(ctx, req)
if err != nil {
s.log.Error(err)
return nil, err
}
res := &proto.ChatCompletionResponse{}
bytes, err := json.Marshal(resp)
if err != nil {
s.log.Error(err)
return nil, err
}
err = jsonpb.UnmarshalString(string(bytes), res)
if err != nil {
s.log.Error(err)
return nil, err
}
go func() {
reqContext := &chat_context.ChatMessage{
ID: in.Id,
PID: in.Pid,
Message: currMessage,
Tokens: currTokens,
}
err := app.saveContext(reqContext)
if err != nil {
s.log.Error(err)
return
}
resContext := &chat_context.ChatMessage{
ID: resp.ID,
PID: reqContext.ID,
Message: resp.Choices[0].Message,
Tokens: resp.Usage.CompletionTokens,
}
err = app.saveContext(resContext)
if err != nil {
s.log.Error(err)
return
}
}()
go func() {
records := &data.ChatRecord{
UserMsg: in.Message,
UserMsgTokens: currTokens,
UserMsgKeywords: keywords,
AIMsg: resp.Choices[0].Message.Content,
AIMsgTokens: resp.Usage.CompletionTokens,
ReqTokens: tokens,
CreateAt: time.Now().Unix(),
}
err := s.data.Add(records)
if err != nil {
s.log.Error(err)
return
}
//保存到向量数据库
if len(keywords) > 0 {
list := []*vector_data.ChatRecord{
{
ID: strconv.FormatInt(records.ID, 10),
KVs: map[string]string{
"keywords": strings.Join(keywords, ","),
},
},
}
err = s.vectorData.UpsertData(context.Background(), list)
if err != nil {
s.log.Error(err)
return
}
}
}()
return res, err
}
func (s *chatService) ChatCompletionStream(in *proto.ChatCompletionRequest, stream proto.Chat_ChatCompletionStreamServer) error {
redisContextCache := chat_context.NewRedisCache()
defer redisContextCache.Close()
app := s.newApp(in, redisContextCache)
//敏感词过滤
ok, msg, err := app.sensitive(in)
if err != nil {
s.busMetrics.ErrQuestionsTotalCounter.Inc()
s.log.Error(err)
return err
}
if !ok {
s.busMetrics.SensitiveQuestionsTotalCounter.Inc()
resId := uuid.New().String()
startRes := app.buildChatCompletionStreamResponse(resId, "", "")
endRes := app.buildChatCompletionStreamResponse(resId, "", "stop")
err = stream.Send(startRes)
if err != nil {
s.log.Error(err)
return err
}
resList := app.buildChatCompletionStreamResponseList(resId, msg)
for _, res := range resList {
err = stream.Send(res)
if err != nil {
s.log.Error(err)
return err
}
}
err = stream.Send(endRes)
if err != nil {
s.log.Error(err)
return err
}
return nil
}
//关键词提取
keywords := app.keywords(in)
if len(keywords) > 0 {
s.busMetrics.KeywordsQuestionsTotalCounter.Inc()
idStr, score, err := s.vectorData.QueryData(context.Background(), map[string][]string{"keywords": {strings.Join(keywords, ",")}})
if err != nil {
s.log.Error(err)
} else if score > 0.99 {
id, err := strconv.ParseInt(idStr, 10, 64)
if err != nil {
s.log.Error(err)
} else {
record, err := s.data.GetById(id)
if err != nil {
s.log.Error(err)
} else {
resId := uuid.New().String()
startRes := app.buildChatCompletionStreamResponse(resId, "", "")
endRes := app.buildChatCompletionStreamResponse(resId, "", "stop")
err = stream.Send(startRes)
if err != nil {
s.log.Error(err)
return err
}
resList := app.buildChatCompletionStreamResponseList(resId, record.AIMsg)
for _, res := range resList {
err = stream.Send(res)
if err != nil {
s.log.Error(err)
return err
}
}
err = stream.Send(endRes)
if err != nil {
s.log.Error(err)
return err
}
return nil
}
}
}
}
client := app.getOpenaiClient()
req, tokens, currTokens, currMessage, err := app.buildChatCompletionRequest(in, false)
chatStream, err := client.CreateChatCompletionStream(stream.Context(), req)
if err != nil {
s.busMetrics.ErrQuestionsTotalCounter.Inc()
s.log.Error(err)
return err
}
defer chatStream.Close()
completionContent := ""
resultID := ""
for {
resp, err := chatStream.Recv()
if err != nil && err != io.EOF {
s.log.Error(err)
return err
}
if err == io.EOF {
break
}
if resultID == "" {
resultID = resp.ID
}
completionContent += resp.Choices[0].Delta.Content
res := &proto.ChatCompletionStreamResponse{}
bytes, err := json.Marshal(resp)
if err != nil {
s.log.Error(err)
return err
}
err = jsonpb.UnmarshalString(string(bytes), res)
if err != nil {
s.log.Error(err)
return err
}
err = stream.Send(res)
if err != nil {
s.log.Error(err)
return err
}
}
resultMessage := openai.ChatCompletionMessage{
Role: openai.ChatMessageRoleAssistant,
Content: completionContent,
}
model := s.config.Chat.Model
if in.ChatParam != nil && in.ChatParam.Model != "" {
model = in.ChatParam.Model
}
resultTokens, err := tokenizer.GetTokens(&resultMessage, model)
if err != nil {
s.busMetrics.ErrQuestionsTotalCounter.Inc()
s.log.Error(err)
return err
}
go func() {
reqContext := &chat_context.ChatMessage{
ID: in.Id,
PID: in.Pid,
Message: currMessage,
Tokens: currTokens,
}
err := app.saveContext(reqContext)
if err != nil {
s.log.Error(err)
return
}
resContext := &chat_context.ChatMessage{
ID: resultID,
PID: reqContext.ID,
Message: resultMessage,
Tokens: resultTokens,
}
err = app.saveContext(resContext)
if err != nil {
s.log.Error(err)
return
}
}()
go func() {
s.busMetrics.QuestionsTotalCounter.Inc()
records := &data.ChatRecord{
UserMsg: in.Message,
UserMsgTokens: currTokens,
UserMsgKeywords: keywords,
AIMsg: completionContent,
AIMsgTokens: resultTokens,
ReqTokens: tokens,
CreateAt: time.Now().Unix(),
}
err := s.data.Add(records)
if err != nil {
s.log.Error(err)
return
}
//保存到向量数据库
if len(keywords) > 0 {
list := []*vector_data.ChatRecord{
{
ID: strconv.FormatInt(records.ID, 10),
KVs: map[string]string{
"keywords": strings.Join(keywords, ","),
},
},
}
err = s.vectorData.UpsertData(context.Background(), list)
if err != nil {
s.log.Error(err)
return
}
}
}()
return nil
}

69
ai-chat-service/chat-server/vector-data/chat_records.go Normal file
View File

@@ -0,0 +1,69 @@
package vector_data
import (
"ai-chat-service/pkg/config"
"context"
"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
)
const CHAT_RECORDS = "chat_records"
type ChatRecord struct {
ID string
KVs map[string]string
}
type IChatRecordsData interface {
UpsertData(ctx context.Context, list []*ChatRecord) error
QueryData(ctx context.Context, text map[string][]string) (id string, score float32, err error)
}
type chatRecordsData struct {
config *config.Config
vectorDB *tcvectordb.Client
}
func NewChatRecordsData(config *config.Config, vectorDB *tcvectordb.Client) IChatRecordsData {
return &chatRecordsData{
config: config,
vectorDB: vectorDB,
}
}
func (data *chatRecordsData) UpsertData(ctx context.Context, list []*ChatRecord) error {
database := data.config.VectorDB.Database
collection := CHAT_RECORDS
coll := data.vectorDB.Database(database).Collection(collection)
documentList := make([]tcvectordb.Document, 0, len(list))
for _, l := range list {
doc := tcvectordb.Document{
Id: l.ID,
}
doc.Fields = make(map[string]tcvectordb.Field, len(l.KVs))
for k, v := range l.KVs {
doc.Fields[k] = tcvectordb.Field{Val: v}
}
documentList = append(documentList, doc)
}
_, err := coll.Upsert(ctx, documentList)
if err != nil {
return err
}
return nil
}
func (data *chatRecordsData) QueryData(ctx context.Context, text map[string][]string) (id string, score float32, err error) {
database := data.config.VectorDB.Database
collection := CHAT_RECORDS
coll := data.vectorDB.Database(database).Collection(collection)
result, err := coll.SearchByText(ctx, text, &tcvectordb.SearchDocumentParams{
Params: &tcvectordb.SearchDocParams{Ef: 100},
Limit: 1,
})
if err != nil {
return "", 0, err
}
if len(result.Documents) > 0 && len(result.Documents[0]) > 0 {
doc := result.Documents[0][0]
return doc.Id, doc.Score, nil
}
return "", 0, nil
}

72
ai-chat-service/dev.config.yaml Normal file
View File

@@ -0,0 +1,72 @@
server:
ip: 0.0.0.0
port: 50055
accessToken: "me256487ang1chubdpdialoud22sev1ozhoguumyqca"
log:
# panic,fatal,error,warn,warning,info,debug,trace
level: "info"
logPath: "runtime/logs/app.log"
chat:
# openai key
api_key: "i0jey84SdkFdw5u43780yjr3h7se8nth0yi295nr94ksDngKprEh"
# openai 接口地址
base_url: "http://localhost:8084/v1"
# 使用的训练模型
model: "gpt-3.5-turbo"
# 单次请求的上下文总长度,包括：请求消息+响应消息
max_tokens: 4096
# 表示语言模型输出的随机性和创造性
# 取值范围 0 ~ 1，值越大随机性越高
temperature: 0.8
# 用于生成文本时控制选词的随机程度
# 即下一个预测单词考虑的概率范围
# 取值范围0 ~ 1
top_p: 0.9
# 存在惩罚，用于生成文本时控制重复使用单词的程度
# 取值范围 0 ~ 1 ，0表示不惩罚，1表示禁止重复
presence_penalty: 0.8
# 用于控制模型生成回复时重复单词出现的频率
# 取值范围 0~1，值越大表示回复时会更注重避免使用已经出现的单词
frequency_penalty: 0.5
# AI助手特征描述
bot_desc: "你是一个AI助手，我需要你模拟一名资深的软件工程师来回答我的问题"
# 单次请求，保留的响应tokens数量
min_response_tokens: 2048
# 上下文缓存时长，单位s
context_ttl: 1800
# 上下文消息条数
context_len: 4
redis:
host: "192.168.239.161"
port: 6379
pwd: "123456"
mysql:
dsn: "root:123456@tcp(192.168.239.161:3306)/ai_chat?collation=utf8mb4_unicode_ci&charset=utf8mb4"
maxLifeTime: 3600
maxOpenConn: 10
maxIdleConn: 10
dependOn:
sensitive:
address: "localhost:50053"
accessToken: "ang1chubdev1ozhome256487d22sapguuv1ozhom"
keywords:
address: "localhost:50054"
accessToken: "ang1chubdev1ozhome256487d22sapguuv1ozhom"
tokenizer:
address: "http://192.168.239.161:3002"
vectorDB:
# 访问地址
url: "http://lb-4u4r1fk4-1ys6gv3rpmdan420.clb.ap-guangzhou.tencentclb.com:60000"
# 用户名
username: "root"
# 密码
pwd: "YaUfVueWZJ20e4ghyLlBT8Dou5OapwpFTUq50oft"
database: "ai-chat"
# 请求超时时长s
timeout: 5
# 最大空闲连接数
maxIdleConnPerHost: 2
# 读一致性： strongConsistency(强一致性)，eventualConsistency(最终一致性)
readConsistency: "eventualConsistency"
# 空闲连接超时时长s
idleConnTimeout: 60

56
ai-chat-service/go.mod Normal file
View File

@@ -0,0 +1,56 @@
module ai-chat-service
go 1.20
require (
github.com/go-sql-driver/mysql v1.8.1
github.com/golang/protobuf v1.5.4
github.com/google/uuid v1.6.0
github.com/prometheus/client_golang v1.20.4
github.com/redis/go-redis/v9 v9.6.1
github.com/sashabaranov/go-openai v1.9.4
github.com/sirupsen/logrus v1.9.3
github.com/spf13/viper v1.19.0
github.com/tencent/vectordatabase-sdk-go v1.3.5
google.golang.org/grpc v1.65.0
google.golang.org/protobuf v1.34.2
gopkg.in/natefinch/lumberjack.v2 v2.2.1
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/clbanning/mxj v1.8.4 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/google/go-querystring v1.0.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tencentyun/cos-go-sdk-v5 v0.7.54 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

2740
ai-chat-service/go.sum Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
ai-chat-service/grpc_health_probe-linux-amd64 Normal file
View File

Binary file not shown.

38
ai-chat-service/interceptor/auth.go Normal file
View File

@@ -0,0 +1,38 @@
package interceptor
import (
"ai-chat-service/pkg/config"
"ai-chat-service/pkg/zerror"
"context"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"strings"
)
func UnaryAuthInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) {
if info.FullMethod != "/grpc.health.v1.Health/Check" {
err = oauth2Valid(ctx)
if err != nil {
return
}
}
return handler(ctx, req)
}
func oauth2Valid(ctx context.Context) error {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return zerror.NewByMsg("元数据获取失败")
}
authorization := md["authorization"]
if len(authorization) < 1 {
return zerror.NewByMsg("元数据获取失败")
}
token := strings.TrimPrefix(authorization[0], "Bearer ")
cnf := config.GetConfig()
if token != cnf.Server.AccessToken {
return zerror.NewByMsg("鉴权失败")
}
return nil
}

90
ai-chat-service/pkg/config/config.go Normal file
View File

@@ -0,0 +1,90 @@
package config
import (
"github.com/spf13/viper"
"log"
)
type Config struct {
Server struct {
IP string
Port int
AccessToken string
}
Log struct {
Level string
LogPath string `mapstructure:"logPath"`
} `mapstructure:"log"`
Chat struct {
ApiKey string `mapstructure:"api_key"`
BaseUrl string `mapstructure:"base_url"`
Model string `mapstructure:"model"`
MaxTokens int `mapstructure:"max_tokens"`
Temperature float32 `mapstructure:"temperature"`
TopP float32 `mapstructure:"top_p"`
PresencePenalty float32 `mapstructure:"presence_penalty"`
FrequencyPenalty float32 `mapstructure:"frequency_penalty"`
BotDesc string `mapstructure:"bot_desc"`
MinResponseTokens int `mapstructure:"min_response_tokens"`
ContextTTL int `mapstructure:"context_ttl"`
ContextLen int `mapstructure:"context_len"`
}
Mysql struct {
DSN string
MaxLifeTime int
MaxOpenConn int
MaxIdleConn int
}
Redis struct {
Host string
Port int
Pwd string `mapstructure:"pwd"`
}
DependOn struct {
Sensitive struct {
Address string
AccessToken string
}
Keywords struct {
Address string
AccessToken string
}
Tokenizer struct {
Address string
}
}
VectorDB struct {
Url string
Username string
Pwd string
Database string
Timeout int
MaxIdleConnPerHost int
ReadConsistency string
IdleConnTimeout int
}
}
var conf *Config
func InitConfig(filePath string, typ ...string) {
v := viper.New()
v.SetConfigFile(filePath)
if len(typ) > 0 {
v.SetConfigType(typ[0])
}
err := v.ReadInConfig()
if err != nil {
log.Fatal(err)
}
conf = &Config{}
err = v.Unmarshal(conf)
if err != nil {
log.Fatal(err)
}
}
func GetConfig() *Config {
return conf
}

28
ai-chat-service/pkg/db/mysql/mysql.go Normal file
View File

@@ -0,0 +1,28 @@
package mysql
import (
"ai-chat-service/pkg/config"
"database/sql"
_ "github.com/go-sql-driver/mysql"
"time"
)
var db *sql.DB
func InitMysql(cnf *config.Config) {
var err error
if cnf.Mysql.DSN == "" {
panic("数据库连接字符串不能为空")
}
db, err = sql.Open("mysql", cnf.Mysql.DSN)
if err != nil {
panic(err)
}
db.SetMaxOpenConns(cnf.Mysql.MaxOpenConn)
db.SetMaxIdleConns(cnf.Mysql.MaxIdleConn)
db.SetConnMaxLifetime(time.Second * time.Duration(cnf.Mysql.MaxLifeTime))
}
func GetDB() *sql.DB {
return db
}

14
ai-chat-service/pkg/db/redis/prefix.go Normal file
View File

@@ -0,0 +1,14 @@
package redis
import "strings"
const ServicePrefix = "ai_chat_service_"
func GetKey(key string, parts ...string) string {
key = ServicePrefix + key
if len(parts) == 0 {
return key
}
key += "_" + strings.Join(parts, "_")
return key
}

57
ai-chat-service/pkg/db/redis/redis.go Normal file
View File

@@ -0,0 +1,57 @@
package redis
import (
"ai-chat-service/pkg/config"
"context"
"fmt"
redis "github.com/redis/go-redis/v9"
"sync"
)
type RedisPool interface {
Get() *redis.Client
Put(client *redis.Client)
}
var pool RedisPool
type redisPool struct {
pool sync.Pool
}
func (p *redisPool) Get() *redis.Client {
client := p.pool.Get().(*redis.Client)
if client.Ping(context.Background()).Err() != nil {
client = p.pool.New().(*redis.Client)
}
return client
}
func (p *redisPool) Put(client *redis.Client) {
if client.Ping(context.Background()).Err() != nil {
return
}
p.pool.Put(client)
}
func getPool(cnf *config.Config) RedisPool {
return &redisPool{
pool: sync.Pool{
New: func() any {
rdb := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%d", cnf.Redis.Host, cnf.Redis.Port),
Password: cnf.Redis.Pwd,
})
return rdb
},
},
}
}
func InitRedisPool(cnf *config.Config) {
pool = getPool(cnf)
}
func GetPool() RedisPool {
return pool
}

29
ai-chat-service/pkg/db/vector/vector.go Normal file
View File

@@ -0,0 +1,29 @@
package vector
import (
"ai-chat-service/pkg/config"
"ai-chat-service/pkg/log"
"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
"time"
)
var vdb *tcvectordb.Client
func InitDB(config *config.Config) {
var defaultOption = &tcvectordb.ClientOption{
Timeout: time.Second * time.Duration(config.VectorDB.Timeout),
MaxIdldConnPerHost: config.VectorDB.MaxIdleConnPerHost,
IdleConnTimeout: time.Second * time.Duration(config.VectorDB.IdleConnTimeout),
ReadConsistency: tcvectordb.ReadConsistency(config.VectorDB.ReadConsistency),
}
var err error
vdb, err = tcvectordb.NewClient(config.VectorDB.Url, config.VectorDB.Username, config.VectorDB.Pwd, defaultOption)
if err != nil {
log.Error(err)
return
}
}
func GetVdb() *tcvectordb.Client {
return vdb
}

3
ai-chat-service/pkg/log/README.md Normal file
View File

@@ -0,0 +1,3 @@
# 日志框架
1. 可通过包调用日志打印，也可以通过对象调用日志打印
2. 可以自动切分日志文件

19
ai-chat-service/pkg/log/hook.go Normal file
View File

@@ -0,0 +1,19 @@
package log
import "github.com/sirupsen/logrus"
import nativeLog "log"
type errorHook struct {
}
func (*errorHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.PanicLevel,
logrus.FatalLevel,
logrus.ErrorLevel,
}
}
func (*errorHook) Fire(entry *logrus.Entry) error {
nativeLog.Println(entry.Message, entry.Data)
return nil
}

233
ai-chat-service/pkg/log/log.go Normal file
View File

@@ -0,0 +1,233 @@
package log
import (
"errors"
"fmt"
"github.com/sirupsen/logrus"
"io"
"runtime"
)
type ILogger interface {
SetLevel(lvl string)
SetOutput(writer io.Writer)
SetPrintCaller(bool)
SetCaller(caller func() (file string, line int, funcName string, err error))
Trace(args ...interface{})
Debug(args ...interface{})
Info(args ...interface{})
Warning(args ...interface{})
Error(args ...interface{})
Fatal(args ...interface{})
Panic(args ...interface{})
TraceF(format string, args ...interface{})
DebugF(format string, args ...interface{})
InfoF(format string, args ...interface{})
WarningF(format string, args ...interface{})
ErrorF(format string, args ...interface{})
FatalF(format string, args ...interface{})
PanicF(format string, args ...interface{})
WithFields(fields map[string]interface{}) ILogger
}
type Logger struct {
entry *logrus.Entry
// panic,fatal,error,warn,warning,info,debug,trace
level string
printCaller bool
caller func() (file string, line int, funcName string, err error)
}
// 设置日志打印级别
func (l *Logger) SetLevel(lvl string) {
if lvl == "" {
return
}
level, err := logrus.ParseLevel(lvl)
if err == nil {
l.level = lvl
l.entry.Logger.Level = level
}
}
// 设置日志输出位置
func (l *Logger) SetOutput(writer io.Writer) {
l.entry.Logger.SetOutput(writer)
}
// 设置是否打印调用信息
func (l *Logger) SetPrintCaller(printCaller bool) {
l.printCaller = printCaller
}
func (l *Logger) SetCaller(caller func() (file string, line int, funcName string, err error)) {
l.caller = caller
}
// 获取caller信息
func (l *Logger) getCallerInfo(level logrus.Level) map[string]interface{} {
mp := make(map[string]interface{})
if l.printCaller == true || level != logrus.InfoLevel {
file, line, funcName, err := l.caller()
if err == nil {
mp["file"] = fmt.Sprintf("%s:%d", file, line)
mp["func"] = funcName
}
}
return mp
}
func (l *Logger) log(level logrus.Level, args ...interface{}) {
l.entry.WithFields(l.getCallerInfo(level)).Log(level, args...)
}
func (l *Logger) logf(level logrus.Level, format string, args ...interface{}) {
l.entry.WithFields(l.getCallerInfo(level)).Logf(level, format, args...)
}
func (l *Logger) Trace(args ...interface{}) {
l.log(logrus.TraceLevel, args...)
}
func (l *Logger) Debug(args ...interface{}) {
l.log(logrus.DebugLevel, args...)
}
func (l *Logger) Info(args ...interface{}) {
l.log(logrus.InfoLevel, args...)
}
func (l *Logger) Warning(args ...interface{}) {
l.log(logrus.WarnLevel, args...)
}
func (l *Logger) Error(args ...interface{}) {
l.log(logrus.ErrorLevel, args...)
}
func (l *Logger) Fatal(args ...interface{}) {
l.log(logrus.FatalLevel, args...)
}
func (l *Logger) Panic(args ...interface{}) {
l.log(logrus.PanicLevel, args...)
}
func (l *Logger) TraceF(format string, args ...interface{}) {
l.logf(logrus.TraceLevel, format, args...)
}
func (l *Logger) DebugF(format string, args ...interface{}) {
l.logf(logrus.DebugLevel, format, args...)
}
func (l *Logger) InfoF(format string, args ...interface{}) {
l.logf(logrus.InfoLevel, format, args...)
}
func (l *Logger) WarningF(format string, args ...interface{}) {
l.logf(logrus.WarnLevel, format, args...)
}
func (l *Logger) ErrorF(format string, args ...interface{}) {
l.logf(logrus.ErrorLevel, format, args...)
}
func (l *Logger) FatalF(format string, args ...interface{}) {
l.logf(logrus.FatalLevel, format, args...)
}
func (l *Logger) PanicF(format string, args ...interface{}) {
l.logf(logrus.PanicLevel, format, args...)
}
func (l *Logger) WithFields(fields map[string]interface{}) ILogger {
entry := l.entry.WithFields(fields)
return &Logger{entry: entry, level: l.level, printCaller: l.printCaller, caller: l.caller}
}
var log *Logger
func NewLogger() ILogger {
return newLogger()
}
func newLogger() *Logger {
log := logrus.New()
log.SetLevel(logrus.InfoLevel)
log.AddHook(&errorHook{})
logger := &Logger{
entry: logrus.NewEntry(log),
caller: defaultCaller,
}
return logger
}
func init() {
log = newLogger()
}
// 设置日志打印级别
func SetLevel(lvl string) {
if lvl == "" {
return
}
level, err := logrus.ParseLevel(lvl)
if err == nil {
log.level = lvl
log.entry.Logger.Level = level
}
}
// 设置日志的输出位置
func SetOutput(writer io.Writer) {
log.entry.Logger.SetOutput(writer)
}
// 设置是否打印调用信息
func SetPrintCaller(printCaller bool) {
log.printCaller = printCaller
}
func SetCaller(caller func() (file string, line int, funcName string, err error)) {
log.caller = caller
}
func defaultCaller() (file string, line int, funcName string, err error) {
pc, f, l, ok := runtime.Caller(4)
if !ok {
err = errors.New("caller failure")
return
}
funcName = runtime.FuncForPC(pc).Name()
file, line = f, l
return
}
func Trace(args ...interface{}) {
log.log(logrus.TraceLevel, args...)
}
func Debug(args ...interface{}) {
log.log(logrus.DebugLevel, args...)
}
func Info(args ...interface{}) {
log.log(logrus.InfoLevel, args...)
}
func Warning(args ...interface{}) {
log.log(logrus.WarnLevel, args...)
}
func Error(args ...interface{}) {
log.log(logrus.ErrorLevel, args...)
}
func Fatal(args ...interface{}) {
log.log(logrus.FatalLevel, args...)
}
func Panic(args ...interface{}) {
log.log(logrus.PanicLevel, args...)
}
func TraceF(format string, args ...interface{}) {
log.logf(logrus.TraceLevel, format, args...)
}
func DebugF(format string, args ...interface{}) {
log.logf(logrus.DebugLevel, format, args...)
}
func InfoF(format string, args ...interface{}) {
log.logf(logrus.InfoLevel, format, args...)
}
func WarningF(format string, args ...interface{}) {
log.logf(logrus.WarnLevel, format, args...)
}
func ErrorF(format string, args ...interface{}) {
log.logf(logrus.ErrorLevel, format, args...)
}
func FatalF(format string, args ...interface{}) {
log.logf(logrus.FatalLevel, format, args...)
}
func PanicF(format string, args ...interface{}) {
log.logf(logrus.PanicLevel, format, args...)
}
func WithFields(fields map[string]interface{}) *Logger {
entry := log.entry.WithFields(fields)
return &Logger{entry: entry, level: log.level, printCaller: log.printCaller, caller: log.caller}
}

58
ai-chat-service/pkg/log/rotate_writer.go Normal file
View File

@@ -0,0 +1,58 @@
package log
import (
"gopkg.in/natefinch/lumberjack.v2"
"io"
"sync"
)
type fileRotateWriter struct {
data map[string]io.Writer
sync.RWMutex
}
func (frw *fileRotateWriter) getWriter(logPath string) io.Writer {
frw.RLock()
defer frw.RUnlock()
w, ok := frw.data[logPath]
if !ok {
return nil
}
return w
}
func (frw *fileRotateWriter) setWriter(logPath string, w io.Writer) io.Writer {
frw.Lock()
defer frw.Unlock()
frw.data[logPath] = w
return w
}
var _fileRotateWriter *fileRotateWriter
func init() {
_fileRotateWriter = &fileRotateWriter{
data: map[string]io.Writer{},
}
}
func GetRotateWriter(logPath string) io.Writer {
if logPath == "" {
panic("日志文件路径不能为空")
}
writer := _fileRotateWriter.getWriter(logPath)
if writer != nil {
return writer
}
writer = &lumberjack.Logger{
//文件名
Filename: logPath,
//单个文件大小单位MB
MaxSize: 1,
//最多保留文件数
MaxBackups: 15,
//最长保留时间（天）
MaxAge: 7,
LocalTime: true,
}
return _fileRotateWriter.setWriter(logPath, writer)
}

14
ai-chat-service/pkg/zerror/error_code.go Normal file
View File

@@ -0,0 +1,14 @@
package zerror
type ZErrorCode string
func getErrMsg(errCode ZErrorCode) string {
msg, ok := errorMsgs[errCode]
if ok {
return msg
}
return ""
}
// 错误码与之对应的错误消息
var errorMsgs = map[ZErrorCode]string{}

101
ai-chat-service/pkg/zerror/zerror.go Normal file
View File

@@ -0,0 +1,101 @@
package zerror
import (
"errors"
"fmt"
)
type ZError struct {
ErrCode ZErrorCode `json:"err_code,omitempty"`
ErrMsg string `json:"err_msg,omitempty"`
errs []error
}
func (e *ZError) Error() string {
if e == nil {
return ""
}
if e.ErrMsg != "" {
return fmt.Sprintf("ErrCode:%s; ErrMsg:%s;", e.ErrCode, e.ErrMsg)
}
res := ""
if e.errs == nil || len(e.errs) == 0 {
return res
}
var first = true
for _, err := range e.errs {
if first {
res = err.Error()
first = false
} else {
res += ";" + err.Error()
}
}
return res
}
func (e *ZError) Errors() []error {
if e == nil {
return nil
}
return e.errs
}
func (e *ZError) Append(err error) {
if e == nil || err == nil {
return
}
ze, ok := err.(*ZError)
if ok {
e.errs = append(e.errs, ze.errs...)
} else {
e.errs = append(e.errs, err)
}
}
func NewByErr(err ...error) error {
res := &ZError{
errs: make([]error, 0),
}
for i, e := range err {
if e == nil {
continue
}
ze, ok := err[i].(*ZError)
if ok {
res.errs = append(res.errs, ze.errs...)
} else {
res.errs = append(res.errs, err[i])
}
}
if len(res.errs) > 0 {
return res
}
return nil
}
func NewByCode(errCode ZErrorCode, errMsg ...string) error {
msg := ""
if len(errMsg) > 0 {
msg = errMsg[0]
} else {
msg = getErrMsg(errCode)
}
return &ZError{
ErrCode: errCode,
ErrMsg: msg,
}
}
func NewByMsg(msg string) error {
err := errors.New(msg)
return NewByErr(err)
}
func Errors(err error) []error {
if err == nil {
return nil
}
ze, ok := err.(*ZError)
if !ok {
return []error{err}
}
//将一个空的[]error切片与ze.Errors()返回的错误切片合并在一起
//返回一个新切片
return append(([]error)(nil), ze.Errors()...)
}

1000
ai-chat-service/proto/chat.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

72
ai-chat-service/proto/chat.proto Normal file
View File

@@ -0,0 +1,72 @@
syntax = "proto3";
option go_package = "ai-chat-service/proto";
package ai_chat_service.zvoice.com;
message ChatCompletionRequest {
string message = 1[json_name = "message"];
string id = 2[json_name = "id"];
string pid = 3[json_name = "p_id"];
bool enableContext = 4[json_name = "enable_context"];
ChatParam chatParam = 5[json_name = "chat_param"];
}
message ChatParam {
string model = 1[json_name = "model"];
int32 maxTokens = 2[json_name = "max_tokens"];
float temperature = 3[json_name = "temperature"];
float topP = 4[json_name = "top_p"];
float presencePenalty = 5[json_name = "presence_penalty"];
float frequencyPenalty = 6[json_name = "frequency_penalty"];
string botDesc =7[json_name = "bot_desc"];
int32 minResponseTokens = 8[json_name = "min_response_tokens"];
int32 contextTTL = 9[json_name = "context_ttl"];
int32 contextLen = 10[json_name = "context_len"];
}
// 服务响应消息，非流式响应
message ChatCompletionResponse {
string id = 1 [json_name = "id"];
string object = 2 [json_name = "object"];
int64 created = 3 [json_name = "created"];
string model = 4 [json_name = "model"];
repeated ChatCompletionChoice choices = 5 [json_name = "choices"];
Usage usage = 6[json_name = "usage"];
}
message ChatCompletionChoice {
int32 index = 1[json_name = "index"];
ChatCompletionMessage message = 2 [json_name = "message"];
string finishReason = 3[json_name = "finish_reason"];
}
message ChatCompletionMessage {
string role=1[json_name = "role"];
string content=2[json_name = "content"];
string name=3[json_name = "name"];
}
message Usage {
int32 promptTokens = 1 [json_name = "prompt_tokens"];
int32 completionTokens =2 [json_name = "completion_tokens"];
int32 totalTokens = 3 [json_name = "total_tokens"];
}
// 服务响应消息，流式响应
message ChatCompletionStreamResponse {
string id = 1 [json_name = "id"];
string object = 2 [json_name = "object"];
int64 created = 3 [json_name = "created"];
string model = 4 [json_name = "model"];
repeated ChatCompletionStreamChoice choices = 5 [json_name = "choices"];
}
message ChatCompletionStreamChoice {
int32 index = 1[json_name = "index"];
ChatCompletionStreamChoiceDelta delta =2 [json_name = "delta"];
string finishReason = 3[json_name="finish_reason"];
}
message ChatCompletionStreamChoiceDelta {
string content = 1 [json_name = "content"];
string role = 2 [json_name = "role"];
}
service Chat {
rpc ChatCompletion(ChatCompletionRequest) returns (ChatCompletionResponse);
rpc ChatCompletionStream(ChatCompletionRequest) returns (stream ChatCompletionStreamResponse);
}

169
ai-chat-service/proto/chat_grpc.pb.go Normal file
View File

@@ -0,0 +1,169 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v4.22.0
// source: proto/chat.proto
package proto
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// ChatClient is the client API for Chat service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type ChatClient interface {
ChatCompletion(ctx context.Context, in *ChatCompletionRequest, opts ...grpc.CallOption) (*ChatCompletionResponse, error)
ChatCompletionStream(ctx context.Context, in *ChatCompletionRequest, opts ...grpc.CallOption) (Chat_ChatCompletionStreamClient, error)
}
type chatClient struct {
cc grpc.ClientConnInterface
}
func NewChatClient(cc grpc.ClientConnInterface) ChatClient {
return &chatClient{cc}
}
func (c *chatClient) ChatCompletion(ctx context.Context, in *ChatCompletionRequest, opts ...grpc.CallOption) (*ChatCompletionResponse, error) {
out := new(ChatCompletionResponse)
err := c.cc.Invoke(ctx, "/ai_chat_service.zvoice.com.Chat/ChatCompletion", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *chatClient) ChatCompletionStream(ctx context.Context, in *ChatCompletionRequest, opts ...grpc.CallOption) (Chat_ChatCompletionStreamClient, error) {
stream, err := c.cc.NewStream(ctx, &Chat_ServiceDesc.Streams[0], "/ai_chat_service.zvoice.com.Chat/ChatCompletionStream", opts...)
if err != nil {
return nil, err
}
x := &chatChatCompletionStreamClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type Chat_ChatCompletionStreamClient interface {
Recv() (*ChatCompletionStreamResponse, error)
grpc.ClientStream
}
type chatChatCompletionStreamClient struct {
grpc.ClientStream
}
func (x *chatChatCompletionStreamClient) Recv() (*ChatCompletionStreamResponse, error) {
m := new(ChatCompletionStreamResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
// ChatServer is the server API for Chat service.
// All implementations must embed UnimplementedChatServer
// for forward compatibility
type ChatServer interface {
ChatCompletion(context.Context, *ChatCompletionRequest) (*ChatCompletionResponse, error)
ChatCompletionStream(*ChatCompletionRequest, Chat_ChatCompletionStreamServer) error
mustEmbedUnimplementedChatServer()
}
// UnimplementedChatServer must be embedded to have forward compatible implementations.
type UnimplementedChatServer struct {
}
func (UnimplementedChatServer) ChatCompletion(context.Context, *ChatCompletionRequest) (*ChatCompletionResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ChatCompletion not implemented")
}
func (UnimplementedChatServer) ChatCompletionStream(*ChatCompletionRequest, Chat_ChatCompletionStreamServer) error {
return status.Errorf(codes.Unimplemented, "method ChatCompletionStream not implemented")
}
func (UnimplementedChatServer) mustEmbedUnimplementedChatServer() {}
// UnsafeChatServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to ChatServer will
// result in compilation errors.
type UnsafeChatServer interface {
mustEmbedUnimplementedChatServer()
}
func RegisterChatServer(s grpc.ServiceRegistrar, srv ChatServer) {
s.RegisterService(&Chat_ServiceDesc, srv)
}
func _Chat_ChatCompletion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ChatCompletionRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ChatServer).ChatCompletion(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/ai_chat_service.zvoice.com.Chat/ChatCompletion",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ChatServer).ChatCompletion(ctx, req.(*ChatCompletionRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Chat_ChatCompletionStream_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(ChatCompletionRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(ChatServer).ChatCompletionStream(m, &chatChatCompletionStreamServer{stream})
}
type Chat_ChatCompletionStreamServer interface {
Send(*ChatCompletionStreamResponse) error
grpc.ServerStream
}
type chatChatCompletionStreamServer struct {
grpc.ServerStream
}
func (x *chatChatCompletionStreamServer) Send(m *ChatCompletionStreamResponse) error {
return x.ServerStream.SendMsg(m)
}
// Chat_ServiceDesc is the grpc.ServiceDesc for Chat service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var Chat_ServiceDesc = grpc.ServiceDesc{
ServiceName: "ai_chat_service.zvoice.com.Chat",
HandlerType: (*ChatServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "ChatCompletion",
Handler: _Chat_ChatCompletion_Handler,
},
},
Streams: []grpc.StreamDesc{
{
StreamName: "ChatCompletionStream",
Handler: _Chat_ChatCompletionStream_Handler,
ServerStreams: true,
},
},
Metadata: "proto/chat.proto",
}

1
ai-chat-service/runtime/logs/app.log Normal file
View File

@@ -0,0 +1 @@
time="2024-09-20T14:44:21+08:00" level=error msg="grpc: no transport security set (use grpc.WithTransportCredentials(insecure.NewCredentials()) explicitly or set credentials)" file="E:/Work/Code/go/5.0/2404/ai-chat/ai-chat-service/services/grpc-client/grpc_client_pool.go:25" func=ai-chat-service/services/grpc-client.NewPool.func1

28
ai-chat-service/services/grpc-client/default.go Normal file
View File

@@ -0,0 +1,28 @@
package grpc_client
import (
"ai-chat-service/pkg/log"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
type ServiceClient interface {
GetPool(addr string) ClientPool
}
type DefaultClient struct {
}
func (c *DefaultClient) GetPool(addr string) ClientPool {
pool, err := NewPool(addr, c.getOptions()...)
if err != nil {
log.Error(err)
return nil
}
return pool
}
func (c *DefaultClient) getOptions() []grpc.DialOption {
opts := make([]grpc.DialOption, 0)
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
return opts
}

48
ai-chat-service/services/grpc-client/grpc_client_pool.go Normal file
View File

@@ -0,0 +1,48 @@
package grpc_client
import (
"ai-chat-service/pkg/log"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"sync"
)
type ClientPool interface {
Get() *grpc.ClientConn
Put(*grpc.ClientConn)
}
type clientPool struct {
pool sync.Pool
}
func NewPool(target string, opts ...grpc.DialOption) (ClientPool, error) {
return &clientPool{
pool: sync.Pool{
New: func() any {
conn, err := grpc.NewClient(target, opts...)
if err != nil {
log.Error(err)
return nil
}
return conn
},
},
}, nil
}
func (c *clientPool) Get() *grpc.ClientConn {
conn := c.pool.Get().(*grpc.ClientConn)
if conn.GetState() == connectivity.Shutdown || conn.GetState() == connectivity.TransientFailure {
conn.Close()
conn = c.pool.New().(*grpc.ClientConn)
}
return conn
}
func (c *clientPool) Put(conn *grpc.ClientConn) {
if conn.GetState() == connectivity.Shutdown || conn.GetState() == connectivity.TransientFailure {
conn.Close()
return
}
c.pool.Put(conn)
}

23
ai-chat-service/services/keywords-filter/keywords.go Normal file
View File

@@ -0,0 +1,23 @@
package keywords_filter
import (
"ai-chat-service/pkg/config"
grpc_client "ai-chat-service/services/grpc-client"
"sync"
)
var keywordsPool grpc_client.ClientPool
var keywordsOnce sync.Once
type keywordsClient struct {
grpc_client.DefaultClient
}
func GetKeywordsClientPool() grpc_client.ClientPool {
keywordsOnce.Do(func() {
cnf := config.GetConfig()
c := &keywordsClient{}
keywordsPool = c.GetPool(cnf.DependOn.Keywords.Address)
})
return keywordsPool
}

294
ai-chat-service/services/keywords-filter/proto/filter.pb.go Normal file
View File

@@ -0,0 +1,294 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc v4.22.0
// source: proto/filter.proto
package proto
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type FilterReq struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Text string `protobuf:"bytes,1,opt,name=text,proto3" json:"text,omitempty"`
}
func (x *FilterReq) Reset() {
*x = FilterReq{}
if protoimpl.UnsafeEnabled {
mi := &file_proto_filter_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FilterReq) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FilterReq) ProtoMessage() {}
func (x *FilterReq) ProtoReflect() protoreflect.Message {
mi := &file_proto_filter_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FilterReq.ProtoReflect.Descriptor instead.
func (*FilterReq) Descriptor() ([]byte, []int) {
return file_proto_filter_proto_rawDescGZIP(), []int{0}
}
func (x *FilterReq) GetText() string {
if x != nil {
return x.Text
}
return ""
}
type ValidateRes struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Ok bool `protobuf:"varint,1,opt,name=ok,proto3" json:"ok,omitempty"`
Keyword string `protobuf:"bytes,2,opt,name=keyword,proto3" json:"keyword,omitempty"`
}
func (x *ValidateRes) Reset() {
*x = ValidateRes{}
if protoimpl.UnsafeEnabled {
mi := &file_proto_filter_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ValidateRes) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ValidateRes) ProtoMessage() {}
func (x *ValidateRes) ProtoReflect() protoreflect.Message {
mi := &file_proto_filter_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ValidateRes.ProtoReflect.Descriptor instead.
func (*ValidateRes) Descriptor() ([]byte, []int) {
return file_proto_filter_proto_rawDescGZIP(), []int{1}
}
func (x *ValidateRes) GetOk() bool {
if x != nil {
return x.Ok
}
return false
}
func (x *ValidateRes) GetKeyword() string {
if x != nil {
return x.Keyword
}
return ""
}
type FindAllRes struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Keywords []string `protobuf:"bytes,1,rep,name=keywords,proto3" json:"keywords,omitempty"`
}
func (x *FindAllRes) Reset() {
*x = FindAllRes{}
if protoimpl.UnsafeEnabled {
mi := &file_proto_filter_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FindAllRes) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FindAllRes) ProtoMessage() {}
func (x *FindAllRes) ProtoReflect() protoreflect.Message {
mi := &file_proto_filter_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FindAllRes.ProtoReflect.Descriptor instead.
func (*FindAllRes) Descriptor() ([]byte, []int) {
return file_proto_filter_proto_rawDescGZIP(), []int{2}
}
func (x *FindAllRes) GetKeywords() []string {
if x != nil {
return x.Keywords
}
return nil
}
var File_proto_filter_proto protoreflect.FileDescriptor
var file_proto_filter_proto_rawDesc = []byte{
0x0a, 0x12, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1a, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66,
0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d,
0x22, 0x1f, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x12, 0x0a,
0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78,
0x74, 0x22, 0x37, 0x0a, 0x0b, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73,
0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b,
0x12, 0x18, 0x0a, 0x07, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x28, 0x0a, 0x0a, 0x46, 0x69,
0x6e, 0x64, 0x41, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77,
0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77,
0x6f, 0x72, 0x64, 0x73, 0x32, 0xbe, 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12,
0x5a, 0x0a, 0x08, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x12, 0x25, 0x2e, 0x6b, 0x65,
0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76,
0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52,
0x65, 0x71, 0x1a, 0x27, 0x2e, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69,
0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e,
0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x07, 0x46,
0x69, 0x6e, 0x64, 0x41, 0x6c, 0x6c, 0x12, 0x25, 0x2e, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64,
0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e,
0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x26, 0x2e,
0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e,
0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x41,
0x6c, 0x6c, 0x52, 0x65, 0x73, 0x42, 0x17, 0x5a, 0x15, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64,
0x73, 0x2d, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_proto_filter_proto_rawDescOnce sync.Once
file_proto_filter_proto_rawDescData = file_proto_filter_proto_rawDesc
)
func file_proto_filter_proto_rawDescGZIP() []byte {
file_proto_filter_proto_rawDescOnce.Do(func() {
file_proto_filter_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_filter_proto_rawDescData)
})
return file_proto_filter_proto_rawDescData
}
var file_proto_filter_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_proto_filter_proto_goTypes = []interface{}{
(*FilterReq)(nil), // 0: keywords_filter.zvoice.com.FilterReq
(*ValidateRes)(nil), // 1: keywords_filter.zvoice.com.ValidateRes
(*FindAllRes)(nil), // 2: keywords_filter.zvoice.com.FindAllRes
}
var file_proto_filter_proto_depIdxs = []int32{
0, // 0: keywords_filter.zvoice.com.Filter.Validate:input_type -> keywords_filter.zvoice.com.FilterReq
0, // 1: keywords_filter.zvoice.com.Filter.FindAll:input_type -> keywords_filter.zvoice.com.FilterReq
1, // 2: keywords_filter.zvoice.com.Filter.Validate:output_type -> keywords_filter.zvoice.com.ValidateRes
2, // 3: keywords_filter.zvoice.com.Filter.FindAll:output_type -> keywords_filter.zvoice.com.FindAllRes
2, // [2:4] is the sub-list for method output_type
0, // [0:2] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_proto_filter_proto_init() }
func file_proto_filter_proto_init() {
if File_proto_filter_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_proto_filter_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FilterReq); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_proto_filter_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ValidateRes); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_proto_filter_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FindAllRes); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_proto_filter_proto_rawDesc,
NumEnums: 0,
NumMessages: 3,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_proto_filter_proto_goTypes,
DependencyIndexes: file_proto_filter_proto_depIdxs,
MessageInfos: file_proto_filter_proto_msgTypes,
}.Build()
File_proto_filter_proto = out.File
file_proto_filter_proto_rawDesc = nil
file_proto_filter_proto_goTypes = nil
file_proto_filter_proto_depIdxs = nil
}

20
ai-chat-service/services/keywords-filter/proto/filter.proto Normal file
View File

@@ -0,0 +1,20 @@
syntax = "proto3";
option go_package = "keywords-filter/proto";
package keywords_filter.zvoice.com;
message FilterReq {
string text = 1;
}
message ValidateRes {
bool ok = 1;
string keyword = 2;
}
message FindAllRes {
repeated string keywords = 1;
}
service Filter {
rpc Validate(FilterReq) returns (ValidateRes);
rpc FindAll(FilterReq) returns (FindAllRes);
}

141
ai-chat-service/services/keywords-filter/proto/filter_grpc.pb.go Normal file
View File

@@ -0,0 +1,141 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v4.22.0
// source: proto/filter.proto
package proto
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// FilterClient is the client API for Filter service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type FilterClient interface {
Validate(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*ValidateRes, error)
FindAll(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*FindAllRes, error)
}
type filterClient struct {
cc grpc.ClientConnInterface
}
func NewFilterClient(cc grpc.ClientConnInterface) FilterClient {
return &filterClient{cc}
}
func (c *filterClient) Validate(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*ValidateRes, error) {
out := new(ValidateRes)
err := c.cc.Invoke(ctx, "/keywords_filter.zvoice.com.Filter/Validate", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *filterClient) FindAll(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*FindAllRes, error) {
out := new(FindAllRes)
err := c.cc.Invoke(ctx, "/keywords_filter.zvoice.com.Filter/FindAll", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// FilterServer is the server API for Filter service.
// All implementations must embed UnimplementedFilterServer
// for forward compatibility
type FilterServer interface {
Validate(context.Context, *FilterReq) (*ValidateRes, error)
FindAll(context.Context, *FilterReq) (*FindAllRes, error)
mustEmbedUnimplementedFilterServer()
}
// UnimplementedFilterServer must be embedded to have forward compatible implementations.
type UnimplementedFilterServer struct {
}
func (UnimplementedFilterServer) Validate(context.Context, *FilterReq) (*ValidateRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method Validate not implemented")
}
func (UnimplementedFilterServer) FindAll(context.Context, *FilterReq) (*FindAllRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method FindAll not implemented")
}
func (UnimplementedFilterServer) mustEmbedUnimplementedFilterServer() {}
// UnsafeFilterServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to FilterServer will
// result in compilation errors.
type UnsafeFilterServer interface {
mustEmbedUnimplementedFilterServer()
}
func RegisterFilterServer(s grpc.ServiceRegistrar, srv FilterServer) {
s.RegisterService(&Filter_ServiceDesc, srv)
}
func _Filter_Validate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FilterReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(FilterServer).Validate(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/keywords_filter.zvoice.com.Filter/Validate",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(FilterServer).Validate(ctx, req.(*FilterReq))
}
return interceptor(ctx, in, info, handler)
}
func _Filter_FindAll_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FilterReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(FilterServer).FindAll(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/keywords_filter.zvoice.com.Filter/FindAll",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(FilterServer).FindAll(ctx, req.(*FilterReq))
}
return interceptor(ctx, in, info, handler)
}
// Filter_ServiceDesc is the grpc.ServiceDesc for Filter service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var Filter_ServiceDesc = grpc.ServiceDesc{
ServiceName: "keywords_filter.zvoice.com.Filter",
HandlerType: (*FilterServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Validate",
Handler: _Filter_Validate_Handler,
},
{
MethodName: "FindAll",
Handler: _Filter_FindAll_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "proto/filter.proto",
}

23
ai-chat-service/services/keywords-filter/sensitive.go Normal file
View File

@@ -0,0 +1,23 @@
package keywords_filter
import (
"ai-chat-service/pkg/config"
grpc_client "ai-chat-service/services/grpc-client"
"sync"
)
var sensitivePool grpc_client.ClientPool
var sensitiveOnce sync.Once
type sensitiveClient struct {
grpc_client.DefaultClient
}
func GetSensitiveClientPool() grpc_client.ClientPool {
sensitiveOnce.Do(func() {
cnf := config.GetConfig()
c := &sensitiveClient{}
sensitivePool = c.GetPool(cnf.DependOn.Sensitive.Address)
})
return sensitivePool
}

11
ai-chat-service/services/services.go Normal file
View File

@@ -0,0 +1,11 @@
package services
import (
"context"
"google.golang.org/grpc/metadata"
)
func AppendBearerTokenToContext(ctx context.Context, accessToken string) context.Context {
md := metadata.Pairs("Authorization", "Bearer "+accessToken)
return metadata.NewOutgoingContext(ctx, md)
}

43
ai-chat-service/services/tokenizer/tokenizer.go Normal file
View File

@@ -0,0 +1,43 @@
package tokenizer
import (
"ai-chat-service/pkg/config"
"bytes"
"encoding/json"
"fmt"
"github.com/sashabaranov/go-openai"
"net/http"
)
type tokensInfo struct {
Code int `json:"code"`
Tokens int `json:"num_tokens"`
Msg string `json:"msg"`
}
var httpClient = &http.Client{}
func GetTokens(message *openai.ChatCompletionMessage, model string) (int, error) {
cnf := config.GetConfig()
url := fmt.Sprintf("%s/tokenizer/%s", cnf.DependOn.Tokenizer.Address, model)
info := &tokensInfo{}
if err := postJSON(url, message, info); err != nil {
return 0, err
}
if info.Code != 200 {
return 0, fmt.Errorf("%v", info.Msg)
}
return info.Tokens, nil
}
func postJSON(url string, requestData *openai.ChatCompletionMessage, responseData *tokensInfo) error {
requestBody, err := json.Marshal(requestData)
if err != nil {
return err
}
resp, err := httpClient.Post(url, "application/json", bytes.NewReader(requestBody))
if err != nil {
return err
}
return json.NewDecoder(resp.Body).Decode(responseData)
}

2
chatgpt-web-backend/cmd/main.go
View File

@@ -50,7 +50,7 @@ func (r *ChatGPTWebServer) Run(cmd *cobra.Command, args []string) error {
if err := r.updateAssetsFiles(); err != nil {
return err
}
go r.startTokenizer(cmd.Context())
// go r.startTokenizer(cmd.Context())
go r.httpServer(cmd.Context())
<-cmd.Context().Done()

9
docker-compose.yml Normal file
View File

@@ -0,0 +1,9 @@
version: '3.3'
services:
tokenizer:
image: tokenizer:1.0.0
container_name: tokenizer
ports:
- "5000:3002"
restart: unless-stopped

1
keywords-filter/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
runtime

19
keywords-filter/Dockerfile Normal file
View File

@@ -0,0 +1,19 @@
# 编译阶段
FROM quay.io/0voice/golang:1.20 as stage0
RUN go env -w GOPROXY=https://proxy.golang.com.cn,https://goproxy.cn,direct
ADD ./ /src/keywords-filter
WORKDIR /src/keywords-filter
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o keywords-filter ./filter-server
FROM quay.io/0voice/alpine:3.18 as stage1
ADD ./grpc_health_probe-linux-amd64 /usr/bin/grpc_health_probe
RUN chmod +x /usr/bin/grpc_health_probe
MAINTAINER nick
WORKDIR /app/
ADD ./dev.config.yaml /app/config.yaml
ADD ./dict.txt /app/dict.txt
COPY --from=stage0 /src/keywords-filter/keywords-filter ./
# 指定入口程序
ENTRYPOINT ["./keywords-filter"]
# 指定容器的启动命令或者入口程序的参数
CMD ["--config=config.yaml","--dict=dict.txt"]

8
keywords-filter/dev.config.yaml Normal file
View File

@@ -0,0 +1,8 @@
server:
ip: 0.0.0.0
port: 50053
accessToken: "ang1chubdev1ozhome256487d22sapguuv1ozhom"
log:
# panic,fatal,errro,warn,info,debug,trace
level: "info"
logPath: "runtime/logs/app.log"

8
keywords-filter/dev.kw.config.yaml Normal file
View File

@@ -0,0 +1,8 @@
server:
ip: 0.0.0.0
port: 50054
accessToken: "ang1chubdev1ozhome256487d22sapguuv1ozhom"
log:
# panic,fatal,errro,warn,info,debug,trace
level: "info"
logPath: "runtime/logs/app.log"

13992
keywords-filter/dict.txt Normal file
View File

File diff suppressed because it is too large Load Diff

38
keywords-filter/filter-server/interceptor/auth.go Normal file
View File

@@ -0,0 +1,38 @@
package interceptor
import (
"context"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"keywords-filter/pkg/config"
"keywords-filter/pkg/zerror"
"strings"
)
func UnaryAuthInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) {
if info.FullMethod != "/grpc.health.v1.Health/Check" {
err = oauth2Valid(ctx)
if err != nil {
return nil, err
}
}
return handler(ctx, req)
}
func oauth2Valid(ctx context.Context) error {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return zerror.NewByMsg("元数据获取失败")
}
authorization := md["authorization"]
if len(authorization) < 1 {
return zerror.NewByMsg("元数据获取失败")
}
token := strings.TrimPrefix(authorization[0], "Bearer ")
cnf := config.GetConfig()
if token != cnf.Server.AccessToken {
return zerror.NewByMsg("鉴权失败")
}
return nil
}

55
keywords-filter/filter-server/main.go Normal file
View File

@@ -0,0 +1,55 @@
package main
import (
"flag"
"fmt"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
"google.golang.org/grpc/health/grpc_health_v1"
"keywords-filter/filter-server/interceptor"
"keywords-filter/filter-server/server"
"keywords-filter/pkg/config"
"keywords-filter/pkg/filter"
"keywords-filter/pkg/log"
"keywords-filter/proto"
"net"
)
var (
configFile = flag.String("config", "dev.config.yaml", "")
dictFile = flag.String("dict", "dict.txt", "")
formatDict = flag.Bool("format", false, "")
)
func main() {
flag.Parse()
if *formatDict {
filter.OverwriteDict(*dictFile)
return
}
//初始化配置文件
config.InitConfig(*configFile)
cnf := config.GetConfig()
//初始化日志
log.SetLevel(cnf.Log.Level)
log.SetOutput(log.GetRotateWriter(cnf.Log.LogPath))
log.SetPrintCaller(true)
//初始话filter
filter.InitFilter(*dictFile)
lis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", cnf.Server.IP, cnf.Server.Port))
if err != nil {
log.Fatal(err)
}
s := grpc.NewServer(grpc.UnaryInterceptor(interceptor.UnaryAuthInterceptor))
service := server.NewFilterService(filter.GetFilter())
proto.RegisterFilterServer(s, service)
healthCheckSrv := health.NewServer()
grpc_health_v1.RegisterHealthServer(s, healthCheckSrv)
if err = s.Serve(lis); err != nil {
log.Fatal(err)
}
}

32
keywords-filter/filter-server/server/server.go Normal file
View File

@@ -0,0 +1,32 @@
package server
import (
"context"
"keywords-filter/pkg/filter"
"keywords-filter/proto"
)
type filterService struct {
proto.UnimplementedFilterServer
filter filter.IFilter
}
func NewFilterService(filter filter.IFilter) proto.FilterServer {
return &filterService{
filter: filter,
}
}
func (s *filterService) Validate(_ context.Context, in *proto.FilterReq) (*proto.ValidateRes, error) {
ok, word := s.filter.Validate(in.Text)
return &proto.ValidateRes{
Ok: ok,
Keyword: word,
}, nil
}
func (s *filterService) FindAll(_ context.Context, in *proto.FilterReq) (*proto.FindAllRes, error) {
words := s.filter.FindAll(in.Text)
return &proto.FindAllRes{
Keywords: words,
}, nil
}

37
keywords-filter/go.mod Normal file
View File

@@ -0,0 +1,37 @@
module keywords-filter
go 1.20
require (
github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b
github.com/sirupsen/logrus v1.9.3
github.com/spf13/viper v1.19.0
google.golang.org/grpc v1.62.1
google.golang.org/protobuf v1.34.2
gopkg.in/natefinch/lumberjack.v2 v2.2.1
)
require (
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

86
keywords-filter/go.sum Normal file
View File

@@ -0,0 +1,86 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b h1:9hudrgWUhyfR4FRMOfL9KB1uYw48DUdHkkgr9ODOw7Y=
github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b/go.mod h1:zLVdX6Ed2SvCbEamKmve16U0E03UkdJo4ls1TBfmc8Q=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

BIN
keywords-filter/grpc_health_probe-linux-amd64 Normal file
View File

Binary file not shown.

192
keywords-filter/keyword-dict.txt Normal file
View File

@@ -0,0 +1,192 @@
golang
defer
recover
sync
Protobuf
gin
grpc-gateway
OpenTelemetry
OTel
otel
k8s
Kubernetes
kubernetes
Docker
docker
Istio
istio
Prometheus
prometheus
cadvisor
cAdvisor
Elastic
Kibana
Grafana
apiserver
CI/CD
ci/cd
ArgoCD
argo
Argo
kaniko
Mesh
Volume
volume
promQL
PromQL
kafka
ingress
StorageClass
VolumeClaim
gitlab
openflow
dpdk
vpp
ovs
spdk
virtio
vhost
qemu
vSwitch
bridge
hugepage
nvme
dpvs
iperf3
rfc2544
ioengine
PCI
vxlan
gre
kni
Kernel
内核
KernelThread
内核线程
Virtual
memory
虚拟内存
内存屏障
内存管理
Scheduler
调度器
File
文件系统
Device
driver
设备驱动程序
Syscall
系统调用
Process
scheduling
进程调度
Page
页表
Swap
交换空间
Mount
Inode
挂载
索引节点
Block
块设备
Character
字符设备
IRQ
Kconfig
内核配置
Perf
Ftrace
内核跟踪工具
Valgrind
内存调试工具
System
系统定时器
DMA
伙伴系统
信号与槽
Signals
Slots
事件处理程序
Event
QML
多线程编程
Multithreading
Programming
QThread
QtQuick
模型
视图架构
Model/View
QObject
QWidget
QRegularExpression
QDesktopWidget
QNetworkAccessManager
QTcpServer
QTcpSocket
QUdpSocket
QMutex
SQLite/MySQL
MySQL编程
SQLite编程
OpenCV
OpenGL
Qt数据库编程
Qt网络编程
Linux
tcp
redis
mysql
网络
nginx
协程
io_uring
内存泄漏
bpf
ebpf
skynet
openresty
RocksDB
TiDB
ceph
etcd
fuse
p2p
http
mqtt
cuda
mutex
spinlock
hash
rbtree
btree
Makefile
git
wrk
Cuda
CUDA
D3D
d3d
ffmpeg
RTSP
WebRTC
PCM
RGB
YUv
MP4
FLV
TS
VLC
EasylCE
flvAnalyser
mp4box
audacity
Elecard
AAC
h264
SDL
AVFormat
AVCodec
AVPacket

42
keywords-filter/pkg/config/config.go Normal file
View File

@@ -0,0 +1,42 @@
package config
import (
"github.com/spf13/viper"
"log"
)
type Config struct {
Server struct {
IP string
Port int
AccessToken string
}
Log struct {
Level string
LogPath string `mapstructure:"logPath"`
} `mapstructure:"log"`
}
var conf *Config
func InitConfig(filePath string, typ ...string) {
v := viper.New()
v.SetConfigFile(filePath)
if len(typ) > 0 {
v.SetConfigType(typ[0])
}
err := v.ReadInConfig()
if err != nil {
log.Fatal(err)
}
conf = &Config{}
err = v.Unmarshal(conf)
if err != nil {
log.Fatal(err)
}
}
func GetConfig() *Config {
return conf
}

98
keywords-filter/pkg/filter/filter.go Normal file
View File

@@ -0,0 +1,98 @@
package filter
import (
"bufio"
"github.com/importcjj/sensitive"
"keywords-filter/pkg/log"
"os"
"regexp"
"strings"
)
type IFilter interface {
Validate(text string) (bool, string)
FindAll(text string) []string
}
type filter struct {
filter *sensitive.Filter
}
func (f *filter) Validate(text string) (bool, string) {
text = " " + strings.Trim(text, " ") + " "
ok, word := f.filter.Validate(text)
return ok, word
}
func (f *filter) FindAll(text string) []string {
text = " " + strings.Trim(text, " ") + " "
list := f.filter.FindAll(text)
for i := 0; i < len(list); i++ {
list[i] = strings.Trim(list[i], " ")
}
return list
}
var _filter *filter
func GetFilter() IFilter {
return _filter
}
func InitFilter(dictFilePath string) {
if dictFilePath == "" {
log.Fatal("请指定词库文件")
}
_, err := os.Stat(dictFilePath)
if os.IsNotExist(err) {
log.Fatal("词库文件不存在，请指定正确的词库文件")
}
f := sensitive.New()
f.UpdateNoisePattern("")
f.LoadWordDict(dictFilePath)
_filter = &filter{
filter: f,
}
}
func OverwriteDict(dictFilePath string) error {
file, err := os.Open(dictFilePath)
if err != nil {
log.Fatal(err)
return err
}
re := regexp.MustCompile(`\p{Han}+`)
newContent := ""
kwMp := make(map[string]struct{}, 0)
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
line = strings.Trim(line, " ")
//去重
if _, ok := kwMp[line]; ok {
continue
}
kwMp[line] = struct{}{}
match := re.FindString(line)
if match == "" {
newContent += " " + line + " \n"
} else {
newContent += line + "\n"
}
}
newContent = strings.Trim(newContent, "\n")
file.Close()
file, err = os.OpenFile(dictFilePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
defer file.Close()
if err != nil {
log.Fatal(err)
return err
}
_, err = file.WriteString(newContent)
if err != nil {
log.Fatal(err)
return err
}
return err
}

3
keywords-filter/pkg/log/README.md Normal file
View File

@@ -0,0 +1,3 @@
# 日志框架
1. 可通过包调用日志打印，也可以通过对象调用日志打印
2. 可以自动切分日志文件

19
keywords-filter/pkg/log/hook.go Normal file
View File

@@ -0,0 +1,19 @@
package log
import "github.com/sirupsen/logrus"
import nativeLog "log"
type errorHook struct {
}
func (*errorHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.PanicLevel,
logrus.FatalLevel,
logrus.ErrorLevel,
}
}
func (*errorHook) Fire(entry *logrus.Entry) error {
nativeLog.Println(entry.Message, entry.Data)
return nil
}

233
keywords-filter/pkg/log/log.go Normal file
View File

@@ -0,0 +1,233 @@
package log
import (
"errors"
"fmt"
"github.com/sirupsen/logrus"
"io"
"runtime"
)
type ILogger interface {
SetLevel(lvl string)
SetOutput(writer io.Writer)
SetPrintCaller(bool)
SetCaller(caller func() (file string, line int, funcName string, err error))
Trace(args ...interface{})
Debug(args ...interface{})
Info(args ...interface{})
Warning(args ...interface{})
Error(args ...interface{})
Fatal(args ...interface{})
Panic(args ...interface{})
TraceF(format string, args ...interface{})
DebugF(format string, args ...interface{})
InfoF(format string, args ...interface{})
WarningF(format string, args ...interface{})
ErrorF(format string, args ...interface{})
FatalF(format string, args ...interface{})
PanicF(format string, args ...interface{})
WithFields(fields map[string]interface{}) ILogger
}
type Logger struct {
entry *logrus.Entry
// panic,fatal,error,warn,warning,info,debug,trace
level string
printCaller bool
caller func() (file string, line int, funcName string, err error)
}
// 设置日志打印级别
func (l *Logger) SetLevel(lvl string) {
if lvl == "" {
return
}
level, err := logrus.ParseLevel(lvl)
if err == nil {
l.level = lvl
l.entry.Logger.Level = level
}
}
// 设置日志输出位置
func (l *Logger) SetOutput(writer io.Writer) {
l.entry.Logger.SetOutput(writer)
}
// 设置是否打印调用信息
func (l *Logger) SetPrintCaller(printCaller bool) {
l.printCaller = printCaller
}
func (l *Logger) SetCaller(caller func() (file string, line int, funcName string, err error)) {
l.caller = caller
}
// 获取caller信息
func (l *Logger) getCallerInfo(level logrus.Level) map[string]interface{} {
mp := make(map[string]interface{})
if l.printCaller == true || level != logrus.InfoLevel {
file, line, funcName, err := l.caller()
if err == nil {
mp["file"] = fmt.Sprintf("%s:%d", file, line)
mp["func"] = funcName
}
}
return mp
}
func (l *Logger) log(level logrus.Level, args ...interface{}) {
l.entry.WithFields(l.getCallerInfo(level)).Log(level, args...)
}
func (l *Logger) logf(level logrus.Level, format string, args ...interface{}) {
l.entry.WithFields(l.getCallerInfo(level)).Logf(level, format, args...)
}
func (l *Logger) Trace(args ...interface{}) {
l.log(logrus.TraceLevel, args...)
}
func (l *Logger) Debug(args ...interface{}) {
l.log(logrus.DebugLevel, args...)
}
func (l *Logger) Info(args ...interface{}) {
l.log(logrus.InfoLevel, args...)
}
func (l *Logger) Warning(args ...interface{}) {
l.log(logrus.WarnLevel, args...)
}
func (l *Logger) Error(args ...interface{}) {
l.log(logrus.ErrorLevel, args...)
}
func (l *Logger) Fatal(args ...interface{}) {
l.log(logrus.FatalLevel, args...)
}
func (l *Logger) Panic(args ...interface{}) {
l.log(logrus.PanicLevel, args...)
}
func (l *Logger) TraceF(format string, args ...interface{}) {
l.logf(logrus.TraceLevel, format, args...)
}
func (l *Logger) DebugF(format string, args ...interface{}) {
l.logf(logrus.DebugLevel, format, args...)
}
func (l *Logger) InfoF(format string, args ...interface{}) {
l.logf(logrus.InfoLevel, format, args...)
}
func (l *Logger) WarningF(format string, args ...interface{}) {
l.logf(logrus.WarnLevel, format, args...)
}
func (l *Logger) ErrorF(format string, args ...interface{}) {
l.logf(logrus.ErrorLevel, format, args...)
}
func (l *Logger) FatalF(format string, args ...interface{}) {
l.logf(logrus.FatalLevel, format, args...)
}
func (l *Logger) PanicF(format string, args ...interface{}) {
l.logf(logrus.PanicLevel, format, args...)
}
func (l *Logger) WithFields(fields map[string]interface{}) ILogger {
entry := l.entry.WithFields(fields)
return &Logger{entry: entry, level: l.level, printCaller: l.printCaller, caller: l.caller}
}
var log *Logger
func NewLogger() ILogger {
return newLogger()
}
func newLogger() *Logger {
log := logrus.New()
log.SetLevel(logrus.InfoLevel)
log.AddHook(&errorHook{})
logger := &Logger{
entry: logrus.NewEntry(log),
caller: defaultCaller,
}
return logger
}
func init() {
log = newLogger()
}
// 设置日志打印级别
func SetLevel(lvl string) {
if lvl == "" {
return
}
level, err := logrus.ParseLevel(lvl)
if err == nil {
log.level = lvl
log.entry.Logger.Level = level
}
}
// 设置日志的输出位置
func SetOutput(writer io.Writer) {
log.entry.Logger.SetOutput(writer)
}
// 设置是否打印调用信息
func SetPrintCaller(printCaller bool) {
log.printCaller = printCaller
}
func SetCaller(caller func() (file string, line int, funcName string, err error)) {
log.caller = caller
}
func defaultCaller() (file string, line int, funcName string, err error) {
pc, f, l, ok := runtime.Caller(4)
if !ok {
err = errors.New("caller failure")
return
}
funcName = runtime.FuncForPC(pc).Name()
file, line = f, l
return
}
func Trace(args ...interface{}) {
log.log(logrus.TraceLevel, args...)
}
func Debug(args ...interface{}) {
log.log(logrus.DebugLevel, args...)
}
func Info(args ...interface{}) {
log.log(logrus.InfoLevel, args...)
}
func Warning(args ...interface{}) {
log.log(logrus.WarnLevel, args...)
}
func Error(args ...interface{}) {
log.log(logrus.ErrorLevel, args...)
}
func Fatal(args ...interface{}) {
log.log(logrus.FatalLevel, args...)
}
func Panic(args ...interface{}) {
log.log(logrus.PanicLevel, args...)
}
func TraceF(format string, args ...interface{}) {
log.logf(logrus.TraceLevel, format, args...)
}
func DebugF(format string, args ...interface{}) {
log.logf(logrus.DebugLevel, format, args...)
}
func InfoF(format string, args ...interface{}) {
log.logf(logrus.InfoLevel, format, args...)
}
func WarningF(format string, args ...interface{}) {
log.logf(logrus.WarnLevel, format, args...)
}
func ErrorF(format string, args ...interface{}) {
log.logf(logrus.ErrorLevel, format, args...)
}
func FatalF(format string, args ...interface{}) {
log.logf(logrus.FatalLevel, format, args...)
}
func PanicF(format string, args ...interface{}) {
log.logf(logrus.PanicLevel, format, args...)
}
func WithFields(fields map[string]interface{}) *Logger {
entry := log.entry.WithFields(fields)
return &Logger{entry: entry, level: log.level, printCaller: log.printCaller, caller: log.caller}
}

58
keywords-filter/pkg/log/rotate_writer.go Normal file
View File

@@ -0,0 +1,58 @@
package log
import (
"gopkg.in/natefinch/lumberjack.v2"
"io"
"sync"
)
type fileRotateWriter struct {
data map[string]io.Writer
sync.RWMutex
}
func (frw *fileRotateWriter) getWriter(logPath string) io.Writer {
frw.RLock()
defer frw.RUnlock()
w, ok := frw.data[logPath]
if !ok {
return nil
}
return w
}
func (frw *fileRotateWriter) setWriter(logPath string, w io.Writer) io.Writer {
frw.Lock()
defer frw.Unlock()
frw.data[logPath] = w
return w
}
var _fileRotateWriter *fileRotateWriter
func init() {
_fileRotateWriter = &fileRotateWriter{
data: map[string]io.Writer{},
}
}
func GetRotateWriter(logPath string) io.Writer {
if logPath == "" {
panic("日志文件路径不能为空")
}
writer := _fileRotateWriter.getWriter(logPath)
if writer != nil {
return writer
}
writer = &lumberjack.Logger{
//文件名
Filename: logPath,
//单个文件大小单位MB
MaxSize: 1,
//最多保留文件数
MaxBackups: 15,
//最长保留时间（天）
MaxAge: 7,
LocalTime: true,
}
return _fileRotateWriter.setWriter(logPath, writer)
}

14
keywords-filter/pkg/zerror/error_code.go Normal file
View File

@@ -0,0 +1,14 @@
package zerror
type ZErrorCode string
func getErrMsg(errCode ZErrorCode) string {
msg, ok := errorMsgs[errCode]
if ok {
return msg
}
return ""
}
// 错误码与之对应的错误消息
var errorMsgs = map[ZErrorCode]string{}

101
keywords-filter/pkg/zerror/zerror.go Normal file
View File

@@ -0,0 +1,101 @@
package zerror
import (
"errors"
"fmt"
)
type ZError struct {
ErrCode ZErrorCode `json:"err_code,omitempty"`
ErrMsg string `json:"err_msg,omitempty"`
errs []error
}
func (e *ZError) Error() string {
if e == nil {
return ""
}
if e.ErrMsg != "" {
return fmt.Sprintf("ErrCode:%s; ErrMsg:%s;", e.ErrCode, e.ErrMsg)
}
res := ""
if e.errs == nil || len(e.errs) == 0 {
return res
}
var first = true
for _, err := range e.errs {
if first {
res = err.Error()
first = false
} else {
res += ";" + err.Error()
}
}
return res
}
func (e *ZError) Errors() []error {
if e == nil {
return nil
}
return e.errs
}
func (e *ZError) Append(err error) {
if e == nil || err == nil {
return
}
ze, ok := err.(*ZError)
if ok {
e.errs = append(e.errs, ze.errs...)
} else {
e.errs = append(e.errs, err)
}
}
func NewByErr(err ...error) error {
res := &ZError{
errs: make([]error, 0),
}
for i, e := range err {
if e == nil {
continue
}
ze, ok := err[i].(*ZError)
if ok {
res.errs = append(res.errs, ze.errs...)
} else {
res.errs = append(res.errs, err[i])
}
}
if len(res.errs) > 0 {
return res
}
return nil
}
func NewByCode(errCode ZErrorCode, errMsg ...string) error {
msg := ""
if len(errMsg) > 0 {
msg = errMsg[0]
} else {
msg = getErrMsg(errCode)
}
return &ZError{
ErrCode: errCode,
ErrMsg: msg,
}
}
func NewByMsg(msg string) error {
err := errors.New(msg)
return NewByErr(err)
}
func Errors(err error) []error {
if err == nil {
return nil
}
ze, ok := err.(*ZError)
if !ok {
return []error{err}
}
//将一个空的[]error切片与ze.Errors()返回的错误切片合并在一起
//返回一个新切片
return append(([]error)(nil), ze.Errors()...)
}

294
keywords-filter/proto/filter.pb.go Normal file
View File

@@ -0,0 +1,294 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc v4.22.0
// source: proto/filter.proto
package proto
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type FilterReq struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Text string `protobuf:"bytes,1,opt,name=text,proto3" json:"text,omitempty"`
}
func (x *FilterReq) Reset() {
*x = FilterReq{}
if protoimpl.UnsafeEnabled {
mi := &file_proto_filter_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FilterReq) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FilterReq) ProtoMessage() {}
func (x *FilterReq) ProtoReflect() protoreflect.Message {
mi := &file_proto_filter_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FilterReq.ProtoReflect.Descriptor instead.
func (*FilterReq) Descriptor() ([]byte, []int) {
return file_proto_filter_proto_rawDescGZIP(), []int{0}
}
func (x *FilterReq) GetText() string {
if x != nil {
return x.Text
}
return ""
}
type ValidateRes struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Ok bool `protobuf:"varint,1,opt,name=ok,proto3" json:"ok,omitempty"`
Keyword string `protobuf:"bytes,2,opt,name=keyword,proto3" json:"keyword,omitempty"`
}
func (x *ValidateRes) Reset() {
*x = ValidateRes{}
if protoimpl.UnsafeEnabled {
mi := &file_proto_filter_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ValidateRes) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ValidateRes) ProtoMessage() {}
func (x *ValidateRes) ProtoReflect() protoreflect.Message {
mi := &file_proto_filter_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ValidateRes.ProtoReflect.Descriptor instead.
func (*ValidateRes) Descriptor() ([]byte, []int) {
return file_proto_filter_proto_rawDescGZIP(), []int{1}
}
func (x *ValidateRes) GetOk() bool {
if x != nil {
return x.Ok
}
return false
}
func (x *ValidateRes) GetKeyword() string {
if x != nil {
return x.Keyword
}
return ""
}
type FindAllRes struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Keywords []string `protobuf:"bytes,1,rep,name=keywords,proto3" json:"keywords,omitempty"`
}
func (x *FindAllRes) Reset() {
*x = FindAllRes{}
if protoimpl.UnsafeEnabled {
mi := &file_proto_filter_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FindAllRes) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FindAllRes) ProtoMessage() {}
func (x *FindAllRes) ProtoReflect() protoreflect.Message {
mi := &file_proto_filter_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FindAllRes.ProtoReflect.Descriptor instead.
func (*FindAllRes) Descriptor() ([]byte, []int) {
return file_proto_filter_proto_rawDescGZIP(), []int{2}
}
func (x *FindAllRes) GetKeywords() []string {
if x != nil {
return x.Keywords
}
return nil
}
var File_proto_filter_proto protoreflect.FileDescriptor
var file_proto_filter_proto_rawDesc = []byte{
0x0a, 0x12, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1a, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66,
0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d,
0x22, 0x1f, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x12, 0x0a,
0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78,
0x74, 0x22, 0x37, 0x0a, 0x0b, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73,
0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b,
0x12, 0x18, 0x0a, 0x07, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x28, 0x0a, 0x0a, 0x46, 0x69,
0x6e, 0x64, 0x41, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77,
0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77,
0x6f, 0x72, 0x64, 0x73, 0x32, 0xbe, 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12,
0x5a, 0x0a, 0x08, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x12, 0x25, 0x2e, 0x6b, 0x65,
0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76,
0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52,
0x65, 0x71, 0x1a, 0x27, 0x2e, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69,
0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e,
0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x07, 0x46,
0x69, 0x6e, 0x64, 0x41, 0x6c, 0x6c, 0x12, 0x25, 0x2e, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64,
0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e,
0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x26, 0x2e,
0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e,
0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x41,
0x6c, 0x6c, 0x52, 0x65, 0x73, 0x42, 0x17, 0x5a, 0x15, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64,
0x73, 0x2d, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_proto_filter_proto_rawDescOnce sync.Once
file_proto_filter_proto_rawDescData = file_proto_filter_proto_rawDesc
)
func file_proto_filter_proto_rawDescGZIP() []byte {
file_proto_filter_proto_rawDescOnce.Do(func() {
file_proto_filter_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_filter_proto_rawDescData)
})
return file_proto_filter_proto_rawDescData
}
var file_proto_filter_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_proto_filter_proto_goTypes = []interface{}{
(*FilterReq)(nil), // 0: keywords_filter.zvoice.com.FilterReq
(*ValidateRes)(nil), // 1: keywords_filter.zvoice.com.ValidateRes
(*FindAllRes)(nil), // 2: keywords_filter.zvoice.com.FindAllRes
}
var file_proto_filter_proto_depIdxs = []int32{
0, // 0: keywords_filter.zvoice.com.Filter.Validate:input_type -> keywords_filter.zvoice.com.FilterReq
0, // 1: keywords_filter.zvoice.com.Filter.FindAll:input_type -> keywords_filter.zvoice.com.FilterReq
1, // 2: keywords_filter.zvoice.com.Filter.Validate:output_type -> keywords_filter.zvoice.com.ValidateRes
2, // 3: keywords_filter.zvoice.com.Filter.FindAll:output_type -> keywords_filter.zvoice.com.FindAllRes
2, // [2:4] is the sub-list for method output_type
0, // [0:2] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_proto_filter_proto_init() }
func file_proto_filter_proto_init() {
if File_proto_filter_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_proto_filter_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FilterReq); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_proto_filter_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ValidateRes); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_proto_filter_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FindAllRes); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_proto_filter_proto_rawDesc,
NumEnums: 0,
NumMessages: 3,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_proto_filter_proto_goTypes,
DependencyIndexes: file_proto_filter_proto_depIdxs,
MessageInfos: file_proto_filter_proto_msgTypes,
}.Build()
File_proto_filter_proto = out.File
file_proto_filter_proto_rawDesc = nil
file_proto_filter_proto_goTypes = nil
file_proto_filter_proto_depIdxs = nil
}

20
keywords-filter/proto/filter.proto Normal file
View File

@@ -0,0 +1,20 @@
syntax = "proto3";
option go_package = "keywords-filter/proto";
package keywords_filter.zvoice.com;
message FilterReq {
string text = 1;
}
message ValidateRes {
bool ok = 1;
string keyword = 2;
}
message FindAllRes {
repeated string keywords = 1;
}
service Filter {
rpc Validate(FilterReq) returns (ValidateRes);
rpc FindAll(FilterReq) returns (FindAllRes);
}

141
keywords-filter/proto/filter_grpc.pb.go Normal file
View File

@@ -0,0 +1,141 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v4.22.0
// source: proto/filter.proto
package proto
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// FilterClient is the client API for Filter service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type FilterClient interface {
Validate(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*ValidateRes, error)
FindAll(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*FindAllRes, error)
}
type filterClient struct {
cc grpc.ClientConnInterface
}
func NewFilterClient(cc grpc.ClientConnInterface) FilterClient {
return &filterClient{cc}
}
func (c *filterClient) Validate(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*ValidateRes, error) {
out := new(ValidateRes)
err := c.cc.Invoke(ctx, "/keywords_filter.zvoice.com.Filter/Validate", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *filterClient) FindAll(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*FindAllRes, error) {
out := new(FindAllRes)
err := c.cc.Invoke(ctx, "/keywords_filter.zvoice.com.Filter/FindAll", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// FilterServer is the server API for Filter service.
// All implementations must embed UnimplementedFilterServer
// for forward compatibility
type FilterServer interface {
Validate(context.Context, *FilterReq) (*ValidateRes, error)
FindAll(context.Context, *FilterReq) (*FindAllRes, error)
mustEmbedUnimplementedFilterServer()
}
// UnimplementedFilterServer must be embedded to have forward compatible implementations.
type UnimplementedFilterServer struct {
}
func (UnimplementedFilterServer) Validate(context.Context, *FilterReq) (*ValidateRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method Validate not implemented")
}
func (UnimplementedFilterServer) FindAll(context.Context, *FilterReq) (*FindAllRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method FindAll not implemented")
}
func (UnimplementedFilterServer) mustEmbedUnimplementedFilterServer() {}
// UnsafeFilterServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to FilterServer will
// result in compilation errors.
type UnsafeFilterServer interface {
mustEmbedUnimplementedFilterServer()
}
func RegisterFilterServer(s grpc.ServiceRegistrar, srv FilterServer) {
s.RegisterService(&Filter_ServiceDesc, srv)
}
func _Filter_Validate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FilterReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(FilterServer).Validate(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/keywords_filter.zvoice.com.Filter/Validate",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(FilterServer).Validate(ctx, req.(*FilterReq))
}
return interceptor(ctx, in, info, handler)
}
func _Filter_FindAll_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FilterReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(FilterServer).FindAll(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/keywords_filter.zvoice.com.Filter/FindAll",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(FilterServer).FindAll(ctx, req.(*FilterReq))
}
return interceptor(ctx, in, info, handler)
}
// Filter_ServiceDesc is the grpc.ServiceDesc for Filter service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var Filter_ServiceDesc = grpc.ServiceDesc{
ServiceName: "keywords_filter.zvoice.com.Filter",
HandlerType: (*FilterServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Validate",
Handler: _Filter_Validate_Handler,
},
{
MethodName: "FindAll",
Handler: _Filter_FindAll_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "proto/filter.proto",
}

11
tokenizer/Dockerfile Normal file
View File

@@ -0,0 +1,11 @@
FROM quay.io/0voice/python:3.10-alpine
WORKDIR /app
ENV PORT 3002
ADD ./tokenizer.py /app
ADD ./requirements.txt /app
RUN pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn --upgrade pip
RUN pip install --root-user-action=ignore -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements.txt
CMD ["sh","-c","nuxt --port ${PORT} --module tokenizer.py --workers 2"]

6
tokenizer/README.md Normal file
View File

@@ -0,0 +1,6 @@
# tokenizer
## 镜像构建
```
docker build -t tokenizer:1.0.0 .
```

2
tokenizer/requirements.txt Normal file
View File

@@ -0,0 +1,2 @@
nuxt>=0.2.0
tiktoken>=0.3.3

65
tokenizer/tokenizer.py Normal file
View File

@@ -0,0 +1,65 @@
from nuxt import route, logger, Request
from nuxt.repositorys.validation import fields, use_args
import traceback
import tiktoken
encoding_cache = {}
support_models = set(["kimi-k2.5",
"gpt-3.5-turbo",
"gpt-3.5-turbo-16k",
"gpt-4",
"gpt-4-32k"])
support_model_prefixes = (
"gpt-3.5-turbo-",
"gpt-4-",
"moonshot-",
"kimi-",
)
@route("/tokenizer/<str:model_name>", methods=["POST"])
@use_args({
"role": fields.Str(required=True),
"content": fields.Str(required=True),
"name": fields.Str(required=False)
}, location="json")
def get_num_tokens(req: Request, message: dict, model_name: str):
try:
return {
"code": 200,
"num_tokens": num_tokens_from_messages([message], model=model_name)
}
except Exception as e:
logger.error(traceback.format_exc())
return {
"code": 500,
"msg": "{}".format(e)
}
def num_tokens_from_messages(messages, model="gpt-3.5-turbo"):
"""Returns the number of tokens used by a list of messages."""
encoding = None
if model in encoding_cache:
encoding = encoding_cache.get(model)
else:
try:
encoding = tiktoken.encoding_for_model(model)
except KeyError:
encoding = tiktoken.get_encoding("cl100k_base")
encoding_cache[model] = encoding
if model in support_models or model.startswith(support_model_prefixes):
num_tokens = 0
for message in messages:
num_tokens += 4 # every message follows <im_start>{role/name}\n{content}<im_end>\n
for key, value in message.items():
num_tokens += len(encoding.encode(value))
if key == "name": # if there's a name, the role is omitted
num_tokens += -1 # role is always required and always 1 token
num_tokens += 3 # every reply is primed with <im_start>assistant
return num_tokens
else:
raise NotImplementedError(f"""num_tokens_from_messages() is not presently implemented for model {model}.
See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""")