tokenizer

2026-04-03 10:29:38 +08:00
parent de99cb2806
commit c1a895258f
70 changed files with 22320 additions and 239 deletions
--- a/ai-chat-service/services/grpc-client/default.go
+++ b/ai-chat-service/services/grpc-client/default.go
@@ -0,0 +1,28 @@
+package grpc_client
+
+import (
+	"ai-chat-service/pkg/log"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+type ServiceClient interface {
+	GetPool(addr string) ClientPool
+}
+type DefaultClient struct {
+}
+
+func (c *DefaultClient) GetPool(addr string) ClientPool {
+	pool, err := NewPool(addr, c.getOptions()...)
+	if err != nil {
+		log.Error(err)
+		return nil
+	}
+	return pool
+}
+
+func (c *DefaultClient) getOptions() []grpc.DialOption {
+	opts := make([]grpc.DialOption, 0)
+	opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	return opts
+}
--- a/ai-chat-service/services/grpc-client/grpc_client_pool.go
+++ b/ai-chat-service/services/grpc-client/grpc_client_pool.go
@@ -0,0 +1,48 @@
+package grpc_client
+
+import (
+	"ai-chat-service/pkg/log"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"sync"
+)
+
+type ClientPool interface {
+	Get() *grpc.ClientConn
+	Put(*grpc.ClientConn)
+}
+
+type clientPool struct {
+	pool sync.Pool
+}
+
+func NewPool(target string, opts ...grpc.DialOption) (ClientPool, error) {
+	return &clientPool{
+		pool: sync.Pool{
+			New: func() any {
+				conn, err := grpc.NewClient(target, opts...)
+				if err != nil {
+					log.Error(err)
+					return nil
+				}
+				return conn
+			},
+		},
+	}, nil
+}
+
+func (c *clientPool) Get() *grpc.ClientConn {
+	conn := c.pool.Get().(*grpc.ClientConn)
+	if conn.GetState() == connectivity.Shutdown || conn.GetState() == connectivity.TransientFailure {
+		conn.Close()
+		conn = c.pool.New().(*grpc.ClientConn)
+	}
+	return conn
+}
+func (c *clientPool) Put(conn *grpc.ClientConn) {
+	if conn.GetState() == connectivity.Shutdown || conn.GetState() == connectivity.TransientFailure {
+		conn.Close()
+		return
+	}
+	c.pool.Put(conn)
+}
--- a/ai-chat-service/services/keywords-filter/keywords.go
+++ b/ai-chat-service/services/keywords-filter/keywords.go
@@ -0,0 +1,23 @@
+package keywords_filter
+
+import (
+	"ai-chat-service/pkg/config"
+	grpc_client "ai-chat-service/services/grpc-client"
+	"sync"
+)
+
+var keywordsPool grpc_client.ClientPool
+var keywordsOnce sync.Once
+
+type keywordsClient struct {
+	grpc_client.DefaultClient
+}
+
+func GetKeywordsClientPool() grpc_client.ClientPool {
+	keywordsOnce.Do(func() {
+		cnf := config.GetConfig()
+		c := &keywordsClient{}
+		keywordsPool = c.GetPool(cnf.DependOn.Keywords.Address)
+	})
+	return keywordsPool
+}
--- a/ai-chat-service/services/keywords-filter/proto/filter.pb.go
+++ b/ai-chat-service/services/keywords-filter/proto/filter.pb.go
@@ -0,0 +1,294 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.1
+// 	protoc        v4.22.0
+// source: proto/filter.proto
+
+package proto
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type FilterReq struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Text string `protobuf:"bytes,1,opt,name=text,proto3" json:"text,omitempty"`
+}
+
+func (x *FilterReq) Reset() {
+	*x = FilterReq{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_filter_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FilterReq) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FilterReq) ProtoMessage() {}
+
+func (x *FilterReq) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_filter_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FilterReq.ProtoReflect.Descriptor instead.
+func (*FilterReq) Descriptor() ([]byte, []int) {
+	return file_proto_filter_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *FilterReq) GetText() string {
+	if x != nil {
+		return x.Text
+	}
+	return ""
+}
+
+type ValidateRes struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Ok      bool   `protobuf:"varint,1,opt,name=ok,proto3" json:"ok,omitempty"`
+	Keyword string `protobuf:"bytes,2,opt,name=keyword,proto3" json:"keyword,omitempty"`
+}
+
+func (x *ValidateRes) Reset() {
+	*x = ValidateRes{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_filter_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ValidateRes) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ValidateRes) ProtoMessage() {}
+
+func (x *ValidateRes) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_filter_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ValidateRes.ProtoReflect.Descriptor instead.
+func (*ValidateRes) Descriptor() ([]byte, []int) {
+	return file_proto_filter_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *ValidateRes) GetOk() bool {
+	if x != nil {
+		return x.Ok
+	}
+	return false
+}
+
+func (x *ValidateRes) GetKeyword() string {
+	if x != nil {
+		return x.Keyword
+	}
+	return ""
+}
+
+type FindAllRes struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Keywords []string `protobuf:"bytes,1,rep,name=keywords,proto3" json:"keywords,omitempty"`
+}
+
+func (x *FindAllRes) Reset() {
+	*x = FindAllRes{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_filter_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FindAllRes) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FindAllRes) ProtoMessage() {}
+
+func (x *FindAllRes) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_filter_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FindAllRes.ProtoReflect.Descriptor instead.
+func (*FindAllRes) Descriptor() ([]byte, []int) {
+	return file_proto_filter_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *FindAllRes) GetKeywords() []string {
+	if x != nil {
+		return x.Keywords
+	}
+	return nil
+}
+
+var File_proto_filter_proto protoreflect.FileDescriptor
+
+var file_proto_filter_proto_rawDesc = []byte{
+	0x0a, 0x12, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1a, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66,
+	0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d,
+	0x22, 0x1f, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x12, 0x0a,
+	0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78,
+	0x74, 0x22, 0x37, 0x0a, 0x0b, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73,
+	0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b,
+	0x12, 0x18, 0x0a, 0x07, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x28, 0x0a, 0x0a, 0x46, 0x69,
+	0x6e, 0x64, 0x41, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77,
+	0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77,
+	0x6f, 0x72, 0x64, 0x73, 0x32, 0xbe, 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12,
+	0x5a, 0x0a, 0x08, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x12, 0x25, 0x2e, 0x6b, 0x65,
+	0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76,
+	0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52,
+	0x65, 0x71, 0x1a, 0x27, 0x2e, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69,
+	0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e,
+	0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x07, 0x46,
+	0x69, 0x6e, 0x64, 0x41, 0x6c, 0x6c, 0x12, 0x25, 0x2e, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64,
+	0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e,
+	0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x26, 0x2e,
+	0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e,
+	0x7a, 0x76, 0x6f, 0x69, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x41,
+	0x6c, 0x6c, 0x52, 0x65, 0x73, 0x42, 0x17, 0x5a, 0x15, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64,
+	0x73, 0x2d, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_proto_filter_proto_rawDescOnce sync.Once
+	file_proto_filter_proto_rawDescData = file_proto_filter_proto_rawDesc
+)
+
+func file_proto_filter_proto_rawDescGZIP() []byte {
+	file_proto_filter_proto_rawDescOnce.Do(func() {
+		file_proto_filter_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_filter_proto_rawDescData)
+	})
+	return file_proto_filter_proto_rawDescData
+}
+
+var file_proto_filter_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_proto_filter_proto_goTypes = []interface{}{
+	(*FilterReq)(nil),   // 0: keywords_filter.zvoice.com.FilterReq
+	(*ValidateRes)(nil), // 1: keywords_filter.zvoice.com.ValidateRes
+	(*FindAllRes)(nil),  // 2: keywords_filter.zvoice.com.FindAllRes
+}
+var file_proto_filter_proto_depIdxs = []int32{
+	0, // 0: keywords_filter.zvoice.com.Filter.Validate:input_type -> keywords_filter.zvoice.com.FilterReq
+	0, // 1: keywords_filter.zvoice.com.Filter.FindAll:input_type -> keywords_filter.zvoice.com.FilterReq
+	1, // 2: keywords_filter.zvoice.com.Filter.Validate:output_type -> keywords_filter.zvoice.com.ValidateRes
+	2, // 3: keywords_filter.zvoice.com.Filter.FindAll:output_type -> keywords_filter.zvoice.com.FindAllRes
+	2, // [2:4] is the sub-list for method output_type
+	0, // [0:2] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_proto_filter_proto_init() }
+func file_proto_filter_proto_init() {
+	if File_proto_filter_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_proto_filter_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FilterReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_filter_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ValidateRes); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_filter_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FindAllRes); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_proto_filter_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   3,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_proto_filter_proto_goTypes,
+		DependencyIndexes: file_proto_filter_proto_depIdxs,
+		MessageInfos:      file_proto_filter_proto_msgTypes,
+	}.Build()
+	File_proto_filter_proto = out.File
+	file_proto_filter_proto_rawDesc = nil
+	file_proto_filter_proto_goTypes = nil
+	file_proto_filter_proto_depIdxs = nil
+}
--- a/ai-chat-service/services/keywords-filter/proto/filter.proto
+++ b/ai-chat-service/services/keywords-filter/proto/filter.proto
@@ -0,0 +1,20 @@
+syntax = "proto3";
+option go_package = "keywords-filter/proto";
+package keywords_filter.zvoice.com;
+
+message FilterReq {
+  string text = 1;
+}
+
+message ValidateRes {
+  bool ok = 1;
+  string keyword = 2;
+}
+
+message FindAllRes {
+  repeated string keywords = 1;
+}
+service Filter {
+  rpc Validate(FilterReq) returns (ValidateRes);
+  rpc FindAll(FilterReq) returns (FindAllRes);
+}
--- a/ai-chat-service/services/keywords-filter/proto/filter_grpc.pb.go
+++ b/ai-chat-service/services/keywords-filter/proto/filter_grpc.pb.go
@@ -0,0 +1,141 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v4.22.0
+// source: proto/filter.proto
+
+package proto
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
+const _ = grpc.SupportPackageIsVersion7
+
+// FilterClient is the client API for Filter service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type FilterClient interface {
+	Validate(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*ValidateRes, error)
+	FindAll(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*FindAllRes, error)
+}
+
+type filterClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewFilterClient(cc grpc.ClientConnInterface) FilterClient {
+	return &filterClient{cc}
+}
+
+func (c *filterClient) Validate(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*ValidateRes, error) {
+	out := new(ValidateRes)
+	err := c.cc.Invoke(ctx, "/keywords_filter.zvoice.com.Filter/Validate", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *filterClient) FindAll(ctx context.Context, in *FilterReq, opts ...grpc.CallOption) (*FindAllRes, error) {
+	out := new(FindAllRes)
+	err := c.cc.Invoke(ctx, "/keywords_filter.zvoice.com.Filter/FindAll", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// FilterServer is the server API for Filter service.
+// All implementations must embed UnimplementedFilterServer
+// for forward compatibility
+type FilterServer interface {
+	Validate(context.Context, *FilterReq) (*ValidateRes, error)
+	FindAll(context.Context, *FilterReq) (*FindAllRes, error)
+	mustEmbedUnimplementedFilterServer()
+}
+
+// UnimplementedFilterServer must be embedded to have forward compatible implementations.
+type UnimplementedFilterServer struct {
+}
+
+func (UnimplementedFilterServer) Validate(context.Context, *FilterReq) (*ValidateRes, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Validate not implemented")
+}
+func (UnimplementedFilterServer) FindAll(context.Context, *FilterReq) (*FindAllRes, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method FindAll not implemented")
+}
+func (UnimplementedFilterServer) mustEmbedUnimplementedFilterServer() {}
+
+// UnsafeFilterServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to FilterServer will
+// result in compilation errors.
+type UnsafeFilterServer interface {
+	mustEmbedUnimplementedFilterServer()
+}
+
+func RegisterFilterServer(s grpc.ServiceRegistrar, srv FilterServer) {
+	s.RegisterService(&Filter_ServiceDesc, srv)
+}
+
+func _Filter_Validate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(FilterReq)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(FilterServer).Validate(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/keywords_filter.zvoice.com.Filter/Validate",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(FilterServer).Validate(ctx, req.(*FilterReq))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Filter_FindAll_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(FilterReq)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(FilterServer).FindAll(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/keywords_filter.zvoice.com.Filter/FindAll",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(FilterServer).FindAll(ctx, req.(*FilterReq))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// Filter_ServiceDesc is the grpc.ServiceDesc for Filter service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var Filter_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "keywords_filter.zvoice.com.Filter",
+	HandlerType: (*FilterServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Validate",
+			Handler:    _Filter_Validate_Handler,
+		},
+		{
+			MethodName: "FindAll",
+			Handler:    _Filter_FindAll_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "proto/filter.proto",
+}
--- a/ai-chat-service/services/keywords-filter/sensitive.go
+++ b/ai-chat-service/services/keywords-filter/sensitive.go
@@ -0,0 +1,23 @@
+package keywords_filter
+
+import (
+	"ai-chat-service/pkg/config"
+	grpc_client "ai-chat-service/services/grpc-client"
+	"sync"
+)
+
+var sensitivePool grpc_client.ClientPool
+var sensitiveOnce sync.Once
+
+type sensitiveClient struct {
+	grpc_client.DefaultClient
+}
+
+func GetSensitiveClientPool() grpc_client.ClientPool {
+	sensitiveOnce.Do(func() {
+		cnf := config.GetConfig()
+		c := &sensitiveClient{}
+		sensitivePool = c.GetPool(cnf.DependOn.Sensitive.Address)
+	})
+	return sensitivePool
+}
--- a/ai-chat-service/services/services.go
+++ b/ai-chat-service/services/services.go
@@ -0,0 +1,11 @@
+package services
+
+import (
+	"context"
+	"google.golang.org/grpc/metadata"
+)
+
+func AppendBearerTokenToContext(ctx context.Context, accessToken string) context.Context {
+	md := metadata.Pairs("Authorization", "Bearer "+accessToken)
+	return metadata.NewOutgoingContext(ctx, md)
+}
--- a/ai-chat-service/services/tokenizer/tokenizer.go
+++ b/ai-chat-service/services/tokenizer/tokenizer.go
@@ -0,0 +1,43 @@
+package tokenizer
+
+import (
+	"ai-chat-service/pkg/config"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/sashabaranov/go-openai"
+	"net/http"
+)
+
+type tokensInfo struct {
+	Code   int    `json:"code"`
+	Tokens int    `json:"num_tokens"`
+	Msg    string `json:"msg"`
+}
+
+var httpClient = &http.Client{}
+
+func GetTokens(message *openai.ChatCompletionMessage, model string) (int, error) {
+	cnf := config.GetConfig()
+	url := fmt.Sprintf("%s/tokenizer/%s", cnf.DependOn.Tokenizer.Address, model)
+	info := &tokensInfo{}
+	if err := postJSON(url, message, info); err != nil {
+		return 0, err
+	}
+	if info.Code != 200 {
+		return 0, fmt.Errorf("%v", info.Msg)
+	}
+	return info.Tokens, nil
+}
+
+func postJSON(url string, requestData *openai.ChatCompletionMessage, responseData *tokensInfo) error {
+	requestBody, err := json.Marshal(requestData)
+	if err != nil {
+		return err
+	}
+	resp, err := httpClient.Post(url, "application/json", bytes.NewReader(requestBody))
+	if err != nil {
+		return err
+	}
+	return json.NewDecoder(resp.Body).Decode(responseData)
+}