first commit

chatgpt-web-backend/pkg/tokenizer/tokenizer.go

@@ -0,0 +1,43 @@
+package tokenizer
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/sashabaranov/go-openai"
+)
+
+type tokenInfo struct {
+	Code  int    `json:"code"`
+	Count int    `json:"num_tokens"`
+	Msg   string `json:"msg"`
+}
+
+func GetTokenCount(message openai.ChatCompletionMessage, model string) (int, error) {
+	url := fmt.Sprintf("http://127.0.0.1:5000/tokenizer/%s", model)
+	info := tokenInfo{}
+	if err := postJSON(url, &message, &info); err != nil {
+		return 0, err
+	}
+	if info.Code != 200 {
+		return 0, fmt.Errorf("%v", info.Msg)
+	}
+	return info.Count, nil
+}
+
+func postJSON(url string, requestData *openai.ChatCompletionMessage, responseData *tokenInfo) error {
+	requestBody, err := json.Marshal(requestData)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	return json.NewDecoder(resp.Body).Decode(responseData)
+}