From 774f674524365ec6e12b407315cfcc0e51f4c264 Mon Sep 17 00:00:00 2001 From: Eg <1711788888@qq.com> Date: Wed, 25 May 2022 01:56:38 +0800 Subject: [PATCH] :sparkles: init project --- .gitignore | 27 +++------------ README.md | 3 -- config.go | 25 ++++++++++++++ console.go | 37 ++++++++++++++++++++ file.go | 44 ++++++++++++++++++++++++ go.mod | 52 ++++++++++++++++++++++++++++ gorm.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++ gorm_test.go | 45 ++++++++++++++++++++++++ log/say.go | 75 ++++++++++++++++++++++++++++++++++++++++ logger.go | 55 +++++++++++++++++++++++++++++ logger_test.go | 23 +++++++++++++ loki.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++ readme.md | 28 +++++++++++++++ 13 files changed, 574 insertions(+), 26 deletions(-) delete mode 100644 README.md create mode 100644 config.go create mode 100644 console.go create mode 100644 file.go create mode 100644 go.mod create mode 100644 gorm.go create mode 100644 gorm_test.go create mode 100644 log/say.go create mode 100644 logger.go create mode 100644 logger_test.go create mode 100644 loki.go create mode 100644 readme.md diff --git a/.gitignore b/.gitignore index adf8f72..7d5048b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,4 @@ -# ---> Go -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file -go.work - +.idea +vendor +logs +cache \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index e5de160..0000000 --- a/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# logger - -基于zap封装的日志库 \ No newline at end of file diff --git a/config.go b/config.go new file mode 100644 index 0000000..72be93f --- /dev/null +++ b/config.go @@ -0,0 +1,25 @@ +package logger + +import "fmt" + +type mode string + +const ( + Dev mode = "development" + Prod mode = "production" +) + +// LogConfig 日志配置 +type LogConfig struct { + Mode mode `env:"LOG_MODE" envDefault:"production"` // dev, prod + LokiEnable bool `env:"LOG_LOKI_ENABLE"` // 是否启用Loki + FileEnable bool `env:"LOG_FILE_ENABLE"` // 是否输出到文件 + LokiHost string `env:"LOG_LOKI_HOST"` // Loki地址 + LokiPort int `env:"LOG_LOKI_PORT"` // Loki端口 + LokiSource string `env:"LOG_LOKI_SOURCE_NAME"` // Loki的source名称 + LokiJob string `env:"LOG_LOKI_JOB_NAME"` // Loki的job名称 +} + +func (c LogConfig) getLokiPushURL() string { + return fmt.Sprintf("http://%v:%v/loki/api/v1/push", c.LokiHost, c.LokiPort) +} diff --git a/console.go b/console.go new file mode 100644 index 0000000..24a0743 --- /dev/null +++ b/console.go @@ -0,0 +1,37 @@ +package logger + +import ( + "fmt" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "os" + "time" +) + +// 初始化打印到控制台的ZapCore +func initConsoleCore() zapcore.Core { + // 配置 sugaredLogger + writer := zapcore.AddSync(os.Stdout) + + // 自定义时间输出格式 + customTimeEncoder := func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString(fmt.Sprintf("[%v]", t.Format("2006-01-02 15:04:05.000"))) + } + + // 格式相关的配置 + encoderConfig := zap.NewProductionEncoderConfig() + // 修改时间戳的格式 + encoderConfig.EncodeTime = customTimeEncoder + // 日志级别使用大写带颜色显示 + encoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + encoder := zapcore.NewConsoleEncoder(encoderConfig) + + // 设置日志等级，如果是release模式，控制台只打印Error级别以上的日志 + logLevel := zapcore.DebugLevel + if config.Mode == Prod { + logLevel = zapcore.ErrorLevel + } + // 设置日志级别 + core := zapcore.NewCore(encoder, writer, logLevel) + return core +} diff --git a/file.go b/file.go new file mode 100644 index 0000000..a89c072 --- /dev/null +++ b/file.go @@ -0,0 +1,44 @@ +package logger + +import ( + "fmt" + "github.com/natefinch/lumberjack" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "time" +) + +// 初始化LokiCore，使日志可以输出到文件 +func initFileCore() zapcore.Core { + lumberJackLogger := &lumberjack.Logger{ + Filename: "logs/runtime.log", // 日志文件的位置 + MaxSize: 10, // 最大10M + MaxBackups: 5, // 保留旧文件的最大个数 + MaxAge: 30, // 保留旧文件的最大天数 + Compress: false, // 是否压缩/归档旧文件 + } + // 配置 sugaredLogger + writer := zapcore.AddSync(lumberJackLogger) + + // 自定义时间输出格式 + customTimeEncoder := func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString(fmt.Sprintf("[%v]", t.Format("2006-01-02 15:04:05.000"))) + } + + // 格式相关的配置 + encoderConfig := zap.NewProductionEncoderConfig() + // 修改时间戳的格式 + encoderConfig.EncodeTime = customTimeEncoder + // 日志级别使用大写显示 + encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder + encoder := zapcore.NewConsoleEncoder(encoderConfig) + + // 设置日志等级，如果是release模式，控制台只打印Error级别以上的日志 + logLevel := zapcore.DebugLevel + if config.Mode == Prod { + logLevel = zapcore.ErrorLevel + } + // 设置日志级别 + core := zapcore.NewCore(encoder, writer, logLevel) + return core +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..dff1c37 --- /dev/null +++ b/go.mod @@ -0,0 +1,52 @@ +module git.echol.cn/loser/logger + +go 1.17 + +require ( + github.com/caarlos0/env/v6 v6.9.2 + github.com/go-kit/kit v0.12.0 + github.com/lixh00/loki-client-go v1.0.1 + github.com/natefinch/lumberjack v2.0.0+incompatible + github.com/prometheus/common v0.34.0 + go.uber.org/zap v1.21.0 + gorm.io/driver/mysql v1.3.2 + gorm.io/gorm v1.23.5 +) + +require ( + github.com/BurntSushi/toml v0.4.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.4 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.12.2 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + golang.org/x/net v0.0.0-20220517181318-183a9ca12b87 // indirect + golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect + golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 // indirect + google.golang.org/grpc v1.46.2 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/gorm.go b/gorm.go new file mode 100644 index 0000000..35c5516 --- /dev/null +++ b/gorm.go @@ -0,0 +1,93 @@ +package logger + +import ( + "context" + "errors" + "fmt" + "go.uber.org/zap" + gl "gorm.io/gorm/logger" + "time" +) + +var gormZap *zap.SugaredLogger + +// 基于Gorm的日志实现 +type gormLogger struct { + gl.Config +} + +// LogMode 实现LogMode接口 +func (l *gormLogger) LogMode(level gl.LogLevel) gl.Interface { + nl := *l + nl.LogLevel = level + return &nl +} + +// Info 实现Info接口 +func (l gormLogger) Info(ctx context.Context, msg string, data ...interface{}) { + if l.LogLevel >= gl.Info { + // // 去掉第一行 + // msg = strings.Join(strings.Split(msg, "\n")[1:], " ") + // gormZap.Info(msg) + // + // l.Printf(msg, append([]interface{}{utils.FileWithLineNum()}, data...)...) + } +} + +// Warn 实现Warn接口 +func (l gormLogger) Warn(ctx context.Context, msg string, data ...interface{}) { + if l.LogLevel >= gl.Warn { + // + } +} + +// 实现Error接口 +func (l gormLogger) Error(ctx context.Context, msg string, data ...interface{}) { + if l.LogLevel >= gl.Error { + // + } +} + +// Trace 实现Trace接口 +func (l gormLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { + if l.LogLevel <= gl.Silent { + return + } + + elapsed := time.Since(begin) + sql, rows := fc() + msg := fmt.Sprintf("[%v] [rows:%v] %s", elapsed.String(), rows, sql) + if rows == -1 { + msg = fmt.Sprintf("[%s] [-] %s", elapsed.String(), sql) + } + + switch { + case err != nil && l.LogLevel >= gl.Error && (!errors.Is(err, gl.ErrRecordNotFound) || !l.IgnoreRecordNotFoundError): + gormZap.Errorf("%s -> %s", err.Error(), sql) + case elapsed > l.SlowThreshold && l.SlowThreshold != 0 && l.LogLevel >= gl.Warn: + slowLog := fmt.Sprintf("SLOW SQL >= %v", l.SlowThreshold) + gormZap.Warnf("%v -> %v", slowLog, sql) + case l.LogLevel == gl.Info: + gormZap.Info(msg) + } +} + +// NewGormLoggerWithConfig ... +func NewGormLoggerWithConfig(config gl.Config) gl.Interface { + return &gormLogger{config} +} + +// DefaultGormLogger 默认的日志实现 +func DefaultGormLogger() gl.Interface { + // 默认日志级别为Info，如果是生产环境，就是Error + logLevel := gl.Info + if config.Mode == Prod { + logLevel = gl.Error + } + return &gormLogger{gl.Config{ + SlowThreshold: time.Second, // Slow SQL threshold + IgnoreRecordNotFoundError: false, // 忽略没找到结果的错误 + LogLevel: logLevel, // Log level + Colorful: false, // Disable color + }} +} diff --git a/gorm_test.go b/gorm_test.go new file mode 100644 index 0000000..836bd84 --- /dev/null +++ b/gorm_test.go @@ -0,0 +1,45 @@ +package logger + +import ( + "gitee.ltd/lxh/logger/log" + "gorm.io/driver/mysql" + "gorm.io/gorm" + gl "gorm.io/gorm/logger" + "testing" + "time" +) + +func TestGormLogger(t *testing.T) { + dsn := "saas:saas123@tcp(10.11.0.10:3307)/saas_tenant?charset=utf8mb4&parseTime=True&loc=Local" + + engine, err := gorm.Open(mysql.Open(dsn), &gorm.Config{Logger: DefaultGormLogger()}) + if err != nil { + log.Panicf("mysql connect error: %s", err.Error()) + } + + var count int64 + if err := engine.Table("t_tenant").Count(&count).Error; err != nil { + t.Log(err) + } + t.Logf("count: %d", count) +} + +func TestGormLoggerWithConfig(t *testing.T) { + dsn := "saas:saas123@tcp(10.11.0.10:3307)/saas_tenant?charset=utf8mb4&parseTime=True&loc=Local" + + engine, err := gorm.Open(mysql.Open(dsn), &gorm.Config{Logger: NewGormLoggerWithConfig(gl.Config{ + SlowThreshold: time.Second, // Slow SQL threshold + IgnoreRecordNotFoundError: false, // 忽略没找到结果的错误 + LogLevel: gl.Warn, // Log level + Colorful: false, // Disable color + })}) + if err != nil { + log.Panicf("mysql connect error: %s", err.Error()) + } + + var count int64 + if err := engine.Table("t_tenant1").Count(&count).Error; err != nil { + t.Log(err) + } + t.Logf("count: %d", count) +} diff --git a/log/say.go b/log/say.go new file mode 100644 index 0000000..ee76012 --- /dev/null +++ b/log/say.go @@ -0,0 +1,75 @@ +package log + +import "go.uber.org/zap" + +// Debug uses fmt.Sprint to construct and log a message. +func Debug(args ...interface{}) { + defer zap.S().Sync() + zap.S().Debug(args...) +} + +// Info uses fmt.Sprint to construct and log a message. +func Info(args ...interface{}) { + defer zap.S().Sync() + zap.S().Info(args...) +} + +// Warn uses fmt.Sprint to construct and log a message. +func Warn(args ...interface{}) { + defer zap.S().Sync() + zap.S().Warn(args...) +} + +// Error uses fmt.Sprint to construct and log a message. +func Error(args ...interface{}) { + defer zap.S().Sync() + zap.S().Error(args...) +} + +// Panic uses fmt.Sprint to construct and log a message, then panics. +func Panic(args ...interface{}) { + defer zap.S().Sync() + zap.S().Panic(args...) +} + +// Fatal uses fmt.Sprint to construct and log a message, then calls os.Exit. +func Fatal(args ...interface{}) { + defer zap.S().Sync() + zap.S().Fatal(args...) +} + +// Debugf uses fmt.Sprintf to log a templated message. +func Debugf(template string, args ...interface{}) { + defer zap.S().Sync() + zap.S().Debugf(template, args...) +} + +// Infof uses fmt.Sprintf to log a templated message. +func Infof(template string, args ...interface{}) { + defer zap.S().Sync() + zap.S().Infof(template, args...) +} + +// Warnf uses fmt.Sprintf to log a templated message. +func Warnf(template string, args ...interface{}) { + defer zap.S().Sync() + zap.S().Warnf(template, args...) +} + +// Errorf uses fmt.Sprintf to log a templated message. +func Errorf(template string, args ...interface{}) { + defer zap.S().Sync() + zap.S().Errorf(template, args...) +} + +// Panicf uses fmt.Sprintf to log a templated message, then panics. +func Panicf(template string, args ...interface{}) { + defer zap.S().Sync() + zap.S().Panicf(template, args...) +} + +// Fatalf uses fmt.Sprintf to log a templated message, then calls os.Exit. +func Fatalf(template string, args ...interface{}) { + defer zap.S().Sync() + zap.S().Fatalf(template, args...) +} diff --git a/logger.go b/logger.go new file mode 100644 index 0000000..9d2dc2b --- /dev/null +++ b/logger.go @@ -0,0 +1,55 @@ +package logger + +import ( + "fmt" + "github.com/caarlos0/env/v6" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var config LogConfig +var initialized bool + +// 避免异常，在第一次调用时初始化一个只打印到控制台的logger +func init() { + if !initialized { + // 从环境变量读取配置 + var c LogConfig + if err := env.Parse(&c); err != nil { + fmt.Println("日志配置解析错误: " + err.Error()) + c = LogConfig{Mode: Dev, LokiEnable: false, FileEnable: false} + } + // 如果值错了，直接默认为Prod + if c.Mode != Dev && c.Mode != Prod { + c.Mode = Prod + } + InitLogger(c) + } +} + +// InitLogger 初始化日志工具 +func InitLogger(c LogConfig) { + config = c + var cores []zapcore.Core + // 生成输出到控制台的Core + consoleCore := initConsoleCore() + cores = append(cores, consoleCore) + // 生成输出到Loki的Core + if config.LokiEnable { + lokiCore := initLokiCore() + cores = append(cores, lokiCore) + } + // 输出到文件的Core + if config.FileEnable { + fileCore := initFileCore() + cores = append(cores, fileCore) + } + + // 增加 caller 信息 + // AddCallerSkip 输出的文件名和行号是调用封装函数的位置，而不是调用日志函数的位置 + logger := zap.New(zapcore.NewTee(cores...), zap.AddCaller(), zap.AddCallerSkip(1)) + initialized = true + // 给GORM单独生成一个 + gormZap = zap.New(zapcore.NewTee(cores...), zap.AddCaller(), zap.AddCallerSkip(3)).Sugar() + zap.ReplaceGlobals(logger) +} diff --git a/logger_test.go b/logger_test.go new file mode 100644 index 0000000..ecc7aa3 --- /dev/null +++ b/logger_test.go @@ -0,0 +1,23 @@ +package logger + +import ( + "gitee.ltd/lxh/logger/log" + "testing" + "time" +) + +func TestLogger(t *testing.T) { + InitLogger(LogConfig{Mode: Dev, LokiEnable: false, FileEnable: true}) + log.Debug("芜湖") +} + +func TestLogger1(t *testing.T) { + log.Info("我是测试消息") + time.Sleep(5 * time.Second) +} + +func TestLogger2(t *testing.T) { + InitLogger(LogConfig{Mode: Dev, LokiEnable: false, FileEnable: true}) + log.Info("我是测试消息") + //time.Sleep(5 * time.Second) +} diff --git a/loki.go b/loki.go new file mode 100644 index 0000000..718b3e5 --- /dev/null +++ b/loki.go @@ -0,0 +1,93 @@ +package logger + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/go-kit/kit/log" + "github.com/lixh00/loki-client-go/loki" + "github.com/prometheus/common/model" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "time" +) + +// Loki连接对象 +var lokiClient *loki.Client + +// 日志输出 +type lokiWriter struct{} + +// 初始化LokiCore，使日志可以推送到Loki +func initLokiCore() zapcore.Core { + initLokiClient() + // 日志输出到控制台和Loki + writer := zapcore.AddSync(newLokiWriter()) + + // 自定义时间输出格式 + customTimeEncoder := func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString(t.Format("2006-01-02 15:04:05.000")) + } + + // 格式相关的配置 + encoderConfig := zap.NewProductionEncoderConfig() + // 修改时间戳的格式 + encoderConfig.EncodeTime = customTimeEncoder // zapcore.EpochNanosTimeEncoder + // 日志级别使用大写 + encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder + // 将日志级别设置为 DEBUG + return zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), writer, zapcore.DebugLevel) +} + +// 初始化LokiClient +func initLokiClient() { + // 如果Loki配置错误，返回一个nil + if config.LokiHost == "" || config.LokiPort < 1 { + panic(errors.New("Loki配置错误")) + } + // 初始化配置 + cfg, _ := loki.NewDefaultConfig(config.getLokiPushURL()) + // 创建连接对象 + client, err := loki.NewWithLogger(cfg, log.NewNopLogger()) + if err != nil { + panic("Loki初始化失败: " + err.Error()) + } + lokiClient = client +} + +// 实现Write接口，使lokiClient可以作为zap的扩展 +func (c lokiWriter) Write(p []byte) (int, error) { + type logInfo struct { + Level string `json:"level"` // 日志级别 + Ts string `json:"ts"` // 格式化后的时间(在zap那边配置的) + Caller string `json:"caller"` // 日志输出的文件名和行号 + Msg string `json:"msg"` // 日志内容 + } + var li logInfo + err := json.Unmarshal(p, &li) + if err != nil { + return 0, err + } + + label := model.LabelSet{"job": model.LabelValue(config.LokiJob)} + label["source"] = model.LabelValue(config.LokiSource) + label["level"] = model.LabelValue(li.Level) + label["caller"] = model.LabelValue(li.Caller) + // 异步推送消息到服务器 + go func() { + t, e := time.ParseInLocation("2006-01-02 15:04:05.000", li.Ts, time.Local) + if e != nil { + t = time.Now().Local() + } + if err = lokiClient.Handle(label, t, li.Msg); err != nil { + fmt.Printf("日志推送到Loki失败: %v\n", err.Error()) + } + }() + + return 0, nil +} + +// NewLokiWriter +func newLokiWriter() *lokiWriter { + return &lokiWriter{} +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..af971e0 --- /dev/null +++ b/readme.md @@ -0,0 +1,28 @@ +### Logger +基于Zap整合的日志框架，可自由组合输出到Console、File、Loki + +### Demo +```go +package main + +import ( + "gitee.ltd/lxh/logger" + "gitee.ltd/lxh/logger/log" +) + +func main() { + logger.InitLogger(logger.LogConfig{Mode: logger.Dev, LokiEnable: false, FileEnable: true}) + log.Debug("芜湖") +} +``` + +### 环境变量 +```shell +export LOG_MODE=0 # development | production +export LOG_LOKI_ENABLE=1 # 是否启用Loki 0: disable, 1: enable +export LOG_FILE_ENABLE=0 # 是否启用输出到文件 0: disable, 1: enable +export LOG_LOKI_HOST=10.0.0.31 # Loki地址 +export LOG_LOKI_PORT=3100 # Loki端口 +export LOG_LOKI_SOURCE_NAME=tests # Loki Source 名称 +export LOG_LOKI_JOB_NAME=testj # Loki Job 名称 +``` \ No newline at end of file