✨ 迁移项目

pkg/backoff/backoff.go

@@ -0,0 +1,117 @@
+package backoff
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math/rand"
+	"time"
+)
+
+// BackoffConfig configures a Backoff
+type BackoffConfig struct {
+	MinBackoff time.Duration `yaml:"min_period"`  // start backoff at this level
+	MaxBackoff time.Duration `yaml:"max_period"`  // increase exponentially to this level
+	MaxRetries int           `yaml:"max_retries"` // give up after this many; zero means infinite retries
+}
+
+// RegisterFlags for BackoffConfig.
+func (cfg *BackoffConfig) RegisterFlags(prefix string, f *flag.FlagSet) {
+	f.DurationVar(&cfg.MinBackoff, prefix+".backoff-min-period", 100*time.Millisecond, "Minimum delay when backing off.")
+	f.DurationVar(&cfg.MaxBackoff, prefix+".backoff-max-period", 10*time.Second, "Maximum delay when backing off.")
+	f.IntVar(&cfg.MaxRetries, prefix+".backoff-retries", 10, "Number of times to backoff and retry before failing.")
+}
+
+// Backoff implements exponential backoff with randomized wait times
+type Backoff struct {
+	cfg          BackoffConfig
+	ctx          context.Context
+	numRetries   int
+	nextDelayMin time.Duration
+	nextDelayMax time.Duration
+}
+
+// New creates a Backoff object. Pass a Context that can also terminate the operation.
+func New(ctx context.Context, cfg BackoffConfig) *Backoff {
+	return &Backoff{
+		cfg:          cfg,
+		ctx:          ctx,
+		nextDelayMin: cfg.MinBackoff,
+		nextDelayMax: doubleDuration(cfg.MinBackoff, cfg.MaxBackoff),
+	}
+}
+
+// Reset the Backoff back to its initial condition
+func (b *Backoff) Reset() {
+	b.numRetries = 0
+	b.nextDelayMin = b.cfg.MinBackoff
+	b.nextDelayMax = doubleDuration(b.cfg.MinBackoff, b.cfg.MaxBackoff)
+}
+
+// Ongoing returns true if caller should keep going
+func (b *Backoff) Ongoing() bool {
+	// Stop if Context has errored or max retry count is exceeded
+	return b.ctx.Err() == nil && (b.cfg.MaxRetries == 0 || b.numRetries < b.cfg.MaxRetries)
+}
+
+// Err returns the reason for terminating the backoff, or nil if it didn't terminate
+func (b *Backoff) Err() error {
+	if b.ctx.Err() != nil {
+		return b.ctx.Err()
+	}
+	if b.cfg.MaxRetries != 0 && b.numRetries >= b.cfg.MaxRetries {
+		return fmt.Errorf("terminated after %d retries", b.numRetries)
+	}
+	return nil
+}
+
+// NumRetries returns the number of retries so far
+func (b *Backoff) NumRetries() int {
+	return b.numRetries
+}
+
+// Wait sleeps for the backoff time then increases the retry count and backoff time
+// Returns immediately if Context is terminated
+func (b *Backoff) Wait() {
+	// Increase the number of retries and get the next delay
+	sleepTime := b.NextDelay()
+
+	if b.Ongoing() {
+		select {
+		case <-b.ctx.Done():
+		case <-time.After(sleepTime):
+		}
+	}
+}
+
+func (b *Backoff) NextDelay() time.Duration {
+	b.numRetries++
+
+	// Handle the edge case the min and max have the same value
+	// (or due to some misconfig max is < min)
+	if b.nextDelayMin >= b.nextDelayMax {
+		return b.nextDelayMin
+	}
+
+	// Add a jitter within the next exponential backoff range
+	sleepTime := b.nextDelayMin + time.Duration(rand.Int63n(int64(b.nextDelayMax-b.nextDelayMin)))
+
+	// Apply the exponential backoff to calculate the next jitter
+	// range, unless we've already reached the max
+	if b.nextDelayMax < b.cfg.MaxBackoff {
+		b.nextDelayMin = doubleDuration(b.nextDelayMin, b.cfg.MaxBackoff)
+		b.nextDelayMax = doubleDuration(b.nextDelayMax, b.cfg.MaxBackoff)
+	}
+
+	return sleepTime
+}
+
+func doubleDuration(value time.Duration, max time.Duration) time.Duration {
+	value = value * 2
+
+	if value <= max {
+		return value
+	}
+
+	return max
+}

pkg/helpers/config.go

@@ -0,0 +1,18 @@
+package helpers
+
+import (
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v2"
+)
+
+// LoadConfig read YAML-formatted config from filename into cfg.
+func LoadConfig(filename string, cfg interface{}) error {
+	buf, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return errors.Wrap(err, "Error reading config file")
+	}
+
+	return yaml.UnmarshalStrict(buf, cfg)
+}

pkg/helpers/logerror.go

@@ -0,0 +1,13 @@
+package helpers
+
+import (
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+)
+
+// LogError logs any error returned by f; useful when deferring Close etc.
+func LogError(logger log.Logger, message string, f func() error) {
+	if err := f(); err != nil {
+		level.Error(logger).Log("message", message, "error", err)
+	}
+}

pkg/helpers/math.go

@@ -0,0 +1,9 @@
+package helpers
+
+// MinUint32 return the min of a and b.
+func MinUint32(a, b uint32) uint32 {
+	if a < b {
+		return a
+	}
+	return b
+}

pkg/httputil/http.go

@@ -0,0 +1,164 @@
+package httputil
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/blang/semver"
+	"github.com/gogo/protobuf/proto"
+	"github.com/golang/snappy"
+	"github.com/opentracing/opentracing-go"
+	otlog "github.com/opentracing/opentracing-go/log"
+)
+
+// WriteJSONResponse writes some JSON as a HTTP response.
+func WriteJSONResponse(w http.ResponseWriter, v interface{}) {
+	data, err := json.Marshal(v)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if _, err = w.Write(data); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+}
+
+// RenderHTTPResponse either responds with json or a rendered html page using the passed in template
+// by checking the Accepts header
+func RenderHTTPResponse(w http.ResponseWriter, v interface{}, t *template.Template, r *http.Request) {
+	accept := r.Header.Get("Accept")
+	if strings.Contains(accept, "application/json") {
+		WriteJSONResponse(w, v)
+		return
+	}
+
+	err := t.Execute(w, v)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+// CompressionType for encoding and decoding requests and responses.
+type CompressionType int
+
+// Values for CompressionType
+const (
+	NoCompression CompressionType = iota
+	FramedSnappy
+	RawSnappy
+)
+
+var rawSnappyFromVersion = semver.MustParse("0.1.0")
+
+// CompressionTypeFor a given version of the Prometheus remote storage protocol.
+// See https://github.com/prometheus/prometheus/issues/2692.
+func CompressionTypeFor(version string) CompressionType {
+	ver, err := semver.Make(version)
+	if err != nil {
+		return FramedSnappy
+	}
+
+	if ver.GTE(rawSnappyFromVersion) {
+		return RawSnappy
+	}
+	return FramedSnappy
+}
+
+// ParseProtoReader parses a compressed proto from an io.Reader.
+func ParseProtoReader(ctx context.Context, reader io.Reader, expectedSize, maxSize int, req proto.Message, compression CompressionType) error {
+	var body []byte
+	var err error
+	sp := opentracing.SpanFromContext(ctx)
+	if sp != nil {
+		sp.LogFields(otlog.String("event", "util.ParseProtoRequest[start reading]"))
+	}
+	var buf bytes.Buffer
+	if expectedSize > 0 {
+		if expectedSize > maxSize {
+			return fmt.Errorf("message expected size larger than max (%d vs %d)", expectedSize, maxSize)
+		}
+		buf.Grow(expectedSize + bytes.MinRead) // extra space guarantees no reallocation
+	}
+	switch compression {
+	case NoCompression:
+		// Read from LimitReader with limit max+1. So if the underlying
+		// reader is over limit, the result will be bigger than max.
+		_, err = buf.ReadFrom(io.LimitReader(reader, int64(maxSize)+1))
+		body = buf.Bytes()
+	case FramedSnappy:
+		_, err = buf.ReadFrom(io.LimitReader(snappy.NewReader(reader), int64(maxSize)+1))
+		body = buf.Bytes()
+	case RawSnappy:
+		_, err = buf.ReadFrom(reader)
+		body = buf.Bytes()
+		if sp != nil {
+			sp.LogFields(otlog.String("event", "util.ParseProtoRequest[decompress]"),
+				otlog.Int("size", len(body)))
+		}
+		if err == nil && len(body) <= maxSize {
+			body, err = snappy.Decode(nil, body)
+		}
+	}
+	if err != nil {
+		return err
+	}
+	if len(body) > maxSize {
+		return fmt.Errorf("received message larger than max (%d vs %d)", len(body), maxSize)
+	}
+
+	if sp != nil {
+		sp.LogFields(otlog.String("event", "util.ParseProtoRequest[unmarshal]"),
+			otlog.Int("size", len(body)))
+	}
+
+	// We re-implement proto.Unmarshal here as it calls XXX_Unmarshal first,
+	// which we can't override without upsetting golint.
+	req.Reset()
+	if u, ok := req.(proto.Unmarshaler); ok {
+		err = u.Unmarshal(body)
+	} else {
+		err = proto.NewBuffer(body).Unmarshal(req)
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SerializeProtoResponse serializes a protobuf response into an HTTP response.
+func SerializeProtoResponse(w http.ResponseWriter, resp proto.Message, compression CompressionType) error {
+	data, err := proto.Marshal(resp)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return fmt.Errorf("error marshaling proto response: %v", err)
+	}
+
+	switch compression {
+	case NoCompression:
+	case FramedSnappy:
+		buf := bytes.Buffer{}
+		writer := snappy.NewBufferedWriter(&buf)
+		if _, err := writer.Write(data); err != nil {
+			return err
+		}
+		writer.Close()
+		data = buf.Bytes()
+	case RawSnappy:
+		data = snappy.Encode(nil, data)
+	}
+
+	if _, err := w.Write(data); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return fmt.Errorf("error sending proto response: %v", err)
+	}
+	return nil
+}

pkg/labelutil/label.go

@@ -0,0 +1,99 @@
+package labelutil
+
+import (
+	"bytes"
+	"encoding/csv"
+	"fmt"
+	"strings"
+
+	"github.com/prometheus/common/model"
+	"gopkg.in/yaml.v2"
+)
+
+// LabelSet is a labelSet that can be used as a flag.
+type LabelSet struct {
+	model.LabelSet `yaml:",inline"`
+}
+
+// String implements flag.Value
+// Format: a=1,b=2
+func (v LabelSet) String() string {
+	if v.LabelSet == nil {
+		return ""
+	}
+	records := make([]string, 0, len(v.LabelSet)>>1)
+	for k, v := range v.LabelSet {
+		records = append(records, string(k)+"="+string(v))
+	}
+
+	var buf bytes.Buffer
+	w := csv.NewWriter(&buf)
+	if err := w.Write(records); err != nil {
+		panic(err)
+	}
+	w.Flush()
+	return "[" + strings.TrimSpace(buf.String()) + "]"
+}
+
+// Set implements flag.Value
+func (v *LabelSet) Set(s string) error {
+	var ss []string
+	n := strings.Count(s, "=")
+	switch n {
+	case 0:
+		return fmt.Errorf("%s must be formatted as key=value", s)
+	case 1:
+		ss = append(ss, strings.Trim(s, `"`))
+	default:
+		r := csv.NewReader(strings.NewReader(s))
+		var err error
+		ss, err = r.Read()
+		if err != nil {
+			return err
+		}
+	}
+
+	out := model.LabelSet{}
+	for _, pair := range ss {
+		kv := strings.SplitN(pair, "=", 2)
+		if len(kv) != 2 {
+			return fmt.Errorf("%s must be formatted as key=value", pair)
+		}
+		out[model.LabelName(kv[0])] = model.LabelValue(kv[1])
+	}
+
+	if err := out.Validate(); err != nil {
+		return err
+	}
+	v.LabelSet = out
+	return nil
+}
+
+// UnmarshalYAML the Unmarshaler interface of the yaml pkg.
+func (v *LabelSet) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	lbSet := model.LabelSet{}
+	err := unmarshal(&lbSet)
+	if err != nil {
+		return err
+	}
+	v.LabelSet = lbSet
+	return nil
+}
+
+// MarshalYAML implements yaml.Marshaller.
+func (v LabelSet) MarshalYAML() (interface{}, error) {
+	out, err := yaml.Marshal(ModelLabelSetToMap(v.LabelSet))
+	if err != nil {
+		return nil, err
+	}
+	return string(out), nil
+}
+
+// ModelLabelSetToMap convert a model.LabelSet to a map[string]string
+func ModelLabelSetToMap(m model.LabelSet) map[string]string {
+	result := map[string]string{}
+	for k, v := range m {
+		result[string(k)] = string(v)
+	}
+	return result
+}

pkg/logproto/extensions.go

@@ -0,0 +1,23 @@
+package logproto
+
+import "github.com/prometheus/prometheus/pkg/labels"
+
+// Note, this is not very efficient and use should be minimized as it requires label construction on each comparison
+type SeriesIdentifiers []SeriesIdentifier
+
+func (ids SeriesIdentifiers) Len() int      { return len(ids) }
+func (ids SeriesIdentifiers) Swap(i, j int) { ids[i], ids[j] = ids[j], ids[i] }
+func (ids SeriesIdentifiers) Less(i, j int) bool {
+	a, b := labels.FromMap(ids[i].Labels), labels.FromMap(ids[j].Labels)
+	return labels.Compare(a, b) <= 0
+}
+
+type Streams []Stream
+
+func (xs Streams) Len() int           { return len(xs) }
+func (xs Streams) Swap(i, j int)      { xs[i], xs[j] = xs[j], xs[i] }
+func (xs Streams) Less(i, j int) bool { return xs[i].Labels <= xs[j].Labels }
+
+func (s Series) Len() int           { return len(s.Samples) }
+func (s Series) Swap(i, j int)      { s.Samples[i], s.Samples[j] = s.Samples[j], s.Samples[i] }
+func (s Series) Less(i, j int) bool { return s.Samples[i].Timestamp < s.Samples[j].Timestamp }

pkg/logproto/logproto.proto

@@ -0,0 +1,166 @@
+syntax = "proto3";
+
+package logproto;
+
+option go_package = "github.com/lixh00/loki-client-go/pkg/logproto";
+
+import "google/protobuf/timestamp.proto";
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+service Pusher {
+  rpc Push(PushRequest) returns (PushResponse) {};
+}
+
+service Querier {
+  rpc Query(QueryRequest) returns (stream QueryResponse) {};
+  rpc QuerySample(SampleQueryRequest) returns (stream SampleQueryResponse) {};
+  rpc Label(LabelRequest) returns (LabelResponse) {};
+  rpc Tail(TailRequest) returns (stream TailResponse) {};
+  rpc Series(SeriesRequest) returns (SeriesResponse) {};
+  rpc TailersCount(TailersCountRequest) returns (TailersCountResponse) {};
+  rpc GetChunkIDs(GetChunkIDsRequest) returns (GetChunkIDsResponse) {}; // GetChunkIDs returns ChunkIDs from the index store holding logs for given selectors and time-range.
+}
+
+service Ingester {
+  rpc TransferChunks(stream TimeSeriesChunk) returns (TransferChunksResponse) {};
+}
+
+message PushRequest {
+  repeated StreamAdapter streams = 1 [(gogoproto.jsontag) = "streams", (gogoproto.customtype) = "Stream"];
+}
+
+message PushResponse {
+}
+
+message QueryRequest {
+  string selector = 1;
+  uint32 limit = 2;
+  google.protobuf.Timestamp start = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  google.protobuf.Timestamp end = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  Direction direction = 5;
+  reserved 6;
+  repeated string shards = 7 [(gogoproto.jsontag) = "shards,omitempty"];
+}
+
+message SampleQueryRequest {
+  string selector = 1;
+  google.protobuf.Timestamp start = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  google.protobuf.Timestamp end = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  repeated string shards = 4 [(gogoproto.jsontag) = "shards,omitempty"];
+}
+
+message SampleQueryResponse {
+  repeated Series series = 1 [(gogoproto.customtype) = "Series", (gogoproto.nullable) = true];
+}
+
+enum Direction {
+  FORWARD = 0;
+  BACKWARD = 1;
+}
+
+message QueryResponse {
+  repeated StreamAdapter streams = 1 [(gogoproto.customtype) = "Stream", (gogoproto.nullable) = true];
+}
+
+message LabelRequest {
+  string name = 1;
+  bool values = 2; // True to fetch label values, false for fetch labels names.
+  google.protobuf.Timestamp start = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = true];
+  google.protobuf.Timestamp end = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = true];
+}
+
+message LabelResponse {
+  repeated string values = 1;
+}
+
+message StreamAdapter {
+  string labels = 1 [(gogoproto.jsontag) = "labels"];
+  repeated EntryAdapter entries = 2 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "entries"];
+}
+
+message EntryAdapter {
+  google.protobuf.Timestamp timestamp = 1 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false, (gogoproto.jsontag) = "ts"];
+  string line = 2 [(gogoproto.jsontag) = "line"];
+}
+
+message Sample {
+  int64 timestamp = 1 [(gogoproto.jsontag) = "ts"];
+  double value = 2 [(gogoproto.jsontag) = "value"];
+  uint64 hash = 3 [(gogoproto.jsontag) = "hash"];
+}
+
+message Series {
+  string labels = 1 [(gogoproto.jsontag) = "labels"];
+  repeated Sample samples = 2 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "samples"];
+}
+
+message TailRequest {
+  string query = 1;
+  reserved 2;
+  uint32 delayFor = 3;
+  uint32 limit = 4;
+  google.protobuf.Timestamp start = 5 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+}
+
+message TailResponse {
+  StreamAdapter stream = 1 [(gogoproto.customtype) = "Stream"];
+  repeated DroppedStream droppedStreams = 2;
+}
+
+message SeriesRequest {
+  google.protobuf.Timestamp start = 1 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  google.protobuf.Timestamp end = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  repeated string groups = 3;
+}
+
+message SeriesResponse {
+  repeated SeriesIdentifier series = 1 [(gogoproto.nullable) = false];
+}
+
+message SeriesIdentifier {
+  map<string,string> labels = 1;
+}
+
+message DroppedStream {
+  google.protobuf.Timestamp from = 1 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  google.protobuf.Timestamp to = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  string labels = 3;
+}
+
+message TimeSeriesChunk {
+  string from_ingester_id = 1;
+  string user_id = 2;
+  repeated LabelPair labels = 3;
+  repeated Chunk chunks = 4;
+}
+
+message LabelPair {
+  string name = 1;
+  string value = 2;
+}
+
+message Chunk {
+  bytes data = 1;
+}
+
+message TransferChunksResponse {
+
+}
+
+message TailersCountRequest {
+
+}
+
+message TailersCountResponse {
+  uint32 count = 1;
+}
+
+message GetChunkIDsRequest {
+  string matchers = 1;
+  google.protobuf.Timestamp start = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+  google.protobuf.Timestamp end = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+}
+
+message GetChunkIDsResponse {
+  repeated string chunkIDs = 1;
+}

pkg/logproto/timestamp.go

@@ -0,0 +1,106 @@
+package logproto
+
+import (
+	"errors"
+	strconv "strconv"
+	time "time"
+
+	"github.com/gogo/protobuf/types"
+)
+
+const (
+	// Seconds field of the earliest valid Timestamp.
+	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	minValidSeconds = -62135596800
+	// Seconds field just after the latest valid Timestamp.
+	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	maxValidSeconds = 253402300800
+)
+
+// validateTimestamp determines whether a Timestamp is valid.
+// A valid timestamp represents a time in the range
+// [0001-01-01, 10000-01-01) and has a Nanos field
+// in the range [0, 1e9).
+//
+// If the Timestamp is valid, validateTimestamp returns nil.
+// Otherwise, it returns an error that describes
+// the problem.
+//
+// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
+func validateTimestamp(ts *types.Timestamp) error {
+	if ts == nil {
+		return errors.New("timestamp: nil Timestamp")
+	}
+	if ts.Seconds < minValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
+	}
+	if ts.Seconds >= maxValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
+	}
+	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
+		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
+	}
+	return nil
+}
+
+// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
+// but avoids the escape incurred by using fmt.Sprintf, eliminating
+// unnecessary heap allocations.
+func formatTimestamp(ts *types.Timestamp) string {
+	if ts == nil {
+		return "nil"
+	}
+
+	seconds := strconv.FormatInt(ts.Seconds, 10)
+	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
+	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
+}
+
+func SizeOfStdTime(t time.Time) int {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0
+	}
+	return ts.Size()
+}
+
+func StdTimeMarshalTo(t time.Time, data []byte) (int, error) {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0, err
+	}
+	return ts.MarshalTo(data)
+}
+
+func StdTimeUnmarshal(t *time.Time, data []byte) error {
+	ts := &types.Timestamp{}
+	if err := ts.Unmarshal(data); err != nil {
+		return err
+	}
+	tt, err := timestampFromProto(ts)
+	if err != nil {
+		return err
+	}
+	*t = tt
+	return nil
+}
+
+func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
+	// Don't return the zero value on error, because corresponds to a valid
+	// timestamp. Instead return whatever time.Unix gives us.
+	var t time.Time
+	if ts == nil {
+		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
+	} else {
+		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	}
+	return t, validateTimestamp(ts)
+}
+
+func timestampProto(t time.Time) (types.Timestamp, error) {
+	ts := types.Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.Nanosecond()),
+	}
+	return ts, validateTimestamp(&ts)
+}

pkg/logproto/types.go

@@ -0,0 +1,475 @@
+package logproto
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/prometheus/prometheus/promql/parser"
+)
+
+// Stream contains a unique labels set as a string and a set of entries for it.
+// We are not using the proto generated version but this custom one so that we
+// can improve serialization see benchmark.
+type Stream struct {
+	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
+	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
+}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (r *PushRequest) MarshalJSON() ([]byte, error) {
+	stream := jsoniter.ConfigDefault.BorrowStream(nil)
+	defer jsoniter.ConfigDefault.ReturnStream(stream)
+
+	stream.WriteObjectStart()
+	stream.WriteObjectField("streams")
+	stream.WriteArrayStart()
+	for i, s := range r.Streams {
+		stream.WriteObjectStart()
+		stream.WriteObjectField("stream")
+		stream.WriteObjectStart()
+		lbs, err := parser.ParseMetric(s.Labels)
+		if err != nil {
+			continue
+		}
+		for i, lb := range lbs {
+			stream.WriteObjectField(lb.Name)
+			stream.WriteStringWithHTMLEscaped(lb.Value)
+			if i != len(lbs)-1 {
+				stream.WriteMore()
+			}
+		}
+		stream.WriteObjectEnd()
+		stream.WriteMore()
+		stream.WriteObjectField("values")
+		stream.WriteArrayStart()
+		for i, entry := range s.Entries {
+			stream.WriteArrayStart()
+			stream.WriteRaw(fmt.Sprintf(`"%d"`, entry.Timestamp.UnixNano()))
+			stream.WriteMore()
+			stream.WriteStringWithHTMLEscaped(entry.Line)
+			stream.WriteArrayEnd()
+			if i != len(s.Entries)-1 {
+				stream.WriteMore()
+			}
+		}
+		stream.WriteArrayEnd()
+		stream.WriteObjectEnd()
+		if i != len(r.Streams)-1 {
+			stream.WriteMore()
+		}
+	}
+	stream.WriteArrayEnd()
+	stream.WriteObjectEnd()
+
+	return stream.Buffer(), nil
+}
+
+// Entry is a log entry with a timestamp.
+type Entry struct {
+	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
+	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
+}
+
+func (m *Stream) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Labels) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintLogproto(dAtA, i, uint64(len(m.Labels)))
+		i += copy(dAtA[i:], m.Labels)
+	}
+	if len(m.Entries) > 0 {
+		for _, msg := range m.Entries {
+			dAtA[i] = 0x12
+			i++
+			i = encodeVarintLogproto(dAtA, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(dAtA[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	return i, nil
+}
+
+func (m *Entry) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	dAtA[i] = 0xa
+	i++
+	i = encodeVarintLogproto(dAtA, i, uint64(SizeOfStdTime(m.Timestamp)))
+	n5, err := StdTimeMarshalTo(m.Timestamp, dAtA[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n5
+	if len(m.Line) > 0 {
+		dAtA[i] = 0x12
+		i++
+		i = encodeVarintLogproto(dAtA, i, uint64(len(m.Line)))
+		i += copy(dAtA[i:], m.Line)
+	}
+	return i, nil
+}
+
+func (m *Stream) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowLogproto
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: Stream: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: Stream: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLogproto
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Labels = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLogproto
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Entries = append(m.Entries, Entry{})
+			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipLogproto(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+func (m *Entry) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowLogproto
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: Entry: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: Entry: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLogproto
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := StdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLogproto
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Line = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipLogproto(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+func (m *Stream) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Labels)
+	if l > 0 {
+		n += 1 + l + sovLogproto(uint64(l))
+	}
+	if len(m.Entries) > 0 {
+		for _, e := range m.Entries {
+			l = e.Size()
+			n += 1 + l + sovLogproto(uint64(l))
+		}
+	}
+	return n
+}
+
+func (m *Entry) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = SizeOfStdTime(m.Timestamp)
+	n += 1 + l + sovLogproto(uint64(l))
+	l = len(m.Line)
+	if l > 0 {
+		n += 1 + l + sovLogproto(uint64(l))
+	}
+	return n
+}
+
+func (m *Stream) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Stream)
+	if !ok {
+		that2, ok := that.(Stream)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if m.Labels != that1.Labels {
+		return false
+	}
+	if len(m.Entries) != len(that1.Entries) {
+		return false
+	}
+	for i := range m.Entries {
+		if !m.Entries[i].Equal(that1.Entries[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+func (m *Entry) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Entry)
+	if !ok {
+		that2, ok := that.(Entry)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if !m.Timestamp.Equal(that1.Timestamp) {
+		return false
+	}
+	if m.Line != that1.Line {
+		return false
+	}
+	return true
+}

pkg/logproto/types_test.go

@@ -0,0 +1,111 @@
+package logproto
+
+import (
+	"testing"
+	time "time"
+
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	now    = time.Now().UTC()
+	line   = `level=info ts=2019-12-12T15:00:08.325Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576130400000 maxt=1576152000000 ulid=01DVX9ZHNM71GRCJS7M34Q0EV7 sources="[01DVWNC6NWY1A60AZV3Z6DGS65 01DVWW7XXX75GHA6ZDTD170CSZ 01DVX33N5W86CWJJVRPAVXJRWJ]" duration=2.897213221s`
+	stream = Stream{
+		Labels: `{job="foobar", cluster="foo-central1", namespace="bar", container_name="buzz"}`,
+		Entries: []Entry{
+			{now, line},
+			{now.Add(1 * time.Second), line},
+			{now.Add(2 * time.Second), line},
+			{now.Add(3 * time.Second), line},
+		},
+	}
+	streamAdapter = StreamAdapter{
+		Labels: `{job="foobar", cluster="foo-central1", namespace="bar", container_name="buzz"}`,
+		Entries: []EntryAdapter{
+			{now, line},
+			{now.Add(1 * time.Second), line},
+			{now.Add(2 * time.Second), line},
+			{now.Add(3 * time.Second), line},
+		},
+	}
+)
+
+func TestStream(t *testing.T) {
+	avg := testing.AllocsPerRun(200, func() {
+		b, err := stream.Marshal()
+		require.NoError(t, err)
+
+		var new Stream
+		err = new.Unmarshal(b)
+		require.NoError(t, err)
+
+		require.Equal(t, stream, new)
+	})
+	t.Log("avg allocs per run:", avg)
+}
+
+func TestStreamAdapter(t *testing.T) {
+	avg := testing.AllocsPerRun(200, func() {
+		b, err := streamAdapter.Marshal()
+		require.NoError(t, err)
+
+		var new StreamAdapter
+		err = new.Unmarshal(b)
+		require.NoError(t, err)
+
+		require.Equal(t, streamAdapter, new)
+	})
+	t.Log("avg allocs per run:", avg)
+}
+
+func TestCompatibility(t *testing.T) {
+	b, err := stream.Marshal()
+	require.NoError(t, err)
+
+	var adapter StreamAdapter
+	err = adapter.Unmarshal(b)
+	require.NoError(t, err)
+	require.Equal(t, streamAdapter, adapter)
+
+	ba, err := adapter.Marshal()
+	require.NoError(t, err)
+	require.Equal(t, b, ba)
+
+	var new Stream
+	err = new.Unmarshal(ba)
+	require.NoError(t, err)
+
+	require.Equal(t, stream, new)
+}
+
+func BenchmarkStream(b *testing.B) {
+	b.ReportAllocs()
+	for n := 0; n < b.N; n++ {
+		by, err := stream.Marshal()
+		if err != nil {
+			b.Fatal(err)
+		}
+		var new Stream
+		err = new.Unmarshal(by)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+}
+
+func BenchmarkStreamAdapter(b *testing.B) {
+	b.ReportAllocs()
+	for n := 0; n < b.N; n++ {
+		by, err := streamAdapter.Marshal()
+		if err != nil {
+			b.Fatal(err)
+		}
+		var new StreamAdapter
+		err = new.Unmarshal(by)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+}

pkg/metric/counters.go

@@ -0,0 +1,117 @@
+package metric
+
+import (
+	"strings"
+	"time"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/model"
+)
+
+const (
+	CounterInc = "inc"
+	CounterAdd = "add"
+
+	ErrCounterActionRequired          = "counter action must be defined as either `inc` or `add`"
+	ErrCounterInvalidAction           = "action %s is not valid, action must be either `inc` or `add`"
+	ErrCounterInvalidMatchAll         = "`match_all: true` cannot be combined with `value`, please remove `match_all` or `value`"
+	ErrCounterInvalidCountBytes       = "`count_entry_bytes: true` can only be set with `match_all: true`"
+	ErrCounterInvalidCountBytesAction = "`count_entry_bytes: true` can only be used with `action: add`"
+)
+
+type CounterConfig struct {
+	MatchAll   *bool   `mapstructure:"match_all"`
+	CountBytes *bool   `mapstructure:"count_entry_bytes"`
+	Value      *string `mapstructure:"value"`
+	Action     string  `mapstructure:"action"`
+}
+
+func validateCounterConfig(config *CounterConfig) error {
+	if config.Action == "" {
+		return errors.New(ErrCounterActionRequired)
+	}
+	config.Action = strings.ToLower(config.Action)
+	if config.Action != CounterInc && config.Action != CounterAdd {
+		return errors.Errorf(ErrCounterInvalidAction, config.Action)
+	}
+	if config.MatchAll != nil && *config.MatchAll && config.Value != nil {
+		return errors.Errorf(ErrCounterInvalidMatchAll)
+	}
+	if config.CountBytes != nil && *config.CountBytes && (config.MatchAll == nil || !*config.MatchAll) {
+		return errors.New(ErrCounterInvalidCountBytes)
+	}
+	if config.CountBytes != nil && *config.CountBytes && config.Action != CounterAdd {
+		return errors.New(ErrCounterInvalidCountBytesAction)
+	}
+	return nil
+}
+
+func parseCounterConfig(config interface{}) (*CounterConfig, error) {
+	cfg := &CounterConfig{}
+	err := mapstructure.Decode(config, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+// Counters is a vec tor of counters for a each log stream.
+type Counters struct {
+	*metricVec
+	Cfg *CounterConfig
+}
+
+// NewCounters creates a new counter vec.
+func NewCounters(name, help string, config interface{}, maxIdleSec int64) (*Counters, error) {
+	cfg, err := parseCounterConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	err = validateCounterConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &Counters{
+		metricVec: newMetricVec(func(labels map[string]string) prometheus.Metric {
+			return &expiringCounter{prometheus.NewCounter(prometheus.CounterOpts{
+				Help:        help,
+				Name:        name,
+				ConstLabels: labels,
+			}),
+				0,
+			}
+		}, maxIdleSec),
+		Cfg: cfg,
+	}, nil
+}
+
+// With returns the counter associated with a stream labelset.
+func (c *Counters) With(labels model.LabelSet) prometheus.Counter {
+	return c.metricVec.With(labels).(prometheus.Counter)
+}
+
+type expiringCounter struct {
+	prometheus.Counter
+	lastModSec int64
+}
+
+// Inc increments the counter by 1. Use Add to increment it by arbitrary
+// non-negative values.
+func (e *expiringCounter) Inc() {
+	e.Counter.Inc()
+	e.lastModSec = time.Now().Unix()
+}
+
+// Add adds the given value to the counter. It panics if the value is <
+// 0.
+func (e *expiringCounter) Add(val float64) {
+	e.Counter.Add(val)
+	e.lastModSec = time.Now().Unix()
+}
+
+// HasExpired implements Expirable
+func (e *expiringCounter) HasExpired(currentTimeSec int64, maxAgeSec int64) bool {
+	return currentTimeSec-e.lastModSec >= maxAgeSec
+}

pkg/metric/counters_test.go

@@ -0,0 +1,140 @@
+package metric
+
+import (
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	counterTestTrue  = true
+	counterTestFalse = false
+	counterTestVal   = "some val"
+)
+
+func Test_validateCounterConfig(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name   string
+		config CounterConfig
+		err    error
+	}{
+		{"invalid action",
+			CounterConfig{
+				Action: "del",
+			},
+			errors.Errorf(ErrCounterInvalidAction, "del"),
+		},
+		{"invalid counter match all",
+			CounterConfig{
+				MatchAll: &counterTestTrue,
+				Value:    &counterTestVal,
+				Action:   "inc",
+			},
+			errors.New(ErrCounterInvalidMatchAll),
+		},
+		{"invalid counter match bytes",
+			CounterConfig{
+				MatchAll:   nil,
+				CountBytes: &counterTestTrue,
+				Action:     "add",
+			},
+			errors.New(ErrCounterInvalidCountBytes),
+		},
+		{"invalid counter match bytes action",
+			CounterConfig{
+				MatchAll:   &counterTestTrue,
+				CountBytes: &counterTestTrue,
+				Action:     "inc",
+			},
+			errors.New(ErrCounterInvalidCountBytesAction),
+		},
+		{"valid counter match bytes",
+			CounterConfig{
+				MatchAll:   &counterTestTrue,
+				CountBytes: &counterTestTrue,
+				Action:     "add",
+			},
+			nil,
+		},
+		{"valid",
+			CounterConfig{
+				Value:  &counterTestVal,
+				Action: "inc",
+			},
+			nil,
+		},
+		{"valid match all is false",
+			CounterConfig{
+				MatchAll: &counterTestFalse,
+				Value:    &counterTestVal,
+				Action:   "inc",
+			},
+			nil,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			err := validateCounterConfig(&tt.config)
+			if ((err != nil) && (err.Error() != tt.err.Error())) || (err == nil && tt.err != nil) {
+				t.Errorf("Metrics stage validation error, expected error = %v, actual error = %v", tt.err, err)
+				return
+			}
+		})
+	}
+}
+
+func TestCounterExpiration(t *testing.T) {
+	t.Parallel()
+	cfg := CounterConfig{
+		Action: "inc",
+	}
+
+	cnt, err := NewCounters("test1", "HELP ME!!!!!", cfg, 1)
+	assert.Nil(t, err)
+
+	// Create a label and increment the counter
+	lbl1 := model.LabelSet{}
+	lbl1["test"] = "i don't wanna make this a constant"
+	cnt.With(lbl1).Inc()
+
+	// Collect the metrics, should still find the metric in the map
+	collect(cnt)
+	assert.Contains(t, cnt.metrics, lbl1.Fingerprint())
+
+	time.Sleep(1100 * time.Millisecond) // Wait just past our max idle of 1 sec
+
+	//Add another counter with new label val
+	lbl2 := model.LabelSet{}
+	lbl2["test"] = "eat this linter"
+	cnt.With(lbl2).Inc()
+
+	// Collect the metrics, first counter should have expired and removed, second should still be present
+	collect(cnt)
+	assert.NotContains(t, cnt.metrics, lbl1.Fingerprint())
+	assert.Contains(t, cnt.metrics, lbl2.Fingerprint())
+}
+
+func collect(c prometheus.Collector) {
+	done := make(chan struct{})
+	collector := make(chan prometheus.Metric)
+
+	go func() {
+		defer close(done)
+		c.Collect(collector)
+	}()
+
+	for {
+		select {
+		case <-collector:
+		case <-done:
+			return
+		}
+	}
+}

pkg/metric/gauges.go

@@ -0,0 +1,136 @@
+package metric
+
+import (
+	"strings"
+	"time"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/model"
+)
+
+const (
+	GaugeSet = "set"
+	GaugeInc = "inc"
+	GaugeDec = "dec"
+	GaugeAdd = "add"
+	GaugeSub = "sub"
+
+	ErrGaugeActionRequired = "gauge action must be defined as `set`, `inc`, `dec`, `add`, or `sub`"
+	ErrGaugeInvalidAction  = "action %s is not valid, action must be `set`, `inc`, `dec`, `add`, or `sub`"
+)
+
+type GaugeConfig struct {
+	Value  *string `mapstructure:"value"`
+	Action string  `mapstructure:"action"`
+}
+
+func validateGaugeConfig(config *GaugeConfig) error {
+	if config.Action == "" {
+		return errors.New(ErrGaugeActionRequired)
+	}
+	config.Action = strings.ToLower(config.Action)
+	if config.Action != GaugeSet &&
+		config.Action != GaugeInc &&
+		config.Action != GaugeDec &&
+		config.Action != GaugeAdd &&
+		config.Action != GaugeSub {
+		return errors.Errorf(ErrGaugeInvalidAction, config.Action)
+	}
+	return nil
+}
+
+func parseGaugeConfig(config interface{}) (*GaugeConfig, error) {
+	cfg := &GaugeConfig{}
+	err := mapstructure.Decode(config, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+// Gauges is a vector of gauges for a each log stream.
+type Gauges struct {
+	*metricVec
+	Cfg *GaugeConfig
+}
+
+// NewGauges creates a new gauge vec.
+func NewGauges(name, help string, config interface{}, maxIdleSec int64) (*Gauges, error) {
+	cfg, err := parseGaugeConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	err = validateGaugeConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &Gauges{
+		metricVec: newMetricVec(func(labels map[string]string) prometheus.Metric {
+			return &expiringGauge{prometheus.NewGauge(prometheus.GaugeOpts{
+				Help:        help,
+				Name:        name,
+				ConstLabels: labels,
+			}),
+				0,
+			}
+		}, maxIdleSec),
+		Cfg: cfg,
+	}, nil
+}
+
+// With returns the gauge associated with a stream labelset.
+func (g *Gauges) With(labels model.LabelSet) prometheus.Gauge {
+	return g.metricVec.With(labels).(prometheus.Gauge)
+}
+
+type expiringGauge struct {
+	prometheus.Gauge
+	lastModSec int64
+}
+
+// Set sets the Gauge to an arbitrary value.
+func (g *expiringGauge) Set(val float64) {
+	g.Gauge.Set(val)
+	g.lastModSec = time.Now().Unix()
+}
+
+// Inc increments the Gauge by 1. Use Add to increment it by arbitrary
+// values.
+func (g *expiringGauge) Inc() {
+	g.Gauge.Inc()
+	g.lastModSec = time.Now().Unix()
+}
+
+// Dec decrements the Gauge by 1. Use Sub to decrement it by arbitrary
+// values.
+func (g *expiringGauge) Dec() {
+	g.Gauge.Dec()
+	g.lastModSec = time.Now().Unix()
+}
+
+// Add adds the given value to the Gauge. (The value can be negative,
+// resulting in a decrease of the Gauge.)
+func (g *expiringGauge) Add(val float64) {
+	g.Gauge.Add(val)
+	g.lastModSec = time.Now().Unix()
+}
+
+// Sub subtracts the given value from the Gauge. (The value can be
+// negative, resulting in an increase of the Gauge.)
+func (g *expiringGauge) Sub(val float64) {
+	g.Gauge.Sub(val)
+	g.lastModSec = time.Now().Unix()
+}
+
+// SetToCurrentTime sets the Gauge to the current Unix time in seconds.
+func (g *expiringGauge) SetToCurrentTime() {
+	g.Gauge.SetToCurrentTime()
+	g.lastModSec = time.Now().Unix()
+}
+
+// HasExpired implements Expirable
+func (g *expiringGauge) HasExpired(currentTimeSec int64, maxAgeSec int64) bool {
+	return currentTimeSec-g.lastModSec >= maxAgeSec
+}

pkg/metric/gauges_test.go

@@ -0,0 +1,40 @@
+package metric
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGaugeExpiration(t *testing.T) {
+	t.Parallel()
+	cfg := GaugeConfig{
+		Action: "inc",
+	}
+
+	gag, err := NewGauges("test1", "HELP ME!!!!!", cfg, 1)
+	assert.Nil(t, err)
+
+	// Create a label and increment the gauge
+	lbl1 := model.LabelSet{}
+	lbl1["test"] = "app"
+	gag.With(lbl1).Inc()
+
+	// Collect the metrics, should still find the metric in the map
+	collect(gag)
+	assert.Contains(t, gag.metrics, lbl1.Fingerprint())
+
+	time.Sleep(1100 * time.Millisecond) // Wait just past our max idle of 1 sec
+
+	//Add another gauge with new label val
+	lbl2 := model.LabelSet{}
+	lbl2["test"] = "app2"
+	gag.With(lbl2).Inc()
+
+	// Collect the metrics, first gauge should have expired and removed, second should still be present
+	collect(gag)
+	assert.NotContains(t, gag.metrics, lbl1.Fingerprint())
+	assert.Contains(t, gag.metrics, lbl2.Fingerprint())
+}

pkg/metric/histograms.go

@@ -0,0 +1,79 @@
+package metric
+
+import (
+	"time"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/model"
+)
+
+type HistogramConfig struct {
+	Value   *string   `mapstructure:"value"`
+	Buckets []float64 `mapstructure:"buckets"`
+}
+
+func validateHistogramConfig(config *HistogramConfig) error {
+	return nil
+}
+
+func parseHistogramConfig(config interface{}) (*HistogramConfig, error) {
+	cfg := &HistogramConfig{}
+	err := mapstructure.Decode(config, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+// Histograms is a vector of histograms for a each log stream.
+type Histograms struct {
+	*metricVec
+	Cfg *HistogramConfig
+}
+
+// NewHistograms creates a new histogram vec.
+func NewHistograms(name, help string, config interface{}, maxIdleSec int64) (*Histograms, error) {
+	cfg, err := parseHistogramConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	err = validateHistogramConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &Histograms{
+		metricVec: newMetricVec(func(labels map[string]string) prometheus.Metric {
+			return &expiringHistogram{prometheus.NewHistogram(prometheus.HistogramOpts{
+				Help:        help,
+				Name:        name,
+				ConstLabels: labels,
+				Buckets:     cfg.Buckets,
+			}),
+				0,
+			}
+		}, maxIdleSec),
+		Cfg: cfg,
+	}, nil
+}
+
+// With returns the histogram associated with a stream labelset.
+func (h *Histograms) With(labels model.LabelSet) prometheus.Histogram {
+	return h.metricVec.With(labels).(prometheus.Histogram)
+}
+
+type expiringHistogram struct {
+	prometheus.Histogram
+	lastModSec int64
+}
+
+// Observe adds a single observation to the histogram.
+func (h *expiringHistogram) Observe(val float64) {
+	h.Histogram.Observe(val)
+	h.lastModSec = time.Now().Unix()
+}
+
+// HasExpired implements Expirable
+func (h *expiringHistogram) HasExpired(currentTimeSec int64, maxAgeSec int64) bool {
+	return currentTimeSec-h.lastModSec >= maxAgeSec
+}

pkg/metric/histograms_test.go

@@ -0,0 +1,38 @@
+package metric
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHistogramExpiration(t *testing.T) {
+	t.Parallel()
+	cfg := HistogramConfig{}
+
+	hist, err := NewHistograms("test1", "HELP ME!!!!!", cfg, 1)
+	assert.Nil(t, err)
+
+	// Create a label and increment the histogram
+	lbl1 := model.LabelSet{}
+	lbl1["test"] = "app"
+	hist.With(lbl1).Observe(23)
+
+	// Collect the metrics, should still find the metric in the map
+	collect(hist)
+	assert.Contains(t, hist.metrics, lbl1.Fingerprint())
+
+	time.Sleep(1100 * time.Millisecond) // Wait just past our max idle of 1 sec
+
+	//Add another histogram with new label val
+	lbl2 := model.LabelSet{}
+	lbl2["test"] = "app2"
+	hist.With(lbl2).Observe(2)
+
+	// Collect the metrics, first histogram should have expired and removed, second should still be present
+	collect(hist)
+	assert.NotContains(t, hist.metrics, lbl1.Fingerprint())
+	assert.Contains(t, hist.metrics, lbl2.Fingerprint())
+}

pkg/metric/metricvec.go

@@ -0,0 +1,82 @@
+package metric
+
+import (
+	"sync"
+	"time"
+
+	"github.com/lixh00/loki-client-go/pkg/labelutil"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/model"
+)
+
+// Expirable allows checking if something has exceeded the provided maxAge based on the provided currentTime
+type Expirable interface {
+	HasExpired(currentTimeSec int64, maxAgeSec int64) bool
+}
+
+type metricVec struct {
+	factory   func(labels map[string]string) prometheus.Metric
+	mtx       sync.Mutex
+	metrics   map[model.Fingerprint]prometheus.Metric
+	maxAgeSec int64
+}
+
+func newMetricVec(factory func(labels map[string]string) prometheus.Metric, maxAgeSec int64) *metricVec {
+	return &metricVec{
+		metrics:   map[model.Fingerprint]prometheus.Metric{},
+		factory:   factory,
+		maxAgeSec: maxAgeSec,
+	}
+}
+
+// Describe implements prometheus.Collector and doesn't declare any metrics on purpose to bypass prometheus validation.
+// see https://godoc.org/github.com/prometheus/client_golang/prometheus#hdr-Custom_Collectors_and_constant_Metrics search for "unchecked"
+func (c *metricVec) Describe(ch chan<- *prometheus.Desc) {}
+
+// Collect implements prometheus.Collector
+func (c *metricVec) Collect(ch chan<- prometheus.Metric) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	for _, m := range c.metrics {
+		ch <- m
+	}
+	c.prune()
+}
+
+// With returns the metric associated with the labelset.
+func (c *metricVec) With(labels model.LabelSet) prometheus.Metric {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	fp := labels.Fingerprint()
+	var ok bool
+	var metric prometheus.Metric
+	if metric, ok = c.metrics[fp]; !ok {
+		metric = c.factory(labelutil.ModelLabelSetToMap(labels))
+		c.metrics[fp] = metric
+	}
+	return metric
+}
+
+func (c *metricVec) Delete(labels model.LabelSet) bool {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	fp := labels.Fingerprint()
+	_, ok := c.metrics[fp]
+	if ok {
+		delete(c.metrics, fp)
+	}
+	return ok
+}
+
+// prune will remove all metrics which implement the Expirable interface and have expired
+// it does not take out a lock on the metrics map so whoever calls this function should do so.
+func (c *metricVec) prune() {
+	currentTimeSec := time.Now().Unix()
+	for fp, m := range c.metrics {
+		if em, ok := m.(Expirable); ok {
+			if em.HasExpired(currentTimeSec, c.maxAgeSec) {
+				delete(c.metrics, fp)
+			}
+		}
+	}
+}

pkg/urlutil/url.go

@@ -0,0 +1,59 @@
+package urlutil
+
+import "net/url"
+
+// URLValue is a url.URL that can be used as a flag.
+type URLValue struct {
+	*url.URL
+}
+
+// String implements flag.Value
+func (v URLValue) String() string {
+	if v.URL == nil {
+		return ""
+	}
+	return v.URL.String()
+}
+
+// Set implements flag.Value
+func (v *URLValue) Set(s string) error {
+	u, err := url.Parse(s)
+	if err != nil {
+		return err
+	}
+	v.URL = u
+	return nil
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler.
+func (v *URLValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	// An empty string means no URL has been configured.
+	if s == "" {
+		v.URL = nil
+		return nil
+	}
+
+	return v.Set(s)
+}
+
+// MarshalYAML implements yaml.Marshaler.
+func (v URLValue) MarshalYAML() (interface{}, error) {
+	if v.URL == nil {
+		return "", nil
+	}
+
+	// Mask out passwords when marshalling URLs back to YAML.
+	u := *v.URL
+	if u.User != nil {
+		if _, set := u.User.Password(); set {
+			u.User = url.UserPassword(u.User.Username(), "********")
+		}
+	}
+
+	return u.String(), nil
+}