diff -Nru mtail-3.0.0~rc49/debian/changelog mtail-3.0.0~rc50/debian/changelog --- mtail-3.0.0~rc49/debian/changelog 2022-05-25 18:16:47.000000000 +0000 +++ mtail-3.0.0~rc50/debian/changelog 2022-08-23 14:42:45.000000000 +0000 @@ -1,3 +1,10 @@ +mtail (3.0.0~rc50-1) unstable; urgency=medium + + * New upstream release. + * Add patch to fix build errors in go 1.18. Closes: #1017291 + + -- Martina Ferrari Tue, 23 Aug 2022 14:42:45 +0000 + mtail (3.0.0~rc49-1) unstable; urgency=medium * New upstream release. @@ -156,7 +163,7 @@ [ Alexandre Viau ] * Point Vcs-* urls to salsa.debian.org. - [ Martín Ferrari ] + [ Martina Ferrari ] * New upstream release candidate. * Remove now unneeded patch. * Automated cmd fixes. diff -Nru mtail-3.0.0~rc49/debian/patches/02-Go1.18_imports.patch mtail-3.0.0~rc50/debian/patches/02-Go1.18_imports.patch --- mtail-3.0.0~rc49/debian/patches/02-Go1.18_imports.patch 1970-01-01 00:00:00.000000000 +0000 +++ mtail-3.0.0~rc50/debian/patches/02-Go1.18_imports.patch 2022-08-23 14:42:45.000000000 +0000 @@ -0,0 +1,50 @@ +Author: Martina Ferrari +Date: Tue, 23 Aug 2022 14:25:48 +0000 +Description: Add missing imports that break the build in go 1.18. +Forwarded: No + +--- a/internal/mtail/examples_integration_unix_test.go ++++ b/internal/mtail/examples_integration_unix_test.go +@@ -6,6 +6,27 @@ + + package mtail_test + ++import ( ++ "context" ++ "errors" ++ "fmt" ++ "io" ++ "net" ++ "os" ++ "path/filepath" ++ "sync" ++ "testing" ++ "time" ++ ++ "github.com/golang/glog" ++ "github.com/google/mtail/internal/metrics" ++ "github.com/google/mtail/internal/metrics/datum" ++ "github.com/google/mtail/internal/mtail" ++ "github.com/google/mtail/internal/testutil" ++ "github.com/google/mtail/internal/waker" ++ "golang.org/x/sys/unix" ++) ++ + // TestFilePipeStreamComparison is a unix-specific test since unix.Mkfifo is not defined on Windows. + // Two mtails both alike in dignity. + func TestFilePipeStreamComparison(t *testing.T) { +--- a/internal/mtail/read_pipe_integration_unix_test.go ++++ b/internal/mtail/read_pipe_integration_unix_test.go +@@ -7,10 +7,12 @@ + package mtail_test + + import ( ++ "net" + "os" + "path/filepath" + "syscall" + "testing" ++ "time" + + "github.com/google/mtail/internal/mtail" + "github.com/google/mtail/internal/testutil" diff -Nru mtail-3.0.0~rc49/debian/patches/series mtail-3.0.0~rc50/debian/patches/series --- mtail-3.0.0~rc49/debian/patches/series 2022-05-25 18:16:47.000000000 +0000 +++ mtail-3.0.0~rc50/debian/patches/series 2022-08-23 14:42:45.000000000 +0000 @@ -1 +1,2 @@ 01-remove-jaeger.patch +02-Go1.18_imports.patch diff -Nru mtail-3.0.0~rc49/docs/Language.md mtail-3.0.0~rc50/docs/Language.md --- mtail-3.0.0~rc49/docs/Language.md 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/docs/Language.md 2022-07-18 06:49:29.000000000 +0000 @@ -362,7 +362,7 @@ #### Decorated actions Decorated actions are an inversion of nested actions. They allow the program to -define repetetive functions that perform the same extraction across many +define repetitive functions that perform the same extraction across many different actions. For example, most log file formats start with a timestamp prefix. To reduce @@ -401,7 +401,7 @@ #### Types -`mtail` metrics have a *kind* and a *type*. The *kind* effects how the metric is recorded, and the *type* describes the data being recorded. +`mtail` metrics have a *kind* and a *type*. The *kind* affects how the metric is recorded, and the *type* describes the data being recorded. Ordinarily `mtail` doesn't treat kinds specially, except when they are being exported. @@ -456,7 +456,7 @@ * `int(x)`, a function of one argument performs type conversion to integer. If `x` is a type that can be converted to integer, it does so. If the type of `x` cannot be converted to an integer, a compile error is triggered. If the - valye of `x` cannot be converted to an integer, then a runtime error is + value of `x` cannot be converted to an integer, then a runtime error is triggered. * `float(x)`, a function of one argument that performs type conversion to floating point numbers. The same rules apply as for `int()` above. @@ -535,7 +535,7 @@ generate faster bytecode if it knows at compile-time the types to expect. If `mtail` can't infer the value types, they default to `String` and `mtail` will attempt a value conversion at runtime if necessary. Runtime conversion errors -will be emitted to the standard INFO log, and terminate program exection for +will be emitted to the standard INFO log, and terminate program execution for that log line. #### Variable Storage Management diff -Nru mtail-3.0.0~rc49/.github/workflows/automerge.yml mtail-3.0.0~rc50/.github/workflows/automerge.yml --- mtail-3.0.0~rc49/.github/workflows/automerge.yml 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/.github/workflows/automerge.yml 2022-07-18 06:49:29.000000000 +0000 @@ -39,7 +39,7 @@ # wait-on-check requires only checks read checks: read steps: - - uses: lewagon/wait-on-check-action@v1.1.1 + - uses: lewagon/wait-on-check-action@v1.1.2 with: ref: ${{ github.event.pull_request.head.sha }} check-regexp: "test.*" diff -Nru mtail-3.0.0~rc49/.github/workflows/auto-review.yml mtail-3.0.0~rc50/.github/workflows/auto-review.yml --- mtail-3.0.0~rc49/.github/workflows/auto-review.yml 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/.github/workflows/auto-review.yml 2022-07-18 06:49:29.000000000 +0000 @@ -29,7 +29,7 @@ # create review pull-requests: write steps: - - uses: lewagon/wait-on-check-action@v1.1.1 + - uses: lewagon/wait-on-check-action@v1.1.2 with: ref: ${{ github.event.pull_request.head.sha }} repo-token: ${{ github.token }} diff -Nru mtail-3.0.0~rc49/.github/workflows/ci.yml mtail-3.0.0~rc50/.github/workflows/ci.yml --- mtail-3.0.0~rc49/.github/workflows/ci.yml 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/.github/workflows/ci.yml 2022-07-18 06:49:29.000000000 +0000 @@ -48,8 +48,13 @@ mkdir -p test-results # Don't use GITHUB_SHA as we need the head of the branch, not the # secret merge commit of the PR itself. https://help.github.com/en/actions/automating-your-workflow-with-github-actions/events-that-trigger-workflows#pull-request-event-pull_request - echo ${{ github.event.pull_request.head.sha }} > test-results/sha-number + if [[ ${{ github.event_name }} == 'pull_request' ]]; then + echo ${{ github.event.pull_request.head.sha }} > test-results/sha-number + else + echo ${{ github.sha }} > test-result/sha-number + fi make --debug junit-regtest TESTCOVERPROFILE=coverprofile + shell: bash - uses: codecov/codecov-action@v3 if: always() with: diff -Nru mtail-3.0.0~rc49/.github/workflows/release.yml mtail-3.0.0~rc50/.github/workflows/release.yml --- mtail-3.0.0~rc49/.github/workflows/release.yml 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/.github/workflows/release.yml 2022-07-18 06:49:29.000000000 +0000 @@ -20,7 +20,7 @@ - uses: actions/setup-go@v3 with: go-version: '^1.x' - - uses: goreleaser/goreleaser-action@v2 + - uses: goreleaser/goreleaser-action@v3 with: version: latest args: release --rm-dist diff -Nru mtail-3.0.0~rc49/go.mod mtail-3.0.0~rc50/go.mod --- mtail-3.0.0~rc49/go.mod 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/go.mod 2022-07-18 06:49:29.000000000 +0000 @@ -1,6 +1,6 @@ module github.com/google/mtail -go 1.16 +go 1.17 require ( contrib.go.opencensus.io/exporter/jaeger v0.2.1 @@ -9,7 +9,22 @@ github.com/google/go-cmp v0.5.8 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.2 - github.com/prometheus/common v0.34.0 + github.com/prometheus/common v0.37.0 go.opencensus.io v0.23.0 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 ) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect + golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect + google.golang.org/api v0.30.0 // indirect + google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect + google.golang.org/grpc v1.33.2 // indirect + google.golang.org/protobuf v1.26.0 // indirect +) diff -Nru mtail-3.0.0~rc49/go.sum mtail-3.0.0~rc50/go.sum --- mtail-3.0.0~rc49/go.sum 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/go.sum 2022-07-18 06:49:29.000000000 +0000 @@ -179,8 +179,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= -github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= diff -Nru mtail-3.0.0~rc49/internal/exporter/export.go mtail-3.0.0~rc50/internal/exporter/export.go --- mtail-3.0.0~rc49/internal/exporter/export.go 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/internal/exporter/export.go 2022-07-18 06:49:29.000000000 +0000 @@ -13,6 +13,7 @@ "io" "net" "os" + "sort" "strings" "sync" "time" @@ -140,10 +141,15 @@ func formatLabels(name string, m map[string]string, ksep, sep, rep string) string { r := name if len(m) > 0 { + var keys []string + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) var s []string - for k, v := range m { + for _, k := range keys { k1 := strings.ReplaceAll(strings.ReplaceAll(k, ksep, rep), sep, rep) - v1 := strings.ReplaceAll(strings.ReplaceAll(v, ksep, rep), sep, rep) + v1 := strings.ReplaceAll(strings.ReplaceAll(m[k], ksep, rep), sep, rep) s = append(s, fmt.Sprintf("%s%s%s", k1, ksep, v1)) } return r + sep + strings.Join(s, sep) diff -Nru mtail-3.0.0~rc49/internal/exporter/export_test.go mtail-3.0.0~rc50/internal/exporter/export_test.go --- mtail-3.0.0~rc49/internal/exporter/export_test.go 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/internal/exporter/export_test.go 2022-07-18 06:49:29.000000000 +0000 @@ -195,6 +195,15 @@ t.Errorf("String didn't match:\n\texpected: %v\n\treceived: %v", expected, r) } + multiLabelMetric := metrics.NewMetric("bar", "prog", metrics.Gauge, metrics.Int, "c", "a", "b") + d, _ = multiLabelMetric.GetDatum("x", "z", "y") + datum.SetInt(d, 37, ts) + r = FakeSocketWrite(metricToStatsd, multiLabelMetric) + expected = []string{"prog.bar.a.z.b.y.c.x:37|g"} + if !reflect.DeepEqual(expected, r) { + t.Errorf("String didn't match:\n\texpected: %v\n\treceived: %v", expected, r) + } + timingMetric := metrics.NewMetric("foo", "prog", metrics.Timer, metrics.Int) d, _ = timingMetric.GetDatum() datum.SetInt(d, 37, ts) diff -Nru mtail-3.0.0~rc49/internal/metrics/metric.go mtail-3.0.0~rc50/internal/metrics/metric.go --- mtail-3.0.0~rc49/internal/metrics/metric.go 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/internal/metrics/metric.go 2022-07-18 06:49:29.000000000 +0000 @@ -204,7 +204,8 @@ k := buildLabelValueKey(labelvalues) olv, ok := m.labelValuesMap[k] if ok { - for i, lv := range m.LabelValues { + for i := 0; i < len(m.LabelValues); i++ { + lv := m.LabelValues[i] if lv == olv { // remove from the slice m.LabelValues = append(m.LabelValues[:i], m.LabelValues[i+1:]...) diff -Nru mtail-3.0.0~rc49/internal/metrics/store.go mtail-3.0.0~rc50/internal/metrics/store.go --- mtail-3.0.0~rc49/internal/metrics/store.go 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/internal/metrics/store.go 2022-07-18 06:49:29.000000000 +0000 @@ -162,7 +162,8 @@ m.RemoveOldestDatum() } } - for _, lv := range m.LabelValues { + for i := 0; i < len(m.LabelValues); i++ { + lv := m.LabelValues[i] if lv.Expiry <= 0 { continue } @@ -171,6 +172,7 @@ if err != nil { return err } + i-- } } return nil diff -Nru mtail-3.0.0~rc49/internal/metrics/store_test.go mtail-3.0.0~rc50/internal/metrics/store_test.go --- mtail-3.0.0~rc49/internal/metrics/store_test.go 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/internal/metrics/store_test.go 2022-07-18 06:49:29.000000000 +0000 @@ -4,6 +4,7 @@ package metrics import ( + "strconv" "testing" "time" @@ -149,5 +150,44 @@ if x := m.FindLabelValueOrNil([]string{"a"}); x != nil { t.Errorf("found label a which is unexpected: %#v", x) } +} + +func TestExpireManyMetrics(t *testing.T) { + s := NewStore() + m := NewMetric("foo", "prog", Counter, Int, "id") + testutil.FatalIfErr(t, s.Add(m)) + d, err := m.GetDatum("0") + if err != nil { + t.Error(err) + } + datum.SetInt(d, 1, time.Now().Add(-time.Hour)) + lv := m.FindLabelValueOrNil([]string{"0"}) + if lv == nil { + t.Fatal("couldn't find lv") + } + for i := 1; i < 10; i++ { + d, err := m.GetDatum(strconv.Itoa(i)) + if err != nil { + t.Error(err) + } + datum.SetInt(d, 1, time.Now().Add(-time.Hour)) + lv = m.FindLabelValueOrNil([]string{strconv.Itoa(i)}) + if lv == nil { + t.Fatal("couldn't find lv") + } + lv.Expiry = time.Minute + } + + testutil.FatalIfErr(t, s.Gc()) + lv = m.FindLabelValueOrNil([]string{"8"}) + if lv != nil { + t.Errorf("lv not expired: %#v", lv) + t.Logf("Store: %#v", s) + } + lv = m.FindLabelValueOrNil([]string{"0"}) + if lv == nil { + t.Errorf("lv expired") + t.Logf("Store: %#v", s) + } } diff -Nru mtail-3.0.0~rc49/internal/tailer/logstream/decode.go mtail-3.0.0~rc50/internal/tailer/logstream/decode.go --- mtail-3.0.0~rc49/internal/tailer/logstream/decode.go 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/internal/tailer/logstream/decode.go 2022-07-18 06:49:29.000000000 +0000 @@ -17,13 +17,17 @@ var logLines = expvar.NewMap("log_lines_total") // decodeAndSend transforms the byte array `b` into unicode in `partial`, sending to the llp as each newline is decoded. -func decodeAndSend(ctx context.Context, lines chan<- *logline.LogLine, pathname string, n int, b []byte, partial *bytes.Buffer) { +func decodeAndSend(ctx context.Context, lines chan<- *logline.LogLine, pathname string, n int, b []byte, partial *bytes.Buffer) int { var ( r rune width int + count int ) for i := 0; i < len(b) && i < n; i += width { r, width = utf8.DecodeRune(b[i:]) + if r == utf8.RuneError { + return count + } // Most file-based log sources will end with \n on Unixlike systems. // On Windows they appear to be both \r\n. syslog disallows \r (and \t // and others) and writes them escaped, per syslog(7). [RFC @@ -39,7 +43,9 @@ default: partial.WriteRune(r) } + count += width } + return count } func sendLine(ctx context.Context, pathname string, partial *bytes.Buffer, lines chan<- *logline.LogLine) { diff -Nru mtail-3.0.0~rc49/internal/tailer/logstream/filestream.go mtail-3.0.0~rc50/internal/tailer/logstream/filestream.go --- mtail-3.0.0~rc49/internal/tailer/logstream/filestream.go 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/internal/tailer/logstream/filestream.go 2022-07-18 06:49:29.000000000 +0000 @@ -82,6 +82,7 @@ glog.V(2).Infof("%v: seeked to end", fd) } b := make([]byte, defaultReadBufferSize) + var lastBytes []byte partial := bytes.NewBufferString("") started := make(chan struct{}) var total int @@ -106,7 +107,14 @@ if count > 0 { total += count glog.V(2).Infof("%v: decode and send", fd) - decodeAndSend(ctx, fs.lines, fs.pathname, count, b[:count], partial) + needSend := lastBytes + needSend = append(needSend, b[:count]...) + sendCount := decodeAndSend(ctx, fs.lines, fs.pathname, len(needSend), needSend, partial) + if sendCount < len(needSend) { + lastBytes = append([]byte{}, needSend[sendCount:]...) + } else { + lastBytes = []byte{} + } fs.mu.Lock() fs.lastReadTime = time.Now() fs.mu.Unlock() diff -Nru mtail-3.0.0~rc49/internal/tailer/logstream/filestream_test.go mtail-3.0.0~rc50/internal/tailer/logstream/filestream_test.go --- mtail-3.0.0~rc49/internal/tailer/logstream/filestream_test.go 2022-05-15 06:21:36.000000000 +0000 +++ mtail-3.0.0~rc50/internal/tailer/logstream/filestream_test.go 2022-07-18 06:49:29.000000000 +0000 @@ -50,6 +50,47 @@ wg.Wait() } +func TestFileStreamReadNonSingleByteEnd(t *testing.T) { + var wg sync.WaitGroup + + tmpDir := testutil.TestTempDir(t) + + name := filepath.Join(tmpDir, "log") + f := testutil.TestOpenFile(t, name) + defer f.Close() + + lines := make(chan *logline.LogLine, 1) + ctx, cancel := context.WithCancel(context.Background()) + waker, awaken := waker.NewTest(ctx, 1) + fs, err := logstream.New(ctx, &wg, waker, name, lines, true) + testutil.FatalIfErr(t, err) + awaken(1) + + s := "a" + for i := 0; i < 4094; i++ { + s += "a" + } + + s += "中" + testutil.WriteString(t, f, s+"\n") + awaken(1) + + fs.Stop() + wg.Wait() + close(lines) + received := testutil.LinesReceived(lines) + expected := []*logline.LogLine{ + {context.TODO(), name, s}, + } + testutil.ExpectNoDiff(t, expected, received, testutil.IgnoreFields(logline.LogLine{}, "Context")) + + if !fs.IsComplete() { + t.Errorf("expecting filestream to be complete because stopped") + } + cancel() + wg.Wait() +} + func TestFileStreamTruncation(t *testing.T) { var wg sync.WaitGroup