Update dependencies

This commit is contained in:
Manuel de Brito Fontes 2017-10-06 17:33:32 -03:00
parent bf5616c65b
commit d6d374b28d
13962 changed files with 48226 additions and 3618880 deletions

View file

@ -22,6 +22,8 @@ type (
Memresident uint64
Memvirtual uint64
OldestStartTime time.Time
OpenFDs uint64
WorstFDratio float64
}
)
@ -84,7 +86,7 @@ func (g *Grouper) Update(iter ProcIter) (int, error) {
// Step 1: track any new proc that should be tracked based on its name and cmdline.
untracked := make(map[ProcId]ProcIdInfo)
for _, idinfo := range newProcs {
wanted, gname := g.namer.MatchAndName(common.NameAndCmdline{idinfo.Name, idinfo.Cmdline})
wanted, gname := g.namer.MatchAndName(common.NameAndCmdline{Name: idinfo.Name, Cmdline: idinfo.Cmdline})
if !wanted {
untracked[idinfo.ProcId] = idinfo
continue
@ -121,14 +123,19 @@ func (g *Grouper) groups() GroupCountMap {
}
cur := gcounts[tinfo.GroupName]
cur.Procs++
_, counts, mem, start := tinfo.GetStats()
cur.Memresident += mem.Resident
cur.Memvirtual += mem.Virtual
cur.Counts.Cpu += counts.Cpu
cur.Counts.ReadBytes += counts.ReadBytes
cur.Counts.WriteBytes += counts.WriteBytes
if cur.OldestStartTime == zeroTime || start.Before(cur.OldestStartTime) {
cur.OldestStartTime = start
tstats := tinfo.GetStats()
cur.Memresident += tstats.Memory.Resident
cur.Memvirtual += tstats.Memory.Virtual
cur.OpenFDs += tstats.Filedesc.Open
openratio := float64(tstats.Filedesc.Open) / float64(tstats.Filedesc.Limit)
if cur.WorstFDratio < openratio {
cur.WorstFDratio = openratio
}
cur.Counts.Cpu += tstats.latest.Cpu
cur.Counts.ReadBytes += tstats.latest.ReadBytes
cur.Counts.WriteBytes += tstats.latest.WriteBytes
if cur.OldestStartTime == zeroTime || tstats.start.Before(cur.OldestStartTime) {
cur.OldestStartTime = tstats.start
}
gcounts[tinfo.GroupName] = cur
}

View file

@ -36,8 +36,8 @@ func (s MySuite) TestGrouperBasic(c *C) {
}
}
gr := NewGrouper(false, newNamer("g1", "g2"))
p1 := newProc(1, "g1", ProcMetrics{1, 2, 3, 4, 5})
p2 := newProc(2, "g2", ProcMetrics{2, 3, 4, 5, 6})
p1 := newProc(1, "g1", ProcMetrics{1, 2, 3, 4, 5, 4, 400})
p2 := newProc(2, "g2", ProcMetrics{2, 3, 4, 5, 6, 40, 400})
p3 := newProc(3, "g3", ProcMetrics{})
_, err := gr.Update(procInfoIter(p1, p2, p3))
@ -45,52 +45,53 @@ func (s MySuite) TestGrouperBasic(c *C) {
got1 := gr.groups()
want1 := GroupCountMap{
"g1": GroupCounts{Counts{0, 0, 0}, 1, 4, 5, time.Time{}},
"g2": GroupCounts{Counts{0, 0, 0}, 1, 5, 6, time.Time{}},
"g1": GroupCounts{Counts{0, 0, 0}, 1, 4, 5, time.Time{}, 4, 0.01},
"g2": GroupCounts{Counts{0, 0, 0}, 1, 5, 6, time.Time{}, 40, 0.1},
}
c.Check(got1, DeepEquals, want1)
c.Check(got1, DeepEquals, want1, Commentf("diff %s", pretty.Compare(got1, want1)))
// Now increment counts and memory and make sure group counts updated.
p1.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6}
p2.ProcMetrics = ProcMetrics{4, 5, 6, 7, 8}
p1.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6, 4, 400}
p2.ProcMetrics = ProcMetrics{4, 5, 6, 7, 8, 40, 400}
_, err = gr.Update(procInfoIter(p1, p2, p3))
c.Assert(err, IsNil)
got2 := gr.groups()
want2 := GroupCountMap{
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}},
"g2": GroupCounts{Counts{2, 2, 2}, 1, 7, 8, time.Time{}},
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}, 4, 0.01},
"g2": GroupCounts{Counts{2, 2, 2}, 1, 7, 8, time.Time{}, 40, 0.1},
}
c.Check(got2, DeepEquals, want2)
c.Check(got2, DeepEquals, want2, Commentf("diff %s", pretty.Compare(got2, want2)))
// Now add a new proc and update p2's metrics. The
// counts for p4 won't be factored into the total yet
// because we only add to counts starting with the
// second time we see a proc. Memory is affected though.
p4 := newProc(4, "g2", ProcMetrics{1, 1, 1, 1, 1})
p2.ProcMetrics = ProcMetrics{5, 6, 7, 8, 9}
// second time we see a proc. Memory and FDs are affected
// though.
p4 := newProc(4, "g2", ProcMetrics{1, 1, 1, 1, 1, 80, 400})
p2.ProcMetrics = ProcMetrics{5, 6, 7, 8, 9, 40, 400}
_, err = gr.Update(procInfoIter(p1, p2, p3, p4))
c.Assert(err, IsNil)
got3 := gr.groups()
want3 := GroupCountMap{
"g1": GroupCounts{Counts{0, 0, 0}, 1, 5, 6, time.Time{}},
"g2": GroupCounts{Counts{1, 1, 1}, 2, 9, 10, time.Time{}},
"g1": GroupCounts{Counts{0, 0, 0}, 1, 5, 6, time.Time{}, 4, 0.01},
"g2": GroupCounts{Counts{1, 1, 1}, 2, 9, 10, time.Time{}, 120, 0.2},
}
c.Check(got3, DeepEquals, want3, Commentf("diff %s", pretty.Compare(got3, want3)))
p4.ProcMetrics = ProcMetrics{2, 2, 2, 2, 2}
p2.ProcMetrics = ProcMetrics{6, 7, 8, 8, 9}
p4.ProcMetrics = ProcMetrics{2, 2, 2, 2, 2, 100, 400}
p2.ProcMetrics = ProcMetrics{6, 7, 8, 8, 9, 40, 400}
_, err = gr.Update(procInfoIter(p1, p2, p3, p4))
c.Assert(err, IsNil)
got4 := gr.groups()
want4 := GroupCountMap{
"g1": GroupCounts{Counts{0, 0, 0}, 1, 5, 6, time.Time{}},
"g2": GroupCounts{Counts{2, 2, 2}, 2, 10, 11, time.Time{}},
"g1": GroupCounts{Counts{0, 0, 0}, 1, 5, 6, time.Time{}, 4, 0.01},
"g2": GroupCounts{Counts{2, 2, 2}, 2, 10, 11, time.Time{}, 140, 0.25},
}
c.Check(got4, DeepEquals, want4, Commentf("diff %s", pretty.Compare(got4, want4)))
@ -118,8 +119,8 @@ func (s MySuite) TestGrouperParents(c *C) {
got1 := gr.groups()
want1 := GroupCountMap{
"g1": GroupCounts{Counts{}, 1, 0, 0, time.Time{}},
"g2": GroupCounts{Counts{}, 1, 0, 0, time.Time{}},
"g1": GroupCounts{Counts{}, 1, 0, 0, time.Time{}, 0, 0},
"g2": GroupCounts{Counts{}, 1, 0, 0, time.Time{}, 0, 0},
}
c.Check(got1, DeepEquals, want1, Commentf("diff %s", pretty.Compare(got1, want1)))
@ -135,8 +136,8 @@ func (s MySuite) TestGrouperParents(c *C) {
got2 := gr.groups()
want2 := GroupCountMap{
"g1": GroupCounts{Counts{}, 2, 0, 0, time.Time{}},
"g2": GroupCounts{Counts{}, 2, 0, 0, time.Time{}},
"g1": GroupCounts{Counts{}, 2, 0, 0, time.Time{}, 0, 0},
"g2": GroupCounts{Counts{}, 2, 0, 0, time.Time{}, 0, 0},
}
c.Check(got2, DeepEquals, want2, Commentf("diff %s", pretty.Compare(got2, want2)))
@ -151,8 +152,8 @@ func (s MySuite) TestGrouperParents(c *C) {
got3 := gr.groups()
want3 := GroupCountMap{
"g1": GroupCounts{Counts{}, 1, 0, 0, time.Time{}},
"g2": GroupCounts{Counts{}, 5, 0, 0, time.Time{}},
"g1": GroupCounts{Counts{}, 1, 0, 0, time.Time{}, 0, 0},
"g2": GroupCounts{Counts{}, 5, 0, 0, time.Time{}, 0, 0},
}
c.Check(got3, DeepEquals, want3, Commentf("diff %s", pretty.Compare(got3, want3)))
}
@ -171,23 +172,23 @@ func (s MySuite) TestGrouperGroup(c *C) {
gr := NewGrouper(false, newNamer("g1"))
// First call should return zero CPU/IO.
p1 := newProc(1, "g1", ProcMetrics{1, 2, 3, 4, 5})
p1 := newProc(1, "g1", ProcMetrics{1, 2, 3, 4, 5, 8, 400})
_, err := gr.Update(procInfoIter(p1))
c.Assert(err, IsNil)
got1 := gr.Groups()
want1 := GroupCountMap{
"g1": GroupCounts{Counts{0, 0, 0}, 1, 4, 5, time.Time{}},
"g1": GroupCounts{Counts{0, 0, 0}, 1, 4, 5, time.Time{}, 8, 0.02},
}
c.Check(got1, DeepEquals, want1)
// Second call should return the delta CPU/IO from first observance,
// as well as latest memory/proccount.
p1.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6}
p1.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6, 12, 400}
_, err = gr.Update(procInfoIter(p1))
c.Assert(err, IsNil)
got2 := gr.Groups()
want2 := GroupCountMap{
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}},
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}, 12, 0.03},
}
c.Check(got2, DeepEquals, want2)
@ -196,7 +197,7 @@ func (s MySuite) TestGrouperGroup(c *C) {
c.Assert(err, IsNil)
got3 := gr.Groups()
want3 := GroupCountMap{
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}},
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}, 12, 0.03},
}
c.Check(got3, DeepEquals, want3, Commentf("diff %s", pretty.Compare(got3, want3)))
}
@ -213,54 +214,54 @@ func (s MySuite) TestGrouperNonDecreasing(c *C) {
}
}
gr := NewGrouper(false, newNamer("g1", "g2"))
p1 := newProc(1, "g1", ProcMetrics{1, 2, 3, 4, 5})
p2 := newProc(2, "g2", ProcMetrics{2, 3, 4, 5, 6})
p1 := newProc(1, "g1", ProcMetrics{1, 2, 3, 4, 5, 4, 400})
p2 := newProc(2, "g2", ProcMetrics{2, 3, 4, 5, 6, 40, 400})
_, err := gr.Update(procInfoIter(p1, p2))
c.Assert(err, IsNil)
got1 := gr.Groups()
want1 := GroupCountMap{
"g1": GroupCounts{Counts{0, 0, 0}, 1, 4, 5, time.Time{}},
"g2": GroupCounts{Counts{0, 0, 0}, 1, 5, 6, time.Time{}},
"g1": GroupCounts{Counts{0, 0, 0}, 1, 4, 5, time.Time{}, 4, 0.01},
"g2": GroupCounts{Counts{0, 0, 0}, 1, 5, 6, time.Time{}, 40, 0.1},
}
c.Check(got1, DeepEquals, want1)
c.Check(got1, DeepEquals, want1, Commentf("diff %s", pretty.Compare(got1, want1)))
// Now add a new proc p3 to g2, and increment p1/p2's metrics.
p1.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6}
p2.ProcMetrics = ProcMetrics{4, 5, 6, 7, 8}
p3 := newProc(3, "g2", ProcMetrics{1, 1, 1, 1, 1})
p1.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6, 8, 400}
p2.ProcMetrics = ProcMetrics{4, 5, 6, 7, 8, 80, 400}
p3 := newProc(3, "g2", ProcMetrics{1, 1, 1, 1, 1, 8, 400})
_, err = gr.Update(procInfoIter(p1, p2, p3))
c.Assert(err, IsNil)
got2 := gr.Groups()
want2 := GroupCountMap{
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}},
"g2": GroupCounts{Counts{2, 2, 2}, 2, 8, 9, time.Time{}},
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}, 8, 0.02},
"g2": GroupCounts{Counts{2, 2, 2}, 2, 8, 9, time.Time{}, 88, 0.2},
}
c.Check(got2, DeepEquals, want2)
c.Check(got2, DeepEquals, want2, Commentf("diff %s", pretty.Compare(got2, want2)))
// Now update p3's metrics and kill p2.
p3.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6}
p3.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6, 8, 400}
_, err = gr.Update(procInfoIter(p1, p3))
c.Assert(err, IsNil)
got3 := gr.Groups()
want3 := GroupCountMap{
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}},
"g2": GroupCounts{Counts{3, 4, 5}, 1, 5, 6, time.Time{}},
"g1": GroupCounts{Counts{1, 1, 1}, 1, 5, 6, time.Time{}, 8, 0.02},
"g2": GroupCounts{Counts{3, 4, 5}, 1, 5, 6, time.Time{}, 8, 0.02},
}
c.Check(got3, DeepEquals, want3, Commentf("diff %s", pretty.Compare(got3, want3)))
// Now update p3's metrics and kill p1.
p3.ProcMetrics = ProcMetrics{4, 4, 4, 2, 1}
p3.ProcMetrics = ProcMetrics{4, 4, 4, 2, 1, 4, 400}
_, err = gr.Update(procInfoIter(p3))
c.Assert(err, IsNil)
got4 := gr.Groups()
want4 := GroupCountMap{
"g1": GroupCounts{Counts{1, 1, 1}, 0, 0, 0, time.Time{}},
"g2": GroupCounts{Counts{5, 5, 5}, 1, 2, 1, time.Time{}},
"g1": GroupCounts{Counts{1, 1, 1}, 0, 0, 0, time.Time{}, 0, 0},
"g2": GroupCounts{Counts{5, 5, 5}, 1, 2, 1, time.Time{}, 4, 0.01},
}
c.Check(got4, DeepEquals, want4, Commentf("diff %s\n%s", pretty.Compare(got4, want4), pretty.Sprint(gr)))

View file

@ -2,8 +2,9 @@ package proc
import (
"fmt"
"github.com/prometheus/procfs"
"time"
"github.com/prometheus/procfs"
)
func newProcIdStatic(pid, ppid int, startTime uint64, name string, cmdline []string) ProcIdStatic {
@ -35,6 +36,8 @@ type (
WriteBytes uint64
ResidentBytes uint64
VirtualBytes uint64
OpenFDs uint64
MaxFDs uint64
}
ProcIdStatic struct {
@ -79,7 +82,7 @@ type (
stat *procfs.ProcStat
cmdline []string
io *procfs.ProcIO
bootTime int64
bootTime uint64
}
procs interface {
@ -89,7 +92,7 @@ type (
procfsprocs struct {
Procs []procfs.Proc
bootTime int64
bootTime uint64
}
// ProcIter is an iterator over a sequence of procs.
@ -230,7 +233,7 @@ func (p proc) GetStatic() (ProcStatic, error) {
if err != nil {
return ProcStatic{}, err
}
startTime := time.Unix(p.bootTime, 0)
startTime := time.Unix(int64(p.bootTime), 0)
startTime = startTime.Add(time.Second / userHZ * time.Duration(stat.Starttime))
return ProcStatic{
Name: stat.Comm,
@ -249,18 +252,28 @@ func (p proc) GetMetrics() (ProcMetrics, error) {
if err != nil {
return ProcMetrics{}, err
}
numfds, err := p.Proc.FileDescriptorsLen()
if err != nil {
return ProcMetrics{}, err
}
limits, err := p.NewLimits()
if err != nil {
return ProcMetrics{}, err
}
return ProcMetrics{
CpuTime: stat.CPUTime(),
ReadBytes: io.ReadBytes,
WriteBytes: io.WriteBytes,
ResidentBytes: uint64(stat.ResidentMemory()),
VirtualBytes: uint64(stat.VirtualMemory()),
OpenFDs: uint64(numfds),
MaxFDs: uint64(limits.OpenFiles),
}, nil
}
type FS struct {
procfs.FS
BootTime int64
BootTime uint64
}
// See https://github.com/prometheus/procfs/blob/master/proc_stat.go for details on userHZ.

View file

@ -18,6 +18,11 @@ type (
Virtual uint64
}
Filedesc struct {
Open uint64
Limit uint64
}
// Tracker tracks processes and records metrics.
Tracker struct {
// Tracked holds the processes are being monitored. Processes
@ -43,6 +48,13 @@ type (
// GroupName is an optional tag for this proc.
GroupName string
}
trackedStats struct {
aggregate, latest Counts
Memory
Filedesc
start time.Time
}
)
func (tp *TrackedProc) GetName() string {
@ -53,8 +65,16 @@ func (tp *TrackedProc) GetCmdLine() []string {
return tp.info.Cmdline
}
func (tp *TrackedProc) GetStats() (aggregate, latest Counts, mem Memory, start time.Time) {
return tp.accum, tp.lastaccum, Memory{Resident: tp.info.ResidentBytes, Virtual: tp.info.VirtualBytes}, tp.info.StartTime
func (tp *TrackedProc) GetStats() trackedStats {
mem := Memory{Resident: tp.info.ResidentBytes, Virtual: tp.info.VirtualBytes}
fd := Filedesc{Open: tp.info.OpenFDs, Limit: tp.info.MaxFDs}
return trackedStats{
aggregate: tp.accum,
latest: tp.lastaccum,
Memory: mem,
Filedesc: fd,
start: tp.info.StartTime,
}
}
func NewTracker() *Tracker {

View file

@ -60,7 +60,7 @@ func (s MySuite) TestTrackerCounts(c *C) {
tr := NewTracker()
// Test that p1 is seen as new
p1 := newProc(1, 1, "p1", ProcMetrics{1, 2, 3, 4, 5})
p1 := newProc(1, 1, "p1", ProcMetrics{1, 2, 3, 4, 5, 6, 4096})
want1 := []ProcIdInfo{p1}
got1, _, err := tr.Update(procInfoIter(p1))
c.Assert(err, IsNil)
@ -73,7 +73,7 @@ func (s MySuite) TestTrackerCounts(c *C) {
c.Check(got2, DeepEquals, []ProcIdInfo(nil))
// Now update p1's metrics
p1.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6}
p1.ProcMetrics = ProcMetrics{2, 3, 4, 5, 6, 7, 4096}
got3, _, err := tr.Update(procInfoIter(p1))
c.Assert(err, IsNil)
c.Check(got3, DeepEquals, []ProcIdInfo(nil))
@ -83,7 +83,7 @@ func (s MySuite) TestTrackerCounts(c *C) {
c.Check(tr.Tracked[p1.ProcId].info, DeepEquals, ProcInfo{p1.ProcStatic, p1.ProcMetrics})
// Now update p1's metrics again
p1.ProcMetrics = ProcMetrics{4, 6, 8, 9, 10}
p1.ProcMetrics = ProcMetrics{4, 6, 8, 9, 10, 11, 4096}
got4, _, err := tr.Update(procInfoIter(p1))
c.Assert(err, IsNil)
c.Check(got4, DeepEquals, []ProcIdInfo(nil))