Update godeps

vendor/github.com/ncabatoff/process-exporter/proc/grouper.go

@@ -0,0 +1,166 @@
+package proc
+
+import (
+	common "github.com/ncabatoff/process-exporter"
+	"time"
+)
+
+type (
+	Grouper struct {
+		namer         common.MatchNamer
+		trackChildren bool
+		// track how much was seen last time so we can report the delta
+		GroupStats map[string]Counts
+		tracker    *Tracker
+	}
+
+	GroupCountMap map[string]GroupCounts
+
+	GroupCounts struct {
+		Counts
+		Procs           int
+		Memresident     uint64
+		Memvirtual      uint64
+		OldestStartTime time.Time
+	}
+)
+
+func NewGrouper(trackChildren bool, namer common.MatchNamer) *Grouper {
+	g := Grouper{
+		trackChildren: trackChildren,
+		namer:         namer,
+		GroupStats:    make(map[string]Counts),
+		tracker:       NewTracker(),
+	}
+	return &g
+}
+
+func (g *Grouper) checkAncestry(idinfo ProcIdInfo, newprocs map[ProcId]ProcIdInfo) string {
+	ppid := idinfo.ParentPid
+	pProcId := g.tracker.ProcIds[ppid]
+	if pProcId.Pid < 1 {
+		// Reached root of process tree without finding a tracked parent.
+		g.tracker.Ignore(idinfo.ProcId)
+		return ""
+	}
+
+	// Is the parent already known to the tracker?
+	if ptproc, ok := g.tracker.Tracked[pProcId]; ok {
+		if ptproc != nil {
+			// We've found a tracked parent.
+			g.tracker.Track(ptproc.GroupName, idinfo)
+			return ptproc.GroupName
+		} else {
+			// We've found an untracked parent.
+			g.tracker.Ignore(idinfo.ProcId)
+			return ""
+		}
+	}
+
+	// Is the parent another new process?
+	if pinfoid, ok := newprocs[pProcId]; ok {
+		if name := g.checkAncestry(pinfoid, newprocs); name != "" {
+			// We've found a tracked parent, which implies this entire lineage should be tracked.
+			g.tracker.Track(name, idinfo)
+			return name
+		}
+	}
+
+	// Parent is dead, i.e. we never saw it, or there's no tracked proc in our ancestry.
+	g.tracker.Ignore(idinfo.ProcId)
+	return ""
+
+}
+
+// Update tracks any new procs that should be according to policy, and updates
+// the metrics for already tracked procs.  Permission errors are returned as a
+// count, and will not affect the error return value.
+func (g *Grouper) Update(iter ProcIter) (int, error) {
+	newProcs, permErrs, err := g.tracker.Update(iter)
+	if err != nil {
+		return permErrs, err
+	}
+
+	// Step 1: track any new proc that should be tracked based on its name and cmdline.
+	untracked := make(map[ProcId]ProcIdInfo)
+	for _, idinfo := range newProcs {
+		wanted, gname := g.namer.MatchAndName(common.NameAndCmdline{idinfo.Name, idinfo.Cmdline})
+		if !wanted {
+			untracked[idinfo.ProcId] = idinfo
+			continue
+		}
+
+		g.tracker.Track(gname, idinfo)
+	}
+
+	// Step 2: track any untracked new proc that should be tracked because its parent is tracked.
+	if !g.trackChildren {
+		return permErrs, nil
+	}
+
+	for _, idinfo := range untracked {
+		if _, ok := g.tracker.Tracked[idinfo.ProcId]; ok {
+			// Already tracked or ignored
+			continue
+		}
+
+		g.checkAncestry(idinfo, untracked)
+	}
+	return permErrs, nil
+}
+
+// groups returns the aggregate metrics for all groups tracked.  This reflects
+// solely what's currently running.
+func (g *Grouper) groups() GroupCountMap {
+	gcounts := make(GroupCountMap)
+
+	var zeroTime time.Time
+	for _, tinfo := range g.tracker.Tracked {
+		if tinfo == nil {
+			continue
+		}
+		cur := gcounts[tinfo.GroupName]
+		cur.Procs++
+		_, counts, mem, start := tinfo.GetStats()
+		cur.Memresident += mem.Resident
+		cur.Memvirtual += mem.Virtual
+		cur.Counts.Cpu += counts.Cpu
+		cur.Counts.ReadBytes += counts.ReadBytes
+		cur.Counts.WriteBytes += counts.WriteBytes
+		if cur.OldestStartTime == zeroTime || start.Before(cur.OldestStartTime) {
+			cur.OldestStartTime = start
+		}
+		gcounts[tinfo.GroupName] = cur
+	}
+
+	return gcounts
+}
+
+// Groups returns GroupCounts with Counts that never decrease in value from one
+// call to the next.  Even if processes exit, their CPU and IO contributions up
+// to that point are included in the results.  Even if no processes remain
+// in a group it will still be included in the results.
+func (g *Grouper) Groups() GroupCountMap {
+	groups := g.groups()
+
+	// First add any accumulated counts to what was just observed,
+	// and update the accumulators.
+	for gname, group := range groups {
+		if oldcounts, ok := g.GroupStats[gname]; ok {
+			group.Counts.Cpu += oldcounts.Cpu
+			group.Counts.ReadBytes += oldcounts.ReadBytes
+			group.Counts.WriteBytes += oldcounts.WriteBytes
+		}
+		g.GroupStats[gname] = group.Counts
+		groups[gname] = group
+	}
+
+	// Now add any groups that were observed in the past but aren't running now.
+	for gname, gcounts := range g.GroupStats {
+		if _, ok := groups[gname]; !ok {
+			groups[gname] = GroupCounts{Counts: gcounts}
+		}
+	}
+
+	return groups
+}

vendor/github.com/ncabatoff/process-exporter/proc/read.go

@@ -0,0 +1,306 @@
+package proc
+
+import (
+	"fmt"
+	"github.com/prometheus/procfs"
+	"time"
+)
+
+func newProcIdStatic(pid, ppid int, startTime uint64, name string, cmdline []string) ProcIdStatic {
+	return ProcIdStatic{ProcId{pid, startTime}, ProcStatic{name, cmdline, ppid, time.Time{}}}
+}
+
+type (
+	// ProcId uniquely identifies a process.
+	ProcId struct {
+		// UNIX process id
+		Pid int
+		// The time the process started after system boot, the value is expressed
+		// in clock ticks.
+		StartTimeRel uint64
+	}
+
+	// ProcStatic contains data read from /proc/pid/*
+	ProcStatic struct {
+		Name      string
+		Cmdline   []string
+		ParentPid int
+		StartTime time.Time
+	}
+
+	// ProcMetrics contains data read from /proc/pid/*
+	ProcMetrics struct {
+		CpuTime       float64
+		ReadBytes     uint64
+		WriteBytes    uint64
+		ResidentBytes uint64
+		VirtualBytes  uint64
+	}
+
+	ProcIdStatic struct {
+		ProcId
+		ProcStatic
+	}
+
+	ProcInfo struct {
+		ProcStatic
+		ProcMetrics
+	}
+
+	ProcIdInfo struct {
+		ProcId
+		ProcStatic
+		ProcMetrics
+	}
+
+	// Proc wraps the details of the underlying procfs-reading library.
+	Proc interface {
+		// GetPid() returns the POSIX PID (process id).  They may be reused over time.
+		GetPid() int
+		// GetProcId() returns (pid,starttime), which can be considered a unique process id.
+		// It may fail if the caller doesn't have permission to read /proc/<pid>/stat, or if
+		// the process has disapeared.
+		GetProcId() (ProcId, error)
+		// GetStatic() returns various details read from files under /proc/<pid>/.  Technically
+		// name may not be static, but we'll pretend it is.
+		// It may fail if the caller doesn't have permission to read those files, or if
+		// the process has disapeared.
+		GetStatic() (ProcStatic, error)
+		// GetMetrics() returns various metrics read from files under /proc/<pid>/.
+		// It may fail if the caller doesn't have permission to read those files, or if
+		// the process has disapeared.
+		GetMetrics() (ProcMetrics, error)
+	}
+
+	// proc is a wrapper for procfs.Proc that caches results of some reads and implements Proc.
+	proc struct {
+		procfs.Proc
+		procid   *ProcId
+		stat     *procfs.ProcStat
+		cmdline  []string
+		io       *procfs.ProcIO
+		bootTime int64
+	}
+
+	procs interface {
+		get(int) Proc
+		length() int
+	}
+
+	procfsprocs struct {
+		Procs    []procfs.Proc
+		bootTime int64
+	}
+
+	// ProcIter is an iterator over a sequence of procs.
+	ProcIter interface {
+		// Next returns true if the iterator is not exhausted.
+		Next() bool
+		// Close releases any resources the iterator uses.
+		Close() error
+		// The iterator satisfies the Proc interface.
+		Proc
+	}
+
+	// procIterator implements the ProcIter interface using procfs.
+	procIterator struct {
+		// procs is the list of Proc we're iterating over.
+		procs
+		// idx is the current iteration, i.e. it's an index into procs.
+		idx int
+		// err is set with an error when Next() fails.  It is not affected by failures accessing
+		// the current iteration variable, e.g. with GetProcId.
+		err error
+		// Proc is the current iteration variable, or nil if Next() has never been called or the
+		// iterator is exhausted.
+		Proc
+	}
+
+	procIdInfos []ProcIdInfo
+)
+
+func procInfoIter(ps ...ProcIdInfo) ProcIter {
+	return &procIterator{procs: procIdInfos(ps), idx: -1}
+}
+
+func Info(p Proc) (ProcIdInfo, error) {
+	id, err := p.GetProcId()
+	if err != nil {
+		return ProcIdInfo{}, err
+	}
+	static, err := p.GetStatic()
+	if err != nil {
+		return ProcIdInfo{}, err
+	}
+	metrics, err := p.GetMetrics()
+	if err != nil {
+		return ProcIdInfo{}, err
+	}
+	return ProcIdInfo{id, static, metrics}, nil
+}
+
+func (p procIdInfos) get(i int) Proc {
+	return &p[i]
+}
+
+func (p procIdInfos) length() int {
+	return len(p)
+}
+
+func (p ProcIdInfo) GetPid() int {
+	return p.ProcId.Pid
+}
+
+func (p ProcIdInfo) GetProcId() (ProcId, error) {
+	return p.ProcId, nil
+}
+
+func (p ProcIdInfo) GetStatic() (ProcStatic, error) {
+	return p.ProcStatic, nil
+}
+
+func (p ProcIdInfo) GetMetrics() (ProcMetrics, error) {
+	return p.ProcMetrics, nil
+}
+
+func (p procfsprocs) get(i int) Proc {
+	return &proc{Proc: p.Procs[i], bootTime: p.bootTime}
+}
+
+func (p procfsprocs) length() int {
+	return len(p.Procs)
+}
+
+func (p *proc) GetPid() int {
+	return p.Proc.PID
+}
+
+func (p *proc) GetStat() (procfs.ProcStat, error) {
+	if p.stat == nil {
+		stat, err := p.Proc.NewStat()
+		if err != nil {
+			return procfs.ProcStat{}, err
+		}
+		p.stat = &stat
+	}
+
+	return *p.stat, nil
+}
+
+func (p *proc) GetProcId() (ProcId, error) {
+	if p.procid == nil {
+		stat, err := p.GetStat()
+		if err != nil {
+			return ProcId{}, err
+		}
+		p.procid = &ProcId{Pid: p.GetPid(), StartTimeRel: stat.Starttime}
+	}
+
+	return *p.procid, nil
+}
+
+func (p *proc) GetCmdLine() ([]string, error) {
+	if p.cmdline == nil {
+		cmdline, err := p.Proc.CmdLine()
+		if err != nil {
+			return nil, err
+		}
+		p.cmdline = cmdline
+	}
+	return p.cmdline, nil
+}
+
+func (p *proc) GetIo() (procfs.ProcIO, error) {
+	if p.io == nil {
+		io, err := p.Proc.NewIO()
+		if err != nil {
+			return procfs.ProcIO{}, err
+		}
+		p.io = &io
+	}
+	return *p.io, nil
+}
+
+func (p proc) GetStatic() (ProcStatic, error) {
+	cmdline, err := p.GetCmdLine()
+	if err != nil {
+		return ProcStatic{}, err
+	}
+	stat, err := p.GetStat()
+	if err != nil {
+		return ProcStatic{}, err
+	}
+	startTime := time.Unix(p.bootTime, 0)
+	startTime = startTime.Add(time.Second / userHZ * time.Duration(stat.Starttime))
+	return ProcStatic{
+		Name:      stat.Comm,
+		Cmdline:   cmdline,
+		ParentPid: stat.PPID,
+		StartTime: startTime,
+	}, nil
+}
+
+func (p proc) GetMetrics() (ProcMetrics, error) {
+	io, err := p.GetIo()
+	if err != nil {
+		return ProcMetrics{}, err
+	}
+	stat, err := p.GetStat()
+	if err != nil {
+		return ProcMetrics{}, err
+	}
+	return ProcMetrics{
+		CpuTime:       stat.CPUTime(),
+		ReadBytes:     io.ReadBytes,
+		WriteBytes:    io.WriteBytes,
+		ResidentBytes: uint64(stat.ResidentMemory()),
+		VirtualBytes:  uint64(stat.VirtualMemory()),
+	}, nil
+}
+
+type FS struct {
+	procfs.FS
+	BootTime int64
+}
+
+// See https://github.com/prometheus/procfs/blob/master/proc_stat.go for details on userHZ.
+const userHZ = 100
+
+// NewFS returns a new FS mounted under the given mountPoint. It will error
+// if the mount point can't be read.
+func NewFS(mountPoint string) (*FS, error) {
+	fs, err := procfs.NewFS(mountPoint)
+	if err != nil {
+		return nil, err
+	}
+	stat, err := fs.NewStat()
+	if err != nil {
+		return nil, err
+	}
+	return &FS{fs, stat.BootTime}, nil
+}
+
+func (fs *FS) AllProcs() ProcIter {
+	procs, err := fs.FS.AllProcs()
+	if err != nil {
+		err = fmt.Errorf("Error reading procs: %v", err)
+	}
+	return &procIterator{procs: procfsprocs{procs, fs.BootTime}, err: err, idx: -1}
+}
+
+func (pi *procIterator) Next() bool {
+	pi.idx++
+	if pi.idx < pi.procs.length() {
+		pi.Proc = pi.procs.get(pi.idx)
+	} else {
+		pi.Proc = nil
+	}
+	return pi.idx < pi.procs.length()
+}
+
+func (pi *procIterator) Close() error {
+	pi.Next()
+	pi.procs = nil
+	pi.Proc = nil
+	return pi.err
+}

vendor/github.com/ncabatoff/process-exporter/proc/tracker.go

@@ -0,0 +1,160 @@
+package proc
+
+import (
+	"fmt"
+	"os"
+	"time"
+)
+
+type (
+	Counts struct {
+		Cpu        float64
+		ReadBytes  uint64
+		WriteBytes uint64
+	}
+
+	Memory struct {
+		Resident uint64
+		Virtual  uint64
+	}
+
+	// Tracker tracks processes and records metrics.
+	Tracker struct {
+		// Tracked holds the processes are being monitored.  Processes
+		// may be blacklisted such that they no longer get tracked by
+		// setting their value in the Tracked map to nil.
+		Tracked map[ProcId]*TrackedProc
+		// ProcIds is a map from pid to ProcId.  This is a convenience
+		// to allow finding the Tracked entry of a parent process.
+		ProcIds map[int]ProcId
+	}
+
+	// TrackedProc accumulates metrics for a process, as well as
+	// remembering an optional GroupName tag associated with it.
+	TrackedProc struct {
+		// lastUpdate is used internally during the update cycle to find which procs have exited
+		lastUpdate time.Time
+		// info is the most recently obtained info for this proc
+		info ProcInfo
+		// accum is the total CPU and IO accrued since we started tracking this proc
+		accum Counts
+		// lastaccum is the CPU and IO accrued in the last Update()
+		lastaccum Counts
+		// GroupName is an optional tag for this proc.
+		GroupName string
+	}
+)
+
+func (tp *TrackedProc) GetName() string {
+	return tp.info.Name
+}
+
+func (tp *TrackedProc) GetCmdLine() []string {
+	return tp.info.Cmdline
+}
+
+func (tp *TrackedProc) GetStats() (aggregate, latest Counts, mem Memory, start time.Time) {
+	return tp.accum, tp.lastaccum, Memory{Resident: tp.info.ResidentBytes, Virtual: tp.info.VirtualBytes}, tp.info.StartTime
+}
+
+func NewTracker() *Tracker {
+	return &Tracker{Tracked: make(map[ProcId]*TrackedProc), ProcIds: make(map[int]ProcId)}
+}
+
+func (t *Tracker) Track(groupName string, idinfo ProcIdInfo) {
+	info := ProcInfo{idinfo.ProcStatic, idinfo.ProcMetrics}
+	t.Tracked[idinfo.ProcId] = &TrackedProc{GroupName: groupName, info: info}
+}
+
+func (t *Tracker) Ignore(id ProcId) {
+	t.Tracked[id] = nil
+}
+
+// Scan procs and update metrics for those which are tracked.  Processes that have gone
+// away get removed from the Tracked map.  New processes are returned, along with the count
+// of permission errors.
+func (t *Tracker) Update(procs ProcIter) ([]ProcIdInfo, int, error) {
+	now := time.Now()
+	var newProcs []ProcIdInfo
+	var permissionErrors int
+
+	for procs.Next() {
+		procId, err := procs.GetProcId()
+		if err != nil {
+			continue
+		}
+
+		last, known := t.Tracked[procId]
+
+		// Are we ignoring this proc?
+		if known && last == nil {
+			continue
+		}
+
+		// TODO if just the io file is unreadable, should we still return the other metrics?
+		metrics, err := procs.GetMetrics()
+		if err != nil {
+			if os.IsPermission(err) {
+				permissionErrors++
+				t.Ignore(procId)
+			}
+			continue
+		}
+
+		if known {
+			var newaccum, lastaccum Counts
+			dcpu := metrics.CpuTime - last.info.CpuTime
+			drbytes := metrics.ReadBytes - last.info.ReadBytes
+			dwbytes := metrics.WriteBytes - last.info.WriteBytes
+
+			lastaccum = Counts{Cpu: dcpu, ReadBytes: drbytes, WriteBytes: dwbytes}
+			newaccum = Counts{
+				Cpu:        last.accum.Cpu + lastaccum.Cpu,
+				ReadBytes:  last.accum.ReadBytes + lastaccum.ReadBytes,
+				WriteBytes: last.accum.WriteBytes + lastaccum.WriteBytes,
+			}
+
+			last.info.ProcMetrics = metrics
+			last.lastUpdate = now
+			last.accum = newaccum
+			last.lastaccum = lastaccum
+		} else {
+			static, err := procs.GetStatic()
+			if err != nil {
+				continue
+			}
+			newProcs = append(newProcs, ProcIdInfo{procId, static, metrics})
+
+			// Is this a new process with the same pid as one we already know?
+			if oldProcId, ok := t.ProcIds[procId.Pid]; ok {
+				// Delete it from known, otherwise the cleanup below will remove the
+				// ProcIds entry we're about to create
+				delete(t.Tracked, oldProcId)
+			}
+			t.ProcIds[procId.Pid] = procId
+		}
+
+	}
+	err := procs.Close()
+	if err != nil {
+		return nil, permissionErrors, fmt.Errorf("Error reading procs: %v", err)
+	}
+
+	// Rather than allocating a new map each time to detect procs that have
+	// disappeared, we bump the last update time on those that are still
+	// present.  Then as a second pass we traverse the map looking for
+	// stale procs and removing them.
+	for procId, pinfo := range t.Tracked {
+		if pinfo == nil {
+			// TODO is this a bug? we're not tracking the proc so we don't see it go away so ProcIds
+			// and Tracked are leaking?
+			continue
+		}
+		if pinfo.lastUpdate != now {
+			delete(t.Tracked, procId)
+			delete(t.ProcIds, procId.Pid)
+		}
+	}
+
+	return newProcs, permissionErrors, nil
+}