Update go dependencies to v1.17.0

This commit is contained in:
Manuel Alejandro de Brito Fontes 2019-12-10 21:55:54 -03:00
parent 67dce30ba6
commit 75c3c47f81
516 changed files with 48300 additions and 15154 deletions

View file

@ -27,7 +27,7 @@
[people.akihirosuda]
Name = "Akihiro Suda"
Email = "suda.akihiro@lab.ntt.co.jp"
Email = "akihiro.suda.cz@hco.ntt.co.jp"
GitHub = "AkihiroSuda"
[people.dnephin]

View file

@ -1,7 +1,7 @@
dependencies:
post:
# install golint
- go get github.com/golang/lint/golint
- go get golang.org/x/lint/golint
test:
pre:

View file

@ -18,7 +18,7 @@ func HumanDuration(d time.Duration) string {
return fmt.Sprintf("%d seconds", seconds)
} else if minutes := int(d.Minutes()); minutes == 1 {
return "About a minute"
} else if minutes < 46 {
} else if minutes < 60 {
return fmt.Sprintf("%d minutes", minutes)
} else if hours := int(d.Hours() + 0.5); hours == 1 {
return "About an hour"

View file

@ -96,8 +96,13 @@ func ParseUlimit(val string) (*Ulimit, error) {
return nil, fmt.Errorf("too many limit value arguments - %s, can only have up to two, `soft[:hard]`", parts[1])
}
if soft > *hard {
return nil, fmt.Errorf("ulimit soft limit must be less than or equal to hard limit: %d > %d", soft, *hard)
if *hard != -1 {
if soft == -1 {
return nil, fmt.Errorf("ulimit soft limit must be less than or equal to hard limit: soft: -1 (unlimited), hard: %d", *hard)
}
if soft > *hard {
return nil, fmt.Errorf("ulimit soft limit must be less than or equal to hard limit: %d > %d", soft, *hard)
}
}
return &Ulimit{Name: parts[0], Soft: soft, Hard: *hard}, nil

View file

@ -1,8 +1,7 @@
language: go
go:
- "1.3"
- "1.4"
- "1.10"
- 1.3
- 1.4
script:
- go test
- go build

View file

@ -4,7 +4,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"reflect"
"strconv"
@ -27,19 +26,15 @@ func Marshal(o interface{}) ([]byte, error) {
return y, nil
}
// JSONOpt is a decoding option for decoding from JSON format.
type JSONOpt func(*json.Decoder) *json.Decoder
// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object,
// optionally configuring the behavior of the JSON unmarshal.
func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error {
// Converts YAML to JSON then uses JSON to unmarshal into an object.
func Unmarshal(y []byte, o interface{}) error {
vo := reflect.ValueOf(o)
j, err := yamlToJSON(y, &vo, yaml.Unmarshal)
j, err := yamlToJSON(y, &vo)
if err != nil {
return fmt.Errorf("error converting YAML to JSON: %v", err)
}
err = jsonUnmarshal(bytes.NewReader(j), o, opts...)
err = json.Unmarshal(j, o)
if err != nil {
return fmt.Errorf("error unmarshaling JSON: %v", err)
}
@ -47,21 +42,6 @@ func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error {
return nil
}
// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the
// object, optionally applying decoder options prior to decoding. We are not
// using json.Unmarshal directly as we want the chance to pass in non-default
// options.
func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error {
d := json.NewDecoder(r)
for _, opt := range opts {
d = opt(d)
}
if err := d.Decode(&o); err != nil {
return fmt.Errorf("while decoding JSON: %v", err)
}
return nil
}
// Convert JSON to YAML.
func JSONToYAML(j []byte) ([]byte, error) {
// Convert the JSON to an object.
@ -80,8 +60,8 @@ func JSONToYAML(j []byte) ([]byte, error) {
return yaml.Marshal(jsonObj)
}
// YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML,
// passing JSON through this method should be a no-op.
// Convert YAML to JSON. Since JSON is a subset of YAML, passing JSON through
// this method should be a no-op.
//
// Things YAML can do that are not supported by JSON:
// * In YAML you can have binary and null keys in your maps. These are invalid
@ -90,22 +70,14 @@ func JSONToYAML(j []byte) ([]byte, error) {
// use binary data with this library, encode the data as base64 as usual but do
// not use the !!binary tag in your YAML. This will ensure the original base64
// encoded data makes it all the way through to the JSON.
//
// For strict decoding of YAML, use YAMLToJSONStrict.
func YAMLToJSON(y []byte) ([]byte, error) {
return yamlToJSON(y, nil, yaml.Unmarshal)
return yamlToJSON(y, nil)
}
// YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding,
// returning an error on any duplicate field names.
func YAMLToJSONStrict(y []byte) ([]byte, error) {
return yamlToJSON(y, nil, yaml.UnmarshalStrict)
}
func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) {
func yamlToJSON(y []byte, jsonTarget *reflect.Value) ([]byte, error) {
// Convert the YAML to an object.
var yamlObj interface{}
err := yamlUnmarshal(y, &yamlObj)
err := yaml.Unmarshal(y, &yamlObj)
if err != nil {
return nil, err
}

View file

@ -1,14 +0,0 @@
// This file contains changes that are only compatible with go 1.10 and onwards.
// +build go1.10
package yaml
import "encoding/json"
// DisallowUnknownFields configures the JSON decoder to error out if unknown
// fields come along, instead of dropping them by default.
func DisallowUnknownFields(d *json.Decoder) *json.Decoder {
d.DisallowUnknownFields()
return d
}

View file

@ -1,6 +1,9 @@
module github.com/go-openapi/jsonpointer
require (
github.com/go-openapi/swag v0.19.2
github.com/go-openapi/swag v0.19.5
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
github.com/stretchr/testify v1.3.0
)
go 1.13

View file

@ -1,8 +1,8 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -10,6 +10,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

View file

@ -135,7 +135,7 @@ func getSingleImpl(node interface{}, decodedToken string, nameProvider *swag.Nam
kv := reflect.ValueOf(decodedToken)
mv := rValue.MapIndex(kv)
if mv.IsValid() && !swag.IsZero(mv) {
if mv.IsValid() {
return mv.Interface(), kind, nil
}
return nil, kind, fmt.Errorf("object has no key %q", decodedToken)

View file

@ -3,8 +3,10 @@ module github.com/go-openapi/jsonreference
require (
github.com/PuerkitoBio/purell v1.1.1
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/go-openapi/jsonpointer v0.19.2
github.com/go-openapi/jsonpointer v0.19.3
github.com/stretchr/testify v1.3.0
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
golang.org/x/text v0.3.2 // indirect
)
go 1.13

View file

@ -7,8 +7,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -16,6 +20,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -24,6 +30,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

View file

@ -21,7 +21,7 @@ import (
func bindataRead(data []byte, name string) ([]byte, error) {
gz, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return nil, fmt.Errorf("Read %q: %v", name, err)
return nil, fmt.Errorf("read %q: %v", name, err)
}
var buf bytes.Buffer
@ -29,7 +29,7 @@ func bindataRead(data []byte, name string) ([]byte, error) {
clErr := gz.Close()
if err != nil {
return nil, fmt.Errorf("Read %q: %v", name, err)
return nil, fmt.Errorf("read %q: %v", name, err)
}
if clErr != nil {
return nil, err
@ -85,7 +85,7 @@ func jsonschemaDraft04JSON() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "jsonschema-draft-04.json", size: 4357, mode: os.FileMode(436), modTime: time.Unix(1540282154, 0)}
info := bindataFileInfo{name: "jsonschema-draft-04.json", size: 4357, mode: os.FileMode(0644), modTime: time.Unix(1567900649, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe1, 0x48, 0x9d, 0xb, 0x47, 0x55, 0xf0, 0x27, 0x93, 0x30, 0x25, 0x91, 0xd3, 0xfc, 0xb8, 0xf0, 0x7b, 0x68, 0x93, 0xa8, 0x2a, 0x94, 0xf2, 0x48, 0x95, 0xf8, 0xe4, 0xed, 0xf1, 0x1b, 0x82, 0xe2}}
return a, nil
}
@ -105,7 +105,7 @@ func v2SchemaJSON() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "v2/schema.json", size: 40249, mode: os.FileMode(436), modTime: time.Unix(1540282154, 0)}
info := bindataFileInfo{name: "v2/schema.json", size: 40249, mode: os.FileMode(0644), modTime: time.Unix(1567900649, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xcb, 0x25, 0x27, 0xe8, 0x46, 0xae, 0x22, 0xc4, 0xf4, 0x8b, 0x1, 0x32, 0x4d, 0x1f, 0xf8, 0xdf, 0x75, 0x15, 0xc8, 0x2d, 0xc7, 0xed, 0xe, 0x7e, 0x0, 0x75, 0xc0, 0xf9, 0xd2, 0x1f, 0x75, 0x57}}
return a, nil
}

View file

@ -1,14 +1,17 @@
module github.com/go-openapi/spec
require (
github.com/go-openapi/jsonpointer v0.19.2
github.com/go-openapi/jsonpointer v0.19.3
github.com/go-openapi/jsonreference v0.19.2
github.com/go-openapi/swag v0.19.2
github.com/go-openapi/swag v0.19.5
github.com/kr/pty v1.1.5 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/stretchr/testify v1.3.0
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 // indirect
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f // indirect
golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 // indirect
gopkg.in/yaml.v2 v2.2.2
)
go 1.13

View file

@ -13,6 +13,8 @@ github.com/go-openapi/jsonpointer v0.19.0 h1:FTUMcX77w5rQkClIzDtTxvn6Bsa894CcrzN
github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.19.0 h1:BqWKpV1dFd+AuiKlgtddwVIFQsuMpxfBDBHGfM2yNpk=
github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
@ -21,6 +23,8 @@ github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi88
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -31,6 +35,8 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -47,6 +53,8 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View file

@ -160,6 +160,7 @@ func (r *schemaLoader) load(refURL *url.URL) (interface{}, url.URL, bool, error)
if !fromCache {
b, err := r.loadDoc(normalized)
if err != nil {
debugLog("unable to load the document: %v", err)
return nil, url.URL{}, false, err
}

View file

@ -1,3 +1,4 @@
secrets.yml
vendor
Godeps
.idea

View file

@ -39,11 +39,12 @@ func IsFloat64AJSONInteger(f float64) bool {
diff := math.Abs(f - g)
// more info: https://floating-point-gui.de/errors/comparison/#look-out-for-edge-cases
if f == g { // best case
switch {
case f == g: // best case
return true
} else if f == float64(int64(f)) || f == float64(uint64(f)) { // optimistic case
case f == float64(int64(f)) || f == float64(uint64(f)): // optimistic case
return true
} else if f == 0 || g == 0 || diff < math.SmallestNonzeroFloat64 { // very close to 0 values
case f == 0 || g == 0 || diff < math.SmallestNonzeroFloat64: // very close to 0 values
return diff < (epsilon * math.SmallestNonzeroFloat64)
}
// check the relative error

View file

@ -99,7 +99,7 @@ func ConcatJSON(blobs ...[]byte) []byte {
last := len(blobs) - 1
for blobs[last] == nil || bytes.Equal(blobs[last], nullJSON) {
// strips trailing null objects
last = last - 1
last--
if last < 0 {
// there was nothing but "null"s or nil...
return nil

View file

@ -28,6 +28,14 @@ var initialisms []string
var isInitialism func(string) bool
// GoNamePrefixFunc sets an optional rule to prefix go names
// which do not start with a letter.
//
// e.g. to help converting "123" into "{prefix}123"
//
// The default is to prefix with "X"
var GoNamePrefixFunc func(string) string
func init() {
// Taken from https://github.com/golang/lint/blob/3390df4df2787994aea98de825b964ac7944b817/lint.go#L732-L769
var configuredInitialisms = map[string]bool{
@ -288,8 +296,17 @@ func ToGoName(name string) string {
}
if len(result) > 0 {
if !unicode.IsUpper([]rune(result)[0]) {
result = "X" + result
// Only prefix with X when the first character isn't an ascii letter
first := []rune(result)[0]
if !unicode.IsLetter(first) || (first > unicode.MaxASCII && !unicode.IsUpper(first)) {
if GoNamePrefixFunc == nil {
return "X" + result
}
result = GoNamePrefixFunc(name) + result
}
first = []rune(result)[0]
if unicode.IsLetter(first) && !unicode.IsUpper(first) {
result = string(append([]rune{unicode.ToUpper(first)}, []rune(result)[1:]...))
}
}

View file

@ -143,19 +143,43 @@ func (s *JSONMapItem) UnmarshalEasyJSON(in *jlexer.Lexer) {
}
func transformData(input interface{}) (out interface{}, err error) {
format := func(t interface{}) (string, error) {
switch k := t.(type) {
case string:
return k, nil
case uint:
return strconv.FormatUint(uint64(k), 10), nil
case uint8:
return strconv.FormatUint(uint64(k), 10), nil
case uint16:
return strconv.FormatUint(uint64(k), 10), nil
case uint32:
return strconv.FormatUint(uint64(k), 10), nil
case uint64:
return strconv.FormatUint(k, 10), nil
case int:
return strconv.Itoa(k), nil
case int8:
return strconv.FormatInt(int64(k), 10), nil
case int16:
return strconv.FormatInt(int64(k), 10), nil
case int32:
return strconv.FormatInt(int64(k), 10), nil
case int64:
return strconv.FormatInt(k, 10), nil
default:
return "", fmt.Errorf("unexpected map key type, got: %T", k)
}
}
switch in := input.(type) {
case yaml.MapSlice:
o := make(JSONMapSlice, len(in))
for i, mi := range in {
var nmi JSONMapItem
switch k := mi.Key.(type) {
case string:
nmi.Key = k
case int:
nmi.Key = strconv.Itoa(k)
default:
return nil, fmt.Errorf("types don't match expect map key string or int got: %T", mi.Key)
if nmi.Key, err = format(mi.Key); err != nil {
return nil, err
}
v, ert := transformData(mi.Value)
@ -170,13 +194,8 @@ func transformData(input interface{}) (out interface{}, err error) {
o := make(JSONMapSlice, 0, len(in))
for ke, va := range in {
var nmi JSONMapItem
switch k := ke.(type) {
case string:
nmi.Key = k
case int:
nmi.Key = strconv.Itoa(k)
default:
return nil, fmt.Errorf("types don't match expect map key string or int got: %T", ke)
if nmi.Key, err = format(ke); err != nil {
return nil, err
}
v, ert := transformData(va)

View file

@ -4,6 +4,7 @@ go:
- 1.6.x
- 1.7.x
- 1.8.x
- 1.9.x
- master
matrix:
allow_failures:

View file

@ -3,7 +3,7 @@ httpcache
[![Build Status](https://travis-ci.org/gregjones/httpcache.svg?branch=master)](https://travis-ci.org/gregjones/httpcache) [![GoDoc](https://godoc.org/github.com/gregjones/httpcache?status.svg)](https://godoc.org/github.com/gregjones/httpcache)
Package httpcache provides a http.RoundTripper implementation that works as a mostly RFC-compliant cache for http responses.
Package httpcache provides a http.RoundTripper implementation that works as a mostly [RFC 7234](https://tools.ietf.org/html/rfc7234) compliant cache for http responses.
It is only suitable for use as a 'private' cache (i.e. for a web-browser or an API-client and not for a shared proxy).
@ -17,6 +17,7 @@ Cache Backends
- [`github.com/gregjones/httpcache/leveldbcache`](https://github.com/gregjones/httpcache/tree/master/leveldbcache) provides a filesystem-backed cache using [leveldb](https://github.com/syndtr/goleveldb/leveldb).
- [`github.com/die-net/lrucache`](https://github.com/die-net/lrucache) provides an in-memory cache that will evict least-recently used entries.
- [`github.com/die-net/lrucache/twotier`](https://github.com/die-net/lrucache/tree/master/twotier) allows caches to be combined, for example to use lrucache above with a persistent disk-cache.
- [`github.com/birkelund/boltdbcache`](https://github.com/birkelund/boltdbcache) provides a BoltDB implementation (based on the [bbolt](https://github.com/coreos/bbolt) fork).
License
-------

View file

@ -10,7 +10,6 @@ import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
@ -41,7 +40,11 @@ type Cache interface {
// cacheKey returns the cache key for req.
func cacheKey(req *http.Request) string {
return req.URL.String()
if req.Method == http.MethodGet {
return req.URL.String()
} else {
return req.Method + " " + req.URL.String()
}
}
// CachedResponse returns the cached http.Response for req if present, and nil
@ -189,16 +192,11 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
for _, header := range endToEndHeaders {
cachedResp.Header[header] = resp.Header[header]
}
cachedResp.Status = fmt.Sprintf("%d %s", http.StatusOK, http.StatusText(http.StatusOK))
cachedResp.StatusCode = http.StatusOK
resp = cachedResp
} else if (err != nil || (cachedResp != nil && resp.StatusCode >= 500)) &&
req.Method == "GET" && canStaleOnError(cachedResp.Header, req.Header) {
// In case of transport failure and stale-if-error activated, returns cached content
// when available
cachedResp.Status = fmt.Sprintf("%d %s", http.StatusOK, http.StatusText(http.StatusOK))
cachedResp.StatusCode = http.StatusOK
return cachedResp, nil
} else {
if err != nil || resp.StatusCode != http.StatusOK {

View file

@ -74,6 +74,7 @@ type Iterator struct {
buf []byte
head int
tail int
depth int
captureStartedAt int
captured []byte
Error error
@ -88,6 +89,7 @@ func NewIterator(cfg API) *Iterator {
buf: nil,
head: 0,
tail: 0,
depth: 0,
}
}
@ -99,6 +101,7 @@ func Parse(cfg API, reader io.Reader, bufSize int) *Iterator {
buf: make([]byte, bufSize),
head: 0,
tail: 0,
depth: 0,
}
}
@ -110,6 +113,7 @@ func ParseBytes(cfg API, input []byte) *Iterator {
buf: input,
head: 0,
tail: len(input),
depth: 0,
}
}
@ -128,6 +132,7 @@ func (iter *Iterator) Reset(reader io.Reader) *Iterator {
iter.reader = reader
iter.head = 0
iter.tail = 0
iter.depth = 0
return iter
}
@ -137,6 +142,7 @@ func (iter *Iterator) ResetBytes(input []byte) *Iterator {
iter.buf = input
iter.head = 0
iter.tail = len(input)
iter.depth = 0
return iter
}
@ -320,3 +326,24 @@ func (iter *Iterator) Read() interface{} {
return nil
}
}
// limit maximum depth of nesting, as allowed by https://tools.ietf.org/html/rfc7159#section-9
const maxDepth = 10000
func (iter *Iterator) incrementDepth() (success bool) {
iter.depth++
if iter.depth <= maxDepth {
return true
}
iter.ReportError("incrementDepth", "exceeded max depth")
return false
}
func (iter *Iterator) decrementDepth() (success bool) {
iter.depth--
if iter.depth >= 0 {
return true
}
iter.ReportError("decrementDepth", "unexpected negative nesting")
return false
}

View file

@ -28,26 +28,32 @@ func (iter *Iterator) ReadArray() (ret bool) {
func (iter *Iterator) ReadArrayCB(callback func(*Iterator) bool) (ret bool) {
c := iter.nextToken()
if c == '[' {
if !iter.incrementDepth() {
return false
}
c = iter.nextToken()
if c != ']' {
iter.unreadByte()
if !callback(iter) {
iter.decrementDepth()
return false
}
c = iter.nextToken()
for c == ',' {
if !callback(iter) {
iter.decrementDepth()
return false
}
c = iter.nextToken()
}
if c != ']' {
iter.ReportError("ReadArrayCB", "expect ] in the end, but found "+string([]byte{c}))
iter.decrementDepth()
return false
}
return true
return iter.decrementDepth()
}
return true
return iter.decrementDepth()
}
if c == 'n' {
iter.skipThreeBytes('u', 'l', 'l')

View file

@ -112,6 +112,9 @@ func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool {
c := iter.nextToken()
var field string
if c == '{' {
if !iter.incrementDepth() {
return false
}
c = iter.nextToken()
if c == '"' {
iter.unreadByte()
@ -121,6 +124,7 @@ func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool {
iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))
}
if !callback(iter, field) {
iter.decrementDepth()
return false
}
c = iter.nextToken()
@ -131,20 +135,23 @@ func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool {
iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))
}
if !callback(iter, field) {
iter.decrementDepth()
return false
}
c = iter.nextToken()
}
if c != '}' {
iter.ReportError("ReadObjectCB", `object not ended with }`)
iter.decrementDepth()
return false
}
return true
return iter.decrementDepth()
}
if c == '}' {
return true
return iter.decrementDepth()
}
iter.ReportError("ReadObjectCB", `expect " after }, but found `+string([]byte{c}))
iter.decrementDepth()
return false
}
if c == 'n' {
@ -159,15 +166,20 @@ func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool {
func (iter *Iterator) ReadMapCB(callback func(*Iterator, string) bool) bool {
c := iter.nextToken()
if c == '{' {
if !iter.incrementDepth() {
return false
}
c = iter.nextToken()
if c == '"' {
iter.unreadByte()
field := iter.ReadString()
if iter.nextToken() != ':' {
iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c}))
iter.decrementDepth()
return false
}
if !callback(iter, field) {
iter.decrementDepth()
return false
}
c = iter.nextToken()
@ -175,23 +187,27 @@ func (iter *Iterator) ReadMapCB(callback func(*Iterator, string) bool) bool {
field = iter.ReadString()
if iter.nextToken() != ':' {
iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c}))
iter.decrementDepth()
return false
}
if !callback(iter, field) {
iter.decrementDepth()
return false
}
c = iter.nextToken()
}
if c != '}' {
iter.ReportError("ReadMapCB", `object not ended with }`)
iter.decrementDepth()
return false
}
return true
return iter.decrementDepth()
}
if c == '}' {
return true
return iter.decrementDepth()
}
iter.ReportError("ReadMapCB", `expect " after }, but found `+string([]byte{c}))
iter.decrementDepth()
return false
}
if c == 'n' {

View file

@ -22,6 +22,9 @@ func (iter *Iterator) skipNumber() {
func (iter *Iterator) skipArray() {
level := 1
if !iter.incrementDepth() {
return
}
for {
for i := iter.head; i < iter.tail; i++ {
switch iter.buf[i] {
@ -31,8 +34,14 @@ func (iter *Iterator) skipArray() {
i = iter.head - 1 // it will be i++ soon
case '[': // If open symbol, increase level
level++
if !iter.incrementDepth() {
return
}
case ']': // If close symbol, increase level
level--
if !iter.decrementDepth() {
return
}
// If we have returned to the original level, we're done
if level == 0 {
@ -50,6 +59,10 @@ func (iter *Iterator) skipArray() {
func (iter *Iterator) skipObject() {
level := 1
if !iter.incrementDepth() {
return
}
for {
for i := iter.head; i < iter.tail; i++ {
switch iter.buf[i] {
@ -59,8 +72,14 @@ func (iter *Iterator) skipObject() {
i = iter.head - 1 // it will be i++ soon
case '{': // If open symbol, increase level
level++
if !iter.incrementDepth() {
return
}
case '}': // If close symbol, increase level
level--
if !iter.decrementDepth() {
return
}
// If we have returned to the original level, we're done
if level == 0 {

View file

@ -60,6 +60,7 @@ func (b *ctx) append(prefix string) *ctx {
// ReadVal copy the underlying JSON into go interface, same as json.Unmarshal
func (iter *Iterator) ReadVal(obj interface{}) {
depth := iter.depth
cacheKey := reflect2.RTypeOf(obj)
decoder := iter.cfg.getDecoderFromCache(cacheKey)
if decoder == nil {
@ -76,6 +77,10 @@ func (iter *Iterator) ReadVal(obj interface{}) {
return
}
decoder.Decode(ptr, iter)
if iter.depth != depth {
iter.ReportError("ReadVal", "unexpected mismatched nesting")
return
}
}
// WriteVal copy the go interface into underlying JSON, same as json.Marshal

View file

@ -341,10 +341,10 @@ func describeStruct(ctx *ctx, typ reflect2.Type) *StructDescriptor {
if ctx.onlyTaggedField && !hastag && !field.Anonymous() {
continue
}
tagParts := strings.Split(tag, ",")
if tag == "-" {
continue
}
tagParts := strings.Split(tag, ",")
if field.Anonymous() && (tag == "" || tagParts[0] == "") {
if field.Type().Kind() == reflect.Struct {
structDescriptor := describeStruct(ctx, field.Type())

View file

@ -249,6 +249,10 @@ type mapEncoder struct {
}
func (encoder *mapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
if *(*unsafe.Pointer)(ptr) == nil {
stream.WriteNil()
return
}
stream.WriteObjectStart()
iter := encoder.mapType.UnsafeIterate(ptr)
for i := 0; iter.HasNext(); i++ {

View file

@ -3,8 +3,9 @@ package jsoniter
import (
"encoding"
"encoding/json"
"github.com/modern-go/reflect2"
"unsafe"
"github.com/modern-go/reflect2"
)
var marshalerType = reflect2.TypeOfPtr((*json.Marshaler)(nil)).Elem()
@ -93,10 +94,17 @@ func (encoder *marshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
stream.WriteNil()
return
}
bytes, err := json.Marshal(obj)
marshaler := obj.(json.Marshaler)
bytes, err := marshaler.MarshalJSON()
if err != nil {
stream.Error = err
} else {
// html escape was already done by jsoniter
// but the extra '\n' should be trimed
l := len(bytes)
if l > 0 && bytes[l-1] == '\n' {
bytes = bytes[:l-1]
}
stream.Write(bytes)
}
}

View file

@ -500,6 +500,9 @@ func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator)
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
var c byte
for c = ','; c == ','; c = iter.nextToken() {
decoder.decodeOneField(ptr, iter)
@ -510,6 +513,7 @@ func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator)
if c != '}' {
iter.ReportError("struct Decode", `expect }, but found `+string([]byte{c}))
}
iter.decrementDepth()
}
func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *Iterator) {
@ -571,6 +575,9 @@ func (decoder *oneFieldStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator)
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
if iter.readFieldHash() == decoder.fieldHash {
decoder.fieldDecoder.Decode(ptr, iter)
@ -584,6 +591,7 @@ func (decoder *oneFieldStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator)
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type twoFieldsStructDecoder struct {
@ -598,6 +606,9 @@ func (decoder *twoFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -614,6 +625,7 @@ func (decoder *twoFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type threeFieldsStructDecoder struct {
@ -630,6 +642,9 @@ func (decoder *threeFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterat
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -648,6 +663,7 @@ func (decoder *threeFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterat
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type fourFieldsStructDecoder struct {
@ -666,6 +682,9 @@ func (decoder *fourFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterato
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -686,6 +705,7 @@ func (decoder *fourFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterato
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type fiveFieldsStructDecoder struct {
@ -706,6 +726,9 @@ func (decoder *fiveFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterato
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -728,6 +751,7 @@ func (decoder *fiveFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterato
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type sixFieldsStructDecoder struct {
@ -750,6 +774,9 @@ func (decoder *sixFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -774,6 +801,7 @@ func (decoder *sixFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type sevenFieldsStructDecoder struct {
@ -798,6 +826,9 @@ func (decoder *sevenFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterat
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -824,6 +855,7 @@ func (decoder *sevenFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterat
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type eightFieldsStructDecoder struct {
@ -850,6 +882,9 @@ func (decoder *eightFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterat
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -878,6 +913,7 @@ func (decoder *eightFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterat
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type nineFieldsStructDecoder struct {
@ -906,6 +942,9 @@ func (decoder *nineFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterato
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -936,6 +975,7 @@ func (decoder *nineFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterato
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type tenFieldsStructDecoder struct {
@ -966,6 +1006,9 @@ func (decoder *tenFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator
if !iter.readObjectStart() {
return
}
if !iter.incrementDepth() {
return
}
for {
switch iter.readFieldHash() {
case decoder.fieldHash1:
@ -998,6 +1041,7 @@ func (decoder *tenFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
iter.decrementDepth()
}
type structFieldDecoder struct {

11
vendor/github.com/liggitt/tabwriter/.travis.yml generated vendored Normal file
View file

@ -0,0 +1,11 @@
language: go
go:
- "1.8"
- "1.9"
- "1.10"
- "1.11"
- "1.12"
- master
script: go test -v ./...

27
vendor/github.com/liggitt/tabwriter/LICENSE generated vendored Normal file
View file

@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

7
vendor/github.com/liggitt/tabwriter/README.md generated vendored Normal file
View file

@ -0,0 +1,7 @@
This repo is a drop-in replacement for the golang [text/tabwriter](https://golang.org/pkg/text/tabwriter/) package.
It is based on that package at [cf2c2ea8](https://github.com/golang/go/tree/cf2c2ea89d09d486bb018b1817c5874388038c3a/src/text/tabwriter) and inherits its license.
The following additional features are supported:
* `RememberWidths` flag allows remembering maximum widths seen per column even after Flush() is called.
* `RememberedWidths() []int` and `SetRememberedWidths([]int) *Writer` allows obtaining and transferring remembered column width between writers.

637
vendor/github.com/liggitt/tabwriter/tabwriter.go generated vendored Normal file
View file

@ -0,0 +1,637 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package tabwriter implements a write filter (tabwriter.Writer) that
// translates tabbed columns in input into properly aligned text.
//
// It is a drop-in replacement for the golang text/tabwriter package (https://golang.org/pkg/text/tabwriter),
// based on that package at https://github.com/golang/go/tree/cf2c2ea89d09d486bb018b1817c5874388038c3a
// with support for additional features.
//
// The package is using the Elastic Tabstops algorithm described at
// http://nickgravgaard.com/elastictabstops/index.html.
package tabwriter
import (
"io"
"unicode/utf8"
)
// ----------------------------------------------------------------------------
// Filter implementation
// A cell represents a segment of text terminated by tabs or line breaks.
// The text itself is stored in a separate buffer; cell only describes the
// segment's size in bytes, its width in runes, and whether it's an htab
// ('\t') terminated cell.
//
type cell struct {
size int // cell size in bytes
width int // cell width in runes
htab bool // true if the cell is terminated by an htab ('\t')
}
// A Writer is a filter that inserts padding around tab-delimited
// columns in its input to align them in the output.
//
// The Writer treats incoming bytes as UTF-8-encoded text consisting
// of cells terminated by horizontal ('\t') or vertical ('\v') tabs,
// and newline ('\n') or formfeed ('\f') characters; both newline and
// formfeed act as line breaks.
//
// Tab-terminated cells in contiguous lines constitute a column. The
// Writer inserts padding as needed to make all cells in a column have
// the same width, effectively aligning the columns. It assumes that
// all characters have the same width, except for tabs for which a
// tabwidth must be specified. Column cells must be tab-terminated, not
// tab-separated: non-tab terminated trailing text at the end of a line
// forms a cell but that cell is not part of an aligned column.
// For instance, in this example (where | stands for a horizontal tab):
//
// aaaa|bbb|d
// aa |b |dd
// a |
// aa |cccc|eee
//
// the b and c are in distinct columns (the b column is not contiguous
// all the way). The d and e are not in a column at all (there's no
// terminating tab, nor would the column be contiguous).
//
// The Writer assumes that all Unicode code points have the same width;
// this may not be true in some fonts or if the string contains combining
// characters.
//
// If DiscardEmptyColumns is set, empty columns that are terminated
// entirely by vertical (or "soft") tabs are discarded. Columns
// terminated by horizontal (or "hard") tabs are not affected by
// this flag.
//
// If a Writer is configured to filter HTML, HTML tags and entities
// are passed through. The widths of tags and entities are
// assumed to be zero (tags) and one (entities) for formatting purposes.
//
// A segment of text may be escaped by bracketing it with Escape
// characters. The tabwriter passes escaped text segments through
// unchanged. In particular, it does not interpret any tabs or line
// breaks within the segment. If the StripEscape flag is set, the
// Escape characters are stripped from the output; otherwise they
// are passed through as well. For the purpose of formatting, the
// width of the escaped text is always computed excluding the Escape
// characters.
//
// The formfeed character acts like a newline but it also terminates
// all columns in the current line (effectively calling Flush). Tab-
// terminated cells in the next line start new columns. Unless found
// inside an HTML tag or inside an escaped text segment, formfeed
// characters appear as newlines in the output.
//
// The Writer must buffer input internally, because proper spacing
// of one line may depend on the cells in future lines. Clients must
// call Flush when done calling Write.
//
type Writer struct {
// configuration
output io.Writer
minwidth int
tabwidth int
padding int
padbytes [8]byte
flags uint
// current state
buf []byte // collected text excluding tabs or line breaks
pos int // buffer position up to which cell.width of incomplete cell has been computed
cell cell // current incomplete cell; cell.width is up to buf[pos] excluding ignored sections
endChar byte // terminating char of escaped sequence (Escape for escapes, '>', ';' for HTML tags/entities, or 0)
lines [][]cell // list of lines; each line is a list of cells
widths []int // list of column widths in runes - re-used during formatting
maxwidths []int // list of max column widths in runes
}
// addLine adds a new line.
// flushed is a hint indicating whether the underlying writer was just flushed.
// If so, the previous line is not likely to be a good indicator of the new line's cells.
func (b *Writer) addLine(flushed bool) {
// Grow slice instead of appending,
// as that gives us an opportunity
// to re-use an existing []cell.
if n := len(b.lines) + 1; n <= cap(b.lines) {
b.lines = b.lines[:n]
b.lines[n-1] = b.lines[n-1][:0]
} else {
b.lines = append(b.lines, nil)
}
if !flushed {
// The previous line is probably a good indicator
// of how many cells the current line will have.
// If the current line's capacity is smaller than that,
// abandon it and make a new one.
if n := len(b.lines); n >= 2 {
if prev := len(b.lines[n-2]); prev > cap(b.lines[n-1]) {
b.lines[n-1] = make([]cell, 0, prev)
}
}
}
}
// Reset the current state.
func (b *Writer) reset() {
b.buf = b.buf[:0]
b.pos = 0
b.cell = cell{}
b.endChar = 0
b.lines = b.lines[0:0]
b.widths = b.widths[0:0]
b.addLine(true)
}
// Internal representation (current state):
//
// - all text written is appended to buf; tabs and line breaks are stripped away
// - at any given time there is a (possibly empty) incomplete cell at the end
// (the cell starts after a tab or line break)
// - cell.size is the number of bytes belonging to the cell so far
// - cell.width is text width in runes of that cell from the start of the cell to
// position pos; html tags and entities are excluded from this width if html
// filtering is enabled
// - the sizes and widths of processed text are kept in the lines list
// which contains a list of cells for each line
// - the widths list is a temporary list with current widths used during
// formatting; it is kept in Writer because it's re-used
//
// |<---------- size ---------->|
// | |
// |<- width ->|<- ignored ->| |
// | | | |
// [---processed---tab------------<tag>...</tag>...]
// ^ ^ ^
// | | |
// buf start of incomplete cell pos
// Formatting can be controlled with these flags.
const (
// Ignore html tags and treat entities (starting with '&'
// and ending in ';') as single characters (width = 1).
FilterHTML uint = 1 << iota
// Strip Escape characters bracketing escaped text segments
// instead of passing them through unchanged with the text.
StripEscape
// Force right-alignment of cell content.
// Default is left-alignment.
AlignRight
// Handle empty columns as if they were not present in
// the input in the first place.
DiscardEmptyColumns
// Always use tabs for indentation columns (i.e., padding of
// leading empty cells on the left) independent of padchar.
TabIndent
// Print a vertical bar ('|') between columns (after formatting).
// Discarded columns appear as zero-width columns ("||").
Debug
// Remember maximum widths seen per column even after Flush() is called.
RememberWidths
)
// A Writer must be initialized with a call to Init. The first parameter (output)
// specifies the filter output. The remaining parameters control the formatting:
//
// minwidth minimal cell width including any padding
// tabwidth width of tab characters (equivalent number of spaces)
// padding padding added to a cell before computing its width
// padchar ASCII char used for padding
// if padchar == '\t', the Writer will assume that the
// width of a '\t' in the formatted output is tabwidth,
// and cells are left-aligned independent of align_left
// (for correct-looking results, tabwidth must correspond
// to the tab width in the viewer displaying the result)
// flags formatting control
//
func (b *Writer) Init(output io.Writer, minwidth, tabwidth, padding int, padchar byte, flags uint) *Writer {
if minwidth < 0 || tabwidth < 0 || padding < 0 {
panic("negative minwidth, tabwidth, or padding")
}
b.output = output
b.minwidth = minwidth
b.tabwidth = tabwidth
b.padding = padding
for i := range b.padbytes {
b.padbytes[i] = padchar
}
if padchar == '\t' {
// tab padding enforces left-alignment
flags &^= AlignRight
}
b.flags = flags
b.reset()
return b
}
// debugging support (keep code around)
func (b *Writer) dump() {
pos := 0
for i, line := range b.lines {
print("(", i, ") ")
for _, c := range line {
print("[", string(b.buf[pos:pos+c.size]), "]")
pos += c.size
}
print("\n")
}
print("\n")
}
// local error wrapper so we can distinguish errors we want to return
// as errors from genuine panics (which we don't want to return as errors)
type osError struct {
err error
}
func (b *Writer) write0(buf []byte) {
n, err := b.output.Write(buf)
if n != len(buf) && err == nil {
err = io.ErrShortWrite
}
if err != nil {
panic(osError{err})
}
}
func (b *Writer) writeN(src []byte, n int) {
for n > len(src) {
b.write0(src)
n -= len(src)
}
b.write0(src[0:n])
}
var (
newline = []byte{'\n'}
tabs = []byte("\t\t\t\t\t\t\t\t")
)
func (b *Writer) writePadding(textw, cellw int, useTabs bool) {
if b.padbytes[0] == '\t' || useTabs {
// padding is done with tabs
if b.tabwidth == 0 {
return // tabs have no width - can't do any padding
}
// make cellw the smallest multiple of b.tabwidth
cellw = (cellw + b.tabwidth - 1) / b.tabwidth * b.tabwidth
n := cellw - textw // amount of padding
if n < 0 {
panic("internal error")
}
b.writeN(tabs, (n+b.tabwidth-1)/b.tabwidth)
return
}
// padding is done with non-tab characters
b.writeN(b.padbytes[0:], cellw-textw)
}
var vbar = []byte{'|'}
func (b *Writer) writeLines(pos0 int, line0, line1 int) (pos int) {
pos = pos0
for i := line0; i < line1; i++ {
line := b.lines[i]
// if TabIndent is set, use tabs to pad leading empty cells
useTabs := b.flags&TabIndent != 0
for j, c := range line {
if j > 0 && b.flags&Debug != 0 {
// indicate column break
b.write0(vbar)
}
if c.size == 0 {
// empty cell
if j < len(b.widths) {
b.writePadding(c.width, b.widths[j], useTabs)
}
} else {
// non-empty cell
useTabs = false
if b.flags&AlignRight == 0 { // align left
b.write0(b.buf[pos : pos+c.size])
pos += c.size
if j < len(b.widths) {
b.writePadding(c.width, b.widths[j], false)
}
} else { // align right
if j < len(b.widths) {
b.writePadding(c.width, b.widths[j], false)
}
b.write0(b.buf[pos : pos+c.size])
pos += c.size
}
}
}
if i+1 == len(b.lines) {
// last buffered line - we don't have a newline, so just write
// any outstanding buffered data
b.write0(b.buf[pos : pos+b.cell.size])
pos += b.cell.size
} else {
// not the last line - write newline
b.write0(newline)
}
}
return
}
// Format the text between line0 and line1 (excluding line1); pos
// is the buffer position corresponding to the beginning of line0.
// Returns the buffer position corresponding to the beginning of
// line1 and an error, if any.
//
func (b *Writer) format(pos0 int, line0, line1 int) (pos int) {
pos = pos0
column := len(b.widths)
for this := line0; this < line1; this++ {
line := b.lines[this]
if column >= len(line)-1 {
continue
}
// cell exists in this column => this line
// has more cells than the previous line
// (the last cell per line is ignored because cells are
// tab-terminated; the last cell per line describes the
// text before the newline/formfeed and does not belong
// to a column)
// print unprinted lines until beginning of block
pos = b.writeLines(pos, line0, this)
line0 = this
// column block begin
width := b.minwidth // minimal column width
discardable := true // true if all cells in this column are empty and "soft"
for ; this < line1; this++ {
line = b.lines[this]
if column >= len(line)-1 {
break
}
// cell exists in this column
c := line[column]
// update width
if w := c.width + b.padding; w > width {
width = w
}
// update discardable
if c.width > 0 || c.htab {
discardable = false
}
}
// column block end
// discard empty columns if necessary
if discardable && b.flags&DiscardEmptyColumns != 0 {
width = 0
}
if b.flags&RememberWidths != 0 {
if len(b.maxwidths) < len(b.widths) {
b.maxwidths = append(b.maxwidths, b.widths[len(b.maxwidths):]...)
}
switch {
case len(b.maxwidths) == len(b.widths):
b.maxwidths = append(b.maxwidths, width)
case b.maxwidths[len(b.widths)] > width:
width = b.maxwidths[len(b.widths)]
case b.maxwidths[len(b.widths)] < width:
b.maxwidths[len(b.widths)] = width
}
}
// format and print all columns to the right of this column
// (we know the widths of this column and all columns to the left)
b.widths = append(b.widths, width) // push width
pos = b.format(pos, line0, this)
b.widths = b.widths[0 : len(b.widths)-1] // pop width
line0 = this
}
// print unprinted lines until end
return b.writeLines(pos, line0, line1)
}
// Append text to current cell.
func (b *Writer) append(text []byte) {
b.buf = append(b.buf, text...)
b.cell.size += len(text)
}
// Update the cell width.
func (b *Writer) updateWidth() {
b.cell.width += utf8.RuneCount(b.buf[b.pos:])
b.pos = len(b.buf)
}
// To escape a text segment, bracket it with Escape characters.
// For instance, the tab in this string "Ignore this tab: \xff\t\xff"
// does not terminate a cell and constitutes a single character of
// width one for formatting purposes.
//
// The value 0xff was chosen because it cannot appear in a valid UTF-8 sequence.
//
const Escape = '\xff'
// Start escaped mode.
func (b *Writer) startEscape(ch byte) {
switch ch {
case Escape:
b.endChar = Escape
case '<':
b.endChar = '>'
case '&':
b.endChar = ';'
}
}
// Terminate escaped mode. If the escaped text was an HTML tag, its width
// is assumed to be zero for formatting purposes; if it was an HTML entity,
// its width is assumed to be one. In all other cases, the width is the
// unicode width of the text.
//
func (b *Writer) endEscape() {
switch b.endChar {
case Escape:
b.updateWidth()
if b.flags&StripEscape == 0 {
b.cell.width -= 2 // don't count the Escape chars
}
case '>': // tag of zero width
case ';':
b.cell.width++ // entity, count as one rune
}
b.pos = len(b.buf)
b.endChar = 0
}
// Terminate the current cell by adding it to the list of cells of the
// current line. Returns the number of cells in that line.
//
func (b *Writer) terminateCell(htab bool) int {
b.cell.htab = htab
line := &b.lines[len(b.lines)-1]
*line = append(*line, b.cell)
b.cell = cell{}
return len(*line)
}
func handlePanic(err *error, op string) {
if e := recover(); e != nil {
if nerr, ok := e.(osError); ok {
*err = nerr.err
return
}
panic("tabwriter: panic during " + op)
}
}
// RememberedWidths returns a copy of the remembered per-column maximum widths.
// Requires use of the RememberWidths flag, and is not threadsafe.
func (b *Writer) RememberedWidths() []int {
retval := make([]int, len(b.maxwidths))
copy(retval, b.maxwidths)
return retval
}
// SetRememberedWidths sets the remembered per-column maximum widths.
// Requires use of the RememberWidths flag, and is not threadsafe.
func (b *Writer) SetRememberedWidths(widths []int) *Writer {
b.maxwidths = make([]int, len(widths))
copy(b.maxwidths, widths)
return b
}
// Flush should be called after the last call to Write to ensure
// that any data buffered in the Writer is written to output. Any
// incomplete escape sequence at the end is considered
// complete for formatting purposes.
func (b *Writer) Flush() error {
return b.flush()
}
func (b *Writer) flush() (err error) {
defer b.reset() // even in the presence of errors
defer handlePanic(&err, "Flush")
// add current cell if not empty
if b.cell.size > 0 {
if b.endChar != 0 {
// inside escape - terminate it even if incomplete
b.endEscape()
}
b.terminateCell(false)
}
// format contents of buffer
b.format(0, 0, len(b.lines))
return nil
}
var hbar = []byte("---\n")
// Write writes buf to the writer b.
// The only errors returned are ones encountered
// while writing to the underlying output stream.
//
func (b *Writer) Write(buf []byte) (n int, err error) {
defer handlePanic(&err, "Write")
// split text into cells
n = 0
for i, ch := range buf {
if b.endChar == 0 {
// outside escape
switch ch {
case '\t', '\v', '\n', '\f':
// end of cell
b.append(buf[n:i])
b.updateWidth()
n = i + 1 // ch consumed
ncells := b.terminateCell(ch == '\t')
if ch == '\n' || ch == '\f' {
// terminate line
b.addLine(ch == '\f')
if ch == '\f' || ncells == 1 {
// A '\f' always forces a flush. Otherwise, if the previous
// line has only one cell which does not have an impact on
// the formatting of the following lines (the last cell per
// line is ignored by format()), thus we can flush the
// Writer contents.
if err = b.Flush(); err != nil {
return
}
if ch == '\f' && b.flags&Debug != 0 {
// indicate section break
b.write0(hbar)
}
}
}
case Escape:
// start of escaped sequence
b.append(buf[n:i])
b.updateWidth()
n = i
if b.flags&StripEscape != 0 {
n++ // strip Escape
}
b.startEscape(Escape)
case '<', '&':
// possibly an html tag/entity
if b.flags&FilterHTML != 0 {
// begin of tag/entity
b.append(buf[n:i])
b.updateWidth()
n = i
b.startEscape(ch)
}
}
} else {
// inside escape
if ch == b.endChar {
// end of tag/entity
j := i + 1
if ch == Escape && b.flags&StripEscape != 0 {
j = i // strip Escape
}
b.append(buf[n:j])
n = i + 1 // ch consumed
b.endEscape()
}
}
}
// append leftover text
b.append(buf[n:])
n = len(buf)
return
}
// NewWriter allocates and initializes a new tabwriter.Writer.
// The parameters are the same as for the Init function.
//
func NewWriter(output io.Writer, minwidth, tabwidth, padding int, padchar byte, flags uint) *Writer {
return new(Writer).Init(output, minwidth, tabwidth, padding, padchar, flags)
}

View file

@ -1,24 +0,0 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

View file

@ -1,23 +0,0 @@
language: go
go:
- 1.7.3
- tip
env:
- GO15VENDOREXPERIMENT=1
matrix:
allow_failures:
- go: tip
before_install:
- go get -u github.com/axw/gocov/gocov
- go get -u github.com/mattn/goveralls
- go get golang.org/x/tools/cmd/cover
script:
- make build
- make test
- make cover
- goveralls -service=travis-ci -v -covermode=count -coverprofile=profile.out

View file

@ -1,22 +0,0 @@
The MIT License (MIT)
Copyright (c) 2015 Manfred Touron
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View file

@ -1,50 +0,0 @@
# Project-specific variables
CONVEY_PORT ?= 9042
# Common variables
SOURCES := $(shell find . -type f -name "*.go")
COMMANDS := $(shell go list ./... | grep -v /vendor/ | grep /cmd/)
PACKAGES := $(shell go list ./... | grep -v /vendor/ | grep -v /cmd/)
GOENV ?= GO15VENDOREXPERIMENT=1
GO ?= $(GOENV) go
GODEP ?= $(GOENV) godep
USER ?= $(shell whoami)
all: build
.PHONY: build
build:
echo "nothing to do"
.PHONY: test
test:
$(GO) get -t .
$(GO) test -v .
.PHONY: godep-save
godep-save:
$(GODEP) save $(PACKAGES) $(COMMANDS)
.PHONY: re
re: clean all
.PHONY: convey
convey:
$(GO) get github.com/smartystreets/goconvey
goconvey -cover -port=$(CONVEY_PORT) -workDir="$(realpath .)" -depth=1
.PHONY: cover
cover: profile.out
profile.out: $(SOURCES)
rm -f $@
$(GO) test -covermode=count -coverpkg=. -coverprofile=$@ .

View file

@ -1,42 +0,0 @@
# http2curl
:triangular_ruler: Convert Golang's http.Request to CURL command line
[![Build Status](https://travis-ci.org/moul/http2curl.svg?branch=master)](https://travis-ci.org/moul/http2curl)
[![GoDoc](https://godoc.org/github.com/moul/http2curl?status.svg)](https://godoc.org/github.com/moul/http2curl)
[![Coverage Status](https://coveralls.io/repos/moul/http2curl/badge.svg)](https://coveralls.io/github/moul/http2curl)
To do the reverse, check out [mholt/curl-to-go](https://github.com/mholt/curl-to-go).
## Example
```go
import "http"
import "github.com/moul/http2curl"
data := bytes.NewBufferString(`{"hello":"world","answer":42}`)
req, _ := http.NewRequest("PUT", "http://www.example.com/abc/def.ghi?jlk=mno&pqr=stu", data)
req.Header.Set("Content-Type", "application/json")
command, _ := http2curl.GetCurlCommand(req)
fmt.Println(command)
// Output: curl -X PUT -d "{\"hello\":\"world\",\"answer\":42}" -H "Content-Type: application/json" http://www.example.com/abc/def.ghi?jlk=mno&pqr=stu
```
## Install
```php
$ go get github.com/moul/http2curl
```
## Usages
- https://github.com/parnurzeal/gorequest
- https://github.com/scaleway/scaleway-cli
- https://github.com/nmonterroso/cowsay-slackapp
- https://github.com/moul/as-a-service
- https://github.com/gavv/httpexpect
- https://github.com/smallnest/goreq
## License
MIT

View file

@ -1,71 +0,0 @@
package http2curl
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"sort"
"strings"
)
// CurlCommand contains exec.Command compatible slice + helpers
type CurlCommand struct {
slice []string
}
// append appends a string to the CurlCommand
func (c *CurlCommand) append(newSlice ...string) {
c.slice = append(c.slice, newSlice...)
}
// String returns a ready to copy/paste command
func (c *CurlCommand) String() string {
return strings.Join(c.slice, " ")
}
// nopCloser is used to create a new io.ReadCloser for req.Body
type nopCloser struct {
io.Reader
}
func bashEscape(str string) string {
return `'` + strings.Replace(str, `'`, `'\''`, -1) + `'`
}
func (nopCloser) Close() error { return nil }
// GetCurlCommand returns a CurlCommand corresponding to an http.Request
func GetCurlCommand(req *http.Request) (*CurlCommand, error) {
command := CurlCommand{}
command.append("curl")
command.append("-X", bashEscape(req.Method))
if req.Body != nil {
body, err := ioutil.ReadAll(req.Body)
if err != nil {
return nil, err
}
req.Body = nopCloser{bytes.NewBuffer(body)}
bodyEscaped := bashEscape(string(body))
command.append("-d", bodyEscaped)
}
var keys []string
for k := range req.Header {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
command.append("-H", bashEscape(fmt.Sprintf("%s: %s", k, strings.Join(req.Header[k], " "))))
}
command.append(bashEscape(req.URL.String()))
return &command, nil
}

View file

@ -11,6 +11,8 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
units "github.com/docker/go-units"
@ -22,6 +24,11 @@ const (
CgroupProcesses = "cgroup.procs"
)
var (
isUnifiedOnce sync.Once
isUnified bool
)
// HugePageSizeUnitList is a list of the units used by the linux kernel when
// naming the HugePage control files.
// https://www.kernel.org/doc/Documentation/cgroup-v1/hugetlb.txt
@ -29,6 +36,18 @@ const (
// depends on https://github.com/docker/go-units/commit/a09cd47f892041a4fac473133d181f5aea6fa393
var HugePageSizeUnitList = []string{"B", "KB", "MB", "GB", "TB", "PB"}
// IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode.
func IsCgroup2UnifiedMode() bool {
isUnifiedOnce.Do(func() {
var st syscall.Statfs_t
if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil {
panic("cannot statfs cgroup root")
}
isUnified = st.Type == unix.CGROUP2_SUPER_MAGIC
})
return isUnified
}
// https://www.kernel.org/doc/Documentation/cgroup-v1/cgroups.txt
func FindCgroupMountpoint(cgroupPath, subsystem string) (string, error) {
mnt, _, err := FindCgroupMountpointAndRoot(cgroupPath, subsystem)
@ -49,6 +68,10 @@ func FindCgroupMountpointAndRoot(cgroupPath, subsystem string) (string, string,
}
defer f.Close()
if IsCgroup2UnifiedMode() {
subsystem = ""
}
return findCgroupMountpointAndRootFromReader(f, cgroupPath, subsystem)
}
@ -57,12 +80,12 @@ func findCgroupMountpointAndRootFromReader(reader io.Reader, cgroupPath, subsyst
for scanner.Scan() {
txt := scanner.Text()
fields := strings.Fields(txt)
if len(fields) < 5 {
if len(fields) < 9 {
continue
}
if strings.HasPrefix(fields[4], cgroupPath) {
for _, opt := range strings.Split(fields[len(fields)-1], ",") {
if opt == subsystem {
if (subsystem == "" && fields[9] == "cgroup2") || opt == subsystem {
return fields[4], fields[3], nil
}
}
@ -76,6 +99,19 @@ func findCgroupMountpointAndRootFromReader(reader io.Reader, cgroupPath, subsyst
}
func isSubsystemAvailable(subsystem string) bool {
if IsCgroup2UnifiedMode() {
controllers, err := GetAllSubsystems()
if err != nil {
return false
}
for _, c := range controllers {
if c == subsystem {
return true
}
}
return false
}
cgroups, err := ParseCgroupFile("/proc/self/cgroup")
if err != nil {
return false
@ -120,7 +156,7 @@ func FindCgroupMountpointDir() (string, error) {
return "", fmt.Errorf("Found no fields post '-' in %q", text)
}
if postSeparatorFields[0] == "cgroup" {
if postSeparatorFields[0] == "cgroup" || postSeparatorFields[0] == "cgroup2" {
// Check that the mount is properly formatted.
if numPostFields < 3 {
return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text)
@ -193,6 +229,19 @@ func getCgroupMountsHelper(ss map[string]bool, mi io.Reader, all bool) ([]Mount,
// GetCgroupMounts returns the mounts for the cgroup subsystems.
// all indicates whether to return just the first instance or all the mounts.
func GetCgroupMounts(all bool) ([]Mount, error) {
if IsCgroup2UnifiedMode() {
availableControllers, err := GetAllSubsystems()
if err != nil {
return nil, err
}
m := Mount{
Mountpoint: "/sys/fs/cgroup",
Root: "/sys/fs/cgroup",
Subsystems: availableControllers,
}
return []Mount{m}, nil
}
f, err := os.Open("/proc/self/mountinfo")
if err != nil {
return nil, err
@ -356,6 +405,9 @@ func parseCgroupFromReader(r io.Reader) (map[string]string, error) {
}
func getControllerPath(subsystem string, cgroups map[string]string) (string, error) {
if IsCgroup2UnifiedMode() {
return "/", nil
}
if p, ok := cgroups[subsystem]; ok {
return p, nil

View file

@ -59,3 +59,8 @@ func NewThrottleDevice(major, minor int64, rate uint64) *ThrottleDevice {
func (td *ThrottleDevice) String() string {
return fmt.Sprintf("%d:%d %d", td.Major, td.Minor, td.Rate)
}
// StringName formats the struct to be writable to the cgroup specific file
func (td *ThrottleDevice) StringName(name string) string {
return fmt.Sprintf("%d:%d %s=%d", td.Major, td.Minor, name, td.Rate)
}

View file

@ -119,4 +119,12 @@ type Resources struct {
// Set class identifier for container's network packets
NetClsClassid uint32 `json:"net_cls_classid_u"`
// Used on cgroups v2:
// CpuWeight sets a proportional bandwidth limit.
CpuWeight uint64 `json:"cpu_weight"`
// CpuMax sets she maximum bandwidth limit (format: max period).
CpuMax string `json:"cpu_max"`
}

View file

@ -1,3 +1,5 @@
// +build !linux
package configs
// TODO Windows: This can ultimately be entirely factored out on Windows as

View file

@ -44,6 +44,7 @@ const (
Trap
Allow
Trace
Log
)
// Operator is a comparison operator to be used when matching syscall arguments in Seccomp

View file

@ -4,7 +4,7 @@ import "os"
// Spec is the base configuration for the container.
type Spec struct {
// Version of the Open Container Runtime Specification with which the bundle complies.
// Version of the Open Container Initiative Runtime Specification with which the bundle complies.
Version string `json:"ociVersion"`
// Process configures the container process.
Process *Process `json:"process,omitempty"`

View file

@ -8,7 +8,7 @@ const (
// VersionMinor is for functionality in a backwards-compatible manner
VersionMinor = 0
// VersionPatch is for backwards-compatible bug fixes
VersionPatch = 0
VersionPatch = 1
// VersionDev indicates development branch. Releases will be empty string.
VersionDev = ""

View file

@ -7,7 +7,7 @@ GoRequest -- Simplified HTTP client ( inspired by famous SuperAgent lib in Node.
#### "Shooting Requests like a Machine Gun" - Gopher
Sending request would never been fun and easier than this. It comes with lots of feature:
Sending request has never been as fun nor easier than this. It comes with lots of features:
* Get/Post/Put/Head/Delete/Patch/Options
* Set - simple header setting
@ -59,7 +59,7 @@ Or below if you don't want to reuse it for other requests.
resp, body, errs := gorequest.New().Get("http://example.com/").End()
```
How about getting control over HTTP client headers, redirect policy, and etc. Things is getting more complicated in golang. You need to create a Client, setting header in different command, ... to do just only one __GET__
How about getting control over HTTP client headers, redirect policy, and etc. Things can quickly get more complicated in golang. You need to create a Client, set headers in a different command, ... just to do only one __GET__
```go
client := &http.Client{
@ -72,7 +72,7 @@ req.Header.Add("If-None-Match", `W/"wyzzy"`)
resp, err := client.Do(req)
```
Why making things ugly while you can just do as follows:
Why make things ugly while you can just do it as follows:
```go
request := gorequest.New()
@ -82,7 +82,7 @@ resp, body, errs := request.Get("http://example.com").
End()
```
__DELETE__, __HEAD__, __POST__, __PUT__, __PATCH__ are now supported and can be used the same way as __GET__:
__DELETE__, __HEAD__, __POST__, __PUT__, __PATCH__ are now supported and can be used in the same way as __GET__:
```go
request := gorequest.New()
@ -95,7 +95,7 @@ resp, body, errs := request.Post("http://example.com").End()
### JSON
For a __JSON POST__ with standard libraries, you might need to marshal map data structure to json format, setting header to 'application/json' (and other headers if you need to) and declare http.Client. So, you code become longer and hard to maintain:
For a __JSON POST__ with standard libraries, you might need to marshal map data structure to json format, set headers to 'application/json' (and other headers if you need to) and declare http.Client. So, your code becomes longer and harder to maintain:
```go
m := map[string]interface{}{
@ -242,7 +242,7 @@ Supposing you need retry 3 times, with 5 seconds between each attempt when gets
```go
request := gorequest.New()
resp, body, errs := request.Get("http://example.com/").
Retry(3, 5 * time.seconds, http.StatusBadRequest, http.StatusInternalServerError).
Retry(3, 5 * time.Second, http.StatusBadRequest, http.StatusInternalServerError).
End()
```
@ -268,6 +268,23 @@ resp, body, errs := request.Get("http://example.com/").
End()
```
## Clone
You can reuse settings of a Request by cloning it _before_ making any requests. This can be useful if you wish to re-use the SuperAgent across multiple requests without worrying about concurrency or having too many Transports being created.
Clones will copy the same settings (headers, query, etc..), but will only shallow copy any "Data" given to it. They will also share the same Transport and http.Client.
```go
baseRequest := gorequest.New()
// apply anything you want to these settings. Eg:
baseRequest.Timeout(10 * time.Millisecond).
BasicAuth("user", "password")
// then reuse the base request elsewhere, cloning before modifying or using it.
resp, body, errs := baseRequest.Clone().Get("http://exmaple.com/").End()
```
## Debug
For debugging, GoRequest leverages `httputil` to dump details of every request/response. (Thanks to @dafang)

View file

@ -5,31 +5,26 @@ import (
"bytes"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"mime/multipart"
"net/http"
"net/http/cookiejar"
"net/http/httputil"
"net/textproto"
"net/url"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"mime/multipart"
"net/textproto"
"fmt"
"path/filepath"
"github.com/moul/http2curl"
"golang.org/x/net/publicsuffix"
"moul.io/http2curl"
)
type Request *http.Request
@ -46,35 +41,51 @@ const (
OPTIONS = "OPTIONS"
)
// Types we support.
const (
TypeJSON = "json"
TypeXML = "xml"
TypeUrlencoded = "urlencoded"
TypeForm = "form"
TypeFormData = "form-data"
TypeHTML = "html"
TypeText = "text"
TypeMultipart = "multipart"
)
type superAgentRetryable struct {
RetryableStatus []int
RetryerTime time.Duration
RetryerCount int
Attempt int
Enable bool
}
// A SuperAgent is a object storing all request data for client.
type SuperAgent struct {
Url string
Method string
Header map[string]string
TargetType string
ForceType string
Data map[string]interface{}
SliceData []interface{}
FormData url.Values
QueryData url.Values
FileData []File
BounceToRawString bool
RawString string
Client *http.Client
Transport *http.Transport
Cookies []*http.Cookie
Errors []error
BasicAuth struct{ Username, Password string }
Debug bool
CurlCommand bool
logger *log.Logger
Retryable struct {
RetryableStatus []int
RetryerTime time.Duration
RetryerCount int
Attempt int
Enable bool
}
Url string
Method string
Header http.Header
TargetType string
ForceType string
Data map[string]interface{}
SliceData []interface{}
FormData url.Values
QueryData url.Values
FileData []File
BounceToRawString bool
RawString string
Client *http.Client
Transport *http.Transport
Cookies []*http.Cookie
Errors []error
BasicAuth struct{ Username, Password string }
Debug bool
CurlCommand bool
logger Logger
Retryable superAgentRetryable
DoNotClearSuperAgent bool
isClone bool
}
var DisableTransportSwap = false
@ -89,9 +100,9 @@ func New() *SuperAgent {
debug := os.Getenv("GOREQUEST_DEBUG") == "1"
s := &SuperAgent{
TargetType: "json",
TargetType: TypeJSON,
Data: make(map[string]interface{}),
Header: make(map[string]string),
Header: http.Header{},
RawString: "",
SliceData: []interface{}{},
FormData: url.Values{},
@ -106,12 +117,122 @@ func New() *SuperAgent {
Debug: debug,
CurlCommand: false,
logger: log.New(os.Stderr, "[gorequest]", log.LstdFlags),
isClone: false,
}
// disable keep alives by default, see this issue https://github.com/parnurzeal/gorequest/issues/75
s.Transport.DisableKeepAlives = true
return s
}
func cloneMapArray(old map[string][]string) map[string][]string {
newMap := make(map[string][]string, len(old))
for k, vals := range old {
newMap[k] = make([]string, len(vals))
for i := range vals {
newMap[k][i] = vals[i]
}
}
return newMap
}
func shallowCopyData(old map[string]interface{}) map[string]interface{} {
if old == nil {
return nil
}
newData := make(map[string]interface{})
for k, val := range old {
newData[k] = val
}
return newData
}
func shallowCopyDataSlice(old []interface{}) []interface{} {
if old == nil {
return nil
}
newData := make([]interface{}, len(old))
for i := range old {
newData[i] = old[i]
}
return newData
}
func shallowCopyFileArray(old []File) []File {
if old == nil {
return nil
}
newData := make([]File, len(old))
for i := range old {
newData[i] = old[i]
}
return newData
}
func shallowCopyCookies(old []*http.Cookie) []*http.Cookie {
if old == nil {
return nil
}
newData := make([]*http.Cookie, len(old))
for i := range old {
newData[i] = old[i]
}
return newData
}
func shallowCopyErrors(old []error) []error {
if old == nil {
return nil
}
newData := make([]error, len(old))
for i := range old {
newData[i] = old[i]
}
return newData
}
// just need to change the array pointer?
func copyRetryable(old superAgentRetryable) superAgentRetryable {
newRetryable := old
newRetryable.RetryableStatus = make([]int, len(old.RetryableStatus))
for i := range old.RetryableStatus {
newRetryable.RetryableStatus[i] = old.RetryableStatus[i]
}
return newRetryable
}
// Returns a copy of this superagent. Useful if you want to reuse the client/settings
// concurrently.
// Note: This does a shallow copy of the parent. So you will need to be
// careful of Data provided
// Note: It also directly re-uses the client and transport. If you modify the Timeout,
// or RedirectPolicy on a clone, the clone will have a new http.client. It is recommended
// that the base request set your timeout and redirect polices, and no modification of
// the client or transport happen after cloning.
// Note: DoNotClearSuperAgent is forced to "true" after Clone
func (s *SuperAgent) Clone() *SuperAgent {
clone := &SuperAgent{
Url: s.Url,
Method: s.Method,
Header: http.Header(cloneMapArray(s.Header)),
TargetType: s.TargetType,
ForceType: s.ForceType,
Data: shallowCopyData(s.Data),
SliceData: shallowCopyDataSlice(s.SliceData),
FormData: url.Values(cloneMapArray(s.FormData)),
QueryData: url.Values(cloneMapArray(s.QueryData)),
FileData: shallowCopyFileArray(s.FileData),
BounceToRawString: s.BounceToRawString,
RawString: s.RawString,
Client: s.Client,
Transport: s.Transport,
Cookies: shallowCopyCookies(s.Cookies),
Errors: shallowCopyErrors(s.Errors),
BasicAuth: s.BasicAuth,
Debug: s.Debug,
CurlCommand: s.CurlCommand,
logger: s.logger, // thread safe.. anyway
Retryable: copyRetryable(s.Retryable),
DoNotClearSuperAgent: true,
isClone: true,
}
return clone
}
// Enable the debug mode which logs request/response detail
func (s *SuperAgent) SetDebug(enable bool) *SuperAgent {
s.Debug = enable
@ -124,16 +245,25 @@ func (s *SuperAgent) SetCurlCommand(enable bool) *SuperAgent {
return s
}
func (s *SuperAgent) SetLogger(logger *log.Logger) *SuperAgent {
// Enable the DoNotClear mode for not clearing super agent and reuse for the next request
func (s *SuperAgent) SetDoNotClearSuperAgent(enable bool) *SuperAgent {
s.DoNotClearSuperAgent = enable
return s
}
func (s *SuperAgent) SetLogger(logger Logger) *SuperAgent {
s.logger = logger
return s
}
// Clear SuperAgent data for another new request.
func (s *SuperAgent) ClearSuperAgent() {
if s.DoNotClearSuperAgent {
return
}
s.Url = ""
s.Method = ""
s.Header = make(map[string]string)
s.Header = http.Header{}
s.Data = make(map[string]interface{})
s.SliceData = []interface{}{}
s.FormData = url.Values{}
@ -142,7 +272,7 @@ func (s *SuperAgent) ClearSuperAgent() {
s.BounceToRawString = false
s.RawString = ""
s.ForceType = ""
s.TargetType = "json"
s.TargetType = TypeJSON
s.Cookies = make([]*http.Cookie, 0)
s.Errors = nil
}
@ -229,7 +359,8 @@ func (s *SuperAgent) Options(targetUrl string) *SuperAgent {
return s
}
// Set is used for setting header fields.
// Set is used for setting header fields,
// this will overwrite the existed values of Header through AppendHeader().
// Example. To set `Accept` as `application/json`
//
// gorequest.New().
@ -237,7 +368,20 @@ func (s *SuperAgent) Options(targetUrl string) *SuperAgent {
// Set("Accept", "application/json").
// End()
func (s *SuperAgent) Set(param string, value string) *SuperAgent {
s.Header[param] = value
s.Header.Set(param, value)
return s
}
// AppendHeader is used for setting header fileds with multiple values,
// Example. To set `Accept` as `application/json, text/plain`
//
// gorequest.New().
// Post("/gamelist").
// AppendHeader("Accept", "application/json").
// AppendHeader("Accept", "text/plain").
// End()
func (s *SuperAgent) AppendHeader(param string, value string) *SuperAgent {
s.Header.Add(param, value)
return s
}
@ -299,14 +443,14 @@ func (s *SuperAgent) AddCookies(cookies []*http.Cookie) *SuperAgent {
}
var Types = map[string]string{
"html": "text/html",
"json": "application/json",
"xml": "application/xml",
"text": "text/plain",
"urlencoded": "application/x-www-form-urlencoded",
"form": "application/x-www-form-urlencoded",
"form-data": "application/x-www-form-urlencoded",
"multipart": "multipart/form-data",
TypeJSON: "application/json",
TypeXML: "application/xml",
TypeForm: "application/x-www-form-urlencoded",
TypeFormData: "application/x-www-form-urlencoded",
TypeUrlencoded: "application/x-www-form-urlencoded",
TypeHTML: "text/html",
TypeText: "text/plain",
TypeMultipart: "multipart/form-data",
}
// Type is a convenience function to specify the data type to send.
@ -448,19 +592,6 @@ func (s *SuperAgent) Param(key string, value string) *SuperAgent {
return s
}
func (s *SuperAgent) Timeout(timeout time.Duration) *SuperAgent {
s.Transport.Dial = func(network, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(network, addr, timeout)
if err != nil {
s.Errors = append(s.Errors, err)
return nil, err
}
conn.SetDeadline(time.Now().Add(timeout))
return conn, nil
}
return s
}
// Set TLSClientConfig for underling Transport.
// One example is you can use it to disable security check (https):
//
@ -469,6 +600,7 @@ func (s *SuperAgent) Timeout(timeout time.Duration) *SuperAgent {
// End()
//
func (s *SuperAgent) TLSClientConfig(config *tls.Config) *SuperAgent {
s.safeModifyTransport()
s.Transport.TLSClientConfig = config
return s
}
@ -494,8 +626,10 @@ func (s *SuperAgent) Proxy(proxyUrl string) *SuperAgent {
if err != nil {
s.Errors = append(s.Errors, err)
} else if proxyUrl == "" {
s.safeModifyTransport()
s.Transport.Proxy = nil
} else {
s.safeModifyTransport()
s.Transport.Proxy = http.ProxyURL(parsedProxyUrl)
}
return s
@ -508,6 +642,7 @@ func (s *SuperAgent) Proxy(proxyUrl string) *SuperAgent {
// The policy function's arguments are the Request about to be made and the
// past requests in order of oldest first.
func (s *SuperAgent) RedirectPolicy(policy func(req Request, via []Request) error) *SuperAgent {
s.safeModifyHttpClient()
s.Client.CheckRedirect = func(r *http.Request, v []*http.Request) error {
vv := make([]Request, len(v))
for i, r := range v {
@ -683,7 +818,7 @@ func (s *SuperAgent) SendString(content string) *SuperAgent {
}
}
}
s.TargetType = "form"
s.TargetType = TypeForm
} else {
s.BounceToRawString = true
}
@ -905,7 +1040,7 @@ func changeMapToURLValues(data map[string]interface{}) url.Values {
// For example:
//
// resp, body, errs := gorequest.New().Get("http://www.google.com").End()
// if (errs != nil) {
// if errs != nil {
// fmt.Println(errs)
// }
// fmt.Println(resp, body)
@ -1010,13 +1145,14 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) {
}
// check if there is forced type
switch s.ForceType {
case "json", "form", "xml", "text", "multipart":
case TypeJSON, TypeForm, TypeXML, TypeText, TypeMultipart:
s.TargetType = s.ForceType
// If forcetype is not set, check whether user set Content-Type header.
// If yes, also bounce to the correct supported TargetType automatically.
default:
contentType := s.Header.Get("Content-Type")
for k, v := range Types {
if s.Header["Content-Type"] == v {
if contentType == v {
s.TargetType = k
}
}
@ -1079,24 +1215,41 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) {
}
}
body, _ := ioutil.ReadAll(resp.Body)
body, err := ioutil.ReadAll(resp.Body)
// Reset resp.Body so it can be use again
resp.Body = ioutil.NopCloser(bytes.NewBuffer(body))
if err != nil {
return nil, nil, []error{err}
}
return resp, body, nil
}
func (s *SuperAgent) MakeRequest() (*http.Request, error) {
var (
req *http.Request
err error
req *http.Request
contentType string // This is only set when the request body content is non-empty.
contentReader io.Reader
err error
)
if s.Method == "" {
return nil, errors.New("No method specified")
}
if s.TargetType == "json" {
// !!! Important Note !!!
//
// Throughout this region, contentReader and contentType are only set when
// the contents will be non-empty.
// This is done avoid ever sending a non-nil request body with nil contents
// to http.NewRequest, because it contains logic which dependends on
// whether or not the body is "nil".
//
// See PR #136 for more information:
//
// https://github.com/parnurzeal/gorequest/pull/136
//
switch s.TargetType {
case TypeJSON:
// If-case to give support to json array. we check if
// 1) Map only: send it as json map from s.Data
// 2) Array or Mix of map & array or others: send it as rawstring from s.RawString
@ -1108,13 +1261,11 @@ func (s *SuperAgent) MakeRequest() (*http.Request, error) {
} else if len(s.SliceData) != 0 {
contentJson, _ = json.Marshal(s.SliceData)
}
contentReader := bytes.NewReader(contentJson)
req, err = http.NewRequest(s.Method, s.Url, contentReader)
if err != nil {
return nil, err
if contentJson != nil {
contentReader = bytes.NewReader(contentJson)
contentType = "application/json"
}
req.Header.Set("Content-Type", "application/json")
} else if s.TargetType == "form" || s.TargetType == "form-data" || s.TargetType == "urlencoded" {
case TypeForm, TypeFormData, TypeUrlencoded:
var contentForm []byte
if s.BounceToRawString || len(s.SliceData) != 0 {
contentForm = []byte(s.RawString)
@ -1122,30 +1273,34 @@ func (s *SuperAgent) MakeRequest() (*http.Request, error) {
formData := changeMapToURLValues(s.Data)
contentForm = []byte(formData.Encode())
}
contentReader := bytes.NewReader(contentForm)
req, err = http.NewRequest(s.Method, s.Url, contentReader)
if err != nil {
return nil, err
if len(contentForm) != 0 {
contentReader = bytes.NewReader(contentForm)
contentType = "application/x-www-form-urlencoded"
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
} else if s.TargetType == "text" {
req, err = http.NewRequest(s.Method, s.Url, strings.NewReader(s.RawString))
req.Header.Set("Content-Type", "text/plain")
} else if s.TargetType == "xml" {
req, err = http.NewRequest(s.Method, s.Url, strings.NewReader(s.RawString))
req.Header.Set("Content-Type", "application/xml")
} else if s.TargetType == "multipart" {
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
case TypeText:
if len(s.RawString) != 0 {
contentReader = strings.NewReader(s.RawString)
contentType = "text/plain"
}
case TypeXML:
if len(s.RawString) != 0 {
contentReader = strings.NewReader(s.RawString)
contentType = "application/xml"
}
case TypeMultipart:
var (
buf = &bytes.Buffer{}
mw = multipart.NewWriter(buf)
)
if s.BounceToRawString {
fieldName, ok := s.Header["data_fieldname"]
if !ok {
fieldName := s.Header.Get("data_fieldname")
if fieldName == "" {
fieldName = "data"
}
fw, _ := mw.CreateFormField(fieldName)
fw.Write([]byte(s.RawString))
contentReader = buf
}
if len(s.Data) != 0 {
@ -1156,11 +1311,12 @@ func (s *SuperAgent) MakeRequest() (*http.Request, error) {
fw.Write([]byte(value))
}
}
contentReader = buf
}
if len(s.SliceData) != 0 {
fieldName, ok := s.Header["json_fieldname"]
if !ok {
fieldName := s.Header.Get("json_fieldname")
if fieldName == "" {
fieldName = "data"
}
// copied from CreateFormField() in mime/multipart/writer.go
@ -1174,6 +1330,7 @@ func (s *SuperAgent) MakeRequest() (*http.Request, error) {
return nil, err
}
fw.Write(contentJson)
contentReader = buf
}
// add the files
@ -1182,25 +1339,40 @@ func (s *SuperAgent) MakeRequest() (*http.Request, error) {
fw, _ := mw.CreateFormFile(file.Fieldname, file.Filename)
fw.Write(file.Data)
}
contentReader = buf
}
// close before call to FormDataContentType ! otherwise its not valid multipart
mw.Close()
req, err = http.NewRequest(s.Method, s.Url, &buf)
req.Header.Set("Content-Type", mw.FormDataContentType())
} else {
if contentReader != nil {
contentType = mw.FormDataContentType()
}
default:
// let's return an error instead of an nil pointer exception here
return nil, errors.New("TargetType '" + s.TargetType + "' could not be determined")
}
for k, v := range s.Header {
req.Header.Set(k, v)
// Setting the host header is a special case, see this issue: https://github.com/golang/go/issues/7682
if strings.EqualFold(k, "host") {
req.Host = v
if req, err = http.NewRequest(s.Method, s.Url, contentReader); err != nil {
return nil, err
}
for k, vals := range s.Header {
for _, v := range vals {
req.Header.Add(k, v)
}
// Setting the Host header is a special case, see this issue: https://github.com/golang/go/issues/7682
if strings.EqualFold(k, "Host") {
req.Host = vals[0]
}
}
// https://github.com/parnurzeal/gorequest/issues/164
// Don't infer the content type header if an overrride is already provided.
if len(contentType) != 0 && req.Header.Get("Content-Type") == "" {
req.Header.Set("Content-Type", contentType)
}
// Add all querystring from Query func
q := req.URL.Query()
for k, v := range s.QueryData {

View file

@ -0,0 +1,38 @@
// +build go1.2
// +build !go1.3
package gorequest
import (
"net"
"net/http"
"time"
)
// we don't want to mess up other clones when we modify the client..
// so unfortantely we need to create a new client
func (s *SuperAgent) safeModifyHttpClient() {
if !s.isClone {
return
}
oldClient := s.Client
s.Client = &http.Client{}
s.Client.Jar = oldClient.Jar
s.Client.Transport = oldClient.Transport
s.Client.CheckRedirect = oldClient.CheckRedirect
}
// I'm not sure how this func will work with Clone.
func (s *SuperAgent) Timeout(timeout time.Duration) *SuperAgent {
s.Transport.Dial = func(network, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(network, addr, timeout)
if err != nil {
s.Errors = append(s.Errors, err)
return nil, err
}
conn.SetDeadline(time.Now().Add(timeout))
return conn, nil
}
return s
}

View file

@ -0,0 +1,30 @@
// +build go1.3
package gorequest
import (
"time"
"net/http"
)
// we don't want to mess up other clones when we modify the client..
// so unfortantely we need to create a new client
func (s *SuperAgent) safeModifyHttpClient() {
if !s.isClone {
return
}
oldClient := s.Client
s.Client = &http.Client{}
s.Client.Jar = oldClient.Jar
s.Client.Transport = oldClient.Transport
s.Client.Timeout = oldClient.Timeout
s.Client.CheckRedirect = oldClient.CheckRedirect
}
func (s *SuperAgent) Timeout(timeout time.Duration) *SuperAgent {
s.safeModifyHttpClient()
s.Client.Timeout = timeout
return s
}

View file

@ -0,0 +1,25 @@
// +build go1.2
// +build !go1.3
package gorequest
import (
"net/http"
)
// does a shallow clone of the transport
func (s *SuperAgent) safeModifyTransport() {
if !s.isClone {
return
}
oldTransport := s.Transport
s.Transport = &http.Transport{
Proxy: oldTransport.Proxy,
Dial: oldTransport.Dial,
TLSClientConfig: oldTransport.TLSClientConfig,
DisableKeepAlives: oldTransport.DisableKeepAlives,
DisableCompression: oldTransport.DisableCompression,
MaxIdleConnsPerHost: oldTransport.MaxIdleConnsPerHost,
ResponseHeaderTimeout: oldTransport.ResponseHeaderTimeout,
}
}

View file

@ -0,0 +1,27 @@
// +build go1.3
// +build !go1.4
package gorequest
import (
"net/http"
)
// does a shallow clone of the transport
func (s *SuperAgent) safeModifyTransport() {
if !s.isClone {
return
}
oldTransport := s.Transport
s.Transport = &http.Transport{
Proxy: oldTransport.Proxy,
Dial: oldTransport.Dial,
TLSClientConfig: oldTransport.TLSClientConfig,
DisableKeepAlives: oldTransport.DisableKeepAlives,
DisableCompression: oldTransport.DisableCompression,
MaxIdleConnsPerHost: oldTransport.MaxIdleConnsPerHost,
ResponseHeaderTimeout: oldTransport.ResponseHeaderTimeout,
// new in 1.3
TLSHandshakeTimeout: oldTransport.TLSHandshakeTimeout,
}
}

View file

@ -0,0 +1,28 @@
// +build go1.4
// +build !go1.6
package gorequest
import (
"net/http"
)
// does a shallow clone of the transport
func (s *SuperAgent) safeModifyTransport() {
if !s.isClone {
return
}
oldTransport := s.Transport
s.Transport = &http.Transport{
Proxy: oldTransport.Proxy,
Dial: oldTransport.Dial,
TLSClientConfig: oldTransport.TLSClientConfig,
TLSHandshakeTimeout: oldTransport.TLSHandshakeTimeout,
DisableKeepAlives: oldTransport.DisableKeepAlives,
DisableCompression: oldTransport.DisableCompression,
MaxIdleConnsPerHost: oldTransport.MaxIdleConnsPerHost,
ResponseHeaderTimeout: oldTransport.ResponseHeaderTimeout,
// new in go1.4
DialTLS: oldTransport.DialTLS,
}
}

View file

@ -0,0 +1,30 @@
// +build go1.6
// +build !go1.7
package gorequest
import (
"net/http"
)
// does a shallow clone of the transport
func (s *SuperAgent) safeModifyTransport() {
if !s.isClone {
return
}
oldTransport := s.Transport
s.Transport = &http.Transport{
Proxy: oldTransport.Proxy,
Dial: oldTransport.Dial,
DialTLS: oldTransport.DialTLS,
TLSClientConfig: oldTransport.TLSClientConfig,
TLSHandshakeTimeout: oldTransport.TLSHandshakeTimeout,
DisableKeepAlives: oldTransport.DisableKeepAlives,
DisableCompression: oldTransport.DisableCompression,
MaxIdleConnsPerHost: oldTransport.MaxIdleConnsPerHost,
ResponseHeaderTimeout: oldTransport.ResponseHeaderTimeout,
// new in 1.6
ExpectContinueTimeout: oldTransport.ExpectContinueTimeout,
TLSNextProto: oldTransport.TLSNextProto,
}
}

View file

@ -0,0 +1,34 @@
// +build go1.7
// +build !go1.8
package gorequest
import (
"net/http"
)
// does a shallow clone of the transport
func (s *SuperAgent) safeModifyTransport() {
if !s.isClone {
return
}
oldTransport := s.Transport
s.Transport = &http.Transport{
Proxy: oldTransport.Proxy,
Dial: oldTransport.Dial,
DialTLS: oldTransport.DialTLS,
TLSClientConfig: oldTransport.TLSClientConfig,
TLSHandshakeTimeout: oldTransport.TLSHandshakeTimeout,
DisableKeepAlives: oldTransport.DisableKeepAlives,
DisableCompression: oldTransport.DisableCompression,
MaxIdleConns: oldTransport.MaxIdleConns,
MaxIdleConnsPerHost: oldTransport.MaxIdleConnsPerHost,
ResponseHeaderTimeout: oldTransport.ResponseHeaderTimeout,
ExpectContinueTimeout: oldTransport.ExpectContinueTimeout,
TLSNextProto: oldTransport.TLSNextProto,
// new in go1.7
DialContext: oldTransport.DialContext,
IdleConnTimeout: oldTransport.IdleConnTimeout,
MaxResponseHeaderBytes: oldTransport.MaxResponseHeaderBytes,
}
}

View file

@ -0,0 +1,34 @@
// +build go1.8
package gorequest
import (
"net/http"
)
// does a shallow clone of the transport
func (s *SuperAgent) safeModifyTransport() {
if !s.isClone {
return
}
oldTransport := s.Transport
s.Transport = &http.Transport{
Proxy: oldTransport.Proxy,
DialContext: oldTransport.DialContext,
Dial: oldTransport.Dial,
DialTLS: oldTransport.DialTLS,
TLSClientConfig: oldTransport.TLSClientConfig,
TLSHandshakeTimeout: oldTransport.TLSHandshakeTimeout,
DisableKeepAlives: oldTransport.DisableKeepAlives,
DisableCompression: oldTransport.DisableCompression,
MaxIdleConns: oldTransport.MaxIdleConns,
MaxIdleConnsPerHost: oldTransport.MaxIdleConnsPerHost,
IdleConnTimeout: oldTransport.IdleConnTimeout,
ResponseHeaderTimeout: oldTransport.ResponseHeaderTimeout,
ExpectContinueTimeout: oldTransport.ExpectContinueTimeout,
TLSNextProto: oldTransport.TLSNextProto,
MaxResponseHeaderBytes: oldTransport.MaxResponseHeaderBytes,
// new in go1.8
ProxyConnectHeader: oldTransport.ProxyConnectHeader,
}
}

7
vendor/github.com/parnurzeal/gorequest/logger.go generated vendored Normal file
View file

@ -0,0 +1,7 @@
package gorequest
type Logger interface {
SetPrefix(string)
Printf(format string, v ...interface{})
Println(v ...interface{})
}

View file

@ -0,0 +1,29 @@
// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.12
package prometheus
import "runtime/debug"
// readBuildInfo is a wrapper around debug.ReadBuildInfo for Go 1.12+.
func readBuildInfo() (path, version, sum string) {
path, version, sum = "unknown", "unknown", "unknown"
if bi, ok := debug.ReadBuildInfo(); ok {
path = bi.Main.Path
version = bi.Main.Version
sum = bi.Main.Sum
}
return
}

View file

@ -0,0 +1,22 @@
// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.12
package prometheus
// readBuildInfo is a wrapper around debug.ReadBuildInfo for Go versions before
// 1.12. Remove this whole file once the minimum supported Go version is 1.12.
func readBuildInfo() (path, version, sum string) {
return "unknown", "unknown", "unknown"
}

View file

@ -183,7 +183,6 @@
// method can then expose the gathered metrics in some way. Usually, the metrics
// are served via HTTP on the /metrics endpoint. That's happening in the example
// above. The tools to expose metrics via HTTP are in the promhttp sub-package.
// (The top-level functions in the prometheus package are deprecated.)
//
// Pushing to the Pushgateway
//

View file

@ -14,9 +14,9 @@
package prometheus
import (
"fmt"
"runtime"
"runtime/debug"
"sync"
"time"
)
@ -26,16 +26,41 @@ type goCollector struct {
gcDesc *Desc
goInfoDesc *Desc
// metrics to describe and collect
metrics memStatsMetrics
// ms... are memstats related.
msLast *runtime.MemStats // Previously collected memstats.
msLastTimestamp time.Time
msMtx sync.Mutex // Protects msLast and msLastTimestamp.
msMetrics memStatsMetrics
msRead func(*runtime.MemStats) // For mocking in tests.
msMaxWait time.Duration // Wait time for fresh memstats.
msMaxAge time.Duration // Maximum allowed age of old memstats.
}
// NewGoCollector returns a collector which exports metrics about the current Go
// NewGoCollector returns a collector that exports metrics about the current Go
// process. This includes memory stats. To collect those, runtime.ReadMemStats
// is called. This causes a stop-the-world, which is very short with Go1.9+
// (~25µs). However, with older Go versions, the stop-the-world duration depends
// on the heap size and can be quite significant (~1.7 ms/GiB as per
// is called. This requires to “stop the world”, which usually only happens for
// garbage collection (GC). Take the following implications into account when
// deciding whether to use the Go collector:
//
// 1. The performance impact of stopping the world is the more relevant the more
// frequently metrics are collected. However, with Go1.9 or later the
// stop-the-world time per metrics collection is very short (~25µs) so that the
// performance impact will only matter in rare cases. However, with older Go
// versions, the stop-the-world duration depends on the heap size and can be
// quite significant (~1.7 ms/GiB as per
// https://go-review.googlesource.com/c/go/+/34937).
//
// 2. During an ongoing GC, nothing else can stop the world. Therefore, if the
// metrics collection happens to coincide with GC, it will only complete after
// GC has finished. Usually, GC is fast enough to not cause problems. However,
// with a very large heap, GC might take multiple seconds, which is enough to
// cause scrape timeouts in common setups. To avoid this problem, the Go
// collector will use the memstats from a previous collection if
// runtime.ReadMemStats takes more than 1s. However, if there are no previously
// collected memstats, or their collection is more than 5m ago, the collection
// will block until runtime.ReadMemStats succeeds. (The problem might be solved
// in Go1.13, see https://github.com/golang/go/issues/19812 for the related Go
// issue.)
func NewGoCollector() Collector {
return &goCollector{
goroutinesDesc: NewDesc(
@ -54,7 +79,11 @@ func NewGoCollector() Collector {
"go_info",
"Information about the Go environment.",
nil, Labels{"version": runtime.Version()}),
metrics: memStatsMetrics{
msLast: &runtime.MemStats{},
msRead: runtime.ReadMemStats,
msMaxWait: time.Second,
msMaxAge: 5 * time.Minute,
msMetrics: memStatsMetrics{
{
desc: NewDesc(
memstatNamespace("alloc_bytes"),
@ -253,7 +282,7 @@ func NewGoCollector() Collector {
}
func memstatNamespace(s string) string {
return fmt.Sprintf("go_memstats_%s", s)
return "go_memstats_" + s
}
// Describe returns all descriptions of the collector.
@ -262,13 +291,27 @@ func (c *goCollector) Describe(ch chan<- *Desc) {
ch <- c.threadsDesc
ch <- c.gcDesc
ch <- c.goInfoDesc
for _, i := range c.metrics {
for _, i := range c.msMetrics {
ch <- i.desc
}
}
// Collect returns the current state of all metrics of the collector.
func (c *goCollector) Collect(ch chan<- Metric) {
var (
ms = &runtime.MemStats{}
done = make(chan struct{})
)
// Start reading memstats first as it might take a while.
go func() {
c.msRead(ms)
c.msMtx.Lock()
c.msLast = ms
c.msLastTimestamp = time.Now()
c.msMtx.Unlock()
close(done)
}()
ch <- MustNewConstMetric(c.goroutinesDesc, GaugeValue, float64(runtime.NumGoroutine()))
n, _ := runtime.ThreadCreateProfile(nil)
ch <- MustNewConstMetric(c.threadsDesc, GaugeValue, float64(n))
@ -286,9 +329,31 @@ func (c *goCollector) Collect(ch chan<- Metric) {
ch <- MustNewConstMetric(c.goInfoDesc, GaugeValue, 1)
ms := &runtime.MemStats{}
runtime.ReadMemStats(ms)
for _, i := range c.metrics {
timer := time.NewTimer(c.msMaxWait)
select {
case <-done: // Our own ReadMemStats succeeded in time. Use it.
timer.Stop() // Important for high collection frequencies to not pile up timers.
c.msCollect(ch, ms)
return
case <-timer.C: // Time out, use last memstats if possible. Continue below.
}
c.msMtx.Lock()
if time.Since(c.msLastTimestamp) < c.msMaxAge {
// Last memstats are recent enough. Collect from them under the lock.
c.msCollect(ch, c.msLast)
c.msMtx.Unlock()
return
}
// If we are here, the last memstats are too old or don't exist. We have
// to wait until our own ReadMemStats finally completes. For that to
// happen, we have to release the lock.
c.msMtx.Unlock()
<-done
c.msCollect(ch, ms)
}
func (c *goCollector) msCollect(ch chan<- Metric, ms *runtime.MemStats) {
for _, i := range c.msMetrics {
ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms))
}
}
@ -299,3 +364,33 @@ type memStatsMetrics []struct {
eval func(*runtime.MemStats) float64
valType ValueType
}
// NewBuildInfoCollector returns a collector collecting a single metric
// "go_build_info" with the constant value 1 and three labels "path", "version",
// and "checksum". Their label values contain the main module path, version, and
// checksum, respectively. The labels will only have meaningful values if the
// binary is built with Go module support and from source code retrieved from
// the source repository (rather than the local file system). This is usually
// accomplished by building from outside of GOPATH, specifying the full address
// of the main package, e.g. "GO111MODULE=on go run
// github.com/prometheus/client_golang/examples/random". If built without Go
// module support, all label values will be "unknown". If built with Go module
// support but using the source code from the local file system, the "path" will
// be set appropriately, but "checksum" will be empty and "version" will be
// "(devel)".
//
// This collector uses only the build information for the main module. See
// https://github.com/povilasv/prommod for an example of a collector for the
// module dependencies.
func NewBuildInfoCollector() Collector {
path, version, sum := readBuildInfo()
c := &selfCollector{MustNewConstMetric(
NewDesc(
"go_build_info",
"Build information about the main Go module.",
nil, Labels{"path": path, "version": version, "checksum": sum},
),
GaugeValue, 1)}
c.init(c.self)
return c
}

View file

@ -224,18 +224,21 @@ type histogramCounts struct {
}
type histogram struct {
// countAndHotIdx is a complicated one. For lock-free yet atomic
// observations, we need to save the total count of observations again,
// combined with the index of the currently-hot counts struct, so that
// we can perform the operation on both values atomically. The least
// significant bit defines the hot counts struct. The remaining 63 bits
// represent the total count of observations. This happens under the
// assumption that the 63bit count will never overflow. Rationale: An
// observations takes about 30ns. Let's assume it could happen in
// 10ns. Overflowing the counter will then take at least (2^63)*10ns,
// which is about 3000 years.
// countAndHotIdx enables lock-free writes with use of atomic updates.
// The most significant bit is the hot index [0 or 1] of the count field
// below. Observe calls update the hot one. All remaining bits count the
// number of Observe calls. Observe starts by incrementing this counter,
// and finish by incrementing the count field in the respective
// histogramCounts, as a marker for completion.
//
// This has to be first in the struct for 64bit alignment. See
// Calls of the Write method (which are non-mutating reads from the
// perspective of the histogram) swap the hotcold under the writeMtx
// lock. A cooldown is awaited (while locked) by comparing the number of
// observations with the initiation count. Once they match, then the
// last observation on the now cool one has completed. All cool fields must
// be merged into the new hot before releasing writeMtx.
//
// Fields with atomic access first! See alignment constraint:
// http://golang.org/pkg/sync/atomic/#pkg-note-BUG
countAndHotIdx uint64
@ -243,16 +246,14 @@ type histogram struct {
desc *Desc
writeMtx sync.Mutex // Only used in the Write method.
upperBounds []float64
// Two counts, one is "hot" for lock-free observations, the other is
// "cold" for writing out a dto.Metric. It has to be an array of
// pointers to guarantee 64bit alignment of the histogramCounts, see
// http://golang.org/pkg/sync/atomic/#pkg-note-BUG.
counts [2]*histogramCounts
hotIdx int // Index of currently-hot counts. Only used within Write.
labelPairs []*dto.LabelPair
upperBounds []float64
labelPairs []*dto.LabelPair
}
func (h *histogram) Desc() *Desc {
@ -271,11 +272,11 @@ func (h *histogram) Observe(v float64) {
// 300 buckets: 154 ns/op linear - binary 61.6 ns/op
i := sort.SearchFloat64s(h.upperBounds, v)
// We increment h.countAndHotIdx by 2 so that the counter in the upper
// 63 bits gets incremented by 1. At the same time, we get the new value
// We increment h.countAndHotIdx so that the counter in the lower
// 63 bits gets incremented. At the same time, we get the new value
// back, which we can use to find the currently-hot counts.
n := atomic.AddUint64(&h.countAndHotIdx, 2)
hotCounts := h.counts[n%2]
n := atomic.AddUint64(&h.countAndHotIdx, 1)
hotCounts := h.counts[n>>63]
if i < len(h.upperBounds) {
atomic.AddUint64(&hotCounts.buckets[i], 1)
@ -293,72 +294,43 @@ func (h *histogram) Observe(v float64) {
}
func (h *histogram) Write(out *dto.Metric) error {
var (
his = &dto.Histogram{}
buckets = make([]*dto.Bucket, len(h.upperBounds))
hotCounts, coldCounts *histogramCounts
count uint64
)
// For simplicity, we mutex the rest of this method. It is not in the
// hot path, i.e. Observe is called much more often than Write. The
// complication of making Write lock-free isn't worth it.
// For simplicity, we protect this whole method by a mutex. It is not in
// the hot path, i.e. Observe is called much more often than Write. The
// complication of making Write lock-free isn't worth it, if possible at
// all.
h.writeMtx.Lock()
defer h.writeMtx.Unlock()
// This is a bit arcane, which is why the following spells out this if
// clause in English:
//
// If the currently-hot counts struct is #0, we atomically increment
// h.countAndHotIdx by 1 so that from now on Observe will use the counts
// struct #1. Furthermore, the atomic increment gives us the new value,
// which, in its most significant 63 bits, tells us the count of
// observations done so far up to and including currently ongoing
// observations still using the counts struct just changed from hot to
// cold. To have a normal uint64 for the count, we bitshift by 1 and
// save the result in count. We also set h.hotIdx to 1 for the next
// Write call, and we will refer to counts #1 as hotCounts and to counts
// #0 as coldCounts.
//
// If the currently-hot counts struct is #1, we do the corresponding
// things the other way round. We have to _decrement_ h.countAndHotIdx
// (which is a bit arcane in itself, as we have to express -1 with an
// unsigned int...).
if h.hotIdx == 0 {
count = atomic.AddUint64(&h.countAndHotIdx, 1) >> 1
h.hotIdx = 1
hotCounts = h.counts[1]
coldCounts = h.counts[0]
} else {
count = atomic.AddUint64(&h.countAndHotIdx, ^uint64(0)) >> 1 // Decrement.
h.hotIdx = 0
hotCounts = h.counts[0]
coldCounts = h.counts[1]
}
// Adding 1<<63 switches the hot index (from 0 to 1 or from 1 to 0)
// without touching the count bits. See the struct comments for a full
// description of the algorithm.
n := atomic.AddUint64(&h.countAndHotIdx, 1<<63)
// count is contained unchanged in the lower 63 bits.
count := n & ((1 << 63) - 1)
// The most significant bit tells us which counts is hot. The complement
// is thus the cold one.
hotCounts := h.counts[n>>63]
coldCounts := h.counts[(^n)>>63]
// Now we have to wait for the now-declared-cold counts to actually cool
// down, i.e. wait for all observations still using it to finish. That's
// the case once the count in the cold counts struct is the same as the
// one atomically retrieved from the upper 63bits of h.countAndHotIdx.
for {
if count == atomic.LoadUint64(&coldCounts.count) {
break
}
// Await cooldown.
for count != atomic.LoadUint64(&coldCounts.count) {
runtime.Gosched() // Let observations get work done.
}
his.SampleCount = proto.Uint64(count)
his.SampleSum = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits)))
his := &dto.Histogram{
Bucket: make([]*dto.Bucket, len(h.upperBounds)),
SampleCount: proto.Uint64(count),
SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
}
var cumCount uint64
for i, upperBound := range h.upperBounds {
cumCount += atomic.LoadUint64(&coldCounts.buckets[i])
buckets[i] = &dto.Bucket{
his.Bucket[i] = &dto.Bucket{
CumulativeCount: proto.Uint64(cumCount),
UpperBound: proto.Float64(upperBound),
}
}
his.Bucket = buckets
out.Histogram = his
out.Label = h.labelPairs

View file

@ -1,503 +0,0 @@
// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"bufio"
"compress/gzip"
"io"
"net"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/prometheus/common/expfmt"
)
// TODO(beorn7): Remove this whole file. It is a partial mirror of
// promhttp/http.go (to avoid circular import chains) where everything HTTP
// related should live. The functions here are just for avoiding
// breakage. Everything is deprecated.
const (
contentTypeHeader = "Content-Type"
contentEncodingHeader = "Content-Encoding"
acceptEncodingHeader = "Accept-Encoding"
)
var gzipPool = sync.Pool{
New: func() interface{} {
return gzip.NewWriter(nil)
},
}
// Handler returns an HTTP handler for the DefaultGatherer. It is
// already instrumented with InstrumentHandler (using "prometheus" as handler
// name).
//
// Deprecated: Please note the issues described in the doc comment of
// InstrumentHandler. You might want to consider using promhttp.Handler instead.
func Handler() http.Handler {
return InstrumentHandler("prometheus", UninstrumentedHandler())
}
// UninstrumentedHandler returns an HTTP handler for the DefaultGatherer.
//
// Deprecated: Use promhttp.HandlerFor(DefaultGatherer, promhttp.HandlerOpts{})
// instead. See there for further documentation.
func UninstrumentedHandler() http.Handler {
return http.HandlerFunc(func(rsp http.ResponseWriter, req *http.Request) {
mfs, err := DefaultGatherer.Gather()
if err != nil {
httpError(rsp, err)
return
}
contentType := expfmt.Negotiate(req.Header)
header := rsp.Header()
header.Set(contentTypeHeader, string(contentType))
w := io.Writer(rsp)
if gzipAccepted(req.Header) {
header.Set(contentEncodingHeader, "gzip")
gz := gzipPool.Get().(*gzip.Writer)
defer gzipPool.Put(gz)
gz.Reset(w)
defer gz.Close()
w = gz
}
enc := expfmt.NewEncoder(w, contentType)
for _, mf := range mfs {
if err := enc.Encode(mf); err != nil {
httpError(rsp, err)
return
}
}
})
}
var instLabels = []string{"method", "code"}
type nower interface {
Now() time.Time
}
type nowFunc func() time.Time
func (n nowFunc) Now() time.Time {
return n()
}
var now nower = nowFunc(func() time.Time {
return time.Now()
})
// InstrumentHandler wraps the given HTTP handler for instrumentation. It
// registers four metric collectors (if not already done) and reports HTTP
// metrics to the (newly or already) registered collectors: http_requests_total
// (CounterVec), http_request_duration_microseconds (Summary),
// http_request_size_bytes (Summary), http_response_size_bytes (Summary). Each
// has a constant label named "handler" with the provided handlerName as
// value. http_requests_total is a metric vector partitioned by HTTP method
// (label name "method") and HTTP status code (label name "code").
//
// Deprecated: InstrumentHandler has several issues. Use the tooling provided in
// package promhttp instead. The issues are the following: (1) It uses Summaries
// rather than Histograms. Summaries are not useful if aggregation across
// multiple instances is required. (2) It uses microseconds as unit, which is
// deprecated and should be replaced by seconds. (3) The size of the request is
// calculated in a separate goroutine. Since this calculator requires access to
// the request header, it creates a race with any writes to the header performed
// during request handling. httputil.ReverseProxy is a prominent example for a
// handler performing such writes. (4) It has additional issues with HTTP/2, cf.
// https://github.com/prometheus/client_golang/issues/272.
func InstrumentHandler(handlerName string, handler http.Handler) http.HandlerFunc {
return InstrumentHandlerFunc(handlerName, handler.ServeHTTP)
}
// InstrumentHandlerFunc wraps the given function for instrumentation. It
// otherwise works in the same way as InstrumentHandler (and shares the same
// issues).
//
// Deprecated: InstrumentHandlerFunc is deprecated for the same reasons as
// InstrumentHandler is. Use the tooling provided in package promhttp instead.
func InstrumentHandlerFunc(handlerName string, handlerFunc func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
return InstrumentHandlerFuncWithOpts(
SummaryOpts{
Subsystem: "http",
ConstLabels: Labels{"handler": handlerName},
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
handlerFunc,
)
}
// InstrumentHandlerWithOpts works like InstrumentHandler (and shares the same
// issues) but provides more flexibility (at the cost of a more complex call
// syntax). As InstrumentHandler, this function registers four metric
// collectors, but it uses the provided SummaryOpts to create them. However, the
// fields "Name" and "Help" in the SummaryOpts are ignored. "Name" is replaced
// by "requests_total", "request_duration_microseconds", "request_size_bytes",
// and "response_size_bytes", respectively. "Help" is replaced by an appropriate
// help string. The names of the variable labels of the http_requests_total
// CounterVec are "method" (get, post, etc.), and "code" (HTTP status code).
//
// If InstrumentHandlerWithOpts is called as follows, it mimics exactly the
// behavior of InstrumentHandler:
//
// prometheus.InstrumentHandlerWithOpts(
// prometheus.SummaryOpts{
// Subsystem: "http",
// ConstLabels: prometheus.Labels{"handler": handlerName},
// },
// handler,
// )
//
// Technical detail: "requests_total" is a CounterVec, not a SummaryVec, so it
// cannot use SummaryOpts. Instead, a CounterOpts struct is created internally,
// and all its fields are set to the equally named fields in the provided
// SummaryOpts.
//
// Deprecated: InstrumentHandlerWithOpts is deprecated for the same reasons as
// InstrumentHandler is. Use the tooling provided in package promhttp instead.
func InstrumentHandlerWithOpts(opts SummaryOpts, handler http.Handler) http.HandlerFunc {
return InstrumentHandlerFuncWithOpts(opts, handler.ServeHTTP)
}
// InstrumentHandlerFuncWithOpts works like InstrumentHandlerFunc (and shares
// the same issues) but provides more flexibility (at the cost of a more complex
// call syntax). See InstrumentHandlerWithOpts for details how the provided
// SummaryOpts are used.
//
// Deprecated: InstrumentHandlerFuncWithOpts is deprecated for the same reasons
// as InstrumentHandler is. Use the tooling provided in package promhttp instead.
func InstrumentHandlerFuncWithOpts(opts SummaryOpts, handlerFunc func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
reqCnt := NewCounterVec(
CounterOpts{
Namespace: opts.Namespace,
Subsystem: opts.Subsystem,
Name: "requests_total",
Help: "Total number of HTTP requests made.",
ConstLabels: opts.ConstLabels,
},
instLabels,
)
if err := Register(reqCnt); err != nil {
if are, ok := err.(AlreadyRegisteredError); ok {
reqCnt = are.ExistingCollector.(*CounterVec)
} else {
panic(err)
}
}
opts.Name = "request_duration_microseconds"
opts.Help = "The HTTP request latencies in microseconds."
reqDur := NewSummary(opts)
if err := Register(reqDur); err != nil {
if are, ok := err.(AlreadyRegisteredError); ok {
reqDur = are.ExistingCollector.(Summary)
} else {
panic(err)
}
}
opts.Name = "request_size_bytes"
opts.Help = "The HTTP request sizes in bytes."
reqSz := NewSummary(opts)
if err := Register(reqSz); err != nil {
if are, ok := err.(AlreadyRegisteredError); ok {
reqSz = are.ExistingCollector.(Summary)
} else {
panic(err)
}
}
opts.Name = "response_size_bytes"
opts.Help = "The HTTP response sizes in bytes."
resSz := NewSummary(opts)
if err := Register(resSz); err != nil {
if are, ok := err.(AlreadyRegisteredError); ok {
resSz = are.ExistingCollector.(Summary)
} else {
panic(err)
}
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
now := time.Now()
delegate := &responseWriterDelegator{ResponseWriter: w}
out := computeApproximateRequestSize(r)
_, cn := w.(http.CloseNotifier)
_, fl := w.(http.Flusher)
_, hj := w.(http.Hijacker)
_, rf := w.(io.ReaderFrom)
var rw http.ResponseWriter
if cn && fl && hj && rf {
rw = &fancyResponseWriterDelegator{delegate}
} else {
rw = delegate
}
handlerFunc(rw, r)
elapsed := float64(time.Since(now)) / float64(time.Microsecond)
method := sanitizeMethod(r.Method)
code := sanitizeCode(delegate.status)
reqCnt.WithLabelValues(method, code).Inc()
reqDur.Observe(elapsed)
resSz.Observe(float64(delegate.written))
reqSz.Observe(float64(<-out))
})
}
func computeApproximateRequestSize(r *http.Request) <-chan int {
// Get URL length in current goroutine for avoiding a race condition.
// HandlerFunc that runs in parallel may modify the URL.
s := 0
if r.URL != nil {
s += len(r.URL.String())
}
out := make(chan int, 1)
go func() {
s += len(r.Method)
s += len(r.Proto)
for name, values := range r.Header {
s += len(name)
for _, value := range values {
s += len(value)
}
}
s += len(r.Host)
// N.B. r.Form and r.MultipartForm are assumed to be included in r.URL.
if r.ContentLength != -1 {
s += int(r.ContentLength)
}
out <- s
close(out)
}()
return out
}
type responseWriterDelegator struct {
http.ResponseWriter
status int
written int64
wroteHeader bool
}
func (r *responseWriterDelegator) WriteHeader(code int) {
r.status = code
r.wroteHeader = true
r.ResponseWriter.WriteHeader(code)
}
func (r *responseWriterDelegator) Write(b []byte) (int, error) {
if !r.wroteHeader {
r.WriteHeader(http.StatusOK)
}
n, err := r.ResponseWriter.Write(b)
r.written += int64(n)
return n, err
}
type fancyResponseWriterDelegator struct {
*responseWriterDelegator
}
func (f *fancyResponseWriterDelegator) CloseNotify() <-chan bool {
return f.ResponseWriter.(http.CloseNotifier).CloseNotify()
}
func (f *fancyResponseWriterDelegator) Flush() {
f.ResponseWriter.(http.Flusher).Flush()
}
func (f *fancyResponseWriterDelegator) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return f.ResponseWriter.(http.Hijacker).Hijack()
}
func (f *fancyResponseWriterDelegator) ReadFrom(r io.Reader) (int64, error) {
if !f.wroteHeader {
f.WriteHeader(http.StatusOK)
}
n, err := f.ResponseWriter.(io.ReaderFrom).ReadFrom(r)
f.written += n
return n, err
}
func sanitizeMethod(m string) string {
switch m {
case "GET", "get":
return "get"
case "PUT", "put":
return "put"
case "HEAD", "head":
return "head"
case "POST", "post":
return "post"
case "DELETE", "delete":
return "delete"
case "CONNECT", "connect":
return "connect"
case "OPTIONS", "options":
return "options"
case "NOTIFY", "notify":
return "notify"
default:
return strings.ToLower(m)
}
}
func sanitizeCode(s int) string {
switch s {
case 100:
return "100"
case 101:
return "101"
case 200:
return "200"
case 201:
return "201"
case 202:
return "202"
case 203:
return "203"
case 204:
return "204"
case 205:
return "205"
case 206:
return "206"
case 300:
return "300"
case 301:
return "301"
case 302:
return "302"
case 304:
return "304"
case 305:
return "305"
case 307:
return "307"
case 400:
return "400"
case 401:
return "401"
case 402:
return "402"
case 403:
return "403"
case 404:
return "404"
case 405:
return "405"
case 406:
return "406"
case 407:
return "407"
case 408:
return "408"
case 409:
return "409"
case 410:
return "410"
case 411:
return "411"
case 412:
return "412"
case 413:
return "413"
case 414:
return "414"
case 415:
return "415"
case 416:
return "416"
case 417:
return "417"
case 418:
return "418"
case 500:
return "500"
case 501:
return "501"
case 502:
return "502"
case 503:
return "503"
case 504:
return "504"
case 505:
return "505"
case 428:
return "428"
case 429:
return "429"
case 431:
return "431"
case 511:
return "511"
default:
return strconv.Itoa(s)
}
}
// gzipAccepted returns whether the client will accept gzip-encoded content.
func gzipAccepted(header http.Header) bool {
a := header.Get(acceptEncodingHeader)
parts := strings.Split(a, ",")
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "gzip" || strings.HasPrefix(part, "gzip;") {
return true
}
}
return false
}
// httpError removes any content-encoding header and then calls http.Error with
// the provided error and http.StatusInternalServerErrer. Error contents is
// supposed to be uncompressed plain text. However, same as with a plain
// http.Error, any header settings will be void if the header has already been
// sent. The error message will still be written to the writer, but it will
// probably be of limited use.
func httpError(rsp http.ResponseWriter, err error) {
rsp.Header().Del(contentEncodingHeader)
http.Error(
rsp,
"An error has occurred while serving metrics:\n\n"+err.Error(),
http.StatusInternalServerError,
)
}

View file

@ -16,8 +16,6 @@ package prometheus
import (
"errors"
"os"
"github.com/prometheus/procfs"
)
type processCollector struct {
@ -59,20 +57,9 @@ type ProcessCollectorOpts struct {
// collector for the current process with an empty namespace string and no error
// reporting.
//
// Currently, the collector depends on a Linux-style proc filesystem and
// therefore only exports metrics for Linux.
//
// Note: An older version of this function had the following signature:
//
// NewProcessCollector(pid int, namespace string) Collector
//
// Most commonly, it was called as
//
// NewProcessCollector(os.Getpid(), "")
//
// The following call of the current version is equivalent to the above:
//
// NewProcessCollector(ProcessCollectorOpts{})
// The collector only works on operating systems with a Linux-style proc
// filesystem and on Microsoft Windows. On other operating systems, it will not
// collect any metrics.
func NewProcessCollector(opts ProcessCollectorOpts) Collector {
ns := ""
if len(opts.Namespace) > 0 {
@ -126,7 +113,7 @@ func NewProcessCollector(opts ProcessCollectorOpts) Collector {
}
// Set up process metric collection if supported by the runtime.
if _, err := procfs.NewStat(); err == nil {
if canCollectProcess() {
c.collectFn = c.processCollect
} else {
c.collectFn = func(ch chan<- Metric) {
@ -153,46 +140,6 @@ func (c *processCollector) Collect(ch chan<- Metric) {
c.collectFn(ch)
}
func (c *processCollector) processCollect(ch chan<- Metric) {
pid, err := c.pidFn()
if err != nil {
c.reportError(ch, nil, err)
return
}
p, err := procfs.NewProc(pid)
if err != nil {
c.reportError(ch, nil, err)
return
}
if stat, err := p.NewStat(); err == nil {
ch <- MustNewConstMetric(c.cpuTotal, CounterValue, stat.CPUTime())
ch <- MustNewConstMetric(c.vsize, GaugeValue, float64(stat.VirtualMemory()))
ch <- MustNewConstMetric(c.rss, GaugeValue, float64(stat.ResidentMemory()))
if startTime, err := stat.StartTime(); err == nil {
ch <- MustNewConstMetric(c.startTime, GaugeValue, startTime)
} else {
c.reportError(ch, c.startTime, err)
}
} else {
c.reportError(ch, nil, err)
}
if fds, err := p.FileDescriptorsLen(); err == nil {
ch <- MustNewConstMetric(c.openFDs, GaugeValue, float64(fds))
} else {
c.reportError(ch, c.openFDs, err)
}
if limits, err := p.NewLimits(); err == nil {
ch <- MustNewConstMetric(c.maxFDs, GaugeValue, float64(limits.OpenFiles))
ch <- MustNewConstMetric(c.maxVsize, GaugeValue, float64(limits.AddressSpace))
} else {
c.reportError(ch, nil, err)
}
}
func (c *processCollector) reportError(ch chan<- Metric, desc *Desc, err error) {
if !c.reportErrors {
return

View file

@ -0,0 +1,65 @@
// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !windows
package prometheus
import (
"github.com/prometheus/procfs"
)
func canCollectProcess() bool {
_, err := procfs.NewDefaultFS()
return err == nil
}
func (c *processCollector) processCollect(ch chan<- Metric) {
pid, err := c.pidFn()
if err != nil {
c.reportError(ch, nil, err)
return
}
p, err := procfs.NewProc(pid)
if err != nil {
c.reportError(ch, nil, err)
return
}
if stat, err := p.Stat(); err == nil {
ch <- MustNewConstMetric(c.cpuTotal, CounterValue, stat.CPUTime())
ch <- MustNewConstMetric(c.vsize, GaugeValue, float64(stat.VirtualMemory()))
ch <- MustNewConstMetric(c.rss, GaugeValue, float64(stat.ResidentMemory()))
if startTime, err := stat.StartTime(); err == nil {
ch <- MustNewConstMetric(c.startTime, GaugeValue, startTime)
} else {
c.reportError(ch, c.startTime, err)
}
} else {
c.reportError(ch, nil, err)
}
if fds, err := p.FileDescriptorsLen(); err == nil {
ch <- MustNewConstMetric(c.openFDs, GaugeValue, float64(fds))
} else {
c.reportError(ch, c.openFDs, err)
}
if limits, err := p.Limits(); err == nil {
ch <- MustNewConstMetric(c.maxFDs, GaugeValue, float64(limits.OpenFiles))
ch <- MustNewConstMetric(c.maxVsize, GaugeValue, float64(limits.AddressSpace))
} else {
c.reportError(ch, nil, err)
}
}

View file

@ -0,0 +1,112 @@
// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
func canCollectProcess() bool {
return true
}
var (
modpsapi = syscall.NewLazyDLL("psapi.dll")
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
procGetProcessMemoryInfo = modpsapi.NewProc("GetProcessMemoryInfo")
procGetProcessHandleCount = modkernel32.NewProc("GetProcessHandleCount")
)
type processMemoryCounters struct {
// https://docs.microsoft.com/en-us/windows/desktop/api/psapi/ns-psapi-_process_memory_counters_ex
_ uint32
PageFaultCount uint32
PeakWorkingSetSize uint64
WorkingSetSize uint64
QuotaPeakPagedPoolUsage uint64
QuotaPagedPoolUsage uint64
QuotaPeakNonPagedPoolUsage uint64
QuotaNonPagedPoolUsage uint64
PagefileUsage uint64
PeakPagefileUsage uint64
PrivateUsage uint64
}
func getProcessMemoryInfo(handle windows.Handle) (processMemoryCounters, error) {
mem := processMemoryCounters{}
r1, _, err := procGetProcessMemoryInfo.Call(
uintptr(handle),
uintptr(unsafe.Pointer(&mem)),
uintptr(unsafe.Sizeof(mem)),
)
if r1 != 1 {
return mem, err
} else {
return mem, nil
}
}
func getProcessHandleCount(handle windows.Handle) (uint32, error) {
var count uint32
r1, _, err := procGetProcessHandleCount.Call(
uintptr(handle),
uintptr(unsafe.Pointer(&count)),
)
if r1 != 1 {
return 0, err
} else {
return count, nil
}
}
func (c *processCollector) processCollect(ch chan<- Metric) {
h, err := windows.GetCurrentProcess()
if err != nil {
c.reportError(ch, nil, err)
return
}
var startTime, exitTime, kernelTime, userTime windows.Filetime
err = windows.GetProcessTimes(h, &startTime, &exitTime, &kernelTime, &userTime)
if err != nil {
c.reportError(ch, nil, err)
return
}
ch <- MustNewConstMetric(c.startTime, GaugeValue, float64(startTime.Nanoseconds()/1e9))
ch <- MustNewConstMetric(c.cpuTotal, CounterValue, fileTimeToSeconds(kernelTime)+fileTimeToSeconds(userTime))
mem, err := getProcessMemoryInfo(h)
if err != nil {
c.reportError(ch, nil, err)
return
}
ch <- MustNewConstMetric(c.vsize, GaugeValue, float64(mem.PrivateUsage))
ch <- MustNewConstMetric(c.rss, GaugeValue, float64(mem.WorkingSetSize))
handles, err := getProcessHandleCount(h)
if err != nil {
c.reportError(ch, nil, err)
return
}
ch <- MustNewConstMetric(c.openFDs, GaugeValue, float64(handles))
ch <- MustNewConstMetric(c.maxFDs, GaugeValue, float64(16*1024*1024)) // Windows has a hard-coded max limit, not per-process.
}
func fileTimeToSeconds(ft windows.Filetime) float64 {
return float64(uint64(ft.HighDateTime)<<32+uint64(ft.LowDateTime)) / 1e7
}

View file

@ -74,8 +74,11 @@ type closeNotifierDelegator struct{ *responseWriterDelegator }
type flusherDelegator struct{ *responseWriterDelegator }
type hijackerDelegator struct{ *responseWriterDelegator }
type readerFromDelegator struct{ *responseWriterDelegator }
type pusherDelegator struct{ *responseWriterDelegator }
func (d closeNotifierDelegator) CloseNotify() <-chan bool {
//lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to
//remove support from client_golang yet.
return d.ResponseWriter.(http.CloseNotifier).CloseNotify()
}
func (d flusherDelegator) Flush() {
@ -92,6 +95,9 @@ func (d readerFromDelegator) ReadFrom(re io.Reader) (int64, error) {
d.written += n
return n, err
}
func (d pusherDelegator) Push(target string, opts *http.PushOptions) error {
return d.ResponseWriter.(http.Pusher).Push(target, opts)
}
var pickDelegator = make([]func(*responseWriterDelegator) delegator, 32)
@ -195,4 +201,157 @@ func init() {
http.CloseNotifier
}{d, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher] = func(d *responseWriterDelegator) delegator { // 16
return pusherDelegator{d}
}
pickDelegator[pusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 17
return struct {
*responseWriterDelegator
http.Pusher
http.CloseNotifier
}{d, pusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+flusher] = func(d *responseWriterDelegator) delegator { // 18
return struct {
*responseWriterDelegator
http.Pusher
http.Flusher
}{d, pusherDelegator{d}, flusherDelegator{d}}
}
pickDelegator[pusher+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 19
return struct {
*responseWriterDelegator
http.Pusher
http.Flusher
http.CloseNotifier
}{d, pusherDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+hijacker] = func(d *responseWriterDelegator) delegator { // 20
return struct {
*responseWriterDelegator
http.Pusher
http.Hijacker
}{d, pusherDelegator{d}, hijackerDelegator{d}}
}
pickDelegator[pusher+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 21
return struct {
*responseWriterDelegator
http.Pusher
http.Hijacker
http.CloseNotifier
}{d, pusherDelegator{d}, hijackerDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 22
return struct {
*responseWriterDelegator
http.Pusher
http.Hijacker
http.Flusher
}{d, pusherDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}}
}
pickDelegator[pusher+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { //23
return struct {
*responseWriterDelegator
http.Pusher
http.Hijacker
http.Flusher
http.CloseNotifier
}{d, pusherDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+readerFrom] = func(d *responseWriterDelegator) delegator { // 24
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
}{d, pusherDelegator{d}, readerFromDelegator{d}}
}
pickDelegator[pusher+readerFrom+closeNotifier] = func(d *responseWriterDelegator) delegator { // 25
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.CloseNotifier
}{d, pusherDelegator{d}, readerFromDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+readerFrom+flusher] = func(d *responseWriterDelegator) delegator { // 26
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Flusher
}{d, pusherDelegator{d}, readerFromDelegator{d}, flusherDelegator{d}}
}
pickDelegator[pusher+readerFrom+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 27
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Flusher
http.CloseNotifier
}{d, pusherDelegator{d}, readerFromDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+readerFrom+hijacker] = func(d *responseWriterDelegator) delegator { // 28
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Hijacker
}{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}}
}
pickDelegator[pusher+readerFrom+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 29
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Hijacker
http.CloseNotifier
}{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+readerFrom+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 30
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Hijacker
http.Flusher
}{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}}
}
pickDelegator[pusher+readerFrom+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 31
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Hijacker
http.Flusher
http.CloseNotifier
}{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
}
func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator {
d := &responseWriterDelegator{
ResponseWriter: w,
observeWriteHeader: observeWriteHeaderFunc,
}
id := 0
//lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to
//remove support from client_golang yet.
if _, ok := w.(http.CloseNotifier); ok {
id += closeNotifier
}
if _, ok := w.(http.Flusher); ok {
id += flusher
}
if _, ok := w.(http.Hijacker); ok {
id += hijacker
}
if _, ok := w.(io.ReaderFrom); ok {
id += readerFrom
}
if _, ok := w.(http.Pusher); ok {
id += pusher
}
return pickDelegator[id](d)
}

View file

@ -1,181 +0,0 @@
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.8
package promhttp
import (
"io"
"net/http"
)
type pusherDelegator struct{ *responseWriterDelegator }
func (d pusherDelegator) Push(target string, opts *http.PushOptions) error {
return d.ResponseWriter.(http.Pusher).Push(target, opts)
}
func init() {
pickDelegator[pusher] = func(d *responseWriterDelegator) delegator { // 16
return pusherDelegator{d}
}
pickDelegator[pusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 17
return struct {
*responseWriterDelegator
http.Pusher
http.CloseNotifier
}{d, pusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+flusher] = func(d *responseWriterDelegator) delegator { // 18
return struct {
*responseWriterDelegator
http.Pusher
http.Flusher
}{d, pusherDelegator{d}, flusherDelegator{d}}
}
pickDelegator[pusher+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 19
return struct {
*responseWriterDelegator
http.Pusher
http.Flusher
http.CloseNotifier
}{d, pusherDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+hijacker] = func(d *responseWriterDelegator) delegator { // 20
return struct {
*responseWriterDelegator
http.Pusher
http.Hijacker
}{d, pusherDelegator{d}, hijackerDelegator{d}}
}
pickDelegator[pusher+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 21
return struct {
*responseWriterDelegator
http.Pusher
http.Hijacker
http.CloseNotifier
}{d, pusherDelegator{d}, hijackerDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 22
return struct {
*responseWriterDelegator
http.Pusher
http.Hijacker
http.Flusher
}{d, pusherDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}}
}
pickDelegator[pusher+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { //23
return struct {
*responseWriterDelegator
http.Pusher
http.Hijacker
http.Flusher
http.CloseNotifier
}{d, pusherDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+readerFrom] = func(d *responseWriterDelegator) delegator { // 24
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
}{d, pusherDelegator{d}, readerFromDelegator{d}}
}
pickDelegator[pusher+readerFrom+closeNotifier] = func(d *responseWriterDelegator) delegator { // 25
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.CloseNotifier
}{d, pusherDelegator{d}, readerFromDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+readerFrom+flusher] = func(d *responseWriterDelegator) delegator { // 26
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Flusher
}{d, pusherDelegator{d}, readerFromDelegator{d}, flusherDelegator{d}}
}
pickDelegator[pusher+readerFrom+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 27
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Flusher
http.CloseNotifier
}{d, pusherDelegator{d}, readerFromDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+readerFrom+hijacker] = func(d *responseWriterDelegator) delegator { // 28
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Hijacker
}{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}}
}
pickDelegator[pusher+readerFrom+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 29
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Hijacker
http.CloseNotifier
}{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, closeNotifierDelegator{d}}
}
pickDelegator[pusher+readerFrom+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 30
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Hijacker
http.Flusher
}{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}}
}
pickDelegator[pusher+readerFrom+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 31
return struct {
*responseWriterDelegator
http.Pusher
io.ReaderFrom
http.Hijacker
http.Flusher
http.CloseNotifier
}{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}}
}
}
func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator {
d := &responseWriterDelegator{
ResponseWriter: w,
observeWriteHeader: observeWriteHeaderFunc,
}
id := 0
if _, ok := w.(http.CloseNotifier); ok {
id += closeNotifier
}
if _, ok := w.(http.Flusher); ok {
id += flusher
}
if _, ok := w.(http.Hijacker); ok {
id += hijacker
}
if _, ok := w.(io.ReaderFrom); ok {
id += readerFrom
}
if _, ok := w.(http.Pusher); ok {
id += pusher
}
return pickDelegator[id](d)
}

View file

@ -1,44 +0,0 @@
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.8
package promhttp
import (
"io"
"net/http"
)
func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator {
d := &responseWriterDelegator{
ResponseWriter: w,
observeWriteHeader: observeWriteHeaderFunc,
}
id := 0
if _, ok := w.(http.CloseNotifier); ok {
id += closeNotifier
}
if _, ok := w.(http.Flusher); ok {
id += flusher
}
if _, ok := w.(http.Hijacker); ok {
id += hijacker
}
if _, ok := w.(io.ReaderFrom); ok {
id += readerFrom
}
return pickDelegator[id](d)
}

View file

@ -84,10 +84,32 @@ func Handler() http.Handler {
// instrumentation. Use the InstrumentMetricHandler function to apply the same
// kind of instrumentation as it is used by the Handler function.
func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
var inFlightSem chan struct{}
var (
inFlightSem chan struct{}
errCnt = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "promhttp_metric_handler_errors_total",
Help: "Total number of internal errors encountered by the promhttp metric handler.",
},
[]string{"cause"},
)
)
if opts.MaxRequestsInFlight > 0 {
inFlightSem = make(chan struct{}, opts.MaxRequestsInFlight)
}
if opts.Registry != nil {
// Initialize all possibilites that can occur below.
errCnt.WithLabelValues("gathering")
errCnt.WithLabelValues("encoding")
if err := opts.Registry.Register(errCnt); err != nil {
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
errCnt = are.ExistingCollector.(*prometheus.CounterVec)
} else {
panic(err)
}
}
}
h := http.HandlerFunc(func(rsp http.ResponseWriter, req *http.Request) {
if inFlightSem != nil {
@ -106,6 +128,7 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
if opts.ErrorLog != nil {
opts.ErrorLog.Println("error gathering metrics:", err)
}
errCnt.WithLabelValues("gathering").Inc()
switch opts.ErrorHandling {
case PanicOnError:
panic(err)
@ -146,6 +169,7 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
if opts.ErrorLog != nil {
opts.ErrorLog.Println("error encoding and sending metric family:", err)
}
errCnt.WithLabelValues("encoding").Inc()
switch opts.ErrorHandling {
case PanicOnError:
panic(err)
@ -236,9 +260,12 @@ const (
// Ignore errors and try to serve as many metrics as possible. However,
// if no metrics can be served, serve an HTTP status code 500 and the
// last error message in the body. Only use this in deliberate "best
// effort" metrics collection scenarios. It is recommended to at least
// log errors (by providing an ErrorLog in HandlerOpts) to not mask
// errors completely.
// effort" metrics collection scenarios. In this case, it is highly
// recommended to provide other means of detecting errors: By setting an
// ErrorLog in HandlerOpts, the errors are logged. By providing a
// Registry in HandlerOpts, the exposed metrics include an error counter
// "promhttp_metric_handler_errors_total", which can be used for
// alerts.
ContinueOnError
// Panic upon the first error encountered (useful for "crash only" apps).
PanicOnError
@ -261,6 +288,18 @@ type HandlerOpts struct {
// logged regardless of the configured ErrorHandling provided ErrorLog
// is not nil.
ErrorHandling HandlerErrorHandling
// If Registry is not nil, it is used to register a metric
// "promhttp_metric_handler_errors_total", partitioned by "cause". A
// failed registration causes a panic. Note that this error counter is
// different from the instrumentation you get from the various
// InstrumentHandler... helpers. It counts errors that don't necessarily
// result in a non-2xx HTTP status code. There are two typical cases:
// (1) Encoding errors that only happen after streaming of the HTTP body
// has already started (and the status code 200 has been sent). This
// should only happen with custom collectors. (2) Collection errors with
// no effect on the HTTP status code because ErrorHandling is set to
// ContinueOnError.
Registry prometheus.Registerer
// If DisableCompression is true, the handler will never compress the
// response, even if requested by the client.
DisableCompression bool

View file

@ -14,7 +14,9 @@
package promhttp
import (
"crypto/tls"
"net/http"
"net/http/httptrace"
"time"
"github.com/prometheus/client_golang/prometheus"
@ -95,3 +97,123 @@ func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundT
return resp, err
})
}
// InstrumentTrace is used to offer flexibility in instrumenting the available
// httptrace.ClientTrace hook functions. Each function is passed a float64
// representing the time in seconds since the start of the http request. A user
// may choose to use separately buckets Histograms, or implement custom
// instance labels on a per function basis.
type InstrumentTrace struct {
GotConn func(float64)
PutIdleConn func(float64)
GotFirstResponseByte func(float64)
Got100Continue func(float64)
DNSStart func(float64)
DNSDone func(float64)
ConnectStart func(float64)
ConnectDone func(float64)
TLSHandshakeStart func(float64)
TLSHandshakeDone func(float64)
WroteHeaders func(float64)
Wait100Continue func(float64)
WroteRequest func(float64)
}
// InstrumentRoundTripperTrace is a middleware that wraps the provided
// RoundTripper and reports times to hook functions provided in the
// InstrumentTrace struct. Hook functions that are not present in the provided
// InstrumentTrace struct are ignored. Times reported to the hook functions are
// time since the start of the request. Only with Go1.9+, those times are
// guaranteed to never be negative. (Earlier Go versions are not using a
// monotonic clock.) Note that partitioning of Histograms is expensive and
// should be used judiciously.
//
// For hook functions that receive an error as an argument, no observations are
// made in the event of a non-nil error value.
//
// See the example for ExampleInstrumentRoundTripperDuration for example usage.
func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc {
return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
start := time.Now()
trace := &httptrace.ClientTrace{
GotConn: func(_ httptrace.GotConnInfo) {
if it.GotConn != nil {
it.GotConn(time.Since(start).Seconds())
}
},
PutIdleConn: func(err error) {
if err != nil {
return
}
if it.PutIdleConn != nil {
it.PutIdleConn(time.Since(start).Seconds())
}
},
DNSStart: func(_ httptrace.DNSStartInfo) {
if it.DNSStart != nil {
it.DNSStart(time.Since(start).Seconds())
}
},
DNSDone: func(_ httptrace.DNSDoneInfo) {
if it.DNSDone != nil {
it.DNSDone(time.Since(start).Seconds())
}
},
ConnectStart: func(_, _ string) {
if it.ConnectStart != nil {
it.ConnectStart(time.Since(start).Seconds())
}
},
ConnectDone: func(_, _ string, err error) {
if err != nil {
return
}
if it.ConnectDone != nil {
it.ConnectDone(time.Since(start).Seconds())
}
},
GotFirstResponseByte: func() {
if it.GotFirstResponseByte != nil {
it.GotFirstResponseByte(time.Since(start).Seconds())
}
},
Got100Continue: func() {
if it.Got100Continue != nil {
it.Got100Continue(time.Since(start).Seconds())
}
},
TLSHandshakeStart: func() {
if it.TLSHandshakeStart != nil {
it.TLSHandshakeStart(time.Since(start).Seconds())
}
},
TLSHandshakeDone: func(_ tls.ConnectionState, err error) {
if err != nil {
return
}
if it.TLSHandshakeDone != nil {
it.TLSHandshakeDone(time.Since(start).Seconds())
}
},
WroteHeaders: func() {
if it.WroteHeaders != nil {
it.WroteHeaders(time.Since(start).Seconds())
}
},
Wait100Continue: func() {
if it.Wait100Continue != nil {
it.Wait100Continue(time.Since(start).Seconds())
}
},
WroteRequest: func(_ httptrace.WroteRequestInfo) {
if it.WroteRequest != nil {
it.WroteRequest(time.Since(start).Seconds())
}
},
}
r = r.WithContext(httptrace.WithClientTrace(r.Context(), trace))
return next.RoundTrip(r)
})
}

View file

@ -1,144 +0,0 @@
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.8
package promhttp
import (
"context"
"crypto/tls"
"net/http"
"net/http/httptrace"
"time"
)
// InstrumentTrace is used to offer flexibility in instrumenting the available
// httptrace.ClientTrace hook functions. Each function is passed a float64
// representing the time in seconds since the start of the http request. A user
// may choose to use separately buckets Histograms, or implement custom
// instance labels on a per function basis.
type InstrumentTrace struct {
GotConn func(float64)
PutIdleConn func(float64)
GotFirstResponseByte func(float64)
Got100Continue func(float64)
DNSStart func(float64)
DNSDone func(float64)
ConnectStart func(float64)
ConnectDone func(float64)
TLSHandshakeStart func(float64)
TLSHandshakeDone func(float64)
WroteHeaders func(float64)
Wait100Continue func(float64)
WroteRequest func(float64)
}
// InstrumentRoundTripperTrace is a middleware that wraps the provided
// RoundTripper and reports times to hook functions provided in the
// InstrumentTrace struct. Hook functions that are not present in the provided
// InstrumentTrace struct are ignored. Times reported to the hook functions are
// time since the start of the request. Only with Go1.9+, those times are
// guaranteed to never be negative. (Earlier Go versions are not using a
// monotonic clock.) Note that partitioning of Histograms is expensive and
// should be used judiciously.
//
// For hook functions that receive an error as an argument, no observations are
// made in the event of a non-nil error value.
//
// See the example for ExampleInstrumentRoundTripperDuration for example usage.
func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc {
return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
start := time.Now()
trace := &httptrace.ClientTrace{
GotConn: func(_ httptrace.GotConnInfo) {
if it.GotConn != nil {
it.GotConn(time.Since(start).Seconds())
}
},
PutIdleConn: func(err error) {
if err != nil {
return
}
if it.PutIdleConn != nil {
it.PutIdleConn(time.Since(start).Seconds())
}
},
DNSStart: func(_ httptrace.DNSStartInfo) {
if it.DNSStart != nil {
it.DNSStart(time.Since(start).Seconds())
}
},
DNSDone: func(_ httptrace.DNSDoneInfo) {
if it.DNSDone != nil {
it.DNSDone(time.Since(start).Seconds())
}
},
ConnectStart: func(_, _ string) {
if it.ConnectStart != nil {
it.ConnectStart(time.Since(start).Seconds())
}
},
ConnectDone: func(_, _ string, err error) {
if err != nil {
return
}
if it.ConnectDone != nil {
it.ConnectDone(time.Since(start).Seconds())
}
},
GotFirstResponseByte: func() {
if it.GotFirstResponseByte != nil {
it.GotFirstResponseByte(time.Since(start).Seconds())
}
},
Got100Continue: func() {
if it.Got100Continue != nil {
it.Got100Continue(time.Since(start).Seconds())
}
},
TLSHandshakeStart: func() {
if it.TLSHandshakeStart != nil {
it.TLSHandshakeStart(time.Since(start).Seconds())
}
},
TLSHandshakeDone: func(_ tls.ConnectionState, err error) {
if err != nil {
return
}
if it.TLSHandshakeDone != nil {
it.TLSHandshakeDone(time.Since(start).Seconds())
}
},
WroteHeaders: func() {
if it.WroteHeaders != nil {
it.WroteHeaders(time.Since(start).Seconds())
}
},
Wait100Continue: func() {
if it.Wait100Continue != nil {
it.Wait100Continue(time.Since(start).Seconds())
}
},
WroteRequest: func(_ httptrace.WroteRequestInfo) {
if it.WroteRequest != nil {
it.WroteRequest(time.Since(start).Seconds())
}
},
}
r = r.WithContext(httptrace.WithClientTrace(context.Background(), trace))
return next.RoundTrip(r)
})
}

View file

@ -325,9 +325,17 @@ func (r *Registry) Register(c Collector) error {
return nil
}
if existing, exists := r.collectorsByID[collectorID]; exists {
return AlreadyRegisteredError{
ExistingCollector: existing,
NewCollector: c,
switch e := existing.(type) {
case *wrappingCollector:
return AlreadyRegisteredError{
ExistingCollector: e.unwrapRecursively(),
NewCollector: c,
}
default:
return AlreadyRegisteredError{
ExistingCollector: e,
NewCollector: c,
}
}
}
// If the collectorID is new, but at least one of the descs existed

View file

@ -39,7 +39,7 @@ const quantileLabel = "quantile"
// A typical use-case is the observation of request latencies. By default, a
// Summary provides the median, the 90th and the 99th percentile of the latency
// as rank estimations. However, the default behavior will change in the
// upcoming v0.10 of the library. There will be no rank estimations at all by
// upcoming v1.0.0 of the library. There will be no rank estimations at all by
// default. For a sane transition, it is recommended to set the desired rank
// estimations explicitly.
//
@ -58,16 +58,8 @@ type Summary interface {
Observe(float64)
}
// DefObjectives are the default Summary quantile values.
//
// Deprecated: DefObjectives will not be used as the default objectives in
// v0.10 of the library. The default Summary will have no quantiles then.
var (
DefObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}
errQuantileLabelNotAllowed = fmt.Errorf(
"%q is not allowed as label name in summaries", quantileLabel,
)
var errQuantileLabelNotAllowed = fmt.Errorf(
"%q is not allowed as label name in summaries", quantileLabel,
)
// Default values for SummaryOpts.
@ -86,7 +78,7 @@ const (
// mandatory to set Name to a non-empty string. While all other fields are
// optional and can safely be left at their zero value, it is recommended to set
// a help string and to explicitly set the Objectives field to the desired value
// as the default value will change in the upcoming v0.10 of the library.
// as the default value will change in the upcoming v1.0.0 of the library.
type SummaryOpts struct {
// Namespace, Subsystem, and Name are components of the fully-qualified
// name of the Summary (created by joining these components with
@ -123,13 +115,8 @@ type SummaryOpts struct {
// Objectives defines the quantile rank estimates with their respective
// absolute error. If Objectives[q] = e, then the value reported for q
// will be the φ-quantile value for some φ between q-e and q+e. The
// default value is DefObjectives. It is used if Objectives is left at
// its zero value (i.e. nil). To create a Summary without Objectives,
// set it to an empty map (i.e. map[float64]float64{}).
//
// Deprecated: Note that the current value of DefObjectives is
// deprecated. It will be replaced by an empty map in v0.10 of the
// library. Please explicitly set Objectives to the desired value.
// default value is an empty map, resulting in a summary without
// quantiles.
Objectives map[float64]float64
// MaxAge defines the duration for which an observation stays relevant
@ -198,7 +185,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
}
if opts.Objectives == nil {
opts.Objectives = DefObjectives
opts.Objectives = map[float64]float64{}
}
if opts.MaxAge < 0 {
@ -405,18 +392,21 @@ type summaryCounts struct {
}
type noObjectivesSummary struct {
// countAndHotIdx is a complicated one. For lock-free yet atomic
// observations, we need to save the total count of observations again,
// combined with the index of the currently-hot counts struct, so that
// we can perform the operation on both values atomically. The least
// significant bit defines the hot counts struct. The remaining 63 bits
// represent the total count of observations. This happens under the
// assumption that the 63bit count will never overflow. Rationale: An
// observations takes about 30ns. Let's assume it could happen in
// 10ns. Overflowing the counter will then take at least (2^63)*10ns,
// which is about 3000 years.
// countAndHotIdx enables lock-free writes with use of atomic updates.
// The most significant bit is the hot index [0 or 1] of the count field
// below. Observe calls update the hot one. All remaining bits count the
// number of Observe calls. Observe starts by incrementing this counter,
// and finish by incrementing the count field in the respective
// summaryCounts, as a marker for completion.
//
// This has to be first in the struct for 64bit alignment. See
// Calls of the Write method (which are non-mutating reads from the
// perspective of the summary) swap the hotcold under the writeMtx
// lock. A cooldown is awaited (while locked) by comparing the number of
// observations with the initiation count. Once they match, then the
// last observation on the now cool one has completed. All cool fields must
// be merged into the new hot before releasing writeMtx.
// Fields with atomic access first! See alignment constraint:
// http://golang.org/pkg/sync/atomic/#pkg-note-BUG
countAndHotIdx uint64
@ -429,7 +419,6 @@ type noObjectivesSummary struct {
// pointers to guarantee 64bit alignment of the histogramCounts, see
// http://golang.org/pkg/sync/atomic/#pkg-note-BUG.
counts [2]*summaryCounts
hotIdx int // Index of currently-hot counts. Only used within Write.
labelPairs []*dto.LabelPair
}
@ -439,11 +428,11 @@ func (s *noObjectivesSummary) Desc() *Desc {
}
func (s *noObjectivesSummary) Observe(v float64) {
// We increment s.countAndHotIdx by 2 so that the counter in the upper
// 63 bits gets incremented by 1. At the same time, we get the new value
// We increment h.countAndHotIdx so that the counter in the lower
// 63 bits gets incremented. At the same time, we get the new value
// back, which we can use to find the currently-hot counts.
n := atomic.AddUint64(&s.countAndHotIdx, 2)
hotCounts := s.counts[n%2]
n := atomic.AddUint64(&s.countAndHotIdx, 1)
hotCounts := s.counts[n>>63]
for {
oldBits := atomic.LoadUint64(&hotCounts.sumBits)
@ -458,61 +447,33 @@ func (s *noObjectivesSummary) Observe(v float64) {
}
func (s *noObjectivesSummary) Write(out *dto.Metric) error {
var (
sum = &dto.Summary{}
hotCounts, coldCounts *summaryCounts
count uint64
)
// For simplicity, we mutex the rest of this method. It is not in the
// hot path, i.e. Observe is called much more often than Write. The
// complication of making Write lock-free isn't worth it.
// For simplicity, we protect this whole method by a mutex. It is not in
// the hot path, i.e. Observe is called much more often than Write. The
// complication of making Write lock-free isn't worth it, if possible at
// all.
s.writeMtx.Lock()
defer s.writeMtx.Unlock()
// This is a bit arcane, which is why the following spells out this if
// clause in English:
//
// If the currently-hot counts struct is #0, we atomically increment
// s.countAndHotIdx by 1 so that from now on Observe will use the counts
// struct #1. Furthermore, the atomic increment gives us the new value,
// which, in its most significant 63 bits, tells us the count of
// observations done so far up to and including currently ongoing
// observations still using the counts struct just changed from hot to
// cold. To have a normal uint64 for the count, we bitshift by 1 and
// save the result in count. We also set s.hotIdx to 1 for the next
// Write call, and we will refer to counts #1 as hotCounts and to counts
// #0 as coldCounts.
//
// If the currently-hot counts struct is #1, we do the corresponding
// things the other way round. We have to _decrement_ s.countAndHotIdx
// (which is a bit arcane in itself, as we have to express -1 with an
// unsigned int...).
if s.hotIdx == 0 {
count = atomic.AddUint64(&s.countAndHotIdx, 1) >> 1
s.hotIdx = 1
hotCounts = s.counts[1]
coldCounts = s.counts[0]
} else {
count = atomic.AddUint64(&s.countAndHotIdx, ^uint64(0)) >> 1 // Decrement.
s.hotIdx = 0
hotCounts = s.counts[0]
coldCounts = s.counts[1]
}
// Adding 1<<63 switches the hot index (from 0 to 1 or from 1 to 0)
// without touching the count bits. See the struct comments for a full
// description of the algorithm.
n := atomic.AddUint64(&s.countAndHotIdx, 1<<63)
// count is contained unchanged in the lower 63 bits.
count := n & ((1 << 63) - 1)
// The most significant bit tells us which counts is hot. The complement
// is thus the cold one.
hotCounts := s.counts[n>>63]
coldCounts := s.counts[(^n)>>63]
// Now we have to wait for the now-declared-cold counts to actually cool
// down, i.e. wait for all observations still using it to finish. That's
// the case once the count in the cold counts struct is the same as the
// one atomically retrieved from the upper 63bits of s.countAndHotIdx.
for {
if count == atomic.LoadUint64(&coldCounts.count) {
break
}
// Await cooldown.
for count != atomic.LoadUint64(&coldCounts.count) {
runtime.Gosched() // Let observations get work done.
}
sum.SampleCount = proto.Uint64(count)
sum.SampleSum = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits)))
sum := &dto.Summary{
SampleCount: proto.Uint64(count),
SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
}
out.Summary = sum
out.Label = s.labelPairs

View file

@ -32,6 +32,12 @@ import (
// WrapRegistererWith provides a way to add fixed labels to a subset of
// Collectors. It should not be used to add fixed labels to all metrics exposed.
//
// Conflicts between Collectors registered through the original Registerer with
// Collectors registered through the wrapping Registerer will still be
// detected. Any AlreadyRegisteredError returned by the Register method of
// either Registerer will contain the ExistingCollector in the form it was
// provided to the respective registry.
//
// The Collector example demonstrates a use of WrapRegistererWith.
func WrapRegistererWith(labels Labels, reg Registerer) Registerer {
return &wrappingRegisterer{
@ -54,6 +60,12 @@ func WrapRegistererWith(labels Labels, reg Registerer) Registerer {
// (see NewGoCollector) and the process collector (see NewProcessCollector). (In
// fact, those metrics are already prefixed with “go_” or “process_”,
// respectively.)
//
// Conflicts between Collectors registered through the original Registerer with
// Collectors registered through the wrapping Registerer will still be
// detected. Any AlreadyRegisteredError returned by the Register method of
// either Registerer will contain the ExistingCollector in the form it was
// provided to the respective registry.
func WrapRegistererWithPrefix(prefix string, reg Registerer) Registerer {
return &wrappingRegisterer{
wrappedRegisterer: reg,
@ -123,6 +135,15 @@ func (c *wrappingCollector) Describe(ch chan<- *Desc) {
}
}
func (c *wrappingCollector) unwrapRecursively() Collector {
switch wc := c.wrappedCollector.(type) {
case *wrappingCollector:
return wc.unwrapRecursively()
default:
return wc
}
}
type wrappingMetric struct {
wrappedMetric Metric
prefix string

View file

@ -150,7 +150,13 @@ func (t *Time) UnmarshalJSON(b []byte) error {
return err
}
*t = Time(v + va)
// If the value was something like -0.1 the negative is lost in the
// parsing because of the leading zero, this ensures that we capture it.
if len(p[0]) > 0 && p[0][0] == '-' && v+va > 0 {
*t = Time(v+va) * -1
} else {
*t = Time(v + va)
}
default:
return fmt.Errorf("invalid time %q", string(b))

6
vendor/github.com/prometheus/procfs/.golangci.yml generated vendored Normal file
View file

@ -0,0 +1,6 @@
# Run only staticcheck for now. Additional linters will be enabled one-by-one.
linters:
enable:
- staticcheck
- govet
disable-all: true

View file

@ -1,2 +1,2 @@
* Tobias Schmidt <tobidt@gmail.com> @grobie
* Johannes 'fish' Ziemke <github@freigeist.org> @discordianfish
* Paul Gier <pgier@redhat.com> @pgier

View file

@ -14,6 +14,7 @@
include Makefile.common
%/.unpacked: %.ttar
@echo ">> extracting fixtures"
./ttar -C $(dir $*) -x -f $*.ttar
touch $@

View file

@ -29,12 +29,15 @@ GO ?= go
GOFMT ?= $(GO)fmt
FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH)))
GOOPTS ?=
GOHOSTOS ?= $(shell $(GO) env GOHOSTOS)
GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH)
GO_VERSION ?= $(shell $(GO) version)
GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION))
PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.')
unexport GOVENDOR
GOVENDOR :=
GO111MODULE :=
ifeq (, $(PRE_GO_111))
ifneq (,$(wildcard go.mod))
# Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI).
@ -55,32 +58,57 @@ $(warning Some recipes may not work as expected as the current Go runtime is '$(
# This repository isn't using Go modules (yet).
GOVENDOR := $(FIRST_GOPATH)/bin/govendor
endif
unexport GO111MODULE
endif
PROMU := $(FIRST_GOPATH)/bin/promu
STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck
pkgs = ./...
GO_VERSION ?= $(shell $(GO) version)
GO_BUILD_PLATFORM ?= $(subst /,-,$(lastword $(GO_VERSION)))
ifeq (arm, $(GOHOSTARCH))
GOHOSTARM ?= $(shell GOARM= $(GO) env GOARM)
GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)v$(GOHOSTARM)
else
GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)
endif
PROMU_VERSION ?= 0.2.0
PROMU_VERSION ?= 0.4.0
PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
GOLANGCI_LINT :=
GOLANGCI_LINT_OPTS ?=
GOLANGCI_LINT_VERSION ?= v1.16.0
# golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
# windows isn't included here because of the path separator being different.
ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
endif
endif
PREFIX ?= $(shell pwd)
BIN_DIR ?= $(shell pwd)
DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
DOCKER_REPO ?= prom
.PHONY: all
all: precheck style staticcheck unused build test
DOCKER_ARCHS ?= amd64
BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS))
PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS))
TAG_DOCKER_ARCHS = $(addprefix common-docker-tag-latest-,$(DOCKER_ARCHS))
ifeq ($(GOHOSTARCH),amd64)
ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin windows))
# Only supported on amd64
test-flags := -race
endif
endif
# This rule is used to forward a target like "build" to "common-build". This
# allows a new "build" target to be defined in a Makefile which includes this
# one and override "common-build" without override warnings.
%: common-% ;
.PHONY: common-all
common-all: precheck style check_license lint unused build test
.PHONY: common-style
common-style:
@echo ">> checking code style"
@ -102,6 +130,15 @@ common-check_license:
exit 1; \
fi
.PHONY: common-deps
common-deps:
@echo ">> getting dependencies"
ifdef GO111MODULE
GO111MODULE=$(GO111MODULE) $(GO) mod download
else
$(GO) get $(GOOPTS) -t ./...
endif
.PHONY: common-test-short
common-test-short:
@echo ">> running short tests"
@ -110,26 +147,35 @@ common-test-short:
.PHONY: common-test
common-test:
@echo ">> running all tests"
GO111MODULE=$(GO111MODULE) $(GO) test -race $(GOOPTS) $(pkgs)
GO111MODULE=$(GO111MODULE) $(GO) test $(test-flags) $(GOOPTS) $(pkgs)
.PHONY: common-format
common-format:
@echo ">> formatting code"
GO111MODULE=$(GO111MODULE) $(GO) fmt $(GOOPTS) $(pkgs)
GO111MODULE=$(GO111MODULE) $(GO) fmt $(pkgs)
.PHONY: common-vet
common-vet:
@echo ">> vetting code"
GO111MODULE=$(GO111MODULE) $(GO) vet $(GOOPTS) $(pkgs)
.PHONY: common-staticcheck
common-staticcheck: $(STATICCHECK)
@echo ">> running staticcheck"
.PHONY: common-lint
common-lint: $(GOLANGCI_LINT)
ifdef GOLANGCI_LINT
@echo ">> running golangci-lint"
ifdef GO111MODULE
GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" -checks "SA*" $(pkgs)
# 'go list' needs to be executed before staticcheck to prepopulate the modules cache.
# Otherwise staticcheck might fail randomly for some reason not yet explained.
GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null
GO111MODULE=$(GO111MODULE) $(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs)
else
$(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs)
$(GOLANGCI_LINT) run $(pkgs)
endif
endif
# For backward-compatibility.
.PHONY: common-staticcheck
common-staticcheck: lint
.PHONY: common-unused
common-unused: $(GOVENDOR)
@ -140,8 +186,9 @@ else
ifdef GO111MODULE
@echo ">> running check for unused/missing packages in go.mod"
GO111MODULE=$(GO111MODULE) $(GO) mod tidy
ifeq (,$(wildcard vendor))
@git diff --exit-code -- go.sum go.mod
ifneq (,$(wildcard vendor))
else
@echo ">> running check for unused packages in vendor/"
GO111MODULE=$(GO111MODULE) $(GO) mod vendor
@git diff --exit-code -- go.sum go.mod vendor/
@ -159,45 +206,48 @@ common-tarball: promu
@echo ">> building release tarball"
$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
.PHONY: common-docker
common-docker:
docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
.PHONY: common-docker $(BUILD_DOCKER_ARCHS)
common-docker: $(BUILD_DOCKER_ARCHS)
$(BUILD_DOCKER_ARCHS): common-docker-%:
docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" \
--build-arg ARCH="$*" \
--build-arg OS="linux" \
.
.PHONY: common-docker-publish
common-docker-publish:
docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)"
.PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS)
common-docker-publish: $(PUBLISH_DOCKER_ARCHS)
$(PUBLISH_DOCKER_ARCHS): common-docker-publish-%:
docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)"
.PHONY: common-docker-tag-latest
common-docker-tag-latest:
docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):latest"
.PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS)
common-docker-tag-latest: $(TAG_DOCKER_ARCHS)
$(TAG_DOCKER_ARCHS): common-docker-tag-latest-%:
docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest"
.PHONY: common-docker-manifest
common-docker-manifest:
DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(DOCKER_IMAGE_TAG))
DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)"
.PHONY: promu
promu: $(PROMU)
$(PROMU):
curl -s -L $(PROMU_URL) | tar -xvz -C /tmp
mkdir -v -p $(FIRST_GOPATH)/bin
cp -v /tmp/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(PROMU)
$(eval PROMU_TMP := $(shell mktemp -d))
curl -s -L $(PROMU_URL) | tar -xvzf - -C $(PROMU_TMP)
mkdir -p $(FIRST_GOPATH)/bin
cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu
rm -r $(PROMU_TMP)
.PHONY: proto
proto:
@echo ">> generating code from proto files"
@./scripts/genproto.sh
.PHONY: $(STATICCHECK)
$(STATICCHECK):
ifdef GO111MODULE
# Get staticcheck from a temporary directory to avoid modifying the local go.{mod,sum}.
# See https://github.com/golang/go/issues/27643.
# For now, we are using the next branch of staticcheck because master isn't compatible yet with Go modules.
tmpModule=$$(mktemp -d 2>&1) && \
mkdir -p $${tmpModule}/staticcheck && \
cd "$${tmpModule}"/staticcheck && \
GO111MODULE=on $(GO) mod init example.com/staticcheck && \
GO111MODULE=on GOOS= GOARCH= $(GO) get -u honnef.co/go/tools/cmd/staticcheck@next && \
rm -rf $${tmpModule};
else
GOOS= GOARCH= GO111MODULE=off $(GO) get -u honnef.co/go/tools/cmd/staticcheck
ifdef GOLANGCI_LINT
$(GOLANGCI_LINT):
mkdir -p $(FIRST_GOPATH)/bin
curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(FIRST_GOPATH)/bin $(GOLANGCI_LINT_VERSION)
endif
ifdef GOVENDOR
@ -212,7 +262,6 @@ precheck::
define PRECHECK_COMMAND_template =
precheck:: $(1)_precheck
PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1)))
.PHONY: $(1)_precheck
$(1)_precheck:

View file

@ -1,7 +1,7 @@
# procfs
This procfs package provides functions to retrieve system, kernel and process
metrics from the pseudo-filesystem proc.
metrics from the pseudo-filesystems /proc and /sys.
*WARNING*: This package is a work in progress. Its API may still break in
backwards-incompatible ways without warnings. Use it at your own risk.
@ -9,3 +9,45 @@ backwards-incompatible ways without warnings. Use it at your own risk.
[![GoDoc](https://godoc.org/github.com/prometheus/procfs?status.png)](https://godoc.org/github.com/prometheus/procfs)
[![Build Status](https://travis-ci.org/prometheus/procfs.svg?branch=master)](https://travis-ci.org/prometheus/procfs)
[![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/procfs)](https://goreportcard.com/report/github.com/prometheus/procfs)
## Usage
The procfs library is organized by packages based on whether the gathered data is coming from
/proc, /sys, or both. Each package contains an `FS` type which represents the path to either /proc, /sys, or both. For example, current cpu statistics are gathered from
`/proc/stat` and are available via the root procfs package. First, the proc filesystem mount
point is initialized, and then the stat information is read.
```go
fs, err := procfs.NewFS("/proc")
stats, err := fs.Stat()
```
Some sub-packages such as `blockdevice`, require access to both the proc and sys filesystems.
```go
fs, err := blockdevice.NewFS("/proc", "/sys")
stats, err := fs.ProcDiskstats()
```
## Building and Testing
The procfs library is normally built as part of another application. However, when making
changes to the library, the `make test` command can be used to run the API test suite.
### Updating Test Fixtures
The procfs library includes a set of test fixtures which include many example files from
the `/proc` and `/sys` filesystems. These fixtures are included as a [ttar](https://github.com/ideaship/ttar) file
which is extracted automatically during testing. To add/update the test fixtures, first
ensure the `fixtures` directory is up to date by removing the existing directory and then
extracting the ttar file using `make fixtures/.unpacked` or just `make test`.
```bash
rm -rf fixtures
make test
```
Next, make the required changes to the extracted files in the `fixtures` directory. When
the changes are complete, run `make update_fixtures` to create a new `fixtures.ttar` file
based on the updated `fixtures` directory. And finally, verify the changes using
`git diff fixtures.ttar`.

View file

@ -31,19 +31,9 @@ type BuddyInfo struct {
Sizes []float64
}
// NewBuddyInfo reads the buddyinfo statistics.
func NewBuddyInfo() ([]BuddyInfo, error) {
fs, err := NewFS(DefaultMountPoint)
if err != nil {
return nil, err
}
return fs.NewBuddyInfo()
}
// NewBuddyInfo reads the buddyinfo statistics from the specified `proc` filesystem.
func (fs FS) NewBuddyInfo() ([]BuddyInfo, error) {
file, err := os.Open(fs.Path("buddyinfo"))
func (fs FS) BuddyInfo() ([]BuddyInfo, error) {
file, err := os.Open(fs.proc.Path("buddyinfo"))
if err != nil {
return nil, err
}

View file

@ -75,13 +75,13 @@ Max realtime timeout unlimited unlimited us
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/26231/mountstats
Lines: 19
Lines: 20
device rootfs mounted on / with fstype rootfs
device sysfs mounted on /sys with fstype sysfs
device proc mounted on /proc with fstype proc
device /dev/sda1 mounted on / with fstype ext4
device 192.168.1.1:/srv/test mounted on /mnt/nfs/test with fstype nfs4 statvers=1.1
opts: rw,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=192.168.1.5,local_lock=none
opts: rw,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,mountaddr=192.168.1.1,clientaddr=192.168.1.5,local_lock=none
age: 13968
caps: caps=0xfff7,wtmult=512,dtsize=32768,bsize=0,namlen=255
nfsv4: bm0=0xfdffafff,bm1=0xf9be3e,bm2=0x0,acl=0x0,pnfs=not configured
@ -94,6 +94,7 @@ device 192.168.1.1:/srv/test mounted on /mnt/nfs/test with fstype nfs4 statvers=
NULL: 0 0 0 0 0 0 0 0
READ: 1298 1298 0 207680 1210292152 6 79386 79407
WRITE: 0 0 0 0 0 0 0 0
ACCESS: 2927395007 2927394995 0 526931094212 362996810236 18446743919241604546 1667369447 1953587717
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@ -125,6 +126,63 @@ Lines: 1
26231 (vim) R 5392 7446 5392 34835 7446 4218880 32533 309516 26 82 1677 44 158 99 20 0 1 0 82375 56274944 1981 18446744073709551615 4194304 6294284 140736914091744 140736914087944 139965136429984 0 0 12288 1870679807 0 0 0 17 0 0 0 31 0 0 8391624 8481048 16420864 140736914093252 140736914093279 140736914093279 140736914096107 0
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/26231/status
Lines: 53
Name: prometheus
Umask: 0022
State: S (sleeping)
Tgid: 1
Ngid: 0
Pid: 1
PPid: 0
TracerPid: 0
Uid: 0 0 0 0
Gid: 0 0 0 0
FDSize: 128
Groups:
NStgid: 1
NSpid: 1
NSpgid: 1
NSsid: 1
VmPeak: 58472 kB
VmSize: 58440 kB
VmLck: 0 kB
VmPin: 0 kB
VmHWM: 8028 kB
VmRSS: 6716 kB
RssAnon: 2092 kB
RssFile: 4624 kB
RssShmem: 0 kB
VmData: 2580 kB
VmStk: 136 kB
VmExe: 948 kB
VmLib: 6816 kB
VmPTE: 128 kB
VmPMD: 12 kB
VmSwap: 660 kB
HugetlbPages: 0 kB
Threads: 1
SigQ: 8/63965
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 7be3c0fe28014a03
SigIgn: 0000000000001000
SigCgt: 00000001800004ec
CapInh: 0000000000000000
CapPrm: 0000003fffffffff
CapEff: 0000003fffffffff
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000
Seccomp: 0
Cpus_allowed: ff
Cpus_allowed_list: 0-7
Mems_allowed: 00000000,00000001
Mems_allowed_list: 0
voluntary_ctxt_switches: 4742839
nonvoluntary_ctxt_switches: 1727500
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Directory: fixtures/proc/26232
Mode: 755
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@ -160,23 +218,23 @@ SymlinkTo: ../../symlinktargets/xyz
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/26232/limits
Lines: 17
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 29436 29436 processes
Max open files 1024 4096 files
Max locked memory 65536 65536 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 29436 29436 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 29436 29436 processes
Max open files 1024 4096 files
Max locked memory 65536 65536 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 29436 29436 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/26232/root
@ -206,9 +264,9 @@ Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/buddyinfo
Lines: 3
Node 0, zone DMA 1 0 1 0 2 1 1 0 1 1 3
Node 0, zone DMA32 759 572 791 475 194 45 12 0 0 0 0
Node 0, zone Normal 4381 1093 185 1530 567 102 4 0 0 0 0
Node 0, zone DMA 1 0 1 0 2 1 1 0 1 1 3
Node 0, zone DMA32 759 572 791 475 194 45 12 0 0 0 0
Node 0, zone Normal 4381 1093 185 1530 567 102 4 0 0 0 0
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/diskstats
@ -302,13 +360,13 @@ Lines: 26
Personalities : [linear] [multipath] [raid0] [raid1] [raid6] [raid5] [raid4] [raid10]
md3 : active raid6 sda1[8] sdh1[7] sdg1[6] sdf1[5] sde1[11] sdd1[3] sdc1[10] sdb1[9]
5853468288 blocks super 1.2 level 6, 64k chunk, algorithm 2 [8/8] [UUUUUUUU]
md127 : active raid1 sdi2[0] sdj2[1]
312319552 blocks [2/2] [UU]
md0 : active raid1 sdk[2](S) sdi1[0] sdj1[1]
248896 blocks [2/2] [UU]
md4 : inactive raid1 sda3[0] sdb3[1]
4883648 blocks [2/2] [UU]
@ -402,6 +460,26 @@ proc4 2 2 10853
proc4ops 72 0 0 0 1098 2 0 0 0 0 8179 5896 0 0 0 0 5900 0 0 2 0 2 0 9609 0 2 150 1272 0 0 0 1236 0 0 0 0 3 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/net/unix
Lines: 6
Num RefCount Protocol Flags Type St Inode Path
0000000000000000: 00000002 00000000 00010000 0001 01 3442596 /var/run/postgresql/.s.PGSQL.5432
0000000000000000: 0000000a 00000000 00010000 0005 01 10061 /run/udev/control
0000000000000000: 00000007 00000000 00000000 0002 01 12392 /dev/log
0000000000000000: 00000003 00000000 00000000 0001 03 4787297 /var/run/postgresql/.s.PGSQL.5432
0000000000000000: 00000003 00000000 00000000 0001 03 5091797
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/net/unix_without_inode
Lines: 6
Num RefCount Protocol Flags Type St Path
0000000000000000: 00000002 00000000 00010000 0001 01 /var/run/postgresql/.s.PGSQL.5432
0000000000000000: 0000000a 00000000 00010000 0005 01 /run/udev/control
0000000000000000: 00000007 00000000 00000000 0002 01 /dev/log
0000000000000000: 00000003 00000000 00000000 0001 03 /var/run/postgresql/.s.PGSQL.5432
0000000000000000: 00000003 00000000 00000000 0001 03
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/proc/net/xfrm_stat
Lines: 28
XfrmInError 1
@ -1107,6 +1185,22 @@ Mode: 644
Directory: fixtures/sys/devices/system
Mode: 775
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Directory: fixtures/sys/devices/system/clocksource
Mode: 775
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Directory: fixtures/sys/devices/system/clocksource/clocksource0
Mode: 775
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/sys/devices/system/clocksource/clocksource0/available_clocksource
Lines: 1
tsc hpet acpi_pm
Mode: 444
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Path: fixtures/sys/devices/system/clocksource/clocksource0/current_clocksource
Lines: 1
tsc
Mode: 644
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Directory: fixtures/sys/devices/system/cpu
Mode: 775
# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

View file

@ -14,33 +14,30 @@
package procfs
import (
"fmt"
"os"
"path"
"github.com/prometheus/procfs/internal/fs"
)
// FS represents the pseudo-filesystem proc, which provides an interface to
// FS represents the pseudo-filesystem sys, which provides an interface to
// kernel data structures.
type FS string
type FS struct {
proc fs.FS
}
// DefaultMountPoint is the common mount point of the proc filesystem.
const DefaultMountPoint = "/proc"
const DefaultMountPoint = fs.DefaultProcMountPoint
// NewFS returns a new FS mounted under the given mountPoint. It will error
// if the mount point can't be read.
// NewDefaultFS returns a new proc FS mounted under the default proc mountPoint.
// It will error if the mount point directory can't be read or is a file.
func NewDefaultFS() (FS, error) {
return NewFS(DefaultMountPoint)
}
// NewFS returns a new proc FS mounted under the given proc mountPoint. It will error
// if the mount point directory can't be read or is a file.
func NewFS(mountPoint string) (FS, error) {
info, err := os.Stat(mountPoint)
fs, err := fs.NewFS(mountPoint)
if err != nil {
return "", fmt.Errorf("could not read %s: %s", mountPoint, err)
return FS{}, err
}
if !info.IsDir() {
return "", fmt.Errorf("mount point %s is not a directory", mountPoint)
}
return FS(mountPoint), nil
}
// Path returns the path of the given subsystem relative to the procfs root.
func (fs FS) Path(p ...string) string {
return path.Join(append([]string{string(fs)}, p...)...)
return FS{fs}, nil
}

52
vendor/github.com/prometheus/procfs/internal/fs/fs.go generated vendored Normal file
View file

@ -0,0 +1,52 @@
// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fs
import (
"fmt"
"os"
"path/filepath"
)
const (
// DefaultProcMountPoint is the common mount point of the proc filesystem.
DefaultProcMountPoint = "/proc"
// DefaultSysMountPoint is the common mount point of the sys filesystem.
DefaultSysMountPoint = "/sys"
)
// FS represents a pseudo-filesystem, normally /proc or /sys, which provides an
// interface to kernel data structures.
type FS string
// NewFS returns a new FS mounted under the given mountPoint. It will error
// if the mount point can't be read.
func NewFS(mountPoint string) (FS, error) {
info, err := os.Stat(mountPoint)
if err != nil {
return "", fmt.Errorf("could not read %s: %s", mountPoint, err)
}
if !info.IsDir() {
return "", fmt.Errorf("mount point %s is not a directory", mountPoint)
}
return FS(mountPoint), nil
}
// Path appends the given path elements to the filesystem path, adding separators
// as necessary.
func (fs FS) Path(p ...string) string {
return filepath.Join(append([]string{string(fs)}, p...)...)
}

View file

@ -62,19 +62,9 @@ type IPVSBackendStatus struct {
Weight uint64
}
// NewIPVSStats reads the IPVS statistics.
func NewIPVSStats() (IPVSStats, error) {
fs, err := NewFS(DefaultMountPoint)
if err != nil {
return IPVSStats{}, err
}
return fs.NewIPVSStats()
}
// NewIPVSStats reads the IPVS statistics from the specified `proc` filesystem.
func (fs FS) NewIPVSStats() (IPVSStats, error) {
file, err := os.Open(fs.Path("net/ip_vs_stats"))
// IPVSStats reads the IPVS statistics from the specified `proc` filesystem.
func (fs FS) IPVSStats() (IPVSStats, error) {
file, err := os.Open(fs.proc.Path("net/ip_vs_stats"))
if err != nil {
return IPVSStats{}, err
}
@ -131,19 +121,9 @@ func parseIPVSStats(file io.Reader) (IPVSStats, error) {
return stats, nil
}
// NewIPVSBackendStatus reads and returns the status of all (virtual,real) server pairs.
func NewIPVSBackendStatus() ([]IPVSBackendStatus, error) {
fs, err := NewFS(DefaultMountPoint)
if err != nil {
return []IPVSBackendStatus{}, err
}
return fs.NewIPVSBackendStatus()
}
// NewIPVSBackendStatus reads and returns the status of all (virtual,real) server pairs from the specified `proc` filesystem.
func (fs FS) NewIPVSBackendStatus() ([]IPVSBackendStatus, error) {
file, err := os.Open(fs.Path("net/ip_vs"))
// IPVSBackendStatus reads and returns the status of all (virtual,real) server pairs from the specified `proc` filesystem.
func (fs FS) IPVSBackendStatus() ([]IPVSBackendStatus, error) {
file, err := os.Open(fs.proc.Path("net/ip_vs"))
if err != nil {
return nil, err
}

View file

@ -42,64 +42,64 @@ type MDStat struct {
BlocksSynced int64
}
// ParseMDStat parses an mdstat-file and returns a struct with the relevant infos.
func (fs FS) ParseMDStat() (mdstates []MDStat, err error) {
mdStatusFilePath := fs.Path("mdstat")
content, err := ioutil.ReadFile(mdStatusFilePath)
// MDStat parses an mdstat-file (/proc/mdstat) and returns a slice of
// structs containing the relevant info. More information available here:
// https://raid.wiki.kernel.org/index.php/Mdstat
func (fs FS) MDStat() ([]MDStat, error) {
data, err := ioutil.ReadFile(fs.proc.Path("mdstat"))
if err != nil {
return []MDStat{}, fmt.Errorf("error parsing %s: %s", mdStatusFilePath, err)
return nil, fmt.Errorf("error parsing mdstat %s: %s", fs.proc.Path("mdstat"), err)
}
mdstat, err := parseMDStat(data)
if err != nil {
return nil, fmt.Errorf("error parsing mdstat %s: %s", fs.proc.Path("mdstat"), err)
}
return mdstat, nil
}
mdStates := []MDStat{}
lines := strings.Split(string(content), "\n")
// parseMDStat parses data from mdstat file (/proc/mdstat) and returns a slice of
// structs containing the relevant info.
func parseMDStat(mdstatData []byte) ([]MDStat, error) {
mdStats := []MDStat{}
lines := strings.Split(string(mdstatData), "\n")
for i, l := range lines {
if l == "" {
continue
}
if l[0] == ' ' {
continue
}
if strings.HasPrefix(l, "Personalities") || strings.HasPrefix(l, "unused") {
if strings.TrimSpace(l) == "" || l[0] == ' ' ||
strings.HasPrefix(l, "Personalities") || strings.HasPrefix(l, "unused") {
continue
}
mainLine := strings.Split(l, " ")
if len(mainLine) < 3 {
return mdStates, fmt.Errorf("error parsing mdline: %s", l)
deviceFields := strings.Fields(l)
if len(deviceFields) < 3 {
return nil, fmt.Errorf("not enough fields in mdline (expected at least 3): %s", l)
}
mdName := mainLine[0]
activityState := mainLine[2]
mdName := deviceFields[0]
activityState := deviceFields[2]
if len(lines) <= i+3 {
return mdStates, fmt.Errorf(
"error parsing %s: too few lines for md device %s",
mdStatusFilePath,
mdName,
)
return mdStats, fmt.Errorf("missing lines for md device %s", mdName)
}
active, total, size, err := evalStatusline(lines[i+1])
active, total, size, err := evalStatusLine(lines[i+1])
if err != nil {
return mdStates, fmt.Errorf("error parsing %s: %s", mdStatusFilePath, err)
return nil, err
}
// j is the line number of the syncing-line.
j := i + 2
syncLineIdx := i + 2
if strings.Contains(lines[i+2], "bitmap") { // skip bitmap line
j = i + 3
syncLineIdx++
}
// If device is syncing at the moment, get the number of currently
// If device is recovering/syncing at the moment, get the number of currently
// synced bytes, otherwise that number equals the size of the device.
syncedBlocks := size
if strings.Contains(lines[j], "recovery") || strings.Contains(lines[j], "resync") {
syncedBlocks, err = evalBuildline(lines[j])
if strings.Contains(lines[syncLineIdx], "recovery") || strings.Contains(lines[syncLineIdx], "resync") {
syncedBlocks, err = evalRecoveryLine(lines[syncLineIdx])
if err != nil {
return mdStates, fmt.Errorf("error parsing %s: %s", mdStatusFilePath, err)
return nil, err
}
}
mdStates = append(mdStates, MDStat{
mdStats = append(mdStats, MDStat{
Name: mdName,
ActivityState: activityState,
DisksActive: active,
@ -109,10 +109,10 @@ func (fs FS) ParseMDStat() (mdstates []MDStat, err error) {
})
}
return mdStates, nil
return mdStats, nil
}
func evalStatusline(statusline string) (active, total, size int64, err error) {
func evalStatusLine(statusline string) (active, total, size int64, err error) {
matches := statuslineRE.FindStringSubmatch(statusline)
if len(matches) != 4 {
return 0, 0, 0, fmt.Errorf("unexpected statusline: %s", statusline)
@ -136,7 +136,7 @@ func evalStatusline(statusline string) (active, total, size int64, err error) {
return active, total, size, nil
}
func evalBuildline(buildline string) (syncedBlocks int64, err error) {
func evalRecoveryLine(buildline string) (syncedBlocks int64, err error) {
matches := buildlineRE.FindStringSubmatch(buildline)
if len(matches) != 2 {
return 0, fmt.Errorf("unexpected buildline: %s", buildline)

View file

@ -69,8 +69,8 @@ type MountStats interface {
type MountStatsNFS struct {
// The version of statistics provided.
StatVersion string
// The optional mountaddr of the NFS mount.
MountAddress string
// The mount options of the NFS mount.
Opts map[string]string
// The age of the NFS mount.
Age time.Duration
// Statistics related to byte counters for various operations.
@ -181,11 +181,11 @@ type NFSOperationStats struct {
// Number of bytes received for this operation, including RPC headers and payload.
BytesReceived uint64
// Duration all requests spent queued for transmission before they were sent.
CumulativeQueueTime time.Duration
CumulativeQueueMilliseconds uint64
// Duration it took to get a reply back after the request was transmitted.
CumulativeTotalResponseTime time.Duration
CumulativeTotalResponseMilliseconds uint64
// Duration from when a request was enqueued to when it was completely handled.
CumulativeTotalRequestTime time.Duration
CumulativeTotalRequestMilliseconds uint64
}
// A NFSTransportStats contains statistics for the NFS mount RPC requests and
@ -204,7 +204,7 @@ type NFSTransportStats struct {
// spent waiting for connections to the server to be established.
ConnectIdleTime uint64
// Duration since the NFS mount last saw any RPC traffic.
IdleTime time.Duration
IdleTimeSeconds uint64
// Number of RPC requests for this mount sent to the NFS server.
Sends uint64
// Number of RPC responses for this mount received from the NFS server.
@ -342,10 +342,15 @@ func parseMountStatsNFS(s *bufio.Scanner, statVersion string) (*MountStatsNFS, e
switch ss[0] {
case fieldOpts:
if stats.Opts == nil {
stats.Opts = map[string]string{}
}
for _, opt := range strings.Split(ss[1], ",") {
split := strings.Split(opt, "=")
if len(split) == 2 && split[0] == "mountaddr" {
stats.MountAddress = split[1]
if len(split) == 2 {
stats.Opts[split[0]] = split[1]
} else {
stats.Opts[opt] = ""
}
}
case fieldAge:
@ -519,15 +524,15 @@ func parseNFSOperationStats(s *bufio.Scanner) ([]NFSOperationStats, error) {
}
ops = append(ops, NFSOperationStats{
Operation: strings.TrimSuffix(ss[0], ":"),
Requests: ns[0],
Transmissions: ns[1],
MajorTimeouts: ns[2],
BytesSent: ns[3],
BytesReceived: ns[4],
CumulativeQueueTime: time.Duration(ns[5]) * time.Millisecond,
CumulativeTotalResponseTime: time.Duration(ns[6]) * time.Millisecond,
CumulativeTotalRequestTime: time.Duration(ns[7]) * time.Millisecond,
Operation: strings.TrimSuffix(ss[0], ":"),
Requests: ns[0],
Transmissions: ns[1],
MajorTimeouts: ns[2],
BytesSent: ns[3],
BytesReceived: ns[4],
CumulativeQueueMilliseconds: ns[5],
CumulativeTotalResponseMilliseconds: ns[6],
CumulativeTotalRequestMilliseconds: ns[7],
})
}
@ -603,7 +608,7 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
Bind: ns[1],
Connect: ns[2],
ConnectIdleTime: ns[3],
IdleTime: time.Duration(ns[4]) * time.Second,
IdleTimeSeconds: ns[4],
Sends: ns[5],
Receives: ns[6],
BadTransactionIDs: ns[7],

View file

@ -47,23 +47,13 @@ type NetDevLine struct {
// are interface names.
type NetDev map[string]NetDevLine
// NewNetDev returns kernel/system statistics read from /proc/net/dev.
func NewNetDev() (NetDev, error) {
fs, err := NewFS(DefaultMountPoint)
if err != nil {
return nil, err
}
return fs.NewNetDev()
// NetDev returns kernel/system statistics read from /proc/net/dev.
func (fs FS) NetDev() (NetDev, error) {
return newNetDev(fs.proc.Path("net/dev"))
}
// NewNetDev returns kernel/system statistics read from /proc/net/dev.
func (fs FS) NewNetDev() (NetDev, error) {
return newNetDev(fs.Path("net/dev"))
}
// NewNetDev returns kernel/system statistics read from /proc/[pid]/net/dev.
func (p Proc) NewNetDev() (NetDev, error) {
// NetDev returns kernel/system statistics read from /proc/[pid]/net/dev.
func (p Proc) NetDev() (NetDev, error) {
return newNetDev(p.path("net/dev"))
}
@ -75,7 +65,7 @@ func newNetDev(file string) (NetDev, error) {
}
defer f.Close()
nd := NetDev{}
netDev := NetDev{}
s := bufio.NewScanner(f)
for n := 0; s.Scan(); n++ {
// Skip the 2 header lines.
@ -83,20 +73,20 @@ func newNetDev(file string) (NetDev, error) {
continue
}
line, err := nd.parseLine(s.Text())
line, err := netDev.parseLine(s.Text())
if err != nil {
return nd, err
return netDev, err
}
nd[line.Name] = *line
netDev[line.Name] = *line
}
return nd, s.Err()
return netDev, s.Err()
}
// parseLine parses a single line from the /proc/net/dev file. Header lines
// must be filtered prior to calling this method.
func (nd NetDev) parseLine(rawLine string) (*NetDevLine, error) {
func (netDev NetDev) parseLine(rawLine string) (*NetDevLine, error) {
parts := strings.SplitN(rawLine, ":", 2)
if len(parts) != 2 {
return nil, errors.New("invalid net/dev line, missing colon")
@ -185,11 +175,11 @@ func (nd NetDev) parseLine(rawLine string) (*NetDevLine, error) {
// Total aggregates the values across interfaces and returns a new NetDevLine.
// The Name field will be a sorted comma separated list of interface names.
func (nd NetDev) Total() NetDevLine {
func (netDev NetDev) Total() NetDevLine {
total := NetDevLine{}
names := make([]string, 0, len(nd))
for _, ifc := range nd {
names := make([]string, 0, len(netDev))
for _, ifc := range netDev {
names = append(names, ifc.Name)
total.RxBytes += ifc.RxBytes
total.RxPackets += ifc.RxPackets

275
vendor/github.com/prometheus/procfs/net_unix.go generated vendored Normal file
View file

@ -0,0 +1,275 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package procfs
import (
"bufio"
"errors"
"fmt"
"io"
"os"
"strconv"
"strings"
)
// For the proc file format details,
// see https://elixir.bootlin.com/linux/v4.17/source/net/unix/af_unix.c#L2815
// and https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/net.h#L48.
const (
netUnixKernelPtrIdx = iota
netUnixRefCountIdx
_
netUnixFlagsIdx
netUnixTypeIdx
netUnixStateIdx
netUnixInodeIdx
// Inode and Path are optional.
netUnixStaticFieldsCnt = 6
)
const (
netUnixTypeStream = 1
netUnixTypeDgram = 2
netUnixTypeSeqpacket = 5
netUnixFlagListen = 1 << 16
netUnixStateUnconnected = 1
netUnixStateConnecting = 2
netUnixStateConnected = 3
netUnixStateDisconnected = 4
)
var errInvalidKernelPtrFmt = errors.New("Invalid Num(the kernel table slot number) format")
// NetUnixType is the type of the type field.
type NetUnixType uint64
// NetUnixFlags is the type of the flags field.
type NetUnixFlags uint64
// NetUnixState is the type of the state field.
type NetUnixState uint64
// NetUnixLine represents a line of /proc/net/unix.
type NetUnixLine struct {
KernelPtr string
RefCount uint64
Protocol uint64
Flags NetUnixFlags
Type NetUnixType
State NetUnixState
Inode uint64
Path string
}
// NetUnix holds the data read from /proc/net/unix.
type NetUnix struct {
Rows []*NetUnixLine
}
// NewNetUnix returns data read from /proc/net/unix.
func NewNetUnix() (*NetUnix, error) {
fs, err := NewFS(DefaultMountPoint)
if err != nil {
return nil, err
}
return fs.NewNetUnix()
}
// NewNetUnix returns data read from /proc/net/unix.
func (fs FS) NewNetUnix() (*NetUnix, error) {
return NewNetUnixByPath(fs.proc.Path("net/unix"))
}
// NewNetUnixByPath returns data read from /proc/net/unix by file path.
// It might returns an error with partial parsed data, if an error occur after some data parsed.
func NewNetUnixByPath(path string) (*NetUnix, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
return NewNetUnixByReader(f)
}
// NewNetUnixByReader returns data read from /proc/net/unix by a reader.
// It might returns an error with partial parsed data, if an error occur after some data parsed.
func NewNetUnixByReader(reader io.Reader) (*NetUnix, error) {
nu := &NetUnix{
Rows: make([]*NetUnixLine, 0, 32),
}
scanner := bufio.NewScanner(reader)
// Omit the header line.
scanner.Scan()
header := scanner.Text()
// From the man page of proc(5), it does not contain an Inode field,
// but in actually it exists.
// This code works for both cases.
hasInode := strings.Contains(header, "Inode")
minFieldsCnt := netUnixStaticFieldsCnt
if hasInode {
minFieldsCnt++
}
for scanner.Scan() {
line := scanner.Text()
item, err := nu.parseLine(line, hasInode, minFieldsCnt)
if err != nil {
return nu, err
}
nu.Rows = append(nu.Rows, item)
}
return nu, scanner.Err()
}
func (u *NetUnix) parseLine(line string, hasInode bool, minFieldsCnt int) (*NetUnixLine, error) {
fields := strings.Fields(line)
fieldsLen := len(fields)
if fieldsLen < minFieldsCnt {
return nil, fmt.Errorf(
"Parse Unix domain failed: expect at least %d fields but got %d",
minFieldsCnt, fieldsLen)
}
kernelPtr, err := u.parseKernelPtr(fields[netUnixKernelPtrIdx])
if err != nil {
return nil, fmt.Errorf("Parse Unix domain num(%s) failed: %s", fields[netUnixKernelPtrIdx], err)
}
users, err := u.parseUsers(fields[netUnixRefCountIdx])
if err != nil {
return nil, fmt.Errorf("Parse Unix domain ref count(%s) failed: %s", fields[netUnixRefCountIdx], err)
}
flags, err := u.parseFlags(fields[netUnixFlagsIdx])
if err != nil {
return nil, fmt.Errorf("Parse Unix domain flags(%s) failed: %s", fields[netUnixFlagsIdx], err)
}
typ, err := u.parseType(fields[netUnixTypeIdx])
if err != nil {
return nil, fmt.Errorf("Parse Unix domain type(%s) failed: %s", fields[netUnixTypeIdx], err)
}
state, err := u.parseState(fields[netUnixStateIdx])
if err != nil {
return nil, fmt.Errorf("Parse Unix domain state(%s) failed: %s", fields[netUnixStateIdx], err)
}
var inode uint64
if hasInode {
inodeStr := fields[netUnixInodeIdx]
inode, err = u.parseInode(inodeStr)
if err != nil {
return nil, fmt.Errorf("Parse Unix domain inode(%s) failed: %s", inodeStr, err)
}
}
nuLine := &NetUnixLine{
KernelPtr: kernelPtr,
RefCount: users,
Type: typ,
Flags: flags,
State: state,
Inode: inode,
}
// Path field is optional.
if fieldsLen > minFieldsCnt {
pathIdx := netUnixInodeIdx + 1
if !hasInode {
pathIdx--
}
nuLine.Path = fields[pathIdx]
}
return nuLine, nil
}
func (u NetUnix) parseKernelPtr(str string) (string, error) {
if !strings.HasSuffix(str, ":") {
return "", errInvalidKernelPtrFmt
}
return str[:len(str)-1], nil
}
func (u NetUnix) parseUsers(hexStr string) (uint64, error) {
return strconv.ParseUint(hexStr, 16, 32)
}
func (u NetUnix) parseProtocol(hexStr string) (uint64, error) {
return strconv.ParseUint(hexStr, 16, 32)
}
func (u NetUnix) parseType(hexStr string) (NetUnixType, error) {
typ, err := strconv.ParseUint(hexStr, 16, 16)
if err != nil {
return 0, err
}
return NetUnixType(typ), nil
}
func (u NetUnix) parseFlags(hexStr string) (NetUnixFlags, error) {
flags, err := strconv.ParseUint(hexStr, 16, 32)
if err != nil {
return 0, err
}
return NetUnixFlags(flags), nil
}
func (u NetUnix) parseState(hexStr string) (NetUnixState, error) {
st, err := strconv.ParseInt(hexStr, 16, 8)
if err != nil {
return 0, err
}
return NetUnixState(st), nil
}
func (u NetUnix) parseInode(inodeStr string) (uint64, error) {
return strconv.ParseUint(inodeStr, 10, 64)
}
func (t NetUnixType) String() string {
switch t {
case netUnixTypeStream:
return "stream"
case netUnixTypeDgram:
return "dgram"
case netUnixTypeSeqpacket:
return "seqpacket"
}
return "unknown"
}
func (f NetUnixFlags) String() string {
switch f {
case netUnixFlagListen:
return "listen"
default:
return "default"
}
}
func (s NetUnixState) String() string {
switch s {
case netUnixStateUnconnected:
return "unconnected"
case netUnixStateConnecting:
return "connecting"
case netUnixStateConnected:
return "connected"
case netUnixStateDisconnected:
return "disconnected"
}
return "unknown"
}

View file

@ -20,6 +20,8 @@ import (
"os"
"strconv"
"strings"
"github.com/prometheus/procfs/internal/fs"
)
// Proc provides information about a running process.
@ -27,7 +29,7 @@ type Proc struct {
// The process ID.
PID int
fs FS
fs fs.FS
}
// Procs represents a list of Proc structs.
@ -52,7 +54,7 @@ func NewProc(pid int) (Proc, error) {
if err != nil {
return Proc{}, err
}
return fs.NewProc(pid)
return fs.Proc(pid)
}
// AllProcs returns a list of all currently available processes under /proc.
@ -66,28 +68,35 @@ func AllProcs() (Procs, error) {
// Self returns a process for the current process.
func (fs FS) Self() (Proc, error) {
p, err := os.Readlink(fs.Path("self"))
p, err := os.Readlink(fs.proc.Path("self"))
if err != nil {
return Proc{}, err
}
pid, err := strconv.Atoi(strings.Replace(p, string(fs), "", -1))
pid, err := strconv.Atoi(strings.Replace(p, string(fs.proc), "", -1))
if err != nil {
return Proc{}, err
}
return fs.NewProc(pid)
return fs.Proc(pid)
}
// NewProc returns a process for the given pid.
//
// Deprecated: use fs.Proc() instead
func (fs FS) NewProc(pid int) (Proc, error) {
if _, err := os.Stat(fs.Path(strconv.Itoa(pid))); err != nil {
return fs.Proc(pid)
}
// Proc returns a process for the given pid.
func (fs FS) Proc(pid int) (Proc, error) {
if _, err := os.Stat(fs.proc.Path(strconv.Itoa(pid))); err != nil {
return Proc{}, err
}
return Proc{PID: pid, fs: fs}, nil
return Proc{PID: pid, fs: fs.proc}, nil
}
// AllProcs returns a list of all currently available processes.
func (fs FS) AllProcs() (Procs, error) {
d, err := os.Open(fs.Path())
d, err := os.Open(fs.proc.Path())
if err != nil {
return Procs{}, err
}
@ -104,7 +113,7 @@ func (fs FS) AllProcs() (Procs, error) {
if err != nil {
continue
}
p = append(p, Proc{PID: int(pid), fs: fs})
p = append(p, Proc{PID: int(pid), fs: fs.proc})
}
return p, nil

View file

@ -39,8 +39,8 @@ type ProcIO struct {
CancelledWriteBytes int64
}
// NewIO creates a new ProcIO instance from a given Proc instance.
func (p Proc) NewIO() (ProcIO, error) {
// IO creates a new ProcIO instance from a given Proc instance.
func (p Proc) IO() (ProcIO, error) {
pio := ProcIO{}
f, err := os.Open(p.path("io"))

View file

@ -78,7 +78,14 @@ var (
)
// NewLimits returns the current soft limits of the process.
//
// Deprecated: use p.Limits() instead
func (p Proc) NewLimits() (ProcLimits, error) {
return p.Limits()
}
// Limits returns the current soft limits of the process.
func (p Proc) Limits() (ProcLimits, error) {
f, err := os.Open(p.path("limits"))
if err != nil {
return ProcLimits{}, err

View file

@ -29,9 +29,9 @@ type Namespace struct {
// Namespaces contains all of the namespaces that the process is contained in.
type Namespaces map[string]Namespace
// NewNamespaces reads from /proc/[pid/ns/* to get the namespaces of which the
// Namespaces reads from /proc/<pid>/ns/* to get the namespaces of which the
// process is a member.
func (p Proc) NewNamespaces() (Namespaces, error) {
func (p Proc) Namespaces() (Namespaces, error) {
d, err := os.Open(p.path("ns"))
if err != nil {
return nil, err

View file

@ -51,20 +51,11 @@ type PSIStats struct {
Full *PSILine
}
// NewPSIStatsForResource reads pressure stall information for the specified
// resource. At time of writing this can be either "cpu", "memory" or "io".
func NewPSIStatsForResource(resource string) (PSIStats, error) {
fs, err := NewFS(DefaultMountPoint)
if err != nil {
return PSIStats{}, err
}
return fs.NewPSIStatsForResource(resource)
}
// NewPSIStatsForResource reads pressure stall information from /proc/pressure/<resource>
func (fs FS) NewPSIStatsForResource(resource string) (PSIStats, error) {
file, err := os.Open(fs.Path(fmt.Sprintf("%s/%s", "pressure", resource)))
// PSIStatsForResource reads pressure stall information for the specified
// resource from /proc/pressure/<resource>. At time of writing this can be
// either "cpu", "memory" or "io".
func (fs FS) PSIStatsForResource(resource string) (PSIStats, error) {
file, err := os.Open(fs.proc.Path(fmt.Sprintf("%s/%s", "pressure", resource)))
if err != nil {
return PSIStats{}, fmt.Errorf("psi_stats: unavailable for %s", resource)
}

View file

@ -18,6 +18,8 @@ import (
"fmt"
"io/ioutil"
"os"
"github.com/prometheus/procfs/internal/fs"
)
// Originally, this USER_HZ value was dynamically retrieved via a sysconf call
@ -99,11 +101,18 @@ type ProcStat struct {
// Resident set size in pages.
RSS int
fs FS
proc fs.FS
}
// NewStat returns the current status information of the process.
//
// Deprecated: use NewStat() instead
func (p Proc) NewStat() (ProcStat, error) {
return p.Stat()
}
// Stat returns the current status information of the process.
func (p Proc) Stat() (ProcStat, error) {
f, err := os.Open(p.path("stat"))
if err != nil {
return ProcStat{}, err
@ -118,7 +127,7 @@ func (p Proc) NewStat() (ProcStat, error) {
var (
ignore int
s = ProcStat{PID: p.PID, fs: p.fs}
s = ProcStat{PID: p.PID, proc: p.fs}
l = bytes.Index(data, []byte("("))
r = bytes.LastIndex(data, []byte(")"))
)
@ -175,7 +184,8 @@ func (s ProcStat) ResidentMemory() int {
// StartTime returns the unix timestamp of the process in seconds.
func (s ProcStat) StartTime() (float64, error) {
stat, err := s.fs.NewStat()
fs := FS{proc: s.proc}
stat, err := fs.Stat()
if err != nil {
return 0, err
}

Some files were not shown because too many files have changed in this diff Show more