DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update go dependencies

Browse source
This commit is contained in:
Manuel Alejandro de Brito Fontes 2019-03-28 20:43:46 -03:00
parent 14a9e9f3fa
commit 14f4a7b8e8
No known key found for this signature in database
GPG key ID: 786136016A8BA02A
1349 changed files with 128369 additions and 32627 deletions
Download patch file Download diff file Expand all files Collapse all files

3
vendor/k8s.io/kubernetes/pkg/.import-restrictions generated vendored
View file

@ -6,7 +6,8 @@
""
],
"ForbiddenPrefixes": [
"k8s.io/kubernetes/cmd"
"k8s.io/kubernetes/cmd",
"github.com/ghodss/yaml"
]
}
]

3
vendor/k8s.io/kubernetes/pkg/BUILD generated vendored
View file

@ -19,11 +19,11 @@ filegroup(
"//pkg/api/podsecuritypolicy:all-srcs",
"//pkg/api/ref:all-srcs",
"//pkg/api/resource:all-srcs",
"//pkg/api/resourcequota:all-srcs",
"//pkg/api/service:all-srcs",
"//pkg/api/testapi:all-srcs",
"//pkg/api/testing:all-srcs",
"//pkg/api/v1/endpoints:all-srcs",
"//pkg/api/v1/node:all-srcs",
"//pkg/api/v1/persistentvolume:all-srcs",
"//pkg/api/v1/pod:all-srcs",
"//pkg/api/v1/resource:all-srcs",
@ -44,6 +44,7 @@ filegroup(
"//pkg/apis/extensions:all-srcs",
"//pkg/apis/imagepolicy:all-srcs",
"//pkg/apis/networking:all-srcs",
"//pkg/apis/node:all-srcs",
"//pkg/apis/policy:all-srcs",
"//pkg/apis/rbac:all-srcs",
"//pkg/apis/scheduling:all-srcs",

2
vendor/k8s.io/kubernetes/pkg/OWNERS generated vendored
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- brendandburns
- dchen1107

1
vendor/k8s.io/kubernetes/pkg/api/OWNERS generated vendored
View file

@ -1,4 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
# Disable inheritance as this is an api owners file
options:
no_parent_owners: true

2
vendor/k8s.io/kubernetes/pkg/api/v1/OWNERS generated vendored Executable file → Normal file
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- thockin
- lavalamp

4
vendor/k8s.io/kubernetes/pkg/api/v1/pod/util.go generated vendored
View file

@ -120,6 +120,10 @@ func VisitPodSecretNames(pod *v1.Pod, visitor Visitor) bool {
if source.StorageOS.SecretRef != nil && !visitor(source.StorageOS.SecretRef.Name) {
return false
}
case source.CSI != nil:
if source.CSI.NodePublishSecretRef != nil && !visitor(source.CSI.NodePublishSecretRef.Name) {
return false
}
}
}
return true

16
vendor/k8s.io/kubernetes/pkg/customers.yaml generated vendored
View file

@ -1,16 +0,0 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
name: standard-rdi-hdd-pool
parameters:
adminId: sam
adminSecretName: ceph-sec-admin-rdi
adminSecretNamespace: sam-system
monitors: 10.253.101.170:6789,10.253.101.154:6789,10.253.101.186:6789
pool: sam
userId: sam
userSecretName: ceph-sec-user-rdi
provisioner: kubernetes.io/rbd
reclaimPolicy: Retain
volumeBindingMode: Immediate

11
vendor/k8s.io/kubernetes/pkg/kubelet/BUILD generated vendored
View file

@ -103,7 +103,6 @@ go_library(
"//pkg/security/podsecuritypolicy/sysctl:go_default_library",
"//pkg/securitycontext:go_default_library",
"//pkg/util/dbus:go_default_library",
"//pkg/util/file:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/util/node:go_default_library",
@ -113,6 +112,7 @@ go_library(
"//pkg/volume:go_default_library",
"//pkg/volume/csi:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//pkg/volume/util/types:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
"//pkg/volume/validation:go_default_library",
@ -132,7 +132,6 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
@ -140,9 +139,7 @@ go_library(
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/util/certificate:go_default_library",
"//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
"//staging/src/k8s.io/client-go/util/integer:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
"//third_party/forked/golang/expansion:go_default_library",
"//vendor/github.com/golang/groupcache/lru:go_default_library",
"//vendor/github.com/google/cadvisor/events:go_default_library",
@ -150,6 +147,8 @@ go_library(
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/integer:go_default_library",
"//vendor/k8s.io/utils/path:go_default_library",
],
)
@ -167,7 +166,6 @@ go_test(
"kubelet_test.go",
"kubelet_volumes_linux_test.go",
"kubelet_volumes_test.go",
"main_test.go",
"oom_watcher_test.go",
"pod_container_deletor_test.go",
"pod_workers_test.go",
@ -210,7 +208,7 @@ go_test(
"//pkg/kubelet/util/sliceutils:go_default_library",
"//pkg/kubelet/volumemanager:go_default_library",
"//pkg/scheduler/api:go_default_library",
"//pkg/scheduler/cache:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/util/taints:go_default_library",
"//pkg/version:go_default_library",
@ -221,6 +219,7 @@ go_test(
"//pkg/volume/host_path:go_default_library",
"//pkg/volume/testing:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

2
vendor/k8s.io/kubernetes/pkg/kubelet/OWNERS generated vendored
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- Random-Liu
- dchen1107

2
vendor/k8s.io/kubernetes/pkg/kubelet/apis/BUILD generated vendored
View file

@ -14,6 +14,7 @@ go_library(
],
importpath = "k8s.io/kubernetes/pkg/kubelet/apis",
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:windows": [
@ -43,6 +44,7 @@ filegroup(
"//pkg/kubelet/apis/pluginregistration/v1alpha1:all-srcs",
"//pkg/kubelet/apis/pluginregistration/v1beta1:all-srcs",
"//pkg/kubelet/apis/podresources:all-srcs",
"//pkg/kubelet/apis/resourcemetrics/v1alpha1:all-srcs",
"//pkg/kubelet/apis/stats/v1alpha1:all-srcs",
],
tags = ["automanaged"],

693
vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go generated vendored
View file

@ -736,7 +736,8 @@ type PodSandboxConfig struct {
// operation. The runtime may also use this information to improve UX, such
// as by constructing a readable name.
Metadata *PodSandboxMetadata `protobuf:"bytes,1,opt,name=metadata" json:"metadata,omitempty"`
// Hostname of the sandbox.
// Hostname of the sandbox. Hostname could only be empty when the pod
// network namespace is NODE.
Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3" json:"hostname,omitempty"`
// Path to the directory on the host in which container log files are
// stored.
@ -848,7 +849,7 @@ type RunPodSandboxRequest struct {
// If the runtime handler is unknown, this request should be rejected. An
// empty string should select the default handler, equivalent to the
// behavior before this feature was added.
// See https://git.k8s.io/community/keps/sig-node/0014-runtime-class.md
// See https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md
RuntimeHandler string `protobuf:"bytes,2,opt,name=runtime_handler,json=runtimeHandler,proto3" json:"runtime_handler,omitempty"`
}
@ -1029,6 +1030,8 @@ type PodSandboxStatus struct {
// MUST be identical to that of the corresponding PodSandboxConfig used to
// instantiate the pod sandbox this status represents.
Annotations map[string]string `protobuf:"bytes,8,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// runtime configuration used for this PodSandbox.
RuntimeHandler string `protobuf:"bytes,9,opt,name=runtime_handler,json=runtimeHandler,proto3" json:"runtime_handler,omitempty"`
}
func (m *PodSandboxStatus) Reset() { *m = PodSandboxStatus{} }
@ -1091,6 +1094,13 @@ func (m *PodSandboxStatus) GetAnnotations() map[string]string {
return nil
}
func (m *PodSandboxStatus) GetRuntimeHandler() string {
if m != nil {
return m.RuntimeHandler
}
return ""
}
type PodSandboxStatusResponse struct {
// Status of the PodSandbox.
Status *PodSandboxStatus `protobuf:"bytes,1,opt,name=status" json:"status,omitempty"`
@ -1207,6 +1217,8 @@ type PodSandbox struct {
// MUST be identical to that of the corresponding PodSandboxConfig used to
// instantiate this PodSandbox.
Annotations map[string]string `protobuf:"bytes,6,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// runtime configuration used for this PodSandbox.
RuntimeHandler string `protobuf:"bytes,7,opt,name=runtime_handler,json=runtimeHandler,proto3" json:"runtime_handler,omitempty"`
}
func (m *PodSandbox) Reset() { *m = PodSandbox{} }
@ -1255,6 +1267,13 @@ func (m *PodSandbox) GetAnnotations() map[string]string {
return nil
}
func (m *PodSandbox) GetRuntimeHandler() string {
if m != nil {
return m.RuntimeHandler
}
return ""
}
type ListPodSandboxResponse struct {
// List of PodSandboxes.
Items []*PodSandbox `protobuf:"bytes,1,rep,name=items" json:"items,omitempty"`
@ -3550,7 +3569,7 @@ type ContainerStats struct {
Cpu *CpuUsage `protobuf:"bytes,2,opt,name=cpu" json:"cpu,omitempty"`
// Memory usage gathered from the container.
Memory *MemoryUsage `protobuf:"bytes,3,opt,name=memory" json:"memory,omitempty"`
// Usage of the writeable layer.
// Usage of the writable layer.
WritableLayer *FilesystemUsage `protobuf:"bytes,4,opt,name=writable_layer,json=writableLayer" json:"writable_layer,omitempty"`
}
@ -5774,6 +5793,12 @@ func (m *PodSandboxStatus) MarshalTo(dAtA []byte) (int, error) {
i += copy(dAtA[i:], v)
}
}
if len(m.RuntimeHandler) > 0 {
dAtA[i] = 0x4a
i++
i = encodeVarintApi(dAtA, i, uint64(len(m.RuntimeHandler)))
i += copy(dAtA[i:], m.RuntimeHandler)
}
return i, nil
}
@ -5999,6 +6024,12 @@ func (m *PodSandbox) MarshalTo(dAtA []byte) (int, error) {
i += copy(dAtA[i:], v)
}
}
if len(m.RuntimeHandler) > 0 {
dAtA[i] = 0x3a
i++
i = encodeVarintApi(dAtA, i, uint64(len(m.RuntimeHandler)))
i += copy(dAtA[i:], m.RuntimeHandler)
}
return i, nil
}
@ -9446,6 +9477,10 @@ func (m *PodSandboxStatus) Size() (n int) {
n += mapEntrySize + 1 + sovApi(uint64(mapEntrySize))
}
}
l = len(m.RuntimeHandler)
if l > 0 {
n += 1 + l + sovApi(uint64(l))
}
return n
}
@ -9541,6 +9576,10 @@ func (m *PodSandbox) Size() (n int) {
n += mapEntrySize + 1 + sovApi(uint64(mapEntrySize))
}
}
l = len(m.RuntimeHandler)
if l > 0 {
n += 1 + l + sovApi(uint64(l))
}
return n
}
@ -11127,6 +11166,7 @@ func (this *PodSandboxStatus) String() string {
`Linux:` + strings.Replace(fmt.Sprintf("%v", this.Linux), "LinuxPodSandboxStatus", "LinuxPodSandboxStatus", 1) + `,`,
`Labels:` + mapStringForLabels + `,`,
`Annotations:` + mapStringForAnnotations + `,`,
`RuntimeHandler:` + fmt.Sprintf("%v", this.RuntimeHandler) + `,`,
`}`,
}, "")
return s
@ -11225,6 +11265,7 @@ func (this *PodSandbox) String() string {
`CreatedAt:` + fmt.Sprintf("%v", this.CreatedAt) + `,`,
`Labels:` + mapStringForLabels + `,`,
`Annotations:` + mapStringForAnnotations + `,`,
`RuntimeHandler:` + fmt.Sprintf("%v", this.RuntimeHandler) + `,`,
`}`,
}, "")
return s
@ -15490,6 +15531,35 @@ func (m *PodSandboxStatus) Unmarshal(dAtA []byte) error {
}
m.Annotations[mapkey] = mapvalue
iNdEx = postIndex
case 9:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field RuntimeHandler", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowApi
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthApi
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.RuntimeHandler = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipApi(dAtA[iNdEx:])
@ -16459,6 +16529,35 @@ func (m *PodSandbox) Unmarshal(dAtA []byte) error {
}
m.Annotations[mapkey] = mapvalue
iNdEx = postIndex
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field RuntimeHandler", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowApi
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthApi
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.RuntimeHandler = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipApi(dAtA[iNdEx:])
@ -26861,300 +26960,300 @@ var (
func init() { proto.RegisterFile("api.proto", fileDescriptorApi) }
var fileDescriptorApi = []byte{
// 4708 bytes of a gzipped FileDescriptorProto
// 4720 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5c, 0xcd, 0x6f, 0x1b, 0x49,
0x76, 0x57, 0x93, 0xfa, 0x20, 0x1f, 0x45, 0x89, 0x2a, 0xcb, 0x16, 0x4d, 0x8f, 0x35, 0x56, 0xcf,
0xf8, 0x73, 0x66, 0xe4, 0xb1, 0x66, 0xd7, 0x13, 0xdb, 0xb3, 0xb6, 0x69, 0x49, 0xb6, 0x99, 0xb5,
0x29, 0xa6, 0x29, 0xcd, 0xc7, 0xce, 0x00, 0xbd, 0x2d, 0x76, 0x89, 0xea, 0x35, 0xd9, 0xd5, 0xd3,
0xdd, 0xb4, 0xad, 0x04, 0x08, 0x16, 0x08, 0xb2, 0x87, 0x00, 0x01, 0x72, 0xce, 0x71, 0x73, 0xc8,
0x21, 0xb7, 0x00, 0x41, 0x0e, 0x39, 0x6d, 0xb0, 0x87, 0xbd, 0x04, 0xc8, 0x69, 0x91, 0x8f, 0x4b,
0x66, 0x92, 0x5c, 0x72, 0x08, 0xf2, 0x07, 0xe4, 0x10, 0xd4, 0x57, 0xb3, 0x3f, 0xf9, 0xe1, 0xf1,
0xee, 0x4c, 0x4e, 0xea, 0x7a, 0xfd, 0xde, 0xab, 0x57, 0xaf, 0x5e, 0xbf, 0x7a, 0xf5, 0xab, 0xa2,
0xa0, 0x68, 0x38, 0xd6, 0xa6, 0xe3, 0x12, 0x9f, 0xa0, 0x8a, 0x3b, 0xb0, 0x7d, 0xab, 0x8f, 0x37,
0x9f, 0xdf, 0x30, 0x7a, 0xce, 0xb1, 0xb1, 0x55, 0x7b, 0xaf, 0x6b, 0xf9, 0xc7, 0x83, 0xc3, 0xcd,
0x0e, 0xe9, 0x5f, 0xef, 0x92, 0x2e, 0xb9, 0xce, 0x18, 0x0f, 0x07, 0x47, 0xac, 0xc5, 0x1a, 0xec,
0x89, 0x2b, 0x50, 0xaf, 0xc1, 0xd2, 0xc7, 0xd8, 0xf5, 0x2c, 0x62, 0x6b, 0xf8, 0xcb, 0x01, 0xf6,
0x7c, 0x54, 0x85, 0x85, 0xe7, 0x9c, 0x52, 0x55, 0x2e, 0x28, 0x57, 0x8a, 0x9a, 0x6c, 0xaa, 0x7f,
0xa9, 0xc0, 0x72, 0xc0, 0xec, 0x39, 0xc4, 0xf6, 0x70, 0x36, 0x37, 0xda, 0x80, 0x45, 0x61, 0x9c,
0x6e, 0x1b, 0x7d, 0x5c, 0xcd, 0xb1, 0xd7, 0x25, 0x41, 0x6b, 0x1a, 0x7d, 0x8c, 0x2e, 0xc3, 0xb2,
0x64, 0x91, 0x4a, 0xf2, 0x8c, 0x6b, 0x49, 0x90, 0x45, 0x6f, 0x68, 0x13, 0x4e, 0x49, 0x46, 0xc3,
0xb1, 0x02, 0xe6, 0x59, 0xc6, 0xbc, 0x22, 0x5e, 0xd5, 0x1d, 0x4b, 0xf0, 0xab, 0x9f, 0x43, 0x71,
0xa7, 0xd9, 0xde, 0x26, 0xf6, 0x91, 0xd5, 0xa5, 0x26, 0x7a, 0xd8, 0xa5, 0x32, 0x55, 0xe5, 0x42,
0x9e, 0x9a, 0x28, 0x9a, 0xa8, 0x06, 0x05, 0x0f, 0x1b, 0x6e, 0xe7, 0x18, 0x7b, 0xd5, 0x1c, 0x7b,
0x15, 0xb4, 0xa9, 0x14, 0x71, 0x7c, 0x8b, 0xd8, 0x5e, 0x35, 0xcf, 0xa5, 0x44, 0x53, 0xfd, 0xb9,
0x02, 0xa5, 0x16, 0x71, 0xfd, 0xa7, 0x86, 0xe3, 0x58, 0x76, 0x17, 0xdd, 0x84, 0x02, 0xf3, 0x65,
0x87, 0xf4, 0x98, 0x0f, 0x96, 0xb6, 0x6a, 0x9b, 0xf1, 0x69, 0xd9, 0x6c, 0x09, 0x0e, 0x2d, 0xe0,
0x45, 0x17, 0x61, 0xa9, 0x43, 0x6c, 0xdf, 0xb0, 0x6c, 0xec, 0xea, 0x0e, 0x71, 0x7d, 0xe6, 0xa2,
0x39, 0xad, 0x1c, 0x50, 0x69, 0x2f, 0xe8, 0x1c, 0x14, 0x8f, 0x89, 0xe7, 0x73, 0x8e, 0x3c, 0xe3,
0x28, 0x50, 0x02, 0x7b, 0xb9, 0x06, 0x0b, 0xec, 0xa5, 0xe5, 0x08, 0x67, 0xcc, 0xd3, 0x66, 0xc3,
0x51, 0x7f, 0xad, 0xc0, 0xdc, 0x53, 0x32, 0xb0, 0xfd, 0x58, 0x37, 0x86, 0x7f, 0x2c, 0x26, 0x2a,
0xd4, 0x8d, 0xe1, 0x1f, 0x0f, 0xbb, 0xa1, 0x1c, 0x7c, 0xae, 0x78, 0x37, 0xf4, 0x65, 0x0d, 0x0a,
0x2e, 0x36, 0x4c, 0x62, 0xf7, 0x4e, 0x98, 0x09, 0x05, 0x2d, 0x68, 0xd3, 0x49, 0xf4, 0x70, 0xcf,
0xb2, 0x07, 0x2f, 0x75, 0x17, 0xf7, 0x8c, 0x43, 0xdc, 0x63, 0xa6, 0x14, 0xb4, 0x25, 0x41, 0xd6,
0x38, 0x15, 0xed, 0x40, 0xc9, 0x71, 0x89, 0x63, 0x74, 0x0d, 0xea, 0xc7, 0xea, 0x1c, 0x73, 0x95,
0x9a, 0x74, 0x15, 0x33, 0xbb, 0x35, 0xe4, 0xd4, 0xc2, 0x62, 0xea, 0x5f, 0x2b, 0xb0, 0x4c, 0x83,
0xc7, 0x73, 0x8c, 0x0e, 0xde, 0x63, 0x53, 0x82, 0x6e, 0xc1, 0x82, 0x8d, 0xfd, 0x17, 0xc4, 0x7d,
0x26, 0x26, 0xe0, 0xcd, 0xa4, 0xd6, 0x40, 0xe6, 0x29, 0x31, 0xb1, 0x26, 0xf9, 0xd1, 0x0d, 0xc8,
0x3b, 0x96, 0xc9, 0x06, 0x3c, 0x81, 0x18, 0xe5, 0xa5, 0x22, 0x96, 0xd3, 0x61, 0x7e, 0x98, 0x44,
0xc4, 0x72, 0x3a, 0xaa, 0x0a, 0xd0, 0xb0, 0xfd, 0x9b, 0xdf, 0xfb, 0xd8, 0xe8, 0x0d, 0x30, 0x5a,
0x85, 0xb9, 0xe7, 0xf4, 0x81, 0x19, 0x9b, 0xd7, 0x78, 0x43, 0xfd, 0x2a, 0x0f, 0xe7, 0x9e, 0x50,
0x7f, 0xb5, 0x0d, 0xdb, 0x3c, 0x24, 0x2f, 0xdb, 0xb8, 0x33, 0x70, 0x2d, 0xff, 0x64, 0x9b, 0xd8,
0x3e, 0x7e, 0xe9, 0xa3, 0x26, 0xac, 0xd8, 0x52, 0xb3, 0x2e, 0x43, 0x93, 0x6a, 0x28, 0x6d, 0x6d,
0x8c, 0x30, 0x82, 0xbb, 0x48, 0xab, 0xd8, 0x51, 0x82, 0x87, 0x1e, 0x0f, 0xe7, 0x4d, 0x6a, 0xcb,
0x31, 0x6d, 0x29, 0x43, 0x6a, 0xef, 0x32, 0xcb, 0x84, 0x2e, 0x39, 0xb1, 0x52, 0xd3, 0x47, 0x40,
0xbf, 0x6a, 0xdd, 0xf0, 0xf4, 0x81, 0x87, 0x5d, 0xe6, 0x98, 0xd2, 0xd6, 0x1b, 0x49, 0x2d, 0x43,
0x17, 0x68, 0x45, 0x77, 0x60, 0xd7, 0xbd, 0x03, 0x0f, 0xbb, 0x2c, 0x09, 0x88, 0x58, 0xd2, 0x5d,
0x42, 0xfc, 0x23, 0x4f, 0xc6, 0x8f, 0x24, 0x6b, 0x8c, 0x8a, 0xae, 0xc3, 0x29, 0x6f, 0xe0, 0x38,
0x3d, 0xdc, 0xc7, 0xb6, 0x6f, 0xf4, 0xf4, 0xae, 0x4b, 0x06, 0x8e, 0x57, 0x9d, 0xbb, 0x90, 0xbf,
0x92, 0xd7, 0x50, 0xf8, 0xd5, 0x23, 0xf6, 0x06, 0xad, 0x03, 0x38, 0xae, 0xf5, 0xdc, 0xea, 0xe1,
0x2e, 0x36, 0xab, 0xf3, 0x4c, 0x69, 0x88, 0x82, 0xde, 0x87, 0x55, 0x0f, 0x77, 0x3a, 0xa4, 0xef,
0xe8, 0x8e, 0x4b, 0x8e, 0xac, 0x1e, 0xe6, 0xd1, 0xbf, 0xc0, 0xa2, 0x1f, 0x89, 0x77, 0x2d, 0xfe,
0x8a, 0x7d, 0x07, 0x77, 0x59, 0x4e, 0xa3, 0x23, 0x65, 0x9d, 0x57, 0x0b, 0x13, 0x0c, 0x15, 0xd8,
0x50, 0x99, 0x49, 0xea, 0xcf, 0x73, 0x70, 0x9a, 0x79, 0xb2, 0x45, 0x4c, 0x31, 0xcd, 0x22, 0x49,
0xbd, 0x05, 0xe5, 0x0e, 0xd3, 0xa9, 0x3b, 0x86, 0x8b, 0x6d, 0x5f, 0x7c, 0xa4, 0x8b, 0x9c, 0xd8,
0x62, 0x34, 0xf4, 0x29, 0x54, 0x3c, 0x11, 0x15, 0x7a, 0x87, 0x87, 0x85, 0x98, 0xb3, 0xf7, 0x92,
0x26, 0x8c, 0x88, 0x25, 0x6d, 0xd9, 0x4b, 0x04, 0xd7, 0x82, 0x77, 0xe2, 0x75, 0xfc, 0x1e, 0xcf,
0x76, 0xa5, 0xad, 0xef, 0x65, 0x28, 0x8c, 0x1b, 0xbe, 0xd9, 0xe6, 0x62, 0xbb, 0xb6, 0xef, 0x9e,
0x68, 0x52, 0x49, 0xed, 0x36, 0x2c, 0x86, 0x5f, 0xa0, 0x0a, 0xe4, 0x9f, 0xe1, 0x13, 0x31, 0x28,
0xfa, 0x38, 0xfc, 0x08, 0x78, 0xae, 0xe1, 0x8d, 0xdb, 0xb9, 0xdf, 0x51, 0x54, 0x17, 0xd0, 0xb0,
0x97, 0xa7, 0xd8, 0x37, 0x4c, 0xc3, 0x37, 0x10, 0x82, 0x59, 0xb6, 0x8c, 0x70, 0x15, 0xec, 0x99,
0x6a, 0x1d, 0x88, 0x8f, 0xb7, 0xa8, 0xd1, 0x47, 0xf4, 0x06, 0x14, 0x83, 0x40, 0x17, 0x6b, 0xc9,
0x90, 0x40, 0x73, 0xba, 0xe1, 0xfb, 0xb8, 0xef, 0xf8, 0x2c, 0xc4, 0xca, 0x9a, 0x6c, 0xaa, 0xff,
0x3d, 0x0b, 0x95, 0xc4, 0x9c, 0xdc, 0x87, 0x42, 0x5f, 0x74, 0x2f, 0x3e, 0xb4, 0xb7, 0x53, 0x12,
0x7b, 0xc2, 0x54, 0x2d, 0x90, 0xa2, 0x79, 0x93, 0xe6, 0xd0, 0xd0, 0xfa, 0x17, 0xb4, 0xe9, 0x8c,
0xf7, 0x48, 0x57, 0x37, 0x2d, 0x17, 0x77, 0x7c, 0xe2, 0x9e, 0x08, 0x73, 0x17, 0x7b, 0xa4, 0xbb,
0x23, 0x69, 0xe8, 0x36, 0x80, 0x69, 0x7b, 0x74, 0xb2, 0x8f, 0xac, 0x2e, 0x33, 0xba, 0xb4, 0x75,
0x2e, 0x69, 0x44, 0xb0, 0xd8, 0x69, 0x45, 0xd3, 0xf6, 0x84, 0xf9, 0x0f, 0xa0, 0x4c, 0xd7, 0x0c,
0xbd, 0xcf, 0xd7, 0x29, 0xfe, 0xa5, 0x94, 0xb6, 0xce, 0xa7, 0x8d, 0x21, 0x58, 0xcd, 0xb4, 0x45,
0x67, 0xd8, 0xf0, 0xd0, 0x43, 0x98, 0x67, 0xc9, 0xdb, 0xab, 0xce, 0x33, 0xe1, 0xcd, 0x51, 0x0e,
0x10, 0x11, 0xf1, 0x84, 0x09, 0xf0, 0x80, 0x10, 0xd2, 0xe8, 0x00, 0x4a, 0x86, 0x6d, 0x13, 0xdf,
0xe0, 0x89, 0x66, 0x81, 0x29, 0xfb, 0x60, 0x02, 0x65, 0xf5, 0xa1, 0x14, 0xd7, 0x18, 0xd6, 0x83,
0x7e, 0x00, 0x73, 0x2c, 0x13, 0x89, 0x0f, 0xf1, 0xf2, 0x84, 0x41, 0xab, 0x71, 0xa9, 0xda, 0x2d,
0x28, 0x85, 0x8c, 0x9d, 0x26, 0x48, 0x6b, 0x77, 0xa1, 0x12, 0x37, 0x6d, 0xaa, 0x20, 0xff, 0x03,
0x58, 0xd5, 0x06, 0xf6, 0xd0, 0x30, 0x59, 0x7d, 0xdd, 0x86, 0x79, 0x31, 0xd9, 0x3c, 0xe2, 0xd4,
0xf1, 0x3e, 0xd2, 0x84, 0x44, 0xb8, 0x9c, 0x3a, 0x36, 0x6c, 0xb3, 0x87, 0x5d, 0xd1, 0xaf, 0x2c,
0xa7, 0x1e, 0x73, 0xaa, 0xfa, 0x03, 0x38, 0x1d, 0xeb, 0x5c, 0x54, 0x73, 0x6f, 0xc3, 0x92, 0x43,
0x4c, 0xdd, 0xe3, 0x64, 0xdd, 0x32, 0x65, 0x1a, 0x72, 0x02, 0xde, 0x86, 0x49, 0xc5, 0xdb, 0x3e,
0x71, 0x92, 0xc6, 0x4f, 0x26, 0x5e, 0x85, 0x33, 0x71, 0x71, 0xde, 0xbd, 0x7a, 0x0f, 0xd6, 0x34,
0xdc, 0x27, 0xcf, 0xf1, 0xab, 0xaa, 0xae, 0x41, 0x35, 0xa9, 0x40, 0x28, 0xff, 0x0c, 0xd6, 0x86,
0xd4, 0xb6, 0x6f, 0xf8, 0x03, 0x6f, 0x2a, 0xe5, 0xa2, 0xd4, 0x3d, 0x24, 0x1e, 0x9f, 0xce, 0x82,
0x26, 0x9b, 0xea, 0xd5, 0xb0, 0xea, 0x26, 0xaf, 0x2c, 0x78, 0x0f, 0x68, 0x09, 0x72, 0x96, 0x23,
0xd4, 0xe5, 0x2c, 0x47, 0x7d, 0x0c, 0xc5, 0x60, 0x69, 0x46, 0x77, 0x86, 0x35, 0x66, 0x6e, 0xd2,
0x85, 0x3c, 0x28, 0x43, 0xf7, 0x13, 0x4b, 0x89, 0xe8, 0xf2, 0x0e, 0x40, 0x90, 0xf2, 0x64, 0x85,
0x70, 0x6e, 0x84, 0x62, 0x2d, 0xc4, 0xae, 0xfe, 0x4b, 0x24, 0x11, 0x86, 0x06, 0x61, 0x06, 0x83,
0x30, 0x23, 0x89, 0x31, 0xf7, 0x4a, 0x89, 0xf1, 0x43, 0x98, 0xf3, 0x7c, 0xc3, 0xc7, 0xa2, 0x8a,
0xda, 0x18, 0x25, 0x4e, 0x8d, 0xc0, 0x1a, 0xe7, 0x47, 0xe7, 0x01, 0x3a, 0x2e, 0x36, 0x7c, 0x6c,
0xea, 0x06, 0xcf, 0xe2, 0x79, 0xad, 0x28, 0x28, 0x75, 0x1f, 0x6d, 0x0f, 0x2b, 0xc1, 0x39, 0x66,
0xd8, 0xd5, 0x51, 0x9a, 0x23, 0x53, 0x35, 0xac, 0x09, 0x83, 0xac, 0x32, 0x3f, 0x61, 0x56, 0x11,
0x0a, 0xb8, 0x54, 0x28, 0x67, 0x2e, 0x8c, 0xcf, 0x99, 0x5c, 0x74, 0x92, 0x9c, 0x59, 0x18, 0x9f,
0x33, 0x85, 0xb2, 0x91, 0x39, 0xf3, 0xdb, 0x4c, 0x7a, 0xff, 0xac, 0x40, 0x35, 0xf9, 0x0d, 0x8a,
0xdc, 0x73, 0x1b, 0xe6, 0x3d, 0x46, 0x99, 0x24, 0xf3, 0x09, 0x59, 0x21, 0x81, 0x1e, 0xc3, 0xac,
0x65, 0x1f, 0x11, 0xb6, 0x89, 0x4b, 0xad, 0x5d, 0xb2, 0x7a, 0xdd, 0x6c, 0xd8, 0x47, 0x84, 0x3b,
0x89, 0x69, 0xa8, 0x7d, 0x08, 0xc5, 0x80, 0x34, 0xd5, 0xd8, 0xf6, 0x60, 0x35, 0x16, 0xb2, 0xbc,
0xd8, 0x0f, 0x22, 0x5d, 0x99, 0x2e, 0xd2, 0xd5, 0x9f, 0xe6, 0xc2, 0x5f, 0xe2, 0x43, 0xab, 0xe7,
0x63, 0x37, 0xf1, 0x25, 0x7e, 0x24, 0xb5, 0xf3, 0xcf, 0xf0, 0xd2, 0x58, 0xed, 0xbc, 0x26, 0x15,
0x1f, 0xd3, 0x17, 0xb0, 0xc4, 0x62, 0x4d, 0xf7, 0x70, 0x8f, 0x15, 0x1c, 0xa2, 0xf8, 0xfb, 0xfe,
0x28, 0x35, 0xdc, 0x12, 0x1e, 0xb1, 0x6d, 0x21, 0xc7, 0x3d, 0x58, 0xee, 0x85, 0x69, 0xb5, 0xfb,
0x80, 0x92, 0x4c, 0x53, 0xf9, 0xb4, 0x4d, 0x53, 0x1c, 0xdd, 0xe9, 0xa6, 0xac, 0x92, 0x47, 0xcc,
0x8c, 0x49, 0x62, 0x85, 0x1b, 0xac, 0x09, 0x09, 0xf5, 0x97, 0x79, 0x80, 0xe1, 0xcb, 0xff, 0x47,
0xb9, 0xed, 0x7e, 0x90, 0x57, 0x78, 0x21, 0x77, 0x65, 0x94, 0xe2, 0xd4, 0x8c, 0xb2, 0x17, 0xcd,
0x28, 0xbc, 0xa4, 0x7b, 0x6f, 0xa4, 0x9a, 0xef, 0x6c, 0x2e, 0x79, 0x02, 0x67, 0xe2, 0xb1, 0x21,
0x12, 0xc9, 0x16, 0xcc, 0x59, 0x3e, 0xee, 0x73, 0xb4, 0x27, 0x75, 0x77, 0x16, 0x12, 0xe2, 0xac,
0xea, 0x06, 0x14, 0x1b, 0x7d, 0xa3, 0x8b, 0xdb, 0x0e, 0xee, 0xd0, 0x4e, 0x2d, 0xda, 0x10, 0x86,
0xf0, 0x86, 0xba, 0x05, 0x85, 0x1f, 0xe2, 0x13, 0xfe, 0x51, 0x4f, 0x68, 0xa8, 0xfa, 0xa7, 0x39,
0x58, 0x63, 0x6b, 0xc5, 0xb6, 0xc4, 0x5a, 0x34, 0xec, 0x91, 0x81, 0xdb, 0xc1, 0x1e, 0x9b, 0x6d,
0x67, 0xa0, 0x3b, 0xd8, 0xb5, 0x88, 0x29, 0xa0, 0x80, 0x62, 0xc7, 0x19, 0xb4, 0x18, 0x01, 0x9d,
0x03, 0xda, 0xd0, 0xbf, 0x1c, 0x10, 0x11, 0x88, 0x79, 0xad, 0xd0, 0x71, 0x06, 0xbf, 0x47, 0xdb,
0x52, 0xd6, 0x3b, 0x36, 0x5c, 0xec, 0xb1, 0x38, 0xe3, 0xb2, 0x6d, 0x46, 0x40, 0x37, 0xe0, 0x74,
0x1f, 0xf7, 0x89, 0x7b, 0xa2, 0xf7, 0xac, 0xbe, 0xe5, 0xeb, 0x96, 0xad, 0x1f, 0x9e, 0xf8, 0xd8,
0x13, 0x31, 0x85, 0xf8, 0xcb, 0x27, 0xf4, 0x5d, 0xc3, 0x7e, 0x40, 0xdf, 0x20, 0x15, 0xca, 0x84,
0xf4, 0x75, 0xaf, 0x43, 0x5c, 0xac, 0x1b, 0xe6, 0x4f, 0xd8, 0xf2, 0x99, 0xd7, 0x4a, 0x84, 0xf4,
0xdb, 0x94, 0x56, 0x37, 0x7f, 0x82, 0xde, 0x84, 0x52, 0xc7, 0x19, 0x78, 0xd8, 0xd7, 0xe9, 0x1f,
0xb6, 0x3a, 0x16, 0x35, 0xe0, 0xa4, 0x6d, 0x67, 0xe0, 0x85, 0x18, 0xfa, 0xd4, 0xff, 0x0b, 0x61,
0x86, 0xa7, 0xd4, 0xcd, 0x06, 0x94, 0x23, 0x50, 0x02, 0xdd, 0xd5, 0x31, 0xcc, 0x40, 0xec, 0xea,
0xe8, 0x33, 0xa5, 0xb9, 0xa4, 0x27, 0x3d, 0xc9, 0x9e, 0x29, 0xcd, 0x3f, 0x71, 0xe4, 0x96, 0x8e,
0x3d, 0x53, 0x97, 0xf7, 0xf0, 0x73, 0x01, 0x37, 0x15, 0x35, 0xde, 0x50, 0x4d, 0x80, 0x6d, 0xc3,
0x31, 0x0e, 0xad, 0x9e, 0xe5, 0x9f, 0xa0, 0xab, 0x50, 0x31, 0x4c, 0x53, 0xef, 0x48, 0x8a, 0x85,
0x25, 0x08, 0xb8, 0x6c, 0x98, 0xe6, 0x76, 0x88, 0x8c, 0xde, 0x81, 0x15, 0xd3, 0x25, 0x4e, 0x94,
0x97, 0xa3, 0x82, 0x15, 0xfa, 0x22, 0xcc, 0xac, 0xfe, 0xc7, 0x1c, 0x9c, 0x8f, 0x4e, 0x6c, 0x1c,
0xae, 0xb9, 0x0f, 0x8b, 0xb1, 0x5e, 0x33, 0xa0, 0x82, 0xa1, 0xb5, 0x5a, 0x44, 0x22, 0x06, 0x5f,
0xe4, 0x12, 0xf0, 0x45, 0x2a, 0x20, 0x94, 0x7f, 0xad, 0x80, 0xd0, 0xec, 0x6b, 0x01, 0x84, 0xe6,
0xa6, 0x03, 0x84, 0x2e, 0xb1, 0x6d, 0x8c, 0x94, 0x66, 0x7b, 0x67, 0x1e, 0x6a, 0xe5, 0x80, 0xc7,
0x96, 0xe8, 0x71, 0x0c, 0x38, 0x5a, 0x98, 0x06, 0x38, 0x2a, 0x64, 0x02, 0x47, 0x34, 0x6a, 0x1c,
0xc7, 0x70, 0xfb, 0xc4, 0x95, 0xc8, 0x50, 0xb5, 0xc8, 0x4c, 0x58, 0x96, 0x74, 0x81, 0x0a, 0x65,
0x62, 0x48, 0x90, 0x89, 0x21, 0x5d, 0x80, 0x45, 0x9b, 0xe8, 0x36, 0x7e, 0xa1, 0xd3, 0xb9, 0xf4,
0xaa, 0x25, 0x3e, 0xb1, 0x36, 0x69, 0xe2, 0x17, 0x2d, 0x4a, 0x49, 0xa0, 0x4c, 0x8b, 0xd3, 0xa1,
0x4c, 0x68, 0x03, 0x16, 0xfb, 0x86, 0xf7, 0x0c, 0x9b, 0xcc, 0x14, 0xaf, 0x5a, 0x66, 0x41, 0x5c,
0xe2, 0x34, 0x6a, 0x83, 0x87, 0x2e, 0x42, 0xe0, 0x24, 0xc1, 0xb4, 0xc4, 0x98, 0xca, 0x92, 0xca,
0xd8, 0xd4, 0xbf, 0x53, 0x60, 0x35, 0x1a, 0xe6, 0x02, 0x5b, 0x78, 0x04, 0x45, 0x57, 0x66, 0x32,
0x11, 0xda, 0x57, 0x33, 0xca, 0xe4, 0x64, 0xea, 0xd3, 0x86, 0xb2, 0xe8, 0x47, 0x99, 0x90, 0xd6,
0xf5, 0x71, 0xfa, 0xc6, 0x81, 0x5a, 0x6a, 0x03, 0xde, 0xfc, 0xc4, 0xb2, 0x4d, 0xf2, 0xc2, 0xcb,
0xfc, 0x4a, 0x53, 0x62, 0x4d, 0x49, 0x89, 0x35, 0xf5, 0x17, 0x0a, 0x9c, 0x89, 0xeb, 0x12, 0xae,
0x68, 0x24, 0x5d, 0xf1, 0x4e, 0xd2, 0xf4, 0xb8, 0x70, 0xaa, 0x33, 0xbe, 0xc8, 0x74, 0xc6, 0x8d,
0xf1, 0x1a, 0xc7, 0xba, 0xe3, 0xaf, 0x14, 0x38, 0x9b, 0x69, 0x46, 0x6c, 0x49, 0x51, 0xe2, 0x4b,
0x8a, 0x58, 0x8e, 0x3a, 0x64, 0x60, 0xfb, 0xa1, 0xe5, 0x68, 0x9b, 0x1d, 0x31, 0xf0, 0xbc, 0xaf,
0xf7, 0x8d, 0x97, 0x56, 0x7f, 0xd0, 0x17, 0xeb, 0x11, 0x55, 0xf7, 0x94, 0x53, 0x5e, 0x61, 0x41,
0x52, 0xeb, 0xb0, 0x12, 0x58, 0x39, 0x12, 0x04, 0x0c, 0x81, 0x7a, 0xb9, 0x28, 0xa8, 0x67, 0xc3,
0xfc, 0x0e, 0x7e, 0x6e, 0x75, 0xf0, 0x6b, 0x39, 0x03, 0xb9, 0x00, 0x25, 0x07, 0xbb, 0x7d, 0xcb,
0xf3, 0x82, 0x44, 0x5b, 0xd4, 0xc2, 0x24, 0xf5, 0x3f, 0xe7, 0x61, 0x39, 0x1e, 0x1d, 0xf7, 0x12,
0x18, 0xe2, 0x5b, 0x29, 0x4b, 0x40, 0x7c, 0xa0, 0xa1, 0x6a, 0xf2, 0x86, 0x2c, 0x46, 0x72, 0x59,
0x1b, 0xf9, 0xa0, 0x70, 0x11, 0x95, 0x0a, 0xf5, 0x48, 0x87, 0xf4, 0xfb, 0x86, 0x6d, 0xca, 0xa3,
0x2b, 0xd1, 0xa4, 0xfe, 0x33, 0xdc, 0x2e, 0x75, 0x3b, 0x25, 0xb3, 0x67, 0x3a, 0x79, 0x74, 0xd7,
0x6b, 0xd9, 0x0c, 0x8b, 0x64, 0xc9, 0xba, 0xa8, 0x81, 0x20, 0xed, 0x58, 0x2e, 0xda, 0x84, 0x59,
0x6c, 0x3f, 0x97, 0xe5, 0x62, 0xca, 0xd9, 0x96, 0x2c, 0x8b, 0x34, 0xc6, 0x87, 0xae, 0xc3, 0x7c,
0x9f, 0x86, 0x85, 0xdc, 0xff, 0xae, 0x65, 0x1c, 0xf1, 0x68, 0x82, 0x0d, 0x6d, 0xc1, 0x82, 0xc9,
0xe6, 0x49, 0x6e, 0x72, 0xab, 0x29, 0x08, 0x27, 0x63, 0xd0, 0x24, 0x23, 0xda, 0x0d, 0x8a, 0xe1,
0x62, 0x56, 0x15, 0x1b, 0x9b, 0x8a, 0xd4, 0x8a, 0x78, 0x3f, 0x5a, 0x11, 0x03, 0xd3, 0xb5, 0x35,
0x5e, 0xd7, 0x68, 0x58, 0xf2, 0x2c, 0x14, 0x7a, 0xa4, 0xcb, 0xc3, 0xa8, 0xc4, 0x4f, 0x45, 0x7b,
0xa4, 0xcb, 0xa2, 0x68, 0x95, 0x6e, 0x0e, 0x4c, 0xcb, 0x66, 0x49, 0xbd, 0xa0, 0xf1, 0x06, 0xfd,
0xf8, 0xd8, 0x83, 0x4e, 0xec, 0x0e, 0xae, 0x96, 0xd9, 0xab, 0x22, 0xa3, 0xec, 0xd9, 0x1d, 0x56,
0x6e, 0xfa, 0xfe, 0x49, 0x75, 0x89, 0xd1, 0xe9, 0x23, 0xdd, 0xf7, 0x71, 0x88, 0x62, 0x39, 0x6b,
0xdf, 0x97, 0x96, 0xb6, 0x25, 0x42, 0xf1, 0x00, 0x16, 0x5e, 0xf0, 0x44, 0x50, 0xad, 0x30, 0xf9,
0x2b, 0xe3, 0xd3, 0x8b, 0xd0, 0x20, 0x05, 0xbf, 0xcd, 0xd2, 0xff, 0x97, 0x0a, 0x9c, 0xd9, 0x66,
0xdb, 0xa2, 0x50, 0x1e, 0x9b, 0x06, 0xc9, 0xbb, 0x15, 0x80, 0xac, 0x99, 0xb0, 0x5b, 0x7c, 0xdc,
0x12, 0x63, 0x6d, 0xc0, 0x92, 0x54, 0x2e, 0x54, 0xe4, 0x27, 0xc6, 0x69, 0xcb, 0x5e, 0xb8, 0xa9,
0x7e, 0x04, 0x6b, 0x89, 0x51, 0x88, 0x2d, 0xcc, 0x06, 0x2c, 0x0e, 0xf3, 0x55, 0x30, 0x88, 0x52,
0x40, 0x6b, 0x98, 0xea, 0x6d, 0x38, 0xdd, 0xf6, 0x0d, 0xd7, 0x4f, 0xb8, 0x60, 0x02, 0x59, 0x86,
0xc0, 0x46, 0x65, 0x05, 0x48, 0xda, 0x86, 0xd5, 0xb6, 0x4f, 0x9c, 0x57, 0x50, 0x4a, 0xb3, 0x0e,
0x1d, 0x3f, 0x19, 0xc8, 0xf5, 0x41, 0x36, 0xd5, 0x35, 0x8e, 0x17, 0x27, 0x7b, 0xbb, 0x03, 0x67,
0x38, 0x5c, 0xfb, 0x2a, 0x83, 0x38, 0x2b, 0xc1, 0xe2, 0xa4, 0xde, 0xa7, 0x70, 0x6a, 0xb8, 0x2c,
0x0e, 0xa1, 0x98, 0x9b, 0x51, 0x28, 0xe6, 0xc2, 0x88, 0x59, 0x8f, 0x20, 0x31, 0x7f, 0x91, 0x0b,
0xe5, 0xf5, 0x0c, 0x20, 0xe6, 0x4e, 0x14, 0x88, 0xb9, 0x38, 0x4e, 0x77, 0x04, 0x87, 0x49, 0x46,
0x6d, 0x3e, 0x25, 0x6a, 0x3f, 0x4f, 0xa0, 0x35, 0xb3, 0x59, 0x70, 0x57, 0xcc, 0xda, 0xdf, 0x0a,
0x58, 0xa3, 0x71, 0xb0, 0x26, 0xe8, 0x3a, 0x40, 0xd7, 0x6f, 0xc5, 0xc0, 0x9a, 0x8d, 0xb1, 0xf6,
0x06, 0x58, 0xcd, 0xdf, 0xcc, 0x42, 0x31, 0x78, 0x97, 0xf0, 0x79, 0xd2, 0x6d, 0xb9, 0x14, 0xb7,
0x85, 0x57, 0xe0, 0xfc, 0x37, 0x5a, 0x81, 0x67, 0x27, 0x5e, 0x81, 0xcf, 0x41, 0x91, 0x3d, 0xe8,
0x2e, 0x3e, 0x12, 0x2b, 0x6a, 0x81, 0x11, 0x34, 0x7c, 0x34, 0x0c, 0xc3, 0xf9, 0xa9, 0xc2, 0x30,
0x06, 0x0f, 0x2d, 0xc4, 0xe1, 0xa1, 0x7b, 0xc1, 0x8a, 0xc8, 0x17, 0xd1, 0xcb, 0x23, 0xf4, 0xa6,
0xae, 0x85, 0xcd, 0xe8, 0x5a, 0xc8, 0xd7, 0xd5, 0x77, 0x47, 0x69, 0xf9, 0xce, 0x82, 0x43, 0x07,
0x1c, 0x1c, 0x0a, 0xc7, 0xa2, 0xc8, 0xac, 0x77, 0x00, 0x82, 0x24, 0x22, 0x11, 0xa2, 0x73, 0x23,
0xc6, 0xa8, 0x85, 0xd8, 0xa9, 0xda, 0xc8, 0xd4, 0x0c, 0x4f, 0x90, 0x26, 0xcb, 0x8f, 0x19, 0xc7,
0x47, 0xff, 0x3b, 0x17, 0xca, 0x2f, 0x19, 0x47, 0x2e, 0xf7, 0x12, 0xb0, 0xe4, 0x94, 0x51, 0x7c,
0x33, 0x8a, 0x4a, 0xbe, 0x62, 0xd4, 0x25, 0x40, 0x49, 0x56, 0xb9, 0x18, 0xae, 0x78, 0xcd, 0x41,
0xa3, 0xa2, 0xa0, 0xd4, 0xd9, 0xce, 0xe0, 0xc8, 0xb2, 0x2d, 0xef, 0x98, 0xbf, 0x9f, 0xe7, 0x3b,
0x03, 0x49, 0xaa, 0xb3, 0xdb, 0x4d, 0xf8, 0xa5, 0xe5, 0xeb, 0x1d, 0x62, 0x62, 0x16, 0xd3, 0x73,
0x5a, 0x81, 0x12, 0xb6, 0x89, 0x89, 0x87, 0x5f, 0x5e, 0xe1, 0xd5, 0xbe, 0xbc, 0x62, 0xec, 0xcb,
0x3b, 0x03, 0xf3, 0x2e, 0x36, 0x3c, 0x62, 0x8b, 0xed, 0xb9, 0x68, 0xd1, 0xa9, 0xe9, 0x63, 0xcf,
0xa3, 0x3d, 0x89, 0x72, 0x4d, 0x34, 0x43, 0x65, 0xe6, 0xe2, 0xd8, 0x32, 0x73, 0xc4, 0x51, 0x4e,
0xac, 0xcc, 0x2c, 0x8f, 0x2d, 0x33, 0x27, 0x39, 0xc9, 0x09, 0x15, 0xda, 0x4b, 0x93, 0x15, 0xda,
0xe1, 0xba, 0x74, 0x39, 0x52, 0x97, 0x7e, 0x9b, 0x1f, 0xeb, 0xaf, 0x15, 0x58, 0x4b, 0x7c, 0x56,
0xe2, 0x73, 0xbd, 0x15, 0x3b, 0x14, 0xda, 0x18, 0xeb, 0xb3, 0xe0, 0x4c, 0xe8, 0x51, 0xe4, 0x4c,
0xe8, 0x83, 0xf1, 0x82, 0xaf, 0xfd, 0x48, 0xe8, 0x8f, 0x15, 0x78, 0xf3, 0xc0, 0x31, 0x63, 0x15,
0x9e, 0xd8, 0xf6, 0x4f, 0x9e, 0x38, 0xee, 0xc9, 0x5a, 0x3f, 0x37, 0x2d, 0xce, 0xc2, 0xe5, 0x54,
0x15, 0x2e, 0x64, 0x9b, 0x21, 0x4a, 0xa6, 0x1f, 0xc3, 0xf2, 0xee, 0x4b, 0xdc, 0x69, 0x9f, 0xd8,
0x9d, 0x29, 0x4c, 0xab, 0x40, 0xbe, 0xd3, 0x37, 0x05, 0x4a, 0x4a, 0x1f, 0xc3, 0x55, 0x60, 0x3e,
0x5a, 0x05, 0xea, 0x50, 0x19, 0xf6, 0x20, 0xa6, 0xf7, 0x0c, 0x9d, 0x5e, 0x93, 0x32, 0x53, 0xe5,
0x8b, 0x9a, 0x68, 0x09, 0x3a, 0x76, 0xf9, 0x05, 0x06, 0x4e, 0xc7, 0xae, 0x1b, 0xcd, 0x16, 0xf9,
0x68, 0xb6, 0x50, 0xff, 0x5c, 0x81, 0x12, 0xed, 0xe1, 0x1b, 0xd9, 0x2f, 0xb6, 0x5a, 0xf9, 0xe1,
0x56, 0x2b, 0xd8, 0xb1, 0xcd, 0x86, 0x77, 0x6c, 0x43, 0xcb, 0xe7, 0x18, 0x39, 0x69, 0xf9, 0x7c,
0x40, 0xc7, 0xae, 0xab, 0x5e, 0x80, 0x45, 0x6e, 0x9b, 0x18, 0x79, 0x05, 0xf2, 0x03, 0xb7, 0x27,
0xe3, 0x68, 0xe0, 0xf6, 0xd4, 0x3f, 0x51, 0xa0, 0x5c, 0xf7, 0x7d, 0xa3, 0x73, 0x3c, 0xc5, 0x00,
0x02, 0xe3, 0x72, 0x61, 0xe3, 0x92, 0x83, 0x18, 0x9a, 0x3b, 0x9b, 0x61, 0xee, 0x5c, 0xc4, 0x5c,
0x15, 0x96, 0xa4, 0x2d, 0x99, 0x06, 0x37, 0x01, 0xb5, 0x88, 0xeb, 0x3f, 0x24, 0xee, 0x0b, 0xc3,
0x35, 0xa7, 0xdb, 0x81, 0x21, 0x98, 0x15, 0x37, 0x5e, 0xf3, 0x57, 0xe6, 0x34, 0xf6, 0xac, 0x5e,
0x86, 0x53, 0x11, 0x7d, 0x99, 0x1d, 0xdf, 0x87, 0x12, 0xcb, 0xfb, 0xa2, 0x14, 0xbf, 0x11, 0x3e,
0xae, 0x99, 0x68, 0x95, 0x50, 0x7f, 0x17, 0x56, 0x68, 0x7d, 0xc0, 0xe8, 0xc1, 0xa7, 0xf8, 0xfd,
0x58, 0x9d, 0x7a, 0x3e, 0x43, 0x51, 0xac, 0x46, 0xfd, 0x5b, 0x05, 0xe6, 0x18, 0x3d, 0xb1, 0x66,
0x9f, 0x83, 0xa2, 0x8b, 0x1d, 0xa2, 0xfb, 0x46, 0x37, 0xb8, 0x5f, 0x4c, 0x09, 0xfb, 0x46, 0xd7,
0x63, 0xd7, 0xa3, 0xe9, 0x4b, 0xd3, 0xea, 0x62, 0xcf, 0x97, 0x97, 0x8c, 0x4b, 0x94, 0xb6, 0xc3,
0x49, 0xd4, 0x49, 0x9e, 0xf5, 0xfb, 0xbc, 0xee, 0x9c, 0xd5, 0xd8, 0x33, 0xda, 0xe4, 0x57, 0xde,
0x26, 0x81, 0xd4, 0xd9, 0x85, 0xb8, 0x1a, 0x14, 0x62, 0x28, 0x7a, 0xd0, 0x56, 0x77, 0x01, 0x85,
0xbd, 0x20, 0xfc, 0x7d, 0x1d, 0xe6, 0x99, 0x93, 0x64, 0x75, 0xb4, 0x96, 0xe1, 0x06, 0x4d, 0xb0,
0xa9, 0x06, 0x20, 0xee, 0xe0, 0x48, 0x45, 0x34, 0xfd, 0xac, 0x8c, 0xa8, 0x90, 0xfe, 0x5e, 0x81,
0x53, 0x91, 0x3e, 0x84, 0xad, 0xef, 0x45, 0x3b, 0xc9, 0x34, 0x55, 0x74, 0xb0, 0x1d, 0x59, 0x12,
0xae, 0x67, 0x99, 0xf4, 0x1b, 0x5a, 0x0e, 0xfe, 0x41, 0x01, 0xa8, 0x0f, 0xfc, 0x63, 0x81, 0x0c,
0x86, 0x67, 0x46, 0x89, 0xce, 0x0c, 0x7d, 0xe7, 0x18, 0x9e, 0xf7, 0x82, 0xb8, 0x72, 0x4f, 0x13,
0xb4, 0x19, 0x86, 0x37, 0xf0, 0x8f, 0xe5, 0x51, 0x18, 0x7d, 0x46, 0x17, 0x61, 0x89, 0xdf, 0x69,
0xd7, 0x0d, 0xd3, 0x74, 0xb1, 0xe7, 0x89, 0x33, 0xb1, 0x32, 0xa7, 0xd6, 0x39, 0x91, 0xb2, 0x59,
0x26, 0xb6, 0x7d, 0xcb, 0x3f, 0xd1, 0x7d, 0xf2, 0x0c, 0xdb, 0x62, 0x6f, 0x52, 0x96, 0xd4, 0x7d,
0x4a, 0xe4, 0x87, 0x03, 0x5d, 0xcb, 0xf3, 0x5d, 0xc9, 0x26, 0xcf, 0x5f, 0x04, 0x95, 0xb1, 0xd1,
0x49, 0xa9, 0xb4, 0x06, 0xbd, 0x1e, 0x77, 0xf1, 0xab, 0x4f, 0xfb, 0xfb, 0x62, 0x40, 0xb9, 0xac,
0x98, 0x1e, 0x3a, 0x4d, 0x0c, 0xf7, 0x35, 0x82, 0x30, 0xef, 0xc3, 0x4a, 0x68, 0x0c, 0x22, 0xac,
0x22, 0x45, 0xa4, 0x12, 0x2d, 0x22, 0xd5, 0x47, 0x80, 0x38, 0xee, 0xf0, 0x0d, 0xc7, 0xad, 0x9e,
0x86, 0x53, 0x11, 0x45, 0x62, 0x25, 0xbe, 0x06, 0x65, 0x71, 0x2f, 0x49, 0x04, 0xca, 0x59, 0x28,
0xd0, 0x8c, 0xda, 0xb1, 0x4c, 0x79, 0x4e, 0xba, 0xe0, 0x10, 0x73, 0xdb, 0x32, 0x5d, 0xf5, 0x13,
0x28, 0x6b, 0xbc, 0x1f, 0xc1, 0xfb, 0x10, 0x96, 0xc4, 0x2d, 0x26, 0x3d, 0x72, 0x8d, 0x30, 0xed,
0x9a, 0x7a, 0xb8, 0x13, 0xad, 0x6c, 0x87, 0x9b, 0xaa, 0x09, 0x35, 0x5e, 0x32, 0x44, 0xd4, 0xcb,
0xc1, 0x3e, 0x04, 0x79, 0xa3, 0x70, 0x6c, 0x2f, 0x51, 0xf9, 0xb2, 0x1b, 0x6e, 0xaa, 0xe7, 0xe1,
0x5c, 0x6a, 0x2f, 0xc2, 0x13, 0x0e, 0x54, 0x86, 0x2f, 0x4c, 0x4b, 0x1e, 0x18, 0xb3, 0x83, 0x60,
0x25, 0x74, 0x10, 0x7c, 0x26, 0x28, 0x12, 0x73, 0x72, 0x11, 0x63, 0x15, 0xe0, 0xb0, 0xdc, 0xcf,
0x67, 0x95, 0xfb, 0xb3, 0x91, 0x72, 0x5f, 0x6d, 0x07, 0xfe, 0x14, 0xdb, 0xb0, 0x07, 0x6c, 0xbb,
0xc8, 0xfb, 0x96, 0x09, 0x51, 0x1d, 0x35, 0x4a, 0xce, 0xaa, 0x85, 0xa4, 0xd4, 0xab, 0x50, 0x8e,
0xa6, 0xc6, 0x50, 0x9e, 0x53, 0x12, 0x79, 0x6e, 0x29, 0x96, 0xe2, 0x3e, 0x8c, 0x55, 0xc0, 0xd9,
0x3e, 0x8e, 0xd5, 0xbf, 0x77, 0x23, 0xc9, 0xee, 0x5a, 0xca, 0x19, 0xee, 0x6f, 0x28, 0xcf, 0xad,
0x8a, 0xf5, 0xe0, 0xa1, 0x47, 0xe5, 0xc5, 0xa0, 0xd5, 0xb7, 0xa0, 0x74, 0x90, 0xf5, 0x1b, 0x88,
0x59, 0x79, 0x5f, 0xe2, 0x26, 0xac, 0x3e, 0xb4, 0x7a, 0xd8, 0x3b, 0xf1, 0x7c, 0xdc, 0x6f, 0xb0,
0xa4, 0x74, 0x64, 0x61, 0x17, 0xad, 0x03, 0xb0, 0x2d, 0x8c, 0x43, 0xac, 0xe0, 0x6a, 0x7c, 0x88,
0xa2, 0xfe, 0x97, 0x02, 0xcb, 0x43, 0xc1, 0x03, 0xb6, 0x75, 0x7b, 0x03, 0x8a, 0x74, 0xbc, 0x9e,
0x6f, 0xf4, 0x1d, 0x79, 0x9e, 0x15, 0x10, 0xd0, 0x1d, 0x98, 0x3b, 0xf2, 0x24, 0x64, 0x94, 0x0a,
0xa0, 0xa7, 0x19, 0xa2, 0xcd, 0x1e, 0x79, 0x0d, 0x13, 0x7d, 0x04, 0x30, 0xf0, 0xb0, 0x29, 0xce,
0xb0, 0xf2, 0x59, 0xd5, 0xc2, 0x41, 0xf8, 0x7c, 0x9b, 0x0a, 0xf0, 0xab, 0x16, 0x77, 0xa1, 0x64,
0xd9, 0xc4, 0xc4, 0xec, 0xcc, 0xd1, 0x14, 0xa8, 0xd2, 0x18, 0x71, 0xe0, 0x12, 0x07, 0x1e, 0x36,
0x55, 0x2c, 0xd6, 0x42, 0xe9, 0x5f, 0x11, 0x28, 0x4d, 0x58, 0xe1, 0x49, 0xeb, 0x28, 0x30, 0x5c,
0x46, 0xec, 0xc6, 0xa8, 0xd1, 0x31, 0x6f, 0x69, 0x15, 0x4b, 0x94, 0x36, 0x52, 0x54, 0xbd, 0x0d,
0xa7, 0x23, 0x3b, 0xa4, 0x29, 0xb6, 0x2c, 0x6a, 0x2b, 0x06, 0x94, 0x0c, 0xc3, 0x59, 0xc0, 0x10,
0x32, 0x9a, 0xc7, 0xc1, 0x10, 0x1e, 0x87, 0x21, 0x3c, 0xf5, 0x73, 0x38, 0x1b, 0x41, 0x74, 0x22,
0x16, 0xdd, 0x8d, 0x55, 0x6e, 0x97, 0xc6, 0x69, 0x8d, 0x95, 0x70, 0xff, 0xa3, 0xc0, 0x6a, 0x1a,
0xc3, 0x2b, 0x22, 0x8e, 0x3f, 0xce, 0xb8, 0x56, 0x77, 0x6b, 0x32, 0xb3, 0x7e, 0x2b, 0x68, 0xed,
0x3e, 0xd4, 0xd2, 0xfc, 0x99, 0x9c, 0xa5, 0xfc, 0x34, 0xb3, 0xf4, 0xb3, 0x7c, 0x08, 0x79, 0xaf,
0xfb, 0xbe, 0x6b, 0x1d, 0x0e, 0x68, 0xc8, 0xbf, 0x76, 0x34, 0xab, 0x11, 0xe0, 0x32, 0xdc, 0xb5,
0x37, 0x46, 0x88, 0x0f, 0xed, 0x48, 0xc5, 0x66, 0x3e, 0x8d, 0x62, 0x33, 0x1c, 0x53, 0xbf, 0x39,
0x99, 0xbe, 0xef, 0x2c, 0x00, 0xfa, 0xb3, 0x1c, 0x2c, 0x45, 0xa7, 0x08, 0xed, 0x02, 0x18, 0x81,
0xe5, 0xe2, 0x43, 0xb9, 0x38, 0xd1, 0x30, 0xb5, 0x90, 0x20, 0x7a, 0x17, 0xf2, 0x1d, 0x67, 0x20,
0x66, 0x2d, 0xe5, 0x30, 0x78, 0xdb, 0x19, 0xf0, 0x8c, 0x42, 0xd9, 0xe8, 0x9e, 0x8a, 0x9f, 0xed,
0x67, 0x67, 0xc9, 0xa7, 0xec, 0x3d, 0x97, 0x11, 0xcc, 0xe8, 0x31, 0x2c, 0xbd, 0x70, 0x2d, 0xdf,
0x38, 0xec, 0x61, 0xbd, 0x67, 0x9c, 0x60, 0x57, 0x64, 0xc9, 0x09, 0x12, 0x59, 0x59, 0x0a, 0x3e,
0xa1, 0x72, 0xea, 0x1f, 0x42, 0x41, 0x5a, 0x34, 0x66, 0x45, 0xd8, 0x87, 0xb5, 0x01, 0x65, 0xd3,
0xd9, 0x15, 0x38, 0xdb, 0xb0, 0x89, 0xee, 0x61, 0xba, 0x8c, 0xcb, 0xcb, 0xf9, 0x63, 0x52, 0xf4,
0x2a, 0x93, 0xde, 0x26, 0x2e, 0x6e, 0x1a, 0x36, 0x69, 0x73, 0x51, 0xf5, 0x39, 0x94, 0x42, 0x03,
0x1c, 0x63, 0x42, 0x03, 0x56, 0xe4, 0x51, 0xbc, 0x87, 0x7d, 0xb1, 0xbc, 0x4c, 0xd4, 0xf9, 0xb2,
0x90, 0x6b, 0x63, 0x9f, 0x5f, 0x9f, 0xb8, 0x0b, 0x67, 0x35, 0x4c, 0x1c, 0x6c, 0x07, 0xf3, 0xf9,
0x84, 0x74, 0xa7, 0xc8, 0xe0, 0x6f, 0x40, 0x2d, 0x4d, 0x9e, 0xe7, 0x87, 0x6b, 0x97, 0xa0, 0x20,
0x7f, 0xd0, 0x8a, 0x16, 0x20, 0xbf, 0xbf, 0xdd, 0xaa, 0xcc, 0xd0, 0x87, 0x83, 0x9d, 0x56, 0x45,
0x41, 0x05, 0x98, 0x6d, 0x6f, 0xef, 0xb7, 0x2a, 0xb9, 0x6b, 0x7d, 0xa8, 0xc4, 0x7f, 0xcd, 0x89,
0xd6, 0xe0, 0x54, 0x4b, 0xdb, 0x6b, 0xd5, 0x1f, 0xd5, 0xf7, 0x1b, 0x7b, 0x4d, 0xbd, 0xa5, 0x35,
0x3e, 0xae, 0xef, 0xef, 0x56, 0x66, 0xd0, 0x06, 0x9c, 0x0f, 0xbf, 0x78, 0xbc, 0xd7, 0xde, 0xd7,
0xf7, 0xf7, 0xf4, 0xed, 0xbd, 0xe6, 0x7e, 0xbd, 0xd1, 0xdc, 0xd5, 0x2a, 0x0a, 0x3a, 0x0f, 0x67,
0xc3, 0x2c, 0x0f, 0x1a, 0x3b, 0x0d, 0x6d, 0x77, 0x9b, 0x3e, 0xd7, 0x9f, 0x54, 0x72, 0xd7, 0x6e,
0x40, 0x39, 0xf2, 0xe3, 0x4b, 0x6a, 0x52, 0x6b, 0x6f, 0xa7, 0x32, 0x83, 0xca, 0x50, 0x0c, 0xeb,
0x29, 0xc0, 0x6c, 0x73, 0x6f, 0x67, 0xb7, 0x92, 0xbb, 0x76, 0x1b, 0x96, 0x63, 0xb7, 0x71, 0xd1,
0x0a, 0x94, 0xdb, 0xf5, 0xe6, 0xce, 0x83, 0xbd, 0x4f, 0x75, 0x6d, 0xb7, 0xbe, 0xf3, 0x59, 0x65,
0x06, 0xad, 0x42, 0x45, 0x92, 0x9a, 0x7b, 0xfb, 0x9c, 0xaa, 0x5c, 0x7b, 0x16, 0xfb, 0xc6, 0x30,
0x3a, 0x0d, 0x2b, 0x41, 0x37, 0xfa, 0xb6, 0xb6, 0x5b, 0xdf, 0xdf, 0xa5, 0xbd, 0x47, 0xc8, 0xda,
0x41, 0xb3, 0xd9, 0x68, 0x3e, 0xaa, 0x28, 0x54, 0xeb, 0x90, 0xbc, 0xfb, 0x69, 0x83, 0x32, 0xe7,
0xa2, 0xcc, 0x07, 0xcd, 0x1f, 0x36, 0xf7, 0x3e, 0x69, 0x56, 0xf2, 0x5b, 0xbf, 0x58, 0x81, 0x25,
0x59, 0xe8, 0x61, 0x97, 0xdd, 0x6a, 0x69, 0xc1, 0x82, 0xfc, 0x81, 0x74, 0x4a, 0x86, 0x8e, 0xfe,
0xac, 0xbb, 0xb6, 0x31, 0x82, 0x43, 0xd4, 0xdb, 0x33, 0xe8, 0x90, 0xd5, 0xbf, 0xa1, 0xdb, 0xd1,
0x97, 0x52, 0xab, 0xcd, 0xc4, 0x85, 0xec, 0xda, 0xe5, 0xb1, 0x7c, 0x41, 0x1f, 0x98, 0x96, 0xb8,
0xe1, 0x9f, 0xff, 0xa0, 0xcb, 0x69, 0xb5, 0x69, 0xca, 0xef, 0x8b, 0x6a, 0x57, 0xc6, 0x33, 0x06,
0xdd, 0x3c, 0x83, 0x4a, 0xfc, 0xa7, 0x40, 0x28, 0x05, 0x3a, 0xcd, 0xf8, 0xbd, 0x51, 0xed, 0xda,
0x24, 0xac, 0xe1, 0xce, 0x12, 0x3f, 0x9a, 0xb9, 0x3a, 0xc9, 0xaf, 0x10, 0x32, 0x3b, 0xcb, 0xfa,
0xc1, 0x02, 0x77, 0x60, 0xf4, 0xe6, 0x33, 0x4a, 0xfd, 0x85, 0x4a, 0xca, 0xbd, 0xf9, 0x34, 0x07,
0xa6, 0x5f, 0xa2, 0x56, 0x67, 0xd0, 0x31, 0x2c, 0xc7, 0xae, 0x27, 0xa0, 0x14, 0xf1, 0xf4, 0x7b,
0x18, 0xb5, 0xab, 0x13, 0x70, 0x46, 0x23, 0x22, 0x7c, 0x1d, 0x21, 0x3d, 0x22, 0x52, 0x2e, 0x3b,
0xa4, 0x47, 0x44, 0xea, 0xcd, 0x06, 0x16, 0xdc, 0x91, 0x6b, 0x08, 0x69, 0xc1, 0x9d, 0x76, 0xf9,
0xa1, 0x76, 0x79, 0x2c, 0x5f, 0xd8, 0x69, 0xb1, 0x4b, 0x09, 0x69, 0x4e, 0x4b, 0xbf, 0xf4, 0x50,
0xbb, 0x3a, 0x01, 0x67, 0x3c, 0x0a, 0x86, 0x47, 0x9c, 0x59, 0x51, 0x90, 0x38, 0x90, 0xcf, 0x8a,
0x82, 0xe4, 0x69, 0xa9, 0x88, 0x82, 0xd8, 0xd1, 0xe4, 0x95, 0x09, 0x8e, 0x52, 0xb2, 0xa3, 0x20,
0xfd, 0xd0, 0x45, 0x9d, 0x41, 0x7f, 0xa4, 0x40, 0x35, 0xeb, 0x98, 0x02, 0xa5, 0xd4, 0x77, 0x63,
0x4e, 0x56, 0x6a, 0x5b, 0xd3, 0x88, 0x04, 0x56, 0x7c, 0x09, 0x28, 0xb9, 0xee, 0xa1, 0x77, 0xd2,
0x66, 0x26, 0x63, 0x75, 0xad, 0xbd, 0x3b, 0x19, 0x73, 0xd0, 0x65, 0x1b, 0x0a, 0xf2, 0x60, 0x04,
0xa5, 0x64, 0xe9, 0xd8, 0xb1, 0x4c, 0x4d, 0x1d, 0xc5, 0x12, 0x28, 0x7d, 0x04, 0xb3, 0x94, 0x8a,
0xce, 0xa7, 0x73, 0x4b, 0x65, 0xeb, 0x59, 0xaf, 0x03, 0x45, 0x4f, 0x61, 0x9e, 0x9f, 0x04, 0xa0,
0x14, 0xe4, 0x21, 0x72, 0x5e, 0x51, 0xbb, 0x90, 0xcd, 0x10, 0xa8, 0xfb, 0x82, 0xff, 0xef, 0x0c,
0x01, 0xf2, 0xa3, 0xb7, 0xd3, 0x7f, 0x8c, 0x1c, 0x3d, 0x53, 0xa8, 0x5d, 0x1c, 0xc3, 0x15, 0xfe,
0x28, 0x62, 0x55, 0xef, 0xe5, 0xb1, 0x5b, 0x97, 0xec, 0x8f, 0x22, 0x7d, 0x73, 0xc4, 0x83, 0x24,
0xb9, 0x79, 0x4a, 0x0b, 0x92, 0xcc, 0x2d, 0x6b, 0x5a, 0x90, 0x64, 0xef, 0xc7, 0xd4, 0x19, 0xe4,
0xc3, 0xa9, 0x14, 0xa8, 0x0c, 0xbd, 0x9b, 0x15, 0xe4, 0x69, 0xb8, 0x5d, 0xed, 0xbd, 0x09, 0xb9,
0xc3, 0x93, 0x2f, 0x3e, 0xfa, 0x37, 0xb3, 0xf1, 0xa3, 0xcc, 0xc9, 0x8f, 0x7f, 0xe2, 0x5b, 0xff,
0x9a, 0x87, 0x45, 0x0e, 0x83, 0x8a, 0x0a, 0xe6, 0x33, 0x80, 0xe1, 0x09, 0x04, 0x7a, 0x2b, 0xdd,
0x27, 0x91, 0x53, 0x9a, 0xda, 0xdb, 0xa3, 0x99, 0xc2, 0x81, 0x16, 0x42, 0xf3, 0xd3, 0x02, 0x2d,
0x79, 0x68, 0x91, 0x16, 0x68, 0x29, 0x47, 0x02, 0xea, 0x0c, 0xfa, 0x18, 0x8a, 0x01, 0x6c, 0x8c,
0xd2, 0x60, 0xe7, 0x18, 0x2e, 0x5e, 0x7b, 0x6b, 0x24, 0x4f, 0xd8, 0xea, 0x10, 0x26, 0x9c, 0x66,
0x75, 0x12, 0x7b, 0x4e, 0xb3, 0x3a, 0x0d, 0x58, 0x1e, 0xfa, 0x84, 0x23, 0x47, 0x99, 0x3e, 0x89,
0x00, 0x77, 0x99, 0x3e, 0x89, 0xc2, 0x4f, 0xea, 0xcc, 0x83, 0x4b, 0xbf, 0xfa, 0x6a, 0x5d, 0xf9,
0xa7, 0xaf, 0xd6, 0x67, 0x7e, 0xfa, 0xf5, 0xba, 0xf2, 0xab, 0xaf, 0xd7, 0x95, 0x7f, 0xfc, 0x7a,
0x5d, 0xf9, 0xb7, 0xaf, 0xd7, 0x95, 0x3f, 0xfb, 0xf7, 0xf5, 0x99, 0x1f, 0x15, 0xa4, 0xf4, 0xe1,
0x3c, 0xfb, 0x0f, 0x38, 0x1f, 0xfc, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x15, 0x30, 0x49, 0xcc,
0xc7, 0x48, 0x00, 0x00,
0xf8, 0x73, 0xc7, 0xf2, 0x58, 0xb3, 0xeb, 0x89, 0xed, 0x59, 0xdb, 0xb4, 0x24, 0xdb, 0xcc, 0xda,
0x14, 0xd3, 0x94, 0xe6, 0x63, 0x67, 0x80, 0xde, 0x16, 0xbb, 0x44, 0xf5, 0x9a, 0xec, 0xea, 0xe9,
0x6e, 0xda, 0x56, 0x02, 0x04, 0x0b, 0x2c, 0xb2, 0x87, 0x00, 0x01, 0x72, 0xce, 0x71, 0x73, 0xc8,
0x21, 0xb7, 0x00, 0x41, 0x0e, 0x39, 0x6d, 0x90, 0xc3, 0x5e, 0x02, 0xe4, 0xb4, 0x48, 0x90, 0x4b,
0x66, 0x92, 0x5c, 0x02, 0x24, 0xc8, 0x1f, 0x90, 0x43, 0x50, 0x5f, 0xcd, 0xfe, 0xe4, 0x87, 0xc7,
0xbb, 0xb3, 0x39, 0xa9, 0xfb, 0xf5, 0x7b, 0xaf, 0x5e, 0xbd, 0x7a, 0xf5, 0xea, 0xd5, 0xaf, 0x8a,
0x82, 0xa2, 0xe1, 0x58, 0x9b, 0x8e, 0x4b, 0x7c, 0x82, 0x2a, 0xee, 0xc0, 0xf6, 0xad, 0x3e, 0xde,
0x7c, 0x71, 0xd3, 0xe8, 0x39, 0xc7, 0xc6, 0x56, 0xed, 0x7a, 0xd7, 0xf2, 0x8f, 0x07, 0x87, 0x9b,
0x1d, 0xd2, 0xbf, 0xd1, 0x25, 0x5d, 0x72, 0x83, 0x31, 0x1e, 0x0e, 0x8e, 0xd8, 0x1b, 0x7b, 0x61,
0x4f, 0x5c, 0x81, 0x7a, 0x0d, 0x96, 0x3e, 0xc6, 0xae, 0x67, 0x11, 0x5b, 0xc3, 0x5f, 0x0e, 0xb0,
0xe7, 0xa3, 0x2a, 0x2c, 0xbc, 0xe0, 0x94, 0xaa, 0x72, 0x41, 0xb9, 0x52, 0xd4, 0xe4, 0xab, 0xfa,
0x17, 0x0a, 0x2c, 0x07, 0xcc, 0x9e, 0x43, 0x6c, 0x0f, 0x67, 0x73, 0xa3, 0x0d, 0x58, 0x14, 0xc6,
0xe9, 0xb6, 0xd1, 0xc7, 0xd5, 0x1c, 0xfb, 0x5c, 0x12, 0xb4, 0xa6, 0xd1, 0xc7, 0xe8, 0x32, 0x2c,
0x4b, 0x16, 0xa9, 0x24, 0xcf, 0xb8, 0x96, 0x04, 0x59, 0xb4, 0x86, 0x36, 0xe1, 0x94, 0x64, 0x34,
0x1c, 0x2b, 0x60, 0x9e, 0x65, 0xcc, 0x2b, 0xe2, 0x53, 0xdd, 0xb1, 0x04, 0xbf, 0xfa, 0x39, 0x14,
0x77, 0x9a, 0xed, 0x6d, 0x62, 0x1f, 0x59, 0x5d, 0x6a, 0xa2, 0x87, 0x5d, 0x2a, 0x53, 0x55, 0x2e,
0xe4, 0xa9, 0x89, 0xe2, 0x15, 0xd5, 0xa0, 0xe0, 0x61, 0xc3, 0xed, 0x1c, 0x63, 0xaf, 0x9a, 0x63,
0x9f, 0x82, 0x77, 0x2a, 0x45, 0x1c, 0xdf, 0x22, 0xb6, 0x57, 0xcd, 0x73, 0x29, 0xf1, 0xaa, 0xfe,
0x5c, 0x81, 0x52, 0x8b, 0xb8, 0xfe, 0x33, 0xc3, 0x71, 0x2c, 0xbb, 0x8b, 0x6e, 0x41, 0x81, 0xf9,
0xb2, 0x43, 0x7a, 0xcc, 0x07, 0x4b, 0x5b, 0xb5, 0xcd, 0xf8, 0xb0, 0x6c, 0xb6, 0x04, 0x87, 0x16,
0xf0, 0xa2, 0x8b, 0xb0, 0xd4, 0x21, 0xb6, 0x6f, 0x58, 0x36, 0x76, 0x75, 0x87, 0xb8, 0x3e, 0x73,
0xd1, 0x9c, 0x56, 0x0e, 0xa8, 0xb4, 0x15, 0x74, 0x0e, 0x8a, 0xc7, 0xc4, 0xf3, 0x39, 0x47, 0x9e,
0x71, 0x14, 0x28, 0x81, 0x7d, 0x5c, 0x83, 0x05, 0xf6, 0xd1, 0x72, 0x84, 0x33, 0xe6, 0xe9, 0x6b,
0xc3, 0x51, 0x7f, 0xa5, 0xc0, 0xdc, 0x33, 0x32, 0xb0, 0xfd, 0x58, 0x33, 0x86, 0x7f, 0x2c, 0x06,
0x2a, 0xd4, 0x8c, 0xe1, 0x1f, 0x0f, 0x9b, 0xa1, 0x1c, 0x7c, 0xac, 0x78, 0x33, 0xf4, 0x63, 0x0d,
0x0a, 0x2e, 0x36, 0x4c, 0x62, 0xf7, 0x4e, 0x98, 0x09, 0x05, 0x2d, 0x78, 0xa7, 0x83, 0xe8, 0xe1,
0x9e, 0x65, 0x0f, 0x5e, 0xe9, 0x2e, 0xee, 0x19, 0x87, 0xb8, 0xc7, 0x4c, 0x29, 0x68, 0x4b, 0x82,
0xac, 0x71, 0x2a, 0xda, 0x81, 0x92, 0xe3, 0x12, 0xc7, 0xe8, 0x1a, 0xd4, 0x8f, 0xd5, 0x39, 0xe6,
0x2a, 0x35, 0xe9, 0x2a, 0x66, 0x76, 0x6b, 0xc8, 0xa9, 0x85, 0xc5, 0xd4, 0xbf, 0x52, 0x60, 0x99,
0x06, 0x8f, 0xe7, 0x18, 0x1d, 0xbc, 0xc7, 0x86, 0x04, 0xdd, 0x86, 0x05, 0x1b, 0xfb, 0x2f, 0x89,
0xfb, 0x5c, 0x0c, 0xc0, 0xdb, 0x49, 0xad, 0x81, 0xcc, 0x33, 0x62, 0x62, 0x4d, 0xf2, 0xa3, 0x9b,
0x90, 0x77, 0x2c, 0x93, 0x75, 0x78, 0x02, 0x31, 0xca, 0x4b, 0x45, 0x2c, 0xa7, 0xc3, 0xfc, 0x30,
0x89, 0x88, 0xe5, 0x74, 0x54, 0x15, 0xa0, 0x61, 0xfb, 0xb7, 0xbe, 0xfb, 0xb1, 0xd1, 0x1b, 0x60,
0xb4, 0x0a, 0x73, 0x2f, 0xe8, 0x03, 0x33, 0x36, 0xaf, 0xf1, 0x17, 0xf5, 0xab, 0x3c, 0x9c, 0x7b,
0x4a, 0xfd, 0xd5, 0x36, 0x6c, 0xf3, 0x90, 0xbc, 0x6a, 0xe3, 0xce, 0xc0, 0xb5, 0xfc, 0x93, 0x6d,
0x62, 0xfb, 0xf8, 0x95, 0x8f, 0x9a, 0xb0, 0x62, 0x4b, 0xcd, 0xba, 0x0c, 0x4d, 0xaa, 0xa1, 0xb4,
0xb5, 0x31, 0xc2, 0x08, 0xee, 0x22, 0xad, 0x62, 0x47, 0x09, 0x1e, 0x7a, 0x32, 0x1c, 0x37, 0xa9,
0x2d, 0xc7, 0xb4, 0xa5, 0x74, 0xa9, 0xbd, 0xcb, 0x2c, 0x13, 0xba, 0xe4, 0xc0, 0x4a, 0x4d, 0x1f,
0x01, 0x9d, 0xd5, 0xba, 0xe1, 0xe9, 0x03, 0x0f, 0xbb, 0xcc, 0x31, 0xa5, 0xad, 0xb7, 0x92, 0x5a,
0x86, 0x2e, 0xd0, 0x8a, 0xee, 0xc0, 0xae, 0x7b, 0x07, 0x1e, 0x76, 0x59, 0x12, 0x10, 0xb1, 0xa4,
0xbb, 0x84, 0xf8, 0x47, 0x9e, 0x8c, 0x1f, 0x49, 0xd6, 0x18, 0x15, 0xdd, 0x80, 0x53, 0xde, 0xc0,
0x71, 0x7a, 0xb8, 0x8f, 0x6d, 0xdf, 0xe8, 0xe9, 0x5d, 0x97, 0x0c, 0x1c, 0xaf, 0x3a, 0x77, 0x21,
0x7f, 0x25, 0xaf, 0xa1, 0xf0, 0xa7, 0xc7, 0xec, 0x0b, 0x5a, 0x07, 0x70, 0x5c, 0xeb, 0x85, 0xd5,
0xc3, 0x5d, 0x6c, 0x56, 0xe7, 0x99, 0xd2, 0x10, 0x05, 0xbd, 0x0f, 0xab, 0x1e, 0xee, 0x74, 0x48,
0xdf, 0xd1, 0x1d, 0x97, 0x1c, 0x59, 0x3d, 0xcc, 0xa3, 0x7f, 0x81, 0x45, 0x3f, 0x12, 0xdf, 0x5a,
0xfc, 0x13, 0x9b, 0x07, 0xf7, 0x58, 0x4e, 0xa3, 0x3d, 0x65, 0x8d, 0x57, 0x0b, 0x13, 0x74, 0x15,
0x58, 0x57, 0x99, 0x49, 0xea, 0xcf, 0x73, 0x70, 0x9a, 0x79, 0xb2, 0x45, 0x4c, 0x31, 0xcc, 0x22,
0x49, 0xbd, 0x03, 0xe5, 0x0e, 0xd3, 0xa9, 0x3b, 0x86, 0x8b, 0x6d, 0x5f, 0x4c, 0xd2, 0x45, 0x4e,
0x6c, 0x31, 0x1a, 0xfa, 0x14, 0x2a, 0x9e, 0x88, 0x0a, 0xbd, 0xc3, 0xc3, 0x42, 0x8c, 0xd9, 0xf5,
0xa4, 0x09, 0x23, 0x62, 0x49, 0x5b, 0xf6, 0x12, 0xc1, 0xb5, 0xe0, 0x9d, 0x78, 0x1d, 0xbf, 0xc7,
0xb3, 0x5d, 0x69, 0xeb, 0xbb, 0x19, 0x0a, 0xe3, 0x86, 0x6f, 0xb6, 0xb9, 0xd8, 0xae, 0xed, 0xbb,
0x27, 0x9a, 0x54, 0x52, 0xbb, 0x03, 0x8b, 0xe1, 0x0f, 0xa8, 0x02, 0xf9, 0xe7, 0xf8, 0x44, 0x74,
0x8a, 0x3e, 0x0e, 0x27, 0x01, 0xcf, 0x35, 0xfc, 0xe5, 0x4e, 0xee, 0x77, 0x14, 0xd5, 0x05, 0x34,
0x6c, 0xe5, 0x19, 0xf6, 0x0d, 0xd3, 0xf0, 0x0d, 0x84, 0x60, 0x96, 0x2d, 0x23, 0x5c, 0x05, 0x7b,
0xa6, 0x5a, 0x07, 0x62, 0xf2, 0x16, 0x35, 0xfa, 0x88, 0xde, 0x82, 0x62, 0x10, 0xe8, 0x62, 0x2d,
0x19, 0x12, 0x68, 0x4e, 0x37, 0x7c, 0x1f, 0xf7, 0x1d, 0x9f, 0x85, 0x58, 0x59, 0x93, 0xaf, 0xea,
0x7f, 0xcf, 0x42, 0x25, 0x31, 0x26, 0x0f, 0xa0, 0xd0, 0x17, 0xcd, 0x8b, 0x89, 0xf6, 0x6e, 0x4a,
0x62, 0x4f, 0x98, 0xaa, 0x05, 0x52, 0x34, 0x6f, 0xd2, 0x1c, 0x1a, 0x5a, 0xff, 0x82, 0x77, 0x3a,
0xe2, 0x3d, 0xd2, 0xd5, 0x4d, 0xcb, 0xc5, 0x1d, 0x9f, 0xb8, 0x27, 0xc2, 0xdc, 0xc5, 0x1e, 0xe9,
0xee, 0x48, 0x1a, 0xba, 0x03, 0x60, 0xda, 0x1e, 0x1d, 0xec, 0x23, 0xab, 0xcb, 0x8c, 0x2e, 0x6d,
0x9d, 0x4b, 0x1a, 0x11, 0x2c, 0x76, 0x5a, 0xd1, 0xb4, 0x3d, 0x61, 0xfe, 0x43, 0x28, 0xd3, 0x35,
0x43, 0xef, 0xf3, 0x75, 0x8a, 0xcf, 0x94, 0xd2, 0xd6, 0xf9, 0xb4, 0x3e, 0x04, 0xab, 0x99, 0xb6,
0xe8, 0x0c, 0x5f, 0x3c, 0xf4, 0x08, 0xe6, 0x59, 0xf2, 0xf6, 0xaa, 0xf3, 0x4c, 0x78, 0x73, 0x94,
0x03, 0x44, 0x44, 0x3c, 0x65, 0x02, 0x3c, 0x20, 0x84, 0x34, 0x3a, 0x80, 0x92, 0x61, 0xdb, 0xc4,
0x37, 0x78, 0xa2, 0x59, 0x60, 0xca, 0x3e, 0x98, 0x40, 0x59, 0x7d, 0x28, 0xc5, 0x35, 0x86, 0xf5,
0xa0, 0xef, 0xc3, 0x1c, 0xcb, 0x44, 0x62, 0x22, 0x5e, 0x9e, 0x30, 0x68, 0x35, 0x2e, 0x55, 0xbb,
0x0d, 0xa5, 0x90, 0xb1, 0xd3, 0x04, 0x69, 0xed, 0x1e, 0x54, 0xe2, 0xa6, 0x4d, 0x15, 0xe4, 0x7f,
0x00, 0xab, 0xda, 0xc0, 0x1e, 0x1a, 0x26, 0xab, 0xaf, 0x3b, 0x30, 0x2f, 0x06, 0x9b, 0x47, 0x9c,
0x3a, 0xde, 0x47, 0x9a, 0x90, 0x08, 0x97, 0x53, 0xc7, 0x86, 0x6d, 0xf6, 0xb0, 0x2b, 0xda, 0x95,
0xe5, 0xd4, 0x13, 0x4e, 0x55, 0xbf, 0x0f, 0xa7, 0x63, 0x8d, 0x8b, 0x6a, 0xee, 0x5d, 0x58, 0x72,
0x88, 0xa9, 0x7b, 0x9c, 0xac, 0x5b, 0xa6, 0x4c, 0x43, 0x4e, 0xc0, 0xdb, 0x30, 0xa9, 0x78, 0xdb,
0x27, 0x4e, 0xd2, 0xf8, 0xc9, 0xc4, 0xab, 0x70, 0x26, 0x2e, 0xce, 0x9b, 0x57, 0xef, 0xc3, 0x9a,
0x86, 0xfb, 0xe4, 0x05, 0x7e, 0x5d, 0xd5, 0x35, 0xa8, 0x26, 0x15, 0x08, 0xe5, 0x9f, 0xc1, 0xda,
0x90, 0xda, 0xf6, 0x0d, 0x7f, 0xe0, 0x4d, 0xa5, 0x5c, 0x94, 0xba, 0x87, 0xc4, 0xe3, 0xc3, 0x59,
0xd0, 0xe4, 0xab, 0x7a, 0x35, 0xac, 0xba, 0xc9, 0x2b, 0x0b, 0xde, 0x02, 0x5a, 0x82, 0x9c, 0xe5,
0x08, 0x75, 0x39, 0xcb, 0x51, 0x9f, 0x40, 0x31, 0x58, 0x9a, 0xd1, 0xdd, 0x61, 0x8d, 0x99, 0x9b,
0x74, 0x21, 0x0f, 0xca, 0xd0, 0xfd, 0xc4, 0x52, 0x22, 0x9a, 0xbc, 0x0b, 0x10, 0xa4, 0x3c, 0x59,
0x21, 0x9c, 0x1b, 0xa1, 0x58, 0x0b, 0xb1, 0xab, 0x3f, 0x9d, 0x0b, 0x27, 0xc2, 0x50, 0x27, 0xcc,
0xa0, 0x13, 0x66, 0x24, 0x31, 0xe6, 0x5e, 0x2b, 0x31, 0x7e, 0x08, 0x73, 0x9e, 0x6f, 0xf8, 0x58,
0x54, 0x51, 0x1b, 0xa3, 0xc4, 0xa9, 0x11, 0x58, 0xe3, 0xfc, 0xe8, 0x3c, 0x40, 0xc7, 0xc5, 0x86,
0x8f, 0x4d, 0xdd, 0xe0, 0x59, 0x3c, 0xaf, 0x15, 0x05, 0xa5, 0xee, 0xa3, 0xed, 0x61, 0x25, 0x38,
0xc7, 0x0c, 0xbb, 0x3a, 0x4a, 0x73, 0x64, 0xa8, 0x86, 0x35, 0x61, 0x90, 0x55, 0xe6, 0x27, 0xcc,
0x2a, 0x42, 0x01, 0x97, 0x0a, 0xe5, 0xcc, 0x85, 0xf1, 0x39, 0x93, 0x8b, 0x4e, 0x92, 0x33, 0x0b,
0xe3, 0x73, 0xa6, 0x50, 0x36, 0x3a, 0x67, 0xa6, 0x64, 0x89, 0x62, 0x5a, 0x96, 0xf8, 0x36, 0xb3,
0xe3, 0x3f, 0x2b, 0x50, 0x4d, 0x4e, 0x56, 0x91, 0xa4, 0xee, 0xc0, 0xbc, 0xc7, 0x28, 0x93, 0xa4,
0x48, 0x21, 0x2b, 0x24, 0xd0, 0x13, 0x98, 0xb5, 0xec, 0x23, 0xc2, 0x76, 0x7b, 0xa9, 0x45, 0x4e,
0x56, 0xab, 0x9b, 0x0d, 0xfb, 0x88, 0x70, 0x6f, 0x32, 0x0d, 0xb5, 0x0f, 0xa1, 0x18, 0x90, 0xa6,
0xea, 0xdb, 0x1e, 0xac, 0xc6, 0x62, 0x9b, 0xef, 0x0a, 0x82, 0x29, 0xa1, 0x4c, 0x37, 0x25, 0xd4,
0x9f, 0xe4, 0xc2, 0x53, 0xf6, 0x91, 0xd5, 0xf3, 0xb1, 0x9b, 0x98, 0xb2, 0x1f, 0x49, 0xed, 0x7c,
0xbe, 0x5e, 0x1a, 0xab, 0x9d, 0x17, 0xaf, 0x62, 0xd6, 0x7d, 0x01, 0x4b, 0x2c, 0x28, 0x75, 0x0f,
0xf7, 0x58, 0x65, 0x22, 0xaa, 0xc4, 0xef, 0x8d, 0x52, 0xc3, 0x2d, 0xe1, 0xa1, 0xdd, 0x16, 0x72,
0xdc, 0x83, 0xe5, 0x5e, 0x98, 0x56, 0x7b, 0x00, 0x28, 0xc9, 0x34, 0x95, 0x4f, 0xdb, 0x34, 0x17,
0xd2, 0x2d, 0x71, 0xca, 0x72, 0x7a, 0xc4, 0xcc, 0x98, 0x24, 0x56, 0xb8, 0xc1, 0x9a, 0x90, 0x50,
0xff, 0x2b, 0x0f, 0x30, 0xfc, 0xf8, 0xff, 0x28, 0x09, 0x3e, 0x08, 0x12, 0x10, 0xaf, 0xf8, 0xae,
0x8c, 0x52, 0x9c, 0x9a, 0x7a, 0xf6, 0xa2, 0xa9, 0x87, 0xd7, 0x7e, 0xd7, 0x47, 0xaa, 0x99, 0x3a,
0xe9, 0x2c, 0xfc, 0xb6, 0x25, 0x9d, 0xa7, 0x70, 0x26, 0x1e, 0x44, 0x22, 0xe3, 0x6c, 0xc1, 0x9c,
0xe5, 0xe3, 0x3e, 0xc7, 0x8f, 0x52, 0xf7, 0x7b, 0x21, 0x21, 0xce, 0xaa, 0x6e, 0x40, 0xb1, 0xd1,
0x37, 0xba, 0xb8, 0xed, 0xe0, 0x0e, 0x6d, 0xd4, 0xa2, 0x2f, 0xc2, 0x10, 0xfe, 0xa2, 0x6e, 0x41,
0xe1, 0x07, 0xf8, 0x84, 0xcf, 0xfe, 0x09, 0x0d, 0x55, 0xff, 0x24, 0x07, 0x6b, 0x6c, 0xf5, 0xd9,
0x96, 0xe8, 0x8d, 0x86, 0x3d, 0x32, 0x70, 0x3b, 0xd8, 0x63, 0x61, 0xe1, 0x0c, 0x74, 0x07, 0xbb,
0x16, 0x31, 0x05, 0xb8, 0x50, 0xec, 0x38, 0x83, 0x16, 0x23, 0xa0, 0x73, 0x40, 0x5f, 0xf4, 0x2f,
0x07, 0x44, 0x44, 0x6c, 0x5e, 0x2b, 0x74, 0x9c, 0xc1, 0xef, 0xd1, 0x77, 0x29, 0xeb, 0x1d, 0x1b,
0x2e, 0xf6, 0x58, 0x40, 0x72, 0xd9, 0x36, 0x23, 0xa0, 0x9b, 0x70, 0xba, 0x8f, 0xfb, 0xc4, 0x3d,
0xd1, 0x7b, 0x56, 0xdf, 0xf2, 0x75, 0xcb, 0xd6, 0x0f, 0x4f, 0x7c, 0xec, 0x89, 0xe0, 0x43, 0xfc,
0xe3, 0x53, 0xfa, 0xad, 0x61, 0x3f, 0xa4, 0x5f, 0x90, 0x0a, 0x65, 0x42, 0xfa, 0xba, 0xd7, 0x21,
0x2e, 0xd6, 0x0d, 0xf3, 0xc7, 0x6c, 0x41, 0xce, 0x6b, 0x25, 0x42, 0xfa, 0x6d, 0x4a, 0xab, 0x9b,
0x3f, 0x46, 0x6f, 0x43, 0xa9, 0xe3, 0x0c, 0x3c, 0xec, 0xeb, 0xf4, 0x0f, 0x5b, 0x6f, 0x8b, 0x1a,
0x70, 0xd2, 0xb6, 0x33, 0xf0, 0x42, 0x0c, 0x7d, 0xea, 0xff, 0x85, 0x30, 0xc3, 0x33, 0xea, 0x66,
0x03, 0xca, 0x11, 0x70, 0x82, 0xee, 0x13, 0x19, 0x0a, 0x21, 0xf6, 0x89, 0xf4, 0x99, 0xd2, 0x5c,
0xd2, 0x93, 0x9e, 0x64, 0xcf, 0x94, 0xe6, 0x9f, 0x38, 0x72, 0x93, 0xc8, 0x9e, 0xa9, 0xcb, 0x7b,
0xf8, 0x85, 0x00, 0xb0, 0x8a, 0x1a, 0x7f, 0x51, 0x4d, 0x80, 0x6d, 0xc3, 0x31, 0x0e, 0xad, 0x9e,
0xe5, 0x9f, 0xa0, 0xab, 0x50, 0x31, 0x4c, 0x53, 0xef, 0x48, 0x8a, 0x85, 0x25, 0xac, 0xb8, 0x6c,
0x98, 0xe6, 0x76, 0x88, 0x8c, 0xbe, 0x03, 0x2b, 0xa6, 0x4b, 0x9c, 0x28, 0x2f, 0xc7, 0x19, 0x2b,
0xf4, 0x43, 0x98, 0x59, 0xfd, 0xf7, 0x39, 0x38, 0x1f, 0x1d, 0xd8, 0x38, 0x00, 0xf4, 0x00, 0x16,
0x63, 0xad, 0x66, 0x80, 0x0f, 0x43, 0x6b, 0xb5, 0x88, 0x44, 0x0c, 0x10, 0xc9, 0x25, 0x00, 0x91,
0x54, 0x88, 0x29, 0xff, 0x46, 0x21, 0xa6, 0xd9, 0x37, 0x02, 0x31, 0xcd, 0x4d, 0x07, 0x31, 0x5d,
0x62, 0xd9, 0x47, 0x4a, 0xb3, 0xdd, 0x38, 0x0f, 0xb5, 0x72, 0xc0, 0x63, 0x4b, 0x3c, 0x3a, 0x06,
0x45, 0x2d, 0x4c, 0x03, 0x45, 0x15, 0x32, 0xa1, 0x28, 0x1a, 0x35, 0x8e, 0x63, 0xb8, 0x7d, 0xe2,
0x4a, 0xac, 0x49, 0x54, 0x5d, 0xcb, 0x92, 0x2e, 0x70, 0xa6, 0x4c, 0x54, 0x0a, 0x32, 0x51, 0xa9,
0x0b, 0xb0, 0x68, 0x13, 0xdd, 0xc6, 0x2f, 0x75, 0x3a, 0x96, 0x5e, 0xb5, 0xc4, 0x07, 0xd6, 0x26,
0x4d, 0xfc, 0xb2, 0x45, 0x29, 0x09, 0xdc, 0x6a, 0x71, 0x3a, 0xdc, 0x0a, 0x6d, 0xc0, 0x62, 0xdf,
0xf0, 0x9e, 0x63, 0x93, 0x99, 0xe2, 0x55, 0xcb, 0x2c, 0x88, 0x4b, 0x9c, 0x46, 0x6d, 0xf0, 0xd0,
0x45, 0x08, 0x9c, 0x24, 0x98, 0x96, 0x18, 0x53, 0x59, 0x52, 0x19, 0x9b, 0xfa, 0xb7, 0x0a, 0xac,
0x46, 0xc3, 0x5c, 0xa0, 0x15, 0x8f, 0xa1, 0xe8, 0xca, 0x4c, 0x26, 0x42, 0xfb, 0x6a, 0x46, 0xe1,
0x9d, 0x4c, 0x7d, 0xda, 0x50, 0x16, 0xfd, 0x30, 0x13, 0x24, 0xbb, 0x31, 0x4e, 0xdf, 0x38, 0x98,
0x4c, 0x6d, 0xc0, 0xdb, 0x9f, 0x58, 0xb6, 0x49, 0x5e, 0x7a, 0x99, 0xb3, 0x34, 0x25, 0xd6, 0x94,
0x94, 0x58, 0x53, 0x7f, 0xa1, 0xc0, 0x99, 0xb8, 0x2e, 0xe1, 0x8a, 0x46, 0xd2, 0x15, 0xdf, 0x49,
0x9a, 0x1e, 0x17, 0x4e, 0x75, 0xc6, 0x17, 0x99, 0xce, 0xb8, 0x39, 0x5e, 0xe3, 0x58, 0x77, 0xfc,
0xa5, 0x02, 0x67, 0x33, 0xcd, 0x88, 0x2d, 0x29, 0x4a, 0x7c, 0x49, 0x11, 0xcb, 0x51, 0x87, 0x0c,
0x6c, 0x3f, 0xb4, 0x1c, 0x6d, 0xb3, 0x43, 0x0b, 0x9e, 0xf7, 0xf5, 0xbe, 0xf1, 0xca, 0xea, 0x0f,
0xfa, 0x62, 0x3d, 0xa2, 0xea, 0x9e, 0x71, 0xca, 0x6b, 0x2c, 0x48, 0x6a, 0x1d, 0x56, 0x02, 0x2b,
0x47, 0xc2, 0x8a, 0x21, 0x98, 0x30, 0x17, 0x85, 0x09, 0x6d, 0x98, 0xdf, 0xc1, 0x2f, 0xac, 0x0e,
0x7e, 0x23, 0xa7, 0x2a, 0x17, 0xa0, 0xe4, 0x60, 0xb7, 0x6f, 0x79, 0x5e, 0x90, 0x68, 0x8b, 0x5a,
0x98, 0xa4, 0xfe, 0xc7, 0x3c, 0x2c, 0xc7, 0xa3, 0xe3, 0x7e, 0x02, 0x95, 0x7c, 0x27, 0x65, 0x09,
0x88, 0x77, 0x34, 0x54, 0x76, 0xde, 0x94, 0xc5, 0x48, 0x2e, 0x0b, 0x1a, 0x08, 0x0a, 0x17, 0x51,
0xa9, 0x50, 0x8f, 0x74, 0x48, 0xbf, 0x6f, 0xd8, 0xa6, 0x3c, 0x0c, 0x13, 0xaf, 0xd4, 0x7f, 0x86,
0xdb, 0xa5, 0x6e, 0xa7, 0x64, 0xf6, 0x4c, 0x07, 0x8f, 0xee, 0xa3, 0x2d, 0x9b, 0xa1, 0x9b, 0x2c,
0x59, 0x17, 0x35, 0x10, 0xa4, 0x1d, 0xcb, 0x45, 0x9b, 0x30, 0x8b, 0xed, 0x17, 0xb2, 0xae, 0x4c,
0x39, 0x2d, 0x93, 0x65, 0x91, 0xc6, 0xf8, 0xd0, 0x0d, 0x98, 0xef, 0xd3, 0xb0, 0x90, 0x3b, 0xea,
0xb5, 0x8c, 0x43, 0x23, 0x4d, 0xb0, 0xa1, 0x2d, 0x58, 0x30, 0xd9, 0x38, 0xc9, 0x6d, 0x73, 0x35,
0x05, 0x33, 0x65, 0x0c, 0x9a, 0x64, 0x44, 0xbb, 0x41, 0xd5, 0x5c, 0xcc, 0x2a, 0x77, 0x63, 0x43,
0x91, 0x5a, 0x3a, 0xef, 0x47, 0x4b, 0x67, 0x60, 0xba, 0xb6, 0xc6, 0xeb, 0x1a, 0x5d, 0x3f, 0x9f,
0x85, 0x42, 0x8f, 0x74, 0x79, 0x18, 0x95, 0xf8, 0x39, 0x6b, 0x8f, 0x74, 0x59, 0x14, 0xad, 0xd2,
0x5d, 0x84, 0x69, 0xd9, 0x2c, 0xa9, 0x17, 0x34, 0xfe, 0x42, 0x27, 0x1f, 0x7b, 0xd0, 0x89, 0xdd,
0xc1, 0xd5, 0x32, 0xfb, 0x54, 0x64, 0x94, 0x3d, 0xbb, 0xc3, 0xca, 0x4d, 0xdf, 0x3f, 0xa9, 0x2e,
0x31, 0x3a, 0x7d, 0xa4, 0x1b, 0x44, 0x0e, 0x7a, 0x2c, 0x67, 0x6d, 0x10, 0xd3, 0xd2, 0xb6, 0xc4,
0x3c, 0x1e, 0xc2, 0xc2, 0x4b, 0x9e, 0x08, 0xaa, 0x15, 0x26, 0x7f, 0x65, 0x7c, 0x7a, 0x11, 0x1a,
0xa4, 0xe0, 0xb7, 0x59, 0xfa, 0xff, 0xbd, 0x02, 0x67, 0xb6, 0xd9, 0xfe, 0x29, 0x94, 0xc7, 0xa6,
0xc1, 0x06, 0x6f, 0x07, 0xb0, 0x6d, 0x26, 0x90, 0x17, 0xef, 0xb7, 0x44, 0x6d, 0x1b, 0xb0, 0x24,
0x95, 0x0b, 0x15, 0xf9, 0x89, 0x91, 0xdf, 0xb2, 0x17, 0x7e, 0x55, 0x3f, 0x82, 0xb5, 0x44, 0x2f,
0xc4, 0x16, 0x66, 0x03, 0x16, 0x87, 0xf9, 0x2a, 0xe8, 0x44, 0x29, 0xa0, 0x35, 0x4c, 0xf5, 0x0e,
0x9c, 0x6e, 0xfb, 0x86, 0xeb, 0x27, 0x5c, 0x30, 0x81, 0x2c, 0xc3, 0x74, 0xa3, 0xb2, 0x02, 0x76,
0x6d, 0xc3, 0x6a, 0xdb, 0x27, 0xce, 0x6b, 0x28, 0xa5, 0x59, 0x87, 0xf6, 0x9f, 0x0c, 0xe4, 0xfa,
0x20, 0x5f, 0xd5, 0x35, 0x8e, 0x40, 0x27, 0x5b, 0xbb, 0x0b, 0x67, 0x38, 0x00, 0xfc, 0x3a, 0x9d,
0x38, 0x2b, 0xe1, 0xe7, 0xa4, 0xde, 0x67, 0x70, 0x6a, 0xb8, 0x2c, 0x0e, 0x31, 0x9b, 0x5b, 0x51,
0xcc, 0xe6, 0xc2, 0x88, 0x51, 0x8f, 0x40, 0x36, 0x7f, 0x9e, 0x0b, 0xe5, 0xf5, 0x0c, 0xc4, 0xe6,
0x6e, 0x14, 0xb1, 0xb9, 0x38, 0x4e, 0x77, 0x04, 0xb0, 0x49, 0x46, 0x6d, 0x3e, 0x25, 0x6a, 0x3f,
0x4f, 0xc0, 0x3a, 0xb3, 0x59, 0xb8, 0x58, 0xcc, 0xda, 0xdf, 0x08, 0xaa, 0xa3, 0x71, 0x54, 0x27,
0x68, 0x3a, 0xc0, 0xeb, 0x6f, 0xc7, 0x50, 0x9d, 0x8d, 0xb1, 0xf6, 0x06, 0xa0, 0xce, 0x5f, 0xcf,
0x42, 0x31, 0xf8, 0x96, 0xf0, 0x79, 0xd2, 0x6d, 0xb9, 0x14, 0xb7, 0x85, 0x57, 0xe0, 0xfc, 0x37,
0x5a, 0x81, 0x67, 0x27, 0x5e, 0x81, 0xcf, 0x41, 0x91, 0x3d, 0xe8, 0x2e, 0x3e, 0x12, 0x2b, 0x6a,
0x81, 0x11, 0x34, 0x7c, 0x34, 0x0c, 0xc3, 0xf9, 0xa9, 0xc2, 0x30, 0x86, 0x23, 0x2d, 0xc4, 0x71,
0xa4, 0xfb, 0xc1, 0x8a, 0xc8, 0x17, 0xd1, 0xcb, 0x23, 0xf4, 0xa6, 0xae, 0x85, 0xcd, 0xe8, 0x5a,
0xc8, 0xd7, 0xd5, 0xf7, 0x46, 0x69, 0x19, 0xb9, 0x0a, 0x7e, 0x9b, 0x2b, 0xc4, 0x01, 0x07, 0x87,
0xc2, 0xb1, 0x28, 0x32, 0xeb, 0x5d, 0x80, 0x20, 0x89, 0x48, 0x84, 0xe8, 0xdc, 0x88, 0x3e, 0x6a,
0x21, 0x76, 0xaa, 0x36, 0x32, 0x34, 0xc3, 0x33, 0xa9, 0xc9, 0xf2, 0x63, 0xc6, 0x81, 0xd4, 0xff,
0xce, 0x85, 0xf2, 0x4b, 0xc6, 0x21, 0xce, 0xfd, 0x04, 0x7e, 0x39, 0x65, 0x14, 0xdf, 0x8a, 0xc2,
0x97, 0xaf, 0x19, 0x75, 0x09, 0xf4, 0x92, 0x55, 0x2e, 0x86, 0x2b, 0x3e, 0x73, 0xd0, 0xa8, 0x28,
0x28, 0x75, 0xb6, 0x33, 0x38, 0xb2, 0x6c, 0xcb, 0x3b, 0xe6, 0xdf, 0xe7, 0xf9, 0xce, 0x40, 0x92,
0xea, 0xec, 0xbe, 0x14, 0x7e, 0x65, 0xf9, 0x7a, 0x87, 0x98, 0x98, 0xc5, 0xf4, 0x9c, 0x56, 0xa0,
0x84, 0x6d, 0x62, 0xe2, 0xe1, 0xcc, 0x2b, 0xbc, 0xde, 0xcc, 0x2b, 0xc6, 0x66, 0xde, 0x19, 0x98,
0x77, 0xb1, 0xe1, 0x11, 0x5b, 0x6c, 0xcf, 0xc5, 0x1b, 0x1d, 0x9a, 0x3e, 0xf6, 0x3c, 0xda, 0x92,
0x28, 0xd7, 0xc4, 0x6b, 0xa8, 0xcc, 0x5c, 0x1c, 0x5b, 0x66, 0x8e, 0x38, 0x1c, 0x8a, 0x95, 0x99,
0xe5, 0xb1, 0x65, 0xe6, 0x44, 0x67, 0x43, 0xc3, 0x42, 0x7b, 0x69, 0xb2, 0x42, 0x3b, 0x5c, 0x97,
0x2e, 0x47, 0xea, 0xd2, 0x6f, 0x73, 0xb2, 0xfe, 0x4a, 0x81, 0xb5, 0xc4, 0xb4, 0x12, 0xd3, 0xf5,
0x76, 0xec, 0xf4, 0x68, 0x63, 0xac, 0xcf, 0x82, 0xc3, 0xa3, 0xc7, 0x91, 0xc3, 0xa3, 0x0f, 0xc6,
0x0b, 0xbe, 0xf1, 0xb3, 0xa3, 0x3f, 0x52, 0xe0, 0xed, 0x03, 0xc7, 0x8c, 0x55, 0x78, 0x62, 0xdb,
0x3f, 0x79, 0xe2, 0xb8, 0x2f, 0x6b, 0xfd, 0xdc, 0xb4, 0x38, 0x0b, 0x97, 0x53, 0x55, 0xb8, 0x90,
0x6d, 0x86, 0x28, 0x99, 0x7e, 0x04, 0xcb, 0xbb, 0xaf, 0x70, 0xa7, 0x7d, 0x62, 0x77, 0xa6, 0x30,
0xad, 0x02, 0xf9, 0x4e, 0xdf, 0x14, 0x28, 0x29, 0x7d, 0x0c, 0x57, 0x81, 0xf9, 0x68, 0x15, 0xa8,
0x43, 0x65, 0xd8, 0x82, 0x18, 0xde, 0x33, 0x74, 0x78, 0x4d, 0xca, 0x4c, 0x95, 0x2f, 0x6a, 0xe2,
0x4d, 0xd0, 0xb1, 0xcb, 0xaf, 0x44, 0x70, 0x3a, 0x76, 0xdd, 0x68, 0xb6, 0xc8, 0x47, 0xb3, 0x85,
0xfa, 0x67, 0x0a, 0x94, 0x68, 0x0b, 0xdf, 0xc8, 0x7e, 0xb1, 0xd5, 0xca, 0x0f, 0xb7, 0x5a, 0xc1,
0x8e, 0x6d, 0x36, 0xbc, 0x63, 0x1b, 0x5a, 0x3e, 0xc7, 0xc8, 0x49, 0xcb, 0xe7, 0x03, 0x3a, 0x76,
0x5d, 0xf5, 0x02, 0x2c, 0x72, 0xdb, 0x44, 0xcf, 0x2b, 0x90, 0x1f, 0xb8, 0x3d, 0x19, 0x47, 0x03,
0xb7, 0xa7, 0xfe, 0xb1, 0x02, 0xe5, 0xba, 0xef, 0x1b, 0x9d, 0xe3, 0x29, 0x3a, 0x10, 0x18, 0x97,
0x0b, 0x1b, 0x97, 0xec, 0xc4, 0xd0, 0xdc, 0xd9, 0x0c, 0x73, 0xe7, 0x22, 0xe6, 0xaa, 0xb0, 0x24,
0x6d, 0xc9, 0x34, 0xb8, 0x09, 0xa8, 0x45, 0x5c, 0xff, 0x11, 0x71, 0x5f, 0x1a, 0xae, 0x39, 0xdd,
0x0e, 0x0c, 0xc1, 0xac, 0xb8, 0x43, 0x9b, 0xbf, 0x32, 0xa7, 0xb1, 0x67, 0xf5, 0x32, 0x9c, 0x8a,
0xe8, 0xcb, 0x6c, 0xf8, 0x01, 0x94, 0x58, 0xde, 0x17, 0xa5, 0xf8, 0xcd, 0xf0, 0x71, 0xcd, 0x44,
0xab, 0x84, 0xfa, 0xbb, 0xb0, 0x42, 0xeb, 0x03, 0x46, 0x0f, 0xa6, 0xe2, 0xf7, 0x62, 0x75, 0xea,
0xf9, 0x0c, 0x45, 0xb1, 0x1a, 0xf5, 0x6f, 0x14, 0x98, 0x63, 0xf4, 0xc4, 0x9a, 0x7d, 0x0e, 0x8a,
0x2e, 0x76, 0x88, 0xee, 0x1b, 0xdd, 0xe0, 0xc6, 0x32, 0x25, 0xec, 0x1b, 0x5d, 0x8f, 0x5d, 0xb8,
0xa6, 0x1f, 0x4d, 0xab, 0x8b, 0x3d, 0x5f, 0x5e, 0x5b, 0x2e, 0x51, 0xda, 0x0e, 0x27, 0x51, 0x27,
0x79, 0xd6, 0xef, 0xf3, 0xba, 0x73, 0x56, 0x63, 0xcf, 0x68, 0x93, 0x5f, 0xa2, 0x9b, 0x04, 0x52,
0x67, 0x57, 0xec, 0x6a, 0x50, 0x88, 0xa1, 0xe8, 0xc1, 0xbb, 0xba, 0x0b, 0x28, 0xec, 0x05, 0xe1,
0xef, 0x1b, 0x30, 0xcf, 0x9c, 0x24, 0xab, 0xa3, 0xb5, 0x0c, 0x37, 0x68, 0x82, 0x4d, 0x35, 0x00,
0x71, 0x07, 0x47, 0x2a, 0xa2, 0xe9, 0x47, 0x65, 0x44, 0x85, 0xf4, 0x77, 0x0a, 0x9c, 0x8a, 0xb4,
0x21, 0x6c, 0xbd, 0x1e, 0x6d, 0x24, 0xd3, 0x54, 0xd1, 0xc0, 0x76, 0x64, 0x49, 0xb8, 0x91, 0x65,
0xd2, 0xaf, 0x69, 0x39, 0xf8, 0x07, 0x05, 0xa0, 0x3e, 0xf0, 0x8f, 0x05, 0x32, 0x18, 0x1e, 0x19,
0x25, 0x3a, 0x32, 0xf4, 0x9b, 0x63, 0x78, 0xde, 0x4b, 0xe2, 0xca, 0x3d, 0x4d, 0xf0, 0xce, 0x30,
0xbc, 0x81, 0x7f, 0x2c, 0x8f, 0xc2, 0xe8, 0x33, 0xba, 0x08, 0x4b, 0xfc, 0x96, 0xbc, 0x6e, 0x98,
0xa6, 0x8b, 0x3d, 0x4f, 0x9c, 0x89, 0x95, 0x39, 0xb5, 0xce, 0x89, 0x94, 0xcd, 0x32, 0xb1, 0xed,
0x5b, 0xfe, 0x89, 0xee, 0x93, 0xe7, 0xd8, 0x16, 0x7b, 0x93, 0xb2, 0xa4, 0xee, 0x53, 0x22, 0x3f,
0x1c, 0xe8, 0x5a, 0x9e, 0xef, 0x4a, 0x36, 0x79, 0xfe, 0x22, 0xa8, 0x8c, 0x8d, 0x0e, 0x4a, 0xa5,
0x35, 0xe8, 0xf5, 0xb8, 0x8b, 0x5f, 0x7f, 0xd8, 0xdf, 0x17, 0x1d, 0xca, 0x65, 0xc5, 0xf4, 0xd0,
0x69, 0xa2, 0xbb, 0x6f, 0x10, 0x84, 0x79, 0x1f, 0x56, 0x42, 0x7d, 0x10, 0x61, 0x15, 0x29, 0x22,
0x95, 0x68, 0x11, 0xa9, 0x3e, 0x06, 0xc4, 0x71, 0x87, 0x6f, 0xd8, 0x6f, 0xf5, 0x34, 0x9c, 0x8a,
0x28, 0x12, 0x2b, 0xf1, 0x35, 0x28, 0x8b, 0x9b, 0x4e, 0x22, 0x50, 0xce, 0x42, 0x81, 0x66, 0xd4,
0x8e, 0x65, 0xca, 0x73, 0xd2, 0x05, 0x87, 0x98, 0xdb, 0x96, 0xe9, 0xaa, 0x9f, 0x40, 0x59, 0xe3,
0xed, 0x08, 0xde, 0x47, 0xb0, 0x24, 0xee, 0x45, 0xe9, 0x91, 0x8b, 0x89, 0x69, 0x17, 0xdf, 0xc3,
0x8d, 0x68, 0x65, 0x3b, 0xfc, 0xaa, 0x9a, 0x50, 0xe3, 0x25, 0x43, 0x44, 0xbd, 0xec, 0xec, 0x23,
0x90, 0x17, 0x01, 0xc6, 0xb6, 0x12, 0x95, 0x2f, 0xbb, 0xe1, 0x57, 0xf5, 0x3c, 0x9c, 0x4b, 0x6d,
0x45, 0x78, 0xc2, 0x81, 0xca, 0xf0, 0x83, 0x69, 0xc9, 0x03, 0x63, 0x76, 0x10, 0xac, 0x84, 0x0e,
0x82, 0xcf, 0x04, 0x45, 0x62, 0x4e, 0x2e, 0x62, 0xac, 0x02, 0x1c, 0x96, 0xfb, 0xf9, 0xac, 0x72,
0x7f, 0x36, 0x52, 0xee, 0xab, 0xed, 0xc0, 0x9f, 0x62, 0x1b, 0xf6, 0x90, 0x6d, 0x17, 0x79, 0xdb,
0x32, 0x21, 0xaa, 0xa3, 0x7a, 0xc9, 0x59, 0xb5, 0x90, 0x94, 0x7a, 0x15, 0xca, 0xd1, 0xd4, 0x18,
0xca, 0x73, 0x4a, 0x22, 0xcf, 0x2d, 0xc5, 0x52, 0xdc, 0x87, 0xb1, 0x0a, 0x38, 0xdb, 0xc7, 0xb1,
0xfa, 0xf7, 0x5e, 0x24, 0xd9, 0x5d, 0x4b, 0x39, 0xc3, 0xfd, 0x35, 0xe5, 0xb9, 0x55, 0xb1, 0x1e,
0x3c, 0xf2, 0xa8, 0xbc, 0xe8, 0xb4, 0xfa, 0x0e, 0x94, 0x0e, 0xb2, 0x7e, 0x55, 0x31, 0x2b, 0xef,
0x4b, 0xdc, 0x82, 0xd5, 0x47, 0x56, 0x0f, 0x7b, 0x27, 0x9e, 0x8f, 0xfb, 0x0d, 0x96, 0x94, 0x8e,
0x2c, 0xec, 0xa2, 0x75, 0x00, 0xb6, 0x85, 0x71, 0x88, 0x15, 0x5c, 0xb6, 0x0f, 0x51, 0xd4, 0xff,
0x54, 0x60, 0x79, 0x28, 0x78, 0xc0, 0xb6, 0x6e, 0x6f, 0x41, 0x91, 0xf6, 0xd7, 0xf3, 0x8d, 0xbe,
0x23, 0xcf, 0xb3, 0x02, 0x02, 0xba, 0x0b, 0x73, 0x47, 0x9e, 0x84, 0x8c, 0x52, 0x01, 0xf4, 0x34,
0x43, 0xb4, 0xd9, 0x23, 0xaf, 0x61, 0xa2, 0x8f, 0x00, 0x06, 0x1e, 0x36, 0xc5, 0x19, 0x56, 0x3e,
0xab, 0x5a, 0x38, 0x08, 0x9f, 0x6f, 0x53, 0x01, 0x7e, 0xd5, 0xe2, 0x1e, 0x94, 0x2c, 0x9b, 0x98,
0x98, 0x9d, 0x39, 0x9a, 0x02, 0x55, 0x1a, 0x23, 0x0e, 0x5c, 0xe2, 0xc0, 0xc3, 0xa6, 0x8a, 0xc5,
0x5a, 0x28, 0xfd, 0x2b, 0x02, 0xa5, 0x09, 0x2b, 0x3c, 0x69, 0x1d, 0x05, 0x86, 0xcb, 0x88, 0xdd,
0x18, 0xd5, 0x3b, 0xe6, 0x2d, 0xad, 0x62, 0x89, 0xd2, 0x46, 0x8a, 0xaa, 0x77, 0xe0, 0x74, 0x64,
0x87, 0x34, 0xc5, 0x96, 0x45, 0x6d, 0xc5, 0x80, 0x92, 0x61, 0x38, 0x0b, 0x18, 0x42, 0x46, 0xf3,
0x38, 0x18, 0xc2, 0xe3, 0x30, 0x84, 0xa7, 0x7e, 0x0e, 0x67, 0x23, 0x88, 0x4e, 0xc4, 0xa2, 0x7b,
0xb1, 0xca, 0xed, 0xd2, 0x38, 0xad, 0xb1, 0x12, 0xee, 0x7f, 0x14, 0x58, 0x4d, 0x63, 0x78, 0x4d,
0xc4, 0xf1, 0x47, 0x19, 0xf7, 0xef, 0x6e, 0x4f, 0x66, 0xd6, 0x6f, 0x04, 0xad, 0xdd, 0x87, 0x5a,
0x9a, 0x3f, 0x93, 0xa3, 0x94, 0x9f, 0x66, 0x94, 0x7e, 0x96, 0x0f, 0x21, 0xef, 0x75, 0xdf, 0x77,
0xad, 0xc3, 0x01, 0x0d, 0xf9, 0x37, 0x8e, 0x66, 0x35, 0x02, 0x5c, 0x86, 0xbb, 0xf6, 0xe6, 0x08,
0xf1, 0xa1, 0x1d, 0xa9, 0xd8, 0xcc, 0xa7, 0x51, 0x6c, 0x86, 0x63, 0xea, 0xb7, 0x26, 0xd3, 0xf7,
0x5b, 0x0b, 0x80, 0xfe, 0x2c, 0x07, 0x4b, 0xd1, 0x21, 0x42, 0xbb, 0x00, 0x46, 0x60, 0xb9, 0x98,
0x28, 0x17, 0x27, 0xea, 0xa6, 0x16, 0x12, 0x44, 0xef, 0x41, 0xbe, 0xe3, 0x0c, 0xc4, 0xa8, 0xa5,
0x1c, 0x06, 0x6f, 0x3b, 0x03, 0x9e, 0x51, 0x28, 0x1b, 0xdd, 0x53, 0xf1, 0xb3, 0xfd, 0xec, 0x2c,
0xf9, 0x8c, 0x7d, 0xe7, 0x32, 0x82, 0x19, 0x3d, 0x81, 0xa5, 0x97, 0xae, 0xe5, 0x1b, 0x87, 0x3d,
0xac, 0xf7, 0x8c, 0x13, 0xec, 0x8a, 0x2c, 0x39, 0x41, 0x22, 0x2b, 0x4b, 0xc1, 0xa7, 0x54, 0x4e,
0xfd, 0x43, 0x28, 0x48, 0x8b, 0xc6, 0xac, 0x08, 0xfb, 0xb0, 0x36, 0xa0, 0x6c, 0x3a, 0xbb, 0x02,
0x67, 0x1b, 0x36, 0xd1, 0x3d, 0x4c, 0x97, 0x71, 0x79, 0xdd, 0x7f, 0x4c, 0x8a, 0x5e, 0x65, 0xd2,
0xdb, 0xc4, 0xc5, 0x4d, 0xc3, 0x26, 0x6d, 0x2e, 0xaa, 0xbe, 0x80, 0x52, 0xa8, 0x83, 0x63, 0x4c,
0x68, 0xc0, 0x8a, 0x3c, 0x8a, 0xf7, 0xb0, 0x2f, 0x96, 0x97, 0x89, 0x1a, 0x5f, 0x16, 0x72, 0x6d,
0xec, 0xf3, 0xeb, 0x13, 0xf7, 0xe0, 0xac, 0x86, 0x89, 0x83, 0xed, 0x60, 0x3c, 0x9f, 0x92, 0xee,
0x14, 0x19, 0xfc, 0x2d, 0xa8, 0xa5, 0xc9, 0xf3, 0xfc, 0x70, 0xed, 0x12, 0x14, 0xe4, 0x4f, 0x64,
0xd1, 0x02, 0xe4, 0xf7, 0xb7, 0x5b, 0x95, 0x19, 0xfa, 0x70, 0xb0, 0xd3, 0xaa, 0x28, 0xa8, 0x00,
0xb3, 0xed, 0xed, 0xfd, 0x56, 0x25, 0x77, 0xad, 0x0f, 0x95, 0xf8, 0xef, 0x43, 0xd1, 0x1a, 0x9c,
0x6a, 0x69, 0x7b, 0xad, 0xfa, 0xe3, 0xfa, 0x7e, 0x63, 0xaf, 0xa9, 0xb7, 0xb4, 0xc6, 0xc7, 0xf5,
0xfd, 0xdd, 0xca, 0x0c, 0xda, 0x80, 0xf3, 0xe1, 0x0f, 0x4f, 0xf6, 0xda, 0xfb, 0xfa, 0xfe, 0x9e,
0xbe, 0xbd, 0xd7, 0xdc, 0xaf, 0x37, 0x9a, 0xbb, 0x5a, 0x45, 0x41, 0xe7, 0xe1, 0x6c, 0x98, 0xe5,
0x61, 0x63, 0xa7, 0xa1, 0xed, 0x6e, 0xd3, 0xe7, 0xfa, 0xd3, 0x4a, 0xee, 0xda, 0x4d, 0x28, 0x47,
0x7e, 0xce, 0x49, 0x4d, 0x6a, 0xed, 0xed, 0x54, 0x66, 0x50, 0x19, 0x8a, 0x61, 0x3d, 0x05, 0x98,
0x6d, 0xee, 0xed, 0xec, 0x56, 0x72, 0xd7, 0xee, 0xc0, 0x72, 0xec, 0xda, 0x2e, 0x5a, 0x81, 0x72,
0xbb, 0xde, 0xdc, 0x79, 0xb8, 0xf7, 0xa9, 0xae, 0xed, 0xd6, 0x77, 0x3e, 0xab, 0xcc, 0xa0, 0x55,
0xa8, 0x48, 0x52, 0x73, 0x6f, 0x9f, 0x53, 0x95, 0x6b, 0xcf, 0x63, 0x73, 0x0c, 0xa3, 0xd3, 0xb0,
0x12, 0x34, 0xa3, 0x6f, 0x6b, 0xbb, 0xf5, 0xfd, 0x5d, 0xda, 0x7a, 0x84, 0xac, 0x1d, 0x34, 0x9b,
0x8d, 0xe6, 0xe3, 0x8a, 0x42, 0xb5, 0x0e, 0xc9, 0xbb, 0x9f, 0x36, 0x28, 0x73, 0x2e, 0xca, 0x7c,
0xd0, 0xfc, 0x41, 0x73, 0xef, 0x93, 0x66, 0x25, 0xbf, 0xf5, 0x8b, 0x15, 0x58, 0x92, 0x85, 0x1e,
0x76, 0xd9, 0xad, 0x96, 0x16, 0x2c, 0xc8, 0x9f, 0x5c, 0xa7, 0x64, 0xe8, 0xe8, 0x0f, 0xc5, 0x6b,
0x1b, 0x23, 0x38, 0x44, 0xbd, 0x3d, 0x83, 0x0e, 0x59, 0xfd, 0x1b, 0xba, 0x46, 0x7d, 0x29, 0xb5,
0xda, 0x4c, 0xdc, 0xdc, 0xae, 0x5d, 0x1e, 0xcb, 0x17, 0xb4, 0x81, 0x69, 0x89, 0x1b, 0xfe, 0x41,
0x11, 0xba, 0x9c, 0x56, 0x9b, 0xa6, 0xfc, 0x62, 0xa9, 0x76, 0x65, 0x3c, 0x63, 0xd0, 0xcc, 0x73,
0xa8, 0xc4, 0x7f, 0x5c, 0x84, 0x52, 0xa0, 0xd3, 0x8c, 0x5f, 0x30, 0xd5, 0xae, 0x4d, 0xc2, 0x1a,
0x6e, 0x2c, 0xf1, 0x33, 0x9c, 0xab, 0x93, 0xfc, 0x5c, 0x21, 0xb3, 0xb1, 0xac, 0x5f, 0x36, 0x70,
0x07, 0x46, 0x6f, 0x3e, 0xa3, 0xd4, 0xdf, 0xbc, 0xa4, 0x5c, 0xb0, 0x4f, 0x73, 0x60, 0xfa, 0x25,
0x6a, 0x75, 0x06, 0x1d, 0xc3, 0x72, 0xec, 0x7a, 0x02, 0x4a, 0x11, 0x4f, 0xbf, 0x87, 0x51, 0xbb,
0x3a, 0x01, 0x67, 0x34, 0x22, 0xc2, 0xd7, 0x11, 0xd2, 0x23, 0x22, 0xe5, 0xb2, 0x43, 0x7a, 0x44,
0xa4, 0xde, 0x6c, 0x60, 0xc1, 0x1d, 0xb9, 0x86, 0x90, 0x16, 0xdc, 0x69, 0x97, 0x1f, 0x6a, 0x97,
0xc7, 0xf2, 0x85, 0x9d, 0x16, 0xbb, 0x94, 0x90, 0xe6, 0xb4, 0xf4, 0x4b, 0x0f, 0xb5, 0xab, 0x13,
0x70, 0xc6, 0xa3, 0x60, 0x78, 0xc4, 0x99, 0x15, 0x05, 0x89, 0x03, 0xf9, 0xac, 0x28, 0x48, 0x9e,
0x96, 0x8a, 0x28, 0x88, 0x1d, 0x4d, 0x5e, 0x99, 0xe0, 0x28, 0x25, 0x3b, 0x0a, 0xd2, 0x0f, 0x5d,
0xd4, 0x19, 0xf4, 0x53, 0x05, 0xaa, 0x59, 0xc7, 0x14, 0x28, 0xa5, 0xbe, 0x1b, 0x73, 0xb2, 0x52,
0xdb, 0x9a, 0x46, 0x24, 0xb0, 0xe2, 0x4b, 0x40, 0xc9, 0x75, 0x0f, 0x7d, 0x27, 0x6d, 0x64, 0x32,
0x56, 0xd7, 0xda, 0x7b, 0x93, 0x31, 0x07, 0x4d, 0xb6, 0xa1, 0x20, 0x0f, 0x46, 0x50, 0x4a, 0x96,
0x8e, 0x1d, 0xcb, 0xd4, 0xd4, 0x51, 0x2c, 0x81, 0xd2, 0xc7, 0x30, 0x4b, 0xa9, 0xe8, 0x7c, 0x3a,
0xb7, 0x54, 0xb6, 0x9e, 0xf5, 0x39, 0x50, 0xf4, 0x0c, 0xe6, 0xf9, 0x49, 0x00, 0x4a, 0x41, 0x1e,
0x22, 0xe7, 0x15, 0xb5, 0x0b, 0xd9, 0x0c, 0x81, 0xba, 0x2f, 0xf8, 0x7f, 0xe3, 0x10, 0x20, 0x3f,
0x7a, 0x37, 0xfd, 0xe7, 0xcd, 0xd1, 0x33, 0x85, 0xda, 0xc5, 0x31, 0x5c, 0xe1, 0x49, 0x11, 0xab,
0x7a, 0x2f, 0x8f, 0xdd, 0xba, 0x64, 0x4f, 0x8a, 0xf4, 0xcd, 0x11, 0x0f, 0x92, 0xe4, 0xe6, 0x29,
0x2d, 0x48, 0x32, 0xb7, 0xac, 0x69, 0x41, 0x92, 0xbd, 0x1f, 0x53, 0x67, 0x90, 0x0f, 0xa7, 0x52,
0xa0, 0x32, 0xf4, 0x5e, 0x56, 0x90, 0xa7, 0xe1, 0x76, 0xb5, 0xeb, 0x13, 0x72, 0x87, 0x07, 0x5f,
0x4c, 0xfa, 0xb7, 0xb3, 0xf1, 0xa3, 0xcc, 0xc1, 0x8f, 0x4f, 0xf1, 0xad, 0x7f, 0xc9, 0xc3, 0x22,
0x87, 0x41, 0x45, 0x05, 0xf3, 0x19, 0xc0, 0xf0, 0x04, 0x02, 0xbd, 0x93, 0xee, 0x93, 0xc8, 0x29,
0x4d, 0xed, 0xdd, 0xd1, 0x4c, 0xe1, 0x40, 0x0b, 0xa1, 0xf9, 0x69, 0x81, 0x96, 0x3c, 0xb4, 0x48,
0x0b, 0xb4, 0x94, 0x23, 0x01, 0x75, 0x06, 0x7d, 0x0c, 0xc5, 0x00, 0x36, 0x46, 0x69, 0xb0, 0x73,
0x0c, 0x17, 0xaf, 0xbd, 0x33, 0x92, 0x27, 0x6c, 0x75, 0x08, 0x13, 0x4e, 0xb3, 0x3a, 0x89, 0x3d,
0xa7, 0x59, 0x9d, 0x06, 0x2c, 0x0f, 0x7d, 0xc2, 0x91, 0xa3, 0x4c, 0x9f, 0x44, 0x80, 0xbb, 0x4c,
0x9f, 0x44, 0xe1, 0x27, 0x75, 0xe6, 0xe1, 0xa5, 0x5f, 0x7e, 0xb5, 0xae, 0xfc, 0xd3, 0x57, 0xeb,
0x33, 0x3f, 0xf9, 0x7a, 0x5d, 0xf9, 0xe5, 0xd7, 0xeb, 0xca, 0x3f, 0x7e, 0xbd, 0xae, 0xfc, 0xeb,
0xd7, 0xeb, 0xca, 0x9f, 0xfe, 0xdb, 0xfa, 0xcc, 0x0f, 0x0b, 0x52, 0xfa, 0x70, 0x9e, 0xfd, 0x4f,
0x9d, 0x0f, 0xfe, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x11, 0x3f, 0x6e, 0x39, 0x19, 0x49, 0x00, 0x00,
}

27
vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto generated vendored
View file

@ -1,3 +1,19 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// To regenerate api.pb.go run hack/update-generated-runtime.sh
syntax = 'proto3';
@ -297,7 +313,8 @@ message PodSandboxConfig {
// operation. The runtime may also use this information to improve UX, such
// as by constructing a readable name.
PodSandboxMetadata metadata = 1;
// Hostname of the sandbox.
// Hostname of the sandbox. Hostname could only be empty when the pod
// network namespace is NODE.
string hostname = 2;
// Path to the directory on the host in which container log files are
// stored.
@ -349,7 +366,7 @@ message RunPodSandboxRequest {
// If the runtime handler is unknown, this request should be rejected. An
// empty string should select the default handler, equivalent to the
// behavior before this feature was added.
// See https://git.k8s.io/community/keps/sig-node/0014-runtime-class.md
// See https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md
string runtime_handler = 2;
}
@ -423,6 +440,8 @@ message PodSandboxStatus {
// MUST be identical to that of the corresponding PodSandboxConfig used to
// instantiate the pod sandbox this status represents.
map<string, string> annotations = 8;
// runtime configuration used for this PodSandbox.
string runtime_handler = 9;
}
message PodSandboxStatusResponse {
@ -477,6 +496,8 @@ message PodSandbox {
// MUST be identical to that of the corresponding PodSandboxConfig used to
// instantiate this PodSandbox.
map<string, string> annotations = 6;
// runtime configuration used for this PodSandbox.
string runtime_handler = 7;
}
message ListPodSandboxResponse {
@ -1221,7 +1242,7 @@ message ContainerStats {
CpuUsage cpu = 2;
// Memory usage gathered from the container.
MemoryUsage memory = 3;
// Usage of the writeable layer.
// Usage of the writable layer.
FilesystemUsage writable_layer = 4;
}

2
vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/services.go generated vendored
View file

@ -111,7 +111,7 @@ type ImageManagerService interface {
// ImageStatus returns the status of the image.
ImageStatus(image *runtimeapi.ImageSpec) (*runtimeapi.Image, error)
// PullImage pulls an image with the authentication config.
PullImage(image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig) (string, error)
PullImage(image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error)
// RemoveImage removes the image.
RemoveImage(image *runtimeapi.ImageSpec) error
// ImageFsInfo returns information of the filesystem that is used to store images.

42
vendor/k8s.io/kubernetes/pkg/kubelet/apis/well_known_labels.go generated vendored
View file

@ -19,17 +19,14 @@ package apis
import (
"strings"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
)
const (
LabelHostname = "kubernetes.io/hostname"
LabelZoneFailureDomain = "failure-domain.beta.kubernetes.io/zone"
LabelMultiZoneDelimiter = "__"
LabelZoneRegion = "failure-domain.beta.kubernetes.io/region"
LabelInstanceType = "beta.kubernetes.io/instance-type"
// The OS/Arch labels are promoted to GA in 1.14. kubelet applies both beta
// and GA labels to ensure backward compatibility.
// TODO: stop applying the beta OS/Arch labels in Kubernetes 1.18.
LabelOS = "beta.kubernetes.io/os"
LabelArch = "beta.kubernetes.io/arch"
@ -38,40 +35,27 @@ const (
labelZoneFailureDomainGA = "failure-domain.kubernetes.io/zone"
labelZoneRegionGA = "failure-domain.kubernetes.io/region"
labelInstanceTypeGA = "kubernetes.io/instance-type"
labelOSGA = "kubernetes.io/os"
labelArchGA = "kubernetes.io/arch"
// LabelNamespaceSuffixKubelet is an allowed label namespace suffix kubelets can self-set ([*.]kubelet.kubernetes.io/*)
LabelNamespaceSuffixKubelet = "kubelet.kubernetes.io"
// LabelNamespaceSuffixNode is an allowed label namespace suffix kubelets can self-set ([*.]node.kubernetes.io/*)
LabelNamespaceSuffixNode = "node.kubernetes.io"
// LabelNamespaceNodeRestriction is a forbidden label namespace that kubelets may not self-set when the NodeRestriction admission plugin is enabled
LabelNamespaceNodeRestriction = "node-restriction.kubernetes.io"
)
// When the --failure-domains scheduler flag is not specified,
// DefaultFailureDomains defines the set of label keys used when TopologyKey is empty in PreferredDuringScheduling anti-affinity.
var DefaultFailureDomains string = LabelHostname + "," + LabelZoneFailureDomain + "," + LabelZoneRegion
var kubeletLabels = sets.NewString(
LabelHostname,
LabelZoneFailureDomain,
LabelZoneRegion,
LabelInstanceType,
v1.LabelHostname,
v1.LabelZoneFailureDomain,
v1.LabelZoneRegion,
v1.LabelInstanceType,
v1.LabelOSStable,
v1.LabelArchStable,
LabelOS,
LabelArch,
labelZoneFailureDomainGA,
labelZoneRegionGA,
labelInstanceTypeGA,
labelOSGA,
labelArchGA,
)
var kubeletLabelNamespaces = sets.NewString(
LabelNamespaceSuffixKubelet,
LabelNamespaceSuffixNode,
v1.LabelNamespaceSuffixKubelet,
v1.LabelNamespaceSuffixNode,
)
// KubeletLabels returns the list of label keys kubelets are allowed to set on their own Node objects

1
vendor/k8s.io/kubernetes/pkg/kubelet/container/BUILD generated vendored
View file

@ -29,6 +29,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/tools/reference:go_default_library",
"//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library",

20
vendor/k8s.io/kubernetes/pkg/kubelet/container/helpers.go generated vendored
View file

@ -27,6 +27,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/record"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/kubelet/util/format"
@ -130,9 +131,22 @@ func ExpandContainerCommandOnlyStatic(containerCommand []string, envs []v1.EnvVa
return command
}
func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (expandedSubpath string) {
mapping := expansion.MappingFuncFor(EnvVarsToMap(envs))
return expansion.Expand(mount.SubPath, mapping)
func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (string, error) {
envmap := EnvVarsToMap(envs)
missingKeys := sets.NewString()
expanded := expansion.Expand(mount.SubPathExpr, func(key string) string {
value, ok := envmap[key]
if !ok || len(value) == 0 {
missingKeys.Insert(key)
}
return value
})
if len(missingKeys) > 0 {
return "", fmt.Errorf("missing value for %s", strings.Join(missingKeys.List(), ", "))
}
return expanded, nil
}
func ExpandContainerCommandAndArgs(container *v1.Container, envs []EnvVar) (command []string, args []string) {

26
vendor/k8s.io/kubernetes/pkg/kubelet/container/runtime.go generated vendored
View file

@ -88,7 +88,7 @@ type Runtime interface {
// TODO: Revisit this method and make it cleaner.
GarbageCollect(gcPolicy ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error
// Syncs the running pod into the desired pod.
SyncPod(pod *v1.Pod, apiPodStatus v1.PodStatus, podStatus *PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) PodSyncResult
SyncPod(pod *v1.Pod, podStatus *PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) PodSyncResult
// KillPod kills all the containers of a pod. Pod may be nil, running pod must not be.
// TODO(random-liu): Return PodSyncResult in KillPod.
// gracePeriodOverride if specified allows the caller to override the pod default grace period.
@ -98,17 +98,6 @@ type Runtime interface {
// GetPodStatus retrieves the status of the pod, including the
// information of all containers in the pod that are visible in Runtime.
GetPodStatus(uid types.UID, name, namespace string) (*PodStatus, error)
// Returns the filesystem path of the pod's network namespace; if the
// runtime does not handle namespace creation itself, or cannot return
// the network namespace path, it should return an error.
// TODO: Change ContainerID to a Pod ID since the namespace is shared
// by all containers in the pod.
GetNetNS(containerID ContainerID) (string, error)
// Returns the container ID that represents the Pod, as passed to network
// plugins. For example, if the runtime uses an infra container, returns
// the infra container's ContainerID.
// TODO: Change ContainerID to a Pod ID, see GetNetNS()
GetPodContainerID(*Pod) (ContainerID, error)
// TODO(vmarmol): Unify pod and containerID args.
// GetContainerLogs returns logs of a specific container. By
// default, it returns a snapshot of the container log. Set 'follow' to true to
@ -137,7 +126,7 @@ type StreamingRuntime interface {
type ImageService interface {
// PullImage pulls an image from the network to local storage using the supplied
// secrets if necessary. It returns a reference (digest or ID) to the pulled image.
PullImage(image ImageSpec, pullSecrets []v1.Secret) (string, error)
PullImage(image ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error)
// GetImageRef gets the reference (digest or ID) of the image which has already been in
// the local storage. It returns ("", nil) if the image isn't in the local storage.
GetImageRef(image ImageSpec) (string, error)
@ -155,7 +144,7 @@ type ContainerAttacher interface {
type ContainerCommandRunner interface {
// RunInContainer synchronously executes the command in the container, and returns the output.
// If the command completes with a non-0 exit code, a pkg/util/exec.ExitError will be returned.
// If the command completes with a non-0 exit code, a k8s.io/utils/exec.ExitError will be returned.
RunInContainer(id ContainerID, cmd []string, timeout time.Duration) ([]byte, error)
}
@ -254,13 +243,6 @@ const (
ContainerStateUnknown ContainerState = "unknown"
)
type ContainerType string
const (
ContainerTypeInit ContainerType = "INIT"
ContainerTypeRegular ContainerType = "REGULAR"
)
// Container provides the runtime information for a container, such as ID, hash,
// state of the container.
type Container struct {
@ -289,7 +271,7 @@ type PodStatus struct {
ID types.UID
// Name of the pod.
Name string
// Namspace of the pod.
// Namespace of the pod.
Namespace string
// IP of the pod.
IP string

6
vendor/k8s.io/kubernetes/pkg/kubelet/errors.go generated vendored
View file

@ -16,6 +16,12 @@ limitations under the License.
package kubelet
import "errors"
const (
NetworkNotReadyErrorMsg = "network is not ready"
)
var (
ErrNetworkUnknown = errors.New("network state unknown")
)

94
vendor/k8s.io/kubernetes/pkg/kubelet/kubelet.go generated vendored
View file

@ -43,7 +43,6 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
corelisters "k8s.io/client-go/listers/core/v1"
@ -51,9 +50,7 @@ import (
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/certificate"
"k8s.io/client-go/util/flowcontrol"
"k8s.io/client-go/util/integer"
cloudprovider "k8s.io/cloud-provider"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/klog"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/features"
@ -115,7 +112,9 @@ import (
"k8s.io/kubernetes/pkg/util/oom"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/csi"
"k8s.io/kubernetes/pkg/volume/util/subpath"
utilexec "k8s.io/utils/exec"
"k8s.io/utils/integer"
)
const (
@ -248,13 +247,12 @@ type Dependencies struct {
HeartbeatClient clientset.Interface
OnHeartbeatFailure func()
KubeClient clientset.Interface
CSIClient csiclientset.Interface
DynamicKubeClient dynamic.Interface
Mounter mount.Interface
OOMAdjuster *oom.OOMAdjuster
OSInterface kubecontainer.OSInterface
PodConfig *config.PodConfig
Recorder record.EventRecorder
Subpather subpath.Interface
VolumePlugins []volume.VolumePlugin
DynamicPluginProber volume.DynamicPluginProber
TLSOptions *server.TLSOptions
@ -491,7 +489,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
hostnameOverridden: len(hostnameOverride) > 0,
nodeName: nodeName,
kubeClient: kubeDeps.KubeClient,
csiClient: kubeDeps.CSIClient,
heartbeatClient: kubeDeps.HeartbeatClient,
onRepeatedHeartbeatFailure: kubeDeps.OnHeartbeatFailure,
rootDirectory: rootDirectory,
@ -519,6 +516,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
cgroupsPerQOS: kubeCfg.CgroupsPerQOS,
cgroupRoot: kubeCfg.CgroupRoot,
mounter: kubeDeps.Mounter,
subpather: kubeDeps.Subpather,
maxPods: int(kubeCfg.MaxPods),
podsPerCore: int(kubeCfg.PodsPerCore),
syncLoopMonitor: atomic.Value{},
@ -590,6 +588,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
// podManager is also responsible for keeping secretManager and configMapManager contents up-to-date.
klet.podManager = kubepod.NewBasicPodManager(kubepod.NewBasicMirrorClient(klet.kubeClient), secretManager, configMapManager, checkpointManager)
klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
if remoteRuntimeEndpoint != "" {
// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
if remoteImageEndpoint == "" {
@ -609,10 +609,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.resourceAnalyzer = serverstats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration)
if containerRuntime == "rkt" {
klog.Fatalln("rktnetes has been deprecated in favor of rktlet. Please see https://github.com/kubernetes-incubator/rktlet for more information.")
}
// if left at nil, that means it is unneeded
var legacyLogProvider kuberuntime.LegacyLogProvider
@ -660,8 +656,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
klet.runtimeService = runtimeService
if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.DynamicKubeClient != nil {
klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.DynamicKubeClient)
if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) {
klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient)
}
runtime, err := kuberuntime.NewKubeGenericRuntimeManager(
@ -705,7 +701,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.resourceAnalyzer,
klet.podManager,
klet.runtimeCache,
klet.containerRuntime)
klet.containerRuntime,
klet.statusManager)
} else {
klet.StatsProvider = stats.NewCRIStatsProvider(
klet.cadvisor,
@ -714,7 +711,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.runtimeCache,
runtimeService,
imageService,
stats.NewLogMetricsService())
stats.NewLogMetricsService(),
kubecontainer.RealOS{})
}
klet.pleg = pleg.NewGenericPLEG(klet.containerRuntime, plegChannelCapacity, plegRelistPeriod, klet.podCache, clock.RealClock{})
@ -754,8 +752,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.containerLogManager = logs.NewStubContainerLogManager()
}
klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
if kubeCfg.ServerTLSBootstrap && kubeDeps.TLSOptions != nil && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory)
if err != nil {
@ -779,10 +775,9 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
tokenManager := token.NewManager(kubeDeps.KubeClient)
if !utilfeature.DefaultFeatureGate.Enabled(features.MountPropagation) {
return nil, fmt.Errorf("mount propagation feature gate has been deprecated and will be removed in 1.14")
}
// NewInitializedVolumePluginMgr intializes some storageErrors on the Kubelet runtimeState (in csi_plugin.go init)
// which affects node ready status. This function must be called before Kubelet is initialized so that the Node
// ReadyState is accurate with the storage state.
klet.volumePluginMgr, err =
NewInitializedVolumePluginMgr(klet, secretManager, configMapManager, tokenManager, kubeDeps.VolumePlugins, kubeDeps.DynamicPluginProber)
if err != nil {
@ -827,7 +822,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.podKillingCh = make(chan *kubecontainer.PodPair, podKillingChannelCapacity)
// setup eviction manager
evictionManager, evictionAdmitHandler := eviction.NewManager(klet.resourceAnalyzer, evictionConfig, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.imageManager, klet.containerGC, kubeDeps.Recorder, nodeRef, klet.clock)
evictionManager, evictionAdmitHandler := eviction.NewManager(klet.resourceAnalyzer, evictionConfig, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.podManager.GetMirrorPodByPod, klet.imageManager, klet.containerGC, kubeDeps.Recorder, nodeRef, klet.clock)
klet.evictionManager = evictionManager
klet.admitHandlers.AddPodAdmitHandler(evictionAdmitHandler)
@ -902,7 +897,6 @@ type Kubelet struct {
nodeName types.NodeName
runtimeCache kubecontainer.RuntimeCache
kubeClient clientset.Interface
csiClient csiclientset.Interface
heartbeatClient clientset.Interface
iptClient utilipt.Interface
rootDirectory string
@ -1064,15 +1058,15 @@ type Kubelet struct {
lastStatusReportTime time.Time
// syncNodeStatusMux is a lock on updating the node status, because this path is not thread-safe.
// This lock is used by Kublet.syncNodeStatus function and shouldn't be used anywhere else.
// This lock is used by Kubelet.syncNodeStatus function and shouldn't be used anywhere else.
syncNodeStatusMux sync.Mutex
// updatePodCIDRMux is a lock on updating pod CIDR, because this path is not thread-safe.
// This lock is used by Kublet.syncNodeStatus function and shouldn't be used anywhere else.
// This lock is used by Kubelet.syncNodeStatus function and shouldn't be used anywhere else.
updatePodCIDRMux sync.Mutex
// updateRuntimeMux is a lock on updating runtime, because this path is not thread-safe.
// This lock is used by Kublet.updateRuntimeUp function and shouldn't be used anywhere else.
// This lock is used by Kubelet.updateRuntimeUp function and shouldn't be used anywhere else.
updateRuntimeMux sync.Mutex
// nodeLeaseController claims and renews the node lease for this Kubelet
@ -1102,6 +1096,9 @@ type Kubelet struct {
// Mounter to use for volumes.
mounter mount.Interface
// subpather to execute subpath actions
subpather subpath.Interface
// Manager of non-Runtime containers.
containerManager cm.ContainerManager
@ -1221,29 +1218,6 @@ type Kubelet struct {
runtimeClassManager *runtimeclass.Manager
}
func allGlobalUnicastIPs() ([]net.IP, error) {
interfaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("could not list network interfaces: %v", err)
}
var ips []net.IP
for _, i := range interfaces {
addresses, err := i.Addrs()
if err != nil {
return nil, fmt.Errorf("could not list the addresses for network interface %v: %v", i, err)
}
for _, address := range addresses {
switch v := address.(type) {
case *net.IPNet:
if v.IP.IsGlobalUnicast() {
ips = append(ips, v.IP)
}
}
}
}
return ips, nil
}
// setupDataDirs creates:
// 1. the root directory
// 2. the pods directory
@ -1329,6 +1303,7 @@ func (kl *Kubelet) initializeModules() error {
collectors.NewVolumeStatsCollector(kl),
collectors.NewLogMetricsCollector(kl.StatsProvider.ListPodStats),
)
metrics.SetNodeName(kl.nodeName)
// Setup filesystem directories.
if err := kl.setupDataDirs(); err != nil {
@ -1452,7 +1427,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
// Start syncing RuntimeClasses if enabled.
if kl.runtimeClassManager != nil {
go kl.runtimeClassManager.Run(wait.NeverStop)
kl.runtimeClassManager.Start(wait.NeverStop)
}
// Start the pod lifecycle event generator.
@ -1525,7 +1500,8 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
if !firstSeenTime.IsZero() {
// This is the first time we are syncing the pod. Record the latency
// since kubelet first saw the pod if firstSeenTime is set.
metrics.PodWorkerStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime))
metrics.PodWorkerStartDuration.Observe(metrics.SinceInSeconds(firstSeenTime))
metrics.DeprecatedPodWorkerStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime))
} else {
klog.V(3).Infof("First seen time not recorded for pod %q", pod.UID)
}
@ -1542,7 +1518,8 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
existingStatus, ok := kl.statusManager.GetPodStatus(pod.UID)
if !ok || existingStatus.Phase == v1.PodPending && apiPodStatus.Phase == v1.PodRunning &&
!firstSeenTime.IsZero() {
metrics.PodStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime))
metrics.PodStartDuration.Observe(metrics.SinceInSeconds(firstSeenTime))
metrics.DeprecatedPodStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime))
}
runnable := kl.canRunPod(pod)
@ -1585,9 +1562,9 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
}
// If the network plugin is not ready, only start the pod if it uses the host network
if rs := kl.runtimeState.networkErrors(); len(rs) != 0 && !kubecontainer.IsHostNetworkPod(pod) {
kl.recorder.Eventf(pod, v1.EventTypeWarning, events.NetworkNotReady, "%s: %v", NetworkNotReadyErrorMsg, rs)
return fmt.Errorf("%s: %v", NetworkNotReadyErrorMsg, rs)
if err := kl.runtimeState.networkErrors(); err != nil && !kubecontainer.IsHostNetworkPod(pod) {
kl.recorder.Eventf(pod, v1.EventTypeWarning, events.NetworkNotReady, "%s: %v", NetworkNotReadyErrorMsg, err)
return fmt.Errorf("%s: %v", NetworkNotReadyErrorMsg, err)
}
// Create Cgroups for the pod and apply resource parameters
@ -1686,7 +1663,7 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
pullSecrets := kl.getPullSecretsForPod(pod)
// Call the container runtime's SyncPod callback
result := kl.containerRuntime.SyncPod(pod, apiPodStatus, podStatus, pullSecrets, kl.backOff)
result := kl.containerRuntime.SyncPod(pod, podStatus, pullSecrets, kl.backOff)
kl.reasonCache.Update(pod.UID, result)
if err := result.Error(); err != nil {
// Do not return error if the only failures were pods in backoff
@ -1842,8 +1819,8 @@ func (kl *Kubelet) syncLoop(updates <-chan kubetypes.PodUpdate, handler SyncHand
)
duration := base
for {
if rs := kl.runtimeState.runtimeErrors(); len(rs) != 0 {
klog.Infof("skipping pod synchronization - %v", rs)
if err := kl.runtimeState.runtimeErrors(); err != nil {
klog.Infof("skipping pod synchronization - %v", err)
// exponential backoff
time.Sleep(duration)
duration = time.Duration(math.Min(float64(max), factor*float64(duration)))
@ -2021,7 +1998,8 @@ func (kl *Kubelet) dispatchWork(pod *v1.Pod, syncType kubetypes.SyncPodType, mir
UpdateType: syncType,
OnCompleteFunc: func(err error) {
if err != nil {
metrics.PodWorkerLatency.WithLabelValues(syncType.String()).Observe(metrics.SinceInMicroseconds(start))
metrics.PodWorkerDuration.WithLabelValues(syncType.String()).Observe(metrics.SinceInSeconds(start))
metrics.DeprecatedPodWorkerLatency.WithLabelValues(syncType.String()).Observe(metrics.SinceInMicroseconds(start))
}
},
})

12
vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_getters.go generated vendored
View file

@ -25,14 +25,14 @@ import (
cadvisorapiv1 "github.com/google/cadvisor/info/v1"
"k8s.io/klog"
v1 "k8s.io/api/core/v1"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
utilfile "k8s.io/kubernetes/pkg/util/file"
"k8s.io/kubernetes/pkg/util/mount"
utilnode "k8s.io/kubernetes/pkg/util/node"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
utilpath "k8s.io/utils/path"
)
// getRootDir returns the full path to the directory under which kubelet can
@ -278,7 +278,7 @@ func (kl *Kubelet) getPodVolumePathListFromDisk(podUID types.UID) ([]string, err
volumes := []string{}
podVolDir := kl.getPodVolumesDir(podUID)
if pathExists, pathErr := volumeutil.PathExists(podVolDir); pathErr != nil {
if pathExists, pathErr := mount.PathExists(podVolDir); pathErr != nil {
return volumes, fmt.Errorf("Error checking if path %q exists: %v", podVolDir, pathErr)
} else if !pathExists {
klog.Warningf("Path %q does not exist", podVolDir)
@ -293,7 +293,7 @@ func (kl *Kubelet) getPodVolumePathListFromDisk(podUID types.UID) ([]string, err
for _, volumePluginDir := range volumePluginDirs {
volumePluginName := volumePluginDir.Name()
volumePluginPath := filepath.Join(podVolDir, volumePluginName)
volumeDirs, err := utilfile.ReadDirNoStat(volumePluginPath)
volumeDirs, err := utilpath.ReadDirNoStat(volumePluginPath)
if err != nil {
return volumes, fmt.Errorf("Could not read directory %s: %v", volumePluginPath, err)
}
@ -327,7 +327,7 @@ func (kl *Kubelet) getMountedVolumePathListFromDisk(podUID types.UID) ([]string,
func (kl *Kubelet) podVolumeSubpathsDirExists(podUID types.UID) (bool, error) {
podVolDir := kl.getPodVolumeSubpathsDir(podUID)
if pathExists, pathErr := volumeutil.PathExists(podVolDir); pathErr != nil {
if pathExists, pathErr := mount.PathExists(podVolDir); pathErr != nil {
return true, fmt.Errorf("Error checking if path %q exists: %v", podVolDir, pathErr)
} else if !pathExists {
return false, nil

51
vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_node_status.go generated vendored
View file

@ -134,6 +134,7 @@ func (kl *Kubelet) reconcileExtendedResource(initialNode, node *v1.Node) bool {
requiresUpdate := false
for k := range node.Status.Capacity {
if v1helper.IsExtendedResourceName(k) {
klog.Infof("Zero out resource %s capacity in existing node.", k)
node.Status.Capacity[k] = *resource.NewQuantity(int64(0), resource.DecimalSI)
node.Status.Allocatable[k] = *resource.NewQuantity(int64(0), resource.DecimalSI)
requiresUpdate = true
@ -145,10 +146,12 @@ func (kl *Kubelet) reconcileExtendedResource(initialNode, node *v1.Node) bool {
// updateDefaultLabels will set the default labels on the node
func (kl *Kubelet) updateDefaultLabels(initialNode, existingNode *v1.Node) bool {
defaultLabels := []string{
kubeletapis.LabelHostname,
kubeletapis.LabelZoneFailureDomain,
kubeletapis.LabelZoneRegion,
kubeletapis.LabelInstanceType,
v1.LabelHostname,
v1.LabelZoneFailureDomain,
v1.LabelZoneRegion,
v1.LabelInstanceType,
v1.LabelOSStable,
v1.LabelArchStable,
kubeletapis.LabelOS,
kubeletapis.LabelArch,
}
@ -213,9 +216,11 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
ObjectMeta: metav1.ObjectMeta{
Name: string(kl.nodeName),
Labels: map[string]string{
kubeletapis.LabelHostname: kl.hostname,
kubeletapis.LabelOS: goruntime.GOOS,
kubeletapis.LabelArch: goruntime.GOARCH,
v1.LabelHostname: kl.hostname,
v1.LabelOSStable: goruntime.GOOS,
v1.LabelArchStable: goruntime.GOARCH,
kubeletapis.LabelOS: goruntime.GOOS,
kubeletapis.LabelArch: goruntime.GOARCH,
},
},
Spec: v1.NodeSpec{
@ -323,8 +328,8 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
return nil, err
}
if instanceType != "" {
klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelInstanceType, instanceType)
node.ObjectMeta.Labels[kubeletapis.LabelInstanceType] = instanceType
klog.Infof("Adding node label from cloud provider: %s=%s", v1.LabelInstanceType, instanceType)
node.ObjectMeta.Labels[v1.LabelInstanceType] = instanceType
}
// If the cloud has zone information, label the node with the zone information
zones, ok := kl.cloud.Zones()
@ -334,12 +339,12 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
return nil, fmt.Errorf("failed to get zone from cloud provider: %v", err)
}
if zone.FailureDomain != "" {
klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneFailureDomain, zone.FailureDomain)
node.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain] = zone.FailureDomain
klog.Infof("Adding node label from cloud provider: %s=%s", v1.LabelZoneFailureDomain, zone.FailureDomain)
node.ObjectMeta.Labels[v1.LabelZoneFailureDomain] = zone.FailureDomain
}
if zone.Region != "" {
klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneRegion, zone.Region)
node.ObjectMeta.Labels[kubeletapis.LabelZoneRegion] = zone.Region
klog.Infof("Adding node label from cloud provider: %s=%s", v1.LabelZoneRegion, zone.Region)
node.ObjectMeta.Labels[v1.LabelZoneRegion] = zone.Region
}
}
}
@ -423,6 +428,23 @@ func (kl *Kubelet) tryUpdateNodeStatus(tryNumber int) error {
now := kl.clock.Now()
if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) && now.Before(kl.lastStatusReportTime.Add(kl.nodeStatusReportFrequency)) {
if !podCIDRChanged && !nodeStatusHasChanged(&originalNode.Status, &node.Status) {
// We must mark the volumes as ReportedInUse in volume manager's dsw even
// if no changes were made to the node status (no volumes were added or removed
// from the VolumesInUse list).
//
// The reason is that on a kubelet restart, the volume manager's dsw is
// repopulated and the volume ReportedInUse is initialized to false, while the
// VolumesInUse list from the Node object still contains the state from the
// previous kubelet instantiation.
//
// Once the volumes are added to the dsw, the ReportedInUse field needs to be
// synced from the VolumesInUse list in the Node.Status.
//
// The MarkVolumesAsReportedInUse() call cannot be performed in dsw directly
// because it does not have access to the Node object.
// This also cannot be populated on node status manager init because the volume
// may not have been added to dsw at that time.
kl.volumeManager.MarkVolumesAsReportedInUse(node.Status.VolumesInUse)
return nil
}
}
@ -522,8 +544,9 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error {
nodestatus.MemoryPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderMemoryPressure, kl.recordNodeStatusEvent),
nodestatus.DiskPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderDiskPressure, kl.recordNodeStatusEvent),
nodestatus.PIDPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderPIDPressure, kl.recordNodeStatusEvent),
nodestatus.ReadyCondition(kl.clock.Now, kl.runtimeState.runtimeErrors, kl.runtimeState.networkErrors, validateHostFunc, kl.containerManager.Status, kl.recordNodeStatusEvent),
nodestatus.ReadyCondition(kl.clock.Now, kl.runtimeState.runtimeErrors, kl.runtimeState.networkErrors, kl.runtimeState.storageErrors, validateHostFunc, kl.containerManager.Status, kl.recordNodeStatusEvent),
nodestatus.VolumesInUse(kl.volumeManager.ReconcilerStatesHasBeenSynced, kl.volumeManager.GetVolumesInUse),
nodestatus.RemoveOutOfDiskCondition(),
// TODO(mtaufen): I decided not to move this setter for now, since all it does is send an event
// and record state back to the Kubelet runtime object. In the future, I'd like to isolate
// these side-effects by decoupling the decisions to send events and partial status recording

49
vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods.go generated vendored
View file

@ -61,6 +61,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format"
mountutil "k8s.io/kubernetes/pkg/util/mount"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
"k8s.io/kubernetes/third_party/forked/golang/expansion"
@ -127,7 +128,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol
}
// makeMounts determines the mount points for the given container.
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, subpather subpath.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
// Kubernetes only mounts on /etc/hosts if:
// - container is not an infrastructure (pause) container
// - container is not already mounting on /etc/hosts
@ -159,27 +160,40 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
if err != nil {
return nil, cleanupAction, err
}
if mount.SubPath != "" {
subPath := mount.SubPath
if mount.SubPathExpr != "" {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
}
// Expand subpath variables
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
mount.SubPath = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
return nil, cleanupAction, fmt.Errorf("volume subpath expansion is disabled")
}
if filepath.IsAbs(mount.SubPath) {
return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
}
subPath, err = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
err = volumevalidation.ValidatePathNoBacksteps(mount.SubPath)
if err != nil {
return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
return nil, cleanupAction, err
}
}
if subPath != "" {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
}
if filepath.IsAbs(subPath) {
return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", subPath)
}
err = volumevalidation.ValidatePathNoBacksteps(subPath)
if err != nil {
return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", subPath, err)
}
volumePath := hostPath
hostPath = filepath.Join(volumePath, mount.SubPath)
hostPath = filepath.Join(volumePath, subPath)
if subPathExists, err := mounter.ExistsPath(hostPath); err != nil {
klog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath)
@ -193,13 +207,13 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
if err != nil {
return nil, cleanupAction, err
}
if err := mounter.SafeMakeDir(mount.SubPath, volumePath, perm); err != nil {
if err := subpather.SafeMakeDir(subPath, volumePath, perm); err != nil {
// Don't pass detailed error back to the user because it could give information about host filesystem
klog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name)
}
}
hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{
hostPath, cleanupAction, err = subpather.PrepareSafeSubpath(subpath.Subpath{
VolumeMountIndex: i,
Path: hostPath,
VolumeName: vol.InnerVolumeSpecName,
@ -384,7 +398,6 @@ func truncatePodHostnameIfNeeded(podName, hostname string) (string, error) {
// GeneratePodHostNameAndDomain creates a hostname and domain name for a pod,
// given that pod's spec and annotations or returns an error.
func (kl *Kubelet) GeneratePodHostNameAndDomain(pod *v1.Pod) (string, string, error) {
// TODO(vmarmol): Handle better.
clusterDomain := kl.dnsConfigurer.ClusterDomain
hostname := pod.Name
@ -452,7 +465,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
}
opts.Envs = append(opts.Envs, envs...)
mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, opts.Envs)
mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, kl.subpather, opts.Envs)
if err != nil {
return nil, cleanupAction, err
}
@ -533,6 +546,10 @@ func (kl *Kubelet) getServiceEnvVarMap(ns string, enableServiceLinks bool) (map[
// Make the environment variables for a pod in the given namespace.
func (kl *Kubelet) makeEnvironmentVariables(pod *v1.Pod, container *v1.Container, podIP string) ([]kubecontainer.EnvVar, error) {
if pod.Spec.EnableServiceLinks == nil {
return nil, fmt.Errorf("nil pod.spec.enableServiceLinks encountered, cannot construct envvars")
}
var result []kubecontainer.EnvVar
// Note: These are added to the docker Config, but are not included in the checksum computed
// by kubecontainer.HashContainer(...). That way, we can still determine whether an
@ -852,7 +869,7 @@ func (kl *Kubelet) podIsTerminated(pod *v1.Pod) bool {
return status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded || (pod.DeletionTimestamp != nil && notRunning(status.ContainerStatuses))
}
// IsPodTerminated returns trus if the pod with the provided UID is in a terminated state ("Failed" or "Succeeded")
// IsPodTerminated returns true if the pod with the provided UID is in a terminated state ("Failed" or "Succeeded")
// or if the pod has been deleted or removed
func (kl *Kubelet) IsPodTerminated(uid types.UID) bool {
pod, podFound := kl.podManager.GetPodByUID(uid)

42
vendor/k8s.io/kubernetes/pkg/kubelet/runtime.go generated vendored
View file

@ -17,9 +17,12 @@ limitations under the License.
package kubelet
import (
"errors"
"fmt"
"sync"
"time"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
)
type runtimeState struct {
@ -27,6 +30,7 @@ type runtimeState struct {
lastBaseRuntimeSync time.Time
baseRuntimeSyncThreshold time.Duration
networkError error
storageError error
cidr string
healthChecks []*healthCheck
}
@ -58,6 +62,12 @@ func (s *runtimeState) setNetworkState(err error) {
s.networkError = err
}
func (s *runtimeState) setStorageState(err error) {
s.Lock()
defer s.Unlock()
s.storageError = err
}
func (s *runtimeState) setPodCIDR(cidr string) {
s.Lock()
defer s.Unlock()
@ -70,32 +80,42 @@ func (s *runtimeState) podCIDR() string {
return s.cidr
}
func (s *runtimeState) runtimeErrors() []string {
func (s *runtimeState) runtimeErrors() error {
s.RLock()
defer s.RUnlock()
var ret []string
errs := []error{}
if s.lastBaseRuntimeSync.IsZero() {
ret = append(ret, "container runtime status check may not have completed yet")
errs = append(errs, errors.New("container runtime status check may not have completed yet."))
} else if !s.lastBaseRuntimeSync.Add(s.baseRuntimeSyncThreshold).After(time.Now()) {
ret = append(ret, "container runtime is down")
errs = append(errs, errors.New("container runtime is down."))
}
for _, hc := range s.healthChecks {
if ok, err := hc.fn(); !ok {
ret = append(ret, fmt.Sprintf("%s is not healthy: %v", hc.name, err))
errs = append(errs, fmt.Errorf("%s is not healthy: %v.", hc.name, err))
}
}
return ret
return utilerrors.NewAggregate(errs)
}
func (s *runtimeState) networkErrors() []string {
func (s *runtimeState) networkErrors() error {
s.RLock()
defer s.RUnlock()
var ret []string
errs := []error{}
if s.networkError != nil {
ret = append(ret, s.networkError.Error())
errs = append(errs, s.networkError)
}
return ret
return utilerrors.NewAggregate(errs)
}
func (s *runtimeState) storageErrors() error {
s.RLock()
defer s.RUnlock()
errs := []error{}
if s.storageError != nil {
errs = append(errs, s.storageError)
}
return utilerrors.NewAggregate(errs)
}
func newRuntimeState(
@ -104,6 +124,6 @@ func newRuntimeState(
return &runtimeState{
lastBaseRuntimeSync: time.Time{},
baseRuntimeSyncThreshold: runtimeSyncThreshold,
networkError: fmt.Errorf("network state unknown"),
networkError: ErrNetworkUnknown,
}
}

1
vendor/k8s.io/kubernetes/pkg/kubelet/util/sliceutils/sliceutils.go generated vendored
View file

@ -21,6 +21,7 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
)
// StringInSlice returns true if s is in list
func StringInSlice(s string, list []string) bool {
for _, v := range list {
if v == s {

11
vendor/k8s.io/kubernetes/pkg/kubelet/volume_host.go generated vendored
View file

@ -30,7 +30,6 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/record"
cloudprovider "k8s.io/cloud-provider"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/configmap"
"k8s.io/kubernetes/pkg/kubelet/container"
@ -40,6 +39,7 @@ import (
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
// NewInitializedVolumePluginMgr returns a new instance of
@ -80,6 +80,7 @@ func NewInitializedVolumePluginMgr(
// Compile-time check to ensure kubeletVolumeHost implements the VolumeHost interface
var _ volume.VolumeHost = &kubeletVolumeHost{}
var _ volume.KubeletVolumeHost = &kubeletVolumeHost{}
func (kvh *kubeletVolumeHost) GetPluginDir(pluginName string) string {
return kvh.kubelet.getPluginDir(pluginName)
@ -94,6 +95,10 @@ type kubeletVolumeHost struct {
mountPodManager mountpod.Manager
}
func (kvh *kubeletVolumeHost) SetKubeletError(err error) {
kvh.kubelet.runtimeState.setStorageState(err)
}
func (kvh *kubeletVolumeHost) GetVolumeDevicePluginDir(pluginName string) string {
return kvh.kubelet.getVolumeDevicePluginDir(pluginName)
}
@ -122,8 +127,8 @@ func (kvh *kubeletVolumeHost) GetKubeClient() clientset.Interface {
return kvh.kubelet.kubeClient
}
func (kvh *kubeletVolumeHost) GetCSIClient() csiclientset.Interface {
return kvh.kubelet.csiClient
func (kvh *kubeletVolumeHost) GetSubpather() subpath.Interface {
return kvh.kubelet.subpather
}
func (kvh *kubeletVolumeHost) NewWrapperMounter(

7
vendor/k8s.io/kubernetes/pkg/util/BUILD generated vendored
View file

@ -20,7 +20,6 @@ filegroup(
"//pkg/util/dbus:all-srcs",
"//pkg/util/ebtables:all-srcs",
"//pkg/util/env:all-srcs",
"//pkg/util/file:all-srcs",
"//pkg/util/filesystem:all-srcs",
"//pkg/util/flag:all-srcs",
"//pkg/util/flock:all-srcs",
@ -28,25 +27,22 @@ filegroup(
"//pkg/util/hash:all-srcs",
"//pkg/util/initsystem:all-srcs",
"//pkg/util/interrupt:all-srcs",
"//pkg/util/io:all-srcs",
"//pkg/util/ipconfig:all-srcs",
"//pkg/util/ipset:all-srcs",
"//pkg/util/iptables:all-srcs",
"//pkg/util/ipvs:all-srcs",
"//pkg/util/keymutex:all-srcs",
"//pkg/util/labels:all-srcs",
"//pkg/util/maps:all-srcs",
"//pkg/util/metrics:all-srcs",
"//pkg/util/mount:all-srcs",
"//pkg/util/net:all-srcs",
"//pkg/util/netsh:all-srcs",
"//pkg/util/node:all-srcs",
"//pkg/util/normalizer:all-srcs",
"//pkg/util/nsenter:all-srcs",
"//pkg/util/oom:all-srcs",
"//pkg/util/parsers:all-srcs",
"//pkg/util/pod:all-srcs",
"//pkg/util/procfs:all-srcs",
"//pkg/util/prometheusclientgo:all-srcs",
"//pkg/util/reflector/prometheus:all-srcs",
"//pkg/util/removeall:all-srcs",
"//pkg/util/resizefs:all-srcs",
@ -54,7 +50,6 @@ filegroup(
"//pkg/util/rlimit:all-srcs",
"//pkg/util/selinux:all-srcs",
"//pkg/util/slice:all-srcs",
"//pkg/util/strings:all-srcs",
"//pkg/util/sysctl:all-srcs",
"//pkg/util/system:all-srcs",
"//pkg/util/tail:all-srcs",

36
vendor/k8s.io/kubernetes/pkg/util/file/BUILD generated vendored
View file

@ -1,36 +0,0 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = ["file.go"],
importpath = "k8s.io/kubernetes/pkg/util/file",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
go_test(
name = "go_default_test",
srcs = ["file_test.go"],
embed = [":go_default_library"],
deps = [
"//vendor/github.com/spf13/afero:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
)

57
vendor/k8s.io/kubernetes/pkg/util/file/file.go generated vendored
View file

@ -1,57 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package file
import (
"os"
)
// FileExists checks if specified file exists.
func FileExists(filename string) (bool, error) {
if _, err := os.Stat(filename); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, err
}
return true, nil
}
// FileOrSymlinkExists checks if specified file or symlink exists.
func FileOrSymlinkExists(filename string) (bool, error) {
if _, err := os.Lstat(filename); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, err
}
return true, nil
}
// ReadDirNoStat returns a string of files/directories contained
// in dirname without calling lstat on them.
func ReadDirNoStat(dirname string) ([]string, error) {
if dirname == "" {
dirname = "."
}
f, err := os.Open(dirname)
if err != nil {
return nil, err
}
defer f.Close()
return f.Readdirnames(-1)
}

25
vendor/k8s.io/kubernetes/pkg/util/io/BUILD generated vendored
View file

@ -1,25 +0,0 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["consistentread.go"],
importpath = "k8s.io/kubernetes/pkg/util/io",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

45
vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go generated vendored
View file

@ -1,45 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package io
import (
"bytes"
"fmt"
"io/ioutil"
)
// ConsistentRead repeatedly reads a file until it gets the same content twice.
// This is useful when reading files in /proc that are larger than page size
// and kernel may modify them between individual read() syscalls.
func ConsistentRead(filename string, attempts int) ([]byte, error) {
oldContent, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
for i := 0; i < attempts; i++ {
newContent, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
if bytes.Compare(oldContent, newContent) == 0 {
return newContent, nil
}
// Files are different, continue reading
oldContent = newContent
}
return nil, fmt.Errorf("could not get consistent content of %s after %d attempts", filename, attempts)
}

37
vendor/k8s.io/kubernetes/pkg/util/mount/BUILD generated vendored
View file

@ -9,7 +9,9 @@ go_library(
"exec_mount_unsupported.go",
"fake.go",
"mount.go",
"mount_helper.go",
"mount_helper_common.go",
"mount_helper_unix.go",
"mount_helper_windows.go",
"mount_linux.go",
"mount_unsupported.go",
"mount_windows.go",
@ -23,42 +25,43 @@ go_library(
"//vendor/k8s.io/utils/exec:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/file:go_default_library",
"//pkg/util/io:go_default_library",
"//pkg/util/nsenter:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/utils/io:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
"//vendor/k8s.io/utils/path:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//pkg/util/file:go_default_library",
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/keymutex:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
"//vendor/k8s.io/utils/path:go_default_library",
],
"//conditions:default": [],
}),
@ -80,10 +83,8 @@ go_test(
"//vendor/k8s.io/utils/exec/testing:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/nsenter:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/github.com/stretchr/testify/assert:go_default_library",

2
vendor/k8s.io/kubernetes/pkg/util/mount/OWNERS generated vendored
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- jingxu97
- saad-ali

12
vendor/k8s.io/kubernetes/pkg/util/mount/exec_mount.go generated vendored
View file

@ -144,18 +144,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
return m.wrappedMounter.EvalHostSymlinks(pathname)
}
func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return m.wrappedMounter.PrepareSafeSubpath(subPath)
}
func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return m.wrappedMounter.CleanSubPaths(podDir, volumeName)
}
func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
}
func (m *execMounter) GetMountRefs(pathname string) ([]string, error) {
return m.wrappedMounter.GetMountRefs(pathname)
}

14
vendor/k8s.io/kubernetes/pkg/util/mount/exec_mount_unsupported.go generated vendored
View file

@ -48,7 +48,7 @@ func (mounter *execMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
}
func (mounter *execMounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
return isNotMountPoint(mounter, dir)
}
func (mounter *execMounter) IsLikelyNotMountPoint(file string) (bool, error) {
@ -91,18 +91,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

13
vendor/k8s.io/kubernetes/pkg/util/mount/fake.go generated vendored
View file

@ -137,7 +137,7 @@ func (f *FakeMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
}
func (f *FakeMounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(f, dir)
return isNotMountPoint(f, dir)
}
func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
@ -220,17 +220,6 @@ func (f *FakeMounter) EvalHostSymlinks(pathname string) (string, error) {
return pathname, nil
}
func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) {
realpath, err := filepath.EvalSymlinks(pathname)
if err != nil {

40
vendor/k8s.io/kubernetes/pkg/util/mount/mount.go generated vendored
View file

@ -84,35 +84,12 @@ type Interface interface {
// MakeDir creates a new directory.
// Will operate in the host mount namespace if kubelet is running in a container
MakeDir(pathname string) error
// SafeMakeDir creates subdir within given base. It makes sure that the
// created directory does not escape given base directory mis-using
// symlinks. Note that the function makes sure that it creates the directory
// somewhere under the base, nothing else. E.g. if the directory already
// exists, it may exist outside of the base due to symlinks.
// This method should be used if the directory to create is inside volume
// that's under user control. User must not be able to use symlinks to
// escape the volume to create directories somewhere else.
SafeMakeDir(subdir string, base string, perm os.FileMode) error
// Will operate in the host mount namespace if kubelet is running in a container.
// Error is returned on any other error than "file not found".
ExistsPath(pathname string) (bool, error)
// EvalHostSymlinks returns the path name after evaluating symlinks.
// Will operate in the host mount namespace if kubelet is running in a container.
EvalHostSymlinks(pathname string) (string, error)
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory.
CleanSubPaths(podDir string, volumeName string) error
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
// that's 1) inside given volumePath and 2) immutable after this call.
//
// newHostPath - location of prepared subPath. It should be used instead of
// hostName when running the container.
// cleanupAction - action to run when the container is running or it failed to start.
//
// CleanupAction must be called immediately after the container with given
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
// when the pod finishes.
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
// GetMountRefs finds all mount references to the path, returns a
// list of paths. Path could be a mountpoint path, device or a normal
// directory (for bind mount).
@ -143,7 +120,7 @@ type Subpath struct {
// Exec executes command where mount utilities are. This can be either the host,
// container where kubelet runs or even a remote pod with mount utilities.
// Usual pkg/util/exec interface is not used because kubelet.RunInContainer does
// Usual k8s.io/utils/exec interface is not used because kubelet.RunInContainer does
// not provide stdin/stdout/stderr streams.
type Exec interface {
// Run executes a command and returns its stdout + stderr combined in one
@ -245,12 +222,9 @@ func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, e
return device, refCount, nil
}
// IsNotMountPoint determines if a directory is a mountpoint.
// It should return ErrNotExist when the directory does not exist.
// This method uses the List() of all mountpoints
// It is more extensive than IsLikelyNotMountPoint
// and it detects bind mounts in linux
func IsNotMountPoint(mounter Interface, file string) (bool, error) {
// isNotMountPoint implements Mounter.IsNotMountPoint and is shared by mounter
// implementations.
func isNotMountPoint(mounter Interface, file string) (bool, error) {
// IsLikelyNotMountPoint provides a quick check
// to determine whether file IS A mountpoint
notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file)
@ -358,15 +332,15 @@ func PathWithinBase(fullPath, basePath string) bool {
if err != nil {
return false
}
if startsWithBackstep(rel) {
if StartsWithBackstep(rel) {
// Needed to escape the base path
return false
}
return true
}
// startsWithBackstep checks if the given path starts with a backstep segment
func startsWithBackstep(rel string) bool {
// StartsWithBackstep checks if the given path starts with a backstep segment
func StartsWithBackstep(rel string) bool {
// normalize to / and check for ../
return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
}

23
vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper.go → vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper_common.go generated vendored
View file

@ -19,7 +19,6 @@ package mount
import (
"fmt"
"os"
"syscall"
"k8s.io/klog"
)
@ -56,7 +55,7 @@ func doCleanupMountPoint(mountPath string, mounter Interface, extensiveMountPoin
var notMnt bool
var err error
if extensiveMountPointCheck {
notMnt, err = IsNotMountPoint(mounter, mountPath)
notMnt, err = mounter.IsNotMountPoint(mountPath)
} else {
notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
}
@ -102,23 +101,3 @@ func PathExists(path string) (bool, error) {
return false, err
}
}
// IsCorruptedMnt return true if err is about corrupted mount point
func IsCorruptedMnt(err error) bool {
if err == nil {
return false
}
var underlyingError error
switch pe := err.(type) {
case nil:
return false
case *os.PathError:
underlyingError = pe.Err
case *os.LinkError:
underlyingError = pe.Err
case *os.SyscallError:
underlyingError = pe.Err
}
return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO
}

44
vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper_unix.go generated vendored Normal file
View file

@ -0,0 +1,44 @@
// +build !windows
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
import (
"os"
"syscall"
)
// IsCorruptedMnt return true if err is about corrupted mount point
func IsCorruptedMnt(err error) bool {
if err == nil {
return false
}
var underlyingError error
switch pe := err.(type) {
case nil:
return false
case *os.PathError:
underlyingError = pe.Err
case *os.LinkError:
underlyingError = pe.Err
case *os.SyscallError:
underlyingError = pe.Err
}
return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES
}

68
vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper_windows.go generated vendored Normal file
View file

@ -0,0 +1,68 @@
// +build windows
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
import (
"os"
"syscall"
"k8s.io/klog"
)
// following failure codes are from https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1300-1699-
// ERROR_BAD_NETPATH = 53
// ERROR_NETWORK_BUSY = 54
// ERROR_UNEXP_NET_ERR = 59
// ERROR_NETNAME_DELETED = 64
// ERROR_NETWORK_ACCESS_DENIED = 65
// ERROR_BAD_DEV_TYPE = 66
// ERROR_BAD_NET_NAME = 67
// ERROR_SESSION_CREDENTIAL_CONFLICT = 1219
// ERROR_LOGON_FAILURE = 1326
var errorNoList = [...]int{53, 54, 59, 64, 65, 66, 67, 1219, 1326}
// IsCorruptedMnt return true if err is about corrupted mount point
func IsCorruptedMnt(err error) bool {
if err == nil {
return false
}
var underlyingError error
switch pe := err.(type) {
case nil:
return false
case *os.PathError:
underlyingError = pe.Err
case *os.LinkError:
underlyingError = pe.Err
case *os.SyscallError:
underlyingError = pe.Err
}
if ee, ok := underlyingError.(syscall.Errno); ok {
for _, errno := range errorNoList {
if int(ee) == errno {
klog.Warningf("IsCorruptedMnt failed with error: %v, error code: %v", err, errno)
return true
}
}
}
return false
}

534
vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux.go generated vendored
View file

@ -21,7 +21,6 @@ package mount
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
@ -33,9 +32,9 @@ import (
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
utilfile "k8s.io/kubernetes/pkg/util/file"
utilio "k8s.io/kubernetes/pkg/util/io"
utilexec "k8s.io/utils/exec"
utilio "k8s.io/utils/io"
utilpath "k8s.io/utils/path"
)
const (
@ -53,14 +52,6 @@ const (
fsckErrorsCorrected = 1
// 'fsck' found errors but exited without correcting them
fsckErrorsUncorrected = 4
// place for subpath mounts
// TODO: pass in directory using kubelet_getters instead
containerSubPathDirectoryName = "volume-subpaths"
// syscall.Openat flags used to traverse directories not following symlinks
nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
// flags for getting file descriptor without following the symlink
openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
)
// Mounter provides the default implementation of mount.Interface
@ -229,7 +220,7 @@ func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
}
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
return isNotMountPoint(mounter, dir)
}
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
@ -417,7 +408,7 @@ func (mounter *Mounter) MakeFile(pathname string) error {
}
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
return utilfile.FileExists(pathname)
return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}
func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
@ -726,287 +717,15 @@ func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) {
return false, nil
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
// This implementation is shared between Linux and NsEnterMounter
func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
if !PathWithinBase(subpath.Path, subpath.VolumePath) {
return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
}
fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
if err != nil {
return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
}
return fd, nil
}
// prepareSubpathTarget creates target for bind-mount of subpath. It returns
// "true" when the target already exists and something is mounted there.
// Given Subpath must have all paths with already resolved symlinks and with
// paths relevant to kubelet (when it runs in a container).
// This function is called also by NsEnterMounter. It works because
// /var/lib/kubelet is mounted from the host into the container with Kubelet as
// /var/lib/kubelet too.
func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) {
// Early check for already bind-mounted subpath.
bindPathTarget := getSubpathBindTarget(subpath)
notMount, err := IsNotMountPoint(mounter, bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
return true, bindPathTarget, nil
}
// bindPathTarget is in /var/lib/kubelet and thus reachable without any
// translation even to containerized kubelet.
bindParent := filepath.Dir(bindPathTarget)
err = os.MkdirAll(bindParent, 0750)
if err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
}
t, err := os.Lstat(subpath.Path)
if err != nil {
return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindPathTarget>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
return false, bindPathTarget, nil
}
func getSubpathBindTarget(subpath Subpath) string {
// containerName is DNS label, i.e. safe as a directory name.
return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
}
func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs on the host:
// - safely open the subpath
// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
// Evaluate all symlinks here once for all subsequent functions.
newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
newPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
subpath.VolumePath = newVolumePath
subpath.Path = newPath
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
kubeletPid := os.Getpid()
mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
// Do the bind mount
options := []string{"bind"}
klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
// This implementation is shared between Linux and NsEnterMounter
func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error {
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
containerDirs, err := ioutil.ReadDir(subPathDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("error reading %s: %s", subPathDir, err)
}
for _, containerDir := range containerDirs {
if !containerDir.IsDir() {
klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
continue
}
klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
if path == fullContainerDirPath {
// Skip top level directory
return nil
}
// pass through errors and let doCleanSubPath handle them
if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
return err
}
return nil
})
if err != nil {
return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
}
// Whole container has been processed, remove its directory.
if err := os.Remove(fullContainerDirPath); err != nil {
return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
}
klog.V(5).Infof("Removed %s", fullContainerDirPath)
}
// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
if err := os.Remove(subPathDir); err != nil {
return fmt.Errorf("error deleting %s: %s", subPathDir, err)
}
klog.V(5).Infof("Removed %s", subPathDir)
// Remove entire subpath directory if it's the last one
podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
}
klog.V(5).Infof("Removed %s", podSubPathDir)
return nil
}
// doCleanSubPath tears down the single subpath bind mount
func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error {
// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil {
return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
}
klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
return nil
}
// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
func cleanSubPath(mounter Interface, subpath Subpath) error {
containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
// Clean subdir bindmount
if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
return err
}
// Recusively remove directories if empty
if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
return err
}
return nil
}
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error {
if !PathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
}
for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
s, err := os.Stat(curDir)
if err != nil {
if os.IsNotExist(err) {
klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
continue
}
return fmt.Errorf("error stat %q: %v", curDir, err)
}
if !s.IsDir() {
return fmt.Errorf("path %q not a directory", curDir)
}
err = os.Remove(curDir)
if os.IsExist(err) {
klog.V(5).Infof("Directory %q not empty, not removing", curDir)
break
} else if err != nil {
return fmt.Errorf("error removing directory %q: %v", curDir, err)
}
klog.V(5).Infof("Removed directory %q", curDir)
}
return nil
}
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
if _, err := os.Stat(pathname); os.IsNotExist(err) {
pathExists, pathErr := PathExists(pathname)
if !pathExists {
return []string{}, nil
} else if err != nil {
return nil, err
} else if IsCorruptedMnt(pathErr) {
klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", pathname)
return []string{}, nil
} else if pathErr != nil {
return nil, fmt.Errorf("error checking path %s: %v", pathname, pathErr)
}
realpath, err := filepath.EvalSymlinks(pathname)
if err != nil {
@ -1049,237 +768,6 @@ func getMode(pathname string) (os.FileMode, error) {
return info.Mode(), nil
}
// This implementation is shared between Linux and NsEnterMounter. Both pathname
// and base must be either already resolved symlinks or thet will be resolved in
// kubelet's mount namespace (in case it runs containerized).
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err)
}
if !PathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
parentFD, err := doSafeOpen(fullExistingPath, base)
if err != nil {
return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
}
childFD := -1
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
}
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink.
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
if err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
// Dive into the created directory
childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
if err != nil {
return fmt.Errorf("cannot open %s: %s", currentPath, err)
}
// We can be sure that childFD is safe to use. It could be changed
// by user after Mkdirat() and before Openat(), however:
// - it could not be changed to symlink - we use nofollowFlags
// - it could be changed to a file (or device, pipe, socket, ...)
// but either subsequent Mkdirat() fails or we mount this file
// to user's container. Security is no violated in both cases
// and user either gets error or the file that it can already access.
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
parentFD = childFD
childFD = -1
}
// Everything was created. mkdirat(..., perm) above was affected by current
// umask and we must apply the right permissions to the last directory
// (that's the one that will be available to the container as subpath)
// so user can read/write it. This is the behavior of previous code.
// TODO: chmod all created directories, not just the last one.
// parentFD is the last created directory.
// Translate perm (os.FileMode) to uint32 that fchmod() expects
kernelPerm := uint32(perm & os.ModePerm)
if perm&os.ModeSetgid > 0 {
kernelPerm |= syscall.S_ISGID
}
if perm&os.ModeSetuid > 0 {
kernelPerm |= syscall.S_ISUID
}
if perm&os.ModeSticky > 0 {
kernelPerm |= syscall.S_ISVTX
}
if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
return fmt.Errorf("chmod %q failed: %s", currentPath, err)
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
dirs := strings.Split(rel, string(filepath.Separator))
// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
// This should be faster than looping through all dirs and calling os.Stat()
// on each of them, as the symlinks are resolved only once with OpenAt().
currentPath := base
fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
if err != nil {
return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
}
defer func() {
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
}()
for i, dir := range dirs {
// Using O_PATH here will prevent hangs in case user replaces directory with
// fifo
childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
if err != nil {
if os.IsNotExist(err) {
return currentPath, dirs[i:], nil
}
return base, nil, err
}
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
fd = childFD
currentPath = filepath.Join(currentPath, dir)
}
return pathname, []string{}, nil
}
// This implementation is shared between Linux and NsEnterMounter
// Open path and return its fd.
// Symlinks are disallowed (pathname must already resolve symlinks),
// and the path must be within the base directory.
func doSafeOpen(pathname string, base string) (int, error) {
pathname = filepath.Clean(pathname)
base = filepath.Clean(base)
// Calculate segments to follow
subpath, err := filepath.Rel(base, pathname)
if err != nil {
return -1, err
}
segments := strings.Split(subpath, string(filepath.Separator))
// Assumption: base is the only directory that we have under control.
// Base dir is not allowed to be a symlink.
parentFD, err := syscall.Open(base, nofollowFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
}
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
}
}
}()
childFD := -1
defer func() {
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := base
// Follow the segments one by one using openat() to make
// sure the user cannot change already existing directories into symlinks.
for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg)
if !PathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
}
klog.V(5).Infof("Opening path %s", currentPath)
childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
}
var deviceStat unix.Stat_t
err := unix.Fstat(childFD, &deviceStat)
if err != nil {
return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
}
fileFmt := deviceStat.Mode & syscall.S_IFMT
if fileFmt == syscall.S_IFLNK {
return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
}
// Close parentFD
if err = syscall.Close(parentFD); err != nil {
return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
}
// Set child to new parent
parentFD = childFD
childFD = -1
}
// We made it to the end, return this fd, don't close it
finalFD := parentFD
parentFD = -1
return finalFD, nil
}
// searchMountPoints finds all mount references to the source, returns a list of
// mountpoints.
// This function assumes source cannot be device.

14
vendor/k8s.io/kubernetes/pkg/util/mount/mount_unsupported.go generated vendored
View file

@ -55,7 +55,7 @@ func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
}
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
return isNotMountPoint(mounter, dir)
}
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
@ -110,18 +110,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return "", unsupportedErr
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, unsupportedErr
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return unsupportedErr
}
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return unsupportedErr
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

342
vendor/k8s.io/kubernetes/pkg/util/mount/mount_windows.go generated vendored
View file

@ -26,11 +26,11 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"k8s.io/klog"
"k8s.io/utils/keymutex"
utilfile "k8s.io/kubernetes/pkg/util/file"
utilpath "k8s.io/utils/path"
)
// Mounter provides the default implementation of mount.Interface
@ -49,12 +49,16 @@ func New(mounterPath string) Interface {
}
}
// Mount : mounts source to target as NTFS with given options.
// acquire lock for smb mount
var getSMBMountMutex = keymutex.NewHashed(0)
// Mount : mounts source to target with given options.
// currently only supports cifs(smb), bind mount(for disk)
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
target = normalizeWindowsPath(target)
if source == "tmpfs" {
klog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options)
klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, options)
return os.MkdirAll(target, 0755)
}
@ -63,9 +67,9 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
return err
}
klog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
options, source, target, fstype)
bindSource := ""
bindSource := source
// tell it's going to mount azure disk or azure file according to options
if bind, _, _ := isBind(options); bind {
@ -73,31 +77,32 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
bindSource = normalizeWindowsPath(source)
} else {
if len(options) < 2 {
klog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
options, len(options), source, target)
return nil
}
// currently only cifs mount is supported
if strings.ToLower(fstype) != "cifs" {
return fmt.Errorf("azureMount: only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options)
return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options)
}
bindSource = source
// lock smb mount for the same source
getSMBMountMutex.LockKey(source)
defer getSMBMountMutex.UnlockKey(source)
// use PowerShell Environment Variables to store user input string to prevent command line injection
// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
cmdLine := fmt.Sprintf(`$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`)
cmd := exec.Command("powershell", "/c", cmdLine)
cmd.Env = append(os.Environ(),
fmt.Sprintf("smbuser=%s", options[0]),
fmt.Sprintf("smbpassword=%s", options[1]),
fmt.Sprintf("smbremotepath=%s", source))
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("azureMount: SmbGlobalMapping failed: %v, only SMB mount is supported now, output: %q", err, string(output))
if output, err := newSMBMapping(options[0], options[1], source); err != nil {
if isSMBMappingExist(source) {
klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source)
if output, err := removeSMBMapping(source); err != nil {
return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output)
}
if output, err := newSMBMapping(options[0], options[1], source); err != nil {
return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output)
}
} else {
return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output)
}
}
}
@ -109,6 +114,44 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
return nil
}
// do the SMB mount with username, password, remotepath
// return (output, error)
func newSMBMapping(username, password, remotepath string) (string, error) {
if username == "" || password == "" || remotepath == "" {
return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, password, remotepath)
}
// use PowerShell Environment Variables to store user input string to prevent command line injection
// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`
cmd := exec.Command("powershell", "/c", cmdLine)
cmd.Env = append(os.Environ(),
fmt.Sprintf("smbuser=%s", username),
fmt.Sprintf("smbpassword=%s", password),
fmt.Sprintf("smbremotepath=%s", remotepath))
output, err := cmd.CombinedOutput()
return string(output), err
}
// check whether remotepath is already mounted
func isSMBMappingExist(remotepath string) bool {
cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`)
cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
_, err := cmd.CombinedOutput()
return err == nil
}
// remove SMB mapping
func removeSMBMapping(remotepath string) (string, error) {
cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`)
cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
output, err := cmd.CombinedOutput()
return string(output), err
}
// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
klog.V(4).Infof("azureMount: Unmount target (%q)", target)
@ -132,7 +175,7 @@ func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
// IsNotMountPoint determines if a directory is a mountpoint.
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
return isNotMountPoint(mounter, dir)
}
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
@ -235,7 +278,7 @@ func (mounter *Mounter) MakeFile(pathname string) error {
// ExistsPath checks whether the path exists
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
return utilfile.FileExists(pathname)
return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}
// EvalHostSymlinks returns the path name after evaluating symlinks
@ -243,123 +286,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return filepath.EvalSymlinks(pathname)
}
// check whether hostPath is within volume path
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(hostPath) == 0 {
return []uintptr{}, nil
}
finalSubPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
}
finalVolumePath, err := filepath.EvalSymlinks(volumePath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
}
return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
}
// lock all intermediate subPath directories and check they are all within volumePath
// volumePath & subPath should not contain any symlink, otherwise it will return error
func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(subPath) == 0 {
return []uintptr{}, nil
}
// get relative path to volumePath
relSubPath, err := filepath.Rel(volumePath, subPath)
if err != nil {
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
}
if startsWithBackstep(relSubPath) {
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
}
if relSubPath == "." {
// volumePath and subPath are equal
return []uintptr{}, nil
}
fileHandles := []uintptr{}
var errorResult error
currentFullPath := volumePath
dirs := strings.Split(relSubPath, string(os.PathSeparator))
for _, dir := range dirs {
// lock intermediate subPath directory first
currentFullPath = filepath.Join(currentFullPath, dir)
handle, err := lockPath(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
break
}
fileHandles = append(fileHandles, handle)
// make sure intermediate subPath directory does not contain symlink any more
stat, err := os.Lstat(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
break
}
if stat.Mode()&os.ModeSymlink != 0 {
errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
break
}
if !PathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break
}
}
return fileHandles, errorResult
}
// unlockPath unlock directories
func unlockPath(fileHandles []uintptr) {
if fileHandles != nil {
for _, handle := range fileHandles {
syscall.CloseHandle(syscall.Handle(handle))
}
}
}
// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
func lockPath(path string) (uintptr, error) {
if len(path) == 0 {
return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return uintptr(syscall.InvalidHandle), err
}
access := uint32(syscall.GENERIC_READ)
sharemode := uint32(syscall.FILE_SHARE_READ)
createmode := uint32(syscall.OPEN_EXISTING)
flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
return uintptr(fd), err
}
// Lock all directories in subPath and check they're not symlinks.
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
// Unlock the directories when the container starts
cleanupAction = func() {
unlockPath(handles)
}
return subPath.Path, cleanupAction, err
}
// No bind-mounts for subpaths are necessary on Windows
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
// Try to mount the disk
klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
@ -460,10 +386,15 @@ func getAllParentLinks(path string) ([]string, error) {
// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
if _, err := os.Stat(normalizeWindowsPath(pathname)); os.IsNotExist(err) {
windowsPath := normalizeWindowsPath(pathname)
pathExists, pathErr := PathExists(windowsPath)
if !pathExists {
return []string{}, nil
} else if err != nil {
return nil, err
} else if IsCorruptedMnt(pathErr) {
klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath)
return []string{}, nil
} else if pathErr != nil {
return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr)
}
return []string{pathname}, nil
}
@ -486,126 +417,3 @@ func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
}
return info.Mode(), nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
if len(toCreate) == 0 {
return nil
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
}
fullBasePath, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err)
}
if !PathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
defer unlockPath(fileHandles)
if err != nil {
return err
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink by lock that directory immediately
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
if err := os.Mkdir(currentPath, perm); err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
handle, err := lockPath(currentPath)
if err != nil {
return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
}
defer syscall.CloseHandle(syscall.Handle(handle))
// make sure newly created directory does not contain symlink after lock
stat, err := os.Lstat(currentPath)
if err != nil {
return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
}
if stat.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
}
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
if startsWithBackstep(rel) {
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
}
if rel == "." {
// base and pathname are equal
return pathname, []string{}, nil
}
dirs := strings.Split(rel, string(filepath.Separator))
parent := base
currentPath := base
for i, dir := range dirs {
parent = currentPath
currentPath = filepath.Join(parent, dir)
if _, err := os.Lstat(currentPath); err != nil {
if os.IsNotExist(err) {
return parent, dirs[i:], nil
}
return base, nil, err
}
}
return pathname, []string{}, nil
}

153
vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount.go generated vendored
View file

@ -23,12 +23,10 @@ import (
"os"
"path/filepath"
"strings"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
utilfile "k8s.io/kubernetes/pkg/util/file"
"k8s.io/kubernetes/pkg/util/nsenter"
"k8s.io/utils/nsenter"
utilpath "k8s.io/utils/path"
)
const (
@ -145,7 +143,7 @@ func (*NsenterMounter) List() ([]MountPoint, error) {
}
func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(m, dir)
return isNotMountPoint(m, dir)
}
func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
@ -291,65 +289,19 @@ func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) {
return false, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return utilfile.FileExists(kubeletpath)
return utilpath.Exists(utilpath.CheckFollowSymlink, kubeletpath)
}
func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return mounter.ne.EvalSymlinks(pathname, true)
}
func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(mounter.rootDir)
if PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
// create subpaths for paths in /var/lib/kubelet.
return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
}
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
}
func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
exists, err := mounter.ExistsPath(pathname)
if err != nil {
return nil, err
}
if !exists {
pathExists, pathErr := PathExists(pathname)
if !pathExists || IsCorruptedMnt(pathErr) {
return []string{}, nil
} else if pathErr != nil {
return nil, fmt.Errorf("Error checking path %s: %v", pathname, pathErr)
}
hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
@ -358,95 +310,6 @@ func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return searchMountPoints(hostpath, hostProcMountinfoPath)
}
func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
// - check that we mounted the right thing by comparing device ID and inode
// of the subpath (via safely opened fd) and the target (that's under our
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Leap of faith: optimistically expect that nobody has modified previously
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
// Check that the bind-mount target is the same inode and device as the
// source that we keept open, i.e. we mounted the right thing.
err = checkDeviceInode(fd, bindPathTarget)
if err != nil {
return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// checkDeviceInode checks that opened file and path represent the same file.
func checkDeviceInode(fd int, path string) error {
var srcStat, dstStat unix.Stat_t
err := unix.Fstat(fd, &srcStat)
if err != nil {
return fmt.Errorf("error running fstat on subpath FD: %v", err)
}
err = unix.Stat(path, &dstStat)
if err != nil {
return fmt.Errorf("error running fstat on %s: %v", path, err)
}
if srcStat.Dev != dstStat.Dev {
return fmt.Errorf("different device number")
}
if srcStat.Ino != dstStat.Ino {
return fmt.Errorf("different inode")
}
return nil
}
func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {

16
vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount_unsupported.go generated vendored
View file

@ -22,7 +22,7 @@ import (
"errors"
"os"
"k8s.io/kubernetes/pkg/util/nsenter"
"k8s.io/utils/nsenter"
)
type NsenterMounter struct{}
@ -46,7 +46,7 @@ func (*NsenterMounter) List() ([]MountPoint, error) {
}
func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(m, dir)
return isNotMountPoint(m, dir)
}
func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
@ -93,18 +93,6 @@ func (*NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

76
vendor/k8s.io/kubernetes/pkg/util/nsenter/BUILD generated vendored
View file

@ -1,76 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"exec.go",
"exec_unsupported.go",
"nsenter.go",
"nsenter_unsupported.go",
],
importpath = "k8s.io/kubernetes/pkg/util/nsenter",
visibility = ["//visibility:public"],
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = ["nsenter_test.go"],
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"//conditions:default": [],
}),
)

8
vendor/k8s.io/kubernetes/pkg/util/nsenter/OWNERS generated vendored
View file

@ -1,8 +0,0 @@
reviewers:
- jsafrane
- msau42
- cofyc
approvers:
- jsafrane
- msau42
- cofyc

67
vendor/k8s.io/kubernetes/pkg/util/nsenter/exec.go generated vendored
View file

@ -1,67 +0,0 @@
// +build linux
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"fmt"
"path/filepath"
"k8s.io/klog"
"k8s.io/utils/exec"
)
// Executor wraps executor interface to be executed via nsenter
type Executor struct {
// Exec implementation
executor exec.Interface
// Path to the host's root proc path
hostProcMountNsPath string
}
// NewNsenterExecutor returns new nsenter based executor
func NewNsenterExecutor(hostRootFsPath string, executor exec.Interface) *Executor {
hostProcMountNsPath := filepath.Join(hostRootFsPath, mountNsPath)
nsExecutor := &Executor{
hostProcMountNsPath: hostProcMountNsPath,
executor: executor,
}
return nsExecutor
}
// Command returns a command wrapped with nenter
func (nsExecutor *Executor) Command(cmd string, args ...string) exec.Cmd {
fullArgs := append([]string{fmt.Sprintf("--mount=%s", nsExecutor.hostProcMountNsPath), "--"},
append([]string{cmd}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return nsExecutor.executor.Command(nsenterPath, fullArgs...)
}
// CommandContext returns a CommandContext wrapped with nsenter
func (nsExecutor *Executor) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
fullArgs := append([]string{fmt.Sprintf("--mount=%s", nsExecutor.hostProcMountNsPath), "--"},
append([]string{cmd}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return nsExecutor.executor.CommandContext(ctx, nsenterPath, fullArgs...)
}
// LookPath returns a LookPath wrapped with nsenter
func (nsExecutor *Executor) LookPath(file string) (string, error) {
return "", fmt.Errorf("not implemented, error looking up : %s", file)
}

58
vendor/k8s.io/kubernetes/pkg/util/nsenter/exec_unsupported.go generated vendored
View file

@ -1,58 +0,0 @@
// +build !linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"fmt"
"k8s.io/utils/exec"
)
// Executor wraps executor interface to be executed via nsenter
type Executor struct {
// Exec implementation
executor exec.Interface
// Path to the host's root proc path
hostProcMountNsPath string
}
// NewNsenterExecutor returns new nsenter based executor
func NewNsenterExecutor(hostRootFsPath string, executor exec.Interface) *Executor {
nsExecutor := &Executor{
hostProcMountNsPath: hostRootFsPath,
executor: executor,
}
return nsExecutor
}
// Command returns a command wrapped with nenter
func (nsExecutor *Executor) Command(cmd string, args ...string) exec.Cmd {
return nil
}
// CommandContext returns a CommandContext wrapped with nsenter
func (nsExecutor *Executor) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return nil
}
// LookPath returns a LookPath wrapped with nsenter
func (nsExecutor *Executor) LookPath(file string) (string, error) {
return "", fmt.Errorf("not implemented, error looking up : %s", file)
}

236
vendor/k8s.io/kubernetes/pkg/util/nsenter/nsenter.go generated vendored
View file

@ -1,236 +0,0 @@
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"k8s.io/utils/exec"
"k8s.io/klog"
)
const (
// DefaultHostRootFsPath is path to host's filesystem mounted into container
// with kubelet.
DefaultHostRootFsPath = "/rootfs"
// mountNsPath is the default mount namespace of the host
mountNsPath = "/proc/1/ns/mnt"
// nsenterPath is the default nsenter command
nsenterPath = "nsenter"
)
// Nsenter is part of experimental support for running the kubelet
// in a container.
//
// Nsenter requires:
//
// 1. Docker >= 1.6 due to the dependency on the slave propagation mode
// of the bind-mount of the kubelet root directory in the container.
// Docker 1.5 used a private propagation mode for bind-mounts, so mounts
// performed in the host's mount namespace do not propagate out to the
// bind-mount in this docker version.
// 2. The host's root filesystem must be available at /rootfs
// 3. The nsenter binary must be on the Kubelet process' PATH in the container's
// filesystem.
// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at
// the present, this effectively means that the kubelet is running in a
// privileged container.
// 5. The volume path used by the Kubelet must be the same inside and outside
// the container and be writable by the container (to initialize volume)
// contents. TODO: remove this requirement.
// 6. The host image must have "mount", "findmnt", "umount", "stat", "touch",
// "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin
// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin if
// systemd is installed/enabled in the operating system.
// For more information about mount propagation modes, see:
// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt
type Nsenter struct {
// a map of commands to their paths on the host filesystem
paths map[string]string
// Path to the host filesystem, typically "/rootfs". Used only for testing.
hostRootFsPath string
// Exec implementation, used only for testing
executor exec.Interface
}
// NewNsenter constructs a new instance of Nsenter
func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
ne := &Nsenter{
hostRootFsPath: hostRootFsPath,
executor: executor,
}
if err := ne.initPaths(); err != nil {
return nil, err
}
return ne, nil
}
func (ne *Nsenter) initPaths() error {
ne.paths = map[string]string{}
binaries := []string{
"mount",
"findmnt",
"umount",
"systemd-run",
"stat",
"touch",
"mkdir",
"sh",
"chmod",
"realpath",
}
// search for the required commands in other locations besides /usr/bin
for _, binary := range binaries {
// check for binary under the following directories
for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
binPath := filepath.Join(path, binary)
if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
continue
}
ne.paths[binary] = binPath
break
}
// systemd-run is optional, bailout if we don't find any of the other binaries
if ne.paths[binary] == "" && binary != "systemd-run" {
return fmt.Errorf("unable to find %v", binary)
}
}
return nil
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
append([]string{ne.AbsHostPath(cmd)}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return ne.executor.Command(nsenterPath, fullArgs...)
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *Nsenter) AbsHostPath(command string) string {
path, ok := ne.paths[command]
if !ok {
return command
}
return path
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *Nsenter) SupportsSystemd() (string, bool) {
systemdRunPath, ok := ne.paths["systemd-run"]
return systemdRunPath, ok && systemdRunPath != ""
}
// EvalSymlinks returns the path name on the host after evaluating symlinks on the
// host.
// mustExist makes EvalSymlinks to return error when the path does not
// exist. When it's false, it evaluates symlinks of the existing part and
// blindly adds the non-existing part:
// pathname: /mnt/volume/non/existing/directory
// /mnt/volume exists
// non/existing/directory does not exist
// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
// /mnt/foo/non/existing/directory.
//
// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
var args []string
if mustExist {
// "realpath -e: all components of the path must exist"
args = []string{"-e", pathname}
} else {
// "realpath -m: no path components need exist or be a directory"
args = []string{"-m", pathname}
}
outBytes, err := ne.Exec("realpath", args).CombinedOutput()
if err != nil {
klog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
return "", err
}
return strings.TrimSpace(string(outBytes)), nil
}
// KubeletPath returns the path name that can be accessed by containerized
// kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
// before calling this function
func (ne *Nsenter) KubeletPath(pathname string) string {
return filepath.Join(ne.hostRootFsPath, pathname)
}
// NewFakeNsenter returns a Nsenter that does not run "nsenter --mount=... --",
// but runs everything in the same mount namespace as the unit test binary.
// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
// This fake Nsenter is enough for most operations, e.g. to resolve symlinks,
// but it's not enough to call /bin/mount - unit tests don't run as root.
func NewFakeNsenter(rootfsPath string) (*Nsenter, error) {
executor := &fakeExec{
rootfsPath: rootfsPath,
}
// prepare /rootfs/bin, usr/bin and usr/sbin
bin := filepath.Join(rootfsPath, "bin")
if err := os.Symlink("/bin", bin); err != nil {
return nil, err
}
usr := filepath.Join(rootfsPath, "usr")
if err := os.Mkdir(usr, 0755); err != nil {
return nil, err
}
usrbin := filepath.Join(usr, "bin")
if err := os.Symlink("/usr/bin", usrbin); err != nil {
return nil, err
}
usrsbin := filepath.Join(usr, "sbin")
if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
return nil, err
}
return NewNsenter(rootfsPath, executor)
}
type fakeExec struct {
rootfsPath string
}
func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
// This will intentionaly panic if Nsenter does not provide enough arguments.
realCmd := args[2]
realArgs := args[3:]
return exec.New().Command(realCmd, realArgs...)
}
func (fakeExec) LookPath(file string) (string, error) {
return "", errors.New("not implemented")
}
func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return nil
}
var _ exec.Interface = fakeExec{}

56
vendor/k8s.io/kubernetes/pkg/util/nsenter/nsenter_unsupported.go generated vendored
View file

@ -1,56 +0,0 @@
// +build !linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"k8s.io/utils/exec"
)
const (
// DefaultHostRootFsPath is path to host's filesystem mounted into container
// with kubelet.
DefaultHostRootFsPath = "/rootfs"
)
// Nsenter is part of experimental support for running the kubelet
// in a container.
type Nsenter struct {
// a map of commands to their paths on the host filesystem
Paths map[string]string
}
// NewNsenter constructs a new instance of Nsenter
func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
return &Nsenter{}, nil
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
return nil
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *Nsenter) AbsHostPath(command string) string {
return ""
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *Nsenter) SupportsSystemd() (string, bool) {
return "", false
}

6
vendor/k8s.io/kubernetes/pkg/util/verify-util-pkg.sh generated vendored
View file

@ -21,7 +21,7 @@ set -o errexit
set -o nounset
set -o pipefail
BASH_DIR=$(dirname "${BASH_SOURCE}")
BASH_DIR=$(dirname "${BASH_SOURCE[0]}")
find_go_files() {
find . -maxdepth 1 -not \( \
@ -34,8 +34,8 @@ find_go_files() {
ret=0
pushd "${BASH_DIR}" > /dev/null
for path in `find_go_files`; do
file=$(basename $path)
for path in $(find_go_files); do
file=$(basename "$path")
echo "Found pkg/util/${file}, but should be moved into util sub-pkgs." 1>&2
ret=1
done

3
vendor/k8s.io/kubernetes/pkg/volume/BUILD generated vendored
View file

@ -21,6 +21,7 @@ go_library(
"//pkg/util/mount:go_default_library",
"//pkg/volume/util/fs:go_default_library",
"//pkg/volume/util/recyclerclient:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
@ -29,9 +30,9 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)

5
vendor/k8s.io/kubernetes/pkg/volume/OWNERS generated vendored
View file

@ -1,11 +1,13 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- saad-ali
- thockin
- jsafrane
- matchstick
- rootfs
- gnufied
- childsb
- msau42
reviewers:
- saad-ali
- jsafrane
@ -15,5 +17,6 @@ reviewers:
- gnufied
- verult
- davidz627
- humblec
labels:
- sig/storage

4
vendor/k8s.io/kubernetes/pkg/volume/noop_expandable_plugin.go generated vendored
View file

@ -48,6 +48,10 @@ func (n *noopExpandableVolumePluginInstance) CanSupport(spec *Spec) bool {
return true
}
func (n *noopExpandableVolumePluginInstance) IsMigratedToCSI() bool {
return false
}
func (n *noopExpandableVolumePluginInstance) RequiresRemount() bool {
return false
}

143
vendor/k8s.io/kubernetes/pkg/volume/plugins.go generated vendored
View file

@ -30,12 +30,13 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/validation"
clientset "k8s.io/client-go/kubernetes"
storagelisters "k8s.io/client-go/listers/storage/v1beta1"
"k8s.io/client-go/tools/record"
cloudprovider "k8s.io/cloud-provider"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume/util/recyclerclient"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
type ProbeOperation uint32
@ -45,6 +46,9 @@ type ProbeEvent struct {
Op ProbeOperation // The operation to the plugin
}
// CSIVolumePhaseType stores information about CSI volume path.
type CSIVolumePhaseType string
const (
// Common parameter which can be specified in StorageClass to specify the desired FSType
// Provisioners SHOULD implement support for this if they are block device based
@ -54,6 +58,8 @@ const (
ProbeAddOrUpdate ProbeOperation = 1 << iota
ProbeRemove
CSIVolumeStaged CSIVolumePhaseType = "staged"
CSIVolumePublished CSIVolumePhaseType = "published"
)
// VolumeOptions contains option information about a volume.
@ -84,6 +90,26 @@ type VolumeOptions struct {
Parameters map[string]string
}
// NodeResizeOptions contain options to be passed for node expansion.
type NodeResizeOptions struct {
VolumeSpec *Spec
// DevicePath - location of actual device on the node. In case of CSI
// this just could be volumeID
DevicePath string
// DeviceMountPath location where device is mounted on the node. If volume type
// is attachable - this would be global mount path otherwise
// it would be location where volume was mounted for the pod
DeviceMountPath string
NewSize resource.Quantity
OldSize resource.Quantity
// CSIVolumePhase contains volume phase on the node
CSIVolumePhase CSIVolumePhaseType
}
type DynamicPluginProber interface {
Init() error
@ -118,6 +144,10 @@ type VolumePlugin interface {
// const.
CanSupport(spec *Spec) bool
// IsMigratedToCSI tests whether a CSIDriver implements this plugin's
// functionality
IsMigratedToCSI() bool
// RequiresRemount returns true if this plugin requires mount calls to be
// reexecuted. Atomically updating volumes, like Downward API, depend on
// this to update the contents of the volume.
@ -135,10 +165,10 @@ type VolumePlugin interface {
NewUnmounter(name string, podUID types.UID) (Unmounter, error)
// ConstructVolumeSpec constructs a volume spec based on the given volume name
// and mountPath. The spec may have incomplete information due to limited
// and volumePath. The spec may have incomplete information due to limited
// information from input. This function is used by volume manager to reconstruct
// volume spec by reading the volume directories from disk
ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error)
ConstructVolumeSpec(volumeName, volumePath string) (*Spec, error)
// SupportsMountOption returns true if volume plugins supports Mount options
// Specifying mount options in a volume plugin that doesn't support
@ -184,12 +214,6 @@ type DeletableVolumePlugin interface {
NewDeleter(spec *Spec) (Deleter, error)
}
const (
// Name of a volume in external cloud that is being provisioned and thus
// should be ignored by rest of Kubernetes.
ProvisionedVolumeName = "placeholder-for-provisioning"
)
// ProvisionableVolumePlugin is an extended interface of VolumePlugin and is
// used to create volumes for the cluster.
type ProvisionableVolumePlugin interface {
@ -206,6 +230,8 @@ type AttachableVolumePlugin interface {
DeviceMountableVolumePlugin
NewAttacher() (Attacher, error)
NewDetacher() (Detacher, error)
// CanAttach tests if provided volume spec is attachable
CanAttach(spec *Spec) bool
}
// DeviceMountableVolumePlugin is an extended interface of VolumePlugin and is used
@ -218,18 +244,20 @@ type DeviceMountableVolumePlugin interface {
}
// ExpandableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that can be
// expanded
// expanded via control-plane ExpandVolumeDevice call.
type ExpandableVolumePlugin interface {
VolumePlugin
ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error)
RequiresFSResize() bool
}
// FSResizableVolumePlugin is an extension of ExpandableVolumePlugin and is used for volumes (flex)
// that require extra steps on nodes for expansion to complete
type FSResizableVolumePlugin interface {
ExpandableVolumePlugin
ExpandFS(spec *Spec, devicePath, deviceMountPath string, newSize, oldSize resource.Quantity) error
// NodeExpandableVolumePlugin is an expanded interface of VolumePlugin and is used for volumes that
// require expansion on the node via NodeExpand call.
type NodeExpandableVolumePlugin interface {
VolumePlugin
RequiresFSResize() bool
// NodeExpand expands volume on given deviceMountPath and returns true if resize is successful.
NodeExpand(resizeOptions NodeResizeOptions) (bool, error)
}
// VolumePluginWithAttachLimits is an extended interface of VolumePlugin that restricts number of
@ -275,7 +303,33 @@ type BlockVolumePlugin interface {
// The spec may have incomplete information due to limited information
// from input. This function is used by volume manager to reconstruct
// volume spec by reading the volume directories from disk.
ConstructBlockVolumeSpec(podUID types.UID, volumeName, mountPath string) (*Spec, error)
ConstructBlockVolumeSpec(podUID types.UID, volumeName, volumePath string) (*Spec, error)
}
// TODO(#14217)
// As part of the Volume Host refactor we are starting to create Volume Hosts
// for specific hosts. New methods for each specific host can be added here.
// Currently consumers will do type assertions to get the specific type of Volume
// Host; however, the end result should be that specific Volume Hosts are passed
// to the specific functions they are needed in (instead of using a catch-all
// VolumeHost interface)
// KubeletVolumeHost is a Kubelet specific interface that plugins can use to access the kubelet.
type KubeletVolumeHost interface {
// SetKubeletError lets plugins set an error on the Kubelet runtime status
// that will cause the Kubelet to post NotReady status with the error message provided
SetKubeletError(err error)
}
// AttachDetachVolumeHost is a AttachDetach Controller specific interface that plugins can use
// to access methods on the Attach Detach Controller.
type AttachDetachVolumeHost interface {
// CSINodeLister returns the informer lister for the CSINode API Object
CSINodeLister() storagelisters.CSINodeLister
// IsAttachDetachController is an interface marker to strictly tie AttachDetachVolumeHost
// to the attachDetachController
IsAttachDetachController() bool
}
// VolumeHost is an interface that plugins can use to access the kubelet.
@ -317,9 +371,6 @@ type VolumeHost interface {
// GetKubeClient returns a client interface
GetKubeClient() clientset.Interface
// GetCSIClient returns a client interface to csi.storage.k8s.io
GetCSIClient() csiclientset.Interface
// NewWrapperMounter finds an appropriate plugin with which to handle
// the provided spec. This is used to implement volume plugins which
// "wrap" other plugins. For example, the "secret" volume is
@ -367,6 +418,9 @@ type VolumeHost interface {
// Returns the event recorder of kubelet.
GetEventRecorder() record.EventRecorder
// Returns an interface that should be used to execute subpath operations
GetSubpather() subpath.Interface
}
// VolumePluginMgr tracks registered plugins.
@ -601,6 +655,37 @@ func (pm *VolumePluginMgr) FindPluginBySpec(spec *Spec) (VolumePlugin, error) {
return matches[0], nil
}
// IsPluginMigratableBySpec looks for a plugin that can support a given volume
// specification and whether that plugin is Migratable. If no plugins can
// support or more than one plugin can support it, return error.
func (pm *VolumePluginMgr) IsPluginMigratableBySpec(spec *Spec) (bool, error) {
pm.mutex.Lock()
defer pm.mutex.Unlock()
if spec == nil {
return false, fmt.Errorf("could not find if plugin is migratable because volume spec is nil")
}
matchedPluginNames := []string{}
matches := []VolumePlugin{}
for k, v := range pm.plugins {
if v.CanSupport(spec) {
matchedPluginNames = append(matchedPluginNames, k)
matches = append(matches, v)
}
}
if len(matches) == 0 {
// Not a known plugin (flex) in which case it is not migratable
return false, nil
}
if len(matches) > 1 {
return false, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ","))
}
return matches[0].IsMigratedToCSI(), nil
}
// FindPluginByName fetches a plugin by name or by legacy name. If no plugin
// is found, returns error.
func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) {
@ -788,7 +873,9 @@ func (pm *VolumePluginMgr) FindAttachablePluginBySpec(spec *Spec) (AttachableVol
return nil, err
}
if attachableVolumePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok {
return attachableVolumePlugin, nil
if attachableVolumePlugin.CanAttach(spec) {
return attachableVolumePlugin, nil
}
}
return nil, nil
}
@ -839,10 +926,10 @@ func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVol
if spec.IsKubeletExpandable() {
// for kubelet expandable volumes, return a noop plugin that
// returns success for expand on the controller
klog.Warningf("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name())
klog.V(4).Infof("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name())
return &noopExpandableVolumePluginInstance{spec}, nil
}
klog.Warningf("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err)
klog.V(4).Infof("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err)
return nil, err
}
@ -891,26 +978,26 @@ func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugi
return nil, nil
}
// FindFSResizablePluginBySpec fetches a persistent volume plugin by spec
func (pm *VolumePluginMgr) FindFSResizablePluginBySpec(spec *Spec) (FSResizableVolumePlugin, error) {
// FindNodeExpandablePluginBySpec fetches a persistent volume plugin by spec
func (pm *VolumePluginMgr) FindNodeExpandablePluginBySpec(spec *Spec) (NodeExpandableVolumePlugin, error) {
volumePlugin, err := pm.FindPluginBySpec(spec)
if err != nil {
return nil, err
}
if fsResizablePlugin, ok := volumePlugin.(FSResizableVolumePlugin); ok {
if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok {
return fsResizablePlugin, nil
}
return nil, nil
}
// FindFSResizablePluginByName fetches a persistent volume plugin by name
func (pm *VolumePluginMgr) FindFSResizablePluginByName(name string) (FSResizableVolumePlugin, error) {
// FindNodeExpandablePluginByName fetches a persistent volume plugin by name
func (pm *VolumePluginMgr) FindNodeExpandablePluginByName(name string) (NodeExpandableVolumePlugin, error) {
volumePlugin, err := pm.FindPluginByName(name)
if err != nil {
return nil, err
}
if fsResizablePlugin, ok := volumePlugin.(FSResizableVolumePlugin); ok {
if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok {
return fsResizablePlugin, nil
}

10
vendor/k8s.io/kubernetes/pkg/volume/util/BUILD generated vendored
View file

@ -9,7 +9,6 @@ go_library(
"device_util_linux.go",
"device_util_unsupported.go",
"doc.go",
"error.go",
"finalizer.go",
"io_util.go",
"metrics.go",
@ -23,10 +22,8 @@ go_library(
"//pkg/api/legacyscheme:go_default_library",
"//pkg/apis/core/v1/helper:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/util/resizefs:go_default_library",
"//pkg/util/strings:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/util/types:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
@ -43,6 +40,7 @@ go_library(
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/strings:go_default_library",
],
)
@ -52,7 +50,6 @@ go_test(
"atomic_writer_test.go",
"attach_limit_test.go",
"device_util_linux_test.go",
"main_test.go",
"nested_volumes_test.go",
"resize_util_test.go",
"util_test.go",
@ -61,8 +58,6 @@ go_test(
deps = [
"//pkg/apis/core/install:go_default_library",
"//pkg/apis/core/v1/helper:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/slice:go_default_library",
"//pkg/volume:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
@ -70,8 +65,6 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/client-go/util/testing:go_default_library",
@ -95,6 +88,7 @@ filegroup(
"//pkg/volume/util/nestedpendingoperations:all-srcs",
"//pkg/volume/util/operationexecutor:all-srcs",
"//pkg/volume/util/recyclerclient:all-srcs",
"//pkg/volume/util/subpath:all-srcs",
"//pkg/volume/util/types:all-srcs",
"//pkg/volume/util/volumepathhandler:all-srcs",
],

2
vendor/k8s.io/kubernetes/pkg/volume/util/OWNERS generated vendored Executable file → Normal file
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- saad-ali
reviewers:

7
vendor/k8s.io/kubernetes/pkg/volume/util/attach_limit.go generated vendored
View file

@ -40,6 +40,13 @@ const (
// GCEVolumeLimitKey stores resource name that will store volume limits for GCE node
GCEVolumeLimitKey = "attachable-volumes-gce-pd"
// CinderVolumeLimitKey contains Volume limit key for Cinder
CinderVolumeLimitKey = "attachable-volumes-cinder"
// DefaultMaxCinderVolumes defines the maximum number of PD Volumes for Cinder
// For Openstack we are keeping this to a high enough value so as depending on backend
// cluster admins can configure it.
DefaultMaxCinderVolumes = 256
// CSIAttachLimitPrefix defines prefix used for CSI volumes
CSIAttachLimitPrefix = "attachable-volumes-csi-"

25
vendor/k8s.io/kubernetes/pkg/volume/util/device_util_linux.go generated vendored
View file

@ -21,10 +21,12 @@ package util
import (
"errors"
"fmt"
"k8s.io/klog"
"os"
"path"
"strconv"
"strings"
"k8s.io/klog"
)
// FindMultipathDeviceForDevice given a device name like /dev/sdx, find the devicemapper parent
@ -98,6 +100,9 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (
sysPath := "/sys/class/iscsi_host"
hostDirs, err := io.ReadDir(sysPath)
if err != nil {
if os.IsNotExist(err) {
return portalHostMap, nil
}
return nil, err
}
for _, hostDir := range hostDirs {
@ -135,7 +140,8 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (
targetNamePath := sessionPath + "/iscsi_session/" + sessionName + "/targetname"
targetName, err := io.ReadFile(targetNamePath)
if err != nil {
return nil, err
klog.Infof("Failed to process session %s, assuming this session is unavailable: %s", sessionName, err)
continue
}
// Ignore hosts that don't matchthe target we were looking for.
@ -147,7 +153,8 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (
// for the iSCSI connection.
dirs2, err := io.ReadDir(sessionPath)
if err != nil {
return nil, err
klog.Infof("Failed to process session %s, assuming this session is unavailable: %s", sessionName, err)
continue
}
for _, dir2 := range dirs2 {
// Skip over files that aren't the connection
@ -164,25 +171,29 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (
addrPath := connectionPath + "/address"
addr, err := io.ReadFile(addrPath)
if err != nil {
return nil, err
klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
continue
}
portPath := connectionPath + "/port"
port, err := io.ReadFile(portPath)
if err != nil {
return nil, err
klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
continue
}
persistentAddrPath := connectionPath + "/persistent_address"
persistentAddr, err := io.ReadFile(persistentAddrPath)
if err != nil {
return nil, err
klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
continue
}
persistentPortPath := connectionPath + "/persistent_port"
persistentPort, err := io.ReadFile(persistentPortPath)
if err != nil {
return nil, err
klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
continue
}
// Add entries to the map for both the current and persistent portals

42
vendor/k8s.io/kubernetes/pkg/volume/util/error.go generated vendored
View file

@ -1,42 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
k8stypes "k8s.io/apimachinery/pkg/types"
)
// DanglingAttachError indicates volume is attached to a different node
// than we expected.
type DanglingAttachError struct {
msg string
CurrentNode k8stypes.NodeName
DevicePath string
}
func (err *DanglingAttachError) Error() string {
return err.msg
}
// NewDanglingError create a new dangling error
func NewDanglingError(msg string, node k8stypes.NodeName, devicePath string) error {
return &DanglingAttachError{
msg: msg,
CurrentNode: node,
DevicePath: devicePath,
}
}

100
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/BUILD generated vendored Normal file
View file

@ -0,0 +1,100 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"subpath.go",
"subpath_linux.go",
"subpath_nsenter.go",
"subpath_unsupported.go",
"subpath_windows.go",
],
importpath = "k8s.io/kubernetes/pkg/volume/util/subpath",
visibility = ["//visibility:public"],
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/mount:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"//conditions:default": [],
}),
)
go_test(
name = "go_default_test",
srcs = [
"subpath_linux_test.go",
"subpath_nsenter_test.go",
"subpath_windows_test.go",
],
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/mount:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

13
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/OWNERS generated vendored Normal file
View file

@ -0,0 +1,13 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- jingxu97
- saad-ali
- jsafrane
- msau42
- andyzhangx
approvers:
- jingxu97
- saad-ali
- jsafrane

92
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath.go generated vendored Normal file
View file

@ -0,0 +1,92 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import "os"
// Interface defines the set of methods all subpathers must implement
type Interface interface {
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory.
CleanSubPaths(poodDir string, volumeName string) error
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
// that's 1) inside given volumePath and 2) immutable after this call.
//
// newHostPath - location of prepared subPath. It should be used instead of
// hostName when running the container.
// cleanupAction - action to run when the container is running or it failed to start.
//
// CleanupAction must be called immediately after the container with given
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
// when the pod finishes.
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
// SafeMakeDir creates subdir within given base. It makes sure that the
// created directory does not escape given base directory mis-using
// symlinks. Note that the function makes sure that it creates the directory
// somewhere under the base, nothing else. E.g. if the directory already
// exists, it may exist outside of the base due to symlinks.
// This method should be used if the directory to create is inside volume
// that's under user control. User must not be able to use symlinks to
// escape the volume to create directories somewhere else.
SafeMakeDir(subdir string, base string, perm os.FileMode) error
}
// Subpath defines the attributes of a subpath
type Subpath struct {
// index of the VolumeMount for this container
VolumeMountIndex int
// Full path to the subpath directory on the host
Path string
// name of the volume that is a valid directory name.
VolumeName string
// Full path to the volume path
VolumePath string
// Path to the pod's directory, including pod UID
PodDir string
// Name of the container
ContainerName string
}
// Compile time-check for all implementers of subpath interface
var _ Interface = &subpath{}
var _ Interface = &FakeSubpath{}
// FakeSubpath is a subpather implementation for testing
type FakeSubpath struct{}
// PrepareSafeSubpath is a fake implementation of PrepareSafeSubpath. Always returns
// newHostPath == subPath.Path
func (fs *FakeSubpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
// CleanSubPaths is a fake implementation of CleanSubPaths. It is a noop
func (fs *FakeSubpath) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
// SafeMakeDir is a fake implementation of SafeMakeDir. It is a noop
func (fs *FakeSubpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}

563
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go generated vendored Normal file
View file

@ -0,0 +1,563 @@
// +build linux
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/mount"
)
const (
// place for subpath mounts
// TODO: pass in directory using kubelet_getters instead
containerSubPathDirectoryName = "volume-subpaths"
// syscall.Openat flags used to traverse directories not following symlinks
nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
// flags for getting file descriptor without following the symlink
openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
)
type subpath struct {
mounter mount.Interface
}
// New returns a subpath.Interface for the current system
func New(mounter mount.Interface) Interface {
return &subpath{
mounter: mounter,
}
}
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(sp.mounter, podDir, volumeName)
}
func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(sp.mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
// This implementation is shared between Linux and NsEnter
func safeOpenSubPath(mounter mount.Interface, subpath Subpath) (int, error) {
if !mount.PathWithinBase(subpath.Path, subpath.VolumePath) {
return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
}
fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
if err != nil {
return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
}
return fd, nil
}
// prepareSubpathTarget creates target for bind-mount of subpath. It returns
// "true" when the target already exists and something is mounted there.
// Given Subpath must have all paths with already resolved symlinks and with
// paths relevant to kubelet (when it runs in a container).
// This function is called also by NsEnterMounter. It works because
// /var/lib/kubelet is mounted from the host into the container with Kubelet as
// /var/lib/kubelet too.
func prepareSubpathTarget(mounter mount.Interface, subpath Subpath) (bool, string, error) {
// Early check for already bind-mounted subpath.
bindPathTarget := getSubpathBindTarget(subpath)
notMount, err := mounter.IsNotMountPoint(bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
return true, bindPathTarget, nil
}
// bindPathTarget is in /var/lib/kubelet and thus reachable without any
// translation even to containerized kubelet.
bindParent := filepath.Dir(bindPathTarget)
err = os.MkdirAll(bindParent, 0750)
if err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
}
t, err := os.Lstat(subpath.Path)
if err != nil {
return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindPathTarget>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
return false, bindPathTarget, nil
}
func getSubpathBindTarget(subpath Subpath) string {
// containerName is DNS label, i.e. safe as a directory name.
return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
}
func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs on the host:
// - safely open the subpath
// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
// Evaluate all symlinks here once for all subsequent functions.
newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
newPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
subpath.VolumePath = newVolumePath
subpath.Path = newPath
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
kubeletPid := os.Getpid()
mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
// Do the bind mount
options := []string{"bind"}
klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// This implementation is shared between Linux and NsEnter
func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string) error {
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
containerDirs, err := ioutil.ReadDir(subPathDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("error reading %s: %s", subPathDir, err)
}
for _, containerDir := range containerDirs {
if !containerDir.IsDir() {
klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
continue
}
klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
if path == fullContainerDirPath {
// Skip top level directory
return nil
}
// pass through errors and let doCleanSubPath handle them
if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
return err
}
return nil
})
if err != nil {
return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
}
// Whole container has been processed, remove its directory.
if err := os.Remove(fullContainerDirPath); err != nil {
return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
}
klog.V(5).Infof("Removed %s", fullContainerDirPath)
}
// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
if err := os.Remove(subPathDir); err != nil {
return fmt.Errorf("error deleting %s: %s", subPathDir, err)
}
klog.V(5).Infof("Removed %s", subPathDir)
// Remove entire subpath directory if it's the last one
podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
}
klog.V(5).Infof("Removed %s", podSubPathDir)
return nil
}
// doCleanSubPath tears down the single subpath bind mount
func doCleanSubPath(mounter mount.Interface, fullContainerDirPath, subPathIndex string) error {
// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
if err := mount.CleanupMountPoint(fullSubPath, mounter, true); err != nil {
return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
}
klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
return nil
}
// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
func cleanSubPath(mounter mount.Interface, subpath Subpath) error {
containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
// Clean subdir bindmount
if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
return err
}
// Recusively remove directories if empty
if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
return err
}
return nil
}
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error {
if !mount.PathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
}
for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
s, err := os.Stat(curDir)
if err != nil {
if os.IsNotExist(err) {
klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
continue
}
return fmt.Errorf("error stat %q: %v", curDir, err)
}
if !s.IsDir() {
return fmt.Errorf("path %q not a directory", curDir)
}
err = os.Remove(curDir)
if os.IsExist(err) {
klog.V(5).Infof("Directory %q not empty, not removing", curDir)
break
} else if err != nil {
return fmt.Errorf("error removing directory %q: %v", curDir, err)
}
klog.V(5).Infof("Removed directory %q", curDir)
}
return nil
}
// This implementation is shared between Linux and NsEnterMounter. Both pathname
// and base must be either already resolved symlinks or thet will be resolved in
// kubelet's mount namespace (in case it runs containerized).
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !mount.PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err)
}
if !mount.PathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
parentFD, err := doSafeOpen(fullExistingPath, base)
if err != nil {
return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
}
childFD := -1
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
}
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink.
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
if err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
// Dive into the created directory
childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
if err != nil {
return fmt.Errorf("cannot open %s: %s", currentPath, err)
}
// We can be sure that childFD is safe to use. It could be changed
// by user after Mkdirat() and before Openat(), however:
// - it could not be changed to symlink - we use nofollowFlags
// - it could be changed to a file (or device, pipe, socket, ...)
// but either subsequent Mkdirat() fails or we mount this file
// to user's container. Security is no violated in both cases
// and user either gets error or the file that it can already access.
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
parentFD = childFD
childFD = -1
}
// Everything was created. mkdirat(..., perm) above was affected by current
// umask and we must apply the right permissions to the last directory
// (that's the one that will be available to the container as subpath)
// so user can read/write it. This is the behavior of previous code.
// TODO: chmod all created directories, not just the last one.
// parentFD is the last created directory.
// Translate perm (os.FileMode) to uint32 that fchmod() expects
kernelPerm := uint32(perm & os.ModePerm)
if perm&os.ModeSetgid > 0 {
kernelPerm |= syscall.S_ISGID
}
if perm&os.ModeSetuid > 0 {
kernelPerm |= syscall.S_ISUID
}
if perm&os.ModeSticky > 0 {
kernelPerm |= syscall.S_ISVTX
}
if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
return fmt.Errorf("chmod %q failed: %s", currentPath, err)
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
dirs := strings.Split(rel, string(filepath.Separator))
// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
// This should be faster than looping through all dirs and calling os.Stat()
// on each of them, as the symlinks are resolved only once with OpenAt().
currentPath := base
fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
if err != nil {
return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
}
defer func() {
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
}()
for i, dir := range dirs {
// Using O_PATH here will prevent hangs in case user replaces directory with
// fifo
childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
if err != nil {
if os.IsNotExist(err) {
return currentPath, dirs[i:], nil
}
return base, nil, err
}
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
fd = childFD
currentPath = filepath.Join(currentPath, dir)
}
return pathname, []string{}, nil
}
// This implementation is shared between Linux and NsEnterMounter
// Open path and return its fd.
// Symlinks are disallowed (pathname must already resolve symlinks),
// and the path must be within the base directory.
func doSafeOpen(pathname string, base string) (int, error) {
pathname = filepath.Clean(pathname)
base = filepath.Clean(base)
// Calculate segments to follow
subpath, err := filepath.Rel(base, pathname)
if err != nil {
return -1, err
}
segments := strings.Split(subpath, string(filepath.Separator))
// Assumption: base is the only directory that we have under control.
// Base dir is not allowed to be a symlink.
parentFD, err := syscall.Open(base, nofollowFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
}
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
}
}
}()
childFD := -1
defer func() {
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := base
// Follow the segments one by one using openat() to make
// sure the user cannot change already existing directories into symlinks.
for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg)
if !mount.PathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
}
klog.V(5).Infof("Opening path %s", currentPath)
childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
}
var deviceStat unix.Stat_t
err := unix.Fstat(childFD, &deviceStat)
if err != nil {
return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
}
fileFmt := deviceStat.Mode & syscall.S_IFMT
if fileFmt == syscall.S_IFLNK {
return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
}
// Close parentFD
if err = syscall.Close(parentFD); err != nil {
return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
}
// Set child to new parent
parentFD = childFD
childFD = -1
}
// We made it to the end, return this fd, don't close it
finalFD := parentFD
parentFD = -1
return finalFD, nil
}

186
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_nsenter.go generated vendored Normal file
View file

@ -0,0 +1,186 @@
// +build linux
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"fmt"
"os"
"path/filepath"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
"k8s.io/utils/nsenter"
"k8s.io/kubernetes/pkg/util/mount"
)
type subpathNSE struct {
mounter mount.Interface
ne *nsenter.Nsenter
rootDir string
}
// Compile time-check for all implementers of subpath interface
var _ Interface = &subpathNSE{}
// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter
// It is only valid on Linux systems
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return &subpathNSE{
mounter: mounter,
ne: ne,
rootDir: rootDir,
}
}
func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(sp.mounter, podDir, volumeName)
}
func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = sp.doNsEnterBindSubPath(subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(sp.rootDir)
if mount.PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
// create subpaths for paths in /var/lib/kubelet.
return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
}
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := sp.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
}
func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
// - check that we mounted the right thing by comparing device ID and inode
// of the subpath (via safely opened fd) and the target (that's under our
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(sp.mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Leap of faith: optimistically expect that nobody has modified previously
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
// Check that the bind-mount target is the same inode and device as the
// source that we keept open, i.e. we mounted the right thing.
err = checkDeviceInode(fd, bindPathTarget)
if err != nil {
return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// checkDeviceInode checks that opened file and path represent the same file.
func checkDeviceInode(fd int, path string) error {
var srcStat, dstStat unix.Stat_t
err := unix.Fstat(fd, &srcStat)
if err != nil {
return fmt.Errorf("error running fstat on subpath FD: %v", err)
}
err = unix.Stat(path, &dstStat)
if err != nil {
return fmt.Errorf("error running fstat on %s: %v", path, err)
}
if srcStat.Dev != dstStat.Dev {
return fmt.Errorf("different device number")
}
if srcStat.Ino != dstStat.Ino {
return fmt.Errorf("different inode")
}
return nil
}

54
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_unsupported.go generated vendored Normal file
View file

@ -0,0 +1,54 @@
// +build !linux,!windows
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"errors"
"os"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/utils/nsenter"
)
type subpath struct{}
var errUnsupported = errors.New("util/subpath on this platform is not supported")
// New returns a subpath.Interface for the current system.
func New(mount.Interface) Interface {
return &subpath{}
}
// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
// OS choices. however, NSEnter is only valid on Linux
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return nil
}
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, errUnsupported
}
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return errUnsupported
}
func (sp *subpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return errUnsupported
}

284
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_windows.go generated vendored Normal file
View file

@ -0,0 +1,284 @@
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/utils/nsenter"
)
type subpath struct{}
// New returns a subpath.Interface for the current system
func New(mount.Interface) Interface {
return &subpath{}
}
// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
// OS choices. however, NSEnter is only valid on Linux
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return nil
}
// check whether hostPath is within volume path
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(hostPath) == 0 {
return []uintptr{}, nil
}
finalSubPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
}
finalVolumePath, err := filepath.EvalSymlinks(volumePath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
}
return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
}
// lock all intermediate subPath directories and check they are all within volumePath
// volumePath & subPath should not contain any symlink, otherwise it will return error
func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(subPath) == 0 {
return []uintptr{}, nil
}
// get relative path to volumePath
relSubPath, err := filepath.Rel(volumePath, subPath)
if err != nil {
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
}
if mount.StartsWithBackstep(relSubPath) {
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
}
if relSubPath == "." {
// volumePath and subPath are equal
return []uintptr{}, nil
}
fileHandles := []uintptr{}
var errorResult error
currentFullPath := volumePath
dirs := strings.Split(relSubPath, string(os.PathSeparator))
for _, dir := range dirs {
// lock intermediate subPath directory first
currentFullPath = filepath.Join(currentFullPath, dir)
handle, err := lockPath(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
break
}
fileHandles = append(fileHandles, handle)
// make sure intermediate subPath directory does not contain symlink any more
stat, err := os.Lstat(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
break
}
if stat.Mode()&os.ModeSymlink != 0 {
errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
break
}
if !mount.PathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break
}
}
return fileHandles, errorResult
}
// unlockPath unlock directories
func unlockPath(fileHandles []uintptr) {
if fileHandles != nil {
for _, handle := range fileHandles {
syscall.CloseHandle(syscall.Handle(handle))
}
}
}
// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
func lockPath(path string) (uintptr, error) {
if len(path) == 0 {
return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return uintptr(syscall.InvalidHandle), err
}
access := uint32(syscall.GENERIC_READ)
sharemode := uint32(syscall.FILE_SHARE_READ)
createmode := uint32(syscall.OPEN_EXISTING)
flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
return uintptr(fd), err
}
// Lock all directories in subPath and check they're not symlinks.
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
// Unlock the directories when the container starts
cleanupAction = func() {
unlockPath(handles)
}
return subPath.Path, cleanupAction, err
}
// No bind-mounts for subpaths are necessary on Windows
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !mount.PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
if len(toCreate) == 0 {
return nil
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
}
fullBasePath, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err)
}
if !mount.PathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
defer unlockPath(fileHandles)
if err != nil {
return err
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink by lock that directory immediately
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
if err := os.Mkdir(currentPath, perm); err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
handle, err := lockPath(currentPath)
if err != nil {
return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
}
defer syscall.CloseHandle(syscall.Handle(handle))
// make sure newly created directory does not contain symlink after lock
stat, err := os.Lstat(currentPath)
if err != nil {
return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
}
if stat.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
}
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
if mount.StartsWithBackstep(rel) {
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
}
if rel == "." {
// base and pathname are equal
return pathname, []string{}, nil
}
dirs := strings.Split(rel, string(filepath.Separator))
parent := base
currentPath := base
for i, dir := range dirs {
parent = currentPath
currentPath = filepath.Join(parent, dir)
if _, err := os.Lstat(currentPath); err != nil {
if os.IsNotExist(err) {
return parent, dirs[i:], nil
}
return base, nil, err
}
}
return pathname, []string{}, nil
}

430
vendor/k8s.io/kubernetes/pkg/volume/util/util.go generated vendored
View file

@ -22,13 +22,16 @@ import (
"os"
"path"
"path/filepath"
"reflect"
"strings"
v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
utypes "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
@ -36,29 +39,14 @@ import (
"k8s.io/kubernetes/pkg/api/legacyscheme"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
"k8s.io/kubernetes/pkg/features"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
"k8s.io/kubernetes/pkg/util/mount"
utilstrings "k8s.io/kubernetes/pkg/util/strings"
"k8s.io/kubernetes/pkg/volume"
"reflect"
"hash/fnv"
"math/rand"
"strconv"
"k8s.io/apimachinery/pkg/api/resource"
utypes "k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/volume/util/types"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
utilstrings "k8s.io/utils/strings"
)
const (
// GB - GigaByte size
GB = 1000 * 1000 * 1000
// GIB - GibiByte size
GIB = 1024 * 1024 * 1024
readyFileName = "ready"
// ControllerManagedAttachAnnotation is the key of the annotation on Node
@ -79,16 +67,6 @@ const (
VolumeDynamicallyCreatedByKey = "kubernetes.io/createdby"
)
// VolumeZoneConfig contains config information about zonal volume.
type VolumeZoneConfig struct {
ZonePresent bool
ZonesPresent bool
ReplicaZoneFromNodePresent bool
Zone string
Zones string
ReplicaZoneFromNode string
}
// IsReady checks for the existence of a regular file
// called 'ready' in the given directory and returns
// true if that file exists.
@ -125,35 +103,6 @@ func SetReady(dir string) {
file.Close()
}
// UnmountPath is a common unmount routine that unmounts the given path and
// deletes the remaining directory if successful.
// TODO: Remove this function and change callers to call mount pkg directly
func UnmountPath(mountPath string, mounter mount.Interface) error {
return mount.CleanupMountPoint(mountPath, mounter, false /* extensiveMountPointCheck */)
}
// UnmountMountPoint is a common unmount routine that unmounts the given path and
// deletes the remaining directory if successful.
// if extensiveMountPointCheck is true
// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
// IsNotMountPoint is more expensive but properly handles bind mounts.
// TODO: Change callers to call mount pkg directly
func UnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMountPointCheck bool) error {
return mount.CleanupMountPoint(mountPath, mounter, extensiveMountPointCheck)
}
// PathExists returns true if the specified path exists.
// TODO: Change callers to call mount pkg directly
func PathExists(path string) (bool, error) {
return mount.PathExists(path)
}
// IsCorruptedMnt return true if err is about corrupted mount point
// TODO: Change callers to call mount pkg directly
func IsCorruptedMnt(err error) bool {
return mount.IsCorruptedMnt(err)
}
// GetSecretForPod locates secret by name in the pod's namespace and returns secret map
func GetSecretForPod(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (map[string]string, error) {
secret := make(map[string]string)
@ -249,174 +198,6 @@ func LoadPodFromFile(filePath string) (*v1.Pod, error) {
return pod, nil
}
// SelectZoneForVolume is a wrapper around SelectZonesForVolume
// to select a single zone for a volume based on parameters
func SelectZoneForVolume(zoneParameterPresent, zonesParameterPresent bool, zoneParameter string, zonesParameter, zonesWithNodes sets.String, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, pvcName string) (string, error) {
zones, err := SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent, zoneParameter, zonesParameter, zonesWithNodes, node, allowedTopologies, pvcName, 1)
if err != nil {
return "", err
}
zone, ok := zones.PopAny()
if !ok {
return "", fmt.Errorf("could not determine a zone to provision volume in")
}
return zone, nil
}
// SelectZonesForVolume selects zones for a volume based on several factors:
// node.zone, allowedTopologies, zone/zones parameters from storageclass,
// zones with active nodes from the cluster. The number of zones = replicas.
func SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent bool, zoneParameter string, zonesParameter, zonesWithNodes sets.String, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, pvcName string, numReplicas uint32) (sets.String, error) {
if zoneParameterPresent && zonesParameterPresent {
return nil, fmt.Errorf("both zone and zones StorageClass parameters must not be used at the same time")
}
var zoneFromNode string
// pick one zone from node if present
if node != nil {
// VolumeScheduling implicit since node is not nil
if zoneParameterPresent || zonesParameterPresent {
return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if VolumeBindingMode is set to WaitForFirstConsumer. Please specify allowedTopologies in StorageClass for constraining zones")
}
// pick node's zone for one of the replicas
var ok bool
zoneFromNode, ok = node.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain]
if !ok {
return nil, fmt.Errorf("%s Label for node missing", kubeletapis.LabelZoneFailureDomain)
}
// if single replica volume and node with zone found, return immediately
if numReplicas == 1 {
return sets.NewString(zoneFromNode), nil
}
}
// pick zone from allowedZones if specified
allowedZones, err := ZonesFromAllowedTopologies(allowedTopologies)
if err != nil {
return nil, err
}
if (len(allowedTopologies) > 0) && (allowedZones.Len() == 0) {
return nil, fmt.Errorf("no matchLabelExpressions with %s key found in allowedTopologies. Please specify matchLabelExpressions with %s key", kubeletapis.LabelZoneFailureDomain, kubeletapis.LabelZoneFailureDomain)
}
if allowedZones.Len() > 0 {
// VolumeScheduling implicit since allowedZones present
if zoneParameterPresent || zonesParameterPresent {
return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if allowedTopologies specified")
}
// scheduler will guarantee if node != null above, zoneFromNode is member of allowedZones.
// so if zoneFromNode != "", we can safely assume it is part of allowedZones.
zones, err := chooseZonesForVolumeIncludingZone(allowedZones, pvcName, zoneFromNode, numReplicas)
if err != nil {
return nil, fmt.Errorf("cannot process zones in allowedTopologies: %v", err)
}
return zones, nil
}
// pick zone from parameters if present
if zoneParameterPresent {
if numReplicas > 1 {
return nil, fmt.Errorf("zone cannot be specified if desired number of replicas for pv is greather than 1. Please specify zones or allowedTopologies to specify desired zones")
}
return sets.NewString(zoneParameter), nil
}
if zonesParameterPresent {
if uint32(zonesParameter.Len()) < numReplicas {
return nil, fmt.Errorf("not enough zones found in zones parameter to provision a volume with %d replicas. Found %d zones, need %d zones", numReplicas, zonesParameter.Len(), numReplicas)
}
// directly choose from zones parameter; no zone from node need to be considered
return ChooseZonesForVolume(zonesParameter, pvcName, numReplicas), nil
}
// pick zone from zones with nodes
if zonesWithNodes.Len() > 0 {
// If node != null (and thus zoneFromNode != ""), zoneFromNode will be member of zonesWithNodes
zones, err := chooseZonesForVolumeIncludingZone(zonesWithNodes, pvcName, zoneFromNode, numReplicas)
if err != nil {
return nil, fmt.Errorf("cannot process zones where nodes exist in the cluster: %v", err)
}
return zones, nil
}
return nil, fmt.Errorf("cannot determine zones to provision volume in")
}
// ZonesFromAllowedTopologies returns a list of zones specified in allowedTopologies
func ZonesFromAllowedTopologies(allowedTopologies []v1.TopologySelectorTerm) (sets.String, error) {
zones := make(sets.String)
for _, term := range allowedTopologies {
for _, exp := range term.MatchLabelExpressions {
if exp.Key == kubeletapis.LabelZoneFailureDomain {
for _, value := range exp.Values {
zones.Insert(value)
}
} else {
return nil, fmt.Errorf("unsupported key found in matchLabelExpressions: %s", exp.Key)
}
}
}
return zones, nil
}
// ZonesSetToLabelValue converts zones set to label value
func ZonesSetToLabelValue(strSet sets.String) string {
return strings.Join(strSet.UnsortedList(), kubeletapis.LabelMultiZoneDelimiter)
}
// ZonesToSet converts a string containing a comma separated list of zones to set
func ZonesToSet(zonesString string) (sets.String, error) {
zones, err := stringToSet(zonesString, ",")
if err != nil {
return nil, fmt.Errorf("error parsing zones %s, must be strings separated by commas: %v", zonesString, err)
}
return zones, nil
}
// LabelZonesToSet converts a PV label value from string containing a delimited list of zones to set
func LabelZonesToSet(labelZonesValue string) (sets.String, error) {
return stringToSet(labelZonesValue, kubeletapis.LabelMultiZoneDelimiter)
}
// StringToSet converts a string containing list separated by specified delimiter to a set
func stringToSet(str, delimiter string) (sets.String, error) {
zonesSlice := strings.Split(str, delimiter)
zonesSet := make(sets.String)
for _, zone := range zonesSlice {
trimmedZone := strings.TrimSpace(zone)
if trimmedZone == "" {
return make(sets.String), fmt.Errorf(
"%q separated list (%q) must not contain an empty string",
delimiter,
str)
}
zonesSet.Insert(trimmedZone)
}
return zonesSet, nil
}
// LabelZonesToList converts a PV label value from string containing a delimited list of zones to list
func LabelZonesToList(labelZonesValue string) ([]string, error) {
return stringToList(labelZonesValue, kubeletapis.LabelMultiZoneDelimiter)
}
// StringToList converts a string containing list separated by specified delimiter to a list
func stringToList(str, delimiter string) ([]string, error) {
zonesSlice := make([]string, 0)
for _, zone := range strings.Split(str, delimiter) {
trimmedZone := strings.TrimSpace(zone)
if trimmedZone == "" {
return nil, fmt.Errorf(
"%q separated list (%q) must not contain an empty string",
delimiter,
str)
}
zonesSlice = append(zonesSlice, trimmedZone)
}
return zonesSlice, nil
}
// CalculateTimeoutForVolume calculates time for a Recycler pod to complete a
// recycle operation. The calculation and return value is either the
// minimumTimeout or the timeoutIncrement per Gi of storage size, whichever is
@ -433,57 +214,6 @@ func CalculateTimeoutForVolume(minimumTimeout, timeoutIncrement int, pv *v1.Pers
return timeout
}
// RoundUpSize calculates how many allocation units are needed to accommodate
// a volume of given size. E.g. when user wants 1500MiB volume, while AWS EBS
// allocates volumes in gibibyte-sized chunks,
// RoundUpSize(1500 * 1024*1024, 1024*1024*1024) returns '2'
// (2 GiB is the smallest allocatable volume that can hold 1500MiB)
func RoundUpSize(volumeSizeBytes int64, allocationUnitBytes int64) int64 {
roundedUp := volumeSizeBytes / allocationUnitBytes
if volumeSizeBytes%allocationUnitBytes > 0 {
roundedUp++
}
return roundedUp
}
// RoundUpToGB rounds up given quantity to chunks of GB
func RoundUpToGB(size resource.Quantity) int64 {
requestBytes := size.Value()
return RoundUpSize(requestBytes, GB)
}
// RoundUpToGiB rounds up given quantity upto chunks of GiB
func RoundUpToGiB(size resource.Quantity) int64 {
requestBytes := size.Value()
return RoundUpSize(requestBytes, GIB)
}
// RoundUpSizeInt calculates how many allocation units are needed to accommodate
// a volume of given size. It returns an int instead of an int64 and an error if
// there's overflow
func RoundUpSizeInt(volumeSizeBytes int64, allocationUnitBytes int64) (int, error) {
roundedUp := RoundUpSize(volumeSizeBytes, allocationUnitBytes)
roundedUpInt := int(roundedUp)
if int64(roundedUpInt) != roundedUp {
return 0, fmt.Errorf("capacity %v is too great, casting results in integer overflow", roundedUp)
}
return roundedUpInt, nil
}
// RoundUpToGBInt rounds up given quantity to chunks of GB. It returns an
// int instead of an int64 and an error if there's overflow
func RoundUpToGBInt(size resource.Quantity) (int, error) {
requestBytes := size.Value()
return RoundUpSizeInt(requestBytes, GB)
}
// RoundUpToGiBInt rounds up given quantity upto chunks of GiB. It returns an
// int instead of an int64 and an error if there's overflow
func RoundUpToGiBInt(size resource.Quantity) (int, error) {
requestBytes := size.Value()
return RoundUpSizeInt(requestBytes, GIB)
}
// GenerateVolumeName returns a PV name with clusterName prefix. The function
// should be used to generate a name of GCE PD or Cinder volume. It basically
// adds "<clusterName>-dynamic-" before the PV name, making sure the resulting
@ -509,148 +239,6 @@ func GetPath(mounter volume.Mounter) (string, error) {
return path, nil
}
// ChooseZoneForVolume implements our heuristics for choosing a zone for volume creation based on the volume name
// Volumes are generally round-robin-ed across all active zones, using the hash of the PVC Name.
// However, if the PVCName ends with `-<integer>`, we will hash the prefix, and then add the integer to the hash.
// This means that a StatefulSet's volumes (`claimname-statefulsetname-id`) will spread across available zones,
// assuming the id values are consecutive.
func ChooseZoneForVolume(zones sets.String, pvcName string) string {
// No zones available, return empty string.
if zones.Len() == 0 {
return ""
}
// We create the volume in a zone determined by the name
// Eventually the scheduler will coordinate placement into an available zone
hash, index := getPVCNameHashAndIndexOffset(pvcName)
// Zones.List returns zones in a consistent order (sorted)
// We do have a potential failure case where volumes will not be properly spread,
// if the set of zones changes during StatefulSet volume creation. However, this is
// probably relatively unlikely because we expect the set of zones to be essentially
// static for clusters.
// Hopefully we can address this problem if/when we do full scheduler integration of
// PVC placement (which could also e.g. avoid putting volumes in overloaded or
// unhealthy zones)
zoneSlice := zones.List()
zone := zoneSlice[(hash+index)%uint32(len(zoneSlice))]
klog.V(2).Infof("Creating volume for PVC %q; chose zone=%q from zones=%q", pvcName, zone, zoneSlice)
return zone
}
// chooseZonesForVolumeIncludingZone is a wrapper around ChooseZonesForVolume that ensures zoneToInclude is chosen
// zoneToInclude can either be empty in which case it is ignored. If non-empty, zoneToInclude is expected to be member of zones.
// numReplicas is expected to be > 0 and <= zones.Len()
func chooseZonesForVolumeIncludingZone(zones sets.String, pvcName, zoneToInclude string, numReplicas uint32) (sets.String, error) {
if numReplicas == 0 {
return nil, fmt.Errorf("invalid number of replicas passed")
}
if uint32(zones.Len()) < numReplicas {
return nil, fmt.Errorf("not enough zones found to provision a volume with %d replicas. Need at least %d distinct zones for a volume with %d replicas", numReplicas, numReplicas, numReplicas)
}
if zoneToInclude != "" && !zones.Has(zoneToInclude) {
return nil, fmt.Errorf("zone to be included: %s needs to be member of set: %v", zoneToInclude, zones)
}
if uint32(zones.Len()) == numReplicas {
return zones, nil
}
if zoneToInclude != "" {
zones.Delete(zoneToInclude)
numReplicas = numReplicas - 1
}
zonesChosen := ChooseZonesForVolume(zones, pvcName, numReplicas)
if zoneToInclude != "" {
zonesChosen.Insert(zoneToInclude)
}
return zonesChosen, nil
}
// ChooseZonesForVolume is identical to ChooseZoneForVolume, but selects a multiple zones, for multi-zone disks.
func ChooseZonesForVolume(zones sets.String, pvcName string, numZones uint32) sets.String {
// No zones available, return empty set.
replicaZones := sets.NewString()
if zones.Len() == 0 {
return replicaZones
}
// We create the volume in a zone determined by the name
// Eventually the scheduler will coordinate placement into an available zone
hash, index := getPVCNameHashAndIndexOffset(pvcName)
// Zones.List returns zones in a consistent order (sorted)
// We do have a potential failure case where volumes will not be properly spread,
// if the set of zones changes during StatefulSet volume creation. However, this is
// probably relatively unlikely because we expect the set of zones to be essentially
// static for clusters.
// Hopefully we can address this problem if/when we do full scheduler integration of
// PVC placement (which could also e.g. avoid putting volumes in overloaded or
// unhealthy zones)
zoneSlice := zones.List()
startingIndex := index * numZones
for index = startingIndex; index < startingIndex+numZones; index++ {
zone := zoneSlice[(hash+index)%uint32(len(zoneSlice))]
replicaZones.Insert(zone)
}
klog.V(2).Infof("Creating volume for replicated PVC %q; chosen zones=%q from zones=%q",
pvcName, replicaZones.UnsortedList(), zoneSlice)
return replicaZones
}
func getPVCNameHashAndIndexOffset(pvcName string) (hash uint32, index uint32) {
if pvcName == "" {
// We should always be called with a name; this shouldn't happen
klog.Warningf("No name defined during volume create; choosing random zone")
hash = rand.Uint32()
} else {
hashString := pvcName
// Heuristic to make sure that volumes in a StatefulSet are spread across zones
// StatefulSet PVCs are (currently) named ClaimName-StatefulSetName-Id,
// where Id is an integer index.
// Note though that if a StatefulSet pod has multiple claims, we need them to be
// in the same zone, because otherwise the pod will be unable to mount both volumes,
// and will be unschedulable. So we hash _only_ the "StatefulSetName" portion when
// it looks like `ClaimName-StatefulSetName-Id`.
// We continue to round-robin volume names that look like `Name-Id` also; this is a useful
// feature for users that are creating statefulset-like functionality without using statefulsets.
lastDash := strings.LastIndexByte(pvcName, '-')
if lastDash != -1 {
statefulsetIDString := pvcName[lastDash+1:]
statefulsetID, err := strconv.ParseUint(statefulsetIDString, 10, 32)
if err == nil {
// Offset by the statefulsetID, so we round-robin across zones
index = uint32(statefulsetID)
// We still hash the volume name, but only the prefix
hashString = pvcName[:lastDash]
// In the special case where it looks like `ClaimName-StatefulSetName-Id`,
// hash only the StatefulSetName, so that different claims on the same StatefulSet
// member end up in the same zone.
// Note that StatefulSetName (and ClaimName) might themselves both have dashes.
// We actually just take the portion after the final - of ClaimName-StatefulSetName.
// For our purposes it doesn't much matter (just suboptimal spreading).
lastDash := strings.LastIndexByte(hashString, '-')
if lastDash != -1 {
hashString = hashString[lastDash+1:]
}
klog.V(2).Infof("Detected StatefulSet-style volume name %q; index=%d", pvcName, index)
}
}
// We hash the (base) volume name, so we don't bias towards the first N zones
h := fnv.New32()
h.Write([]byte(hashString))
hash = h.Sum32()
}
return hash, index
}
// UnmountViaEmptyDir delegates the tear down operation for secret, configmap, git_repo and downwardapi
// to empty_dir
func UnmountViaEmptyDir(dir string, host volume.VolumeHost, volName string, volSpec volume.Spec, podUID utypes.UID) error {
@ -699,16 +287,6 @@ func JoinMountOptions(userOptions []string, systemOptions []string) []string {
return allMountOptions.List()
}
// ValidateZone returns:
// - an error in case zone is an empty string or contains only any combination of spaces and tab characters
// - nil otherwise
func ValidateZone(zone string) error {
if strings.TrimSpace(zone) == "" {
return fmt.Errorf("the provided %q zone is not valid, it's an empty string or contains only spaces and tab characters", zone)
}
return nil
}
// AccessModesContains returns whether the requested mode is contained by modes
func AccessModesContains(modes []v1.PersistentVolumeAccessMode, mode v1.PersistentVolumeAccessMode) bool {
for _, m := range modes {

25
vendor/k8s.io/kubernetes/pkg/volume/volume.go generated vendored
View file

@ -260,28 +260,3 @@ type DeviceUnmounter interface {
// unmounted.
UnmountDevice(deviceMountPath string) error
}
// NewDeletedVolumeInUseError returns a new instance of DeletedVolumeInUseError
// error.
func NewDeletedVolumeInUseError(message string) error {
return deletedVolumeInUseError(message)
}
type deletedVolumeInUseError string
var _ error = deletedVolumeInUseError("")
// IsDeletedVolumeInUse returns true if an error returned from Delete() is
// deletedVolumeInUseError
func IsDeletedVolumeInUse(err error) bool {
switch err.(type) {
case deletedVolumeInUseError:
return true
default:
return false
}
}
func (err deletedVolumeInUseError) Error() string {
return string(err)
}

6
vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go generated vendored
View file

@ -28,8 +28,9 @@ import (
)
const (
rwMask = os.FileMode(0660)
roMask = os.FileMode(0440)
rwMask = os.FileMode(0660)
roMask = os.FileMode(0440)
execMask = os.FileMode(0110)
)
// SetVolumeOwnership modifies the given volume to be owned by
@ -79,6 +80,7 @@ func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error {
if info.IsDir() {
mask |= os.ModeSetgid
mask |= execMask
}
err = os.Chmod(path, info.Mode()|mask)