Update go dependencies

vendor/k8s.io/kubernetes/pkg/BUILD

@@ -17,6 +17,7 @@ filegroup(
         "//pkg/api/persistentvolume:all-srcs",
         "//pkg/api/persistentvolumeclaim:all-srcs",
         "//pkg/api/pod:all-srcs",
+        "//pkg/api/podsecuritypolicy:all-srcs",
         "//pkg/api/ref:all-srcs",
         "//pkg/api/resource:all-srcs",
         "//pkg/api/service:all-srcs",
@@ -24,6 +25,7 @@ filegroup(
         "//pkg/api/testing:all-srcs",
         "//pkg/api/v1/endpoints:all-srcs",
         "//pkg/api/v1/node:all-srcs",
+        "//pkg/api/v1/persistentvolume:all-srcs",
         "//pkg/api/v1/pod:all-srcs",
         "//pkg/api/v1/resource:all-srcs",
         "//pkg/api/v1/service:all-srcs",
@@ -36,7 +38,7 @@ filegroup(
         "//pkg/apis/autoscaling:all-srcs",
         "//pkg/apis/batch:all-srcs",
         "//pkg/apis/certificates:all-srcs",
-        "//pkg/apis/componentconfig:all-srcs",
+        "//pkg/apis/coordination:all-srcs",
         "//pkg/apis/core:all-srcs",
         "//pkg/apis/events:all-srcs",
         "//pkg/apis/extensions:all-srcs",
@@ -56,13 +58,13 @@ filegroup(
         "//pkg/client/informers/informers_generated/internalversion:all-srcs",
         "//pkg/client/leaderelectionconfig:all-srcs",
         "//pkg/client/listers/admissionregistration/internalversion:all-srcs",
-        "//pkg/client/listers/apis/admissionregistration:all-srcs",
         "//pkg/client/listers/apps/internalversion:all-srcs",
         "//pkg/client/listers/authentication/internalversion:all-srcs",
         "//pkg/client/listers/authorization/internalversion:all-srcs",
         "//pkg/client/listers/autoscaling/internalversion:all-srcs",
         "//pkg/client/listers/batch/internalversion:all-srcs",
         "//pkg/client/listers/certificates/internalversion:all-srcs",
+        "//pkg/client/listers/coordination/internalversion:all-srcs",
         "//pkg/client/listers/core/internalversion:all-srcs",
         "//pkg/client/listers/extensions/internalversion:all-srcs",
         "//pkg/client/listers/imagepolicy/internalversion:all-srcs",

vendor/k8s.io/kubernetes/pkg/api/OWNERS

@@ -1,4 +1,15 @@
-approvers:
-- api-approvers
-reviewers:
-- api-reviewers
+# See the OWNERS docs at https://go.k8s.io/owners
+
+filters:
+  ".*":
+    approvers:
+      - api-approvers
+    reviewers:
+      - api-reviewers
+
+  # examples:
+  #   pkg/api/types.go
+  #   pkg/api/*/register.go
+  "([^/]+/)?(register|types)\\.go$":
+    labels:
+      - kind/api-change

vendor/k8s.io/kubernetes/pkg/api/legacyscheme/BUILD

@@ -6,8 +6,8 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/api/legacyscheme",
     visibility = ["//visibility:public"],
     deps = [
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/api/v1/pod/BUILD

@@ -11,9 +11,9 @@ go_library(
     srcs = ["util.go"],
     importpath = "k8s.io/kubernetes/pkg/api/v1/pod",
     deps = [
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
     ],
 )
 
@@ -22,12 +22,12 @@ go_test(
     srcs = ["util_test.go"],
     embed = [":go_default_library"],
     deps = [
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/api/v1/pod/util.go

@@ -209,14 +209,10 @@ func GetContainerStatus(statuses []v1.ContainerStatus, name string) (v1.Containe
 }
 
 // GetExistingContainerStatus extracts the status of container "name" from "statuses",
-// and returns empty status if "name" does not exist.
+// It also returns if "name" exists.
 func GetExistingContainerStatus(statuses []v1.ContainerStatus, name string) v1.ContainerStatus {
-	for i := range statuses {
-		if statuses[i].Name == name {
-			return statuses[i]
-		}
-	}
-	return v1.ContainerStatus{}
+	status, _ := GetContainerStatus(statuses, name)
+	return status
 }
 
 // IsPodAvailable returns true if a pod is available; false otherwise.

vendor/k8s.io/kubernetes/pkg/apis/OWNERS

@@ -1,44 +1,55 @@
-approvers:
-- erictune
-- lavalamp
-- smarterclayton
-- thockin
-- liggitt
-# - bgrant0607 # manual escalations only
-reviewers:
-- lavalamp
-- smarterclayton
-- wojtek-t
-- deads2k
-- yujuhong
-- brendandburns
-- derekwaynecarr
-- caesarxuchao
-- vishh
-- mikedanese
-- liggitt
-- nikhiljindal
-- gmarek
-- erictune
-- pmorie
-- sttts
-- dchen1107
-- saad-ali
-- luxas
-- janetkuo
-- justinsb
-- pwittrock
-- ncdc
-- tallclair
-- yifan-gu
-- eparis
-- mwielgus
-- feiskyer
-- soltysh
-- piosz
-- dims
-- errordeveloper
-- madhusudancs
-- krousey
-- rootfs
-- jszczepkowski
+# See the OWNERS docs at https://go.k8s.io/owners
+
+filters:
+  ".*":
+    approvers:
+    - erictune
+    - lavalamp
+    - smarterclayton
+    - thockin
+    - liggitt
+    # - bgrant0607 # manual escalations only
+    reviewers:
+    - lavalamp
+    - smarterclayton
+    - wojtek-t
+    - deads2k
+    - yujuhong
+    - brendandburns
+    - derekwaynecarr
+    - caesarxuchao
+    - vishh
+    - mikedanese
+    - liggitt
+    - nikhiljindal
+    - gmarek
+    - erictune
+    - pmorie
+    - sttts
+    - dchen1107
+    - saad-ali
+    - luxas
+    - janetkuo
+    - justinsb
+    - pwittrock
+    - ncdc
+    - tallclair
+    - yifan-gu
+    - eparis
+    - mwielgus
+    - feiskyer
+    - soltysh
+    - piosz
+    - dims
+    - errordeveloper
+    - madhusudancs
+    - krousey
+    - rootfs
+    - jszczepkowski
+
+  # examples:
+  #   pkg/apis/*/types.go
+  #   pkg/apis/*/*/types.go
+  "[^/]+/([^/]+/)?(register|types)\\.go$":
+    labels:
+      - kind/api-change

vendor/k8s.io/kubernetes/pkg/apis/autoscaling/BUILD

@@ -17,10 +17,10 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/apis/autoscaling",
     deps = [
         "//pkg/apis/core:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
     ],
 )
 
@@ -39,6 +39,7 @@ filegroup(
         "//pkg/apis/autoscaling/install:all-srcs",
         "//pkg/apis/autoscaling/v1:all-srcs",
         "//pkg/apis/autoscaling/v2beta1:all-srcs",
+        "//pkg/apis/autoscaling/v2beta2:all-srcs",
         "//pkg/apis/autoscaling/validation:all-srcs",
     ],
     tags = ["automanaged"],

vendor/k8s.io/kubernetes/pkg/apis/autoscaling/types.go

@@ -154,13 +154,9 @@ type MetricSpec struct {
 // ObjectMetricSource indicates how to scale on a metric describing a
 // kubernetes object (for example, hits-per-second on an Ingress object).
 type ObjectMetricSource struct {
-	// Target is the described Kubernetes object.
-	Target CrossVersionObjectReference
-
-	// MetricName is the name of the metric in question.
-	MetricName string
-	// TargetValue is the target value of the metric (as a quantity).
-	TargetValue resource.Quantity
+	DescribedObject CrossVersionObjectReference
+	Target          MetricTarget
+	Metric          MetricIdentifier
 }
 
 // PodsMetricSource indicates how to scale on a metric describing each pod in
@@ -168,11 +164,10 @@ type ObjectMetricSource struct {
 // The values will be averaged together before being compared to the target
 // value.
 type PodsMetricSource struct {
-	// MetricName is the name of the metric in question
-	MetricName string
-	// TargetAverageValue is the target value of the average of the
-	// metric across all relevant pods (as a quantity)
-	TargetAverageValue resource.Quantity
+	// metric identifies the target metric by name and selector
+	Metric MetricIdentifier
+	// target specifies the target value for the given metric
+	Target MetricTarget
 }
 
 // ResourceMetricSource indicates how to scale on a resource metric known to
@@ -185,38 +180,57 @@ type PodsMetricSource struct {
 type ResourceMetricSource struct {
 	// Name is the name of the resource in question.
 	Name api.ResourceName
-	// TargetAverageUtilization is the target value of the average of the
-	// resource metric across all relevant pods, represented as a percentage of
-	// the requested value of the resource for the pods.
-	// +optional
-	TargetAverageUtilization *int32
-	// TargetAverageValue is the target value of the average of the
-	// resource metric across all relevant pods, as a raw value (instead of as
-	// a percentage of the request), similar to the "pods" metric source type.
-	// +optional
-	TargetAverageValue *resource.Quantity
+	// Target specifies the target value for the given metric
+	Target MetricTarget
 }
 
 // ExternalMetricSource indicates how to scale on a metric not associated with
 // any Kubernetes object (for example length of queue in cloud
 // messaging service, or QPS from loadbalancer running outside of cluster).
 type ExternalMetricSource struct {
-	// metricName is the name of the metric in question.
-	MetricName string
-	// MetricSelector is used to identify a specific time series
-	// within a given metric.
-	// +optional
-	MetricSelector *metav1.LabelSelector
-	// TargetValue is the target value of the metric (as a quantity).
-	// Mutually exclusive with TargetAverageValue.
-	// +optional
-	TargetValue *resource.Quantity
-	// TargetAverageValue is the target per-pod value of global metric (as a quantity).
-	// Mutually exclusive with TargetValue.
-	// +optional
-	TargetAverageValue *resource.Quantity
+	// Metric identifies the target metric by name and selector
+	Metric MetricIdentifier
+	// Target specifies the target value for the given metric
+	Target MetricTarget
 }
 
+// MetricIdentifier defines the name and optionally selector for a metric
+type MetricIdentifier struct {
+	// Name is the name of the given metric
+	Name string
+	// Selector is the selector for the given metric
+	// it is the string-encoded form of a standard kubernetes label selector
+	// +optional
+	Selector *metav1.LabelSelector
+}
+
+// MetricTarget defines the target value, average value, or average utilization of a specific metric
+type MetricTarget struct {
+	// Type represents whether the metric type is Utilization, Value, or AverageValue
+	Type MetricTargetType
+	// Value is the target value of the metric (as a quantity).
+	Value *resource.Quantity
+	// TargetAverageValue is the target value of the average of the
+	// metric across all relevant pods (as a quantity)
+	AverageValue *resource.Quantity
+
+	// AverageUtilization is the target value of the average of the
+	// resource metric across all relevant pods, represented as a percentage of
+	// the requested value of the resource for the pods.
+	// Currently only valid for Resource metric source type
+	AverageUtilization *int32
+}
+
+// MetricTargetType specifies the type of metric being targeted, and should be either
+// "Value", "AverageValue", or "Utilization"
+type MetricTargetType string
+
+var (
+	UtilizationMetricType  MetricTargetType = "Utilization"
+	ValueMetricType        MetricTargetType = "Value"
+	AverageValueMetricType MetricTargetType = "AverageValue"
+)
+
 // HorizontalPodAutoscalerStatus describes the current status of a horizontal pod autoscaler.
 type HorizontalPodAutoscalerStatus struct {
 	// ObservedGeneration is the most recent generation observed by this autoscaler.
@@ -237,6 +251,7 @@ type HorizontalPodAutoscalerStatus struct {
 	DesiredReplicas int32
 
 	// CurrentMetrics is the last read state of the metrics used by this autoscaler.
+	// +optional
 	CurrentMetrics []MetricStatus
 
 	// Conditions is the set of conditions required for this autoscaler to scale its target,
@@ -327,23 +342,17 @@ type MetricStatus struct {
 // ObjectMetricStatus indicates the current value of a metric describing a
 // kubernetes object (for example, hits-per-second on an Ingress object).
 type ObjectMetricStatus struct {
-	// Target is the described Kubernetes object.
-	Target CrossVersionObjectReference
+	Metric  MetricIdentifier
+	Current MetricValueStatus
 
-	// MetricName is the name of the metric in question.
-	MetricName string
-	// CurrentValue is the current value of the metric (as a quantity).
-	CurrentValue resource.Quantity
+	DescribedObject CrossVersionObjectReference
 }
 
 // PodsMetricStatus indicates the current value of a metric describing each pod in
 // the current scale target (for example, transactions-processed-per-second).
 type PodsMetricStatus struct {
-	// MetricName is the name of the metric in question
-	MetricName string
-	// CurrentAverageValue is the current value of the average of the
-	// metric across all relevant pods (as a quantity)
-	CurrentAverageValue resource.Quantity
+	Metric  MetricIdentifier
+	Current MetricValueStatus
 }
 
 // ResourceMetricStatus indicates the current value of a resource metric known to
@@ -353,36 +362,21 @@ type PodsMetricStatus struct {
 // normal per-pod metrics using the "pods" source.
 type ResourceMetricStatus struct {
 	// Name is the name of the resource in question.
-	Name api.ResourceName
-	// CurrentAverageUtilization is the current value of the average of the
-	// resource metric across all relevant pods, represented as a percentage of
-	// the requested value of the resource for the pods.  It will only be
-	// present if `targetAverageValue` was set in the corresponding metric
-	// specification.
-	// +optional
-	CurrentAverageUtilization *int32
-	// CurrentAverageValue is the current value of the average of the
-	// resource metric across all relevant pods, as a raw value (instead of as
-	// a percentage of the request), similar to the "pods" metric source type.
-	// It will always be set, regardless of the corresponding metric specification.
-	CurrentAverageValue resource.Quantity
+	Name    api.ResourceName
+	Current MetricValueStatus
 }
 
 // ExternalMetricStatus indicates the current value of a global metric
 // not associated with any Kubernetes object.
 type ExternalMetricStatus struct {
-	// MetricName is the name of a metric used for autoscaling in
-	// metric system.
-	MetricName string
-	// MetricSelector is used to identify a specific time series
-	// within a given metric.
-	// +optional
-	MetricSelector *metav1.LabelSelector
-	// CurrentValue is the current value of the metric (as a quantity)
-	CurrentValue resource.Quantity
-	// CurrentAverageValue is the current value of metric averaged over autoscaled pods.
-	// +optional
-	CurrentAverageValue *resource.Quantity
+	Metric  MetricIdentifier
+	Current MetricValueStatus
+}
+
+type MetricValueStatus struct {
+	Value              *resource.Quantity
+	AverageValue       *resource.Quantity
+	AverageUtilization *int32
 }
 
 // +genclient

vendor/k8s.io/kubernetes/pkg/apis/autoscaling/zz_generated.deepcopy.go

@@ -44,33 +44,8 @@ func (in *CrossVersionObjectReference) DeepCopy() *CrossVersionObjectReference {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ExternalMetricSource) DeepCopyInto(out *ExternalMetricSource) {
 	*out = *in
-	if in.MetricSelector != nil {
-		in, out := &in.MetricSelector, &out.MetricSelector
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.LabelSelector)
-			(*in).DeepCopyInto(*out)
-		}
-	}
-	if in.TargetValue != nil {
-		in, out := &in.TargetValue, &out.TargetValue
-		if *in == nil {
-			*out = nil
-		} else {
-			x := (*in).DeepCopy()
-			*out = &x
-		}
-	}
-	if in.TargetAverageValue != nil {
-		in, out := &in.TargetAverageValue, &out.TargetAverageValue
-		if *in == nil {
-			*out = nil
-		} else {
-			x := (*in).DeepCopy()
-			*out = &x
-		}
-	}
+	in.Metric.DeepCopyInto(&out.Metric)
+	in.Target.DeepCopyInto(&out.Target)
 	return
 }
 
@@ -87,25 +62,8 @@ func (in *ExternalMetricSource) DeepCopy() *ExternalMetricSource {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ExternalMetricStatus) DeepCopyInto(out *ExternalMetricStatus) {
 	*out = *in
-	if in.MetricSelector != nil {
-		in, out := &in.MetricSelector, &out.MetricSelector
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.LabelSelector)
-			(*in).DeepCopyInto(*out)
-		}
-	}
-	out.CurrentValue = in.CurrentValue.DeepCopy()
-	if in.CurrentAverageValue != nil {
-		in, out := &in.CurrentAverageValue, &out.CurrentAverageValue
-		if *in == nil {
-			*out = nil
-		} else {
-			x := (*in).DeepCopy()
-			*out = &x
-		}
-	}
+	in.Metric.DeepCopyInto(&out.Metric)
+	in.Current.DeepCopyInto(&out.Current)
 	return
 }
 
@@ -203,12 +161,8 @@ func (in *HorizontalPodAutoscalerSpec) DeepCopyInto(out *HorizontalPodAutoscaler
 	out.ScaleTargetRef = in.ScaleTargetRef
 	if in.MinReplicas != nil {
 		in, out := &in.MinReplicas, &out.MinReplicas
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
+		*out = new(int32)
+		**out = **in
 	}
 	if in.Metrics != nil {
 		in, out := &in.Metrics, &out.Metrics
@@ -235,20 +189,12 @@ func (in *HorizontalPodAutoscalerStatus) DeepCopyInto(out *HorizontalPodAutoscal
 	*out = *in
 	if in.ObservedGeneration != nil {
 		in, out := &in.ObservedGeneration, &out.ObservedGeneration
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int64)
-			**out = **in
-		}
+		*out = new(int64)
+		**out = **in
 	}
 	if in.LastScaleTime != nil {
 		in, out := &in.LastScaleTime, &out.LastScaleTime
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = (*in).DeepCopy()
-		}
+		*out = (*in).DeepCopy()
 	}
 	if in.CurrentMetrics != nil {
 		in, out := &in.CurrentMetrics, &out.CurrentMetrics
@@ -277,44 +223,49 @@ func (in *HorizontalPodAutoscalerStatus) DeepCopy() *HorizontalPodAutoscalerStat
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricIdentifier) DeepCopyInto(out *MetricIdentifier) {
+	*out = *in
+	if in.Selector != nil {
+		in, out := &in.Selector, &out.Selector
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricIdentifier.
+func (in *MetricIdentifier) DeepCopy() *MetricIdentifier {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricIdentifier)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MetricSpec) DeepCopyInto(out *MetricSpec) {
 	*out = *in
 	if in.Object != nil {
 		in, out := &in.Object, &out.Object
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(ObjectMetricSource)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(ObjectMetricSource)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Pods != nil {
 		in, out := &in.Pods, &out.Pods
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(PodsMetricSource)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(PodsMetricSource)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Resource != nil {
 		in, out := &in.Resource, &out.Resource
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(ResourceMetricSource)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(ResourceMetricSource)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.External != nil {
 		in, out := &in.External, &out.External
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(ExternalMetricSource)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(ExternalMetricSource)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -334,39 +285,23 @@ func (in *MetricStatus) DeepCopyInto(out *MetricStatus) {
 	*out = *in
 	if in.Object != nil {
 		in, out := &in.Object, &out.Object
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(ObjectMetricStatus)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(ObjectMetricStatus)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Pods != nil {
 		in, out := &in.Pods, &out.Pods
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(PodsMetricStatus)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(PodsMetricStatus)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Resource != nil {
 		in, out := &in.Resource, &out.Resource
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(ResourceMetricStatus)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(ResourceMetricStatus)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.External != nil {
 		in, out := &in.External, &out.External
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(ExternalMetricStatus)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(ExternalMetricStatus)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -381,11 +316,74 @@ func (in *MetricStatus) DeepCopy() *MetricStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricTarget) DeepCopyInto(out *MetricTarget) {
+	*out = *in
+	if in.Value != nil {
+		in, out := &in.Value, &out.Value
+		x := (*in).DeepCopy()
+		*out = &x
+	}
+	if in.AverageValue != nil {
+		in, out := &in.AverageValue, &out.AverageValue
+		x := (*in).DeepCopy()
+		*out = &x
+	}
+	if in.AverageUtilization != nil {
+		in, out := &in.AverageUtilization, &out.AverageUtilization
+		*out = new(int32)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricTarget.
+func (in *MetricTarget) DeepCopy() *MetricTarget {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricTarget)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricValueStatus) DeepCopyInto(out *MetricValueStatus) {
+	*out = *in
+	if in.Value != nil {
+		in, out := &in.Value, &out.Value
+		x := (*in).DeepCopy()
+		*out = &x
+	}
+	if in.AverageValue != nil {
+		in, out := &in.AverageValue, &out.AverageValue
+		x := (*in).DeepCopy()
+		*out = &x
+	}
+	if in.AverageUtilization != nil {
+		in, out := &in.AverageUtilization, &out.AverageUtilization
+		*out = new(int32)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricValueStatus.
+func (in *MetricValueStatus) DeepCopy() *MetricValueStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricValueStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ObjectMetricSource) DeepCopyInto(out *ObjectMetricSource) {
 	*out = *in
-	out.Target = in.Target
-	out.TargetValue = in.TargetValue.DeepCopy()
+	out.DescribedObject = in.DescribedObject
+	in.Target.DeepCopyInto(&out.Target)
+	in.Metric.DeepCopyInto(&out.Metric)
 	return
 }
 
@@ -402,8 +400,9 @@ func (in *ObjectMetricSource) DeepCopy() *ObjectMetricSource {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ObjectMetricStatus) DeepCopyInto(out *ObjectMetricStatus) {
 	*out = *in
-	out.Target = in.Target
-	out.CurrentValue = in.CurrentValue.DeepCopy()
+	in.Metric.DeepCopyInto(&out.Metric)
+	in.Current.DeepCopyInto(&out.Current)
+	out.DescribedObject = in.DescribedObject
 	return
 }
 
@@ -420,7 +419,8 @@ func (in *ObjectMetricStatus) DeepCopy() *ObjectMetricStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PodsMetricSource) DeepCopyInto(out *PodsMetricSource) {
 	*out = *in
-	out.TargetAverageValue = in.TargetAverageValue.DeepCopy()
+	in.Metric.DeepCopyInto(&out.Metric)
+	in.Target.DeepCopyInto(&out.Target)
 	return
 }
 
@@ -437,7 +437,8 @@ func (in *PodsMetricSource) DeepCopy() *PodsMetricSource {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PodsMetricStatus) DeepCopyInto(out *PodsMetricStatus) {
 	*out = *in
-	out.CurrentAverageValue = in.CurrentAverageValue.DeepCopy()
+	in.Metric.DeepCopyInto(&out.Metric)
+	in.Current.DeepCopyInto(&out.Current)
 	return
 }
 
@@ -454,24 +455,7 @@ func (in *PodsMetricStatus) DeepCopy() *PodsMetricStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ResourceMetricSource) DeepCopyInto(out *ResourceMetricSource) {
 	*out = *in
-	if in.TargetAverageUtilization != nil {
-		in, out := &in.TargetAverageUtilization, &out.TargetAverageUtilization
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
-	}
-	if in.TargetAverageValue != nil {
-		in, out := &in.TargetAverageValue, &out.TargetAverageValue
-		if *in == nil {
-			*out = nil
-		} else {
-			x := (*in).DeepCopy()
-			*out = &x
-		}
-	}
+	in.Target.DeepCopyInto(&out.Target)
 	return
 }
 
@@ -488,16 +472,7 @@ func (in *ResourceMetricSource) DeepCopy() *ResourceMetricSource {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ResourceMetricStatus) DeepCopyInto(out *ResourceMetricStatus) {
 	*out = *in
-	if in.CurrentAverageUtilization != nil {
-		in, out := &in.CurrentAverageUtilization, &out.CurrentAverageUtilization
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
-	}
-	out.CurrentAverageValue = in.CurrentAverageValue.DeepCopy()
+	in.Current.DeepCopyInto(&out.Current)
 	return
 }
 

vendor/k8s.io/kubernetes/pkg/apis/core/BUILD

@@ -18,13 +18,13 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/apis/core",
     visibility = ["//visibility:public"],
     deps = [
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/internalversion:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/internalversion:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/core/OWNERS

@@ -41,3 +41,5 @@ reviewers:
 - piosz
 - jsafrane
 - jbeda
+labels:
+- sig/apps

vendor/k8s.io/kubernetes/pkg/apis/core/helper/BUILD

@@ -12,8 +12,8 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
     ],
 )
 
@@ -23,14 +23,14 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/apis/core/helper",
     deps = [
         "//pkg/apis/core:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/selection:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/selection:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/core/helper/helpers.go

@@ -56,7 +56,7 @@ func HugePageResourceName(pageSize resource.Quantity) core.ResourceName {
 // an error is returned.
 func HugePageSizeFromResourceName(name core.ResourceName) (resource.Quantity, error) {
 	if !IsHugePageResourceName(name) {
-		return resource.Quantity{}, fmt.Errorf("resource name: %s is not valid hugepage name", name)
+		return resource.Quantity{}, fmt.Errorf("resource name: %s is an invalid hugepage name", name)
 	}
 	pageSize := strings.TrimPrefix(string(name), core.ResourceHugePagesPrefix)
 	return resource.ParseQuantity(pageSize)

vendor/k8s.io/kubernetes/pkg/apis/core/install/BUILD

@@ -14,8 +14,8 @@ go_library(
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
     ],
 )
 
@@ -26,10 +26,10 @@ go_test(
     deps = [
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/apis/core:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/core/types.go

@@ -35,6 +35,8 @@ const (
 	NamespaceSystem string = "kube-system"
 	// NamespacePublic is the namespace where we place public info (ConfigMaps)
 	NamespacePublic string = "kube-public"
+	// NamespaceNodeLease is the namespace where we place node lease objects (used for node heartbeats)
+	NamespaceNodeLease string = "kube-node-lease"
 	// TerminationMessagePathDefault means the default path to capture the application termination message running in a container
 	TerminationMessagePathDefault string = "/dev/termination-log"
 )
@@ -411,6 +413,16 @@ type PersistentVolumeClaimSpec struct {
 	// This is an alpha feature and may change in the future.
 	// +optional
 	VolumeMode *PersistentVolumeMode
+	// This field requires the VolumeSnapshotDataSource alpha feature gate to be
+	// enabled and currently VolumeSnapshot is the only supported data source.
+	// If the provisioner can support VolumeSnapshot data source, it will create
+	// a new volume and data will be restored to the volume at the same time.
+	// If the provisioner does not support VolumeSnapshot data source, volume will
+	// not be created and the failure will be reported as an event.
+	// In the future, we plan to support more data source types and the behavior
+	// of the provisioner may change.
+	// +optional
+	DataSource *TypedLocalObjectReference
 }
 
 type PersistentVolumeClaimConditionType string
@@ -562,6 +574,8 @@ const (
 	ProtocolTCP Protocol = "TCP"
 	// ProtocolUDP is the UDP protocol.
 	ProtocolUDP Protocol = "UDP"
+	// ProtocolSCTP is the SCTP protocol.
+	ProtocolSCTP Protocol = "SCTP"
 )
 
 // Represents a Persistent Disk resource in Google Compute Engine.
@@ -1264,6 +1278,7 @@ type ScaleIOVolumeSource struct {
 	// +optional
 	StoragePool string
 	// Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned.
+	// Default is ThinProvisioned.
 	// +optional
 	StorageMode string
 	// The name of a volume already created in the ScaleIO system
@@ -1271,7 +1286,8 @@ type ScaleIOVolumeSource struct {
 	VolumeName string
 	// Filesystem type to mount.
 	// Must be a filesystem type supported by the host operating system.
-	// Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified.
+	// Ex. "ext4", "xfs", "ntfs".
+	// Default is "xfs".
 	// +optional
 	FSType string
 	// Defaults to false (read/write). ReadOnly here will force
@@ -1300,6 +1316,7 @@ type ScaleIOPersistentVolumeSource struct {
 	// +optional
 	StoragePool string
 	// Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned.
+	// Default is ThinProvisioned.
 	// +optional
 	StorageMode string
 	// The name of a volume created in the ScaleIO system
@@ -1307,7 +1324,8 @@ type ScaleIOPersistentVolumeSource struct {
 	VolumeName string
 	// Filesystem type to mount.
 	// Must be a filesystem type supported by the host operating system.
-	// Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified.
+	// Ex. "ext4", "xfs", "ntfs".
+	// Default is "xfs".
 	// +optional
 	FSType string
 	// Defaults to false (read/write). ReadOnly here will force
@@ -1495,10 +1513,14 @@ type KeyToPath struct {
 type LocalVolumeSource struct {
 	// The full path to the volume on the node.
 	// It can be either a directory or block device (disk, partition, ...).
-	// Directories can be represented only by PersistentVolume with VolumeMode=Filesystem.
-	// Block devices can be represented only by VolumeMode=Block, which also requires the
-	// BlockVolume alpha feature gate to be enabled.
 	Path string
+
+	// Filesystem type to mount.
+	// It applies only when the Path is a block device.
+	// Must be a filesystem type supported by the host operating system.
+	// Ex. "ext4", "xfs", "ntfs". The default value is to auto-select a fileystem if unspecified.
+	// +optional
+	FSType *string
 }
 
 // Represents storage that is managed by an external CSI volume driver (Beta feature)
@@ -1564,7 +1586,7 @@ type ContainerPort struct {
 	HostPort int32
 	// Required: This must be a valid port number, 0 < x < 65536.
 	ContainerPort int32
-	// Required: Supports "TCP" and "UDP".
+	// Required: Supports "TCP", "UDP" and "SCTP"
 	// +optional
 	Protocol Protocol
 	// Optional: What host IP to bind the external port to.
@@ -2567,6 +2589,14 @@ type PodSpec struct {
 	// More info: https://github.com/kubernetes/community/blob/master/keps/sig-network/0007-pod-ready%2B%2B.md
 	// +optional
 	ReadinessGates []PodReadinessGate
+	// RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used
+	// to run this pod.  If no RuntimeClass resource matches the named class, the pod will not be run.
+	// If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an
+	// empty definition that uses the default runtime handler.
+	// More info: https://github.com/kubernetes/community/blob/master/keps/sig-node/0014-runtime-class.md
+	// This is an alpha feature and may change in the future.
+	// +optional
+	RuntimeClassName *string
 }
 
 // HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the
@@ -2609,7 +2639,7 @@ type PodSecurityContext struct {
 	// in the same pod, and the first process in each container will not be assigned PID 1.
 	// HostPID and ShareProcessNamespace cannot both be set.
 	// Optional: Default to false.
-	// This field is alpha-level and is honored only by servers that enable the PodShareProcessNamespace feature.
+	// This field is beta-level and may be disabled with the PodShareProcessNamespace feature.
 	// +k8s:conversion-gen=false
 	// +optional
 	ShareProcessNamespace *bool
@@ -3161,7 +3191,7 @@ type ServicePort struct {
 	// the 'Name' field in EndpointPort objects.
 	Name string
 
-	// The IP protocol for this port.  Supports "TCP" and "UDP".
+	// The IP protocol for this port.  Supports "TCP", "UDP", and "SCTP".
 	Protocol Protocol
 
 	// The port that will be exposed on the service.
@@ -3944,6 +3974,19 @@ type LocalObjectReference struct {
 	Name string
 }
 
+// TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.
+type TypedLocalObjectReference struct {
+	// APIGroup is the group for the resource being referenced.
+	// If APIGroup is not specified, the specified Kind must be in the core API group.
+	// For any other third-party types, APIGroup is required.
+	// +optional
+	APIGroup *string
+	// Kind is the type of resource being referenced
+	Kind string
+	// Name is the name of resource being referenced
+	Name string
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 type SerializedReference struct {
@@ -4580,8 +4623,27 @@ type SecurityContext struct {
 	// the no_new_privs flag will be set on the container process.
 	// +optional
 	AllowPrivilegeEscalation *bool
+	// ProcMount denotes the type of proc mount to use for the containers.
+	// The default is DefaultProcMount which uses the container runtime defaults for
+	// readonly paths and masked paths.
+	// +optional
+	ProcMount *ProcMountType
 }
 
+type ProcMountType string
+
+const (
+	// DefaultProcMount uses the container runtime defaults for readonly and masked
+	// paths for /proc.  Most container runtimes mask certain paths in /proc to avoid
+	// accidental security exposure of special devices or information.
+	DefaultProcMount ProcMountType = "Default"
+
+	// UnmaskedProcMount bypasses the default masking behavior of the container
+	// runtime and ensures the newly created /proc the container stays intact with
+	// no modifications.
+	UnmaskedProcMount ProcMountType = "Unmasked"
+)
+
 // SELinuxOptions are the labels to be applied to the container.
 type SELinuxOptions struct {
 	// SELinux user label

vendor/k8s.io/kubernetes/pkg/apis/core/v1/BUILD

@@ -17,45 +17,45 @@ go_library(
         "//pkg/apis/extensions:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/util/parsers:go_default_library",
-        "//pkg/util/pointer:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )
 
 go_test(
-    name = "go_default_xtest",
+    name = "go_default_test",
     srcs = [
         "conversion_test.go",
         "defaults_test.go",
     ],
+    embed = [":go_default_library"],
     deps = [
-        ":go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/api/testapi:go_default_library",
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/core/fuzzer:go_default_library",
         "//pkg/apis/extensions:go_default_library",
-        "//pkg/util/pointer:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/testing/fuzzer:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/fuzzer:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/fuzzer:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/core/v1/conversion.go

@@ -30,105 +30,6 @@ import (
 	"k8s.io/kubernetes/pkg/apis/extensions"
 )
 
-// This is a "fast-path" that avoids reflection for common types. It focuses on the objects that are
-// converted the most in the cluster.
-// TODO: generate one of these for every external API group - this is to prove the impact
-func addFastPathConversionFuncs(scheme *runtime.Scheme) error {
-	scheme.AddGenericConversionFunc(func(objA, objB interface{}, s conversion.Scope) (bool, error) {
-		switch a := objA.(type) {
-		case *v1.Pod:
-			switch b := objB.(type) {
-			case *core.Pod:
-				return true, Convert_v1_Pod_To_core_Pod(a, b, s)
-			}
-		case *core.Pod:
-			switch b := objB.(type) {
-			case *v1.Pod:
-				return true, Convert_core_Pod_To_v1_Pod(a, b, s)
-			}
-
-		case *v1.Event:
-			switch b := objB.(type) {
-			case *core.Event:
-				return true, Convert_v1_Event_To_core_Event(a, b, s)
-			}
-		case *core.Event:
-			switch b := objB.(type) {
-			case *v1.Event:
-				return true, Convert_core_Event_To_v1_Event(a, b, s)
-			}
-
-		case *v1.ReplicationController:
-			switch b := objB.(type) {
-			case *core.ReplicationController:
-				return true, Convert_v1_ReplicationController_To_core_ReplicationController(a, b, s)
-			}
-		case *core.ReplicationController:
-			switch b := objB.(type) {
-			case *v1.ReplicationController:
-				return true, Convert_core_ReplicationController_To_v1_ReplicationController(a, b, s)
-			}
-
-		case *v1.Node:
-			switch b := objB.(type) {
-			case *core.Node:
-				return true, Convert_v1_Node_To_core_Node(a, b, s)
-			}
-		case *core.Node:
-			switch b := objB.(type) {
-			case *v1.Node:
-				return true, Convert_core_Node_To_v1_Node(a, b, s)
-			}
-
-		case *v1.Namespace:
-			switch b := objB.(type) {
-			case *core.Namespace:
-				return true, Convert_v1_Namespace_To_core_Namespace(a, b, s)
-			}
-		case *core.Namespace:
-			switch b := objB.(type) {
-			case *v1.Namespace:
-				return true, Convert_core_Namespace_To_v1_Namespace(a, b, s)
-			}
-
-		case *v1.Service:
-			switch b := objB.(type) {
-			case *core.Service:
-				return true, Convert_v1_Service_To_core_Service(a, b, s)
-			}
-		case *core.Service:
-			switch b := objB.(type) {
-			case *v1.Service:
-				return true, Convert_core_Service_To_v1_Service(a, b, s)
-			}
-
-		case *v1.Endpoints:
-			switch b := objB.(type) {
-			case *core.Endpoints:
-				return true, Convert_v1_Endpoints_To_core_Endpoints(a, b, s)
-			}
-		case *core.Endpoints:
-			switch b := objB.(type) {
-			case *v1.Endpoints:
-				return true, Convert_core_Endpoints_To_v1_Endpoints(a, b, s)
-			}
-
-		case *metav1.WatchEvent:
-			switch b := objB.(type) {
-			case *metav1.InternalEvent:
-				return true, metav1.Convert_versioned_Event_to_versioned_InternalEvent(a, b, s)
-			}
-		case *metav1.InternalEvent:
-			switch b := objB.(type) {
-			case *metav1.WatchEvent:
-				return true, metav1.Convert_versioned_InternalEvent_to_versioned_Event(a, b, s)
-			}
-		}
-		return false, nil
-	})
-	return nil
-}
-
 func addConversionFuncs(scheme *runtime.Scheme) error {
 	// Add non-generated conversion functions
 	err := scheme.AddConversionFuncs(
@@ -142,19 +43,19 @@ func addConversionFuncs(scheme *runtime.Scheme) error {
 		Convert_v1_Secret_To_core_Secret,
 		Convert_v1_ServiceSpec_To_core_ServiceSpec,
 		Convert_v1_ResourceList_To_core_ResourceList,
-		Convert_v1_ReplicationController_to_extensions_ReplicaSet,
-		Convert_v1_ReplicationControllerSpec_to_extensions_ReplicaSetSpec,
-		Convert_v1_ReplicationControllerStatus_to_extensions_ReplicaSetStatus,
-		Convert_extensions_ReplicaSet_to_v1_ReplicationController,
-		Convert_extensions_ReplicaSetSpec_to_v1_ReplicationControllerSpec,
-		Convert_extensions_ReplicaSetStatus_to_v1_ReplicationControllerStatus,
+		Convert_v1_ReplicationController_To_extensions_ReplicaSet,
+		Convert_v1_ReplicationControllerSpec_To_extensions_ReplicaSetSpec,
+		Convert_v1_ReplicationControllerStatus_To_extensions_ReplicaSetStatus,
+		Convert_extensions_ReplicaSet_To_v1_ReplicationController,
+		Convert_extensions_ReplicaSetSpec_To_v1_ReplicationControllerSpec,
+		Convert_extensions_ReplicaSetStatus_To_v1_ReplicationControllerStatus,
 	)
 	if err != nil {
 		return err
 	}
 
 	// Add field conversion funcs.
-	err = scheme.AddFieldLabelConversionFunc("v1", "Pod",
+	err = scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.WithKind("Pod"),
 		func(label, value string) (string, string, error) {
 			switch label {
 			case "metadata.name",
@@ -162,6 +63,7 @@ func addConversionFuncs(scheme *runtime.Scheme) error {
 				"spec.nodeName",
 				"spec.restartPolicy",
 				"spec.schedulerName",
+				"spec.serviceAccountName",
 				"status.phase",
 				"status.podIP",
 				"status.nominatedNodeName":
@@ -177,7 +79,7 @@ func addConversionFuncs(scheme *runtime.Scheme) error {
 	if err != nil {
 		return err
 	}
-	err = scheme.AddFieldLabelConversionFunc("v1", "Node",
+	err = scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.WithKind("Node"),
 		func(label, value string) (string, string, error) {
 			switch label {
 			case "metadata.name":
@@ -192,7 +94,7 @@ func addConversionFuncs(scheme *runtime.Scheme) error {
 	if err != nil {
 		return err
 	}
-	err = scheme.AddFieldLabelConversionFunc("v1", "ReplicationController",
+	err = scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.WithKind("ReplicationController"),
 		func(label, value string) (string, string, error) {
 			switch label {
 			case "metadata.name",
@@ -218,23 +120,23 @@ func addConversionFuncs(scheme *runtime.Scheme) error {
 	return nil
 }
 
-func Convert_v1_ReplicationController_to_extensions_ReplicaSet(in *v1.ReplicationController, out *extensions.ReplicaSet, s conversion.Scope) error {
+func Convert_v1_ReplicationController_To_extensions_ReplicaSet(in *v1.ReplicationController, out *extensions.ReplicaSet, s conversion.Scope) error {
 	out.ObjectMeta = in.ObjectMeta
-	if err := Convert_v1_ReplicationControllerSpec_to_extensions_ReplicaSetSpec(&in.Spec, &out.Spec, s); err != nil {
+	if err := Convert_v1_ReplicationControllerSpec_To_extensions_ReplicaSetSpec(&in.Spec, &out.Spec, s); err != nil {
 		return err
 	}
-	if err := Convert_v1_ReplicationControllerStatus_to_extensions_ReplicaSetStatus(&in.Status, &out.Status, s); err != nil {
+	if err := Convert_v1_ReplicationControllerStatus_To_extensions_ReplicaSetStatus(&in.Status, &out.Status, s); err != nil {
 		return err
 	}
 	return nil
 }
 
-func Convert_v1_ReplicationControllerSpec_to_extensions_ReplicaSetSpec(in *v1.ReplicationControllerSpec, out *extensions.ReplicaSetSpec, s conversion.Scope) error {
+func Convert_v1_ReplicationControllerSpec_To_extensions_ReplicaSetSpec(in *v1.ReplicationControllerSpec, out *extensions.ReplicaSetSpec, s conversion.Scope) error {
 	out.Replicas = *in.Replicas
 	out.MinReadySeconds = in.MinReadySeconds
 	if in.Selector != nil {
 		out.Selector = new(metav1.LabelSelector)
-		metav1.Convert_map_to_unversioned_LabelSelector(&in.Selector, out.Selector, s)
+		metav1.Convert_Map_string_To_string_To_v1_LabelSelector(&in.Selector, out.Selector, s)
 	}
 	if in.Template != nil {
 		if err := Convert_v1_PodTemplateSpec_To_core_PodTemplateSpec(in.Template, &out.Template, s); err != nil {
@@ -244,7 +146,7 @@ func Convert_v1_ReplicationControllerSpec_to_extensions_ReplicaSetSpec(in *v1.Re
 	return nil
 }
 
-func Convert_v1_ReplicationControllerStatus_to_extensions_ReplicaSetStatus(in *v1.ReplicationControllerStatus, out *extensions.ReplicaSetStatus, s conversion.Scope) error {
+func Convert_v1_ReplicationControllerStatus_To_extensions_ReplicaSetStatus(in *v1.ReplicationControllerStatus, out *extensions.ReplicaSetStatus, s conversion.Scope) error {
 	out.Replicas = in.Replicas
 	out.FullyLabeledReplicas = in.FullyLabeledReplicas
 	out.ReadyReplicas = in.ReadyReplicas
@@ -262,9 +164,9 @@ func Convert_v1_ReplicationControllerStatus_to_extensions_ReplicaSetStatus(in *v
 	return nil
 }
 
-func Convert_extensions_ReplicaSet_to_v1_ReplicationController(in *extensions.ReplicaSet, out *v1.ReplicationController, s conversion.Scope) error {
+func Convert_extensions_ReplicaSet_To_v1_ReplicationController(in *extensions.ReplicaSet, out *v1.ReplicationController, s conversion.Scope) error {
 	out.ObjectMeta = in.ObjectMeta
-	if err := Convert_extensions_ReplicaSetSpec_to_v1_ReplicationControllerSpec(&in.Spec, &out.Spec, s); err != nil {
+	if err := Convert_extensions_ReplicaSetSpec_To_v1_ReplicationControllerSpec(&in.Spec, &out.Spec, s); err != nil {
 		fieldErr, ok := err.(*field.Error)
 		if !ok {
 			return err
@@ -274,19 +176,19 @@ func Convert_extensions_ReplicaSet_to_v1_ReplicationController(in *extensions.Re
 		}
 		out.Annotations[v1.NonConvertibleAnnotationPrefix+"/"+fieldErr.Field] = reflect.ValueOf(fieldErr.BadValue).String()
 	}
-	if err := Convert_extensions_ReplicaSetStatus_to_v1_ReplicationControllerStatus(&in.Status, &out.Status, s); err != nil {
+	if err := Convert_extensions_ReplicaSetStatus_To_v1_ReplicationControllerStatus(&in.Status, &out.Status, s); err != nil {
 		return err
 	}
 	return nil
 }
 
-func Convert_extensions_ReplicaSetSpec_to_v1_ReplicationControllerSpec(in *extensions.ReplicaSetSpec, out *v1.ReplicationControllerSpec, s conversion.Scope) error {
+func Convert_extensions_ReplicaSetSpec_To_v1_ReplicationControllerSpec(in *extensions.ReplicaSetSpec, out *v1.ReplicationControllerSpec, s conversion.Scope) error {
 	out.Replicas = new(int32)
 	*out.Replicas = in.Replicas
 	out.MinReadySeconds = in.MinReadySeconds
 	var invalidErr error
 	if in.Selector != nil {
-		invalidErr = metav1.Convert_unversioned_LabelSelector_to_map(in.Selector, &out.Selector, s)
+		invalidErr = metav1.Convert_v1_LabelSelector_To_Map_string_To_string(in.Selector, &out.Selector, s)
 	}
 	out.Template = new(v1.PodTemplateSpec)
 	if err := Convert_core_PodTemplateSpec_To_v1_PodTemplateSpec(&in.Template, out.Template, s); err != nil {
@@ -295,7 +197,7 @@ func Convert_extensions_ReplicaSetSpec_to_v1_ReplicationControllerSpec(in *exten
 	return invalidErr
 }
 
-func Convert_extensions_ReplicaSetStatus_to_v1_ReplicationControllerStatus(in *extensions.ReplicaSetStatus, out *v1.ReplicationControllerStatus, s conversion.Scope) error {
+func Convert_extensions_ReplicaSetStatus_To_v1_ReplicationControllerStatus(in *extensions.ReplicaSetStatus, out *v1.ReplicationControllerStatus, s conversion.Scope) error {
 	out.Replicas = in.Replicas
 	out.FullyLabeledReplicas = in.FullyLabeledReplicas
 	out.ReadyReplicas = in.ReadyReplicas
@@ -478,6 +380,11 @@ func Convert_core_SecurityContext_To_v1_SecurityContext(in *core.SecurityContext
 	out.RunAsNonRoot = in.RunAsNonRoot
 	out.ReadOnlyRootFilesystem = in.ReadOnlyRootFilesystem
 	out.AllowPrivilegeEscalation = in.AllowPrivilegeEscalation
+	if in.ProcMount != nil {
+		pm := string(*in.ProcMount)
+		pmt := v1.ProcMountType(pm)
+		out.ProcMount = &pmt
+	}
 	return nil
 }
 
@@ -553,7 +460,7 @@ func Convert_v1_ResourceList_To_core_ResourceList(in *v1.ResourceList, out *core
 }
 
 func AddFieldLabelConversionsForEvent(scheme *runtime.Scheme) error {
-	return scheme.AddFieldLabelConversionFunc("v1", "Event",
+	return scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.WithKind("Event"),
 		func(label, value string) (string, string, error) {
 			switch label {
 			case "involvedObject.kind",
@@ -576,7 +483,7 @@ func AddFieldLabelConversionsForEvent(scheme *runtime.Scheme) error {
 }
 
 func AddFieldLabelConversionsForNamespace(scheme *runtime.Scheme) error {
-	return scheme.AddFieldLabelConversionFunc("v1", "Namespace",
+	return scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.WithKind("Namespace"),
 		func(label, value string) (string, string, error) {
 			switch label {
 			case "status.phase",
@@ -589,7 +496,7 @@ func AddFieldLabelConversionsForNamespace(scheme *runtime.Scheme) error {
 }
 
 func AddFieldLabelConversionsForSecret(scheme *runtime.Scheme) error {
-	return scheme.AddFieldLabelConversionFunc("v1", "Secret",
+	return scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.WithKind("Secret"),
 		func(label, value string) (string, string, error) {
 			switch label {
 			case "type",

vendor/k8s.io/kubernetes/pkg/apis/core/v1/defaults.go

@@ -25,7 +25,7 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/util/parsers"
-	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
+	utilpointer "k8s.io/utils/pointer"
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
@@ -93,6 +93,7 @@ func SetDefaults_Container(obj *v1.Container) {
 		obj.TerminationMessagePolicy = v1.TerminationMessageReadFile
 	}
 }
+
 func SetDefaults_Service(obj *v1.Service) {
 	if obj.Spec.SessionAffinity == "" {
 		obj.Spec.SessionAffinity = v1.ServiceAffinityNone

vendor/k8s.io/kubernetes/pkg/apis/core/v1/helper/BUILD

@@ -11,12 +11,12 @@ go_test(
     srcs = ["helpers_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
     ],
 )
 
@@ -26,12 +26,12 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/apis/core/v1/helper",
     deps = [
         "//pkg/apis/core/helper:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/selection:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/selection:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/core/v1/helper/helpers.go

@@ -79,7 +79,7 @@ func HugePageResourceName(pageSize resource.Quantity) v1.ResourceName {
 // an error is returned.
 func HugePageSizeFromResourceName(name v1.ResourceName) (resource.Quantity, error) {
 	if !IsHugePageResourceName(name) {
-		return resource.Quantity{}, fmt.Errorf("resource name: %s is not valid hugepage name", name)
+		return resource.Quantity{}, fmt.Errorf("resource name: %s is an invalid hugepage name", name)
 	}
 	pageSize := strings.TrimPrefix(string(name), v1.ResourceHugePagesPrefix)
 	return resource.ParseQuantity(pageSize)
@@ -283,6 +283,20 @@ func NodeSelectorRequirementsAsFieldSelector(nsm []v1.NodeSelectorRequirement) (
 	return fields.AndSelectors(selectors...), nil
 }
 
+// NodeSelectorRequirementKeysExistInNodeSelectorTerms checks if a NodeSelectorTerm with key is already specified in terms
+func NodeSelectorRequirementKeysExistInNodeSelectorTerms(reqs []v1.NodeSelectorRequirement, terms []v1.NodeSelectorTerm) bool {
+	for _, req := range reqs {
+		for _, term := range terms {
+			for _, r := range term.MatchExpressions {
+				if r.Key == req.Key {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
 // MatchNodeSelectorTerms checks whether the node labels and fields match node selector terms in ORed;
 // nil or empty term matches no objects.
 func MatchNodeSelectorTerms(

vendor/k8s.io/kubernetes/pkg/apis/core/v1/register.go

@@ -30,7 +30,7 @@ func init() {
 	// We only register manually written functions here. The registration of the
 	// generated functions takes place in the generated files. The separation
 	// makes the code compile even when the generated files are missing.
-	localSchemeBuilder.Register(addDefaultingFuncs, addConversionFuncs, addFastPathConversionFuncs)
+	localSchemeBuilder.Register(addDefaultingFuncs, addConversionFuncs)
 }
 
 // TODO: remove these global varialbes

vendor/k8s.io/kubernetes/pkg/apis/core/validation/BUILD

@@ -25,20 +25,21 @@ go_library(
         "//pkg/fieldpath:go_default_library",
         "//pkg/master/ports:go_default_library",
         "//pkg/security/apparmor:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/validation:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/validation:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/validation:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/validation:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 
@@ -55,15 +56,15 @@ go_test(
         "//pkg/capabilities:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/security/apparmor:go_default_library",
-        "//pkg/util/pointer:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/core/validation/validation.go

@@ -36,6 +36,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -195,23 +196,23 @@ type ValidateNameFunc apimachineryvalidation.ValidateNameFunc
 // ValidatePodName can be used to check whether the given pod name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidatePodName = NameIsDNSSubdomain
+var ValidatePodName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidateReplicationControllerName can be used to check whether the given replication
 // controller name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidateReplicationControllerName = NameIsDNSSubdomain
+var ValidateReplicationControllerName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidateServiceName can be used to check whether the given service name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidateServiceName = NameIsDNS1035Label
+var ValidateServiceName = apimachineryvalidation.NameIsDNS1035Label
 
 // ValidateNodeName can be used to check whether the given node name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidateNodeName = NameIsDNSSubdomain
+var ValidateNodeName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidateNamespaceName can be used to check whether the given namespace name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
@@ -221,18 +222,18 @@ var ValidateNamespaceName = apimachineryvalidation.ValidateNamespaceName
 // ValidateLimitRangeName can be used to check whether the given limit range name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidateLimitRangeName = NameIsDNSSubdomain
+var ValidateLimitRangeName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidateResourceQuotaName can be used to check whether the given
 // resource quota name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidateResourceQuotaName = NameIsDNSSubdomain
+var ValidateResourceQuotaName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidateSecretName can be used to check whether the given secret name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidateSecretName = NameIsDNSSubdomain
+var ValidateSecretName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidateServiceAccountName can be used to check whether the given service account name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
@@ -242,7 +243,7 @@ var ValidateServiceAccountName = apimachineryvalidation.ValidateServiceAccountNa
 // ValidateEndpointsName can be used to check whether the given endpoints name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidateEndpointsName = NameIsDNSSubdomain
+var ValidateEndpointsName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidateClusterName can be used to check whether the given cluster name is valid.
 var ValidateClusterName = apimachineryvalidation.ValidateClusterName
@@ -250,21 +251,21 @@ var ValidateClusterName = apimachineryvalidation.ValidateClusterName
 // ValidateClassName can be used to check whether the given class name is valid.
 // It is defined here to avoid import cycle between pkg/apis/storage/validation
 // (where it should be) and this file.
-var ValidateClassName = NameIsDNSSubdomain
+var ValidateClassName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidatePiorityClassName can be used to check whether the given priority
 // class name is valid.
-var ValidatePriorityClassName = NameIsDNSSubdomain
+var ValidatePriorityClassName = apimachineryvalidation.NameIsDNSSubdomain
 
-// TODO update all references to these functions to point to the apimachineryvalidation ones
-// NameIsDNSSubdomain is a ValidateNameFunc for names that must be a DNS subdomain.
-func NameIsDNSSubdomain(name string, prefix bool) []string {
-	return apimachineryvalidation.NameIsDNSSubdomain(name, prefix)
-}
-
-// NameIsDNS1035Label is a ValidateNameFunc for names that must be a DNS 952 label.
-func NameIsDNS1035Label(name string, prefix bool) []string {
-	return apimachineryvalidation.NameIsDNS1035Label(name, prefix)
+// ValidateRuntimeClassName can be used to check whether the given RuntimeClass name is valid.
+// Prefix indicates this name will be used as part of generation, in which case
+// trailing dashes are allowed.
+func ValidateRuntimeClassName(name string, fldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+	for _, msg := range apimachineryvalidation.NameIsDNSSubdomain(name, false) {
+		allErrs = append(allErrs, field.Invalid(fldPath, name, msg))
+	}
+	return allErrs
 }
 
 // Validates that given value is not negative.
@@ -1022,7 +1023,7 @@ func validateProjectionSources(projection *core.ProjectedVolumeSource, projectio
 				}
 			}
 		}
-		if projPath := fldPath.Child("serviceAccountToken"); source.ServiceAccountToken != nil {
+		if projPath := srcPath.Child("serviceAccountToken"); source.ServiceAccountToken != nil {
 			numSources++
 			if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequestProjection) {
 				allErrs = append(allErrs, field.Forbidden(projPath, "TokenRequestProjection feature is not enabled"))
@@ -1493,7 +1494,7 @@ func validateCSIPersistentVolumeSource(csi *core.CSIPersistentVolumeSource, fldP
 
 // ValidatePersistentVolumeName checks that a name is appropriate for a
 // PersistentVolumeName object.
-var ValidatePersistentVolumeName = NameIsDNSSubdomain
+var ValidatePersistentVolumeName = apimachineryvalidation.NameIsDNSSubdomain
 
 var supportedAccessModes = sets.NewString(string(core.ReadWriteOnce), string(core.ReadOnlyMany), string(core.ReadWriteMany))
 
@@ -1501,6 +1502,10 @@ var supportedReclaimPolicy = sets.NewString(string(core.PersistentVolumeReclaimD
 
 var supportedVolumeModes = sets.NewString(string(core.PersistentVolumeBlock), string(core.PersistentVolumeFilesystem))
 
+var supportedDataSourceAPIGroupKinds = map[schema.GroupKind]bool{
+	{Group: "snapshot.storage.k8s.io", Kind: "VolumeSnapshot"}: true,
+}
+
 func ValidatePersistentVolume(pv *core.PersistentVolume) field.ErrorList {
 	metaPath := field.NewPath("metadata")
 	allErrs := ValidateObjectMeta(&pv.ObjectMeta, false, ValidatePersistentVolumeName, metaPath)
@@ -1824,6 +1829,27 @@ func ValidatePersistentVolumeClaimSpec(spec *core.PersistentVolumeClaimSpec, fld
 	} else if spec.VolumeMode != nil && !supportedVolumeModes.Has(string(*spec.VolumeMode)) {
 		allErrs = append(allErrs, field.NotSupported(fldPath.Child("volumeMode"), *spec.VolumeMode, supportedVolumeModes.List()))
 	}
+
+	if spec.DataSource != nil && !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSnapshotDataSource) {
+		allErrs = append(allErrs, field.Forbidden(fldPath.Child("dataSource"), "VolumeSnapshotDataSource is disabled by feature-gate"))
+	} else if spec.DataSource != nil {
+		if len(spec.DataSource.Name) == 0 {
+			allErrs = append(allErrs, field.Required(fldPath.Child("dataSource", "name"), ""))
+		}
+
+		groupKind := schema.GroupKind{Group: "", Kind: spec.DataSource.Kind}
+		if spec.DataSource.APIGroup != nil {
+			groupKind.Group = string(*spec.DataSource.APIGroup)
+		}
+		groupKindList := make([]string, 0, len(supportedDataSourceAPIGroupKinds))
+		for grp := range supportedDataSourceAPIGroupKinds {
+			groupKindList = append(groupKindList, grp.String())
+		}
+		if !supportedDataSourceAPIGroupKinds[groupKind] {
+			allErrs = append(allErrs, field.NotSupported(fldPath.Child("dataSource"), groupKind.String(), groupKindList))
+		}
+	}
+
 	return allErrs
 }
 
@@ -1918,7 +1944,7 @@ func ValidatePersistentVolumeClaimStatusUpdate(newPvc, oldPvc *core.PersistentVo
 	return allErrs
 }
 
-var supportedPortProtocols = sets.NewString(string(core.ProtocolTCP), string(core.ProtocolUDP))
+var supportedPortProtocols = sets.NewString(string(core.ProtocolTCP), string(core.ProtocolUDP), string(core.ProtocolSCTP))
 
 func validateContainerPorts(ports []core.ContainerPort, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
@@ -1951,6 +1977,8 @@ func validateContainerPorts(ports []core.ContainerPort, fldPath *field.Path) fie
 		}
 		if len(port.Protocol) == 0 {
 			allErrs = append(allErrs, field.Required(idxPath.Child("protocol"), ""))
+		} else if !utilfeature.DefaultFeatureGate.Enabled(features.SCTPSupport) && port.Protocol == core.ProtocolSCTP {
+			allErrs = append(allErrs, field.NotSupported(idxPath.Child("protocol"), port.Protocol, []string{string(core.ProtocolTCP), string(core.ProtocolUDP)}))
 		} else if !supportedPortProtocols.Has(string(port.Protocol)) {
 			allErrs = append(allErrs, field.NotSupported(idxPath.Child("protocol"), port.Protocol, supportedPortProtocols.List()))
 		}
@@ -2913,6 +2941,20 @@ func ValidatePod(pod *core.Pod) field.ErrorList {
 	// this was done to preserve backwards compatibility
 	specPath := field.NewPath("spec")
 
+	if pod.Spec.ServiceAccountName == "" {
+		for vi, volume := range pod.Spec.Volumes {
+			path := specPath.Child("volumes").Index(vi).Child("projected")
+			if volume.Projected != nil {
+				for si, source := range volume.Projected.Sources {
+					saPath := path.Child("sources").Index(si).Child("serviceAccountToken")
+					if source.ServiceAccountToken != nil {
+						allErrs = append(allErrs, field.Forbidden(saPath, "must not be specified when serviceAccountName is not set"))
+					}
+				}
+			}
+		}
+	}
+
 	allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.Containers, specPath.Child("containers"))...)
 	allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.InitContainers, specPath.Child("initContainers"))...)
 
@@ -2996,6 +3038,10 @@ func ValidatePodSpec(spec *core.PodSpec, fldPath *field.Path) field.ErrorList {
 		}
 	}
 
+	if spec.RuntimeClassName != nil && utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) {
+		allErrs = append(allErrs, ValidateRuntimeClassName(*spec.RuntimeClassName, fldPath.Child("runtimeClassName"))...)
+	}
+
 	return allErrs
 }
 
@@ -3087,30 +3133,55 @@ func ValidateNodeSelector(nodeSelector *core.NodeSelector, fldPath *field.Path)
 	return allErrs
 }
 
-// validateTopologySelectorLabelRequirement tests that the specified TopologySelectorLabelRequirement fields has valid data
-func validateTopologySelectorLabelRequirement(rq core.TopologySelectorLabelRequirement, fldPath *field.Path) field.ErrorList {
+// validateTopologySelectorLabelRequirement tests that the specified TopologySelectorLabelRequirement fields has valid data,
+// and constructs a set containing all of its Values.
+func validateTopologySelectorLabelRequirement(rq core.TopologySelectorLabelRequirement, fldPath *field.Path) (sets.String, field.ErrorList) {
 	allErrs := field.ErrorList{}
+	valueSet := make(sets.String)
+	valuesPath := fldPath.Child("values")
 	if len(rq.Values) == 0 {
-		allErrs = append(allErrs, field.Forbidden(fldPath.Child("values"), "must specify as least one value"))
+		allErrs = append(allErrs, field.Required(valuesPath, ""))
 	}
+
+	// Validate set property of Values field
+	for i, value := range rq.Values {
+		if valueSet.Has(value) {
+			allErrs = append(allErrs, field.Duplicate(valuesPath.Index(i), value))
+		}
+		valueSet.Insert(value)
+	}
+
 	allErrs = append(allErrs, unversionedvalidation.ValidateLabelName(rq.Key, fldPath.Child("key"))...)
 
-	return allErrs
+	return valueSet, allErrs
 }
 
-// ValidateTopologySelectorTerm tests that the specified topology selector term has valid data
-func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field.Path) field.ErrorList {
+// ValidateTopologySelectorTerm tests that the specified topology selector term has valid data,
+// and constructs a map representing the term in raw form.
+func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field.Path) (map[string]sets.String, field.ErrorList) {
 	allErrs := field.ErrorList{}
+	exprMap := make(map[string]sets.String)
+	exprPath := fldPath.Child("matchLabelExpressions")
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
+		// Allow empty MatchLabelExpressions, in case this field becomes optional in the future.
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
 		for i, req := range term.MatchLabelExpressions {
-			allErrs = append(allErrs, validateTopologySelectorLabelRequirement(req, fldPath.Child("matchLabelExpressions").Index(i))...)
+			idxPath := exprPath.Index(i)
+			valueSet, exprErrs := validateTopologySelectorLabelRequirement(req, idxPath)
+			allErrs = append(allErrs, exprErrs...)
+
+			// Validate no duplicate keys exist.
+			if _, exists := exprMap[req.Key]; exists {
+				allErrs = append(allErrs, field.Duplicate(idxPath.Child("key"), req.Key))
+			}
+			exprMap[req.Key] = valueSet
 		}
 	} else if len(term.MatchLabelExpressions) != 0 {
-		allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling"))
+		allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate VolumeScheduling"))
 	}
 
-	return allErrs
+	return exprMap, allErrs
 }
 
 // ValidateAvoidPodsInNodeAnnotations tests that the serialized AvoidPods in Node.Annotations has valid data
@@ -3708,8 +3779,10 @@ func ValidateService(service *core.Service) field.ErrorList {
 		includeProtocols := sets.NewString()
 		for i := range service.Spec.Ports {
 			portPath := portsPath.Index(i)
-			if !supportedPortProtocols.Has(string(service.Spec.Ports[i].Protocol)) {
-				allErrs = append(allErrs, field.Invalid(portPath.Child("protocol"), service.Spec.Ports[i].Protocol, "cannot create an external load balancer with non-TCP/UDP ports"))
+			if !utilfeature.DefaultFeatureGate.Enabled(features.SCTPSupport) && service.Spec.Ports[i].Protocol == core.ProtocolSCTP {
+				allErrs = append(allErrs, field.NotSupported(portPath.Child("protocol"), service.Spec.Ports[i].Protocol, []string{string(core.ProtocolTCP), string(core.ProtocolUDP)}))
+			} else if !supportedPortProtocols.Has(string(service.Spec.Ports[i].Protocol)) {
+				allErrs = append(allErrs, field.Invalid(portPath.Child("protocol"), service.Spec.Ports[i].Protocol, "cannot create an external load balancer with non-TCP/UDP/SCTP ports"))
 			} else {
 				includeProtocols.Insert(string(service.Spec.Ports[i].Protocol))
 			}
@@ -3807,6 +3880,8 @@ func validateServicePort(sp *core.ServicePort, requireName, isHeadlessService bo
 
 	if len(sp.Protocol) == 0 {
 		allErrs = append(allErrs, field.Required(fldPath.Child("protocol"), ""))
+	} else if !utilfeature.DefaultFeatureGate.Enabled(features.SCTPSupport) && sp.Protocol == core.ProtocolSCTP {
+		allErrs = append(allErrs, field.NotSupported(fldPath.Child("protocol"), sp.Protocol, []string{string(core.ProtocolTCP), string(core.ProtocolUDP)}))
 	} else if !supportedPortProtocols.Has(string(sp.Protocol)) {
 		allErrs = append(allErrs, field.NotSupported(fldPath.Child("protocol"), sp.Protocol, supportedPortProtocols.List()))
 	}
@@ -4654,7 +4729,7 @@ func ValidateSecretUpdate(newSecret, oldSecret *core.Secret) field.ErrorList {
 // ValidateConfigMapName can be used to check whether the given ConfigMap name is valid.
 // Prefix indicates this name will be used as part of generation, in which case
 // trailing dashes are allowed.
-var ValidateConfigMapName = NameIsDNSSubdomain
+var ValidateConfigMapName = apimachineryvalidation.NameIsDNSSubdomain
 
 // ValidateConfigMap tests whether required fields in the ConfigMap are set.
 func ValidateConfigMap(cfg *core.ConfigMap) field.ErrorList {
@@ -5049,50 +5124,16 @@ func ValidateNamespaceFinalizeUpdate(newNamespace, oldNamespace *core.Namespace)
 	return allErrs
 }
 
-// Construct lookup map of old subset IPs to NodeNames.
-func updateEpAddrToNodeNameMap(ipToNodeName map[string]string, addresses []core.EndpointAddress) {
-	for n := range addresses {
-		if addresses[n].NodeName == nil {
-			continue
-		}
-		ipToNodeName[addresses[n].IP] = *addresses[n].NodeName
-	}
-}
-
-// Build a map across all subsets of IP -> NodeName
-func buildEndpointAddressNodeNameMap(subsets []core.EndpointSubset) map[string]string {
-	ipToNodeName := make(map[string]string)
-	for i := range subsets {
-		updateEpAddrToNodeNameMap(ipToNodeName, subsets[i].Addresses)
-		updateEpAddrToNodeNameMap(ipToNodeName, subsets[i].NotReadyAddresses)
-	}
-	return ipToNodeName
-}
-
-func validateEpAddrNodeNameTransition(addr *core.EndpointAddress, ipToNodeName map[string]string, fldPath *field.Path) field.ErrorList {
-	errList := field.ErrorList{}
-	existingNodeName, found := ipToNodeName[addr.IP]
-	if !found {
-		return errList
-	}
-	if addr.NodeName == nil || *addr.NodeName == existingNodeName {
-		return errList
-	}
-	// NodeName entry found for this endpoint IP, but user is attempting to change NodeName
-	return append(errList, field.Forbidden(fldPath, fmt.Sprintf("Cannot change NodeName for %s to %s", addr.IP, *addr.NodeName)))
-}
-
 // ValidateEndpoints tests if required fields are set.
 func ValidateEndpoints(endpoints *core.Endpoints) field.ErrorList {
 	allErrs := ValidateObjectMeta(&endpoints.ObjectMeta, true, ValidateEndpointsName, field.NewPath("metadata"))
 	allErrs = append(allErrs, ValidateEndpointsSpecificAnnotations(endpoints.Annotations, field.NewPath("annotations"))...)
-	allErrs = append(allErrs, validateEndpointSubsets(endpoints.Subsets, []core.EndpointSubset{}, field.NewPath("subsets"))...)
+	allErrs = append(allErrs, validateEndpointSubsets(endpoints.Subsets, field.NewPath("subsets"))...)
 	return allErrs
 }
 
-func validateEndpointSubsets(subsets []core.EndpointSubset, oldSubsets []core.EndpointSubset, fldPath *field.Path) field.ErrorList {
+func validateEndpointSubsets(subsets []core.EndpointSubset, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
-	ipToNodeName := buildEndpointAddressNodeNameMap(oldSubsets)
 	for i := range subsets {
 		ss := &subsets[i]
 		idxPath := fldPath.Index(i)
@@ -5103,10 +5144,10 @@ func validateEndpointSubsets(subsets []core.EndpointSubset, oldSubsets []core.En
 			allErrs = append(allErrs, field.Required(idxPath, "must specify `addresses` or `notReadyAddresses`"))
 		}
 		for addr := range ss.Addresses {
-			allErrs = append(allErrs, validateEndpointAddress(&ss.Addresses[addr], idxPath.Child("addresses").Index(addr), ipToNodeName)...)
+			allErrs = append(allErrs, validateEndpointAddress(&ss.Addresses[addr], idxPath.Child("addresses").Index(addr))...)
 		}
 		for addr := range ss.NotReadyAddresses {
-			allErrs = append(allErrs, validateEndpointAddress(&ss.NotReadyAddresses[addr], idxPath.Child("notReadyAddresses").Index(addr), ipToNodeName)...)
+			allErrs = append(allErrs, validateEndpointAddress(&ss.NotReadyAddresses[addr], idxPath.Child("notReadyAddresses").Index(addr))...)
 		}
 		for port := range ss.Ports {
 			allErrs = append(allErrs, validateEndpointPort(&ss.Ports[port], len(ss.Ports) > 1, idxPath.Child("ports").Index(port))...)
@@ -5116,7 +5157,7 @@ func validateEndpointSubsets(subsets []core.EndpointSubset, oldSubsets []core.En
 	return allErrs
 }
 
-func validateEndpointAddress(address *core.EndpointAddress, fldPath *field.Path, ipToNodeName map[string]string) field.ErrorList {
+func validateEndpointAddress(address *core.EndpointAddress, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 	for _, msg := range validation.IsValidIP(address.IP) {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("ip"), address.IP, msg))
@@ -5130,10 +5171,6 @@ func validateEndpointAddress(address *core.EndpointAddress, fldPath *field.Path,
 			allErrs = append(allErrs, field.Invalid(fldPath.Child("nodeName"), *address.NodeName, msg))
 		}
 	}
-	allErrs = append(allErrs, validateEpAddrNodeNameTransition(address, ipToNodeName, fldPath.Child("nodeName"))...)
-	if len(allErrs) > 0 {
-		return allErrs
-	}
 	allErrs = append(allErrs, validateNonSpecialIP(address.IP, fldPath.Child("ip"))...)
 	return allErrs
 }
@@ -5176,6 +5213,8 @@ func validateEndpointPort(port *core.EndpointPort, requireName bool, fldPath *fi
 	}
 	if len(port.Protocol) == 0 {
 		allErrs = append(allErrs, field.Required(fldPath.Child("protocol"), ""))
+	} else if !utilfeature.DefaultFeatureGate.Enabled(features.SCTPSupport) && port.Protocol == core.ProtocolSCTP {
+		allErrs = append(allErrs, field.NotSupported(fldPath.Child("protocol"), port.Protocol, []string{string(core.ProtocolTCP), string(core.ProtocolUDP)}))
 	} else if !supportedPortProtocols.Has(string(port.Protocol)) {
 		allErrs = append(allErrs, field.NotSupported(fldPath.Child("protocol"), port.Protocol, supportedPortProtocols.List()))
 	}
@@ -5183,9 +5222,11 @@ func validateEndpointPort(port *core.EndpointPort, requireName bool, fldPath *fi
 }
 
 // ValidateEndpointsUpdate tests to make sure an endpoints update can be applied.
+// NodeName changes are allowed during update to accommodate the case where nodeIP or PodCIDR is reused.
+// An existing endpoint ip will have a different nodeName if this happens.
 func ValidateEndpointsUpdate(newEndpoints, oldEndpoints *core.Endpoints) field.ErrorList {
 	allErrs := ValidateObjectMetaUpdate(&newEndpoints.ObjectMeta, &oldEndpoints.ObjectMeta, field.NewPath("metadata"))
-	allErrs = append(allErrs, validateEndpointSubsets(newEndpoints.Subsets, oldEndpoints.Subsets, field.NewPath("subsets"))...)
+	allErrs = append(allErrs, validateEndpointSubsets(newEndpoints.Subsets, field.NewPath("subsets"))...)
 	allErrs = append(allErrs, ValidateEndpointsSpecificAnnotations(newEndpoints.Annotations, field.NewPath("annotations"))...)
 	return allErrs
 }

vendor/k8s.io/kubernetes/pkg/apis/extensions/BUILD

@@ -19,11 +19,11 @@ go_library(
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/networking:go_default_library",
         "//pkg/apis/policy:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/extensions/OWNERS

@@ -36,3 +36,5 @@ reviewers:
 - mqliang
 - lukaszo
 - jianhuiz
+labels:
+- sig/apps

vendor/k8s.io/kubernetes/pkg/apis/extensions/types.go

@@ -127,8 +127,8 @@ type DeploymentSpec struct {
 	// is considered to be failed. The deployment controller will continue to
 	// process failed deployments and a condition with a ProgressDeadlineExceeded
 	// reason will be surfaced in the deployment status. Note that progress will
-	// not be estimated during the time a deployment is paused. This is not set
-	// by default.
+	// not be estimated during the time a deployment is paused. This is set to
+	// the max value of int32 (i.e. 2147483647) by default, which means "no deadline".
 	// +optional
 	ProgressDeadlineSeconds *int32
 }

vendor/k8s.io/kubernetes/pkg/apis/extensions/zz_generated.deepcopy.go

@@ -188,23 +188,15 @@ func (in *DaemonSetSpec) DeepCopyInto(out *DaemonSetSpec) {
 	*out = *in
 	if in.Selector != nil {
 		in, out := &in.Selector, &out.Selector
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.LabelSelector)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
 	}
 	in.Template.DeepCopyInto(&out.Template)
 	in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy)
 	if in.RevisionHistoryLimit != nil {
 		in, out := &in.RevisionHistoryLimit, &out.RevisionHistoryLimit
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
+		*out = new(int32)
+		**out = **in
 	}
 	return
 }
@@ -224,12 +216,8 @@ func (in *DaemonSetStatus) DeepCopyInto(out *DaemonSetStatus) {
 	*out = *in
 	if in.CollisionCount != nil {
 		in, out := &in.CollisionCount, &out.CollisionCount
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
+		*out = new(int32)
+		**out = **in
 	}
 	if in.Conditions != nil {
 		in, out := &in.Conditions, &out.Conditions
@@ -256,12 +244,8 @@ func (in *DaemonSetUpdateStrategy) DeepCopyInto(out *DaemonSetUpdateStrategy) {
 	*out = *in
 	if in.RollingUpdate != nil {
 		in, out := &in.RollingUpdate, &out.RollingUpdate
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(RollingUpdateDaemonSet)
-			**out = **in
-		}
+		*out = new(RollingUpdateDaemonSet)
+		**out = **in
 	}
 	return
 }
@@ -393,41 +377,25 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
 	if in.Selector != nil {
 		in, out := &in.Selector, &out.Selector
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.LabelSelector)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
 	}
 	in.Template.DeepCopyInto(&out.Template)
 	in.Strategy.DeepCopyInto(&out.Strategy)
 	if in.RevisionHistoryLimit != nil {
 		in, out := &in.RevisionHistoryLimit, &out.RevisionHistoryLimit
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
+		*out = new(int32)
+		**out = **in
 	}
 	if in.RollbackTo != nil {
 		in, out := &in.RollbackTo, &out.RollbackTo
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(RollbackConfig)
-			**out = **in
-		}
+		*out = new(RollbackConfig)
+		**out = **in
 	}
 	if in.ProgressDeadlineSeconds != nil {
 		in, out := &in.ProgressDeadlineSeconds, &out.ProgressDeadlineSeconds
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
+		*out = new(int32)
+		**out = **in
 	}
 	return
 }
@@ -454,12 +422,8 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
 	}
 	if in.CollisionCount != nil {
 		in, out := &in.CollisionCount, &out.CollisionCount
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
+		*out = new(int32)
+		**out = **in
 	}
 	return
 }
@@ -479,12 +443,8 @@ func (in *DeploymentStrategy) DeepCopyInto(out *DeploymentStrategy) {
 	*out = *in
 	if in.RollingUpdate != nil {
 		in, out := &in.RollingUpdate, &out.RollingUpdate
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(RollingUpdateDeployment)
-			**out = **in
-		}
+		*out = new(RollingUpdateDeployment)
+		**out = **in
 	}
 	return
 }
@@ -637,12 +597,8 @@ func (in *IngressRuleValue) DeepCopyInto(out *IngressRuleValue) {
 	*out = *in
 	if in.HTTP != nil {
 		in, out := &in.HTTP, &out.HTTP
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(HTTPIngressRuleValue)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(HTTPIngressRuleValue)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -662,12 +618,8 @@ func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
 	*out = *in
 	if in.Backend != nil {
 		in, out := &in.Backend, &out.Backend
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(IngressBackend)
-			**out = **in
-		}
+		*out = new(IngressBackend)
+		**out = **in
 	}
 	if in.TLS != nil {
 		in, out := &in.TLS, &out.TLS
@@ -817,12 +769,8 @@ func (in *ReplicaSetSpec) DeepCopyInto(out *ReplicaSetSpec) {
 	*out = *in
 	if in.Selector != nil {
 		in, out := &in.Selector, &out.Selector
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.LabelSelector)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
 	}
 	in.Template.DeepCopyInto(&out.Template)
 	return

vendor/k8s.io/kubernetes/pkg/apis/networking/BUILD

@@ -16,10 +16,10 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/apis/networking",
     deps = [
         "//pkg/apis/core:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/networking/types.go

@@ -134,7 +134,7 @@ type NetworkPolicyEgressRule struct {
 
 // NetworkPolicyPort describes a port to allow traffic on
 type NetworkPolicyPort struct {
-	// The protocol (TCP or UDP) which traffic must match. If not specified, this
+	// The protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this
 	// field defaults to TCP.
 	// +optional
 	Protocol *api.Protocol

vendor/k8s.io/kubernetes/pkg/apis/networking/zz_generated.deepcopy.go

@@ -173,30 +173,18 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) {
 	*out = *in
 	if in.PodSelector != nil {
 		in, out := &in.PodSelector, &out.PodSelector
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.LabelSelector)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.NamespaceSelector != nil {
 		in, out := &in.NamespaceSelector, &out.NamespaceSelector
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.LabelSelector)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.IPBlock != nil {
 		in, out := &in.IPBlock, &out.IPBlock
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(IPBlock)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(IPBlock)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -216,21 +204,13 @@ func (in *NetworkPolicyPort) DeepCopyInto(out *NetworkPolicyPort) {
 	*out = *in
 	if in.Protocol != nil {
 		in, out := &in.Protocol, &out.Protocol
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(core.Protocol)
-			**out = **in
-		}
+		*out = new(core.Protocol)
+		**out = **in
 	}
 	if in.Port != nil {
 		in, out := &in.Port, &out.Port
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(intstr.IntOrString)
-			**out = **in
-		}
+		*out = new(intstr.IntOrString)
+		**out = **in
 	}
 	return
 }

vendor/k8s.io/kubernetes/pkg/apis/policy/BUILD

@@ -16,10 +16,10 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/apis/policy",
     deps = [
         "//pkg/apis/core:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/apis/policy/types.go

@@ -63,6 +63,7 @@ type PodDisruptionBudgetStatus struct {
 	// the list automatically by PodDisruptionBudget controller after some time.
 	// If everything goes smooth this map should be empty for the most of the time.
 	// Large number of entries in the map may indicate problems with pod deletions.
+	// +optional
 	DisruptedPods map[string]metav1.Time
 
 	// Number of pod disruptions that are currently allowed.
@@ -227,6 +228,10 @@ type PodSecurityPolicySpec struct {
 	// e.g. "foo.*" forbids "foo.bar", "foo.baz", etc.
 	// +optional
 	ForbiddenSysctls []string
+	// AllowedProcMountTypes is a whitelist of allowed ProcMountTypes.
+	// Empty or nil indicates that only the DefaultProcMountType may be used.
+	// +optional
+	AllowedProcMountTypes []api.ProcMountType
 }
 
 // AllowedHostPath defines the host volume conditions that will be enabled by a policy

vendor/k8s.io/kubernetes/pkg/apis/policy/zz_generated.deepcopy.go

@@ -66,12 +66,8 @@ func (in *Eviction) DeepCopyInto(out *Eviction) {
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
 	if in.DeleteOptions != nil {
 		in, out := &in.DeleteOptions, &out.DeleteOptions
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.DeleteOptions)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.DeleteOptions)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -213,30 +209,18 @@ func (in *PodDisruptionBudgetSpec) DeepCopyInto(out *PodDisruptionBudgetSpec) {
 	*out = *in
 	if in.MinAvailable != nil {
 		in, out := &in.MinAvailable, &out.MinAvailable
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(intstr.IntOrString)
-			**out = **in
-		}
+		*out = new(intstr.IntOrString)
+		**out = **in
 	}
 	if in.Selector != nil {
 		in, out := &in.Selector, &out.Selector
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.LabelSelector)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.MaxUnavailable != nil {
 		in, out := &in.MaxUnavailable, &out.MaxUnavailable
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(intstr.IntOrString)
-			**out = **in
-		}
+		*out = new(intstr.IntOrString)
+		**out = **in
 	}
 	return
 }
@@ -368,12 +352,8 @@ func (in *PodSecurityPolicySpec) DeepCopyInto(out *PodSecurityPolicySpec) {
 	in.FSGroup.DeepCopyInto(&out.FSGroup)
 	if in.DefaultAllowPrivilegeEscalation != nil {
 		in, out := &in.DefaultAllowPrivilegeEscalation, &out.DefaultAllowPrivilegeEscalation
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(bool)
-			**out = **in
-		}
+		*out = new(bool)
+		**out = **in
 	}
 	if in.AllowedHostPaths != nil {
 		in, out := &in.AllowedHostPaths, &out.AllowedHostPaths
@@ -395,6 +375,11 @@ func (in *PodSecurityPolicySpec) DeepCopyInto(out *PodSecurityPolicySpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.AllowedProcMountTypes != nil {
+		in, out := &in.AllowedProcMountTypes, &out.AllowedProcMountTypes
+		*out = make([]core.ProcMountType, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -434,12 +419,8 @@ func (in *SELinuxStrategyOptions) DeepCopyInto(out *SELinuxStrategyOptions) {
 	*out = *in
 	if in.SELinuxOptions != nil {
 		in, out := &in.SELinuxOptions, &out.SELinuxOptions
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(core.SELinuxOptions)
-			**out = **in
-		}
+		*out = new(core.SELinuxOptions)
+		**out = **in
 	}
 	return
 }

vendor/k8s.io/kubernetes/pkg/apis/scheduling/BUILD

@@ -17,9 +17,9 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/apis/scheduling",
     deps = [
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
     ],
 )
 
@@ -47,5 +47,5 @@ go_test(
     name = "go_default_test",
     srcs = ["helpers_test.go"],
     embed = [":go_default_library"],
-    deps = ["//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library"],
+    deps = ["//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library"],
 )

vendor/k8s.io/kubernetes/pkg/cloudprovider/BUILD

@@ -15,10 +15,10 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/cloudprovider",
     deps = [
         "//pkg/controller:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/client-go/informers:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/cloudprovider/OWNERS

@@ -2,6 +2,7 @@ approvers:
 - mikedanese
 - dims
 - wlan0
+- andrewsykim
 reviewers:
 - thockin
 - lavalamp
@@ -42,3 +43,5 @@ reviewers:
 - wlan0
 - cheftako
 - andrewsykim
+labels:
+- sig/cloud-provider

vendor/k8s.io/kubernetes/pkg/cloudprovider/README.md

@@ -1,16 +1,18 @@
-##### Deprecation Notice: This directory has entered maintenance mode and will not be accepting new providers. Cloud Providers in this directory will continue to be actively developed or maintained and supported at their current level of support as a longer-term solution evolves. 
- 
-## Overview: 
+##### Deprecation Notice: cloud providers in this directory are deprecated and will be removed in favor of external (a.k.a out-of-tree) providers. Existing providers in this directory (a.k.a in-tree providers) should only make small incremental changes as needed and avoid large refactors or new features. New providers seeking to support Kubernetes should follow the out-of-tree model as specified in the [Running Kubernetes Cloud Controller Manager docs](https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/). For more information on the future of Kubernetes cloud providers see [KEP-0002](https://github.com/kubernetes/community/blob/master/keps/sig-cloud-provider/0002-cloud-controller-manager.md) and [KEP-0013](https://github.com/kubernetes/community/blob/master/keps/sig-cloud-provider/0013-build-deploy-ccm.md).
+
+Cloud Providers in this directory will continue to be actively developed or maintained and supported at their current level of support as a longer-term solution evolves.
+
+## Overview:
 The mechanism for supporting cloud providers is currently in transition:  the original method of implementing cloud provider-specific functionality within the main kubernetes tree (here) is no longer advised; however, the proposed solution is still in development.
- 
-#### Guidance for potential cloud providers: 
-* Support for cloud providers is currently in a state of flux. Background information on motivation and the proposal for improving is in the github [proposal](https://git.k8s.io/community/contributors/design-proposals/cloud-provider/cloud-provider-refactoring.md). 
-* In support of this plan, a new cloud-controller-manager binary was added in 1.6. This was the first of several steps (see the proposal for more information). 
-* Attempts to contribute new cloud providers or (to a lesser extent) persistent volumes to the core repo will likely meet with some pushback from reviewers/approvers. 
-* It is understood that this is an unfortunate situation in which 'the old way is no longer supported but the new way is not ready yet', but the initial path is unsustainable, and contributors are encouraged to participate in the implementation of the proposed long-term solution, as there is risk that PRs for new cloud providers here will not be approved. 
-* Though the fully productized support envisioned in the proposal is still 2 - 3 releases out, the foundational work is underway, and a motivated cloud provider could accomplish the work in a forward-looking way. Contributors are encouraged to assist with the implementation of the design outlined in the proposal. 
- 
-#### Some additional context on status / direction: 
+
+#### Guidance for potential cloud providers:
+* Support for cloud providers is currently in a state of flux. Background information on motivation and the proposal for improving is in the github [proposal](https://git.k8s.io/community/contributors/design-proposals/cloud-provider/cloud-provider-refactoring.md).
+* In support of this plan, a new cloud-controller-manager binary was added in 1.6. This was the first of several steps (see the proposal for more information).
+* Attempts to contribute new cloud providers or (to a lesser extent) persistent volumes to the core repo will likely meet with some pushback from reviewers/approvers.
+* It is understood that this is an unfortunate situation in which 'the old way is no longer supported but the new way is not ready yet', but the initial path is unsustainable, and contributors are encouraged to participate in the implementation of the proposed long-term solution, as there is risk that PRs for new cloud providers here will not be approved.
+* Though the fully productized support envisioned in the proposal is still 2 - 3 releases out, the foundational work is underway, and a motivated cloud provider could accomplish the work in a forward-looking way. Contributors are encouraged to assist with the implementation of the design outlined in the proposal.
+
+#### Some additional context on status / direction:
 * 1.6 added a new cloud-controller-manager binary that may be used for testing the new out-of-core cloudprovider flow.
 * Setting cloud-provider=external allows for creation of a separate controller-manager binary
-* 1.7 adds [extensible admission control](https://git.k8s.io/community/contributors/design-proposals/api-machinery/admission_control_extension.md), further enabling topology customization. 
+* 1.7 adds [extensible admission control](https://git.k8s.io/community/contributors/design-proposals/api-machinery/admission_control_extension.md), further enabling topology customization.

vendor/k8s.io/kubernetes/pkg/cloudprovider/cloud.go

@@ -62,9 +62,12 @@ type Clusters interface {
 	Master(ctx context.Context, clusterName string) (string, error)
 }
 
-// TODO(#6812): Use a shorter name that's less likely to be longer than cloud
-// providers' name length limits.
-func GetLoadBalancerName(service *v1.Service) string {
+// (DEPRECATED) DefaultLoadBalancerName is the default load balancer name that is called from
+// LoadBalancer.GetLoadBalancerName. Use this method to maintain backward compatible names for
+// LoadBalancers that were created prior to Kubernetes v1.12. In the future, each provider should
+// replace this method call in GetLoadBalancerName with a provider-specific implementation that
+// is less cryptic than the Service's UUID.
+func DefaultLoadBalancerName(service *v1.Service) string {
 	//GCE requires that the name of a load balancer starts with a lower case letter.
 	ret := "a" + string(service.UID)
 	ret = strings.Replace(ret, "-", "", -1)
@@ -96,6 +99,9 @@ type LoadBalancer interface {
 	// Implementations must treat the *v1.Service parameter as read-only and not modify it.
 	// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager
 	GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error)
+	// GetLoadBalancerName returns the name of the load balancer. Implementations must treat the
+	// *v1.Service parameter as read-only and not modify it.
+	GetLoadBalancerName(ctx context.Context, clusterName string, service *v1.Service) string
 	// EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer
 	// Implementations must treat the *v1.Service and *v1.Node
 	// parameters as read-only and not modify them.
@@ -131,7 +137,8 @@ type Instances interface {
 	// services cannot be used in this method to obtain nodeaddresses
 	NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error)
 	// InstanceID returns the cloud provider ID of the node with the specified NodeName.
-	// Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound)
+	// Note that if the instance does not exist, we must return ("", cloudprovider.InstanceNotFound)
+	// cloudprovider.InstanceNotFound should NOT be returned for instances that exist but are stopped/sleeping
 	InstanceID(ctx context.Context, nodeName types.NodeName) (string, error)
 	// InstanceType returns the type of the specified instance.
 	InstanceType(ctx context.Context, name types.NodeName) (string, error)
@@ -143,8 +150,9 @@ type Instances interface {
 	// CurrentNodeName returns the name of the node we are currently running on
 	// On most clouds (e.g. GCE) this is the hostname, so we provide the hostname
 	CurrentNodeName(ctx context.Context, hostname string) (types.NodeName, error)
-	// InstanceExistsByProviderID returns true if the instance for the given provider id still is running.
+	// InstanceExistsByProviderID returns true if the instance for the given provider exists.
 	// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+	// This method should still return true for instances that exist but are stopped/sleeping.
 	InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error)
 	// InstanceShutdownByProviderID returns true if the instance is shutdown in cloudprovider
 	InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error)
@@ -198,13 +206,13 @@ type Zones interface {
 	// can no longer be called from the kubelets.
 	GetZone(ctx context.Context) (Zone, error)
 
-	// GetZoneByProviderID returns the Zone containing the current zone and locality region of the node specified by providerId
-	// This method is particularly used in the context of external cloud providers where node initialization must be down
+	// GetZoneByProviderID returns the Zone containing the current zone and locality region of the node specified by providerID
+	// This method is particularly used in the context of external cloud providers where node initialization must be done
 	// outside the kubelets.
 	GetZoneByProviderID(ctx context.Context, providerID string) (Zone, error)
 
 	// GetZoneByNodeName returns the Zone containing the current zone and locality region of the node specified by node name
-	// This method is particularly used in the context of external cloud providers where node initialization must be down
+	// This method is particularly used in the context of external cloud providers where node initialization must be done
 	// outside the kubelets.
 	GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) (Zone, error)
 }

vendor/k8s.io/kubernetes/pkg/cloudprovider/plugins.go

@@ -42,6 +42,8 @@ var (
 	}{
 		{"openstack", true, "https://github.com/kubernetes/cloud-provider-openstack"},
 		{"photon", false, "The Photon Controller project is no longer maintained."},
+		{"cloudstack", false, "The CloudStack Controller project is no longer maintained."},
+		{"ovirt", false, "The ovirt Controller project is no longer maintained."},
 	}
 )
 

vendor/k8s.io/kubernetes/pkg/controller/BUILD

@@ -18,24 +18,24 @@ go_test(
         "//pkg/apis/core/install:go_default_library",
         "//pkg/controller/testutil:go_default_library",
         "//pkg/securitycontext:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+        "//staging/src/k8s.io/client-go/util/testing:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/k8s.io/api/apps/v1:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
-        "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
-        "//vendor/k8s.io/client-go/util/testing:go_default_library",
     ],
 )
 
@@ -59,35 +59,37 @@ go_library(
         "//pkg/serviceaccount:go_default_library",
         "//pkg/util/hash:go_default_library",
         "//pkg/util/taints:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
+        "//staging/src/k8s.io/api/authentication/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/watch:go_default_library",
+        "//staging/src/k8s.io/client-go/util/integer:go_default_library",
+        "//staging/src/k8s.io/client-go/util/retry:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/golang/groupcache/lru:go_default_library",
-        "//vendor/k8s.io/api/apps/v1:go_default_library",
-        "//vendor/k8s.io/api/authentication/v1:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/typed/authentication/v1:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
-        "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
-        "//vendor/k8s.io/client-go/util/integer:go_default_library",
-        "//vendor/k8s.io/client-go/util/retry:go_default_library",
     ],
 )
 
@@ -102,6 +104,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//pkg/controller/apis/config:all-srcs",
         "//pkg/controller/bootstrap:all-srcs",
         "//pkg/controller/certificates:all-srcs",
         "//pkg/controller/cloud:all-srcs",
@@ -128,6 +131,7 @@ filegroup(
         "//pkg/controller/statefulset:all-srcs",
         "//pkg/controller/testutil:all-srcs",
         "//pkg/controller/ttl:all-srcs",
+        "//pkg/controller/ttlafterfinished:all-srcs",
         "//pkg/controller/util/node:all-srcs",
         "//pkg/controller/volume/attachdetach:all-srcs",
         "//pkg/controller/volume/events:all-srcs",

vendor/k8s.io/kubernetes/pkg/controller/client_builder.go

@@ -33,6 +33,7 @@ import (
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/serviceaccount"
@@ -46,8 +47,6 @@ type ControllerClientBuilder interface {
 	ConfigOrDie(name string) *restclient.Config
 	Client(name string) (clientset.Interface, error)
 	ClientOrDie(name string) clientset.Interface
-	ClientGoClient(name string) (clientset.Interface, error)
-	ClientGoClientOrDie(name string) clientset.Interface
 }
 
 // SimpleControllerClientBuilder returns a fixed client with different user agents
@@ -85,22 +84,6 @@ func (b SimpleControllerClientBuilder) ClientOrDie(name string) clientset.Interf
 	return client
 }
 
-func (b SimpleControllerClientBuilder) ClientGoClient(name string) (clientset.Interface, error) {
-	clientConfig, err := b.Config(name)
-	if err != nil {
-		return nil, err
-	}
-	return clientset.NewForConfig(clientConfig)
-}
-
-func (b SimpleControllerClientBuilder) ClientGoClientOrDie(name string) clientset.Interface {
-	client, err := b.ClientGoClient(name)
-	if err != nil {
-		glog.Fatal(err)
-	}
-	return client
-}
-
 // SAControllerClientBuilder is a ControllerClientBuilder that returns clients identifying as
 // service accounts
 type SAControllerClientBuilder struct {
@@ -140,7 +123,7 @@ func (b SAControllerClientBuilder) Config(name string) (*restclient.Config, erro
 			return b.CoreClient.Secrets(b.Namespace).Watch(options)
 		},
 	}
-	_, err = cache.ListWatchUntil(30*time.Second, lw,
+	_, err = watchtools.ListWatchUntil(30*time.Second, lw,
 		func(event watch.Event) (bool, error) {
 			switch event.Type {
 			case watch.Deleted:
@@ -274,19 +257,3 @@ func (b SAControllerClientBuilder) ClientOrDie(name string) clientset.Interface
 	}
 	return client
 }
-
-func (b SAControllerClientBuilder) ClientGoClient(name string) (clientset.Interface, error) {
-	clientConfig, err := b.Config(name)
-	if err != nil {
-		return nil, err
-	}
-	return clientset.NewForConfig(clientConfig)
-}
-
-func (b SAControllerClientBuilder) ClientGoClientOrDie(name string) clientset.Interface {
-	client, err := b.ClientGoClient(name)
-	if err != nil {
-		glog.Fatal(err)
-	}
-	return client
-}

vendor/k8s.io/kubernetes/pkg/controller/controller_utils.go

@@ -27,12 +27,14 @@ import (
 
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apimachinery/pkg/util/rand"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
@@ -594,7 +596,7 @@ func (r RealPodControl) DeletePod(namespace string, podID string, object runtime
 		return fmt.Errorf("object does not have ObjectMeta, %v", err)
 	}
 	glog.V(2).Infof("Controller %v deleting pod %v/%v", accessor.GetName(), namespace, podID)
-	if err := r.KubeClient.CoreV1().Pods(namespace).Delete(podID, nil); err != nil {
+	if err := r.KubeClient.CoreV1().Pods(namespace).Delete(podID, nil); err != nil && !apierrors.IsNotFound(err) {
 		r.Recorder.Eventf(object, v1.EventTypeWarning, FailedDeletePodReason, "Error deleting: %v", err)
 		return fmt.Errorf("unable to delete pods: %v", err)
 	} else {
@@ -1033,8 +1035,10 @@ func WaitForCacheSync(controllerName string, stopCh <-chan struct{}, cacheSyncs
 	return true
 }
 
-// ComputeHash returns a hash value calculated from pod template and a collisionCount to avoid hash collision
-func ComputeHash(template *v1.PodTemplateSpec, collisionCount *int32) uint32 {
+// ComputeHash returns a hash value calculated from pod template and
+// a collisionCount to avoid hash collision. The hash will be safe encoded to
+// avoid bad words.
+func ComputeHash(template *v1.PodTemplateSpec, collisionCount *int32) string {
 	podTemplateSpecHasher := fnv.New32a()
 	hashutil.DeepHashObject(podTemplateSpecHasher, *template)
 
@@ -1045,5 +1049,5 @@ func ComputeHash(template *v1.PodTemplateSpec, collisionCount *int32) uint32 {
 		podTemplateSpecHasher.Write(collisionCountBytes)
 	}
 
-	return podTemplateSpecHasher.Sum32()
+	return rand.SafeEncodeString(fmt.Sprint(podTemplateSpecHasher.Sum32()))
 }

vendor/k8s.io/kubernetes/pkg/features/BUILD

@@ -10,9 +10,9 @@ go_library(
     srcs = ["kube_features.go"],
     importpath = "k8s.io/kubernetes/pkg/features",
     deps = [
-        "//vendor/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/features:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/features/kube_features.go

@@ -69,6 +69,7 @@ const (
 
 	// owner: @mikedanese
 	// alpha: v1.7
+	// beta: v1.12
 	//
 	// Gets a server certificate for the kubelet from the Certificate Signing
 	// Request API instead of generating one self signed and auto rotates the
@@ -111,7 +112,7 @@ const (
 	DebugContainers utilfeature.Feature = "DebugContainers"
 
 	// owner: @verb
-	// alpha: v1.10
+	// beta: v1.12
 	//
 	// Allows all containers in a pod to share a process namespace.
 	PodShareProcessNamespace utilfeature.Feature = "PodShareProcessNamespace"
@@ -129,14 +130,14 @@ const (
 	EnableEquivalenceClassCache utilfeature.Feature = "EnableEquivalenceClassCache"
 
 	// owner: @k82cn
-	// alpha: v1.8
+	// beta: v1.12
 	//
 	// Taint nodes based on their condition status for 'NetworkUnavailable',
 	// 'MemoryPressure', 'OutOfDisk' and 'DiskPressure'.
 	TaintNodesByCondition utilfeature.Feature = "TaintNodesByCondition"
 
 	// owner: @jsafrane
-	// beta: v1.10
+	// GA: v1.12
 	//
 	// Enable mount propagation of volumes.
 	MountPropagation utilfeature.Feature = "MountPropagation"
@@ -154,6 +155,12 @@ const (
 	// Alternative container-level CPU affinity policies.
 	CPUManager utilfeature.Feature = "CPUManager"
 
+	// owner: @szuecs
+	// alpha: v1.12
+	//
+	// Enable nodes to change CPUCFSQuotaPeriod
+	CPUCFSQuotaPeriod utilfeature.Feature = "CustomCPUCFSQuotaPeriod"
+
 	// owner: @derekwaynecarr
 	// beta: v1.10
 	//
@@ -172,13 +179,6 @@ const (
 	// Enable nodes to exclude themselves from service load balancers
 	ServiceNodeExclusion utilfeature.Feature = "ServiceNodeExclusion"
 
-	// owner @brendandburns
-	// deprecated: v1.10
-	//
-	// Enable the service proxy to contact external IP addresses. Note this feature is present
-	// only for backward compatibility, it will be removed in the 1.11 release.
-	ServiceProxyAllowExternalIPs utilfeature.Feature = "ServiceProxyAllowExternalIPs"
-
 	// owner: @jsafrane
 	// alpha: v1.9
 	//
@@ -193,11 +193,21 @@ const (
 	VolumeScheduling utilfeature.Feature = "VolumeScheduling"
 
 	// owner: @vladimirvivien
-	// alpha: v1.9
+	// beta: v1.10
 	//
 	// Enable mount/attachment of Container Storage Interface (CSI) backed PVs
 	CSIPersistentVolume utilfeature.Feature = "CSIPersistentVolume"
 
+	// owner: @saad-ali
+	// alpha: v1.12
+	// Enable all logic related to the CSIDriver API object in csi.storage.k8s.io
+	CSIDriverRegistry utilfeature.Feature = "CSIDriverRegistry"
+
+	// owner: @verult
+	// alpha: v1.12
+	// Enable all logic related to the CSINodeInfo API object in csi.storage.k8s.io
+	CSINodeInfo utilfeature.Feature = "CSINodeInfo"
+
 	// owner @MrHohn
 	// beta: v1.10
 	//
@@ -240,27 +250,20 @@ const (
 	// Enable Hyper-V containers on Windows
 	HyperVContainer utilfeature.Feature = "HyperVContainer"
 
-	// owner: @joelsmith
-	// deprecated: v1.10
-	//
-	// Mount secret, configMap, downwardAPI and projected volumes ReadOnly. Note: this feature
-	// gate is present only for backward compatibility, it will be removed in the 1.11 release.
-	ReadOnlyAPIDataVolumes utilfeature.Feature = "ReadOnlyAPIDataVolumes"
-
 	// owner: @k82cn
-	// alpha: v1.10
+	// beta: v1.12
 	//
 	// Schedule DaemonSet Pods by default scheduler instead of DaemonSet controller
 	ScheduleDaemonSetPods utilfeature.Feature = "ScheduleDaemonSetPods"
 
 	// owner: @mikedanese
-	// alpha: v1.10
+	// beta: v1.12
 	//
 	// Implement TokenRequest endpoint on service account resources.
 	TokenRequest utilfeature.Feature = "TokenRequest"
 
 	// owner: @mikedanese
-	// alpha: v1.11
+	// beta: v1.12
 	//
 	// Enable ServiceAccountTokenVolumeProjection support in ProjectedVolumes.
 	TokenRequestProjection utilfeature.Feature = "TokenRequestProjection"
@@ -291,7 +294,7 @@ const (
 	VolumeSubpath utilfeature.Feature = "VolumeSubpath"
 
 	// owner: @gnufied
-	// alpha : v1.11
+	// beta : v1.12
 	//
 	// Add support for volume plugins to report node specific
 	// volume limits
@@ -311,12 +314,6 @@ const (
 	// Support Pod Ready++
 	PodReadinessGates utilfeature.Feature = "PodReadinessGates"
 
-	// owner: @lichuqiang
-	// alpha: v1.11
-	//
-	// Extend the default scheduler to be aware of volume topology and handle PV provisioning
-	DynamicProvisioningScheduling utilfeature.Feature = "DynamicProvisioningScheduling"
-
 	// owner: @kevtaylor
 	// alpha: v1.11
 	//
@@ -332,7 +329,7 @@ const (
 	KubeletPluginsWatcher utilfeature.Feature = "KubeletPluginsWatcher"
 
 	// owner: @vikaschoudhary16
-	// alpha: v1.11
+	// beta: v1.12
 	//
 	//
 	// Enable resource quota scope selectors
@@ -343,6 +340,43 @@ const (
 	//
 	// Enables CSI to use raw block storage volumes
 	CSIBlockVolume utilfeature.Feature = "CSIBlockVolume"
+
+	// owner: @tallclair
+	// alpha: v1.12
+	//
+	// Enables RuntimeClass, for selecting between multiple runtimes to run a pod.
+	RuntimeClass utilfeature.Feature = "RuntimeClass"
+
+	// owner: @mtaufen
+	// alpha: v1.12
+	//
+	// Kubelet uses the new Lease API to report node heartbeats,
+	// (Kube) Node Lifecycle Controller uses these heartbeats as a node health signal.
+	NodeLease utilfeature.Feature = "NodeLease"
+
+	// owner: @janosi
+	// alpha: v1.12
+	//
+	// Enables SCTP as new protocol for Service ports, NetworkPolicy, and ContainerPort in Pod/Containers definition
+	SCTPSupport utilfeature.Feature = "SCTPSupport"
+
+	// owner: @xing-yang
+	// alpha: v1.12
+	//
+	// Enable volume snapshot data source support.
+	VolumeSnapshotDataSource utilfeature.Feature = "VolumeSnapshotDataSource"
+
+	// owner: @jessfraz
+	// alpha: v1.12
+	//
+	// Enables control over ProcMountType for containers.
+	ProcMountType utilfeature.Feature = "ProcMountType"
+
+	// owner: @janetkuo
+	// alpha: v1.12
+	//
+	// Allow TTL controller to clean up Pods and Jobs after they finish.
+	TTLAfterFinished utilfeature.Feature = "TTLAfterFinished"
 )
 
 func init() {
@@ -359,27 +393,30 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	ExperimentalCriticalPodAnnotation:           {Default: false, PreRelease: utilfeature.Alpha},
 	DevicePlugins:                               {Default: true, PreRelease: utilfeature.Beta},
 	TaintBasedEvictions:                         {Default: false, PreRelease: utilfeature.Alpha},
-	RotateKubeletServerCertificate:              {Default: false, PreRelease: utilfeature.Alpha},
+	RotateKubeletServerCertificate:              {Default: true, PreRelease: utilfeature.Beta},
 	RotateKubeletClientCertificate:              {Default: true, PreRelease: utilfeature.Beta},
 	PersistentLocalVolumes:                      {Default: true, PreRelease: utilfeature.Beta},
 	LocalStorageCapacityIsolation:               {Default: true, PreRelease: utilfeature.Beta},
 	HugePages:                                   {Default: true, PreRelease: utilfeature.Beta},
 	Sysctls:                                     {Default: true, PreRelease: utilfeature.Beta},
 	DebugContainers:                             {Default: false, PreRelease: utilfeature.Alpha},
-	PodShareProcessNamespace:                    {Default: false, PreRelease: utilfeature.Alpha},
+	PodShareProcessNamespace:                    {Default: true, PreRelease: utilfeature.Beta},
 	PodPriority:                                 {Default: true, PreRelease: utilfeature.Beta},
 	EnableEquivalenceClassCache:                 {Default: false, PreRelease: utilfeature.Alpha},
-	TaintNodesByCondition:                       {Default: false, PreRelease: utilfeature.Alpha},
-	MountPropagation:                            {Default: true, PreRelease: utilfeature.Beta},
+	TaintNodesByCondition:                       {Default: true, PreRelease: utilfeature.Beta},
+	MountPropagation:                            {Default: true, PreRelease: utilfeature.GA},
 	QOSReserved:                                 {Default: false, PreRelease: utilfeature.Alpha},
 	ExpandPersistentVolumes:                     {Default: true, PreRelease: utilfeature.Beta},
 	ExpandInUsePersistentVolumes:                {Default: false, PreRelease: utilfeature.Alpha},
-	AttachVolumeLimit:                           {Default: false, PreRelease: utilfeature.Alpha},
+	AttachVolumeLimit:                           {Default: true, PreRelease: utilfeature.Beta},
 	CPUManager:                                  {Default: true, PreRelease: utilfeature.Beta},
+	CPUCFSQuotaPeriod:                           {Default: false, PreRelease: utilfeature.Alpha},
 	ServiceNodeExclusion:                        {Default: false, PreRelease: utilfeature.Alpha},
 	MountContainers:                             {Default: false, PreRelease: utilfeature.Alpha},
 	VolumeScheduling:                            {Default: true, PreRelease: utilfeature.Beta},
 	CSIPersistentVolume:                         {Default: true, PreRelease: utilfeature.Beta},
+	CSIDriverRegistry:                           {Default: false, PreRelease: utilfeature.Alpha},
+	CSINodeInfo:                                 {Default: false, PreRelease: utilfeature.Alpha},
 	CustomPodDNS:                                {Default: true, PreRelease: utilfeature.Beta},
 	BlockVolume:                                 {Default: false, PreRelease: utilfeature.Alpha},
 	StorageObjectInUseProtection:                {Default: true, PreRelease: utilfeature.GA},
@@ -387,28 +424,34 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	SupportIPVSProxyMode:                        {Default: true, PreRelease: utilfeature.GA},
 	SupportPodPidsLimit:                         {Default: false, PreRelease: utilfeature.Alpha},
 	HyperVContainer:                             {Default: false, PreRelease: utilfeature.Alpha},
-	ScheduleDaemonSetPods:                       {Default: false, PreRelease: utilfeature.Alpha},
-	TokenRequest:                                {Default: false, PreRelease: utilfeature.Alpha},
-	TokenRequestProjection:                      {Default: false, PreRelease: utilfeature.Alpha},
+	ScheduleDaemonSetPods:                       {Default: true, PreRelease: utilfeature.Beta},
+	TokenRequest:                                {Default: true, PreRelease: utilfeature.Beta},
+	TokenRequestProjection:                      {Default: true, PreRelease: utilfeature.Beta},
 	CRIContainerLogRotation:                     {Default: true, PreRelease: utilfeature.Beta},
 	GCERegionalPersistentDisk:                   {Default: true, PreRelease: utilfeature.Beta},
 	RunAsGroup:                                  {Default: false, PreRelease: utilfeature.Alpha},
 	VolumeSubpath:                               {Default: true, PreRelease: utilfeature.GA},
 	BalanceAttachedNodeVolumes:                  {Default: false, PreRelease: utilfeature.Alpha},
-	DynamicProvisioningScheduling:               {Default: false, PreRelease: utilfeature.Alpha},
-	PodReadinessGates:                           {Default: false, PreRelease: utilfeature.Beta},
+	PodReadinessGates:                           {Default: true, PreRelease: utilfeature.Beta},
 	VolumeSubpathEnvExpansion:                   {Default: false, PreRelease: utilfeature.Alpha},
-	KubeletPluginsWatcher:                       {Default: false, PreRelease: utilfeature.Alpha},
-	ResourceQuotaScopeSelectors:                 {Default: false, PreRelease: utilfeature.Alpha},
+	KubeletPluginsWatcher:                       {Default: true, PreRelease: utilfeature.Beta},
+	ResourceQuotaScopeSelectors:                 {Default: true, PreRelease: utilfeature.Beta},
 	CSIBlockVolume:                              {Default: false, PreRelease: utilfeature.Alpha},
+	RuntimeClass:                                {Default: false, PreRelease: utilfeature.Alpha},
+	NodeLease:                                   {Default: false, PreRelease: utilfeature.Alpha},
+	SCTPSupport:                                 {Default: false, PreRelease: utilfeature.Alpha},
+	VolumeSnapshotDataSource:                    {Default: false, PreRelease: utilfeature.Alpha},
+	ProcMountType:                               {Default: false, PreRelease: utilfeature.Alpha},
+	TTLAfterFinished:                            {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:
 	genericfeatures.StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta},
-	genericfeatures.AdvancedAuditing:        {Default: true, PreRelease: utilfeature.Beta},
+	genericfeatures.AdvancedAuditing:        {Default: true, PreRelease: utilfeature.GA},
 	genericfeatures.APIResponseCompression:  {Default: false, PreRelease: utilfeature.Alpha},
 	genericfeatures.Initializers:            {Default: false, PreRelease: utilfeature.Alpha},
 	genericfeatures.APIListChunking:         {Default: true, PreRelease: utilfeature.Beta},
+	genericfeatures.DryRun:                  {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from apiextensions-apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:
@@ -416,6 +459,5 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	apiextensionsfeatures.CustomResourceSubresources: {Default: true, PreRelease: utilfeature.Beta},
 
 	// features that enable backwards compatibility but are scheduled to be removed
-	ServiceProxyAllowExternalIPs: {Default: false, PreRelease: utilfeature.Deprecated},
-	ReadOnlyAPIDataVolumes:       {Default: true, PreRelease: utilfeature.Deprecated},
+	// ...
 }

vendor/k8s.io/kubernetes/pkg/fieldpath/BUILD

@@ -14,8 +14,9 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/fieldpath",
     deps = [
-        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
     ],
 )
 
@@ -24,8 +25,8 @@ go_test(
     srcs = ["fieldpath_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/fieldpath/fieldpath.go

@@ -21,13 +21,19 @@ import (
 	"strings"
 
 	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation"
 )
 
 // FormatMap formats map[string]string to a string.
 func FormatMap(m map[string]string) (fmtStr string) {
-	for key, value := range m {
-		fmtStr += fmt.Sprintf("%v=%q\n", key, value)
+	// output with keys in sorted order to provide stable output
+	keys := sets.NewString()
+	for key := range m {
+		keys.Insert(key)
+	}
+	for _, key := range keys.List() {
+		fmtStr += fmt.Sprintf("%v=%q\n", key, m[key])
 	}
 	fmtStr = strings.TrimSuffix(fmtStr, "\n")
 

vendor/k8s.io/kubernetes/pkg/kubelet/BUILD

@@ -14,6 +14,8 @@ go_library(
         "kubelet.go",
         "kubelet_getters.go",
         "kubelet_network.go",
+        "kubelet_network_linux.go",
+        "kubelet_network_others.go",
         "kubelet_node_status.go",
         "kubelet_pods.go",
         "kubelet_resources.go",
@@ -41,12 +43,14 @@ go_library(
         "//pkg/features:go_default_library",
         "//pkg/fieldpath:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
+        "//pkg/kubelet/apis/config:go_default_library",
         "//pkg/kubelet/apis/cri:go_default_library",
         "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library",
-        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
         "//pkg/kubelet/cadvisor:go_default_library",
         "//pkg/kubelet/certificate:go_default_library",
         "//pkg/kubelet/checkpointmanager:go_default_library",
+        "//pkg/kubelet/cloudresource:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/config:go_default_library",
         "//pkg/kubelet/configmap:go_default_library",
@@ -65,12 +69,15 @@ go_library(
         "//pkg/kubelet/metrics/collectors:go_default_library",
         "//pkg/kubelet/mountpod:go_default_library",
         "//pkg/kubelet/network/dns:go_default_library",
+        "//pkg/kubelet/nodelease:go_default_library",
+        "//pkg/kubelet/nodestatus:go_default_library",
         "//pkg/kubelet/pleg:go_default_library",
         "//pkg/kubelet/pod:go_default_library",
         "//pkg/kubelet/preemption:go_default_library",
         "//pkg/kubelet/prober:go_default_library",
         "//pkg/kubelet/prober/results:go_default_library",
         "//pkg/kubelet/remote:go_default_library",
+        "//pkg/kubelet/runtimeclass:go_default_library",
         "//pkg/kubelet/secret:go_default_library",
         "//pkg/kubelet/server:go_default_library",
         "//pkg/kubelet/server/portforward:go_default_library",
@@ -96,49 +103,49 @@ go_library(
         "//pkg/securitycontext:go_default_library",
         "//pkg/util/dbus:go_default_library",
         "//pkg/util/file:go_default_library",
-        "//pkg/util/io:go_default_library",
         "//pkg/util/iptables:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/node:go_default_library",
         "//pkg/util/oom:go_default_library",
         "//pkg/util/removeall:go_default_library",
-        "//pkg/version:go_default_library",
+        "//pkg/util/taints:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/csi:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/util/types:go_default_library",
         "//pkg/volume/util/volumepathhandler:go_default_library",
         "//pkg/volume/validation:go_default_library",
+        "//staging/src/k8s.io/api/authentication/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/client-go/dynamic:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+        "//staging/src/k8s.io/client-go/util/certificate:go_default_library",
+        "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//staging/src/k8s.io/client-go/util/integer:go_default_library",
+        "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
         "//third_party/forked/golang/expansion:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/golang/groupcache/lru:go_default_library",
         "//vendor/github.com/google/cadvisor/events:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
-        "//vendor/k8s.io/api/authentication/v1:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
-        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
-        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
-        "//vendor/k8s.io/client-go/util/certificate:go_default_library",
-        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
-        "//vendor/k8s.io/client-go/util/integer:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ],
 )
@@ -150,7 +157,9 @@ go_test(
         "kubelet_getters_test.go",
         "kubelet_network_test.go",
         "kubelet_node_status_test.go",
+        "kubelet_pods_linux_test.go",
         "kubelet_pods_test.go",
+        "kubelet_pods_windows_test.go",
         "kubelet_resources_test.go",
         "kubelet_test.go",
         "kubelet_volumes_test.go",
@@ -159,19 +168,13 @@ go_test(
         "pod_workers_test.go",
         "reason_cache_test.go",
         "runonce_test.go",
-    ] + select({
-        "@io_bazel_rules_go//go/platform:windows": [
-            "kubelet_pods_windows_test.go",
-        ],
-        "//conditions:default": [],
-    }),
+    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core/install:go_default_library",
         "//pkg/capabilities:go_default_library",
-        "//pkg/cloudprovider/providers/fake:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
-        "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library",
         "//pkg/kubelet/cadvisor/testing:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/config:go_default_library",
@@ -182,6 +185,8 @@ go_test(
         "//pkg/kubelet/images:go_default_library",
         "//pkg/kubelet/lifecycle:go_default_library",
         "//pkg/kubelet/logs:go_default_library",
+        "//pkg/kubelet/network/dns:go_default_library",
+        "//pkg/kubelet/nodestatus:go_default_library",
         "//pkg/kubelet/pleg:go_default_library",
         "//pkg/kubelet/pod:go_default_library",
         "//pkg/kubelet/pod/testing:go_default_library",
@@ -199,8 +204,10 @@ go_test(
         "//pkg/kubelet/util/queue:go_default_library",
         "//pkg/kubelet/util/sliceutils:go_default_library",
         "//pkg/kubelet/volumemanager:go_default_library",
+        "//pkg/scheduler/algorithm:go_default_library",
         "//pkg/scheduler/cache:go_default_library",
         "//pkg/util/mount:go_default_library",
+        "//pkg/util/taints:go_default_library",
         "//pkg/version:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/aws_ebs:go_default_library",
@@ -209,35 +216,40 @@ go_test(
         "//pkg/volume/host_path:go_default_library",
         "//pkg/volume/testing:go_default_library",
         "//pkg/volume/util:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/client-go/testing:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+        "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//staging/src/k8s.io/client-go/util/testing:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
-        "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/testing:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
-        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
-        "//vendor/k8s.io/client-go/util/testing:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(
@@ -257,6 +269,7 @@ filegroup(
         "//pkg/kubelet/checkpoint:all-srcs",
         "//pkg/kubelet/checkpointmanager:all-srcs",
         "//pkg/kubelet/client:all-srcs",
+        "//pkg/kubelet/cloudresource:all-srcs",
         "//pkg/kubelet/cm:all-srcs",
         "//pkg/kubelet/config:all-srcs",
         "//pkg/kubelet/configmap:all-srcs",
@@ -275,12 +288,15 @@ filegroup(
         "//pkg/kubelet/metrics:all-srcs",
         "//pkg/kubelet/mountpod:all-srcs",
         "//pkg/kubelet/network:all-srcs",
+        "//pkg/kubelet/nodelease:all-srcs",
+        "//pkg/kubelet/nodestatus:all-srcs",
         "//pkg/kubelet/pleg:all-srcs",
         "//pkg/kubelet/pod:all-srcs",
         "//pkg/kubelet/preemption:all-srcs",
         "//pkg/kubelet/prober:all-srcs",
         "//pkg/kubelet/qos:all-srcs",
         "//pkg/kubelet/remote:all-srcs",
+        "//pkg/kubelet/runtimeclass:all-srcs",
         "//pkg/kubelet/secret:all-srcs",
         "//pkg/kubelet/server:all-srcs",
         "//pkg/kubelet/stats:all-srcs",

vendor/k8s.io/kubernetes/pkg/kubelet/OWNERS

@@ -7,3 +7,6 @@ approvers:
 - yujuhong
 reviewers:
 - sig-node-reviewers
+labels:
+- area/kubelet
+- sig/node

vendor/k8s.io/kubernetes/pkg/kubelet/apis/BUILD

@@ -9,18 +9,14 @@ go_library(
     name = "go_default_library",
     srcs = [
         "well_known_annotations.go",
+        "well_known_annotations_windows.go",
         "well_known_labels.go",
-    ] + select({
-        "@io_bazel_rules_go//go/platform:windows": [
-            "well_known_annotations_windows.go",
-        ],
-        "//conditions:default": [],
-    }),
+    ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/apis",
     deps = select({
         "@io_bazel_rules_go//go/platform:windows": [
             "//pkg/features:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "//conditions:default": [],
     }),
@@ -37,11 +33,12 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//pkg/kubelet/apis/config:all-srcs",
         "//pkg/kubelet/apis/cri:all-srcs",
         "//pkg/kubelet/apis/deviceplugin/v1alpha:all-srcs",
         "//pkg/kubelet/apis/deviceplugin/v1beta1:all-srcs",
-        "//pkg/kubelet/apis/kubeletconfig:all-srcs",
         "//pkg/kubelet/apis/pluginregistration/v1alpha1:all-srcs",
+        "//pkg/kubelet/apis/pluginregistration/v1beta1:all-srcs",
         "//pkg/kubelet/apis/stats/v1alpha1:all-srcs",
     ],
     tags = ["automanaged"],

vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2/BUILD

@@ -33,8 +33,3 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
-
-filegroup(
-    name = "go_default_library_protos",
-    srcs = ["api.proto"],
-)

vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go

@@ -160,17 +160,20 @@ const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
 type Protocol int32
 
 const (
-	Protocol_TCP Protocol = 0
-	Protocol_UDP Protocol = 1
+	Protocol_TCP  Protocol = 0
+	Protocol_UDP  Protocol = 1
+	Protocol_SCTP Protocol = 2
 )
 
 var Protocol_name = map[int32]string{
 	0: "TCP",
 	1: "UDP",
+	2: "SCTP",
 }
 var Protocol_value = map[string]int32{
-	"TCP": 0,
-	"UDP": 1,
+	"TCP":  0,
+	"UDP":  1,
+	"SCTP": 2,
 }
 
 func (x Protocol) String() string {
@@ -842,6 +845,12 @@ func (m *PodSandboxConfig) GetLinux() *LinuxPodSandboxConfig {
 type RunPodSandboxRequest struct {
 	// Configuration for creating a PodSandbox.
 	Config *PodSandboxConfig `protobuf:"bytes,1,opt,name=config" json:"config,omitempty"`
+	// Named runtime configuration to use for this PodSandbox.
+	// If the runtime handler is unknown, this request should be rejected.  An
+	// empty string should select the default handler, equivalent to the
+	// behavior before this feature was added.
+	// See https://git.k8s.io/community/keps/sig-node/0014-runtime-class.md
+	RuntimeHandler string `protobuf:"bytes,2,opt,name=runtime_handler,json=runtimeHandler,proto3" json:"runtime_handler,omitempty"`
 }
 
 func (m *RunPodSandboxRequest) Reset()                    { *m = RunPodSandboxRequest{} }
@@ -855,6 +864,13 @@ func (m *RunPodSandboxRequest) GetConfig() *PodSandboxConfig {
 	return nil
 }
 
+func (m *RunPodSandboxRequest) GetRuntimeHandler() string {
+	if m != nil {
+		return m.RuntimeHandler
+	}
+	return ""
+}
+
 type RunPodSandboxResponse struct {
 	// ID of the PodSandbox to run.
 	PodSandboxId string `protobuf:"bytes,1,opt,name=pod_sandbox_id,json=podSandboxId,proto3" json:"pod_sandbox_id,omitempty"`
@@ -1497,6 +1513,12 @@ type LinuxContainerSecurityContext struct {
 	// no_new_privs defines if the flag for no_new_privs should be set on the
 	// container.
 	NoNewPrivs bool `protobuf:"varint,11,opt,name=no_new_privs,json=noNewPrivs,proto3" json:"no_new_privs,omitempty"`
+	// masked_paths is a slice of paths that should be masked by the container
+	// runtime, this can be passed directly to the OCI spec.
+	MaskedPaths []string `protobuf:"bytes,13,rep,name=masked_paths,json=maskedPaths" json:"masked_paths,omitempty"`
+	// readonly_paths is a slice of paths that should be set as readonly by the
+	// container runtime, this can be passed directly to the OCI spec.
+	ReadonlyPaths []string `protobuf:"bytes,14,rep,name=readonly_paths,json=readonlyPaths" json:"readonly_paths,omitempty"`
 }
 
 func (m *LinuxContainerSecurityContext) Reset()      { *m = LinuxContainerSecurityContext{} }
@@ -1589,6 +1611,20 @@ func (m *LinuxContainerSecurityContext) GetNoNewPrivs() bool {
 	return false
 }
 
+func (m *LinuxContainerSecurityContext) GetMaskedPaths() []string {
+	if m != nil {
+		return m.MaskedPaths
+	}
+	return nil
+}
+
+func (m *LinuxContainerSecurityContext) GetReadonlyPaths() []string {
+	if m != nil {
+		return m.ReadonlyPaths
+	}
+	return nil
+}
+
 // LinuxContainerConfig contains platform-specific configuration for
 // Linux-based containers.
 type LinuxContainerConfig struct {
@@ -5413,6 +5449,12 @@ func (m *RunPodSandboxRequest) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i += n11
 	}
+	if len(m.RuntimeHandler) > 0 {
+		dAtA[i] = 0x12
+		i++
+		i = encodeVarintApi(dAtA, i, uint64(len(m.RuntimeHandler)))
+		i += copy(dAtA[i:], m.RuntimeHandler)
+	}
 	return i, nil
 }
 
@@ -6321,6 +6363,36 @@ func (m *LinuxContainerSecurityContext) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i += n27
 	}
+	if len(m.MaskedPaths) > 0 {
+		for _, s := range m.MaskedPaths {
+			dAtA[i] = 0x6a
+			i++
+			l = len(s)
+			for l >= 1<<7 {
+				dAtA[i] = uint8(uint64(l)&0x7f | 0x80)
+				l >>= 7
+				i++
+			}
+			dAtA[i] = uint8(l)
+			i++
+			i += copy(dAtA[i:], s)
+		}
+	}
+	if len(m.ReadonlyPaths) > 0 {
+		for _, s := range m.ReadonlyPaths {
+			dAtA[i] = 0x72
+			i++
+			l = len(s)
+			for l >= 1<<7 {
+				dAtA[i] = uint8(uint64(l)&0x7f | 0x80)
+				l >>= 7
+				i++
+			}
+			dAtA[i] = uint8(l)
+			i++
+			i += copy(dAtA[i:], s)
+		}
+	}
 	return i, nil
 }
 
@@ -9260,6 +9332,10 @@ func (m *RunPodSandboxRequest) Size() (n int) {
 		l = m.Config.Size()
 		n += 1 + l + sovApi(uint64(l))
 	}
+	l = len(m.RuntimeHandler)
+	if l > 0 {
+		n += 1 + l + sovApi(uint64(l))
+	}
 	return n
 }
 
@@ -9643,6 +9719,18 @@ func (m *LinuxContainerSecurityContext) Size() (n int) {
 		l = m.RunAsGroup.Size()
 		n += 1 + l + sovApi(uint64(l))
 	}
+	if len(m.MaskedPaths) > 0 {
+		for _, s := range m.MaskedPaths {
+			l = len(s)
+			n += 1 + l + sovApi(uint64(l))
+		}
+	}
+	if len(m.ReadonlyPaths) > 0 {
+		for _, s := range m.ReadonlyPaths {
+			l = len(s)
+			n += 1 + l + sovApi(uint64(l))
+		}
+	}
 	return n
 }
 
@@ -10931,6 +11019,7 @@ func (this *RunPodSandboxRequest) String() string {
 	}
 	s := strings.Join([]string{`&RunPodSandboxRequest{`,
 		`Config:` + strings.Replace(fmt.Sprintf("%v", this.Config), "PodSandboxConfig", "PodSandboxConfig", 1) + `,`,
+		`RuntimeHandler:` + fmt.Sprintf("%v", this.RuntimeHandler) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -11247,6 +11336,8 @@ func (this *LinuxContainerSecurityContext) String() string {
 		`SeccompProfilePath:` + fmt.Sprintf("%v", this.SeccompProfilePath) + `,`,
 		`NoNewPrivs:` + fmt.Sprintf("%v", this.NoNewPrivs) + `,`,
 		`RunAsGroup:` + strings.Replace(fmt.Sprintf("%v", this.RunAsGroup), "Int64Value", "Int64Value", 1) + `,`,
+		`MaskedPaths:` + fmt.Sprintf("%v", this.MaskedPaths) + `,`,
+		`ReadonlyPaths:` + fmt.Sprintf("%v", this.ReadonlyPaths) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -14250,6 +14341,35 @@ func (m *RunPodSandboxRequest) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field RuntimeHandler", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.RuntimeHandler = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipApi(dAtA[iNdEx:])
@@ -17509,6 +17629,64 @@ func (m *LinuxContainerSecurityContext) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 13:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field MaskedPaths", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.MaskedPaths = append(m.MaskedPaths, string(dAtA[iNdEx:postIndex]))
+			iNdEx = postIndex
+		case 14:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ReadonlyPaths", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ReadonlyPaths = append(m.ReadonlyPaths, string(dAtA[iNdEx:postIndex]))
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipApi(dAtA[iNdEx:])
@@ -26658,296 +26836,300 @@ var (
 func init() { proto.RegisterFile("api.proto", fileDescriptorApi) }
 
 var fileDescriptorApi = []byte{
-	// 4649 bytes of a gzipped FileDescriptorProto
+	// 4708 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5c, 0xcd, 0x6f, 0x1b, 0x49,
-	0x76, 0x57, 0x93, 0xfa, 0x20, 0x1f, 0x45, 0x89, 0x2a, 0xcb, 0x16, 0x4d, 0xdb, 0xb2, 0xd5, 0x1e,
-	0x7f, 0xce, 0x58, 0x1e, 0x6b, 0x76, 0x3d, 0xb1, 0x3d, 0x6b, 0x9b, 0x96, 0x64, 0x9b, 0x59, 0x9b,
-	0x52, 0x9a, 0xd2, 0x7c, 0xec, 0x0c, 0xd0, 0xdb, 0x62, 0x97, 0xa8, 0x5e, 0x93, 0x5d, 0x3d, 0xdd,
-	0x4d, 0xdb, 0xca, 0x21, 0x58, 0x20, 0xc8, 0x1e, 0x02, 0x04, 0xc8, 0x79, 0x8f, 0x9b, 0x43, 0x0e,
-	0xb9, 0x05, 0x08, 0x72, 0xc8, 0x69, 0x83, 0x3d, 0xec, 0x25, 0x40, 0x4e, 0x8b, 0x7c, 0x5c, 0x32,
-	0x13, 0xe4, 0x92, 0x43, 0x90, 0x3f, 0x20, 0x87, 0xa0, 0xbe, 0x9a, 0xfd, 0xc9, 0x0f, 0x8f, 0x77,
-	0x67, 0xf6, 0xa4, 0xae, 0xd7, 0xef, 0xbd, 0x7a, 0xf5, 0xea, 0xf5, 0xab, 0x57, 0xbf, 0x2a, 0x0a,
-	0x8a, 0x86, 0x63, 0xad, 0x3b, 0x2e, 0xf1, 0x09, 0xaa, 0xb8, 0x7d, 0xdb, 0xb7, 0x7a, 0x78, 0xfd,
-	0xe5, 0x2d, 0xa3, 0xeb, 0x1c, 0x19, 0x1b, 0xb5, 0x1b, 0x1d, 0xcb, 0x3f, 0xea, 0x1f, 0xac, 0xb7,
-	0x49, 0xef, 0x66, 0x87, 0x74, 0xc8, 0x4d, 0xc6, 0x78, 0xd0, 0x3f, 0x64, 0x2d, 0xd6, 0x60, 0x4f,
-	0x5c, 0x81, 0x7a, 0x1d, 0x16, 0x3e, 0xc6, 0xae, 0x67, 0x11, 0x5b, 0xc3, 0x5f, 0xf6, 0xb1, 0xe7,
-	0xa3, 0x2a, 0xcc, 0xbd, 0xe4, 0x94, 0xaa, 0x72, 0x41, 0xb9, 0x5a, 0xd4, 0x64, 0x53, 0xfd, 0x6b,
-	0x05, 0x16, 0x03, 0x66, 0xcf, 0x21, 0xb6, 0x87, 0xb3, 0xb9, 0xd1, 0x1a, 0xcc, 0x0b, 0xe3, 0x74,
-	0xdb, 0xe8, 0xe1, 0x6a, 0x8e, 0xbd, 0x2e, 0x09, 0x5a, 0xd3, 0xe8, 0x61, 0x74, 0x05, 0x16, 0x25,
-	0x8b, 0x54, 0x92, 0x67, 0x5c, 0x0b, 0x82, 0x2c, 0x7a, 0x43, 0xeb, 0x70, 0x42, 0x32, 0x1a, 0x8e,
-	0x15, 0x30, 0x4f, 0x33, 0xe6, 0x25, 0xf1, 0xaa, 0xee, 0x58, 0x82, 0x5f, 0xfd, 0x1c, 0x8a, 0x5b,
-	0xcd, 0xd6, 0x26, 0xb1, 0x0f, 0xad, 0x0e, 0x35, 0xd1, 0xc3, 0x2e, 0x95, 0xa9, 0x2a, 0x17, 0xf2,
-	0xd4, 0x44, 0xd1, 0x44, 0x35, 0x28, 0x78, 0xd8, 0x70, 0xdb, 0x47, 0xd8, 0xab, 0xe6, 0xd8, 0xab,
-	0xa0, 0x4d, 0xa5, 0x88, 0xe3, 0x5b, 0xc4, 0xf6, 0xaa, 0x79, 0x2e, 0x25, 0x9a, 0xea, 0x2f, 0x14,
-	0x28, 0xed, 0x12, 0xd7, 0x7f, 0x6e, 0x38, 0x8e, 0x65, 0x77, 0xd0, 0x6d, 0x28, 0x30, 0x5f, 0xb6,
-	0x49, 0x97, 0xf9, 0x60, 0x61, 0xa3, 0xb6, 0x1e, 0x9f, 0x96, 0xf5, 0x5d, 0xc1, 0xa1, 0x05, 0xbc,
-	0xe8, 0x12, 0x2c, 0xb4, 0x89, 0xed, 0x1b, 0x96, 0x8d, 0x5d, 0xdd, 0x21, 0xae, 0xcf, 0x5c, 0x34,
-	0xa3, 0x95, 0x03, 0x2a, 0xed, 0x05, 0x9d, 0x81, 0xe2, 0x11, 0xf1, 0x7c, 0xce, 0x91, 0x67, 0x1c,
-	0x05, 0x4a, 0x60, 0x2f, 0x57, 0x60, 0x8e, 0xbd, 0xb4, 0x1c, 0xe1, 0x8c, 0x59, 0xda, 0x6c, 0x38,
-	0xea, 0x6f, 0x14, 0x98, 0x79, 0x4e, 0xfa, 0xb6, 0x1f, 0xeb, 0xc6, 0xf0, 0x8f, 0xc4, 0x44, 0x85,
-	0xba, 0x31, 0xfc, 0xa3, 0x41, 0x37, 0x94, 0x83, 0xcf, 0x15, 0xef, 0x86, 0xbe, 0xac, 0x41, 0xc1,
-	0xc5, 0x86, 0x49, 0xec, 0xee, 0x31, 0x33, 0xa1, 0xa0, 0x05, 0x6d, 0x3a, 0x89, 0x1e, 0xee, 0x5a,
-	0x76, 0xff, 0xb5, 0xee, 0xe2, 0xae, 0x71, 0x80, 0xbb, 0xcc, 0x94, 0x82, 0xb6, 0x20, 0xc8, 0x1a,
-	0xa7, 0xa2, 0x2d, 0x28, 0x39, 0x2e, 0x71, 0x8c, 0x8e, 0x41, 0xfd, 0x58, 0x9d, 0x61, 0xae, 0x52,
-	0x93, 0xae, 0x62, 0x66, 0xef, 0x0e, 0x38, 0xb5, 0xb0, 0x98, 0xfa, 0xb7, 0x0a, 0x2c, 0xd2, 0xe0,
-	0xf1, 0x1c, 0xa3, 0x8d, 0x77, 0xd8, 0x94, 0xa0, 0x3b, 0x30, 0x67, 0x63, 0xff, 0x15, 0x71, 0x5f,
-	0x88, 0x09, 0x38, 0x9f, 0xd4, 0x1a, 0xc8, 0x3c, 0x27, 0x26, 0xd6, 0x24, 0x3f, 0xba, 0x05, 0x79,
-	0xc7, 0x32, 0xd9, 0x80, 0xc7, 0x10, 0xa3, 0xbc, 0x54, 0xc4, 0x72, 0xda, 0xcc, 0x0f, 0xe3, 0x88,
-	0x58, 0x4e, 0x5b, 0x55, 0x01, 0x1a, 0xb6, 0x7f, 0xfb, 0x7b, 0x1f, 0x1b, 0xdd, 0x3e, 0x46, 0xcb,
-	0x30, 0xf3, 0x92, 0x3e, 0x30, 0x63, 0xf3, 0x1a, 0x6f, 0xa8, 0x5f, 0xe5, 0xe1, 0xcc, 0x33, 0xea,
-	0xaf, 0x96, 0x61, 0x9b, 0x07, 0xe4, 0x75, 0x0b, 0xb7, 0xfb, 0xae, 0xe5, 0x1f, 0x6f, 0x12, 0xdb,
-	0xc7, 0xaf, 0x7d, 0xd4, 0x84, 0x25, 0x5b, 0x6a, 0xd6, 0x65, 0x68, 0x52, 0x0d, 0xa5, 0x8d, 0xb5,
-	0x21, 0x46, 0x70, 0x17, 0x69, 0x15, 0x3b, 0x4a, 0xf0, 0xd0, 0xd3, 0xc1, 0xbc, 0x49, 0x6d, 0x39,
-	0xa6, 0x2d, 0x65, 0x48, 0xad, 0x6d, 0x66, 0x99, 0xd0, 0x25, 0x27, 0x56, 0x6a, 0xfa, 0x08, 0xe8,
-	0x57, 0xad, 0x1b, 0x9e, 0xde, 0xf7, 0xb0, 0xcb, 0x1c, 0x53, 0xda, 0x38, 0x9b, 0xd4, 0x32, 0x70,
-	0x81, 0x56, 0x74, 0xfb, 0x76, 0xdd, 0xdb, 0xf7, 0xb0, 0xcb, 0x92, 0x80, 0x88, 0x25, 0xdd, 0x25,
-	0xc4, 0x3f, 0xf4, 0x64, 0xfc, 0x48, 0xb2, 0xc6, 0xa8, 0xe8, 0x26, 0x9c, 0xf0, 0xfa, 0x8e, 0xd3,
-	0xc5, 0x3d, 0x6c, 0xfb, 0x46, 0x57, 0xef, 0xb8, 0xa4, 0xef, 0x78, 0xd5, 0x99, 0x0b, 0xf9, 0xab,
-	0x79, 0x0d, 0x85, 0x5f, 0x3d, 0x61, 0x6f, 0xd0, 0x2a, 0x80, 0xe3, 0x5a, 0x2f, 0xad, 0x2e, 0xee,
-	0x60, 0xb3, 0x3a, 0xcb, 0x94, 0x86, 0x28, 0xe8, 0x7d, 0x58, 0xf6, 0x70, 0xbb, 0x4d, 0x7a, 0x8e,
-	0xee, 0xb8, 0xe4, 0xd0, 0xea, 0x62, 0x1e, 0xfd, 0x73, 0x2c, 0xfa, 0x91, 0x78, 0xb7, 0xcb, 0x5f,
-	0xb1, 0xef, 0xe0, 0x3e, 0xcb, 0x69, 0x74, 0xa4, 0xac, 0xf3, 0x6a, 0x61, 0x8c, 0xa1, 0x02, 0x1b,
-	0x2a, 0x33, 0x49, 0xfd, 0x45, 0x0e, 0x4e, 0x32, 0x4f, 0xee, 0x12, 0x53, 0x4c, 0xb3, 0x48, 0x52,
-	0x17, 0xa1, 0xdc, 0x66, 0x3a, 0x75, 0xc7, 0x70, 0xb1, 0xed, 0x8b, 0x8f, 0x74, 0x9e, 0x13, 0x77,
-	0x19, 0x0d, 0x7d, 0x0a, 0x15, 0x4f, 0x44, 0x85, 0xde, 0xe6, 0x61, 0x21, 0xe6, 0xec, 0x46, 0xd2,
-	0x84, 0x21, 0xb1, 0xa4, 0x2d, 0x7a, 0x89, 0xe0, 0x9a, 0xf3, 0x8e, 0xbd, 0xb6, 0xdf, 0xe5, 0xd9,
-	0xae, 0xb4, 0xf1, 0xbd, 0x0c, 0x85, 0x71, 0xc3, 0xd7, 0x5b, 0x5c, 0x6c, 0xdb, 0xf6, 0xdd, 0x63,
-	0x4d, 0x2a, 0xa9, 0xdd, 0x85, 0xf9, 0xf0, 0x0b, 0x54, 0x81, 0xfc, 0x0b, 0x7c, 0x2c, 0x06, 0x45,
-	0x1f, 0x07, 0x1f, 0x01, 0xcf, 0x35, 0xbc, 0x71, 0x37, 0xf7, 0x07, 0x8a, 0xea, 0x02, 0x1a, 0xf4,
-	0xf2, 0x1c, 0xfb, 0x86, 0x69, 0xf8, 0x06, 0x42, 0x30, 0xcd, 0x96, 0x11, 0xae, 0x82, 0x3d, 0x53,
-	0xad, 0x7d, 0xf1, 0xf1, 0x16, 0x35, 0xfa, 0x88, 0xce, 0x42, 0x31, 0x08, 0x74, 0xb1, 0x96, 0x0c,
-	0x08, 0x34, 0xa7, 0x1b, 0xbe, 0x8f, 0x7b, 0x8e, 0xcf, 0x42, 0xac, 0xac, 0xc9, 0xa6, 0xfa, 0x3f,
-	0xd3, 0x50, 0x49, 0xcc, 0xc9, 0x43, 0x28, 0xf4, 0x44, 0xf7, 0xe2, 0x43, 0x7b, 0x27, 0x25, 0xb1,
-	0x27, 0x4c, 0xd5, 0x02, 0x29, 0x9a, 0x37, 0x69, 0x0e, 0x0d, 0xad, 0x7f, 0x41, 0x9b, 0xce, 0x78,
-	0x97, 0x74, 0x74, 0xd3, 0x72, 0x71, 0xdb, 0x27, 0xee, 0xb1, 0x30, 0x77, 0xbe, 0x4b, 0x3a, 0x5b,
-	0x92, 0x86, 0xee, 0x02, 0x98, 0xb6, 0x47, 0x27, 0xfb, 0xd0, 0xea, 0x30, 0xa3, 0x4b, 0x1b, 0x67,
-	0x92, 0x46, 0x04, 0x8b, 0x9d, 0x56, 0x34, 0x6d, 0x4f, 0x98, 0xff, 0x08, 0xca, 0x74, 0xcd, 0xd0,
-	0x7b, 0x7c, 0x9d, 0xe2, 0x5f, 0x4a, 0x69, 0xe3, 0x5c, 0xda, 0x18, 0x82, 0xd5, 0x4c, 0x9b, 0x77,
-	0x06, 0x0d, 0x0f, 0x3d, 0x86, 0x59, 0x96, 0xbc, 0xbd, 0xea, 0x2c, 0x13, 0x5e, 0x1f, 0xe6, 0x00,
-	0x11, 0x11, 0xcf, 0x98, 0x00, 0x0f, 0x08, 0x21, 0x8d, 0xf6, 0xa1, 0x64, 0xd8, 0x36, 0xf1, 0x0d,
-	0x9e, 0x68, 0xe6, 0x98, 0xb2, 0x0f, 0xc6, 0x50, 0x56, 0x1f, 0x48, 0x71, 0x8d, 0x61, 0x3d, 0xe8,
-	0x07, 0x30, 0xc3, 0x32, 0x91, 0xf8, 0x10, 0xaf, 0x8c, 0x19, 0xb4, 0x1a, 0x97, 0xaa, 0xdd, 0x81,
-	0x52, 0xc8, 0xd8, 0x49, 0x82, 0xb4, 0x76, 0x1f, 0x2a, 0x71, 0xd3, 0x26, 0x0a, 0x72, 0x0d, 0x96,
-	0xb5, 0xbe, 0x3d, 0x30, 0x4c, 0x56, 0x5f, 0x77, 0x61, 0x56, 0x4c, 0x36, 0x8f, 0x38, 0x75, 0xb4,
-	0x8f, 0x34, 0x21, 0xa1, 0xfe, 0x00, 0x4e, 0xc6, 0x74, 0x8a, 0x22, 0xed, 0x1d, 0x58, 0x70, 0x88,
-	0xa9, 0x7b, 0x9c, 0xac, 0x5b, 0xa6, 0xcc, 0x2e, 0x4e, 0xc0, 0xdb, 0x30, 0xa9, 0x78, 0xcb, 0x27,
-	0x4e, 0xd2, 0xa6, 0xf1, 0xc4, 0xab, 0x70, 0x2a, 0x2e, 0xce, 0xbb, 0x57, 0x1f, 0xc0, 0x8a, 0x86,
-	0x7b, 0xe4, 0x25, 0x7e, 0x53, 0xd5, 0x35, 0xa8, 0x26, 0x15, 0x08, 0xe5, 0x9f, 0xc1, 0xca, 0x80,
-	0xda, 0xf2, 0x0d, 0xbf, 0xef, 0x4d, 0xa4, 0x5c, 0x54, 0xb0, 0x07, 0xc4, 0xe3, 0xb3, 0x54, 0xd0,
-	0x64, 0x53, 0xbd, 0x16, 0x56, 0xdd, 0xe4, 0x05, 0x03, 0xef, 0x01, 0x2d, 0x40, 0xce, 0x72, 0x84,
-	0xba, 0x9c, 0xe5, 0xa8, 0x4f, 0xa1, 0x18, 0xac, 0xb8, 0xe8, 0xde, 0xa0, 0x74, 0xcc, 0x8d, 0xbb,
-	0x3e, 0x07, 0xd5, 0xe5, 0x5e, 0x62, 0x85, 0x10, 0x5d, 0xde, 0x03, 0x08, 0x32, 0x99, 0x5c, 0xf8,
-	0xcf, 0x0c, 0x51, 0xac, 0x85, 0xd8, 0xd5, 0x7f, 0x8b, 0xe4, 0xb7, 0xd0, 0x20, 0xcc, 0x60, 0x10,
-	0x66, 0x24, 0xdf, 0xe5, 0xde, 0x28, 0xdf, 0x7d, 0x08, 0x33, 0x9e, 0x6f, 0xf8, 0x58, 0x14, 0x47,
-	0x6b, 0xc3, 0xc4, 0xa9, 0x11, 0x58, 0xe3, 0xfc, 0xe8, 0x1c, 0x40, 0xdb, 0xc5, 0x86, 0x8f, 0x4d,
-	0xdd, 0xe0, 0xc9, 0x39, 0xaf, 0x15, 0x05, 0xa5, 0xee, 0xa3, 0xcd, 0x41, 0x81, 0x37, 0xc3, 0x0c,
-	0xbb, 0x36, 0x4c, 0x73, 0x64, 0xaa, 0x06, 0xa5, 0x5e, 0x90, 0x2c, 0x66, 0xc7, 0x4c, 0x16, 0x42,
-	0x01, 0x97, 0x0a, 0xa5, 0xc2, 0xb9, 0xd1, 0xa9, 0x90, 0x8b, 0x8e, 0x93, 0x0a, 0x0b, 0xa3, 0x53,
-	0xa1, 0x50, 0x36, 0x34, 0x15, 0x7e, 0x9b, 0xb9, 0xec, 0x5f, 0x15, 0xa8, 0x26, 0xbf, 0x41, 0x91,
-	0x7b, 0xee, 0xc2, 0xac, 0xc7, 0x28, 0xe3, 0x24, 0x34, 0x21, 0x2b, 0x24, 0xd0, 0x53, 0x98, 0xb6,
-	0xec, 0x43, 0xc2, 0xf6, 0x66, 0xa9, 0x25, 0x49, 0x56, 0xaf, 0xeb, 0x0d, 0xfb, 0x90, 0x70, 0x27,
-	0x31, 0x0d, 0xb5, 0x0f, 0xa1, 0x18, 0x90, 0x26, 0x1a, 0xdb, 0x0e, 0x2c, 0xc7, 0x42, 0x96, 0xd7,
-	0xf0, 0x41, 0xa4, 0x2b, 0x93, 0x45, 0xba, 0xfa, 0xd3, 0x5c, 0xf8, 0x4b, 0x7c, 0x6c, 0x75, 0x7d,
-	0xec, 0x26, 0xbe, 0xc4, 0x8f, 0xa4, 0x76, 0xfe, 0x19, 0x5e, 0x1e, 0xa9, 0x9d, 0x97, 0x9a, 0xe2,
-	0x63, 0xfa, 0x02, 0x16, 0x58, 0xac, 0xe9, 0x1e, 0xee, 0xb2, 0x3a, 0x42, 0xd4, 0x74, 0xdf, 0x1f,
-	0xa6, 0x86, 0x5b, 0xc2, 0x23, 0xb6, 0x25, 0xe4, 0xb8, 0x07, 0xcb, 0xdd, 0x30, 0xad, 0xf6, 0x10,
-	0x50, 0x92, 0x69, 0x22, 0x9f, 0xb6, 0x68, 0x8a, 0xa3, 0x1b, 0xd8, 0x94, 0xc5, 0xef, 0x90, 0x99,
-	0x31, 0x4e, 0xac, 0x70, 0x83, 0x35, 0x21, 0xa1, 0xfe, 0x2a, 0x0f, 0x30, 0x78, 0xf9, 0x7b, 0x94,
-	0xdb, 0x1e, 0x06, 0x79, 0x85, 0xd7, 0x67, 0x57, 0x87, 0x29, 0x4e, 0xcd, 0x28, 0x3b, 0xd1, 0x8c,
-	0xc2, 0x2b, 0xb5, 0x1b, 0x43, 0xd5, 0x7c, 0x67, 0x73, 0xc9, 0x33, 0x38, 0x15, 0x8f, 0x0d, 0x91,
-	0x48, 0x36, 0x60, 0xc6, 0xf2, 0x71, 0x8f, 0x83, 0x38, 0xa9, 0x9b, 0xae, 0x90, 0x10, 0x67, 0x55,
-	0xd7, 0xa0, 0xd8, 0xe8, 0x19, 0x1d, 0xdc, 0x72, 0x70, 0x9b, 0x76, 0x6a, 0xd1, 0x86, 0x30, 0x84,
-	0x37, 0xd4, 0x0d, 0x28, 0xfc, 0x10, 0x1f, 0xf3, 0x8f, 0x7a, 0x4c, 0x43, 0xd5, 0xbf, 0xc8, 0xc1,
-	0x0a, 0x5b, 0x2b, 0x36, 0x25, 0x84, 0xa2, 0x61, 0x8f, 0xf4, 0xdd, 0x36, 0xf6, 0xd8, 0x6c, 0x3b,
-	0x7d, 0xdd, 0xc1, 0xae, 0x45, 0x4c, 0xb1, 0xc3, 0x2f, 0xb6, 0x9d, 0xfe, 0x2e, 0x23, 0xa0, 0x33,
-	0x40, 0x1b, 0xfa, 0x97, 0x7d, 0x22, 0x02, 0x31, 0xaf, 0x15, 0xda, 0x4e, 0xff, 0x8f, 0x68, 0x5b,
-	0xca, 0x7a, 0x47, 0x86, 0x8b, 0x3d, 0x16, 0x67, 0x5c, 0xb6, 0xc5, 0x08, 0xe8, 0x16, 0x9c, 0xec,
-	0xe1, 0x1e, 0x71, 0x8f, 0xf5, 0xae, 0xd5, 0xb3, 0x7c, 0xdd, 0xb2, 0xf5, 0x83, 0x63, 0x1f, 0x7b,
-	0x22, 0xa6, 0x10, 0x7f, 0xf9, 0x8c, 0xbe, 0x6b, 0xd8, 0x8f, 0xe8, 0x1b, 0xa4, 0x42, 0x99, 0x90,
-	0x9e, 0xee, 0xb5, 0x89, 0x8b, 0x75, 0xc3, 0xfc, 0x09, 0x5b, 0x3e, 0xf3, 0x5a, 0x89, 0x90, 0x5e,
-	0x8b, 0xd2, 0xea, 0xe6, 0x4f, 0xd0, 0x79, 0x28, 0xb5, 0x9d, 0xbe, 0x87, 0x7d, 0x9d, 0xfe, 0x61,
-	0xab, 0x63, 0x51, 0x03, 0x4e, 0xda, 0x74, 0xfa, 0x5e, 0x88, 0xa1, 0x47, 0xfd, 0x3f, 0x17, 0x66,
-	0x78, 0x4e, 0xdd, 0x6c, 0x40, 0x39, 0x82, 0x10, 0xd0, 0xcd, 0x1a, 0x83, 0x02, 0xc4, 0x66, 0x8d,
-	0x3e, 0x53, 0x9a, 0x4b, 0xba, 0xd2, 0x93, 0xec, 0x99, 0xd2, 0xfc, 0x63, 0x47, 0xee, 0xd4, 0xd8,
-	0x33, 0x75, 0x79, 0x17, 0xbf, 0x14, 0x28, 0x52, 0x51, 0xe3, 0x0d, 0xd5, 0x04, 0xd8, 0x34, 0x1c,
-	0xe3, 0xc0, 0xea, 0x5a, 0xfe, 0x31, 0xba, 0x06, 0x15, 0xc3, 0x34, 0xf5, 0xb6, 0xa4, 0x58, 0x58,
-	0x62, 0x7b, 0x8b, 0x86, 0x69, 0x6e, 0x86, 0xc8, 0xe8, 0x5d, 0x58, 0x32, 0x5d, 0xe2, 0x44, 0x79,
-	0x39, 0xd8, 0x57, 0xa1, 0x2f, 0xc2, 0xcc, 0xea, 0xcf, 0x67, 0xe0, 0x5c, 0x74, 0x62, 0xe3, 0x28,
-	0xcc, 0x43, 0x98, 0x8f, 0xf5, 0x9a, 0x81, 0x00, 0x0c, 0xac, 0xd5, 0x22, 0x12, 0x31, 0x54, 0x22,
-	0x97, 0x40, 0x25, 0x52, 0x71, 0x9e, 0xfc, 0x5b, 0xc5, 0x79, 0xa6, 0xdf, 0x0a, 0xce, 0x33, 0x33,
-	0x19, 0xce, 0x73, 0x99, 0x81, 0xbd, 0x52, 0x9a, 0x6d, 0x89, 0x79, 0xa8, 0x95, 0x03, 0x1e, 0x5b,
-	0x82, 0xc2, 0x31, 0x3c, 0x68, 0x6e, 0x12, 0x3c, 0xa8, 0x90, 0x89, 0x07, 0xd1, 0xa8, 0x71, 0x1c,
-	0xc3, 0xed, 0x11, 0x57, 0x02, 0x3e, 0xd5, 0x22, 0x33, 0x61, 0x51, 0xd2, 0x05, 0xd8, 0x93, 0x09,
-	0x0d, 0x41, 0x26, 0x34, 0x74, 0x01, 0xe6, 0x6d, 0xa2, 0xdb, 0xf8, 0x95, 0x4e, 0xe7, 0xd2, 0xab,
-	0x96, 0xf8, 0xc4, 0xda, 0xa4, 0x89, 0x5f, 0xed, 0x52, 0x4a, 0x02, 0x3c, 0x9a, 0x9f, 0x10, 0x3c,
-	0xfa, 0x07, 0x05, 0x96, 0xa3, 0xc1, 0x29, 0x36, 0xfa, 0x4f, 0xa0, 0xe8, 0xca, 0xfc, 0x23, 0x02,
-	0xf2, 0x5a, 0x46, 0x71, 0x9b, 0x4c, 0x58, 0xda, 0x40, 0x16, 0xfd, 0x28, 0x13, 0x5f, 0xba, 0x39,
-	0x4a, 0xdf, 0x28, 0x84, 0x49, 0x6d, 0xc0, 0xf9, 0x4f, 0x2c, 0xdb, 0x24, 0xaf, 0xbc, 0xcc, 0x6f,
-	0x2b, 0x25, 0x42, 0x94, 0x94, 0x08, 0x51, 0x7f, 0xa9, 0xc0, 0xa9, 0xb8, 0x2e, 0xe1, 0x8a, 0x46,
-	0xd2, 0x15, 0xef, 0x26, 0x4d, 0x8f, 0x0b, 0xa7, 0x3a, 0xe3, 0x8b, 0x4c, 0x67, 0xdc, 0x1a, 0xad,
-	0x71, 0xa4, 0x3b, 0xfe, 0x46, 0x81, 0xd3, 0x99, 0x66, 0xc4, 0x16, 0x02, 0x25, 0xbe, 0x10, 0x88,
-	0x45, 0xa4, 0x4d, 0xfa, 0xb6, 0x1f, 0x5a, 0x44, 0x36, 0x19, 0xde, 0xcf, 0xb3, 0xb5, 0xde, 0x33,
-	0x5e, 0x5b, 0xbd, 0x7e, 0x4f, 0xac, 0x22, 0x54, 0xdd, 0x73, 0x4e, 0x79, 0x83, 0x65, 0x44, 0xad,
-	0xc3, 0x52, 0x60, 0xe5, 0x50, 0x44, 0x2e, 0x84, 0xb0, 0xe5, 0xa2, 0x08, 0x9b, 0x0d, 0xb3, 0x5b,
-	0xf8, 0xa5, 0xd5, 0xc6, 0x6f, 0xe5, 0x40, 0xe2, 0x02, 0x94, 0x1c, 0xec, 0xf6, 0x2c, 0xcf, 0x0b,
-	0xd2, 0x63, 0x51, 0x0b, 0x93, 0xd4, 0xff, 0x9a, 0x85, 0xc5, 0x78, 0x74, 0x3c, 0x48, 0x00, 0x7a,
-	0x17, 0x53, 0x12, 0x77, 0x7c, 0xa0, 0xa1, 0x1a, 0xf0, 0x96, 0x2c, 0x21, 0x72, 0x59, 0xdb, 0xef,
-	0xa0, 0xdc, 0x10, 0xf5, 0x05, 0xf5, 0x48, 0x9b, 0xf4, 0x7a, 0x86, 0x6d, 0xca, 0x73, 0x24, 0xd1,
-	0xa4, 0xfe, 0x33, 0xdc, 0x0e, 0x75, 0x3b, 0x25, 0xb3, 0x67, 0x3a, 0x79, 0x74, 0xaf, 0x6a, 0xd9,
-	0x0c, 0x18, 0x64, 0x29, 0xb6, 0xa8, 0x81, 0x20, 0x6d, 0x59, 0x2e, 0x5a, 0x87, 0x69, 0x6c, 0xbf,
-	0x94, 0x45, 0x5e, 0xca, 0x41, 0x93, 0x2c, 0x66, 0x34, 0xc6, 0x87, 0x6e, 0xc2, 0x6c, 0x8f, 0x86,
-	0x85, 0xdc, 0xb5, 0xae, 0x64, 0x9c, 0xb7, 0x68, 0x82, 0x0d, 0x6d, 0xc0, 0x9c, 0xc9, 0xe6, 0x49,
-	0x6e, 0x4d, 0xab, 0x29, 0x70, 0x23, 0x63, 0xd0, 0x24, 0x23, 0xda, 0x0e, 0x4a, 0xd8, 0x62, 0x56,
-	0xed, 0x19, 0x9b, 0x8a, 0xd4, 0x3a, 0x76, 0x2f, 0x5a, 0xc7, 0x02, 0xd3, 0xb5, 0x31, 0x5a, 0xd7,
-	0x70, 0x8c, 0xf0, 0x34, 0x14, 0xba, 0xa4, 0xc3, 0xc3, 0xa8, 0xc4, 0x8f, 0x28, 0xbb, 0xa4, 0xc3,
-	0xa2, 0x68, 0x99, 0x96, 0xf4, 0xa6, 0x65, 0xb3, 0x54, 0x5c, 0xd0, 0x78, 0x83, 0x7e, 0x7c, 0xec,
-	0x41, 0x27, 0x76, 0x1b, 0x57, 0xcb, 0xec, 0x55, 0x91, 0x51, 0x76, 0xec, 0x36, 0x2b, 0x12, 0x7d,
-	0xff, 0xb8, 0xba, 0xc0, 0xe8, 0xf4, 0x91, 0xee, 0xd6, 0x38, 0xb0, 0xb0, 0x98, 0xb5, 0x5b, 0x4b,
-	0x4b, 0xdb, 0x12, 0x57, 0x78, 0x04, 0x73, 0xaf, 0x78, 0x22, 0xa8, 0x56, 0x98, 0xfc, 0xd5, 0xd1,
-	0xe9, 0x45, 0x68, 0x90, 0x82, 0xdf, 0x66, 0xc1, 0xfe, 0x2b, 0x05, 0x4e, 0x6d, 0xb2, 0xcd, 0x4c,
-	0x28, 0x8f, 0x4d, 0x82, 0xbf, 0xdd, 0x09, 0x10, 0xcf, 0x4c, 0xb0, 0x2c, 0x3e, 0x6e, 0x21, 0x80,
-	0x1a, 0xb0, 0x20, 0x95, 0x0b, 0x15, 0xf9, 0xb1, 0x41, 0xd3, 0xb2, 0x17, 0x6e, 0xaa, 0x1f, 0xc1,
-	0x4a, 0x62, 0x14, 0x62, 0xe3, 0xb1, 0x06, 0xf3, 0x83, 0x7c, 0x15, 0x0c, 0xa2, 0x14, 0xd0, 0x1a,
-	0xa6, 0x7a, 0x17, 0x4e, 0xb6, 0x7c, 0xc3, 0xf5, 0x13, 0x2e, 0x18, 0x43, 0x96, 0xe1, 0xa6, 0x51,
-	0x59, 0x01, 0x6d, 0xb6, 0x60, 0xb9, 0xe5, 0x13, 0xe7, 0x0d, 0x94, 0xd2, 0xac, 0x43, 0xc7, 0x4f,
-	0xfa, 0x72, 0x7d, 0x90, 0x4d, 0x75, 0x85, 0xa3, 0xbc, 0xc9, 0xde, 0xee, 0xc1, 0x29, 0x0e, 0xb2,
-	0xbe, 0xc9, 0x20, 0x4e, 0x4b, 0x88, 0x37, 0xa9, 0xf7, 0x39, 0x9c, 0x18, 0x2c, 0x8b, 0x03, 0x00,
-	0xe5, 0x76, 0x14, 0x40, 0xb9, 0x30, 0x64, 0xd6, 0x23, 0xf8, 0xc9, 0x5f, 0xe5, 0x42, 0x79, 0x3d,
-	0x03, 0x3e, 0xb9, 0x17, 0x85, 0x4f, 0x2e, 0x8d, 0xd2, 0x1d, 0x41, 0x4f, 0x92, 0x51, 0x9b, 0x4f,
-	0x89, 0xda, 0xcf, 0x13, 0x18, 0xcb, 0x74, 0x16, 0x48, 0x15, 0xb3, 0xf6, 0x77, 0x02, 0xb1, 0x68,
-	0x1c, 0x62, 0x09, 0xba, 0x0e, 0x30, 0xf1, 0x3b, 0x31, 0x88, 0x65, 0x6d, 0xa4, 0xbd, 0x01, 0xc2,
-	0xf2, 0x77, 0xd3, 0x50, 0x0c, 0xde, 0x25, 0x7c, 0x9e, 0x74, 0x5b, 0x2e, 0xc5, 0x6d, 0xe1, 0x15,
-	0x38, 0xff, 0x8d, 0x56, 0xe0, 0xe9, 0xb1, 0x57, 0xe0, 0x33, 0x50, 0x64, 0x0f, 0xba, 0x8b, 0x0f,
-	0xc5, 0x8a, 0x5a, 0x60, 0x04, 0x0d, 0x1f, 0x0e, 0xc2, 0x70, 0x76, 0xa2, 0x30, 0x8c, 0x81, 0x3a,
-	0x73, 0x71, 0x50, 0xe7, 0x41, 0xb0, 0x22, 0xf2, 0x45, 0xf4, 0xca, 0x10, 0xbd, 0xa9, 0x6b, 0x61,
-	0x33, 0xba, 0x16, 0xf2, 0x75, 0xf5, 0xbd, 0x61, 0x5a, 0xbe, 0xb3, 0x90, 0xce, 0x3e, 0x87, 0x74,
-	0xc2, 0xb1, 0x28, 0x32, 0xeb, 0x3d, 0x80, 0x20, 0x89, 0x48, 0x5c, 0xe7, 0xcc, 0x90, 0x31, 0x6a,
-	0x21, 0x76, 0xaa, 0x36, 0x32, 0x35, 0x83, 0x73, 0x9f, 0xf1, 0xf2, 0x63, 0xc6, 0xa1, 0xcf, 0xff,
-	0xcd, 0x84, 0xf2, 0x4b, 0xc6, 0x41, 0xc9, 0x83, 0x04, 0x98, 0x38, 0x61, 0x14, 0xdf, 0x8e, 0x62,
-	0x89, 0x6f, 0x18, 0x75, 0x09, 0x28, 0x91, 0x55, 0x2e, 0x86, 0x2b, 0x5e, 0x73, 0xa8, 0xa7, 0x28,
-	0x28, 0x75, 0xb6, 0x33, 0x38, 0xb4, 0x6c, 0xcb, 0x3b, 0xe2, 0xef, 0x67, 0xf9, 0xce, 0x40, 0x92,
-	0xea, 0xec, 0xaa, 0x11, 0x7e, 0x6d, 0xf9, 0x7a, 0x9b, 0x98, 0x98, 0xc5, 0xf4, 0x8c, 0x56, 0xa0,
-	0x84, 0x4d, 0x62, 0xe2, 0xc1, 0x97, 0x57, 0x78, 0xb3, 0x2f, 0xaf, 0x18, 0xfb, 0xf2, 0x4e, 0xc1,
-	0xac, 0x8b, 0x0d, 0x8f, 0xd8, 0x62, 0x53, 0x2d, 0x5a, 0x74, 0x6a, 0x7a, 0xd8, 0xf3, 0x68, 0x4f,
-	0xa2, 0x5c, 0x13, 0xcd, 0x50, 0x99, 0x39, 0x3f, 0xb2, 0xcc, 0x1c, 0x72, 0x00, 0x13, 0x2b, 0x33,
-	0xcb, 0x23, 0xcb, 0xcc, 0x71, 0xce, 0x5f, 0x42, 0x85, 0xf6, 0xc2, 0x78, 0x85, 0x76, 0xb8, 0x2e,
-	0x5d, 0x8c, 0xd4, 0xa5, 0xdf, 0xe6, 0xc7, 0xfa, 0x1b, 0x05, 0x56, 0x12, 0x9f, 0x95, 0xf8, 0x5c,
-	0xef, 0xc4, 0x8e, 0x72, 0xd6, 0x46, 0xfa, 0x2c, 0x38, 0xc9, 0x79, 0x12, 0x39, 0xc9, 0xf9, 0x60,
-	0xb4, 0xe0, 0x5b, 0x3f, 0xc8, 0xf9, 0x33, 0x05, 0xce, 0xef, 0x3b, 0x66, 0xac, 0xc2, 0x13, 0xdb,
-	0xfe, 0xf1, 0x13, 0xc7, 0x03, 0x59, 0xeb, 0xe7, 0x26, 0xc5, 0x59, 0xb8, 0x9c, 0xaa, 0xc2, 0x85,
-	0x6c, 0x33, 0x44, 0xc9, 0xf4, 0x63, 0x58, 0xdc, 0x7e, 0x8d, 0xdb, 0xad, 0x63, 0xbb, 0x3d, 0x81,
-	0x69, 0x15, 0xc8, 0xb7, 0x7b, 0xa6, 0xc0, 0x36, 0xe9, 0x63, 0xb8, 0x0a, 0xcc, 0x47, 0xab, 0x40,
-	0x1d, 0x2a, 0x83, 0x1e, 0xc4, 0xf4, 0x9e, 0xa2, 0xd3, 0x6b, 0x52, 0x66, 0xaa, 0x7c, 0x5e, 0x13,
-	0x2d, 0x41, 0xc7, 0xae, 0xcb, 0xc6, 0xcc, 0xe9, 0xd8, 0x75, 0xa3, 0xd9, 0x22, 0x1f, 0xcd, 0x16,
-	0xea, 0xcf, 0x15, 0x28, 0xd1, 0x1e, 0xbe, 0x91, 0xfd, 0x62, 0xab, 0x95, 0x1f, 0x6c, 0xb5, 0x82,
-	0x1d, 0xdb, 0x74, 0x78, 0xc7, 0x36, 0xb0, 0x7c, 0x86, 0x91, 0x93, 0x96, 0xcf, 0x06, 0x74, 0xec,
-	0xba, 0xea, 0x05, 0x98, 0xe7, 0xb6, 0x89, 0x91, 0x57, 0x20, 0xdf, 0x77, 0xbb, 0x32, 0x8e, 0xfa,
-	0x6e, 0x57, 0xfd, 0x73, 0x05, 0xca, 0x75, 0xdf, 0x37, 0xda, 0x47, 0x13, 0x0c, 0x20, 0x30, 0x2e,
-	0x17, 0x36, 0x2e, 0x39, 0x88, 0x81, 0xb9, 0xd3, 0x19, 0xe6, 0xce, 0x44, 0xcc, 0x55, 0x61, 0x41,
-	0xda, 0x92, 0x69, 0x70, 0x13, 0xd0, 0x2e, 0x71, 0xfd, 0xc7, 0xc4, 0x7d, 0x65, 0xb8, 0xe6, 0x64,
-	0x3b, 0x30, 0x04, 0xd3, 0xe2, 0xfa, 0x69, 0xfe, 0xea, 0x8c, 0xc6, 0x9e, 0xd5, 0x2b, 0x70, 0x22,
-	0xa2, 0x2f, 0xb3, 0xe3, 0x87, 0x50, 0x62, 0x79, 0x5f, 0x94, 0xe2, 0xb7, 0xc2, 0x87, 0x2c, 0x63,
-	0xad, 0x12, 0xea, 0x1f, 0xc2, 0x12, 0xad, 0x0f, 0x18, 0x3d, 0xf8, 0x14, 0xbf, 0x1f, 0xab, 0x53,
-	0xcf, 0x65, 0x28, 0x8a, 0xd5, 0xa8, 0x7f, 0xaf, 0xc0, 0x0c, 0xa3, 0x27, 0xd6, 0xec, 0x33, 0x50,
-	0x74, 0xb1, 0x43, 0x74, 0xdf, 0xe8, 0x04, 0x97, 0x7d, 0x29, 0x61, 0xcf, 0xe8, 0x78, 0xec, 0xae,
-	0x32, 0x7d, 0x69, 0x5a, 0x1d, 0xec, 0xf9, 0xf2, 0xc6, 0x6f, 0x89, 0xd2, 0xb6, 0x38, 0x89, 0x3a,
-	0xc9, 0xb3, 0xfe, 0x98, 0xd7, 0x9d, 0xd3, 0x1a, 0x7b, 0x46, 0xeb, 0xfc, 0xfe, 0xd9, 0x38, 0x40,
-	0x38, 0xbb, 0x9d, 0x56, 0x83, 0x42, 0x0c, 0xfb, 0x0e, 0xda, 0xea, 0x36, 0xa0, 0xb0, 0x17, 0x84,
-	0xbf, 0x6f, 0xc2, 0x2c, 0x73, 0x92, 0xac, 0x8e, 0x56, 0x32, 0xdc, 0xa0, 0x09, 0x36, 0xd5, 0x00,
-	0xc4, 0x1d, 0x1c, 0xa9, 0x88, 0x26, 0x9f, 0x95, 0x21, 0x15, 0xd2, 0x3f, 0x2a, 0x70, 0x22, 0xd2,
-	0x87, 0xb0, 0xf5, 0x46, 0xb4, 0x93, 0x4c, 0x53, 0x45, 0x07, 0x9b, 0x91, 0x25, 0xe1, 0x66, 0x96,
-	0x49, 0xbf, 0xa5, 0xe5, 0xe0, 0x9f, 0x14, 0x80, 0x7a, 0xdf, 0x3f, 0x12, 0xc8, 0x60, 0x78, 0x66,
-	0x94, 0xe8, 0xcc, 0xd0, 0x77, 0x8e, 0xe1, 0x79, 0xaf, 0x88, 0x2b, 0xf7, 0x34, 0x41, 0x9b, 0x61,
-	0x78, 0x7d, 0xff, 0x48, 0x1e, 0x60, 0xd1, 0x67, 0x74, 0x09, 0x16, 0xf8, 0x05, 0x73, 0xdd, 0x30,
-	0x4d, 0x17, 0x7b, 0x9e, 0x38, 0xc9, 0x2a, 0x73, 0x6a, 0x9d, 0x13, 0x29, 0x9b, 0x65, 0x62, 0xdb,
-	0xb7, 0xfc, 0x63, 0xdd, 0x27, 0x2f, 0xb0, 0x2d, 0xf6, 0x26, 0x65, 0x49, 0xdd, 0xa3, 0x44, 0xca,
-	0xe6, 0xe2, 0x8e, 0xe5, 0xf9, 0xae, 0x64, 0x93, 0xa7, 0x26, 0x82, 0xca, 0xd8, 0xe8, 0xa4, 0x54,
-	0x76, 0xfb, 0xdd, 0x2e, 0x77, 0xf1, 0x9b, 0x4f, 0xfb, 0xfb, 0x62, 0x40, 0xb9, 0xac, 0x98, 0x1e,
-	0x38, 0x4d, 0x0c, 0xf7, 0x2d, 0x82, 0x30, 0xef, 0xc3, 0x52, 0x68, 0x0c, 0x22, 0xac, 0x22, 0x45,
-	0xa4, 0x12, 0x2d, 0x22, 0xd5, 0x27, 0x80, 0x38, 0xee, 0xf0, 0x0d, 0xc7, 0xad, 0x9e, 0x84, 0x13,
-	0x11, 0x45, 0x62, 0x25, 0xbe, 0x0e, 0x65, 0x71, 0x9b, 0x48, 0x04, 0xca, 0x69, 0x28, 0xd0, 0x8c,
-	0xda, 0xb6, 0x4c, 0x79, 0xba, 0x39, 0xe7, 0x10, 0x73, 0xd3, 0x32, 0x5d, 0xf5, 0x13, 0x28, 0x6b,
-	0xbc, 0x1f, 0xc1, 0xfb, 0x18, 0x16, 0xc4, 0xdd, 0x23, 0x3d, 0x72, 0xa7, 0x2f, 0xed, 0xce, 0x78,
-	0xb8, 0x13, 0xad, 0x6c, 0x87, 0x9b, 0xaa, 0x09, 0x35, 0x5e, 0x32, 0x44, 0xd4, 0xcb, 0xc1, 0x3e,
-	0x06, 0xf9, 0x6b, 0x89, 0x91, 0xbd, 0x44, 0xe5, 0xcb, 0x6e, 0xb8, 0xa9, 0x9e, 0x83, 0x33, 0xa9,
-	0xbd, 0x08, 0x4f, 0x38, 0x50, 0x19, 0xbc, 0x30, 0x2d, 0x79, 0xcc, 0xcb, 0x8e, 0x6f, 0x95, 0xd0,
-	0xf1, 0xed, 0xa9, 0xa0, 0x48, 0xcc, 0xc9, 0x45, 0x8c, 0x55, 0x80, 0x83, 0x72, 0x3f, 0x9f, 0x55,
-	0xee, 0x4f, 0x47, 0xca, 0x7d, 0xb5, 0x15, 0xf8, 0x53, 0x6c, 0xc3, 0x1e, 0xb1, 0xed, 0x22, 0xef,
-	0x5b, 0x26, 0x44, 0x75, 0xd8, 0x28, 0x39, 0xab, 0x16, 0x92, 0x52, 0xaf, 0x41, 0x39, 0x9a, 0x1a,
-	0x43, 0x79, 0x4e, 0x49, 0xe4, 0xb9, 0x85, 0x58, 0x8a, 0xfb, 0x30, 0x56, 0x01, 0x67, 0xfb, 0x38,
-	0x56, 0xff, 0xde, 0x8f, 0x24, 0xbb, 0xeb, 0x29, 0x27, 0xaf, 0xbf, 0xa5, 0x3c, 0xb7, 0x2c, 0xd6,
-	0x83, 0xc7, 0x1e, 0x95, 0x17, 0x83, 0x56, 0x2f, 0x42, 0x69, 0x3f, 0xeb, 0x07, 0x09, 0xd3, 0xf2,
-	0x96, 0xc3, 0x6d, 0x58, 0x7e, 0x6c, 0x75, 0xb1, 0x77, 0xec, 0xf9, 0xb8, 0xd7, 0x60, 0x49, 0xe9,
-	0xd0, 0xc2, 0x2e, 0x5a, 0x05, 0x60, 0x5b, 0x18, 0x87, 0x58, 0xc1, 0x3d, 0xf5, 0x10, 0x45, 0xfd,
-	0x6f, 0x05, 0x16, 0x07, 0x82, 0xfb, 0x6c, 0xeb, 0x76, 0x16, 0x8a, 0x74, 0xbc, 0x9e, 0x6f, 0xf4,
-	0x1c, 0x79, 0x9e, 0x15, 0x10, 0xd0, 0x3d, 0x98, 0x39, 0xf4, 0x24, 0x64, 0x94, 0x0a, 0xa0, 0xa7,
-	0x19, 0xa2, 0x4d, 0x1f, 0x7a, 0x0d, 0x13, 0x7d, 0x04, 0xd0, 0xf7, 0xb0, 0x29, 0xce, 0xb0, 0xf2,
-	0x59, 0xd5, 0xc2, 0x7e, 0xf8, 0x54, 0x9a, 0x0a, 0xf0, 0x0b, 0x12, 0xf7, 0xa1, 0x64, 0xd9, 0xc4,
-	0xc4, 0xec, 0xcc, 0xd1, 0x14, 0xa8, 0xd2, 0x08, 0x71, 0xe0, 0x12, 0xfb, 0x1e, 0x36, 0x55, 0x2c,
-	0xd6, 0x42, 0xe9, 0x5f, 0x11, 0x28, 0x4d, 0x58, 0xe2, 0x49, 0xeb, 0x30, 0x30, 0x5c, 0x46, 0xec,
-	0xda, 0xb0, 0xd1, 0x31, 0x6f, 0x69, 0x15, 0x4b, 0x94, 0x36, 0x52, 0x54, 0xbd, 0x0b, 0x27, 0x23,
-	0x3b, 0xa4, 0x09, 0xb6, 0x2c, 0xea, 0x6e, 0x0c, 0x28, 0x19, 0x84, 0xb3, 0x80, 0x21, 0x64, 0x34,
-	0x8f, 0x82, 0x21, 0x3c, 0x0e, 0x43, 0x78, 0xea, 0xe7, 0x70, 0x3a, 0x82, 0xe8, 0x44, 0x2c, 0xba,
-	0x1f, 0xab, 0xdc, 0x2e, 0x8f, 0xd2, 0x1a, 0x2b, 0xe1, 0xfe, 0x57, 0x81, 0xe5, 0x34, 0x86, 0x37,
-	0x44, 0x1c, 0x7f, 0x9c, 0x71, 0x19, 0xee, 0xce, 0x78, 0x66, 0xfd, 0x4e, 0xd0, 0xda, 0x3d, 0xa8,
-	0xa5, 0xf9, 0x33, 0x39, 0x4b, 0xf9, 0x49, 0x66, 0xe9, 0x67, 0xf9, 0x10, 0xf2, 0x5e, 0xf7, 0x7d,
-	0xd7, 0x3a, 0xe8, 0xd3, 0x90, 0x7f, 0xeb, 0x68, 0x56, 0x23, 0xc0, 0x65, 0xb8, 0x6b, 0x6f, 0x0d,
-	0x11, 0x1f, 0xd8, 0x91, 0x8a, 0xcd, 0x7c, 0x1a, 0xc5, 0x66, 0x38, 0xa6, 0x7e, 0x7b, 0x3c, 0x7d,
-	0xdf, 0x59, 0x00, 0xf4, 0x67, 0x39, 0x58, 0x88, 0x4e, 0x11, 0xda, 0x06, 0x30, 0x02, 0xcb, 0xc5,
-	0x87, 0x72, 0x69, 0xac, 0x61, 0x6a, 0x21, 0x41, 0xf4, 0x1e, 0xe4, 0xdb, 0x4e, 0x5f, 0xcc, 0x5a,
-	0xca, 0x61, 0xf0, 0xa6, 0xd3, 0xe7, 0x19, 0x85, 0xb2, 0xd1, 0x3d, 0x15, 0x3f, 0xdb, 0xcf, 0xce,
-	0x92, 0xcf, 0xd9, 0x7b, 0x2e, 0x23, 0x98, 0xd1, 0x53, 0x58, 0x78, 0xe5, 0x5a, 0xbe, 0x71, 0xd0,
-	0xc5, 0x7a, 0xd7, 0x38, 0xc6, 0xae, 0xc8, 0x92, 0x63, 0x24, 0xb2, 0xb2, 0x14, 0x7c, 0x46, 0xe5,
-	0xd4, 0x3f, 0x81, 0x82, 0xb4, 0x68, 0xc4, 0x8a, 0xb0, 0x07, 0x2b, 0x7d, 0xca, 0xa6, 0xb3, 0x8b,
-	0x6b, 0xb6, 0x61, 0x13, 0xdd, 0xc3, 0x74, 0x19, 0x97, 0x57, 0xea, 0x47, 0xa4, 0xe8, 0x65, 0x26,
-	0xbd, 0x49, 0x5c, 0xdc, 0x34, 0x6c, 0xd2, 0xe2, 0xa2, 0xea, 0x4b, 0x28, 0x85, 0x06, 0x38, 0xc2,
-	0x84, 0x06, 0x2c, 0xc9, 0xa3, 0x78, 0x0f, 0xfb, 0x62, 0x79, 0x19, 0xab, 0xf3, 0x45, 0x21, 0xd7,
-	0xc2, 0x3e, 0xbf, 0x3e, 0x71, 0x1f, 0x4e, 0x6b, 0x98, 0x38, 0xd8, 0x0e, 0xe6, 0xf3, 0x19, 0xe9,
-	0x4c, 0x90, 0xc1, 0xcf, 0x42, 0x2d, 0x4d, 0x9e, 0xe7, 0x87, 0xeb, 0x67, 0xa1, 0x20, 0x7f, 0x5d,
-	0x8a, 0xe6, 0x20, 0xbf, 0xb7, 0xb9, 0x5b, 0x99, 0xa2, 0x0f, 0xfb, 0x5b, 0xbb, 0x15, 0xe5, 0x7a,
-	0x0f, 0x2a, 0xf1, 0x1f, 0x54, 0xa2, 0x15, 0x38, 0xb1, 0xab, 0xed, 0xec, 0xd6, 0x9f, 0xd4, 0xf7,
-	0x1a, 0x3b, 0x4d, 0x7d, 0x57, 0x6b, 0x7c, 0x5c, 0xdf, 0xdb, 0xae, 0x4c, 0xa1, 0x35, 0x38, 0x17,
-	0x7e, 0xf1, 0x74, 0xa7, 0xb5, 0xa7, 0xef, 0xed, 0xe8, 0x9b, 0x3b, 0xcd, 0xbd, 0x7a, 0xa3, 0xb9,
-	0xad, 0x55, 0x14, 0x74, 0x0e, 0x4e, 0x87, 0x59, 0x1e, 0x35, 0xb6, 0x1a, 0xda, 0xf6, 0x26, 0x7d,
-	0xae, 0x3f, 0xab, 0xe4, 0xae, 0xdf, 0x82, 0x72, 0xe4, 0xf7, 0x8f, 0xd4, 0x90, 0xdd, 0x9d, 0xad,
-	0xca, 0x14, 0x2a, 0x43, 0x31, 0xac, 0xa7, 0x00, 0xd3, 0xcd, 0x9d, 0xad, 0xed, 0x4a, 0xee, 0xfa,
-	0x5d, 0x58, 0x8c, 0xdd, 0x9c, 0x45, 0x4b, 0x50, 0x6e, 0xd5, 0x9b, 0x5b, 0x8f, 0x76, 0x3e, 0xd5,
-	0xb5, 0xed, 0xfa, 0xd6, 0x67, 0x95, 0x29, 0xb4, 0x0c, 0x15, 0x49, 0x6a, 0xee, 0xec, 0x71, 0xaa,
-	0x72, 0xfd, 0x45, 0xec, 0xcb, 0xc2, 0xe8, 0x24, 0x2c, 0x05, 0xdd, 0xe8, 0x9b, 0xda, 0x76, 0x7d,
-	0x6f, 0x9b, 0xf6, 0x1e, 0x21, 0x6b, 0xfb, 0xcd, 0x66, 0xa3, 0xf9, 0xa4, 0xa2, 0x50, 0xad, 0x03,
-	0xf2, 0xf6, 0xa7, 0x0d, 0xca, 0x9c, 0x8b, 0x32, 0xef, 0x37, 0x7f, 0xd8, 0xdc, 0xf9, 0xa4, 0x59,
-	0xc9, 0x6f, 0xfc, 0x72, 0x09, 0x16, 0x64, 0x79, 0x87, 0x5d, 0x76, 0x97, 0x65, 0x17, 0xe6, 0xe4,
-	0x6f, 0x94, 0x53, 0xf2, 0x72, 0xf4, 0x97, 0xd5, 0xb5, 0xb5, 0x21, 0x1c, 0xa2, 0xca, 0x9e, 0x42,
-	0x07, 0xac, 0xea, 0x0d, 0xdd, 0x64, 0xbe, 0x9c, 0x5a, 0x63, 0x26, 0x2e, 0x4f, 0xd7, 0xae, 0x8c,
-	0xe4, 0x0b, 0xfa, 0xc0, 0xb4, 0xb0, 0x0d, 0xff, 0x54, 0x07, 0x5d, 0x49, 0xab, 0x48, 0x53, 0x7e,
-	0x0b, 0x54, 0xbb, 0x3a, 0x9a, 0x31, 0xe8, 0xe6, 0x05, 0x54, 0xe2, 0x3f, 0xdb, 0x41, 0x29, 0x80,
-	0x69, 0xc6, 0x6f, 0x83, 0x6a, 0xd7, 0xc7, 0x61, 0x0d, 0x77, 0x96, 0xf8, 0x81, 0xcb, 0xb5, 0x71,
-	0x7e, 0x31, 0x90, 0xd9, 0x59, 0xd6, 0x8f, 0x0b, 0xb8, 0x03, 0xa3, 0xb7, 0x94, 0x51, 0xea, 0xaf,
-	0x49, 0x52, 0xee, 0xb8, 0xa7, 0x39, 0x30, 0xfd, 0xc2, 0xb3, 0x3a, 0x85, 0x8e, 0x60, 0x31, 0x76,
-	0x29, 0x01, 0xa5, 0x88, 0xa7, 0xdf, 0xbe, 0xa8, 0x5d, 0x1b, 0x83, 0x33, 0x1a, 0x11, 0xe1, 0x4b,
-	0x08, 0xe9, 0x11, 0x91, 0x72, 0xc5, 0x21, 0x3d, 0x22, 0x52, 0xef, 0x33, 0xb0, 0xe0, 0x8e, 0x5c,
-	0x3e, 0x48, 0x0b, 0xee, 0xb4, 0x2b, 0x0f, 0xb5, 0x2b, 0x23, 0xf9, 0xc2, 0x4e, 0x8b, 0x5d, 0x45,
-	0x48, 0x73, 0x5a, 0xfa, 0x55, 0x87, 0xda, 0xb5, 0x31, 0x38, 0xe3, 0x51, 0x30, 0x38, 0xd8, 0xcc,
-	0x8a, 0x82, 0xc4, 0x31, 0x7c, 0x56, 0x14, 0x24, 0xcf, 0x48, 0x45, 0x14, 0xc4, 0x0e, 0x24, 0xaf,
-	0x8e, 0x71, 0x80, 0x92, 0x1d, 0x05, 0xe9, 0x47, 0x2d, 0xea, 0x14, 0xfa, 0x53, 0x05, 0xaa, 0x59,
-	0x87, 0x13, 0x28, 0xa5, 0xaa, 0x1b, 0x71, 0x9e, 0x52, 0xdb, 0x98, 0x44, 0x24, 0xb0, 0xe2, 0x4b,
-	0x40, 0xc9, 0xd5, 0x0e, 0xbd, 0x9b, 0x36, 0x33, 0x19, 0x6b, 0x6a, 0xed, 0xbd, 0xf1, 0x98, 0x83,
-	0x2e, 0x5b, 0x50, 0x90, 0xc7, 0x21, 0x28, 0x25, 0x4b, 0xc7, 0x0e, 0x63, 0x6a, 0xea, 0x30, 0x96,
-	0x40, 0xe9, 0x13, 0x98, 0xa6, 0x54, 0x74, 0x2e, 0x9d, 0x5b, 0x2a, 0x5b, 0xcd, 0x7a, 0x1d, 0x28,
-	0x7a, 0x0e, 0xb3, 0x1c, 0xff, 0x47, 0x29, 0x78, 0x43, 0xe4, 0x94, 0xa2, 0x76, 0x21, 0x9b, 0x21,
-	0x50, 0xf7, 0x05, 0xff, 0xf7, 0x15, 0x02, 0xda, 0x47, 0xef, 0xa4, 0xff, 0x1e, 0x38, 0x7a, 0x92,
-	0x50, 0xbb, 0x34, 0x82, 0x2b, 0xfc, 0x51, 0xc4, 0x6a, 0xdd, 0x2b, 0x23, 0x37, 0x2c, 0xd9, 0x1f,
-	0x45, 0xfa, 0x96, 0x88, 0x07, 0x49, 0x72, 0xcb, 0x94, 0x16, 0x24, 0x99, 0x1b, 0xd5, 0xb4, 0x20,
-	0xc9, 0xde, 0x85, 0xa9, 0x53, 0xc8, 0x87, 0x13, 0x29, 0x00, 0x19, 0x7a, 0x2f, 0x2b, 0xc8, 0xd3,
-	0xd0, 0xba, 0xda, 0x8d, 0x31, 0xb9, 0xc3, 0x93, 0x2f, 0x3e, 0xfa, 0xf3, 0xd9, 0xa8, 0x51, 0xe6,
-	0xe4, 0xc7, 0x3f, 0xf1, 0x8d, 0x7f, 0xcf, 0xc3, 0x3c, 0x07, 0x3f, 0x45, 0x05, 0xf3, 0x19, 0xc0,
-	0xe0, 0xdc, 0x01, 0x5d, 0x4c, 0xf7, 0x49, 0xe4, 0x6c, 0xa6, 0xf6, 0xce, 0x70, 0xa6, 0x70, 0xa0,
-	0x85, 0x30, 0xfc, 0xb4, 0x40, 0x4b, 0x1e, 0x55, 0xa4, 0x05, 0x5a, 0xca, 0x41, 0x80, 0x3a, 0x85,
-	0x3e, 0x86, 0x62, 0x00, 0x16, 0xa3, 0x34, 0xb0, 0x39, 0x86, 0x86, 0xd7, 0x2e, 0x0e, 0xe5, 0x09,
-	0x5b, 0x1d, 0x42, 0x82, 0xd3, 0xac, 0x4e, 0x22, 0xce, 0x69, 0x56, 0xa7, 0xc1, 0xc9, 0x03, 0x9f,
-	0x70, 0xbc, 0x28, 0xd3, 0x27, 0x11, 0xb8, 0x2e, 0xd3, 0x27, 0x51, 0xd0, 0x49, 0x9d, 0x7a, 0x74,
-	0xf9, 0xd7, 0x5f, 0xad, 0x2a, 0xff, 0xf2, 0xd5, 0xea, 0xd4, 0x4f, 0xbf, 0x5e, 0x55, 0x7e, 0xfd,
-	0xf5, 0xaa, 0xf2, 0xcf, 0x5f, 0xaf, 0x2a, 0xff, 0xf1, 0xf5, 0xaa, 0xf2, 0x97, 0xff, 0xb9, 0x3a,
-	0xf5, 0xa3, 0x82, 0x94, 0x3e, 0x98, 0x65, 0xff, 0x84, 0xe6, 0x83, 0xff, 0x0f, 0x00, 0x00, 0xff,
-	0xff, 0xb5, 0x80, 0xdd, 0xf4, 0x4a, 0x48, 0x00, 0x00,
+	0x76, 0x57, 0x93, 0xfa, 0x20, 0x1f, 0x45, 0x89, 0x2a, 0xcb, 0x16, 0x4d, 0x8f, 0x35, 0x56, 0xcf,
+	0xf8, 0x73, 0x66, 0xe4, 0xb1, 0x66, 0xd7, 0x13, 0xdb, 0xb3, 0xb6, 0x69, 0x49, 0xb6, 0x99, 0xb5,
+	0x29, 0xa6, 0x29, 0xcd, 0xc7, 0xce, 0x00, 0xbd, 0x2d, 0x76, 0x89, 0xea, 0x35, 0xd9, 0xd5, 0xd3,
+	0xdd, 0xb4, 0xad, 0x04, 0x08, 0x16, 0x08, 0xb2, 0x87, 0x00, 0x01, 0x72, 0xce, 0x71, 0x73, 0xc8,
+	0x21, 0xb7, 0x00, 0x41, 0x0e, 0x39, 0x6d, 0xb0, 0x87, 0xbd, 0x04, 0xc8, 0x69, 0x91, 0x8f, 0x4b,
+	0x66, 0x92, 0x5c, 0x72, 0x08, 0xf2, 0x07, 0xe4, 0x10, 0xd4, 0x57, 0xb3, 0x3f, 0xf9, 0xe1, 0xf1,
+	0xee, 0x4c, 0x4e, 0xea, 0x7a, 0xfd, 0xde, 0xab, 0x57, 0xaf, 0x5e, 0xbf, 0x7a, 0xf5, 0xab, 0xa2,
+	0xa0, 0x68, 0x38, 0xd6, 0xa6, 0xe3, 0x12, 0x9f, 0xa0, 0x8a, 0x3b, 0xb0, 0x7d, 0xab, 0x8f, 0x37,
+	0x9f, 0xdf, 0x30, 0x7a, 0xce, 0xb1, 0xb1, 0x55, 0x7b, 0xaf, 0x6b, 0xf9, 0xc7, 0x83, 0xc3, 0xcd,
+	0x0e, 0xe9, 0x5f, 0xef, 0x92, 0x2e, 0xb9, 0xce, 0x18, 0x0f, 0x07, 0x47, 0xac, 0xc5, 0x1a, 0xec,
+	0x89, 0x2b, 0x50, 0xaf, 0xc1, 0xd2, 0xc7, 0xd8, 0xf5, 0x2c, 0x62, 0x6b, 0xf8, 0xcb, 0x01, 0xf6,
+	0x7c, 0x54, 0x85, 0x85, 0xe7, 0x9c, 0x52, 0x55, 0x2e, 0x28, 0x57, 0x8a, 0x9a, 0x6c, 0xaa, 0x7f,
+	0xa9, 0xc0, 0x72, 0xc0, 0xec, 0x39, 0xc4, 0xf6, 0x70, 0x36, 0x37, 0xda, 0x80, 0x45, 0x61, 0x9c,
+	0x6e, 0x1b, 0x7d, 0x5c, 0xcd, 0xb1, 0xd7, 0x25, 0x41, 0x6b, 0x1a, 0x7d, 0x8c, 0x2e, 0xc3, 0xb2,
+	0x64, 0x91, 0x4a, 0xf2, 0x8c, 0x6b, 0x49, 0x90, 0x45, 0x6f, 0x68, 0x13, 0x4e, 0x49, 0x46, 0xc3,
+	0xb1, 0x02, 0xe6, 0x59, 0xc6, 0xbc, 0x22, 0x5e, 0xd5, 0x1d, 0x4b, 0xf0, 0xab, 0x9f, 0x43, 0x71,
+	0xa7, 0xd9, 0xde, 0x26, 0xf6, 0x91, 0xd5, 0xa5, 0x26, 0x7a, 0xd8, 0xa5, 0x32, 0x55, 0xe5, 0x42,
+	0x9e, 0x9a, 0x28, 0x9a, 0xa8, 0x06, 0x05, 0x0f, 0x1b, 0x6e, 0xe7, 0x18, 0x7b, 0xd5, 0x1c, 0x7b,
+	0x15, 0xb4, 0xa9, 0x14, 0x71, 0x7c, 0x8b, 0xd8, 0x5e, 0x35, 0xcf, 0xa5, 0x44, 0x53, 0xfd, 0xb9,
+	0x02, 0xa5, 0x16, 0x71, 0xfd, 0xa7, 0x86, 0xe3, 0x58, 0x76, 0x17, 0xdd, 0x84, 0x02, 0xf3, 0x65,
+	0x87, 0xf4, 0x98, 0x0f, 0x96, 0xb6, 0x6a, 0x9b, 0xf1, 0x69, 0xd9, 0x6c, 0x09, 0x0e, 0x2d, 0xe0,
+	0x45, 0x17, 0x61, 0xa9, 0x43, 0x6c, 0xdf, 0xb0, 0x6c, 0xec, 0xea, 0x0e, 0x71, 0x7d, 0xe6, 0xa2,
+	0x39, 0xad, 0x1c, 0x50, 0x69, 0x2f, 0xe8, 0x1c, 0x14, 0x8f, 0x89, 0xe7, 0x73, 0x8e, 0x3c, 0xe3,
+	0x28, 0x50, 0x02, 0x7b, 0xb9, 0x06, 0x0b, 0xec, 0xa5, 0xe5, 0x08, 0x67, 0xcc, 0xd3, 0x66, 0xc3,
+	0x51, 0x7f, 0xad, 0xc0, 0xdc, 0x53, 0x32, 0xb0, 0xfd, 0x58, 0x37, 0x86, 0x7f, 0x2c, 0x26, 0x2a,
+	0xd4, 0x8d, 0xe1, 0x1f, 0x0f, 0xbb, 0xa1, 0x1c, 0x7c, 0xae, 0x78, 0x37, 0xf4, 0x65, 0x0d, 0x0a,
+	0x2e, 0x36, 0x4c, 0x62, 0xf7, 0x4e, 0x98, 0x09, 0x05, 0x2d, 0x68, 0xd3, 0x49, 0xf4, 0x70, 0xcf,
+	0xb2, 0x07, 0x2f, 0x75, 0x17, 0xf7, 0x8c, 0x43, 0xdc, 0x63, 0xa6, 0x14, 0xb4, 0x25, 0x41, 0xd6,
+	0x38, 0x15, 0xed, 0x40, 0xc9, 0x71, 0x89, 0x63, 0x74, 0x0d, 0xea, 0xc7, 0xea, 0x1c, 0x73, 0x95,
+	0x9a, 0x74, 0x15, 0x33, 0xbb, 0x35, 0xe4, 0xd4, 0xc2, 0x62, 0xea, 0x5f, 0x2b, 0xb0, 0x4c, 0x83,
+	0xc7, 0x73, 0x8c, 0x0e, 0xde, 0x63, 0x53, 0x82, 0x6e, 0xc1, 0x82, 0x8d, 0xfd, 0x17, 0xc4, 0x7d,
+	0x26, 0x26, 0xe0, 0xcd, 0xa4, 0xd6, 0x40, 0xe6, 0x29, 0x31, 0xb1, 0x26, 0xf9, 0xd1, 0x0d, 0xc8,
+	0x3b, 0x96, 0xc9, 0x06, 0x3c, 0x81, 0x18, 0xe5, 0xa5, 0x22, 0x96, 0xd3, 0x61, 0x7e, 0x98, 0x44,
+	0xc4, 0x72, 0x3a, 0xaa, 0x0a, 0xd0, 0xb0, 0xfd, 0x9b, 0xdf, 0xfb, 0xd8, 0xe8, 0x0d, 0x30, 0x5a,
+	0x85, 0xb9, 0xe7, 0xf4, 0x81, 0x19, 0x9b, 0xd7, 0x78, 0x43, 0xfd, 0x2a, 0x0f, 0xe7, 0x9e, 0x50,
+	0x7f, 0xb5, 0x0d, 0xdb, 0x3c, 0x24, 0x2f, 0xdb, 0xb8, 0x33, 0x70, 0x2d, 0xff, 0x64, 0x9b, 0xd8,
+	0x3e, 0x7e, 0xe9, 0xa3, 0x26, 0xac, 0xd8, 0x52, 0xb3, 0x2e, 0x43, 0x93, 0x6a, 0x28, 0x6d, 0x6d,
+	0x8c, 0x30, 0x82, 0xbb, 0x48, 0xab, 0xd8, 0x51, 0x82, 0x87, 0x1e, 0x0f, 0xe7, 0x4d, 0x6a, 0xcb,
+	0x31, 0x6d, 0x29, 0x43, 0x6a, 0xef, 0x32, 0xcb, 0x84, 0x2e, 0x39, 0xb1, 0x52, 0xd3, 0x47, 0x40,
+	0xbf, 0x6a, 0xdd, 0xf0, 0xf4, 0x81, 0x87, 0x5d, 0xe6, 0x98, 0xd2, 0xd6, 0x1b, 0x49, 0x2d, 0x43,
+	0x17, 0x68, 0x45, 0x77, 0x60, 0xd7, 0xbd, 0x03, 0x0f, 0xbb, 0x2c, 0x09, 0x88, 0x58, 0xd2, 0x5d,
+	0x42, 0xfc, 0x23, 0x4f, 0xc6, 0x8f, 0x24, 0x6b, 0x8c, 0x8a, 0xae, 0xc3, 0x29, 0x6f, 0xe0, 0x38,
+	0x3d, 0xdc, 0xc7, 0xb6, 0x6f, 0xf4, 0xf4, 0xae, 0x4b, 0x06, 0x8e, 0x57, 0x9d, 0xbb, 0x90, 0xbf,
+	0x92, 0xd7, 0x50, 0xf8, 0xd5, 0x23, 0xf6, 0x06, 0xad, 0x03, 0x38, 0xae, 0xf5, 0xdc, 0xea, 0xe1,
+	0x2e, 0x36, 0xab, 0xf3, 0x4c, 0x69, 0x88, 0x82, 0xde, 0x87, 0x55, 0x0f, 0x77, 0x3a, 0xa4, 0xef,
+	0xe8, 0x8e, 0x4b, 0x8e, 0xac, 0x1e, 0xe6, 0xd1, 0xbf, 0xc0, 0xa2, 0x1f, 0x89, 0x77, 0x2d, 0xfe,
+	0x8a, 0x7d, 0x07, 0x77, 0x59, 0x4e, 0xa3, 0x23, 0x65, 0x9d, 0x57, 0x0b, 0x13, 0x0c, 0x15, 0xd8,
+	0x50, 0x99, 0x49, 0xea, 0xcf, 0x73, 0x70, 0x9a, 0x79, 0xb2, 0x45, 0x4c, 0x31, 0xcd, 0x22, 0x49,
+	0xbd, 0x05, 0xe5, 0x0e, 0xd3, 0xa9, 0x3b, 0x86, 0x8b, 0x6d, 0x5f, 0x7c, 0xa4, 0x8b, 0x9c, 0xd8,
+	0x62, 0x34, 0xf4, 0x29, 0x54, 0x3c, 0x11, 0x15, 0x7a, 0x87, 0x87, 0x85, 0x98, 0xb3, 0xf7, 0x92,
+	0x26, 0x8c, 0x88, 0x25, 0x6d, 0xd9, 0x4b, 0x04, 0xd7, 0x82, 0x77, 0xe2, 0x75, 0xfc, 0x1e, 0xcf,
+	0x76, 0xa5, 0xad, 0xef, 0x65, 0x28, 0x8c, 0x1b, 0xbe, 0xd9, 0xe6, 0x62, 0xbb, 0xb6, 0xef, 0x9e,
+	0x68, 0x52, 0x49, 0xed, 0x36, 0x2c, 0x86, 0x5f, 0xa0, 0x0a, 0xe4, 0x9f, 0xe1, 0x13, 0x31, 0x28,
+	0xfa, 0x38, 0xfc, 0x08, 0x78, 0xae, 0xe1, 0x8d, 0xdb, 0xb9, 0xdf, 0x51, 0x54, 0x17, 0xd0, 0xb0,
+	0x97, 0xa7, 0xd8, 0x37, 0x4c, 0xc3, 0x37, 0x10, 0x82, 0x59, 0xb6, 0x8c, 0x70, 0x15, 0xec, 0x99,
+	0x6a, 0x1d, 0x88, 0x8f, 0xb7, 0xa8, 0xd1, 0x47, 0xf4, 0x06, 0x14, 0x83, 0x40, 0x17, 0x6b, 0xc9,
+	0x90, 0x40, 0x73, 0xba, 0xe1, 0xfb, 0xb8, 0xef, 0xf8, 0x2c, 0xc4, 0xca, 0x9a, 0x6c, 0xaa, 0xff,
+	0x3d, 0x0b, 0x95, 0xc4, 0x9c, 0xdc, 0x87, 0x42, 0x5f, 0x74, 0x2f, 0x3e, 0xb4, 0xb7, 0x53, 0x12,
+	0x7b, 0xc2, 0x54, 0x2d, 0x90, 0xa2, 0x79, 0x93, 0xe6, 0xd0, 0xd0, 0xfa, 0x17, 0xb4, 0xe9, 0x8c,
+	0xf7, 0x48, 0x57, 0x37, 0x2d, 0x17, 0x77, 0x7c, 0xe2, 0x9e, 0x08, 0x73, 0x17, 0x7b, 0xa4, 0xbb,
+	0x23, 0x69, 0xe8, 0x36, 0x80, 0x69, 0x7b, 0x74, 0xb2, 0x8f, 0xac, 0x2e, 0x33, 0xba, 0xb4, 0x75,
+	0x2e, 0x69, 0x44, 0xb0, 0xd8, 0x69, 0x45, 0xd3, 0xf6, 0x84, 0xf9, 0x0f, 0xa0, 0x4c, 0xd7, 0x0c,
+	0xbd, 0xcf, 0xd7, 0x29, 0xfe, 0xa5, 0x94, 0xb6, 0xce, 0xa7, 0x8d, 0x21, 0x58, 0xcd, 0xb4, 0x45,
+	0x67, 0xd8, 0xf0, 0xd0, 0x43, 0x98, 0x67, 0xc9, 0xdb, 0xab, 0xce, 0x33, 0xe1, 0xcd, 0x51, 0x0e,
+	0x10, 0x11, 0xf1, 0x84, 0x09, 0xf0, 0x80, 0x10, 0xd2, 0xe8, 0x00, 0x4a, 0x86, 0x6d, 0x13, 0xdf,
+	0xe0, 0x89, 0x66, 0x81, 0x29, 0xfb, 0x60, 0x02, 0x65, 0xf5, 0xa1, 0x14, 0xd7, 0x18, 0xd6, 0x83,
+	0x7e, 0x00, 0x73, 0x2c, 0x13, 0x89, 0x0f, 0xf1, 0xf2, 0x84, 0x41, 0xab, 0x71, 0xa9, 0xda, 0x2d,
+	0x28, 0x85, 0x8c, 0x9d, 0x26, 0x48, 0x6b, 0x77, 0xa1, 0x12, 0x37, 0x6d, 0xaa, 0x20, 0xff, 0x03,
+	0x58, 0xd5, 0x06, 0xf6, 0xd0, 0x30, 0x59, 0x7d, 0xdd, 0x86, 0x79, 0x31, 0xd9, 0x3c, 0xe2, 0xd4,
+	0xf1, 0x3e, 0xd2, 0x84, 0x44, 0xb8, 0x9c, 0x3a, 0x36, 0x6c, 0xb3, 0x87, 0x5d, 0xd1, 0xaf, 0x2c,
+	0xa7, 0x1e, 0x73, 0xaa, 0xfa, 0x03, 0x38, 0x1d, 0xeb, 0x5c, 0x54, 0x73, 0x6f, 0xc3, 0x92, 0x43,
+	0x4c, 0xdd, 0xe3, 0x64, 0xdd, 0x32, 0x65, 0x1a, 0x72, 0x02, 0xde, 0x86, 0x49, 0xc5, 0xdb, 0x3e,
+	0x71, 0x92, 0xc6, 0x4f, 0x26, 0x5e, 0x85, 0x33, 0x71, 0x71, 0xde, 0xbd, 0x7a, 0x0f, 0xd6, 0x34,
+	0xdc, 0x27, 0xcf, 0xf1, 0xab, 0xaa, 0xae, 0x41, 0x35, 0xa9, 0x40, 0x28, 0xff, 0x0c, 0xd6, 0x86,
+	0xd4, 0xb6, 0x6f, 0xf8, 0x03, 0x6f, 0x2a, 0xe5, 0xa2, 0xd4, 0x3d, 0x24, 0x1e, 0x9f, 0xce, 0x82,
+	0x26, 0x9b, 0xea, 0xd5, 0xb0, 0xea, 0x26, 0xaf, 0x2c, 0x78, 0x0f, 0x68, 0x09, 0x72, 0x96, 0x23,
+	0xd4, 0xe5, 0x2c, 0x47, 0x7d, 0x0c, 0xc5, 0x60, 0x69, 0x46, 0x77, 0x86, 0x35, 0x66, 0x6e, 0xd2,
+	0x85, 0x3c, 0x28, 0x43, 0xf7, 0x13, 0x4b, 0x89, 0xe8, 0xf2, 0x0e, 0x40, 0x90, 0xf2, 0x64, 0x85,
+	0x70, 0x6e, 0x84, 0x62, 0x2d, 0xc4, 0xae, 0xfe, 0x4b, 0x24, 0x11, 0x86, 0x06, 0x61, 0x06, 0x83,
+	0x30, 0x23, 0x89, 0x31, 0xf7, 0x4a, 0x89, 0xf1, 0x43, 0x98, 0xf3, 0x7c, 0xc3, 0xc7, 0xa2, 0x8a,
+	0xda, 0x18, 0x25, 0x4e, 0x8d, 0xc0, 0x1a, 0xe7, 0x47, 0xe7, 0x01, 0x3a, 0x2e, 0x36, 0x7c, 0x6c,
+	0xea, 0x06, 0xcf, 0xe2, 0x79, 0xad, 0x28, 0x28, 0x75, 0x1f, 0x6d, 0x0f, 0x2b, 0xc1, 0x39, 0x66,
+	0xd8, 0xd5, 0x51, 0x9a, 0x23, 0x53, 0x35, 0xac, 0x09, 0x83, 0xac, 0x32, 0x3f, 0x61, 0x56, 0x11,
+	0x0a, 0xb8, 0x54, 0x28, 0x67, 0x2e, 0x8c, 0xcf, 0x99, 0x5c, 0x74, 0x92, 0x9c, 0x59, 0x18, 0x9f,
+	0x33, 0x85, 0xb2, 0x91, 0x39, 0xf3, 0xdb, 0x4c, 0x7a, 0xff, 0xac, 0x40, 0x35, 0xf9, 0x0d, 0x8a,
+	0xdc, 0x73, 0x1b, 0xe6, 0x3d, 0x46, 0x99, 0x24, 0xf3, 0x09, 0x59, 0x21, 0x81, 0x1e, 0xc3, 0xac,
+	0x65, 0x1f, 0x11, 0xb6, 0x89, 0x4b, 0xad, 0x5d, 0xb2, 0x7a, 0xdd, 0x6c, 0xd8, 0x47, 0x84, 0x3b,
+	0x89, 0x69, 0xa8, 0x7d, 0x08, 0xc5, 0x80, 0x34, 0xd5, 0xd8, 0xf6, 0x60, 0x35, 0x16, 0xb2, 0xbc,
+	0xd8, 0x0f, 0x22, 0x5d, 0x99, 0x2e, 0xd2, 0xd5, 0x9f, 0xe6, 0xc2, 0x5f, 0xe2, 0x43, 0xab, 0xe7,
+	0x63, 0x37, 0xf1, 0x25, 0x7e, 0x24, 0xb5, 0xf3, 0xcf, 0xf0, 0xd2, 0x58, 0xed, 0xbc, 0x26, 0x15,
+	0x1f, 0xd3, 0x17, 0xb0, 0xc4, 0x62, 0x4d, 0xf7, 0x70, 0x8f, 0x15, 0x1c, 0xa2, 0xf8, 0xfb, 0xfe,
+	0x28, 0x35, 0xdc, 0x12, 0x1e, 0xb1, 0x6d, 0x21, 0xc7, 0x3d, 0x58, 0xee, 0x85, 0x69, 0xb5, 0xfb,
+	0x80, 0x92, 0x4c, 0x53, 0xf9, 0xb4, 0x4d, 0x53, 0x1c, 0xdd, 0xe9, 0xa6, 0xac, 0x92, 0x47, 0xcc,
+	0x8c, 0x49, 0x62, 0x85, 0x1b, 0xac, 0x09, 0x09, 0xf5, 0x97, 0x79, 0x80, 0xe1, 0xcb, 0xff, 0x47,
+	0xb9, 0xed, 0x7e, 0x90, 0x57, 0x78, 0x21, 0x77, 0x65, 0x94, 0xe2, 0xd4, 0x8c, 0xb2, 0x17, 0xcd,
+	0x28, 0xbc, 0xa4, 0x7b, 0x6f, 0xa4, 0x9a, 0xef, 0x6c, 0x2e, 0x79, 0x02, 0x67, 0xe2, 0xb1, 0x21,
+	0x12, 0xc9, 0x16, 0xcc, 0x59, 0x3e, 0xee, 0x73, 0xb4, 0x27, 0x75, 0x77, 0x16, 0x12, 0xe2, 0xac,
+	0xea, 0x06, 0x14, 0x1b, 0x7d, 0xa3, 0x8b, 0xdb, 0x0e, 0xee, 0xd0, 0x4e, 0x2d, 0xda, 0x10, 0x86,
+	0xf0, 0x86, 0xba, 0x05, 0x85, 0x1f, 0xe2, 0x13, 0xfe, 0x51, 0x4f, 0x68, 0xa8, 0xfa, 0xa7, 0x39,
+	0x58, 0x63, 0x6b, 0xc5, 0xb6, 0xc4, 0x5a, 0x34, 0xec, 0x91, 0x81, 0xdb, 0xc1, 0x1e, 0x9b, 0x6d,
+	0x67, 0xa0, 0x3b, 0xd8, 0xb5, 0x88, 0x29, 0xa0, 0x80, 0x62, 0xc7, 0x19, 0xb4, 0x18, 0x01, 0x9d,
+	0x03, 0xda, 0xd0, 0xbf, 0x1c, 0x10, 0x11, 0x88, 0x79, 0xad, 0xd0, 0x71, 0x06, 0xbf, 0x47, 0xdb,
+	0x52, 0xd6, 0x3b, 0x36, 0x5c, 0xec, 0xb1, 0x38, 0xe3, 0xb2, 0x6d, 0x46, 0x40, 0x37, 0xe0, 0x74,
+	0x1f, 0xf7, 0x89, 0x7b, 0xa2, 0xf7, 0xac, 0xbe, 0xe5, 0xeb, 0x96, 0xad, 0x1f, 0x9e, 0xf8, 0xd8,
+	0x13, 0x31, 0x85, 0xf8, 0xcb, 0x27, 0xf4, 0x5d, 0xc3, 0x7e, 0x40, 0xdf, 0x20, 0x15, 0xca, 0x84,
+	0xf4, 0x75, 0xaf, 0x43, 0x5c, 0xac, 0x1b, 0xe6, 0x4f, 0xd8, 0xf2, 0x99, 0xd7, 0x4a, 0x84, 0xf4,
+	0xdb, 0x94, 0x56, 0x37, 0x7f, 0x82, 0xde, 0x84, 0x52, 0xc7, 0x19, 0x78, 0xd8, 0xd7, 0xe9, 0x1f,
+	0xb6, 0x3a, 0x16, 0x35, 0xe0, 0xa4, 0x6d, 0x67, 0xe0, 0x85, 0x18, 0xfa, 0xd4, 0xff, 0x0b, 0x61,
+	0x86, 0xa7, 0xd4, 0xcd, 0x06, 0x94, 0x23, 0x50, 0x02, 0xdd, 0xd5, 0x31, 0xcc, 0x40, 0xec, 0xea,
+	0xe8, 0x33, 0xa5, 0xb9, 0xa4, 0x27, 0x3d, 0xc9, 0x9e, 0x29, 0xcd, 0x3f, 0x71, 0xe4, 0x96, 0x8e,
+	0x3d, 0x53, 0x97, 0xf7, 0xf0, 0x73, 0x01, 0x37, 0x15, 0x35, 0xde, 0x50, 0x4d, 0x80, 0x6d, 0xc3,
+	0x31, 0x0e, 0xad, 0x9e, 0xe5, 0x9f, 0xa0, 0xab, 0x50, 0x31, 0x4c, 0x53, 0xef, 0x48, 0x8a, 0x85,
+	0x25, 0x08, 0xb8, 0x6c, 0x98, 0xe6, 0x76, 0x88, 0x8c, 0xde, 0x81, 0x15, 0xd3, 0x25, 0x4e, 0x94,
+	0x97, 0xa3, 0x82, 0x15, 0xfa, 0x22, 0xcc, 0xac, 0xfe, 0xc7, 0x1c, 0x9c, 0x8f, 0x4e, 0x6c, 0x1c,
+	0xae, 0xb9, 0x0f, 0x8b, 0xb1, 0x5e, 0x33, 0xa0, 0x82, 0xa1, 0xb5, 0x5a, 0x44, 0x22, 0x06, 0x5f,
+	0xe4, 0x12, 0xf0, 0x45, 0x2a, 0x20, 0x94, 0x7f, 0xad, 0x80, 0xd0, 0xec, 0x6b, 0x01, 0x84, 0xe6,
+	0xa6, 0x03, 0x84, 0x2e, 0xb1, 0x6d, 0x8c, 0x94, 0x66, 0x7b, 0x67, 0x1e, 0x6a, 0xe5, 0x80, 0xc7,
+	0x96, 0xe8, 0x71, 0x0c, 0x38, 0x5a, 0x98, 0x06, 0x38, 0x2a, 0x64, 0x02, 0x47, 0x34, 0x6a, 0x1c,
+	0xc7, 0x70, 0xfb, 0xc4, 0x95, 0xc8, 0x50, 0xb5, 0xc8, 0x4c, 0x58, 0x96, 0x74, 0x81, 0x0a, 0x65,
+	0x62, 0x48, 0x90, 0x89, 0x21, 0x5d, 0x80, 0x45, 0x9b, 0xe8, 0x36, 0x7e, 0xa1, 0xd3, 0xb9, 0xf4,
+	0xaa, 0x25, 0x3e, 0xb1, 0x36, 0x69, 0xe2, 0x17, 0x2d, 0x4a, 0x49, 0xa0, 0x4c, 0x8b, 0xd3, 0xa1,
+	0x4c, 0x68, 0x03, 0x16, 0xfb, 0x86, 0xf7, 0x0c, 0x9b, 0xcc, 0x14, 0xaf, 0x5a, 0x66, 0x41, 0x5c,
+	0xe2, 0x34, 0x6a, 0x83, 0x87, 0x2e, 0x42, 0xe0, 0x24, 0xc1, 0xb4, 0xc4, 0x98, 0xca, 0x92, 0xca,
+	0xd8, 0xd4, 0xbf, 0x53, 0x60, 0x35, 0x1a, 0xe6, 0x02, 0x5b, 0x78, 0x04, 0x45, 0x57, 0x66, 0x32,
+	0x11, 0xda, 0x57, 0x33, 0xca, 0xe4, 0x64, 0xea, 0xd3, 0x86, 0xb2, 0xe8, 0x47, 0x99, 0x90, 0xd6,
+	0xf5, 0x71, 0xfa, 0xc6, 0x81, 0x5a, 0x6a, 0x03, 0xde, 0xfc, 0xc4, 0xb2, 0x4d, 0xf2, 0xc2, 0xcb,
+	0xfc, 0x4a, 0x53, 0x62, 0x4d, 0x49, 0x89, 0x35, 0xf5, 0x17, 0x0a, 0x9c, 0x89, 0xeb, 0x12, 0xae,
+	0x68, 0x24, 0x5d, 0xf1, 0x4e, 0xd2, 0xf4, 0xb8, 0x70, 0xaa, 0x33, 0xbe, 0xc8, 0x74, 0xc6, 0x8d,
+	0xf1, 0x1a, 0xc7, 0xba, 0xe3, 0xaf, 0x14, 0x38, 0x9b, 0x69, 0x46, 0x6c, 0x49, 0x51, 0xe2, 0x4b,
+	0x8a, 0x58, 0x8e, 0x3a, 0x64, 0x60, 0xfb, 0xa1, 0xe5, 0x68, 0x9b, 0x1d, 0x31, 0xf0, 0xbc, 0xaf,
+	0xf7, 0x8d, 0x97, 0x56, 0x7f, 0xd0, 0x17, 0xeb, 0x11, 0x55, 0xf7, 0x94, 0x53, 0x5e, 0x61, 0x41,
+	0x52, 0xeb, 0xb0, 0x12, 0x58, 0x39, 0x12, 0x04, 0x0c, 0x81, 0x7a, 0xb9, 0x28, 0xa8, 0x67, 0xc3,
+	0xfc, 0x0e, 0x7e, 0x6e, 0x75, 0xf0, 0x6b, 0x39, 0x03, 0xb9, 0x00, 0x25, 0x07, 0xbb, 0x7d, 0xcb,
+	0xf3, 0x82, 0x44, 0x5b, 0xd4, 0xc2, 0x24, 0xf5, 0x3f, 0xe7, 0x61, 0x39, 0x1e, 0x1d, 0xf7, 0x12,
+	0x18, 0xe2, 0x5b, 0x29, 0x4b, 0x40, 0x7c, 0xa0, 0xa1, 0x6a, 0xf2, 0x86, 0x2c, 0x46, 0x72, 0x59,
+	0x1b, 0xf9, 0xa0, 0x70, 0x11, 0x95, 0x0a, 0xf5, 0x48, 0x87, 0xf4, 0xfb, 0x86, 0x6d, 0xca, 0xa3,
+	0x2b, 0xd1, 0xa4, 0xfe, 0x33, 0xdc, 0x2e, 0x75, 0x3b, 0x25, 0xb3, 0x67, 0x3a, 0x79, 0x74, 0xd7,
+	0x6b, 0xd9, 0x0c, 0x8b, 0x64, 0xc9, 0xba, 0xa8, 0x81, 0x20, 0xed, 0x58, 0x2e, 0xda, 0x84, 0x59,
+	0x6c, 0x3f, 0x97, 0xe5, 0x62, 0xca, 0xd9, 0x96, 0x2c, 0x8b, 0x34, 0xc6, 0x87, 0xae, 0xc3, 0x7c,
+	0x9f, 0x86, 0x85, 0xdc, 0xff, 0xae, 0x65, 0x1c, 0xf1, 0x68, 0x82, 0x0d, 0x6d, 0xc1, 0x82, 0xc9,
+	0xe6, 0x49, 0x6e, 0x72, 0xab, 0x29, 0x08, 0x27, 0x63, 0xd0, 0x24, 0x23, 0xda, 0x0d, 0x8a, 0xe1,
+	0x62, 0x56, 0x15, 0x1b, 0x9b, 0x8a, 0xd4, 0x8a, 0x78, 0x3f, 0x5a, 0x11, 0x03, 0xd3, 0xb5, 0x35,
+	0x5e, 0xd7, 0x68, 0x58, 0xf2, 0x2c, 0x14, 0x7a, 0xa4, 0xcb, 0xc3, 0xa8, 0xc4, 0x4f, 0x45, 0x7b,
+	0xa4, 0xcb, 0xa2, 0x68, 0x95, 0x6e, 0x0e, 0x4c, 0xcb, 0x66, 0x49, 0xbd, 0xa0, 0xf1, 0x06, 0xfd,
+	0xf8, 0xd8, 0x83, 0x4e, 0xec, 0x0e, 0xae, 0x96, 0xd9, 0xab, 0x22, 0xa3, 0xec, 0xd9, 0x1d, 0x56,
+	0x6e, 0xfa, 0xfe, 0x49, 0x75, 0x89, 0xd1, 0xe9, 0x23, 0xdd, 0xf7, 0x71, 0x88, 0x62, 0x39, 0x6b,
+	0xdf, 0x97, 0x96, 0xb6, 0x25, 0x42, 0xf1, 0x00, 0x16, 0x5e, 0xf0, 0x44, 0x50, 0xad, 0x30, 0xf9,
+	0x2b, 0xe3, 0xd3, 0x8b, 0xd0, 0x20, 0x05, 0xbf, 0xcd, 0xd2, 0xff, 0x97, 0x0a, 0x9c, 0xd9, 0x66,
+	0xdb, 0xa2, 0x50, 0x1e, 0x9b, 0x06, 0xc9, 0xbb, 0x15, 0x80, 0xac, 0x99, 0xb0, 0x5b, 0x7c, 0xdc,
+	0x12, 0x63, 0x6d, 0xc0, 0x92, 0x54, 0x2e, 0x54, 0xe4, 0x27, 0xc6, 0x69, 0xcb, 0x5e, 0xb8, 0xa9,
+	0x7e, 0x04, 0x6b, 0x89, 0x51, 0x88, 0x2d, 0xcc, 0x06, 0x2c, 0x0e, 0xf3, 0x55, 0x30, 0x88, 0x52,
+	0x40, 0x6b, 0x98, 0xea, 0x6d, 0x38, 0xdd, 0xf6, 0x0d, 0xd7, 0x4f, 0xb8, 0x60, 0x02, 0x59, 0x86,
+	0xc0, 0x46, 0x65, 0x05, 0x48, 0xda, 0x86, 0xd5, 0xb6, 0x4f, 0x9c, 0x57, 0x50, 0x4a, 0xb3, 0x0e,
+	0x1d, 0x3f, 0x19, 0xc8, 0xf5, 0x41, 0x36, 0xd5, 0x35, 0x8e, 0x17, 0x27, 0x7b, 0xbb, 0x03, 0x67,
+	0x38, 0x5c, 0xfb, 0x2a, 0x83, 0x38, 0x2b, 0xc1, 0xe2, 0xa4, 0xde, 0xa7, 0x70, 0x6a, 0xb8, 0x2c,
+	0x0e, 0xa1, 0x98, 0x9b, 0x51, 0x28, 0xe6, 0xc2, 0x88, 0x59, 0x8f, 0x20, 0x31, 0x7f, 0x91, 0x0b,
+	0xe5, 0xf5, 0x0c, 0x20, 0xe6, 0x4e, 0x14, 0x88, 0xb9, 0x38, 0x4e, 0x77, 0x04, 0x87, 0x49, 0x46,
+	0x6d, 0x3e, 0x25, 0x6a, 0x3f, 0x4f, 0xa0, 0x35, 0xb3, 0x59, 0x70, 0x57, 0xcc, 0xda, 0xdf, 0x0a,
+	0x58, 0xa3, 0x71, 0xb0, 0x26, 0xe8, 0x3a, 0x40, 0xd7, 0x6f, 0xc5, 0xc0, 0x9a, 0x8d, 0xb1, 0xf6,
+	0x06, 0x58, 0xcd, 0xdf, 0xcc, 0x42, 0x31, 0x78, 0x97, 0xf0, 0x79, 0xd2, 0x6d, 0xb9, 0x14, 0xb7,
+	0x85, 0x57, 0xe0, 0xfc, 0x37, 0x5a, 0x81, 0x67, 0x27, 0x5e, 0x81, 0xcf, 0x41, 0x91, 0x3d, 0xe8,
+	0x2e, 0x3e, 0x12, 0x2b, 0x6a, 0x81, 0x11, 0x34, 0x7c, 0x34, 0x0c, 0xc3, 0xf9, 0xa9, 0xc2, 0x30,
+	0x06, 0x0f, 0x2d, 0xc4, 0xe1, 0xa1, 0x7b, 0xc1, 0x8a, 0xc8, 0x17, 0xd1, 0xcb, 0x23, 0xf4, 0xa6,
+	0xae, 0x85, 0xcd, 0xe8, 0x5a, 0xc8, 0xd7, 0xd5, 0x77, 0x47, 0x69, 0xf9, 0xce, 0x82, 0x43, 0x07,
+	0x1c, 0x1c, 0x0a, 0xc7, 0xa2, 0xc8, 0xac, 0x77, 0x00, 0x82, 0x24, 0x22, 0x11, 0xa2, 0x73, 0x23,
+	0xc6, 0xa8, 0x85, 0xd8, 0xa9, 0xda, 0xc8, 0xd4, 0x0c, 0x4f, 0x90, 0x26, 0xcb, 0x8f, 0x19, 0xc7,
+	0x47, 0xff, 0x3b, 0x17, 0xca, 0x2f, 0x19, 0x47, 0x2e, 0xf7, 0x12, 0xb0, 0xe4, 0x94, 0x51, 0x7c,
+	0x33, 0x8a, 0x4a, 0xbe, 0x62, 0xd4, 0x25, 0x40, 0x49, 0x56, 0xb9, 0x18, 0xae, 0x78, 0xcd, 0x41,
+	0xa3, 0xa2, 0xa0, 0xd4, 0xd9, 0xce, 0xe0, 0xc8, 0xb2, 0x2d, 0xef, 0x98, 0xbf, 0x9f, 0xe7, 0x3b,
+	0x03, 0x49, 0xaa, 0xb3, 0xdb, 0x4d, 0xf8, 0xa5, 0xe5, 0xeb, 0x1d, 0x62, 0x62, 0x16, 0xd3, 0x73,
+	0x5a, 0x81, 0x12, 0xb6, 0x89, 0x89, 0x87, 0x5f, 0x5e, 0xe1, 0xd5, 0xbe, 0xbc, 0x62, 0xec, 0xcb,
+	0x3b, 0x03, 0xf3, 0x2e, 0x36, 0x3c, 0x62, 0x8b, 0xed, 0xb9, 0x68, 0xd1, 0xa9, 0xe9, 0x63, 0xcf,
+	0xa3, 0x3d, 0x89, 0x72, 0x4d, 0x34, 0x43, 0x65, 0xe6, 0xe2, 0xd8, 0x32, 0x73, 0xc4, 0x51, 0x4e,
+	0xac, 0xcc, 0x2c, 0x8f, 0x2d, 0x33, 0x27, 0x39, 0xc9, 0x09, 0x15, 0xda, 0x4b, 0x93, 0x15, 0xda,
+	0xe1, 0xba, 0x74, 0x39, 0x52, 0x97, 0x7e, 0x9b, 0x1f, 0xeb, 0xaf, 0x15, 0x58, 0x4b, 0x7c, 0x56,
+	0xe2, 0x73, 0xbd, 0x15, 0x3b, 0x14, 0xda, 0x18, 0xeb, 0xb3, 0xe0, 0x4c, 0xe8, 0x51, 0xe4, 0x4c,
+	0xe8, 0x83, 0xf1, 0x82, 0xaf, 0xfd, 0x48, 0xe8, 0x8f, 0x15, 0x78, 0xf3, 0xc0, 0x31, 0x63, 0x15,
+	0x9e, 0xd8, 0xf6, 0x4f, 0x9e, 0x38, 0xee, 0xc9, 0x5a, 0x3f, 0x37, 0x2d, 0xce, 0xc2, 0xe5, 0x54,
+	0x15, 0x2e, 0x64, 0x9b, 0x21, 0x4a, 0xa6, 0x1f, 0xc3, 0xf2, 0xee, 0x4b, 0xdc, 0x69, 0x9f, 0xd8,
+	0x9d, 0x29, 0x4c, 0xab, 0x40, 0xbe, 0xd3, 0x37, 0x05, 0x4a, 0x4a, 0x1f, 0xc3, 0x55, 0x60, 0x3e,
+	0x5a, 0x05, 0xea, 0x50, 0x19, 0xf6, 0x20, 0xa6, 0xf7, 0x0c, 0x9d, 0x5e, 0x93, 0x32, 0x53, 0xe5,
+	0x8b, 0x9a, 0x68, 0x09, 0x3a, 0x76, 0xf9, 0x05, 0x06, 0x4e, 0xc7, 0xae, 0x1b, 0xcd, 0x16, 0xf9,
+	0x68, 0xb6, 0x50, 0xff, 0x5c, 0x81, 0x12, 0xed, 0xe1, 0x1b, 0xd9, 0x2f, 0xb6, 0x5a, 0xf9, 0xe1,
+	0x56, 0x2b, 0xd8, 0xb1, 0xcd, 0x86, 0x77, 0x6c, 0x43, 0xcb, 0xe7, 0x18, 0x39, 0x69, 0xf9, 0x7c,
+	0x40, 0xc7, 0xae, 0xab, 0x5e, 0x80, 0x45, 0x6e, 0x9b, 0x18, 0x79, 0x05, 0xf2, 0x03, 0xb7, 0x27,
+	0xe3, 0x68, 0xe0, 0xf6, 0xd4, 0x3f, 0x51, 0xa0, 0x5c, 0xf7, 0x7d, 0xa3, 0x73, 0x3c, 0xc5, 0x00,
+	0x02, 0xe3, 0x72, 0x61, 0xe3, 0x92, 0x83, 0x18, 0x9a, 0x3b, 0x9b, 0x61, 0xee, 0x5c, 0xc4, 0x5c,
+	0x15, 0x96, 0xa4, 0x2d, 0x99, 0x06, 0x37, 0x01, 0xb5, 0x88, 0xeb, 0x3f, 0x24, 0xee, 0x0b, 0xc3,
+	0x35, 0xa7, 0xdb, 0x81, 0x21, 0x98, 0x15, 0x37, 0x5e, 0xf3, 0x57, 0xe6, 0x34, 0xf6, 0xac, 0x5e,
+	0x86, 0x53, 0x11, 0x7d, 0x99, 0x1d, 0xdf, 0x87, 0x12, 0xcb, 0xfb, 0xa2, 0x14, 0xbf, 0x11, 0x3e,
+	0xae, 0x99, 0x68, 0x95, 0x50, 0x7f, 0x17, 0x56, 0x68, 0x7d, 0xc0, 0xe8, 0xc1, 0xa7, 0xf8, 0xfd,
+	0x58, 0x9d, 0x7a, 0x3e, 0x43, 0x51, 0xac, 0x46, 0xfd, 0x5b, 0x05, 0xe6, 0x18, 0x3d, 0xb1, 0x66,
+	0x9f, 0x83, 0xa2, 0x8b, 0x1d, 0xa2, 0xfb, 0x46, 0x37, 0xb8, 0x5f, 0x4c, 0x09, 0xfb, 0x46, 0xd7,
+	0x63, 0xd7, 0xa3, 0xe9, 0x4b, 0xd3, 0xea, 0x62, 0xcf, 0x97, 0x97, 0x8c, 0x4b, 0x94, 0xb6, 0xc3,
+	0x49, 0xd4, 0x49, 0x9e, 0xf5, 0xfb, 0xbc, 0xee, 0x9c, 0xd5, 0xd8, 0x33, 0xda, 0xe4, 0x57, 0xde,
+	0x26, 0x81, 0xd4, 0xd9, 0x85, 0xb8, 0x1a, 0x14, 0x62, 0x28, 0x7a, 0xd0, 0x56, 0x77, 0x01, 0x85,
+	0xbd, 0x20, 0xfc, 0x7d, 0x1d, 0xe6, 0x99, 0x93, 0x64, 0x75, 0xb4, 0x96, 0xe1, 0x06, 0x4d, 0xb0,
+	0xa9, 0x06, 0x20, 0xee, 0xe0, 0x48, 0x45, 0x34, 0xfd, 0xac, 0x8c, 0xa8, 0x90, 0xfe, 0x5e, 0x81,
+	0x53, 0x91, 0x3e, 0x84, 0xad, 0xef, 0x45, 0x3b, 0xc9, 0x34, 0x55, 0x74, 0xb0, 0x1d, 0x59, 0x12,
+	0xae, 0x67, 0x99, 0xf4, 0x1b, 0x5a, 0x0e, 0xfe, 0x41, 0x01, 0xa8, 0x0f, 0xfc, 0x63, 0x81, 0x0c,
+	0x86, 0x67, 0x46, 0x89, 0xce, 0x0c, 0x7d, 0xe7, 0x18, 0x9e, 0xf7, 0x82, 0xb8, 0x72, 0x4f, 0x13,
+	0xb4, 0x19, 0x86, 0x37, 0xf0, 0x8f, 0xe5, 0x51, 0x18, 0x7d, 0x46, 0x17, 0x61, 0x89, 0xdf, 0x69,
+	0xd7, 0x0d, 0xd3, 0x74, 0xb1, 0xe7, 0x89, 0x33, 0xb1, 0x32, 0xa7, 0xd6, 0x39, 0x91, 0xb2, 0x59,
+	0x26, 0xb6, 0x7d, 0xcb, 0x3f, 0xd1, 0x7d, 0xf2, 0x0c, 0xdb, 0x62, 0x6f, 0x52, 0x96, 0xd4, 0x7d,
+	0x4a, 0xe4, 0x87, 0x03, 0x5d, 0xcb, 0xf3, 0x5d, 0xc9, 0x26, 0xcf, 0x5f, 0x04, 0x95, 0xb1, 0xd1,
+	0x49, 0xa9, 0xb4, 0x06, 0xbd, 0x1e, 0x77, 0xf1, 0xab, 0x4f, 0xfb, 0xfb, 0x62, 0x40, 0xb9, 0xac,
+	0x98, 0x1e, 0x3a, 0x4d, 0x0c, 0xf7, 0x35, 0x82, 0x30, 0xef, 0xc3, 0x4a, 0x68, 0x0c, 0x22, 0xac,
+	0x22, 0x45, 0xa4, 0x12, 0x2d, 0x22, 0xd5, 0x47, 0x80, 0x38, 0xee, 0xf0, 0x0d, 0xc7, 0xad, 0x9e,
+	0x86, 0x53, 0x11, 0x45, 0x62, 0x25, 0xbe, 0x06, 0x65, 0x71, 0x2f, 0x49, 0x04, 0xca, 0x59, 0x28,
+	0xd0, 0x8c, 0xda, 0xb1, 0x4c, 0x79, 0x4e, 0xba, 0xe0, 0x10, 0x73, 0xdb, 0x32, 0x5d, 0xf5, 0x13,
+	0x28, 0x6b, 0xbc, 0x1f, 0xc1, 0xfb, 0x10, 0x96, 0xc4, 0x2d, 0x26, 0x3d, 0x72, 0x8d, 0x30, 0xed,
+	0x9a, 0x7a, 0xb8, 0x13, 0xad, 0x6c, 0x87, 0x9b, 0xaa, 0x09, 0x35, 0x5e, 0x32, 0x44, 0xd4, 0xcb,
+	0xc1, 0x3e, 0x04, 0x79, 0xa3, 0x70, 0x6c, 0x2f, 0x51, 0xf9, 0xb2, 0x1b, 0x6e, 0xaa, 0xe7, 0xe1,
+	0x5c, 0x6a, 0x2f, 0xc2, 0x13, 0x0e, 0x54, 0x86, 0x2f, 0x4c, 0x4b, 0x1e, 0x18, 0xb3, 0x83, 0x60,
+	0x25, 0x74, 0x10, 0x7c, 0x26, 0x28, 0x12, 0x73, 0x72, 0x11, 0x63, 0x15, 0xe0, 0xb0, 0xdc, 0xcf,
+	0x67, 0x95, 0xfb, 0xb3, 0x91, 0x72, 0x5f, 0x6d, 0x07, 0xfe, 0x14, 0xdb, 0xb0, 0x07, 0x6c, 0xbb,
+	0xc8, 0xfb, 0x96, 0x09, 0x51, 0x1d, 0x35, 0x4a, 0xce, 0xaa, 0x85, 0xa4, 0xd4, 0xab, 0x50, 0x8e,
+	0xa6, 0xc6, 0x50, 0x9e, 0x53, 0x12, 0x79, 0x6e, 0x29, 0x96, 0xe2, 0x3e, 0x8c, 0x55, 0xc0, 0xd9,
+	0x3e, 0x8e, 0xd5, 0xbf, 0x77, 0x23, 0xc9, 0xee, 0x5a, 0xca, 0x19, 0xee, 0x6f, 0x28, 0xcf, 0xad,
+	0x8a, 0xf5, 0xe0, 0xa1, 0x47, 0xe5, 0xc5, 0xa0, 0xd5, 0xb7, 0xa0, 0x74, 0x90, 0xf5, 0x1b, 0x88,
+	0x59, 0x79, 0x5f, 0xe2, 0x26, 0xac, 0x3e, 0xb4, 0x7a, 0xd8, 0x3b, 0xf1, 0x7c, 0xdc, 0x6f, 0xb0,
+	0xa4, 0x74, 0x64, 0x61, 0x17, 0xad, 0x03, 0xb0, 0x2d, 0x8c, 0x43, 0xac, 0xe0, 0x6a, 0x7c, 0x88,
+	0xa2, 0xfe, 0x97, 0x02, 0xcb, 0x43, 0xc1, 0x03, 0xb6, 0x75, 0x7b, 0x03, 0x8a, 0x74, 0xbc, 0x9e,
+	0x6f, 0xf4, 0x1d, 0x79, 0x9e, 0x15, 0x10, 0xd0, 0x1d, 0x98, 0x3b, 0xf2, 0x24, 0x64, 0x94, 0x0a,
+	0xa0, 0xa7, 0x19, 0xa2, 0xcd, 0x1e, 0x79, 0x0d, 0x13, 0x7d, 0x04, 0x30, 0xf0, 0xb0, 0x29, 0xce,
+	0xb0, 0xf2, 0x59, 0xd5, 0xc2, 0x41, 0xf8, 0x7c, 0x9b, 0x0a, 0xf0, 0xab, 0x16, 0x77, 0xa1, 0x64,
+	0xd9, 0xc4, 0xc4, 0xec, 0xcc, 0xd1, 0x14, 0xa8, 0xd2, 0x18, 0x71, 0xe0, 0x12, 0x07, 0x1e, 0x36,
+	0x55, 0x2c, 0xd6, 0x42, 0xe9, 0x5f, 0x11, 0x28, 0x4d, 0x58, 0xe1, 0x49, 0xeb, 0x28, 0x30, 0x5c,
+	0x46, 0xec, 0xc6, 0xa8, 0xd1, 0x31, 0x6f, 0x69, 0x15, 0x4b, 0x94, 0x36, 0x52, 0x54, 0xbd, 0x0d,
+	0xa7, 0x23, 0x3b, 0xa4, 0x29, 0xb6, 0x2c, 0x6a, 0x2b, 0x06, 0x94, 0x0c, 0xc3, 0x59, 0xc0, 0x10,
+	0x32, 0x9a, 0xc7, 0xc1, 0x10, 0x1e, 0x87, 0x21, 0x3c, 0xf5, 0x73, 0x38, 0x1b, 0x41, 0x74, 0x22,
+	0x16, 0xdd, 0x8d, 0x55, 0x6e, 0x97, 0xc6, 0x69, 0x8d, 0x95, 0x70, 0xff, 0xa3, 0xc0, 0x6a, 0x1a,
+	0xc3, 0x2b, 0x22, 0x8e, 0x3f, 0xce, 0xb8, 0x56, 0x77, 0x6b, 0x32, 0xb3, 0x7e, 0x2b, 0x68, 0xed,
+	0x3e, 0xd4, 0xd2, 0xfc, 0x99, 0x9c, 0xa5, 0xfc, 0x34, 0xb3, 0xf4, 0xb3, 0x7c, 0x08, 0x79, 0xaf,
+	0xfb, 0xbe, 0x6b, 0x1d, 0x0e, 0x68, 0xc8, 0xbf, 0x76, 0x34, 0xab, 0x11, 0xe0, 0x32, 0xdc, 0xb5,
+	0x37, 0x46, 0x88, 0x0f, 0xed, 0x48, 0xc5, 0x66, 0x3e, 0x8d, 0x62, 0x33, 0x1c, 0x53, 0xbf, 0x39,
+	0x99, 0xbe, 0xef, 0x2c, 0x00, 0xfa, 0xb3, 0x1c, 0x2c, 0x45, 0xa7, 0x08, 0xed, 0x02, 0x18, 0x81,
+	0xe5, 0xe2, 0x43, 0xb9, 0x38, 0xd1, 0x30, 0xb5, 0x90, 0x20, 0x7a, 0x17, 0xf2, 0x1d, 0x67, 0x20,
+	0x66, 0x2d, 0xe5, 0x30, 0x78, 0xdb, 0x19, 0xf0, 0x8c, 0x42, 0xd9, 0xe8, 0x9e, 0x8a, 0x9f, 0xed,
+	0x67, 0x67, 0xc9, 0xa7, 0xec, 0x3d, 0x97, 0x11, 0xcc, 0xe8, 0x31, 0x2c, 0xbd, 0x70, 0x2d, 0xdf,
+	0x38, 0xec, 0x61, 0xbd, 0x67, 0x9c, 0x60, 0x57, 0x64, 0xc9, 0x09, 0x12, 0x59, 0x59, 0x0a, 0x3e,
+	0xa1, 0x72, 0xea, 0x1f, 0x42, 0x41, 0x5a, 0x34, 0x66, 0x45, 0xd8, 0x87, 0xb5, 0x01, 0x65, 0xd3,
+	0xd9, 0x15, 0x38, 0xdb, 0xb0, 0x89, 0xee, 0x61, 0xba, 0x8c, 0xcb, 0xcb, 0xf9, 0x63, 0x52, 0xf4,
+	0x2a, 0x93, 0xde, 0x26, 0x2e, 0x6e, 0x1a, 0x36, 0x69, 0x73, 0x51, 0xf5, 0x39, 0x94, 0x42, 0x03,
+	0x1c, 0x63, 0x42, 0x03, 0x56, 0xe4, 0x51, 0xbc, 0x87, 0x7d, 0xb1, 0xbc, 0x4c, 0xd4, 0xf9, 0xb2,
+	0x90, 0x6b, 0x63, 0x9f, 0x5f, 0x9f, 0xb8, 0x0b, 0x67, 0x35, 0x4c, 0x1c, 0x6c, 0x07, 0xf3, 0xf9,
+	0x84, 0x74, 0xa7, 0xc8, 0xe0, 0x6f, 0x40, 0x2d, 0x4d, 0x9e, 0xe7, 0x87, 0x6b, 0x97, 0xa0, 0x20,
+	0x7f, 0xd0, 0x8a, 0x16, 0x20, 0xbf, 0xbf, 0xdd, 0xaa, 0xcc, 0xd0, 0x87, 0x83, 0x9d, 0x56, 0x45,
+	0x41, 0x05, 0x98, 0x6d, 0x6f, 0xef, 0xb7, 0x2a, 0xb9, 0x6b, 0x7d, 0xa8, 0xc4, 0x7f, 0xcd, 0x89,
+	0xd6, 0xe0, 0x54, 0x4b, 0xdb, 0x6b, 0xd5, 0x1f, 0xd5, 0xf7, 0x1b, 0x7b, 0x4d, 0xbd, 0xa5, 0x35,
+	0x3e, 0xae, 0xef, 0xef, 0x56, 0x66, 0xd0, 0x06, 0x9c, 0x0f, 0xbf, 0x78, 0xbc, 0xd7, 0xde, 0xd7,
+	0xf7, 0xf7, 0xf4, 0xed, 0xbd, 0xe6, 0x7e, 0xbd, 0xd1, 0xdc, 0xd5, 0x2a, 0x0a, 0x3a, 0x0f, 0x67,
+	0xc3, 0x2c, 0x0f, 0x1a, 0x3b, 0x0d, 0x6d, 0x77, 0x9b, 0x3e, 0xd7, 0x9f, 0x54, 0x72, 0xd7, 0x6e,
+	0x40, 0x39, 0xf2, 0xe3, 0x4b, 0x6a, 0x52, 0x6b, 0x6f, 0xa7, 0x32, 0x83, 0xca, 0x50, 0x0c, 0xeb,
+	0x29, 0xc0, 0x6c, 0x73, 0x6f, 0x67, 0xb7, 0x92, 0xbb, 0x76, 0x1b, 0x96, 0x63, 0xb7, 0x71, 0xd1,
+	0x0a, 0x94, 0xdb, 0xf5, 0xe6, 0xce, 0x83, 0xbd, 0x4f, 0x75, 0x6d, 0xb7, 0xbe, 0xf3, 0x59, 0x65,
+	0x06, 0xad, 0x42, 0x45, 0x92, 0x9a, 0x7b, 0xfb, 0x9c, 0xaa, 0x5c, 0x7b, 0x16, 0xfb, 0xc6, 0x30,
+	0x3a, 0x0d, 0x2b, 0x41, 0x37, 0xfa, 0xb6, 0xb6, 0x5b, 0xdf, 0xdf, 0xa5, 0xbd, 0x47, 0xc8, 0xda,
+	0x41, 0xb3, 0xd9, 0x68, 0x3e, 0xaa, 0x28, 0x54, 0xeb, 0x90, 0xbc, 0xfb, 0x69, 0x83, 0x32, 0xe7,
+	0xa2, 0xcc, 0x07, 0xcd, 0x1f, 0x36, 0xf7, 0x3e, 0x69, 0x56, 0xf2, 0x5b, 0xbf, 0x58, 0x81, 0x25,
+	0x59, 0xe8, 0x61, 0x97, 0xdd, 0x6a, 0x69, 0xc1, 0x82, 0xfc, 0x81, 0x74, 0x4a, 0x86, 0x8e, 0xfe,
+	0xac, 0xbb, 0xb6, 0x31, 0x82, 0x43, 0xd4, 0xdb, 0x33, 0xe8, 0x90, 0xd5, 0xbf, 0xa1, 0xdb, 0xd1,
+	0x97, 0x52, 0xab, 0xcd, 0xc4, 0x85, 0xec, 0xda, 0xe5, 0xb1, 0x7c, 0x41, 0x1f, 0x98, 0x96, 0xb8,
+	0xe1, 0x9f, 0xff, 0xa0, 0xcb, 0x69, 0xb5, 0x69, 0xca, 0xef, 0x8b, 0x6a, 0x57, 0xc6, 0x33, 0x06,
+	0xdd, 0x3c, 0x83, 0x4a, 0xfc, 0xa7, 0x40, 0x28, 0x05, 0x3a, 0xcd, 0xf8, 0xbd, 0x51, 0xed, 0xda,
+	0x24, 0xac, 0xe1, 0xce, 0x12, 0x3f, 0x9a, 0xb9, 0x3a, 0xc9, 0xaf, 0x10, 0x32, 0x3b, 0xcb, 0xfa,
+	0xc1, 0x02, 0x77, 0x60, 0xf4, 0xe6, 0x33, 0x4a, 0xfd, 0x85, 0x4a, 0xca, 0xbd, 0xf9, 0x34, 0x07,
+	0xa6, 0x5f, 0xa2, 0x56, 0x67, 0xd0, 0x31, 0x2c, 0xc7, 0xae, 0x27, 0xa0, 0x14, 0xf1, 0xf4, 0x7b,
+	0x18, 0xb5, 0xab, 0x13, 0x70, 0x46, 0x23, 0x22, 0x7c, 0x1d, 0x21, 0x3d, 0x22, 0x52, 0x2e, 0x3b,
+	0xa4, 0x47, 0x44, 0xea, 0xcd, 0x06, 0x16, 0xdc, 0x91, 0x6b, 0x08, 0x69, 0xc1, 0x9d, 0x76, 0xf9,
+	0xa1, 0x76, 0x79, 0x2c, 0x5f, 0xd8, 0x69, 0xb1, 0x4b, 0x09, 0x69, 0x4e, 0x4b, 0xbf, 0xf4, 0x50,
+	0xbb, 0x3a, 0x01, 0x67, 0x3c, 0x0a, 0x86, 0x47, 0x9c, 0x59, 0x51, 0x90, 0x38, 0x90, 0xcf, 0x8a,
+	0x82, 0xe4, 0x69, 0xa9, 0x88, 0x82, 0xd8, 0xd1, 0xe4, 0x95, 0x09, 0x8e, 0x52, 0xb2, 0xa3, 0x20,
+	0xfd, 0xd0, 0x45, 0x9d, 0x41, 0x7f, 0xa4, 0x40, 0x35, 0xeb, 0x98, 0x02, 0xa5, 0xd4, 0x77, 0x63,
+	0x4e, 0x56, 0x6a, 0x5b, 0xd3, 0x88, 0x04, 0x56, 0x7c, 0x09, 0x28, 0xb9, 0xee, 0xa1, 0x77, 0xd2,
+	0x66, 0x26, 0x63, 0x75, 0xad, 0xbd, 0x3b, 0x19, 0x73, 0xd0, 0x65, 0x1b, 0x0a, 0xf2, 0x60, 0x04,
+	0xa5, 0x64, 0xe9, 0xd8, 0xb1, 0x4c, 0x4d, 0x1d, 0xc5, 0x12, 0x28, 0x7d, 0x04, 0xb3, 0x94, 0x8a,
+	0xce, 0xa7, 0x73, 0x4b, 0x65, 0xeb, 0x59, 0xaf, 0x03, 0x45, 0x4f, 0x61, 0x9e, 0x9f, 0x04, 0xa0,
+	0x14, 0xe4, 0x21, 0x72, 0x5e, 0x51, 0xbb, 0x90, 0xcd, 0x10, 0xa8, 0xfb, 0x82, 0xff, 0xef, 0x0c,
+	0x01, 0xf2, 0xa3, 0xb7, 0xd3, 0x7f, 0x8c, 0x1c, 0x3d, 0x53, 0xa8, 0x5d, 0x1c, 0xc3, 0x15, 0xfe,
+	0x28, 0x62, 0x55, 0xef, 0xe5, 0xb1, 0x5b, 0x97, 0xec, 0x8f, 0x22, 0x7d, 0x73, 0xc4, 0x83, 0x24,
+	0xb9, 0x79, 0x4a, 0x0b, 0x92, 0xcc, 0x2d, 0x6b, 0x5a, 0x90, 0x64, 0xef, 0xc7, 0xd4, 0x19, 0xe4,
+	0xc3, 0xa9, 0x14, 0xa8, 0x0c, 0xbd, 0x9b, 0x15, 0xe4, 0x69, 0xb8, 0x5d, 0xed, 0xbd, 0x09, 0xb9,
+	0xc3, 0x93, 0x2f, 0x3e, 0xfa, 0x37, 0xb3, 0xf1, 0xa3, 0xcc, 0xc9, 0x8f, 0x7f, 0xe2, 0x5b, 0xff,
+	0x9a, 0x87, 0x45, 0x0e, 0x83, 0x8a, 0x0a, 0xe6, 0x33, 0x80, 0xe1, 0x09, 0x04, 0x7a, 0x2b, 0xdd,
+	0x27, 0x91, 0x53, 0x9a, 0xda, 0xdb, 0xa3, 0x99, 0xc2, 0x81, 0x16, 0x42, 0xf3, 0xd3, 0x02, 0x2d,
+	0x79, 0x68, 0x91, 0x16, 0x68, 0x29, 0x47, 0x02, 0xea, 0x0c, 0xfa, 0x18, 0x8a, 0x01, 0x6c, 0x8c,
+	0xd2, 0x60, 0xe7, 0x18, 0x2e, 0x5e, 0x7b, 0x6b, 0x24, 0x4f, 0xd8, 0xea, 0x10, 0x26, 0x9c, 0x66,
+	0x75, 0x12, 0x7b, 0x4e, 0xb3, 0x3a, 0x0d, 0x58, 0x1e, 0xfa, 0x84, 0x23, 0x47, 0x99, 0x3e, 0x89,
+	0x00, 0x77, 0x99, 0x3e, 0x89, 0xc2, 0x4f, 0xea, 0xcc, 0x83, 0x4b, 0xbf, 0xfa, 0x6a, 0x5d, 0xf9,
+	0xa7, 0xaf, 0xd6, 0x67, 0x7e, 0xfa, 0xf5, 0xba, 0xf2, 0xab, 0xaf, 0xd7, 0x95, 0x7f, 0xfc, 0x7a,
+	0x5d, 0xf9, 0xb7, 0xaf, 0xd7, 0x95, 0x3f, 0xfb, 0xf7, 0xf5, 0x99, 0x1f, 0x15, 0xa4, 0xf4, 0xe1,
+	0x3c, 0xfb, 0x0f, 0x38, 0x1f, 0xfc, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x15, 0x30, 0x49, 0xcc,
+	0xc7, 0x48, 0x00, 0x00,
 }

vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto

@@ -143,6 +143,7 @@ message DNSConfig {
 enum Protocol {
     TCP = 0;
     UDP = 1;
+    SCTP = 2;
 }
 
 // PortMapping specifies the port mapping configurations of a sandbox.
@@ -344,6 +345,12 @@ message PodSandboxConfig {
 message RunPodSandboxRequest {
     // Configuration for creating a PodSandbox.
     PodSandboxConfig config = 1;
+    // Named runtime configuration to use for this PodSandbox.
+    // If the runtime handler is unknown, this request should be rejected.  An
+    // empty string should select the default handler, equivalent to the
+    // behavior before this feature was added.
+    // See https://git.k8s.io/community/keps/sig-node/0014-runtime-class.md
+    string runtime_handler = 2;
 }
 
 message RunPodSandboxResponse {
@@ -586,6 +593,12 @@ message LinuxContainerSecurityContext {
     // no_new_privs defines if the flag for no_new_privs should be set on the
     // container.
     bool no_new_privs = 11;
+    // masked_paths is a slice of paths that should be masked by the container
+    // runtime, this can be passed directly to the OCI spec.
+    repeated string masked_paths = 13;
+    // readonly_paths is a slice of paths that should be set as readonly by the
+    // container runtime, this can be passed directly to the OCI spec.
+    repeated string readonly_paths = 14;
 }
 
 // LinuxContainerConfig contains platform-specific configuration for

vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/services.go

@@ -63,7 +63,7 @@ type ContainerManager interface {
 type PodSandboxManager interface {
 	// RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
 	// the sandbox is in ready state.
-	RunPodSandbox(config *runtimeapi.PodSandboxConfig) (string, error)
+	RunPodSandbox(config *runtimeapi.PodSandboxConfig, runtimeHandler string) (string, error)
 	// StopPodSandbox stops the sandbox. If there are any running containers in the
 	// sandbox, they should be force terminated.
 	StopPodSandbox(podSandboxID string) error

vendor/k8s.io/kubernetes/pkg/kubelet/container/BUILD

@@ -23,18 +23,18 @@ go_library(
         "//pkg/kubelet/util/format:go_default_library",
         "//pkg/util/hash:go_default_library",
         "//pkg/volume:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/reference:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library",
+        "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//third_party/forked/golang/expansion:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
-        "//vendor/k8s.io/client-go/tools/reference:go_default_library",
-        "//vendor/k8s.io/client-go/tools/remotecommand:go_default_library",
-        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
     ],
 )
 
@@ -44,24 +44,17 @@ go_test(
         "cache_test.go",
         "helpers_test.go",
         "ref_test.go",
+        "runtime_cache_test.go",
         "sync_result_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core/install:go_default_library",
-        "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_xtest",
-    srcs = ["runtime_cache_test.go"],
-    deps = [
-        ":go_default_library",
         "//pkg/kubelet/container/testing:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/kubelet/container/runtime.go

@@ -17,6 +17,7 @@ limitations under the License.
 package container
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/url"
@@ -113,7 +114,7 @@ type Runtime interface {
 	// default, it returns a snapshot of the container log. Set 'follow' to true to
 	// stream the log. Set 'follow' to false and specify the number of lines (e.g.
 	// "100" or "all") to tail the log.
-	GetContainerLogs(pod *v1.Pod, containerID ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer) (err error)
+	GetContainerLogs(ctx context.Context, pod *v1.Pod, containerID ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer) (err error)
 	// Delete a container. If the container is still running, an error is returned.
 	DeleteContainer(containerID ContainerID) error
 	// ImageService provides methods to image-related methods.

vendor/k8s.io/kubernetes/pkg/kubelet/kubelet.go

@@ -45,6 +45,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	corelisters "k8s.io/client-go/listers/core/v1"
@@ -53,14 +54,17 @@ import (
 	"k8s.io/client-go/util/certificate"
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/client-go/util/integer"
+	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/features"
+	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
-	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	pluginwatcherapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate"
 	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
+	"k8s.io/kubernetes/pkg/kubelet/cloudresource"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	"k8s.io/kubernetes/pkg/kubelet/configmap"
@@ -77,12 +81,14 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
 	"k8s.io/kubernetes/pkg/kubelet/network/dns"
+	"k8s.io/kubernetes/pkg/kubelet/nodelease"
 	"k8s.io/kubernetes/pkg/kubelet/pleg"
 	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
 	"k8s.io/kubernetes/pkg/kubelet/preemption"
 	"k8s.io/kubernetes/pkg/kubelet/prober"
 	proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results"
 	"k8s.io/kubernetes/pkg/kubelet/remote"
+	"k8s.io/kubernetes/pkg/kubelet/runtimeclass"
 	"k8s.io/kubernetes/pkg/kubelet/secret"
 	"k8s.io/kubernetes/pkg/kubelet/server"
 	serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
@@ -102,7 +108,6 @@ import (
 	"k8s.io/kubernetes/pkg/security/apparmor"
 	sysctlwhitelist "k8s.io/kubernetes/pkg/security/podsecuritypolicy/sysctl"
 	utildbus "k8s.io/kubernetes/pkg/util/dbus"
-	kubeio "k8s.io/kubernetes/pkg/util/io"
 	utilipt "k8s.io/kubernetes/pkg/util/iptables"
 	"k8s.io/kubernetes/pkg/util/mount"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
@@ -238,16 +243,16 @@ type Dependencies struct {
 	ContainerManager        cm.ContainerManager
 	DockerClientConfig      *dockershim.ClientConfig
 	EventClient             v1core.EventsGetter
-	HeartbeatClient         v1core.CoreV1Interface
+	HeartbeatClient         clientset.Interface
 	OnHeartbeatFailure      func()
 	KubeClient              clientset.Interface
-	ExternalKubeClient      clientset.Interface
+	CSIClient               csiclientset.Interface
+	DynamicKubeClient       dynamic.Interface
 	Mounter                 mount.Interface
 	OOMAdjuster             *oom.OOMAdjuster
 	OSInterface             kubecontainer.OSInterface
 	PodConfig               *config.PodConfig
 	Recorder                record.EventRecorder
-	Writer                  kubeio.Writer
 	VolumePlugins           []volume.VolumePlugin
 	DynamicPluginProber     volume.DynamicPluginProber
 	TLSOptions              *server.TLSOptions
@@ -369,11 +374,12 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		}
 	}
 
-	hostname := nodeutil.GetHostname(hostnameOverride)
+	hostname, err := nodeutil.GetHostname(hostnameOverride)
+	if err != nil {
+		return nil, err
+	}
 	// Query the cloud provider for our node name, default to hostname
 	nodeName := types.NodeName(hostname)
-	cloudIPs := []net.IP{}
-	cloudNames := []string{}
 	if kubeDeps.Cloud != nil {
 		var err error
 		instances, ok := kubeDeps.Cloud.Instances()
@@ -387,25 +393,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		}
 
 		glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName)
-
-		if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
-			nodeAddresses, err := instances.NodeAddresses(context.TODO(), nodeName)
-			if err != nil {
-				return nil, fmt.Errorf("failed to get the addresses of the current instance from the cloud provider: %v", err)
-			}
-			for _, nodeAddress := range nodeAddresses {
-				switch nodeAddress.Type {
-				case v1.NodeExternalIP, v1.NodeInternalIP:
-					ip := net.ParseIP(nodeAddress.Address)
-					if ip != nil && !ip.IsLoopback() {
-						cloudIPs = append(cloudIPs, ip)
-					}
-				case v1.NodeExternalDNS, v1.NodeInternalDNS, v1.NodeHostName:
-					cloudNames = append(cloudNames, nodeAddress.Address)
-				}
-			}
-		}
-
 	}
 
 	if kubeDeps.PodConfig == nil {
@@ -491,11 +478,18 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	}
 	httpClient := &http.Client{}
 	parsedNodeIP := net.ParseIP(nodeIP)
+	protocol := utilipt.ProtocolIpv4
+	if parsedNodeIP != nil && parsedNodeIP.To4() == nil {
+		glog.V(0).Infof("IPv6 node IP (%s), assume IPv6 operation", nodeIP)
+		protocol = utilipt.ProtocolIpv6
+	}
 
 	klet := &Kubelet{
 		hostname:                       hostname,
+		hostnameOverridden:             len(hostnameOverride) > 0,
 		nodeName:                       nodeName,
 		kubeClient:                     kubeDeps.KubeClient,
+		csiClient:                      kubeDeps.CSIClient,
 		heartbeatClient:                kubeDeps.HeartbeatClient,
 		onRepeatedHeartbeatFailure:     kubeDeps.OnHeartbeatFailure,
 		rootDirectory:                  rootDirectory,
@@ -522,7 +516,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		cgroupsPerQOS:              kubeCfg.CgroupsPerQOS,
 		cgroupRoot:                 kubeCfg.CgroupRoot,
 		mounter:                    kubeDeps.Mounter,
-		writer:                     kubeDeps.Writer,
 		maxPods:                    int(kubeCfg.MaxPods),
 		podsPerCore:                int(kubeCfg.PodsPerCore),
 		syncLoopMonitor:            atomic.Value{},
@@ -534,7 +527,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		nodeIPValidator: validateNodeIP,
 		clock:           clock.RealClock{},
 		enableControllerAttachDetach:            kubeCfg.EnableControllerAttachDetach,
-		iptClient:                               utilipt.New(utilexec.New(), utildbus.New(), utilipt.ProtocolIpv4),
+		iptClient:                               utilipt.New(utilexec.New(), utildbus.New(), protocol),
 		makeIPTablesUtilChains:                  kubeCfg.MakeIPTablesUtilChains,
 		iptablesMasqueradeBit:                   int(kubeCfg.IPTablesMasqueradeBit),
 		iptablesDropBit:                         int(kubeCfg.IPTablesDropBit),
@@ -545,18 +538,28 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	}
 
 	if klet.cloud != nil {
-		klet.cloudproviderRequestParallelism = make(chan int, 1)
-		klet.cloudproviderRequestSync = make(chan int)
-		// TODO(jchaloup): Make it configurable via --cloud-provider-request-timeout
-		klet.cloudproviderRequestTimeout = 10 * time.Second
+		klet.cloudResourceSyncManager = cloudresource.NewSyncManager(klet.cloud, nodeName, klet.nodeStatusUpdateFrequency)
 	}
 
-	secretManager := secret.NewCachingSecretManager(
-		kubeDeps.KubeClient, manager.GetObjectTTLFromNodeFunc(klet.GetNode))
-	klet.secretManager = secretManager
+	var secretManager secret.Manager
+	var configMapManager configmap.Manager
+	switch kubeCfg.ConfigMapAndSecretChangeDetectionStrategy {
+	case kubeletconfiginternal.WatchChangeDetectionStrategy:
+		secretManager = secret.NewWatchingSecretManager(kubeDeps.KubeClient)
+		configMapManager = configmap.NewWatchingConfigMapManager(kubeDeps.KubeClient)
+	case kubeletconfiginternal.TTLCacheChangeDetectionStrategy:
+		secretManager = secret.NewCachingSecretManager(
+			kubeDeps.KubeClient, manager.GetObjectTTLFromNodeFunc(klet.GetNode))
+		configMapManager = configmap.NewCachingConfigMapManager(
+			kubeDeps.KubeClient, manager.GetObjectTTLFromNodeFunc(klet.GetNode))
+	case kubeletconfiginternal.GetChangeDetectionStrategy:
+		secretManager = secret.NewSimpleSecretManager(kubeDeps.KubeClient)
+		configMapManager = configmap.NewSimpleConfigMapManager(kubeDeps.KubeClient)
+	default:
+		return nil, fmt.Errorf("unknown configmap and secret manager mode: %v", kubeCfg.ConfigMapAndSecretChangeDetectionStrategy)
+	}
 
-	configMapManager := configmap.NewCachingConfigMapManager(
-		kubeDeps.KubeClient, manager.GetObjectTTLFromNodeFunc(klet.GetNode))
+	klet.secretManager = secretManager
 	klet.configMapManager = configMapManager
 
 	if klet.experimentalHostUserNamespaceDefaulting {
@@ -615,8 +618,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		// Create and start the CRI shim running as a grpc server.
 		streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
 		ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
-			&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory,
-			crOptions.DockerDisableSharedPID, !crOptions.RedirectContainerStreaming)
+			&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
 		if err != nil {
 			return nil, err
 		}
@@ -654,6 +656,11 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		return nil, err
 	}
 	klet.runtimeService = runtimeService
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.DynamicKubeClient != nil {
+		klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.DynamicKubeClient)
+	}
+
 	runtime, err := kuberuntime.NewKubeGenericRuntimeManager(
 		kubecontainer.FilterEventRecorder(kubeDeps.Recorder),
 		klet.livenessManager,
@@ -669,10 +676,12 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		float32(kubeCfg.RegistryPullQPS),
 		int(kubeCfg.RegistryBurst),
 		kubeCfg.CPUCFSQuota,
+		kubeCfg.CPUCFSQuotaPeriod,
 		runtimeService,
 		imageService,
 		kubeDeps.ContainerManager.InternalContainerLifecycle(),
 		legacyLogProvider,
+		klet.runtimeClassManager,
 	)
 	if err != nil {
 		return nil, err
@@ -681,6 +690,12 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	klet.streamingRuntime = runtime
 	klet.runner = runtime
 
+	runtimeCache, err := kubecontainer.NewRuntimeCache(klet.containerRuntime)
+	if err != nil {
+		return nil, err
+	}
+	klet.runtimeCache = runtimeCache
+
 	if cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) {
 		klet.StatsProvider = stats.NewCadvisorStatsProvider(
 			klet.cadvisor,
@@ -702,7 +717,9 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	klet.pleg = pleg.NewGenericPLEG(klet.containerRuntime, plegChannelCapacity, plegRelistPeriod, klet.podCache, clock.RealClock{})
 	klet.runtimeState = newRuntimeState(maxWaitForContainerRuntime)
 	klet.runtimeState.addHealthCheck("PLEG", klet.pleg.Healthy)
-	klet.updatePodCIDR(kubeCfg.PodCIDR)
+	if err := klet.updatePodCIDR(kubeCfg.PodCIDR); err != nil {
+		glog.Errorf("Pod CIDR update failed %v", err)
+	}
 
 	// setup containerGC
 	containerGC, err := kubecontainer.NewContainerGC(klet.containerRuntime, containerGCPolicy, klet.sourcesReady)
@@ -737,29 +754,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
 
 	if kubeCfg.ServerTLSBootstrap && kubeDeps.TLSOptions != nil && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
-		var (
-			ips   []net.IP
-			names []string
-		)
-
-		// If the address was explicitly configured, use that. Otherwise, try to
-		// discover addresses from the cloudprovider. Otherwise, make a best guess.
-		if cfgAddress := net.ParseIP(kubeCfg.Address); cfgAddress != nil && !cfgAddress.IsUnspecified() {
-			ips = []net.IP{cfgAddress}
-			names = []string{klet.GetHostname(), hostnameOverride}
-		} else if len(cloudIPs) != 0 || len(cloudNames) != 0 {
-			ips = cloudIPs
-			names = cloudNames
-		} else {
-			localIPs, err := allGlobalUnicastIPs()
-			if err != nil {
-				return nil, err
-			}
-			ips = localIPs
-			names = []string{klet.GetHostname(), hostnameOverride}
-		}
-
-		klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names, certDirectory)
+		klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory)
 		if err != nil {
 			return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
 		}
@@ -781,6 +776,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 
 	tokenManager := token.NewManager(kubeDeps.KubeClient)
 
+	if !utilfeature.DefaultFeatureGate.Enabled(features.MountPropagation) {
+		glog.Warning("Mount propagation feature gate has been deprecated and will be removed in the next release")
+	}
+
 	klet.volumePluginMgr, err =
 		NewInitializedVolumePluginMgr(klet, secretManager, configMapManager, tokenManager, kubeDeps.VolumePlugins, kubeDeps.DynamicPluginProber)
 	if err != nil {
@@ -814,18 +813,12 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		experimentalCheckNodeCapabilitiesBeforeMount,
 		keepTerminatedPodVolumes)
 
-	runtimeCache, err := kubecontainer.NewRuntimeCache(klet.containerRuntime)
-	if err != nil {
-		return nil, err
-	}
-	klet.runtimeCache = runtimeCache
 	klet.reasonCache = NewReasonCache()
 	klet.workQueue = queue.NewBasicWorkQueue(klet.clock)
 	klet.podWorkers = newPodWorkers(klet.syncPod, kubeDeps.Recorder, klet.workQueue, klet.resyncInterval, backOffPeriod, klet.podCache)
 
 	klet.backOff = flowcontrol.NewBackOff(backOffPeriod, MaxContainerBackOff)
 	klet.podKillingCh = make(chan *kubecontainer.PodPair, podKillingChannelCapacity)
-	klet.setNodeStatusFuncs = klet.defaultNodeStatusFuncs()
 
 	// setup eviction manager
 	evictionManager, evictionAdmitHandler := eviction.NewManager(klet.resourceAnalyzer, evictionConfig, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.imageManager, klet.containerGC, kubeDeps.Recorder, nodeRef, klet.clock)
@@ -869,9 +862,21 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	klet.appArmorValidator = apparmor.NewValidator(containerRuntime)
 	klet.softAdmitHandlers.AddPodAdmitHandler(lifecycle.NewAppArmorAdmitHandler(klet.appArmorValidator))
 	klet.softAdmitHandlers.AddPodAdmitHandler(lifecycle.NewNoNewPrivsAdmitHandler(klet.containerRuntime))
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) {
+		klet.nodeLeaseController = nodelease.NewController(klet.clock, klet.heartbeatClient, string(klet.nodeName), kubeCfg.NodeLeaseDurationSeconds, klet.onRepeatedHeartbeatFailure)
+	}
+
+	klet.softAdmitHandlers.AddPodAdmitHandler(lifecycle.NewProcMountAdmitHandler(klet.containerRuntime))
+
 	// Finally, put the most recent version of the config on the Kubelet, so
 	// people can see how it was configured.
 	klet.kubeletConfiguration = *kubeCfg
+
+	// Generating the status funcs should be the last thing we do,
+	// since this relies on the rest of the Kubelet having been constructed.
+	klet.setNodeStatusFuncs = klet.defaultNodeStatusFuncs()
+
 	return klet, nil
 }
 
@@ -883,14 +888,22 @@ type serviceLister interface {
 type Kubelet struct {
 	kubeletConfiguration kubeletconfiginternal.KubeletConfiguration
 
-	hostname        string
+	// hostname is the hostname the kubelet detected or was given via flag/config
+	hostname string
+	// hostnameOverridden indicates the hostname was overridden via flag/config
+	hostnameOverridden bool
+
 	nodeName        types.NodeName
 	runtimeCache    kubecontainer.RuntimeCache
 	kubeClient      clientset.Interface
-	heartbeatClient v1core.CoreV1Interface
+	csiClient       csiclientset.Interface
+	heartbeatClient clientset.Interface
 	iptClient       utilipt.Interface
 	rootDirectory   string
 
+	lastObservedNodeAddressesMux sync.Mutex
+	lastObservedNodeAddresses    []v1.NodeAddress
+
 	// onRepeatedHeartbeatFailure is called when a heartbeat operation fails more than once. optional.
 	onRepeatedHeartbeatFailure func()
 
@@ -994,14 +1007,8 @@ type Kubelet struct {
 
 	// Cloud provider interface.
 	cloud cloudprovider.Interface
-	// To keep exclusive access to the cloudproviderRequestParallelism
-	cloudproviderRequestMux sync.Mutex
-	// Keep the count of requests processed in parallel (expected to be 1 at most at a given time)
-	cloudproviderRequestParallelism chan int
-	// Sync with finished requests
-	cloudproviderRequestSync chan int
-	// Request timeout
-	cloudproviderRequestTimeout time.Duration
+	// Handles requests to cloud provider with timeout
+	cloudResourceSyncManager cloudresource.SyncManager
 
 	// Indicates that the node initialization happens in an external cloud controller
 	externalCloudProvider bool
@@ -1042,6 +1049,21 @@ type Kubelet struct {
 	//    as it takes time to gather all necessary node information.
 	nodeStatusUpdateFrequency time.Duration
 
+	// syncNodeStatusMux is a lock on updating the node status, because this path is not thread-safe.
+	// This lock is used by Kublet.syncNodeStatus function and shouldn't be used anywhere else.
+	syncNodeStatusMux sync.Mutex
+
+	// updatePodCIDRMux is a lock on updating pod CIDR, because this path is not thread-safe.
+	// This lock is used by Kublet.syncNodeStatus function and shouldn't be used anywhere else.
+	updatePodCIDRMux sync.Mutex
+
+	// updateRuntimeMux is a lock on updating runtime, because this path is not thread-safe.
+	// This lock is used by Kublet.updateRuntimeUp function and shouldn't be used anywhere else.
+	updateRuntimeMux sync.Mutex
+
+	// nodeLeaseController claims and renews the node lease for this Kubelet
+	nodeLeaseController nodelease.Controller
+
 	// Generates pod events.
 	pleg pleg.PodLifecycleEventGenerator
 
@@ -1066,9 +1088,6 @@ type Kubelet struct {
 	// Mounter to use for volumes.
 	mounter mount.Interface
 
-	// Writer interface to use for volumes.
-	writer kubeio.Writer
-
 	// Manager of non-Runtime containers.
 	containerManager cm.ContainerManager
 
@@ -1109,6 +1128,10 @@ type Kubelet struct {
 	// handlers called during the tryUpdateNodeStatus cycle
 	setNodeStatusFuncs []func(*v1.Node) error
 
+	lastNodeUnschedulableLock sync.Mutex
+	// maintains Node.Spec.Unschedulable value from previous run of tryUpdateNodeStatus()
+	lastNodeUnschedulable bool
+
 	// TODO: think about moving this to be centralized in PodWorkers in follow-on.
 	// the list of handlers to call during pod admission.
 	admitHandlers lifecycle.PodAdmitHandlers
@@ -1172,13 +1195,16 @@ type Kubelet struct {
 	// pluginwatcher is a utility for Kubelet to register different types of node-level plugins
 	// such as device plugins or CSI plugins. It discovers plugins by monitoring inotify events under the
 	// directory returned by kubelet.getPluginsDir()
-	pluginWatcher pluginwatcher.Watcher
+	pluginWatcher *pluginwatcher.Watcher
 
 	// This flag sets a maximum number of images to report in the node status.
 	nodeStatusMaxImages int32
 
 	//  This flag indicates that kubelet should start plugin watcher utility server for discovering Kubelet plugins
 	enablePluginsWatcher bool
+
+	// Handles RuntimeClass objects for the Kubelet.
+	runtimeClassManager *runtimeclass.Manager
 }
 
 func allGlobalUnicastIPs() ([]net.IP, error) {
@@ -1340,7 +1366,9 @@ func (kl *Kubelet) initializeRuntimeDependentModules() {
 	kl.containerLogManager.Start()
 	if kl.enablePluginsWatcher {
 		// Adding Registration Callback function for CSI Driver
-		kl.pluginWatcher.AddHandler("CSIPlugin", csi.RegistrationCallback)
+		kl.pluginWatcher.AddHandler("CSIPlugin", pluginwatcher.PluginHandler(csi.PluginHandler))
+		// Adding Registration Callback function for Device Manager
+		kl.pluginWatcher.AddHandler(pluginwatcherapi.DevicePlugin, kl.containerManager.GetPluginRegistrationHandler())
 		// Start the plugin watcher
 		glog.V(4).Infof("starting watcher")
 		if err := kl.pluginWatcher.Start(); err != nil {
@@ -1359,6 +1387,11 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
 		glog.Warning("No api server defined - no node status update will be sent.")
 	}
 
+	// Start the cloud provider sync manager
+	if kl.cloudResourceSyncManager != nil {
+		go kl.cloudResourceSyncManager.Run(wait.NeverStop)
+	}
+
 	if err := kl.initializeModules(); err != nil {
 		kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.KubeletSetupFailed, err.Error())
 		glog.Fatal(err)
@@ -1370,6 +1403,12 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
 	if kl.kubeClient != nil {
 		// Start syncing node status immediately, this may set up things the runtime needs to run.
 		go wait.Until(kl.syncNodeStatus, kl.nodeStatusUpdateFrequency, wait.NeverStop)
+		go kl.fastStatusUpdateOnce()
+
+		// start syncing lease
+		if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) {
+			go kl.nodeLeaseController.Run(wait.NeverStop)
+		}
 	}
 	go wait.Until(kl.updateRuntimeUp, 5*time.Second, wait.NeverStop)
 
@@ -1382,15 +1421,15 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
 	// handled by pod workers).
 	go wait.Until(kl.podKiller, 1*time.Second, wait.NeverStop)
 
-	// Start gorouting responsible for checking limits in resolv.conf
-	if kl.dnsConfigurer.ResolverConfig != "" {
-		go wait.Until(func() { kl.dnsConfigurer.CheckLimitsForResolvConf() }, 30*time.Second, wait.NeverStop)
-	}
-
 	// Start component sync loops.
 	kl.statusManager.Start()
 	kl.probeManager.Start()
 
+	// Start syncing RuntimeClasses if enabled.
+	if kl.runtimeClassManager != nil {
+		go kl.runtimeClassManager.Run(wait.NeverStop)
+	}
+
 	// Start the pod lifecycle event generator.
 	kl.pleg.Start()
 	kl.syncLoop(updates, kl)
@@ -1625,9 +1664,16 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
 	result := kl.containerRuntime.SyncPod(pod, apiPodStatus, podStatus, pullSecrets, kl.backOff)
 	kl.reasonCache.Update(pod.UID, result)
 	if err := result.Error(); err != nil {
-		// Do not record an event here, as we keep all event logging for sync pod failures
-		// local to container runtime so we get better errors
-		return err
+		// Do not return error if the only failures were pods in backoff
+		for _, r := range result.SyncResults {
+			if r.Error != kubecontainer.ErrCrashLoopBackOff && r.Error != images.ErrImagePullBackOff {
+				// Do not record an event here, as we keep all event logging for sync pod failures
+				// local to container runtime so we get better errors
+				return err
+			}
+		}
+
+		return nil
 	}
 
 	return nil
@@ -1841,7 +1887,7 @@ func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handle
 			// once we have checkpointing.
 			handler.HandlePodAdditions(u.Pods)
 		case kubetypes.UPDATE:
-			glog.V(2).Infof("SyncLoop (UPDATE, %q): %q", u.Source, format.PodsWithDeletiontimestamps(u.Pods))
+			glog.V(2).Infof("SyncLoop (UPDATE, %q): %q", u.Source, format.PodsWithDeletionTimestamps(u.Pods))
 			handler.HandlePodUpdates(u.Pods)
 		case kubetypes.REMOVE:
 			glog.V(2).Infof("SyncLoop (REMOVE, %q): %q", u.Source, format.Pods(u.Pods))
@@ -1976,6 +2022,10 @@ func (kl *Kubelet) HandlePodAdditions(pods []*v1.Pod) {
 	start := kl.clock.Now()
 	sort.Sort(sliceutils.PodsByCreationTime(pods))
 	for _, pod := range pods {
+		// Responsible for checking limits in resolv.conf
+		if kl.dnsConfigurer != nil && kl.dnsConfigurer.ResolverConfig != "" {
+			kl.dnsConfigurer.CheckLimitsForResolvConf()
+		}
 		existingPods := kl.podManager.GetPods()
 		// Always add the pod to the pod manager. Kubelet relies on the pod
 		// manager as the source of truth for the desired state. If a pod does
@@ -2013,6 +2063,10 @@ func (kl *Kubelet) HandlePodAdditions(pods []*v1.Pod) {
 func (kl *Kubelet) HandlePodUpdates(pods []*v1.Pod) {
 	start := kl.clock.Now()
 	for _, pod := range pods {
+		// Responsible for checking limits in resolv.conf
+		if kl.dnsConfigurer != nil && kl.dnsConfigurer.ResolverConfig != "" {
+			kl.dnsConfigurer.CheckLimitsForResolvConf()
+		}
 		kl.podManager.UpdatePod(pod)
 		if kubepod.IsMirrorPod(pod) {
 			kl.handleMirrorPod(pod, start)
@@ -2092,6 +2146,9 @@ func (kl *Kubelet) LatestLoopEntryTime() time.Time {
 // and returns an error if the status check fails.  If the status check is OK,
 // update the container runtime uptime in the kubelet runtimeState.
 func (kl *Kubelet) updateRuntimeUp() {
+	kl.updateRuntimeMux.Lock()
+	defer kl.updateRuntimeMux.Unlock()
+
 	s, err := kl.containerRuntime.Status()
 	if err != nil {
 		glog.Errorf("Container runtime sanity check failed: %v", err)
@@ -2166,6 +2223,31 @@ func (kl *Kubelet) cleanUpContainersInPod(podID types.UID, exitedContainerID str
 	}
 }
 
+// fastStatusUpdateOnce starts a loop that checks the internal node indexer cache for when a CIDR
+// is applied  and tries to update pod CIDR immediately. After pod CIDR is updated it fires off
+// a runtime update and a node status update. Function returns after one successful node status update.
+// Function is executed only during Kubelet start which improves latency to ready node by updating
+// pod CIDR, runtime status and node statuses ASAP.
+func (kl *Kubelet) fastStatusUpdateOnce() {
+	for {
+		time.Sleep(100 * time.Millisecond)
+		node, err := kl.GetNode()
+		if err != nil {
+			glog.Errorf(err.Error())
+			continue
+		}
+		if node.Spec.PodCIDR != "" {
+			if err := kl.updatePodCIDR(node.Spec.PodCIDR); err != nil {
+				glog.Errorf("Pod CIDR update failed %v", err)
+				continue
+			}
+			kl.updateRuntimeUp()
+			kl.syncNodeStatus()
+			return
+		}
+	}
+}
+
 // isSyncPodWorthy filters out events that are not worthy of pod syncing
 func isSyncPodWorthy(event *pleg.PodLifecycleEvent) bool {
 	// ContatnerRemoved doesn't affect pod state

vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_network.go

@@ -56,107 +56,25 @@ func (kl *Kubelet) providerRequiresNetworkingConfiguration() bool {
 
 // updatePodCIDR updates the pod CIDR in the runtime state if it is different
 // from the current CIDR.
-func (kl *Kubelet) updatePodCIDR(cidr string) {
+func (kl *Kubelet) updatePodCIDR(cidr string) error {
+	kl.updatePodCIDRMux.Lock()
+	defer kl.updatePodCIDRMux.Unlock()
+
 	podCIDR := kl.runtimeState.podCIDR()
 
 	if podCIDR == cidr {
-		return
+		return nil
 	}
 
 	// kubelet -> generic runtime -> runtime shim -> network plugin
 	// docker/non-cri implementations have a passthrough UpdatePodCIDR
 	if err := kl.getRuntime().UpdatePodCIDR(cidr); err != nil {
-		glog.Errorf("Failed to update pod CIDR: %v", err)
-		return
+		return fmt.Errorf("failed to update pod CIDR: %v", err)
 	}
 
 	glog.Infof("Setting Pod CIDR: %v -> %v", podCIDR, cidr)
 	kl.runtimeState.setPodCIDR(cidr)
-}
-
-// syncNetworkUtil ensures the network utility are present on host.
-// Network util includes:
-// 1. 	In nat table, KUBE-MARK-DROP rule to mark connections for dropping
-// 	Marked connection will be drop on INPUT/OUTPUT Chain in filter table
-// 2. 	In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT
-// 	Marked connection will get SNAT on POSTROUTING Chain in nat table
-func (kl *Kubelet) syncNetworkUtil() {
-	if kl.iptablesMasqueradeBit < 0 || kl.iptablesMasqueradeBit > 31 {
-		glog.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", kl.iptablesMasqueradeBit)
-		return
-	}
-
-	if kl.iptablesDropBit < 0 || kl.iptablesDropBit > 31 {
-		glog.Errorf("invalid iptables-drop-bit %v not in [0, 31]", kl.iptablesDropBit)
-		return
-	}
-
-	if kl.iptablesDropBit == kl.iptablesMasqueradeBit {
-		glog.Errorf("iptables-masquerade-bit %v and iptables-drop-bit %v must be different", kl.iptablesMasqueradeBit, kl.iptablesDropBit)
-		return
-	}
-
-	// Setup KUBE-MARK-DROP rules
-	dropMark := getIPTablesMark(kl.iptablesDropBit)
-	if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil {
-		glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--set-xmark", dropMark); err != nil {
-		glog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil {
-		glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
-		"-m", "comment", "--comment", "kubernetes firewall for dropping marked packets",
-		"-m", "mark", "--mark", dropMark,
-		"-j", "DROP"); err != nil {
-		glog.Errorf("Failed to ensure rule to drop packet marked by %v in %v chain %v: %v", KubeMarkDropChain, utiliptables.TableFilter, KubeFirewallChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil {
-		glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil {
-		glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err)
-		return
-	}
-
-	// Setup KUBE-MARK-MASQ rules
-	masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit)
-	if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil {
-		glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil {
-		glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--set-xmark", masqueradeMark); err != nil {
-		glog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting,
-		"-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil {
-		glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err)
-		return
-	}
-	if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
-		"-m", "comment", "--comment", "kubernetes service traffic requiring SNAT",
-		"-m", "mark", "--mark", masqueradeMark, "-j", "MASQUERADE"); err != nil {
-		glog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err)
-		return
-	}
-}
-
-// getIPTablesMark returns the fwmark given the bit
-func getIPTablesMark(bit int) string {
-	value := 1 << uint(bit)
-	return fmt.Sprintf("%#08x/%#08x", value, value)
+	return nil
 }
 
 // GetPodDNS returns DNS settings for the pod.

vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_network_linux.go

@@ -0,0 +1,111 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+// syncNetworkUtil ensures the network utility are present on host.
+// Network util includes:
+// 1. 	In nat table, KUBE-MARK-DROP rule to mark connections for dropping
+// 	Marked connection will be drop on INPUT/OUTPUT Chain in filter table
+// 2. 	In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT
+// 	Marked connection will get SNAT on POSTROUTING Chain in nat table
+func (kl *Kubelet) syncNetworkUtil() {
+	if kl.iptablesMasqueradeBit < 0 || kl.iptablesMasqueradeBit > 31 {
+		glog.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", kl.iptablesMasqueradeBit)
+		return
+	}
+
+	if kl.iptablesDropBit < 0 || kl.iptablesDropBit > 31 {
+		glog.Errorf("invalid iptables-drop-bit %v not in [0, 31]", kl.iptablesDropBit)
+		return
+	}
+
+	if kl.iptablesDropBit == kl.iptablesMasqueradeBit {
+		glog.Errorf("iptables-masquerade-bit %v and iptables-drop-bit %v must be different", kl.iptablesMasqueradeBit, kl.iptablesDropBit)
+		return
+	}
+
+	// Setup KUBE-MARK-DROP rules
+	dropMark := getIPTablesMark(kl.iptablesDropBit)
+	if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil {
+		glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--set-xmark", dropMark); err != nil {
+		glog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil {
+		glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
+		"-m", "comment", "--comment", "kubernetes firewall for dropping marked packets",
+		"-m", "mark", "--mark", dropMark,
+		"-j", "DROP"); err != nil {
+		glog.Errorf("Failed to ensure rule to drop packet marked by %v in %v chain %v: %v", KubeMarkDropChain, utiliptables.TableFilter, KubeFirewallChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil {
+		glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil {
+		glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err)
+		return
+	}
+
+	// Setup KUBE-MARK-MASQ rules
+	masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit)
+	if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil {
+		glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil {
+		glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--set-xmark", masqueradeMark); err != nil {
+		glog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting,
+		"-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil {
+		glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err)
+		return
+	}
+	if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
+		"-m", "comment", "--comment", "kubernetes service traffic requiring SNAT",
+		"-m", "mark", "--mark", masqueradeMark, "-j", "MASQUERADE"); err != nil {
+		glog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err)
+		return
+	}
+}
+
+// getIPTablesMark returns the fwmark given the bit
+func getIPTablesMark(bit int) string {
+	value := 1 << uint(bit)
+	return fmt.Sprintf("%#08x/%#08x", value, value)
+}

vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_network_others.go

@@ -1,5 +1,7 @@
+// +build !linux
+
 /*
-Copyright 2016 The Kubernetes Authors.
+Copyright 2018 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,23 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package util
+package kubelet
 
-import (
-	"k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// GetControllerRef gets pod's owner controller reference from a pod object.
-func GetControllerRef(pod *v1.Pod) *metav1.OwnerReference {
-	if len(pod.OwnerReferences) == 0 {
-		return nil
-	}
-	for i := range pod.OwnerReferences {
-		ref := &pod.OwnerReferences[i]
-		if ref.Controller != nil && *ref.Controller {
-			return ref
-		}
-	}
-	return nil
-}
+// Do nothing.
+func (kl *Kubelet) syncNetworkUtil() {}

vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_node_status.go

@@ -19,41 +19,32 @@ package kubelet
 import (
 	"context"
 	"fmt"
-	"math"
 	"net"
 	goruntime "runtime"
-	"strings"
-	"sync"
 	"time"
 
 	"github.com/golang/glog"
+
 	"k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	utilnet "k8s.io/apimachinery/pkg/util/net"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/features"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
-	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	"k8s.io/kubernetes/pkg/kubelet/events"
+	"k8s.io/kubernetes/pkg/kubelet/nodestatus"
 	"k8s.io/kubernetes/pkg/kubelet/util"
 	"k8s.io/kubernetes/pkg/scheduler/algorithm"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
-	"k8s.io/kubernetes/pkg/version"
+	taintutil "k8s.io/kubernetes/pkg/util/taints"
 	volutil "k8s.io/kubernetes/pkg/volume/util"
 )
 
-const (
-	// maxNamesPerImageInNodeStatus is max number of names per image stored in
-	// the node status.
-	maxNamesPerImageInNodeStatus = 5
-)
-
 // registerWithAPIServer registers the node with the cluster master. It is safe
 // to call multiple times, but not concurrently (kl.registrationCompleted is
 // not locked).
@@ -90,9 +81,7 @@ func (kl *Kubelet) registerWithAPIServer() {
 // the API server, returning a boolean indicating whether the attempt was
 // successful.  If a node with the same name already exists, it reconciles the
 // value of the annotation for controller-managed attach-detach of attachable
-// persistent volumes for the node.  If a node of the same name exists but has
-// a different externalID value, it attempts to delete that node so that a
-// later attempt can recreate it.
+// persistent volumes for the node.
 func (kl *Kubelet) tryRegisterWithAPIServer(node *v1.Node) bool {
 	_, err := kl.kubeClient.CoreV1().Nodes().Create(node)
 	if err == nil {
@@ -163,6 +152,9 @@ func (kl *Kubelet) updateDefaultLabels(initialNode, existingNode *v1.Node) bool
 	}
 
 	var needsUpdate bool = false
+	if existingNode.Labels == nil {
+		existingNode.Labels = make(map[string]string)
+	}
 	//Set default labels but make sure to not set labels with empty values
 	for _, label := range defaultLabels {
 		if _, hasInitialValue := initialNode.Labels[label]; !hasInitialValue {
@@ -238,6 +230,21 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
 		}
 		nodeTaints = append(nodeTaints, taints...)
 	}
+
+	unschedulableTaint := v1.Taint{
+		Key:    algorithm.TaintNodeUnschedulable,
+		Effect: v1.TaintEffectNoSchedule,
+	}
+
+	// If TaintNodesByCondition enabled, taint node with TaintNodeUnschedulable when initializing
+	// node to avoid race condition; refer to #63897 for more detail.
+	if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) {
+		if node.Spec.Unschedulable &&
+			!taintutil.TaintExists(nodeTaints, &unschedulableTaint) {
+			nodeTaints = append(nodeTaints, unschedulableTaint)
+		}
+	}
+
 	if kl.externalCloudProvider {
 		taint := v1.Taint{
 			Key:    algorithm.TaintExternalCloudProvider,
@@ -340,34 +347,13 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
 	return node, nil
 }
 
-// setVolumeLimits updates volume limits on the node
-func (kl *Kubelet) setVolumeLimits(node *v1.Node) {
-	if node.Status.Capacity == nil {
-		node.Status.Capacity = v1.ResourceList{}
-	}
-
-	if node.Status.Allocatable == nil {
-		node.Status.Allocatable = v1.ResourceList{}
-	}
-
-	pluginWithLimits := kl.volumePluginMgr.ListVolumePluginWithLimits()
-	for _, volumePlugin := range pluginWithLimits {
-		attachLimits, err := volumePlugin.GetVolumeLimits()
-		if err != nil {
-			glog.V(4).Infof("Error getting volume limit for plugin %s", volumePlugin.GetPluginName())
-			continue
-		}
-		for limitKey, value := range attachLimits {
-			node.Status.Capacity[v1.ResourceName(limitKey)] = *resource.NewQuantity(value, resource.DecimalSI)
-			node.Status.Allocatable[v1.ResourceName(limitKey)] = *resource.NewQuantity(value, resource.DecimalSI)
-		}
-	}
-}
-
 // syncNodeStatus should be called periodically from a goroutine.
 // It synchronizes node status to master, registering the kubelet first if
 // necessary.
 func (kl *Kubelet) syncNodeStatus() {
+	kl.syncNodeStatusMux.Lock()
+	defer kl.syncNodeStatusMux.Unlock()
+
 	if kl.kubeClient == nil || kl.heartbeatClient == nil {
 		return
 	}
@@ -396,8 +382,7 @@ func (kl *Kubelet) updateNodeStatus() error {
 	return fmt.Errorf("update node status exceeds retry count")
 }
 
-// tryUpdateNodeStatus tries to update node status to master. If ReconcileCBR0
-// is set, this function will also confirm that cbr0 is configured correctly.
+// tryUpdateNodeStatus tries to update node status to master.
 func (kl *Kubelet) tryUpdateNodeStatus(tryNumber int) error {
 	// In large clusters, GET and PUT operations on Node objects coming
 	// from here are the majority of load on apiserver and etcd.
@@ -409,7 +394,7 @@ func (kl *Kubelet) tryUpdateNodeStatus(tryNumber int) error {
 	if tryNumber == 0 {
 		util.FromApiserverCache(&opts)
 	}
-	node, err := kl.heartbeatClient.Nodes().Get(string(kl.nodeName), opts)
+	node, err := kl.heartbeatClient.CoreV1().Nodes().Get(string(kl.nodeName), opts)
 	if err != nil {
 		return fmt.Errorf("error getting node %q: %v", kl.nodeName, err)
 	}
@@ -420,15 +405,18 @@ func (kl *Kubelet) tryUpdateNodeStatus(tryNumber int) error {
 	}
 
 	if node.Spec.PodCIDR != "" {
-		kl.updatePodCIDR(node.Spec.PodCIDR)
+		if err := kl.updatePodCIDR(node.Spec.PodCIDR); err != nil {
+			glog.Errorf(err.Error())
+		}
 	}
 
 	kl.setNodeStatus(node)
 	// Patch the current status on the API server
-	updatedNode, _, err := nodeutil.PatchNodeStatus(kl.heartbeatClient, types.NodeName(kl.nodeName), originalNode, node)
+	updatedNode, _, err := nodeutil.PatchNodeStatus(kl.heartbeatClient.CoreV1(), types.NodeName(kl.nodeName), originalNode, node)
 	if err != nil {
 		return err
 	}
+	kl.setLastObservedNodeAddresses(updatedNode.Status.Addresses)
 	// If update finishes successfully, mark the volumeInUse as reportedInUse to indicate
 	// those volumes are already updated in the node's status
 	kl.volumeManager.MarkVolumesAsReportedInUse(updatedNode.Status.VolumesInUse)
@@ -444,663 +432,24 @@ func (kl *Kubelet) recordNodeStatusEvent(eventType, event string) {
 	kl.recorder.Eventf(kl.nodeRef, eventType, event, "Node %s status is now: %s", kl.nodeName, event)
 }
 
-// Set IP and hostname addresses for the node.
-func (kl *Kubelet) setNodeAddress(node *v1.Node) error {
-	if kl.nodeIP != nil {
-		if err := kl.nodeIPValidator(kl.nodeIP); err != nil {
-			return fmt.Errorf("failed to validate nodeIP: %v", err)
-		}
-		glog.V(2).Infof("Using node IP: %q", kl.nodeIP.String())
-	}
-
-	if kl.externalCloudProvider {
-		if kl.nodeIP != nil {
-			if node.ObjectMeta.Annotations == nil {
-				node.ObjectMeta.Annotations = make(map[string]string)
-			}
-			node.ObjectMeta.Annotations[kubeletapis.AnnotationProvidedIPAddr] = kl.nodeIP.String()
-		}
-		// We rely on the external cloud provider to supply the addresses.
-		return nil
-	}
-	if kl.cloud != nil {
-		instances, ok := kl.cloud.Instances()
-		if !ok {
-			return fmt.Errorf("failed to get instances from cloud provider")
-		}
-		// TODO(roberthbailey): Can we do this without having credentials to talk
-		// to the cloud provider?
-		// TODO(justinsb): We can if CurrentNodeName() was actually CurrentNode() and returned an interface
-		// TODO: If IP addresses couldn't be fetched from the cloud provider, should kubelet fallback on the other methods for getting the IP below?
-		var nodeAddresses []v1.NodeAddress
-		var err error
-
-		// Make sure the instances.NodeAddresses returns even if the cloud provider API hangs for a long time
-		func() {
-			kl.cloudproviderRequestMux.Lock()
-			if len(kl.cloudproviderRequestParallelism) > 0 {
-				kl.cloudproviderRequestMux.Unlock()
-				return
-			}
-			kl.cloudproviderRequestParallelism <- 0
-			kl.cloudproviderRequestMux.Unlock()
-
-			go func() {
-				nodeAddresses, err = instances.NodeAddresses(context.TODO(), kl.nodeName)
-
-				kl.cloudproviderRequestMux.Lock()
-				<-kl.cloudproviderRequestParallelism
-				kl.cloudproviderRequestMux.Unlock()
-
-				kl.cloudproviderRequestSync <- 0
-			}()
-		}()
-
-		select {
-		case <-kl.cloudproviderRequestSync:
-		case <-time.After(kl.cloudproviderRequestTimeout):
-			err = fmt.Errorf("Timeout after %v", kl.cloudproviderRequestTimeout)
-		}
-
-		if err != nil {
-			return fmt.Errorf("failed to get node address from cloud provider: %v", err)
-		}
-		if kl.nodeIP != nil {
-			enforcedNodeAddresses := []v1.NodeAddress{}
-
-			var nodeIPType v1.NodeAddressType
-			for _, nodeAddress := range nodeAddresses {
-				if nodeAddress.Address == kl.nodeIP.String() {
-					enforcedNodeAddresses = append(enforcedNodeAddresses, v1.NodeAddress{Type: nodeAddress.Type, Address: nodeAddress.Address})
-					nodeIPType = nodeAddress.Type
-					break
-				}
-			}
-			if len(enforcedNodeAddresses) > 0 {
-				for _, nodeAddress := range nodeAddresses {
-					if nodeAddress.Type != nodeIPType && nodeAddress.Type != v1.NodeHostName {
-						enforcedNodeAddresses = append(enforcedNodeAddresses, v1.NodeAddress{Type: nodeAddress.Type, Address: nodeAddress.Address})
-					}
-				}
-
-				enforcedNodeAddresses = append(enforcedNodeAddresses, v1.NodeAddress{Type: v1.NodeHostName, Address: kl.GetHostname()})
-				node.Status.Addresses = enforcedNodeAddresses
-				return nil
-			}
-			return fmt.Errorf("failed to get node address from cloud provider that matches ip: %v", kl.nodeIP)
-		}
-
-		// Only add a NodeHostName address if the cloudprovider did not specify one
-		// (we assume the cloudprovider knows best)
-		var addressNodeHostName *v1.NodeAddress
-		for i := range nodeAddresses {
-			if nodeAddresses[i].Type == v1.NodeHostName {
-				addressNodeHostName = &nodeAddresses[i]
-				break
-			}
-		}
-		if addressNodeHostName == nil {
-			hostnameAddress := v1.NodeAddress{Type: v1.NodeHostName, Address: kl.GetHostname()}
-			nodeAddresses = append(nodeAddresses, hostnameAddress)
-		} else {
-			glog.V(2).Infof("Using Node Hostname from cloudprovider: %q", addressNodeHostName.Address)
-		}
-		node.Status.Addresses = nodeAddresses
-	} else {
-		var ipAddr net.IP
-		var err error
-
-		// 1) Use nodeIP if set
-		// 2) If the user has specified an IP to HostnameOverride, use it
-		// 3) Lookup the IP from node name by DNS and use the first valid IPv4 address.
-		//    If the node does not have a valid IPv4 address, use the first valid IPv6 address.
-		// 4) Try to get the IP from the network interface used as default gateway
-		if kl.nodeIP != nil {
-			ipAddr = kl.nodeIP
-		} else if addr := net.ParseIP(kl.hostname); addr != nil {
-			ipAddr = addr
-		} else {
-			var addrs []net.IP
-			addrs, _ = net.LookupIP(node.Name)
-			for _, addr := range addrs {
-				if err = kl.nodeIPValidator(addr); err == nil {
-					if addr.To4() != nil {
-						ipAddr = addr
-						break
-					}
-					if addr.To16() != nil && ipAddr == nil {
-						ipAddr = addr
-					}
-				}
-			}
-
-			if ipAddr == nil {
-				ipAddr, err = utilnet.ChooseHostInterface()
-			}
-		}
-
-		if ipAddr == nil {
-			// We tried everything we could, but the IP address wasn't fetchable; error out
-			return fmt.Errorf("can't get ip address of node %s. error: %v", node.Name, err)
-		}
-		node.Status.Addresses = []v1.NodeAddress{
-			{Type: v1.NodeInternalIP, Address: ipAddr.String()},
-			{Type: v1.NodeHostName, Address: kl.GetHostname()},
-		}
-	}
-	return nil
+// recordEvent records an event for this node, the Kubelet's nodeRef is passed to the recorder
+func (kl *Kubelet) recordEvent(eventType, event, message string) {
+	kl.recorder.Eventf(kl.nodeRef, eventType, event, message)
 }
 
-func (kl *Kubelet) setNodeStatusMachineInfo(node *v1.Node) {
-	// Note: avoid blindly overwriting the capacity in case opaque
-	//       resources are being advertised.
-	if node.Status.Capacity == nil {
-		node.Status.Capacity = v1.ResourceList{}
-	}
-
-	var devicePluginAllocatable v1.ResourceList
-	var devicePluginCapacity v1.ResourceList
-	var removedDevicePlugins []string
-
-	// TODO: Post NotReady if we cannot get MachineInfo from cAdvisor. This needs to start
-	// cAdvisor locally, e.g. for test-cmd.sh, and in integration test.
-	info, err := kl.GetCachedMachineInfo()
-	if err != nil {
-		// TODO(roberthbailey): This is required for test-cmd.sh to pass.
-		// See if the test should be updated instead.
-		node.Status.Capacity[v1.ResourceCPU] = *resource.NewMilliQuantity(0, resource.DecimalSI)
-		node.Status.Capacity[v1.ResourceMemory] = resource.MustParse("0Gi")
-		node.Status.Capacity[v1.ResourcePods] = *resource.NewQuantity(int64(kl.maxPods), resource.DecimalSI)
-		glog.Errorf("Error getting machine info: %v", err)
-	} else {
-		node.Status.NodeInfo.MachineID = info.MachineID
-		node.Status.NodeInfo.SystemUUID = info.SystemUUID
-
-		for rName, rCap := range cadvisor.CapacityFromMachineInfo(info) {
-			node.Status.Capacity[rName] = rCap
-		}
-
-		if kl.podsPerCore > 0 {
-			node.Status.Capacity[v1.ResourcePods] = *resource.NewQuantity(
-				int64(math.Min(float64(info.NumCores*kl.podsPerCore), float64(kl.maxPods))), resource.DecimalSI)
-		} else {
-			node.Status.Capacity[v1.ResourcePods] = *resource.NewQuantity(
-				int64(kl.maxPods), resource.DecimalSI)
-		}
-
-		if node.Status.NodeInfo.BootID != "" &&
-			node.Status.NodeInfo.BootID != info.BootID {
-			// TODO: This requires a transaction, either both node status is updated
-			// and event is recorded or neither should happen, see issue #6055.
-			kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.NodeRebooted,
-				"Node %s has been rebooted, boot id: %s", kl.nodeName, info.BootID)
-		}
-		node.Status.NodeInfo.BootID = info.BootID
-
-		if utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) {
-			// TODO: all the node resources should use GetCapacity instead of deriving the
-			// capacity for every node status request
-			initialCapacity := kl.containerManager.GetCapacity()
-			if initialCapacity != nil {
-				node.Status.Capacity[v1.ResourceEphemeralStorage] = initialCapacity[v1.ResourceEphemeralStorage]
-			}
-		}
-
-		devicePluginCapacity, devicePluginAllocatable, removedDevicePlugins = kl.containerManager.GetDevicePluginResourceCapacity()
-		if devicePluginCapacity != nil {
-			for k, v := range devicePluginCapacity {
-				if old, ok := node.Status.Capacity[k]; !ok || old.Value() != v.Value() {
-					glog.V(2).Infof("Update capacity for %s to %d", k, v.Value())
-				}
-				node.Status.Capacity[k] = v
-			}
-		}
-
-		for _, removedResource := range removedDevicePlugins {
-			glog.V(2).Infof("Set capacity for %s to 0 on device removal", removedResource)
-			// Set the capacity of the removed resource to 0 instead of
-			// removing the resource from the node status. This is to indicate
-			// that the resource is managed by device plugin and had been
-			// registered before.
-			//
-			// This is required to differentiate the device plugin managed
-			// resources and the cluster-level resources, which are absent in
-			// node status.
-			node.Status.Capacity[v1.ResourceName(removedResource)] = *resource.NewQuantity(int64(0), resource.DecimalSI)
-		}
-	}
-
-	// Set Allocatable.
-	if node.Status.Allocatable == nil {
-		node.Status.Allocatable = make(v1.ResourceList)
-	}
-	// Remove extended resources from allocatable that are no longer
-	// present in capacity.
-	for k := range node.Status.Allocatable {
-		_, found := node.Status.Capacity[k]
-		if !found && v1helper.IsExtendedResourceName(k) {
-			delete(node.Status.Allocatable, k)
-		}
-	}
-	allocatableReservation := kl.containerManager.GetNodeAllocatableReservation()
-	for k, v := range node.Status.Capacity {
-		value := *(v.Copy())
-		if res, exists := allocatableReservation[k]; exists {
-			value.Sub(res)
-		}
-		if value.Sign() < 0 {
-			// Negative Allocatable resources don't make sense.
-			value.Set(0)
-		}
-		node.Status.Allocatable[k] = value
-	}
-
-	if devicePluginAllocatable != nil {
-		for k, v := range devicePluginAllocatable {
-			if old, ok := node.Status.Allocatable[k]; !ok || old.Value() != v.Value() {
-				glog.V(2).Infof("Update allocatable for %s to %d", k, v.Value())
-			}
-			node.Status.Allocatable[k] = v
-		}
-	}
-	// for every huge page reservation, we need to remove it from allocatable memory
-	for k, v := range node.Status.Capacity {
-		if v1helper.IsHugePageResourceName(k) {
-			allocatableMemory := node.Status.Allocatable[v1.ResourceMemory]
-			value := *(v.Copy())
-			allocatableMemory.Sub(value)
-			if allocatableMemory.Sign() < 0 {
-				// Negative Allocatable resources don't make sense.
-				allocatableMemory.Set(0)
-			}
-			node.Status.Allocatable[v1.ResourceMemory] = allocatableMemory
-		}
-	}
-}
-
-// Set versioninfo for the node.
-func (kl *Kubelet) setNodeStatusVersionInfo(node *v1.Node) {
-	verinfo, err := kl.cadvisor.VersionInfo()
-	if err != nil {
-		glog.Errorf("Error getting version info: %v", err)
-		return
-	}
-
-	node.Status.NodeInfo.KernelVersion = verinfo.KernelVersion
-	node.Status.NodeInfo.OSImage = verinfo.ContainerOsVersion
-
-	runtimeVersion := "Unknown"
-	if runtimeVer, err := kl.containerRuntime.Version(); err == nil {
-		runtimeVersion = runtimeVer.String()
-	}
-	node.Status.NodeInfo.ContainerRuntimeVersion = fmt.Sprintf("%s://%s", kl.containerRuntime.Type(), runtimeVersion)
-
-	node.Status.NodeInfo.KubeletVersion = version.Get().String()
-	// TODO: kube-proxy might be different version from kubelet in the future
-	node.Status.NodeInfo.KubeProxyVersion = version.Get().String()
-}
-
-// Set daemonEndpoints for the node.
-func (kl *Kubelet) setNodeStatusDaemonEndpoints(node *v1.Node) {
-	node.Status.DaemonEndpoints = *kl.daemonEndpoints
-}
-
-// Set images list for the node
-func (kl *Kubelet) setNodeStatusImages(node *v1.Node) {
-	// Update image list of this node
-	var imagesOnNode []v1.ContainerImage
-	containerImages, err := kl.imageManager.GetImageList()
-	if err != nil {
-		glog.Errorf("Error getting image list: %v", err)
-		node.Status.Images = imagesOnNode
-		return
-	}
-	// sort the images from max to min, and only set top N images into the node status.
-	if int(kl.nodeStatusMaxImages) > -1 &&
-		int(kl.nodeStatusMaxImages) < len(containerImages) {
-		containerImages = containerImages[0:kl.nodeStatusMaxImages]
-	}
-
-	for _, image := range containerImages {
-		names := append(image.RepoDigests, image.RepoTags...)
-		// Report up to maxNamesPerImageInNodeStatus names per image.
-		if len(names) > maxNamesPerImageInNodeStatus {
-			names = names[0:maxNamesPerImageInNodeStatus]
-		}
-		imagesOnNode = append(imagesOnNode, v1.ContainerImage{
-			Names:     names,
-			SizeBytes: image.Size,
-		})
-	}
-
-	node.Status.Images = imagesOnNode
-}
-
-// Set the GOOS and GOARCH for this node
-func (kl *Kubelet) setNodeStatusGoRuntime(node *v1.Node) {
-	node.Status.NodeInfo.OperatingSystem = goruntime.GOOS
-	node.Status.NodeInfo.Architecture = goruntime.GOARCH
-}
-
-// Set status for the node.
-func (kl *Kubelet) setNodeStatusInfo(node *v1.Node) {
-	kl.setNodeStatusMachineInfo(node)
-	kl.setNodeStatusVersionInfo(node)
-	kl.setNodeStatusDaemonEndpoints(node)
-	kl.setNodeStatusImages(node)
-	kl.setNodeStatusGoRuntime(node)
-	if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) {
-		kl.setVolumeLimits(node)
-	}
-}
-
-// Set Ready condition for the node.
-func (kl *Kubelet) setNodeReadyCondition(node *v1.Node) {
-	// NOTE(aaronlevy): NodeReady condition needs to be the last in the list of node conditions.
-	// This is due to an issue with version skewed kubelet and master components.
-	// ref: https://github.com/kubernetes/kubernetes/issues/16961
-	currentTime := metav1.NewTime(kl.clock.Now())
-	newNodeReadyCondition := v1.NodeCondition{
-		Type:              v1.NodeReady,
-		Status:            v1.ConditionTrue,
-		Reason:            "KubeletReady",
-		Message:           "kubelet is posting ready status",
-		LastHeartbeatTime: currentTime,
-	}
-	rs := append(kl.runtimeState.runtimeErrors(), kl.runtimeState.networkErrors()...)
-	requiredCapacities := []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory, v1.ResourcePods}
-	if utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) {
-		requiredCapacities = append(requiredCapacities, v1.ResourceEphemeralStorage)
-	}
-	missingCapacities := []string{}
-	for _, resource := range requiredCapacities {
-		if _, found := node.Status.Capacity[resource]; !found {
-			missingCapacities = append(missingCapacities, string(resource))
-		}
-	}
-	if len(missingCapacities) > 0 {
-		rs = append(rs, fmt.Sprintf("Missing node capacity for resources: %s", strings.Join(missingCapacities, ", ")))
-	}
-	if len(rs) > 0 {
-		newNodeReadyCondition = v1.NodeCondition{
-			Type:              v1.NodeReady,
-			Status:            v1.ConditionFalse,
-			Reason:            "KubeletNotReady",
-			Message:           strings.Join(rs, ","),
-			LastHeartbeatTime: currentTime,
-		}
-	}
-	// Append AppArmor status if it's enabled.
-	// TODO(tallclair): This is a temporary message until node feature reporting is added.
-	if newNodeReadyCondition.Status == v1.ConditionTrue &&
-		kl.appArmorValidator != nil && kl.appArmorValidator.ValidateHost() == nil {
-		newNodeReadyCondition.Message = fmt.Sprintf("%s. AppArmor enabled", newNodeReadyCondition.Message)
-	}
-
-	// Record any soft requirements that were not met in the container manager.
-	status := kl.containerManager.Status()
-	if status.SoftRequirements != nil {
-		newNodeReadyCondition.Message = fmt.Sprintf("%s. WARNING: %s", newNodeReadyCondition.Message, status.SoftRequirements.Error())
-	}
-
-	readyConditionUpdated := false
-	needToRecordEvent := false
-	for i := range node.Status.Conditions {
-		if node.Status.Conditions[i].Type == v1.NodeReady {
-			if node.Status.Conditions[i].Status == newNodeReadyCondition.Status {
-				newNodeReadyCondition.LastTransitionTime = node.Status.Conditions[i].LastTransitionTime
-			} else {
-				newNodeReadyCondition.LastTransitionTime = currentTime
-				needToRecordEvent = true
-			}
-			node.Status.Conditions[i] = newNodeReadyCondition
-			readyConditionUpdated = true
-			break
-		}
-	}
-	if !readyConditionUpdated {
-		newNodeReadyCondition.LastTransitionTime = currentTime
-		node.Status.Conditions = append(node.Status.Conditions, newNodeReadyCondition)
-	}
-	if needToRecordEvent {
-		if newNodeReadyCondition.Status == v1.ConditionTrue {
-			kl.recordNodeStatusEvent(v1.EventTypeNormal, events.NodeReady)
-		} else {
-			kl.recordNodeStatusEvent(v1.EventTypeNormal, events.NodeNotReady)
-			glog.Infof("Node became not ready: %+v", newNodeReadyCondition)
-		}
-	}
-}
-
-// setNodeMemoryPressureCondition for the node.
-// TODO: this needs to move somewhere centralized...
-func (kl *Kubelet) setNodeMemoryPressureCondition(node *v1.Node) {
-	currentTime := metav1.NewTime(kl.clock.Now())
-	var condition *v1.NodeCondition
-
-	// Check if NodeMemoryPressure condition already exists and if it does, just pick it up for update.
-	for i := range node.Status.Conditions {
-		if node.Status.Conditions[i].Type == v1.NodeMemoryPressure {
-			condition = &node.Status.Conditions[i]
-		}
-	}
-
-	newCondition := false
-	// If the NodeMemoryPressure condition doesn't exist, create one
-	if condition == nil {
-		condition = &v1.NodeCondition{
-			Type:   v1.NodeMemoryPressure,
-			Status: v1.ConditionUnknown,
-		}
-		// cannot be appended to node.Status.Conditions here because it gets
-		// copied to the slice. So if we append to the slice here none of the
-		// updates we make below are reflected in the slice.
-		newCondition = true
-	}
-
-	// Update the heartbeat time
-	condition.LastHeartbeatTime = currentTime
-
-	// Note: The conditions below take care of the case when a new NodeMemoryPressure condition is
-	// created and as well as the case when the condition already exists. When a new condition
-	// is created its status is set to v1.ConditionUnknown which matches either
-	// condition.Status != v1.ConditionTrue or
-	// condition.Status != v1.ConditionFalse in the conditions below depending on whether
-	// the kubelet is under memory pressure or not.
-	if kl.evictionManager.IsUnderMemoryPressure() {
-		if condition.Status != v1.ConditionTrue {
-			condition.Status = v1.ConditionTrue
-			condition.Reason = "KubeletHasInsufficientMemory"
-			condition.Message = "kubelet has insufficient memory available"
-			condition.LastTransitionTime = currentTime
-			kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasInsufficientMemory")
-		}
-	} else if condition.Status != v1.ConditionFalse {
-		condition.Status = v1.ConditionFalse
-		condition.Reason = "KubeletHasSufficientMemory"
-		condition.Message = "kubelet has sufficient memory available"
-		condition.LastTransitionTime = currentTime
-		kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasSufficientMemory")
-	}
-
-	if newCondition {
-		node.Status.Conditions = append(node.Status.Conditions, *condition)
-	}
-}
-
-// setNodePIDPressureCondition for the node.
-// TODO: this needs to move somewhere centralized...
-func (kl *Kubelet) setNodePIDPressureCondition(node *v1.Node) {
-	currentTime := metav1.NewTime(kl.clock.Now())
-	var condition *v1.NodeCondition
-
-	// Check if NodePIDPressure condition already exists and if it does, just pick it up for update.
-	for i := range node.Status.Conditions {
-		if node.Status.Conditions[i].Type == v1.NodePIDPressure {
-			condition = &node.Status.Conditions[i]
-		}
-	}
-
-	newCondition := false
-	// If the NodePIDPressure condition doesn't exist, create one
-	if condition == nil {
-		condition = &v1.NodeCondition{
-			Type:   v1.NodePIDPressure,
-			Status: v1.ConditionUnknown,
-		}
-		// cannot be appended to node.Status.Conditions here because it gets
-		// copied to the slice. So if we append to the slice here none of the
-		// updates we make below are reflected in the slice.
-		newCondition = true
-	}
-
-	// Update the heartbeat time
-	condition.LastHeartbeatTime = currentTime
-
-	// Note: The conditions below take care of the case when a new NodePIDPressure condition is
-	// created and as well as the case when the condition already exists. When a new condition
-	// is created its status is set to v1.ConditionUnknown which matches either
-	// condition.Status != v1.ConditionTrue or
-	// condition.Status != v1.ConditionFalse in the conditions below depending on whether
-	// the kubelet is under PID pressure or not.
-	if kl.evictionManager.IsUnderPIDPressure() {
-		if condition.Status != v1.ConditionTrue {
-			condition.Status = v1.ConditionTrue
-			condition.Reason = "KubeletHasInsufficientPID"
-			condition.Message = "kubelet has insufficient PID available"
-			condition.LastTransitionTime = currentTime
-			kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasInsufficientPID")
-		}
-	} else if condition.Status != v1.ConditionFalse {
-		condition.Status = v1.ConditionFalse
-		condition.Reason = "KubeletHasSufficientPID"
-		condition.Message = "kubelet has sufficient PID available"
-		condition.LastTransitionTime = currentTime
-		kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasSufficientPID")
-	}
-
-	if newCondition {
-		node.Status.Conditions = append(node.Status.Conditions, *condition)
-	}
-}
-
-// setNodeDiskPressureCondition for the node.
-// TODO: this needs to move somewhere centralized...
-func (kl *Kubelet) setNodeDiskPressureCondition(node *v1.Node) {
-	currentTime := metav1.NewTime(kl.clock.Now())
-	var condition *v1.NodeCondition
-
-	// Check if NodeDiskPressure condition already exists and if it does, just pick it up for update.
-	for i := range node.Status.Conditions {
-		if node.Status.Conditions[i].Type == v1.NodeDiskPressure {
-			condition = &node.Status.Conditions[i]
-		}
-	}
-
-	newCondition := false
-	// If the NodeDiskPressure condition doesn't exist, create one
-	if condition == nil {
-		condition = &v1.NodeCondition{
-			Type:   v1.NodeDiskPressure,
-			Status: v1.ConditionUnknown,
-		}
-		// cannot be appended to node.Status.Conditions here because it gets
-		// copied to the slice. So if we append to the slice here none of the
-		// updates we make below are reflected in the slice.
-		newCondition = true
-	}
-
-	// Update the heartbeat time
-	condition.LastHeartbeatTime = currentTime
-
-	// Note: The conditions below take care of the case when a new NodeDiskPressure condition is
-	// created and as well as the case when the condition already exists. When a new condition
-	// is created its status is set to v1.ConditionUnknown which matches either
-	// condition.Status != v1.ConditionTrue or
-	// condition.Status != v1.ConditionFalse in the conditions below depending on whether
-	// the kubelet is under disk pressure or not.
-	if kl.evictionManager.IsUnderDiskPressure() {
-		if condition.Status != v1.ConditionTrue {
-			condition.Status = v1.ConditionTrue
-			condition.Reason = "KubeletHasDiskPressure"
-			condition.Message = "kubelet has disk pressure"
-			condition.LastTransitionTime = currentTime
-			kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasDiskPressure")
-		}
-	} else if condition.Status != v1.ConditionFalse {
-		condition.Status = v1.ConditionFalse
-		condition.Reason = "KubeletHasNoDiskPressure"
-		condition.Message = "kubelet has no disk pressure"
-		condition.LastTransitionTime = currentTime
-		kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasNoDiskPressure")
-	}
-
-	if newCondition {
-		node.Status.Conditions = append(node.Status.Conditions, *condition)
-	}
-}
-
-// Set OODCondition for the node.
-func (kl *Kubelet) setNodeOODCondition(node *v1.Node) {
-	currentTime := metav1.NewTime(kl.clock.Now())
-	var nodeOODCondition *v1.NodeCondition
-
-	// Check if NodeOutOfDisk condition already exists and if it does, just pick it up for update.
-	for i := range node.Status.Conditions {
-		if node.Status.Conditions[i].Type == v1.NodeOutOfDisk {
-			nodeOODCondition = &node.Status.Conditions[i]
-		}
-	}
-
-	newOODCondition := nodeOODCondition == nil
-	if newOODCondition {
-		nodeOODCondition = &v1.NodeCondition{}
-	}
-	if nodeOODCondition.Status != v1.ConditionFalse {
-		nodeOODCondition.Type = v1.NodeOutOfDisk
-		nodeOODCondition.Status = v1.ConditionFalse
-		nodeOODCondition.Reason = "KubeletHasSufficientDisk"
-		nodeOODCondition.Message = "kubelet has sufficient disk space available"
-		nodeOODCondition.LastTransitionTime = currentTime
-		kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasSufficientDisk")
-	}
-
-	// Update the heartbeat time irrespective of all the conditions.
-	nodeOODCondition.LastHeartbeatTime = currentTime
-
-	if newOODCondition {
-		node.Status.Conditions = append(node.Status.Conditions, *nodeOODCondition)
-	}
-}
-
-// Maintains Node.Spec.Unschedulable value from previous run of tryUpdateNodeStatus()
-// TODO: why is this a package var?
-var (
-	oldNodeUnschedulable     bool
-	oldNodeUnschedulableLock sync.Mutex
-)
-
 // record if node schedulable change.
-func (kl *Kubelet) recordNodeSchedulableEvent(node *v1.Node) {
-	oldNodeUnschedulableLock.Lock()
-	defer oldNodeUnschedulableLock.Unlock()
-	if oldNodeUnschedulable != node.Spec.Unschedulable {
+func (kl *Kubelet) recordNodeSchedulableEvent(node *v1.Node) error {
+	kl.lastNodeUnschedulableLock.Lock()
+	defer kl.lastNodeUnschedulableLock.Unlock()
+	if kl.lastNodeUnschedulable != node.Spec.Unschedulable {
 		if node.Spec.Unschedulable {
 			kl.recordNodeStatusEvent(v1.EventTypeNormal, events.NodeNotSchedulable)
 		} else {
 			kl.recordNodeStatusEvent(v1.EventTypeNormal, events.NodeSchedulable)
 		}
-		oldNodeUnschedulable = node.Spec.Unschedulable
-	}
-}
-
-// Update VolumesInUse field in Node Status only after states are synced up at least once
-// in volume reconciler.
-func (kl *Kubelet) setNodeVolumesInUseStatus(node *v1.Node) {
-	// Make sure to only update node status after reconciler starts syncing up states
-	if kl.volumeManager.ReconcilerStatesHasBeenSynced() {
-		node.Status.VolumesInUse = kl.volumeManager.GetVolumesInUse()
+		kl.lastNodeUnschedulable = node.Spec.Unschedulable
 	}
+	return nil
 }
 
 // setNodeStatus fills in the Status fields of the given Node, overwriting
@@ -1116,27 +465,56 @@ func (kl *Kubelet) setNodeStatus(node *v1.Node) {
 	}
 }
 
+func (kl *Kubelet) setLastObservedNodeAddresses(addresses []v1.NodeAddress) {
+	kl.lastObservedNodeAddressesMux.Lock()
+	defer kl.lastObservedNodeAddressesMux.Unlock()
+	kl.lastObservedNodeAddresses = addresses
+}
+func (kl *Kubelet) getLastObservedNodeAddresses() []v1.NodeAddress {
+	kl.lastObservedNodeAddressesMux.Lock()
+	defer kl.lastObservedNodeAddressesMux.Unlock()
+	return kl.lastObservedNodeAddresses
+}
+
 // defaultNodeStatusFuncs is a factory that generates the default set of
 // setNodeStatus funcs
 func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error {
-	// initial set of node status update handlers, can be modified by Option's
-	withoutError := func(f func(*v1.Node)) func(*v1.Node) error {
-		return func(n *v1.Node) error {
-			f(n)
-			return nil
-		}
+	// if cloud is not nil, we expect the cloud resource sync manager to exist
+	var nodeAddressesFunc func() ([]v1.NodeAddress, error)
+	if kl.cloud != nil {
+		nodeAddressesFunc = kl.cloudResourceSyncManager.NodeAddresses
 	}
-	return []func(*v1.Node) error{
-		kl.setNodeAddress,
-		withoutError(kl.setNodeStatusInfo),
-		withoutError(kl.setNodeOODCondition),
-		withoutError(kl.setNodeMemoryPressureCondition),
-		withoutError(kl.setNodeDiskPressureCondition),
-		withoutError(kl.setNodePIDPressureCondition),
-		withoutError(kl.setNodeReadyCondition),
-		withoutError(kl.setNodeVolumesInUseStatus),
-		withoutError(kl.recordNodeSchedulableEvent),
+	var validateHostFunc func() error
+	if kl.appArmorValidator != nil {
+		validateHostFunc = kl.appArmorValidator.ValidateHost
 	}
+	var setters []func(n *v1.Node) error
+	setters = append(setters,
+		nodestatus.NodeAddress(kl.nodeIP, kl.nodeIPValidator, kl.hostname, kl.hostnameOverridden, kl.externalCloudProvider, kl.cloud, nodeAddressesFunc),
+		nodestatus.MachineInfo(string(kl.nodeName), kl.maxPods, kl.podsPerCore, kl.GetCachedMachineInfo, kl.containerManager.GetCapacity,
+			kl.containerManager.GetDevicePluginResourceCapacity, kl.containerManager.GetNodeAllocatableReservation, kl.recordEvent),
+		nodestatus.VersionInfo(kl.cadvisor.VersionInfo, kl.containerRuntime.Type, kl.containerRuntime.Version),
+		nodestatus.DaemonEndpoints(kl.daemonEndpoints),
+		nodestatus.Images(kl.nodeStatusMaxImages, kl.imageManager.GetImageList),
+		nodestatus.GoRuntime(),
+	)
+	if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) {
+		setters = append(setters, nodestatus.VolumeLimits(kl.volumePluginMgr.ListVolumePluginWithLimits))
+	}
+	setters = append(setters,
+		nodestatus.OutOfDiskCondition(kl.clock.Now, kl.recordNodeStatusEvent),
+		nodestatus.MemoryPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderMemoryPressure, kl.recordNodeStatusEvent),
+		nodestatus.DiskPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderDiskPressure, kl.recordNodeStatusEvent),
+		nodestatus.PIDPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderPIDPressure, kl.recordNodeStatusEvent),
+		nodestatus.ReadyCondition(kl.clock.Now, kl.runtimeState.runtimeErrors, kl.runtimeState.networkErrors, validateHostFunc, kl.containerManager.Status, kl.recordNodeStatusEvent),
+		nodestatus.VolumesInUse(kl.volumeManager.ReconcilerStatesHasBeenSynced, kl.volumeManager.GetVolumesInUse),
+		// TODO(mtaufen): I decided not to move this setter for now, since all it does is send an event
+		// and record state back to the Kubelet runtime object. In the future, I'd like to isolate
+		// these side-effects by decoupling the decisions to send events and partial status recording
+		// from the Node setters.
+		kl.recordNodeSchedulableEvent,
+	)
+	return setters
 }
 
 // Validate given node IP belongs to the current host

vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods.go

@@ -18,6 +18,7 @@ package kubelet
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -231,7 +232,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 		}
 		glog.V(5).Infof("Pod %q container %q mount %q has propagation %q", format.Pod(pod), container.Name, mount.Name, propagation)
 
-		mustMountRO := vol.Mounter.GetAttributes().ReadOnly && utilfeature.DefaultFeatureGate.Enabled(features.ReadOnlyAPIDataVolumes)
+		mustMountRO := vol.Mounter.GetAttributes().ReadOnly
 
 		mounts = append(mounts, kubecontainer.Mount{
 			Name:           mount.Name,
@@ -277,7 +278,7 @@ func translateMountPropagation(mountMode *v1.MountPropagationMode) (runtimeapi.M
 	case *mountMode == v1.MountPropagationNone:
 		return runtimeapi.MountPropagation_PROPAGATION_PRIVATE, nil
 	default:
-		return 0, fmt.Errorf("invalid MountPropagation mode: %q", mountMode)
+		return 0, fmt.Errorf("invalid MountPropagation mode: %q", *mountMode)
 	}
 }
 
@@ -1159,7 +1160,7 @@ func (kl *Kubelet) validateContainerLogStatus(podName string, podStatus *v1.PodS
 // GetKubeletContainerLogs returns logs from the container
 // TODO: this method is returning logs of random container attempts, when it should be returning the most recent attempt
 // or all of them.
-func (kl *Kubelet) GetKubeletContainerLogs(podFullName, containerName string, logOptions *v1.PodLogOptions, stdout, stderr io.Writer) error {
+func (kl *Kubelet) GetKubeletContainerLogs(ctx context.Context, podFullName, containerName string, logOptions *v1.PodLogOptions, stdout, stderr io.Writer) error {
 	// Pod workers periodically write status to statusManager. If status is not
 	// cached there, something is wrong (or kubelet just restarted and hasn't
 	// caught up yet). Just assume the pod is not ready yet.
@@ -1205,9 +1206,9 @@ func (kl *Kubelet) GetKubeletContainerLogs(podFullName, containerName string, lo
 		// dockerLegacyService should only be non-nil when we actually need it, so
 		// inject it into the runtimeService.
 		// TODO(random-liu): Remove this hack after deprecating unsupported log driver.
-		return kl.dockerLegacyService.GetContainerLogs(pod, containerID, logOptions, stdout, stderr)
+		return kl.dockerLegacyService.GetContainerLogs(ctx, pod, containerID, logOptions, stdout, stderr)
 	}
-	return kl.containerRuntime.GetContainerLogs(pod, containerID, logOptions, stdout, stderr)
+	return kl.containerRuntime.GetContainerLogs(ctx, pod, containerID, logOptions, stdout, stderr)
 }
 
 // getPhase returns the phase of a pod given its container info.

vendor/k8s.io/kubernetes/pkg/kubelet/runtime.go

@@ -53,12 +53,6 @@ func (s *runtimeState) setRuntimeSync(t time.Time) {
 	s.lastBaseRuntimeSync = t
 }
 
-func (s *runtimeState) setInternalError(err error) {
-	s.Lock()
-	defer s.Unlock()
-	s.internalError = err
-}
-
 func (s *runtimeState) setNetworkState(err error) {
 	s.Lock()
 	defer s.Unlock()

vendor/k8s.io/kubernetes/pkg/kubelet/types/BUILD

@@ -20,9 +20,11 @@ go_library(
     deps = [
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/scheduling:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//pkg/features:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 
@@ -36,10 +38,11 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/kubelet/types/pod_update.go

@@ -21,8 +21,10 @@ import (
 
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	kubeapi "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/scheduling"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 const (
@@ -141,10 +143,36 @@ func (sp SyncPodType) String() string {
 }
 
 // IsCriticalPod returns true if the pod bears the critical pod annotation key or if pod's priority is greater than
-// or equal to SystemCriticalPriority. Both the rescheduler(deprecated in 1.10) and the kubelet use this function
+// or equal to SystemCriticalPriority. Both the default scheduler and the kubelet use this function
 // to make admission and scheduling decisions.
 func IsCriticalPod(pod *v1.Pod) bool {
-	return IsCritical(pod.Namespace, pod.Annotations) || (pod.Spec.Priority != nil && IsCriticalPodBasedOnPriority(*pod.Spec.Priority))
+	if utilfeature.DefaultFeatureGate.Enabled(features.PodPriority) {
+		if pod.Spec.Priority != nil && IsCriticalPodBasedOnPriority(*pod.Spec.Priority) {
+			return true
+		}
+	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) {
+		if IsCritical(pod.Namespace, pod.Annotations) {
+			return true
+		}
+	}
+	return false
+}
+
+// Preemptable returns true if preemptor pod can preempt preemptee pod
+// if preemptee is not critical or if preemptor's priority is greater than preemptee's priority
+func Preemptable(preemptor, preemptee *v1.Pod) bool {
+	if IsCriticalPod(preemptor) && !IsCriticalPod(preemptee) {
+		return true
+	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.PodPriority) {
+		if (preemptor != nil && preemptor.Spec.Priority != nil) &&
+			(preemptee != nil && preemptee.Spec.Priority != nil) {
+			return *(preemptor.Spec.Priority) > *(preemptee.Spec.Priority)
+		}
+	}
+
+	return false
 }
 
 // IsCritical returns true if parameters bear the critical pod annotation

vendor/k8s.io/kubernetes/pkg/kubelet/util/BUILD

@@ -8,9 +8,7 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = [
-        "util_test.go",
-    ],
+    srcs = ["util_test.go"],
     embed = [":go_default_library"],
     deps = [
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
@@ -22,45 +20,13 @@ go_library(
     srcs = [
         "doc.go",
         "util.go",
-    ] + select({
-        "@io_bazel_rules_go//go/platform:android": [
-            "util_unsupported.go",
-        ],
-        "@io_bazel_rules_go//go/platform:darwin": [
-            "util_unix.go",
-        ],
-        "@io_bazel_rules_go//go/platform:dragonfly": [
-            "util_unsupported.go",
-        ],
-        "@io_bazel_rules_go//go/platform:freebsd": [
-            "util_unix.go",
-        ],
-        "@io_bazel_rules_go//go/platform:linux": [
-            "util_unix.go",
-        ],
-        "@io_bazel_rules_go//go/platform:nacl": [
-            "util_unsupported.go",
-        ],
-        "@io_bazel_rules_go//go/platform:netbsd": [
-            "util_unsupported.go",
-        ],
-        "@io_bazel_rules_go//go/platform:openbsd": [
-            "util_unsupported.go",
-        ],
-        "@io_bazel_rules_go//go/platform:plan9": [
-            "util_unsupported.go",
-        ],
-        "@io_bazel_rules_go//go/platform:solaris": [
-            "util_unsupported.go",
-        ],
-        "@io_bazel_rules_go//go/platform:windows": [
-            "util_windows.go",
-        ],
-        "//conditions:default": [],
-    }),
+        "util_unix.go",
+        "util_unsupported.go",
+        "util_windows.go",
+    ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/util",
     deps = [
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:darwin": [
             "//vendor/github.com/golang/glog:go_default_library",

vendor/k8s.io/kubernetes/pkg/kubelet/util/format/BUILD

@@ -1,10 +1,4 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-    "go_test",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -13,19 +7,26 @@ go_library(
         "resources.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/util/format",
+    visibility = ["//visibility:public"],
     deps = [
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = ["resources_test.go"],
+    srcs = [
+        "pod_test.go",
+        "resources_test.go",
+    ],
     embed = [":go_default_library"],
     deps = [
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
     ],
 )
 
@@ -40,4 +41,5 @@ filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )

vendor/k8s.io/kubernetes/pkg/kubelet/util/format/pod.go

@@ -57,9 +57,9 @@ func Pods(pods []*v1.Pod) string {
 	return aggregatePods(pods, Pod)
 }
 
-// PodsWithDeletiontimestamps is the same as Pods. In addition, it prints the
+// PodsWithDeletionTimestamps is the same as Pods. In addition, it prints the
 // deletion timestamps of the pods if they are not nil.
-func PodsWithDeletiontimestamps(pods []*v1.Pod) string {
+func PodsWithDeletionTimestamps(pods []*v1.Pod) string {
 	return aggregatePods(pods, PodWithDeletionTimestamp)
 }
 

vendor/k8s.io/kubernetes/pkg/kubelet/util/sliceutils/BUILD

@@ -12,7 +12,7 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/kubelet/util/sliceutils",
     deps = [
         "//pkg/kubelet/container:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
     ],
 )
 
@@ -35,7 +35,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/kubelet/container:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
     ],
 )

vendor/k8s.io/kubernetes/pkg/kubelet/volume_host.go

@@ -29,6 +29,7 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
+	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/configmap"
@@ -36,7 +37,6 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/mountpod"
 	"k8s.io/kubernetes/pkg/kubelet/secret"
 	"k8s.io/kubernetes/pkg/kubelet/token"
-	"k8s.io/kubernetes/pkg/util/io"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
@@ -122,6 +122,10 @@ func (kvh *kubeletVolumeHost) GetKubeClient() clientset.Interface {
 	return kvh.kubelet.kubeClient
 }
 
+func (kvh *kubeletVolumeHost) GetCSIClient() csiclientset.Interface {
+	return kvh.kubelet.csiClient
+}
+
 func (kvh *kubeletVolumeHost) NewWrapperMounter(
 	volName string,
 	spec volume.Spec,
@@ -168,10 +172,6 @@ func (kvh *kubeletVolumeHost) GetMounter(pluginName string) mount.Interface {
 	return mount.NewExecMounter(exec, kvh.kubelet.mounter)
 }
 
-func (kvh *kubeletVolumeHost) GetWriter() io.Writer {
-	return kvh.kubelet.writer
-}
-
 func (kvh *kubeletVolumeHost) GetHostName() string {
 	return kvh.kubelet.hostname
 }
@@ -247,7 +247,7 @@ func (kvh *kubeletVolumeHost) getMountExec(pluginName string) (mount.Exec, error
 		glog.V(5).Infof("using default mounter/exec for %s", pluginName)
 		return nil, nil
 	}
-	glog.V(5).Infof("using container %s/%s/%s to execute mount utilites for %s", pod.Namespace, pod.Name, container, pluginName)
+	glog.V(5).Infof("using container %s/%s/%s to execute mount utilities for %s", pod.Namespace, pod.Name, container, pluginName)
 	return &containerExec{
 		pod:           pod,
 		containerName: container,

vendor/k8s.io/kubernetes/pkg/master/BUILD

@@ -1,10 +1,4 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-    "go_test",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -18,6 +12,7 @@ go_library(
         "services.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/master",
+    visibility = ["//visibility:public"],
     deps = [
         "//pkg/apis/admission/install:go_default_library",
         "//pkg/apis/admissionregistration/install:go_default_library",
@@ -27,7 +22,7 @@ go_library(
         "//pkg/apis/autoscaling/install:go_default_library",
         "//pkg/apis/batch/install:go_default_library",
         "//pkg/apis/certificates/install:go_default_library",
-        "//pkg/apis/componentconfig/install:go_default_library",
+        "//pkg/apis/coordination/install:go_default_library",
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/core/install:go_default_library",
         "//pkg/apis/events/install:go_default_library",
@@ -40,6 +35,8 @@ go_library(
         "//pkg/apis/settings/install:go_default_library",
         "//pkg/apis/storage/install:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library",
+        "//pkg/client/informers/informers_generated/internalversion:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/kubeapiserver/options:go_default_library",
         "//pkg/kubelet/client:go_default_library",
         "//pkg/master/reconcilers:go_default_library",
@@ -51,7 +48,7 @@ go_library(
         "//pkg/registry/autoscaling/rest:go_default_library",
         "//pkg/registry/batch/rest:go_default_library",
         "//pkg/registry/certificates/rest:go_default_library",
-        "//pkg/registry/core/endpoint:go_default_library",
+        "//pkg/registry/coordination/rest:go_default_library",
         "//pkg/registry/core/endpoint/storage:go_default_library",
         "//pkg/registry/core/rangeallocation:go_default_library",
         "//pkg/registry/core/rest:go_default_library",
@@ -70,51 +67,54 @@ go_library(
         "//pkg/serviceaccount:go_default_library",
         "//pkg/util/async:go_default_library",
         "//pkg/util/node:go_default_library",
+        "//staging/src/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/api/admissionregistration/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1beta2:go_default_library",
+        "//staging/src/k8s.io/api/authentication/v1:go_default_library",
+        "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/authorization/v1:go_default_library",
+        "//staging/src/k8s.io/api/authorization/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/autoscaling/v1:go_default_library",
+        "//staging/src/k8s.io/api/autoscaling/v2beta1:go_default_library",
+        "//staging/src/k8s.io/api/autoscaling/v2beta2:go_default_library",
+        "//staging/src/k8s.io/api/batch/v1:go_default_library",
+        "//staging/src/k8s.io/api/batch/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/batch/v2alpha1:go_default_library",
+        "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/coordination/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/events/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/networking/v1:go_default_library",
+        "//staging/src/k8s.io/api/policy/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/scheduling/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/api/scheduling/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/settings/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/api/storage/v1:go_default_library",
+        "//staging/src/k8s.io/api/storage/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
-        "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
-        "//vendor/k8s.io/api/admissionregistration/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/apps/v1:go_default_library",
-        "//vendor/k8s.io/api/apps/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/apps/v1beta2:go_default_library",
-        "//vendor/k8s.io/api/authentication/v1:go_default_library",
-        "//vendor/k8s.io/api/authentication/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/authorization/v1:go_default_library",
-        "//vendor/k8s.io/api/authorization/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/autoscaling/v1:go_default_library",
-        "//vendor/k8s.io/api/autoscaling/v2beta1:go_default_library",
-        "//vendor/k8s.io/api/batch/v1:go_default_library",
-        "//vendor/k8s.io/api/batch/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/batch/v2alpha1:go_default_library",
-        "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/api/events/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/networking/v1:go_default_library",
-        "//vendor/k8s.io/api/policy/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/rbac/v1:go_default_library",
-        "//vendor/k8s.io/api/rbac/v1alpha1:go_default_library",
-        "//vendor/k8s.io/api/rbac/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/scheduling/v1alpha1:go_default_library",
-        "//vendor/k8s.io/api/scheduling/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/settings/v1alpha1:go_default_library",
-        "//vendor/k8s.io/api/storage/v1:go_default_library",
-        "//vendor/k8s.io/api/storage/v1alpha1:go_default_library",
-        "//vendor/k8s.io/api/storage/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory:go_default_library",
-        "//vendor/k8s.io/client-go/informers:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
     ],
 )
 
@@ -150,31 +150,31 @@ go_test(
         "//pkg/registry/core/rest:go_default_library",
         "//pkg/registry/registrytest:go_default_library",
         "//pkg/version:go_default_library",
+        "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/client-go/testing:go_default_library",
         "//vendor/github.com/go-openapi/loads:go_default_library",
         "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/go-openapi/strfmt:go_default_library",
         "//vendor/github.com/go-openapi/validate:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
-        "//vendor/k8s.io/client-go/informers:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
-        "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/testing:go_default_library",
     ],
 )
 
@@ -195,4 +195,5 @@ filegroup(
         "//pkg/master/tunneler:all-srcs",
     ],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )

vendor/k8s.io/kubernetes/pkg/master/OWNERS

@@ -39,3 +39,5 @@ reviewers:
 - hongchaodeng
 - jszczepkowski
 - enj
+labels:
+- sig/api-machinery

vendor/k8s.io/kubernetes/pkg/master/client_ca_hook.go

@@ -41,11 +41,6 @@ type ClientCARegistrationHook struct {
 }
 
 func (h ClientCARegistrationHook) PostStartHook(hookContext genericapiserver.PostStartHookContext) error {
-	// no work to do
-	if len(h.ClientCA) == 0 && len(h.RequestHeaderCA) == 0 {
-		return nil
-	}
-
 	// initializing CAs is important so that aggregated API servers can come up with "normal" config.
 	// We've seen lagging etcd before, so we want to retry this a few times before we decide to crashloop
 	// the API server on it.
@@ -68,7 +63,6 @@ func (h ClientCARegistrationHook) PostStartHook(hookContext genericapiserver.Pos
 	}
 
 	return nil
-
 }
 
 // tryToWriteClientCAs is here for unit testing with a fake client.  This is a wait.ConditionFunc so the bool

vendor/k8s.io/kubernetes/pkg/master/controller.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -29,8 +30,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	genericapiserver "k8s.io/apiserver/pkg/server"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/master/reconcilers"
 	"k8s.io/kubernetes/pkg/registry/core/rangeallocation"
 	corerest "k8s.io/kubernetes/pkg/registry/core/rest"
@@ -79,6 +82,16 @@ type Controller struct {
 
 // NewBootstrapController returns a controller for watching the core capabilities of the master
 func (c *completedConfig) NewBootstrapController(legacyRESTStorage corerest.LegacyRESTStorage, serviceClient coreclient.ServicesGetter, nsClient coreclient.NamespacesGetter, eventClient coreclient.EventsGetter) *Controller {
+	_, publicServicePort, err := c.GenericConfig.SecureServing.HostPort()
+	if err != nil {
+		glog.Fatalf("failed to get listener address: %v", err)
+	}
+
+	systemNamespaces := []string{metav1.NamespaceSystem, metav1.NamespacePublic}
+	if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) {
+		systemNamespaces = append(systemNamespaces, corev1.NamespaceNodeLease)
+	}
+
 	return &Controller{
 		ServiceClient:   serviceClient,
 		NamespaceClient: nsClient,
@@ -87,7 +100,7 @@ func (c *completedConfig) NewBootstrapController(legacyRESTStorage corerest.Lega
 		EndpointReconciler: c.ExtraConfig.EndpointReconcilerConfig.Reconciler,
 		EndpointInterval:   c.ExtraConfig.EndpointReconcilerConfig.Interval,
 
-		SystemNamespaces:         []string{metav1.NamespaceSystem, metav1.NamespacePublic},
+		SystemNamespaces:         systemNamespaces,
 		SystemNamespacesInterval: 1 * time.Minute,
 
 		ServiceClusterIPRegistry: legacyRESTStorage.ServiceClusterIPAllocator,
@@ -104,7 +117,7 @@ func (c *completedConfig) NewBootstrapController(legacyRESTStorage corerest.Lega
 		ServicePort:               c.ExtraConfig.APIServerServicePort,
 		ExtraServicePorts:         c.ExtraConfig.ExtraServicePorts,
 		ExtraEndpointPorts:        c.ExtraConfig.ExtraEndpointPorts,
-		PublicServicePort:         c.GenericConfig.ReadWritePort,
+		PublicServicePort:         publicServicePort,
 		KubernetesServiceNodePort: c.ExtraConfig.KubernetesServiceNodePort,
 	}
 }

vendor/k8s.io/kubernetes/pkg/master/import_known_versions.go

@@ -26,7 +26,7 @@ import (
 	_ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
 	_ "k8s.io/kubernetes/pkg/apis/batch/install"
 	_ "k8s.io/kubernetes/pkg/apis/certificates/install"
-	_ "k8s.io/kubernetes/pkg/apis/componentconfig/install"
+	_ "k8s.io/kubernetes/pkg/apis/coordination/install"
 	_ "k8s.io/kubernetes/pkg/apis/core/install"
 	_ "k8s.io/kubernetes/pkg/apis/events/install"
 	_ "k8s.io/kubernetes/pkg/apis/extensions/install"

vendor/k8s.io/kubernetes/pkg/master/master.go

@@ -35,10 +35,12 @@ import (
 	authorizationapiv1beta1 "k8s.io/api/authorization/v1beta1"
 	autoscalingapiv1 "k8s.io/api/autoscaling/v1"
 	autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1"
+	autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2"
 	batchapiv1 "k8s.io/api/batch/v1"
 	batchapiv1beta1 "k8s.io/api/batch/v1beta1"
 	batchapiv2alpha1 "k8s.io/api/batch/v2alpha1"
 	certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1"
+	coordinationapiv1beta1 "k8s.io/api/coordination/v1beta1"
 	apiv1 "k8s.io/api/core/v1"
 	eventsv1beta1 "k8s.io/api/events/v1beta1"
 	extensionsapiv1beta1 "k8s.io/api/extensions/v1beta1"
@@ -65,11 +67,11 @@ import (
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
+	internalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
 	"k8s.io/kubernetes/pkg/master/reconcilers"
 	"k8s.io/kubernetes/pkg/master/tunneler"
-	"k8s.io/kubernetes/pkg/registry/core/endpoint"
 	endpointsstorage "k8s.io/kubernetes/pkg/registry/core/endpoint/storage"
 	"k8s.io/kubernetes/pkg/routes"
 	"k8s.io/kubernetes/pkg/serviceaccount"
@@ -86,6 +88,7 @@ import (
 	autoscalingrest "k8s.io/kubernetes/pkg/registry/autoscaling/rest"
 	batchrest "k8s.io/kubernetes/pkg/registry/batch/rest"
 	certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest"
+	coordinationrest "k8s.io/kubernetes/pkg/registry/coordination/rest"
 	corerest "k8s.io/kubernetes/pkg/registry/core/rest"
 	eventsrest "k8s.io/kubernetes/pkg/registry/events/rest"
 	extensionsrest "k8s.io/kubernetes/pkg/registry/extensions/rest"
@@ -163,8 +166,12 @@ type ExtraConfig struct {
 	// Selects which reconciler to use
 	EndpointReconcilerType reconcilers.Type
 
-	ServiceAccountIssuer       serviceaccount.TokenGenerator
-	ServiceAccountAPIAudiences []string
+	ServiceAccountIssuer        serviceaccount.TokenGenerator
+	ServiceAccountAPIAudiences  []string
+	ServiceAccountMaxExpiration time.Duration
+
+	VersionedInformers informers.SharedInformerFactory
+	InternalInformers  internalinformers.SharedInformerFactory
 }
 
 type Config struct {
@@ -225,9 +232,8 @@ func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler {
 		DeleteCollectionWorkers: 0,
 		ResourcePrefix:          c.ExtraConfig.StorageFactory.ResourcePrefix(api.Resource("endpoints")),
 	})
-	endpointRegistry := endpoint.NewRegistry(endpointsStorage)
 	masterLeases := reconcilers.NewLeases(leaseStorage, "/masterleases/", ttl)
-	return reconcilers.NewLeaseEndpointReconciler(endpointRegistry, masterLeases)
+	return reconcilers.NewLeaseEndpointReconciler(endpointsStorage.Store, masterLeases)
 }
 
 func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler {
@@ -247,9 +253,9 @@ func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler {
 }
 
 // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
-func (cfg *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
+func (cfg *Config) Complete() CompletedConfig {
 	c := completedConfig{
-		cfg.GenericConfig.Complete(informers),
+		cfg.GenericConfig.Complete(cfg.ExtraConfig.VersionedInformers),
 		&cfg.ExtraConfig,
 	}
 
@@ -318,15 +324,16 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 	// install legacy rest storage
 	if c.ExtraConfig.APIResourceConfigSource.VersionEnabled(apiv1.SchemeGroupVersion) {
 		legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{
-			StorageFactory:             c.ExtraConfig.StorageFactory,
-			ProxyTransport:             c.ExtraConfig.ProxyTransport,
-			KubeletClientConfig:        c.ExtraConfig.KubeletClientConfig,
-			EventTTL:                   c.ExtraConfig.EventTTL,
-			ServiceIPRange:             c.ExtraConfig.ServiceIPRange,
-			ServiceNodePortRange:       c.ExtraConfig.ServiceNodePortRange,
-			LoopbackClientConfig:       c.GenericConfig.LoopbackClientConfig,
-			ServiceAccountIssuer:       c.ExtraConfig.ServiceAccountIssuer,
-			ServiceAccountAPIAudiences: c.ExtraConfig.ServiceAccountAPIAudiences,
+			StorageFactory:              c.ExtraConfig.StorageFactory,
+			ProxyTransport:              c.ExtraConfig.ProxyTransport,
+			KubeletClientConfig:         c.ExtraConfig.KubeletClientConfig,
+			EventTTL:                    c.ExtraConfig.EventTTL,
+			ServiceIPRange:              c.ExtraConfig.ServiceIPRange,
+			ServiceNodePortRange:        c.ExtraConfig.ServiceNodePortRange,
+			LoopbackClientConfig:        c.GenericConfig.LoopbackClientConfig,
+			ServiceAccountIssuer:        c.ExtraConfig.ServiceAccountIssuer,
+			ServiceAccountAPIAudiences:  c.ExtraConfig.ServiceAccountAPIAudiences,
+			ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration,
 		}
 		m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider)
 	}
@@ -344,6 +351,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		autoscalingrest.RESTStorageProvider{},
 		batchrest.RESTStorageProvider{},
 		certificatesrest.RESTStorageProvider{},
+		coordinationrest.RESTStorageProvider{},
 		extensionsrest.RESTStorageProvider{},
 		networkingrest.RESTStorageProvider{},
 		policyrest.RESTStorageProvider{},
@@ -364,6 +372,12 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 	}
 
 	m.GenericAPIServer.AddPostStartHookOrDie("ca-registration", c.ExtraConfig.ClientCARegistrationHook.PostStartHook)
+	m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-informers", func(context genericapiserver.PostStartHookContext) error {
+		if c.ExtraConfig.InternalInformers != nil {
+			c.ExtraConfig.InternalInformers.Start(context.StopCh)
+		}
+		return nil
+	})
 
 	return m, nil
 }
@@ -474,9 +488,11 @@ func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
 		authorizationapiv1beta1.SchemeGroupVersion,
 		autoscalingapiv1.SchemeGroupVersion,
 		autoscalingapiv2beta1.SchemeGroupVersion,
+		autoscalingapiv2beta2.SchemeGroupVersion,
 		batchapiv1.SchemeGroupVersion,
 		batchapiv1beta1.SchemeGroupVersion,
 		certificatesapiv1beta1.SchemeGroupVersion,
+		coordinationapiv1beta1.SchemeGroupVersion,
 		eventsv1beta1.SchemeGroupVersion,
 		extensionsapiv1beta1.SchemeGroupVersion,
 		networkingapiv1.SchemeGroupVersion,

vendor/k8s.io/kubernetes/pkg/master/ports/ports.go

@@ -28,9 +28,11 @@ const (
 	SchedulerPort = 10251
 	// InsecureKubeControllerManagerPort is the default port for the controller manager status server.
 	// May be overridden by a flag at startup.
+	// Deprecated: use the secure KubeControllerManagerPort instead.
 	InsecureKubeControllerManagerPort = 10252
 	// InsecureCloudControllerManagerPort is the default port for the cloud controller manager server.
 	// This value may be overridden by a flag at startup.
+	// Deprecated: use the secure CloudControllerManagerPort instead.
 	InsecureCloudControllerManagerPort = 10253
 	// KubeletReadOnlyPort exposes basic read-only services from the kubelet.
 	// May be overridden by a flag at startup.
@@ -41,4 +43,10 @@ const (
 	// ProxyHealthzPort is the default port for the proxy healthz server.
 	// May be overridden by a flag at startup.
 	ProxyHealthzPort = 10256
+	// KubeControllerManagerPort is the default port for the controller manager status server.
+	// May be overridden by a flag at startup.
+	KubeControllerManagerPort = 10257
+	// CloudControllerManagerPort is the default port for the cloud controller manager server.
+	// This value may be overridden by a flag at startup.
+	CloudControllerManagerPort = 10258
 )

vendor/k8s.io/kubernetes/pkg/scheduler/BUILD

@@ -15,20 +15,21 @@ go_test(
         "//pkg/controller/volume/persistentvolume:go_default_library",
         "//pkg/scheduler/algorithm:go_default_library",
         "//pkg/scheduler/algorithm/predicates:go_default_library",
+        "//pkg/scheduler/api:go_default_library",
         "//pkg/scheduler/cache:go_default_library",
         "//pkg/scheduler/core:go_default_library",
         "//pkg/scheduler/testing:go_default_library",
         "//pkg/scheduler/volumebinder:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
     ],
 )
 
@@ -46,18 +47,19 @@ go_library(
         "//pkg/scheduler/api:go_default_library",
         "//pkg/scheduler/cache:go_default_library",
         "//pkg/scheduler/core:go_default_library",
+        "//pkg/scheduler/core/equivalence:go_default_library",
         "//pkg/scheduler/metrics:go_default_library",
         "//pkg/scheduler/util:go_default_library",
         "//pkg/scheduler/volumebinder:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
     ],
 )
 
@@ -75,6 +77,7 @@ filegroup(
         "//pkg/scheduler/algorithm:all-srcs",
         "//pkg/scheduler/algorithmprovider:all-srcs",
         "//pkg/scheduler/api:all-srcs",
+        "//pkg/scheduler/apis/config:all-srcs",
         "//pkg/scheduler/cache:all-srcs",
         "//pkg/scheduler/core:all-srcs",
         "//pkg/scheduler/factory:all-srcs",

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/BUILD

@@ -19,24 +19,20 @@ go_library(
         "//pkg/apis/core:go_default_library",
         "//pkg/scheduler/api:go_default_library",
         "//pkg/scheduler/cache:go_default_library",
-        "//vendor/k8s.io/api/apps/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = [
-        "scheduler_interface_test.go",
-        "types_test.go",
-    ],
+    srcs = ["types_test.go"],
     embed = [":go_default_library"],
     deps = [
         "//pkg/scheduler/cache:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/BUILD

@@ -37,14 +37,14 @@ go_library(
         "//pkg/scheduler/cache:go_default_library",
         "//pkg/util/node:go_default_library",
         "//pkg/util/parsers:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
     ],
 )
 
@@ -74,13 +74,12 @@ go_test(
         "//pkg/scheduler/cache:go_default_library",
         "//pkg/scheduler/testing:go_default_library",
         "//pkg/util/parsers:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/k8s.io/api/apps/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/balanced_resource_allocation.go

@@ -32,7 +32,7 @@ var (
 	// BalancedResourceAllocationMap should **NOT** be used alone, and **MUST** be used together
 	// with LeastRequestedPriority. It calculates the difference between the cpu and memory fraction
 	// of capacity, and prioritizes the host based on how close the two metrics are to each other.
-	// Detail: score = 10 - abs(cpuFraction-memoryFraction)*10. The algorithm is partly inspired by:
+	// Detail: score = 10 - variance(cpuFraction,memoryFraction,volumeFraction)*10. The algorithm is partly inspired by:
 	// "Wei Huang et al. An Energy Efficient Virtual Machine Placement Algorithm with Balanced
 	// Resource Utilization"
 	BalancedResourceAllocationMap = balancedResourcePriority.PriorityMap

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/image_locality.go

@@ -26,11 +26,12 @@ import (
 	"k8s.io/kubernetes/pkg/util/parsers"
 )
 
-// This is a reasonable size range of all container images. 90%ile of images on dockerhub drops into this range.
+// The two thresholds are used as bounds for the image score range. They correspond to a reasonable size range for
+// container images compressed and stored in registries; 90%ile of images on dockerhub drops into this range.
 const (
-	mb         int64 = 1024 * 1024
-	minImgSize int64 = 23 * mb
-	maxImgSize int64 = 1000 * mb
+	mb           int64 = 1024 * 1024
+	minThreshold int64 = 23 * mb
+	maxThreshold int64 = 1000 * mb
 )
 
 // ImageLocalityPriorityMap is a priority function that favors nodes that already have requested pod container's images.
@@ -44,44 +45,55 @@ func ImageLocalityPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *scheduler
 		return schedulerapi.HostPriority{}, fmt.Errorf("node not found")
 	}
 
-	sumSize := totalImageSize(nodeInfo, pod.Spec.Containers)
+	var score int
+	if priorityMeta, ok := meta.(*priorityMetadata); ok {
+		score = calculatePriority(sumImageScores(nodeInfo, pod.Spec.Containers, priorityMeta.totalNumNodes))
+	} else {
+		// if we are not able to parse priority meta data, skip this priority
+		score = 0
+	}
 
 	return schedulerapi.HostPriority{
 		Host:  node.Name,
-		Score: calculateScoreFromSize(sumSize),
+		Score: score,
 	}, nil
 }
 
-// calculateScoreFromSize calculates the priority of a node. sumSize is sum size of requested images on this node.
-// 1. Split image size range into 10 buckets.
-// 2. Decide the priority of a given sumSize based on which bucket it belongs to.
-func calculateScoreFromSize(sumSize int64) int {
-	switch {
-	case sumSize == 0 || sumSize < minImgSize:
-		// 0 means none of the images required by this pod are present on this
-		// node or the total size of the images present is too small to be taken into further consideration.
-		return 0
-
-	case sumSize >= maxImgSize:
-		// If existing images' total size is larger than max, just make it highest priority.
-		return schedulerapi.MaxPriority
+// calculatePriority returns the priority of a node. Given the sumScores of requested images on the node, the node's
+// priority is obtained by scaling the maximum priority value with a ratio proportional to the sumScores.
+func calculatePriority(sumScores int64) int {
+	if sumScores < minThreshold {
+		sumScores = minThreshold
+	} else if sumScores > maxThreshold {
+		sumScores = maxThreshold
 	}
 
-	return int((int64(schedulerapi.MaxPriority) * (sumSize - minImgSize) / (maxImgSize - minImgSize)) + 1)
+	return int(int64(schedulerapi.MaxPriority) * (sumScores - minThreshold) / (maxThreshold - minThreshold))
 }
 
-// totalImageSize returns the total image size of all the containers that are already on the node.
-func totalImageSize(nodeInfo *schedulercache.NodeInfo, containers []v1.Container) int64 {
-	var total int64
+// sumImageScores returns the sum of image scores of all the containers that are already on the node.
+// Each image receives a raw score of its size, scaled by scaledImageScore. The raw scores are later used to calculate
+// the final score. Note that the init containers are not considered for it's rare for users to deploy huge init containers.
+func sumImageScores(nodeInfo *schedulercache.NodeInfo, containers []v1.Container, totalNumNodes int) int64 {
+	var sum int64
+	imageStates := nodeInfo.ImageStates()
 
-	imageSizes := nodeInfo.ImageSizes()
 	for _, container := range containers {
-		if size, ok := imageSizes[normalizedImageName(container.Image)]; ok {
-			total += size
+		if state, ok := imageStates[normalizedImageName(container.Image)]; ok {
+			sum += scaledImageScore(state, totalNumNodes)
 		}
 	}
 
-	return total
+	return sum
+}
+
+// scaledImageScore returns an adaptively scaled score for the given state of an image.
+// The size of the image is used as the base score, scaled by a factor which considers how much nodes the image has "spread" to.
+// This heuristic aims to mitigate the undesirable "node heating problem", i.e., pods get assigned to the same or
+// a few nodes due to image locality.
+func scaledImageScore(imageState *schedulercache.ImageStateSummary, totalNumNodes int) int64 {
+	spread := float64(imageState.NumNodes) / float64(totalNumNodes)
+	return int64(float64(imageState.Size) * spread)
 }
 
 // normalizedImageName returns the CRI compliant name for a given image.

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/least_requested.go

@@ -29,7 +29,7 @@ var (
 	// prioritizes based on the minimum of the average of the fraction of requested to capacity.
 	//
 	// Details:
-	// cpu((capacity-sum(requested))*10/capacity) + memory((capacity-sum(requested))*10/capacity)/2
+	// (cpu((capacity-sum(requested))*10/capacity) + memory((capacity-sum(requested))*10/capacity))/2
 	LeastRequestedPriorityMap = leastResourcePriority.PriorityMap
 )
 

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/metadata.go

@@ -21,7 +21,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/kubernetes/pkg/scheduler/algorithm"
-	priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util"
 	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
 )
 
@@ -52,6 +51,7 @@ type priorityMetadata struct {
 	podSelectors            []labels.Selector
 	controllerRef           *metav1.OwnerReference
 	podFirstServiceSelector labels.Selector
+	totalNumNodes           int
 }
 
 // PriorityMetadata is a PriorityMetadataProducer.  Node info can be nil.
@@ -65,8 +65,9 @@ func (pmf *PriorityMetadataFactory) PriorityMetadata(pod *v1.Pod, nodeNameToInfo
 		podTolerations:          getAllTolerationPreferNoSchedule(pod.Spec.Tolerations),
 		affinity:                pod.Spec.Affinity,
 		podSelectors:            getSelectors(pod, pmf.serviceLister, pmf.controllerLister, pmf.replicaSetLister, pmf.statefulSetLister),
-		controllerRef:           priorityutil.GetControllerRef(pod),
+		controllerRef:           metav1.GetControllerOf(pod),
 		podFirstServiceSelector: getFirstServiceSelector(pod, pmf.serviceLister),
+		totalNumNodes:           len(nodeNameToInfo),
 	}
 }
 

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/most_requested.go

@@ -39,10 +39,10 @@ func mostResourceScorer(requested, allocable *schedulercache.Resource, includeVo
 // The used capacity is calculated on a scale of 0-10
 // 0 being the lowest priority and 10 being the highest.
 // The more resources are used the higher the score is. This function
-// is almost a reversed version of least_requested_priority.calculatUnusedScore
+// is almost a reversed version of least_requested_priority.calculateUnusedScore
 // (10 - calculateUnusedScore). The main difference is in rounding. It was added to
 // keep the final formula clean and not to modify the widely used (by users
-// in their default scheduling policies) calculateUSedScore.
+// in their default scheduling policies) calculateUsedScore.
 func mostRequestedScore(requested, capacity int64) int64 {
 	if capacity == 0 {
 		return 0

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods.go

@@ -22,7 +22,6 @@ import (
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
-	priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
 )
@@ -39,7 +38,7 @@ func CalculateNodePreferAvoidPodsPriorityMap(pod *v1.Pod, meta interface{}, node
 		controllerRef = priorityMeta.controllerRef
 	} else {
 		// We couldn't parse metadata - fallback to the podspec.
-		controllerRef = priorityutil.GetControllerRef(pod)
+		controllerRef = metav1.GetControllerOf(pod)
 	}
 
 	if controllerRef != nil {

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/resource_allocation.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/features"
 	priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
@@ -56,20 +58,31 @@ func (r *ResourceAllocationPriority) PriorityMap(
 	requested.Memory += nodeInfo.NonZeroRequest().Memory
 	var score int64
 	// Check if the pod has volumes and this could be added to scorer function for balanced resource allocation.
-	if len(pod.Spec.Volumes) >= 0 && nodeInfo.TransientInfo != nil {
+	if len(pod.Spec.Volumes) >= 0 && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) && nodeInfo.TransientInfo != nil {
 		score = r.scorer(&requested, &allocatable, true, nodeInfo.TransientInfo.TransNodeInfo.RequestedVolumes, nodeInfo.TransientInfo.TransNodeInfo.AllocatableVolumesCount)
 	} else {
 		score = r.scorer(&requested, &allocatable, false, 0, 0)
 	}
 
 	if glog.V(10) {
-		glog.Infof(
-			"%v -> %v: %v, capacity %d millicores %d memory bytes, total request %d millicores %d memory bytes, score %d",
-			pod.Name, node.Name, r.Name,
-			allocatable.MilliCPU, allocatable.Memory,
-			requested.MilliCPU+allocatable.MilliCPU, requested.Memory+allocatable.Memory,
-			score,
-		)
+		if len(pod.Spec.Volumes) >= 0 && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) && nodeInfo.TransientInfo != nil {
+			glog.Infof(
+				"%v -> %v: %v, capacity %d millicores %d memory bytes, %d volumes, total request %d millicores %d memory bytes %d volumes, score %d",
+				pod.Name, node.Name, r.Name,
+				allocatable.MilliCPU, allocatable.Memory, nodeInfo.TransientInfo.TransNodeInfo.AllocatableVolumesCount,
+				requested.MilliCPU, requested.Memory,
+				nodeInfo.TransientInfo.TransNodeInfo.RequestedVolumes,
+				score,
+			)
+		} else {
+			glog.Infof(
+				"%v -> %v: %v, capacity %d millicores %d memory bytes, total request %d millicores %d memory bytes, score %d",
+				pod.Name, node.Name, r.Name,
+				allocatable.MilliCPU, allocatable.Memory,
+				requested.MilliCPU, requested.Memory,
+				score,
+			)
+		}
 	}
 
 	return schedulerapi.HostPriority{

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/resource_limits.go

@@ -70,7 +70,7 @@ func ResourceLimitsPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *schedule
 	}, nil
 }
 
-// computeScore return 1 if limit value is less than or equal to allocable
+// computeScore returns 1 if limit value is less than or equal to allocatable
 // value, otherwise it returns 0.
 func computeScore(limit, allocatable int64) int64 {
 	if limit != 0 && allocatable != 0 && limit <= allocatable {

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/selector_spreading.go

@@ -97,16 +97,12 @@ func (s *SelectorSpread) CalculateSpreadPriorityMap(pod *v1.Pod, meta interface{
 			glog.V(4).Infof("skipping pending-deleted pod: %s/%s", nodePod.Namespace, nodePod.Name)
 			continue
 		}
-		matches := false
 		for _, selector := range selectors {
 			if selector.Matches(labels.Set(nodePod.ObjectMeta.Labels)) {
-				matches = true
+				count++
 				break
 			}
 		}
-		if matches {
-			count++
-		}
 	}
 	return schedulerapi.HostPriority{
 		Host:  node.Name,

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util/BUILD

@@ -11,17 +11,16 @@ go_test(
     srcs = [
         "non_zero_test.go",
         "topologies_test.go",
-        "util_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/selection:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/selection:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
     ],
 )
 
@@ -30,14 +29,12 @@ go_library(
     srcs = [
         "non_zero.go",
         "topologies.go",
-        "util.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util",
     deps = [
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/scheduler/algorithm/types.go

@@ -17,9 +17,8 @@ limitations under the License.
 package algorithm
 
 import (
-	apps "k8s.io/api/apps/v1beta1"
+	apps "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
-	extensions "k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/labels"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
@@ -120,7 +119,7 @@ type ControllerLister interface {
 // ReplicaSetLister interface represents anything that can produce a list of ReplicaSet; the list is consumed by a scheduler.
 type ReplicaSetLister interface {
 	// Gets the replicasets for the given pod
-	GetPodReplicaSets(*v1.Pod) ([]*extensions.ReplicaSet, error)
+	GetPodReplicaSets(*v1.Pod) ([]*apps.ReplicaSet, error)
 }
 
 var _ ControllerLister = &EmptyControllerLister{}
@@ -144,7 +143,7 @@ var _ ReplicaSetLister = &EmptyReplicaSetLister{}
 type EmptyReplicaSetLister struct{}
 
 // GetPodReplicaSets returns nil
-func (f EmptyReplicaSetLister) GetPodReplicaSets(pod *v1.Pod) (rss []*extensions.ReplicaSet, err error) {
+func (f EmptyReplicaSetLister) GetPodReplicaSets(pod *v1.Pod) (rss []*apps.ReplicaSet, err error) {
 	return nil, nil
 }
 

vendor/k8s.io/kubernetes/pkg/scheduler/api/BUILD

@@ -15,12 +15,12 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/scheduler/api",
     deps = [
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/scheduler/api/types.go

@@ -36,6 +36,9 @@ const (
 	MaxPriority = 10
 	// MaxWeight defines the max weight value.
 	MaxWeight = MaxInt / MaxPriority
+	// DefaultPercentageOfNodesToScore defines the percentage of nodes of all nodes
+	// that once found feasible, the scheduler stops looking for more nodes.
+	DefaultPercentageOfNodesToScore = 50
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

vendor/k8s.io/kubernetes/pkg/scheduler/api/zz_generated.deepcopy.go

@@ -31,33 +31,21 @@ func (in *ExtenderArgs) DeepCopyInto(out *ExtenderArgs) {
 	*out = *in
 	if in.Pod != nil {
 		in, out := &in.Pod, &out.Pod
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.Pod)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.Pod)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Nodes != nil {
 		in, out := &in.Nodes, &out.Nodes
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.NodeList)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.NodeList)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.NodeNames != nil {
 		in, out := &in.NodeNames, &out.NodeNames
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new([]string)
-			if **in != nil {
-				in, out := *in, *out
-				*out = make([]string, len(*in))
-				copy(*out, *in)
-			}
+		*out = new([]string)
+		if **in != nil {
+			in, out := *in, *out
+			*out = make([]string, len(*in))
+			copy(*out, *in)
 		}
 	}
 	return
@@ -110,12 +98,8 @@ func (in *ExtenderConfig) DeepCopyInto(out *ExtenderConfig) {
 	*out = *in
 	if in.TLSConfig != nil {
 		in, out := &in.TLSConfig, &out.TLSConfig
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(rest.TLSClientConfig)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(rest.TLSClientConfig)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.ManagedResources != nil {
 		in, out := &in.ManagedResources, &out.ManagedResources
@@ -140,24 +124,16 @@ func (in *ExtenderFilterResult) DeepCopyInto(out *ExtenderFilterResult) {
 	*out = *in
 	if in.Nodes != nil {
 		in, out := &in.Nodes, &out.Nodes
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.NodeList)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.NodeList)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.NodeNames != nil {
 		in, out := &in.NodeNames, &out.NodeNames
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new([]string)
-			if **in != nil {
-				in, out := *in, *out
-				*out = make([]string, len(*in))
-				copy(*out, *in)
-			}
+		*out = new([]string)
+		if **in != nil {
+			in, out := *in, *out
+			*out = make([]string, len(*in))
+			copy(*out, *in)
 		}
 	}
 	if in.FailedNodes != nil {
@@ -201,35 +177,37 @@ func (in *ExtenderPreemptionArgs) DeepCopyInto(out *ExtenderPreemptionArgs) {
 	*out = *in
 	if in.Pod != nil {
 		in, out := &in.Pod, &out.Pod
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(v1.Pod)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(v1.Pod)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.NodeNameToVictims != nil {
 		in, out := &in.NodeNameToVictims, &out.NodeNameToVictims
 		*out = make(map[string]*Victims, len(*in))
 		for key, val := range *in {
+			var outVal *Victims
 			if val == nil {
 				(*out)[key] = nil
 			} else {
-				(*out)[key] = new(Victims)
-				val.DeepCopyInto((*out)[key])
+				in, out := &val, &outVal
+				*out = new(Victims)
+				(*in).DeepCopyInto(*out)
 			}
+			(*out)[key] = outVal
 		}
 	}
 	if in.NodeNameToMetaVictims != nil {
 		in, out := &in.NodeNameToMetaVictims, &out.NodeNameToMetaVictims
 		*out = make(map[string]*MetaVictims, len(*in))
 		for key, val := range *in {
+			var outVal *MetaVictims
 			if val == nil {
 				(*out)[key] = nil
 			} else {
-				(*out)[key] = new(MetaVictims)
-				val.DeepCopyInto((*out)[key])
+				in, out := &val, &outVal
+				*out = new(MetaVictims)
+				(*in).DeepCopyInto(*out)
 			}
+			(*out)[key] = outVal
 		}
 	}
 	return
@@ -252,12 +230,15 @@ func (in *ExtenderPreemptionResult) DeepCopyInto(out *ExtenderPreemptionResult)
 		in, out := &in.NodeNameToMetaVictims, &out.NodeNameToMetaVictims
 		*out = make(map[string]*MetaVictims, len(*in))
 		for key, val := range *in {
+			var outVal *MetaVictims
 			if val == nil {
 				(*out)[key] = nil
 			} else {
-				(*out)[key] = new(MetaVictims)
-				val.DeepCopyInto((*out)[key])
+				in, out := &val, &outVal
+				*out = new(MetaVictims)
+				(*in).DeepCopyInto(*out)
 			}
+			(*out)[key] = outVal
 		}
 	}
 	return
@@ -391,11 +372,10 @@ func (in *MetaVictims) DeepCopyInto(out *MetaVictims) {
 		in, out := &in.Pods, &out.Pods
 		*out = make([]*MetaPod, len(*in))
 		for i := range *in {
-			if (*in)[i] == nil {
-				(*out)[i] = nil
-			} else {
-				(*out)[i] = new(MetaPod)
-				(*in)[i].DeepCopyInto((*out)[i])
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(MetaPod)
+				**out = **in
 			}
 		}
 	}
@@ -463,21 +443,13 @@ func (in *PredicateArgument) DeepCopyInto(out *PredicateArgument) {
 	*out = *in
 	if in.ServiceAffinity != nil {
 		in, out := &in.ServiceAffinity, &out.ServiceAffinity
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(ServiceAffinity)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(ServiceAffinity)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.LabelsPresence != nil {
 		in, out := &in.LabelsPresence, &out.LabelsPresence
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(LabelsPresence)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(LabelsPresence)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -497,12 +469,8 @@ func (in *PredicatePolicy) DeepCopyInto(out *PredicatePolicy) {
 	*out = *in
 	if in.Argument != nil {
 		in, out := &in.Argument, &out.Argument
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(PredicateArgument)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(PredicateArgument)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -522,30 +490,18 @@ func (in *PriorityArgument) DeepCopyInto(out *PriorityArgument) {
 	*out = *in
 	if in.ServiceAntiAffinity != nil {
 		in, out := &in.ServiceAntiAffinity, &out.ServiceAntiAffinity
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(ServiceAntiAffinity)
-			**out = **in
-		}
+		*out = new(ServiceAntiAffinity)
+		**out = **in
 	}
 	if in.LabelPreference != nil {
 		in, out := &in.LabelPreference, &out.LabelPreference
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(LabelPreference)
-			**out = **in
-		}
+		*out = new(LabelPreference)
+		**out = **in
 	}
 	if in.RequestedToCapacityRatioArguments != nil {
 		in, out := &in.RequestedToCapacityRatioArguments, &out.RequestedToCapacityRatioArguments
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(RequestedToCapacityRatioArguments)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(RequestedToCapacityRatioArguments)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -565,12 +521,8 @@ func (in *PriorityPolicy) DeepCopyInto(out *PriorityPolicy) {
 	*out = *in
 	if in.Argument != nil {
 		in, out := &in.Argument, &out.Argument
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(PriorityArgument)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(PriorityArgument)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -666,11 +618,10 @@ func (in *Victims) DeepCopyInto(out *Victims) {
 		in, out := &in.Pods, &out.Pods
 		*out = make([]*v1.Pod, len(*in))
 		for i := range *in {
-			if (*in)[i] == nil {
-				(*out)[i] = nil
-			} else {
-				(*out)[i] = new(v1.Pod)
-				(*in)[i].DeepCopyInto((*out)[i])
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(v1.Pod)
+				(*in).DeepCopyInto(*out)
 			}
 		}
 	}

vendor/k8s.io/kubernetes/pkg/scheduler/cache/BUILD

@@ -6,6 +6,7 @@ go_library(
         "cache.go",
         "interface.go",
         "node_info.go",
+        "node_tree.go",
         "util.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/scheduler/cache",
@@ -15,13 +16,15 @@ go_library(
         "//pkg/features:go_default_library",
         "//pkg/scheduler/algorithm/priorities/util:go_default_library",
         "//pkg/scheduler/util:go_default_library",
+        "//pkg/util/node:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/policy/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/api/policy/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 
@@ -30,22 +33,25 @@ go_test(
     srcs = [
         "cache_test.go",
         "node_info_test.go",
+        "node_tree_test.go",
+        "util_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/features:go_default_library",
+        "//pkg/kubelet/apis:go_default_library",
         "//pkg/scheduler/algorithm/priorities/util:go_default_library",
         "//pkg/scheduler/util:go_default_library",
-        "//pkg/util/parsers:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/api/policy/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/policy/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/scheduler/cache/cache.go

@@ -23,6 +23,7 @@ import (
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/features"
@@ -51,14 +52,17 @@ type schedulerCache struct {
 	period time.Duration
 
 	// This mutex guards all fields within this cache struct.
-	mu sync.Mutex
+	mu sync.RWMutex
 	// a set of assumed pod keys.
 	// The key could further be used to get an entry in podStates.
 	assumedPods map[string]bool
 	// a map from pod key to podState.
 	podStates map[string]*podState
 	nodes     map[string]*NodeInfo
+	nodeTree  *NodeTree
 	pdbs      map[string]*policy.PodDisruptionBudget
+	// A map from image name to its imageState.
+	imageStates map[string]*imageState
 }
 
 type podState struct {
@@ -69,6 +73,29 @@ type podState struct {
 	bindingFinished bool
 }
 
+type imageState struct {
+	// Size of the image
+	size int64
+	// A set of node names for nodes having this image present
+	nodes sets.String
+}
+
+// ImageStateSummary provides summarized information about the state of an image.
+type ImageStateSummary struct {
+	// Size of the image
+	Size int64
+	// Used to track how many nodes have this image
+	NumNodes int
+}
+
+// createImageStateSummary returns a summarizing snapshot of the given image's state.
+func (cache *schedulerCache) createImageStateSummary(state *imageState) *ImageStateSummary {
+	return &ImageStateSummary{
+		Size:     state.size,
+		NumNodes: len(state.nodes),
+	}
+}
+
 func newSchedulerCache(ttl, period time.Duration, stop <-chan struct{}) *schedulerCache {
 	return &schedulerCache{
 		ttl:    ttl,
@@ -76,17 +103,19 @@ func newSchedulerCache(ttl, period time.Duration, stop <-chan struct{}) *schedul
 		stop:   stop,
 
 		nodes:       make(map[string]*NodeInfo),
+		nodeTree:    newNodeTree(nil),
 		assumedPods: make(map[string]bool),
 		podStates:   make(map[string]*podState),
 		pdbs:        make(map[string]*policy.PodDisruptionBudget),
+		imageStates: make(map[string]*imageState),
 	}
 }
 
 // Snapshot takes a snapshot of the current schedulerCache. The method has performance impact,
 // and should be only used in non-critical path.
 func (cache *schedulerCache) Snapshot() *Snapshot {
-	cache.mu.Lock()
-	defer cache.mu.Unlock()
+	cache.mu.RLock()
+	defer cache.mu.RUnlock()
 
 	nodes := make(map[string]*NodeInfo)
 	for k, v := range cache.nodes {
@@ -113,6 +142,7 @@ func (cache *schedulerCache) Snapshot() *Snapshot {
 func (cache *schedulerCache) UpdateNodeNameToInfoMap(nodeNameToInfo map[string]*NodeInfo) error {
 	cache.mu.Lock()
 	defer cache.mu.Unlock()
+
 	for name, info := range cache.nodes {
 		if utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) && info.TransientInfo != nil {
 			// Transient scheduler info is reset here.
@@ -136,8 +166,8 @@ func (cache *schedulerCache) List(selector labels.Selector) ([]*v1.Pod, error) {
 }
 
 func (cache *schedulerCache) FilteredList(podFilter PodFilter, selector labels.Selector) ([]*v1.Pod, error) {
-	cache.mu.Lock()
-	defer cache.mu.Unlock()
+	cache.mu.RLock()
+	defer cache.mu.RUnlock()
 	// podFilter is expected to return true for most or all of the pods. We
 	// can avoid expensive array growth without wasting too much memory by
 	// pre-allocating capacity.
@@ -188,8 +218,8 @@ func (cache *schedulerCache) finishBinding(pod *v1.Pod, now time.Time) error {
 		return err
 	}
 
-	cache.mu.Lock()
-	defer cache.mu.Unlock()
+	cache.mu.RLock()
+	defer cache.mu.RUnlock()
 
 	glog.V(5).Infof("Finished binding for pod %v. Can be expired.", key)
 	currState, ok := cache.podStates[key]
@@ -317,6 +347,7 @@ func (cache *schedulerCache) UpdatePod(oldPod, newPod *v1.Pod) error {
 		if err := cache.updatePod(oldPod, newPod); err != nil {
 			return err
 		}
+		currState.pod = newPod
 	default:
 		return fmt.Errorf("pod %v is not added to scheduler cache, so cannot be updated", key)
 	}
@@ -358,8 +389,8 @@ func (cache *schedulerCache) IsAssumedPod(pod *v1.Pod) (bool, error) {
 		return false, err
 	}
 
-	cache.mu.Lock()
-	defer cache.mu.Unlock()
+	cache.mu.RLock()
+	defer cache.mu.RUnlock()
 
 	b, found := cache.assumedPods[key]
 	if !found {
@@ -374,8 +405,8 @@ func (cache *schedulerCache) GetPod(pod *v1.Pod) (*v1.Pod, error) {
 		return nil, err
 	}
 
-	cache.mu.Lock()
-	defer cache.mu.Unlock()
+	cache.mu.RLock()
+	defer cache.mu.RUnlock()
 
 	podState, ok := cache.podStates[key]
 	if !ok {
@@ -393,7 +424,12 @@ func (cache *schedulerCache) AddNode(node *v1.Node) error {
 	if !ok {
 		n = NewNodeInfo()
 		cache.nodes[node.Name] = n
+	} else {
+		cache.removeNodeImageStates(n.node)
 	}
+
+	cache.nodeTree.AddNode(node)
+	cache.addNodeImageStates(node, n)
 	return n.SetNode(node)
 }
 
@@ -405,7 +441,12 @@ func (cache *schedulerCache) UpdateNode(oldNode, newNode *v1.Node) error {
 	if !ok {
 		n = NewNodeInfo()
 		cache.nodes[newNode.Name] = n
+	} else {
+		cache.removeNodeImageStates(n.node)
 	}
+
+	cache.nodeTree.UpdateNode(oldNode, newNode)
+	cache.addNodeImageStates(newNode, n)
 	return n.SetNode(newNode)
 }
 
@@ -424,9 +465,63 @@ func (cache *schedulerCache) RemoveNode(node *v1.Node) error {
 	if len(n.pods) == 0 && n.node == nil {
 		delete(cache.nodes, node.Name)
 	}
+
+	cache.nodeTree.RemoveNode(node)
+	cache.removeNodeImageStates(node)
 	return nil
 }
 
+// addNodeImageStates adds states of the images on given node to the given nodeInfo and update the imageStates in
+// scheduler cache. This function assumes the lock to scheduler cache has been acquired.
+func (cache *schedulerCache) addNodeImageStates(node *v1.Node, nodeInfo *NodeInfo) {
+	newSum := make(map[string]*ImageStateSummary)
+
+	for _, image := range node.Status.Images {
+		for _, name := range image.Names {
+			// update the entry in imageStates
+			state, ok := cache.imageStates[name]
+			if !ok {
+				state = &imageState{
+					size:  image.SizeBytes,
+					nodes: sets.NewString(node.Name),
+				}
+				cache.imageStates[name] = state
+			} else {
+				state.nodes.Insert(node.Name)
+			}
+			// create the imageStateSummary for this image
+			if _, ok := newSum[name]; !ok {
+				newSum[name] = cache.createImageStateSummary(state)
+			}
+		}
+	}
+	nodeInfo.imageStates = newSum
+}
+
+// removeNodeImageStates removes the given node record from image entries having the node
+// in imageStates cache. After the removal, if any image becomes free, i.e., the image
+// is no longer available on any node, the image entry will be removed from imageStates.
+func (cache *schedulerCache) removeNodeImageStates(node *v1.Node) {
+	if node == nil {
+		return
+	}
+
+	for _, image := range node.Status.Images {
+		for _, name := range image.Names {
+			state, ok := cache.imageStates[name]
+			if ok {
+				state.nodes.Delete(node.Name)
+				if len(state.nodes) == 0 {
+					// Remove the unused image to make sure the length of
+					// imageStates represents the total number of different
+					// images on all nodes
+					delete(cache.imageStates, name)
+				}
+			}
+		}
+	}
+}
+
 func (cache *schedulerCache) AddPDB(pdb *policy.PodDisruptionBudget) error {
 	cache.mu.Lock()
 	defer cache.mu.Unlock()
@@ -449,8 +544,8 @@ func (cache *schedulerCache) RemovePDB(pdb *policy.PodDisruptionBudget) error {
 }
 
 func (cache *schedulerCache) ListPDBs(selector labels.Selector) ([]*policy.PodDisruptionBudget, error) {
-	cache.mu.Lock()
-	defer cache.mu.Unlock()
+	cache.mu.RLock()
+	defer cache.mu.RUnlock()
 	var pdbs []*policy.PodDisruptionBudget
 	for _, pdb := range cache.pdbs {
 		if selector.Matches(labels.Set(pdb.Labels)) {
@@ -461,8 +556,8 @@ func (cache *schedulerCache) ListPDBs(selector labels.Selector) ([]*policy.PodDi
 }
 
 func (cache *schedulerCache) IsUpToDate(n *NodeInfo) bool {
-	cache.mu.Lock()
-	defer cache.mu.Unlock()
+	cache.mu.RLock()
+	defer cache.mu.RUnlock()
 	node, ok := cache.nodes[n.Node().Name]
 	return ok && n.generation == node.generation
 }
@@ -508,3 +603,7 @@ func (cache *schedulerCache) expirePod(key string, ps *podState) error {
 	delete(cache.podStates, key)
 	return nil
 }
+
+func (cache *schedulerCache) NodeTree() *NodeTree {
+	return cache.nodeTree
+}

vendor/k8s.io/kubernetes/pkg/scheduler/cache/interface.go

@@ -125,6 +125,9 @@ type Cache interface {
 
 	// IsUpToDate returns true if the given NodeInfo matches the current data in the cache.
 	IsUpToDate(n *NodeInfo) bool
+
+	// NodeTree returns a node tree structure
+	NodeTree() *NodeTree
 }
 
 // Snapshot is a snapshot of cache state

vendor/k8s.io/kubernetes/pkg/scheduler/cache/node_info.go

@@ -58,9 +58,10 @@ type NodeInfo struct {
 	taints    []v1.Taint
 	taintsErr error
 
-	// This is a map from image name to image size, also for checking image existence on the node
-	// Cache it here to avoid rebuilding the map during scheduling, e.g., in image_locality.go
-	imageSizes map[string]int64
+	// imageStates holds the entry of an image if and only if this image is on the node. The entry can be used for
+	// checking an image's existence and advanced usage (e.g., image locality scheduling policy) based on the image
+	// state information.
+	imageStates map[string]*ImageStateSummary
 
 	// TransientInfo holds the information pertaining to a scheduling cycle. This will be destructed at the end of
 	// scheduling cycle.
@@ -261,7 +262,7 @@ func NewNodeInfo(pods ...*v1.Pod) *NodeInfo {
 		TransientInfo:       newTransientSchedulerInfo(),
 		generation:          nextGeneration(),
 		usedPorts:           make(util.HostPortInfo),
-		imageSizes:          make(map[string]int64),
+		imageStates:         make(map[string]*ImageStateSummary),
 	}
 	for _, pod := range pods {
 		ni.AddPod(pod)
@@ -293,12 +294,12 @@ func (n *NodeInfo) UsedPorts() util.HostPortInfo {
 	return n.usedPorts
 }
 
-// ImageSizes returns the image size information on this node.
-func (n *NodeInfo) ImageSizes() map[string]int64 {
+// ImageStates returns the state information of all images.
+func (n *NodeInfo) ImageStates() map[string]*ImageStateSummary {
 	if n == nil {
 		return nil
 	}
-	return n.imageSizes
+	return n.imageStates
 }
 
 // PodsWithAffinity return all pods with (anti)affinity constraints on this node.
@@ -392,15 +393,20 @@ func (n *NodeInfo) Clone() *NodeInfo {
 		diskPressureCondition:   n.diskPressureCondition,
 		pidPressureCondition:    n.pidPressureCondition,
 		usedPorts:               make(util.HostPortInfo),
-		imageSizes:              n.imageSizes,
+		imageStates:             n.imageStates,
 		generation:              n.generation,
 	}
 	if len(n.pods) > 0 {
 		clone.pods = append([]*v1.Pod(nil), n.pods...)
 	}
 	if len(n.usedPorts) > 0 {
-		for k, v := range n.usedPorts {
-			clone.usedPorts[k] = v
+		// util.HostPortInfo is a map-in-map struct
+		// make sure it's deep copied
+		for ip, portMap := range n.usedPorts {
+			clone.usedPorts[ip] = make(map[util.ProtocolPort]struct{})
+			for protocolPort, v := range portMap {
+				clone.usedPorts[ip][protocolPort] = v
+			}
 		}
 	}
 	if len(n.podsWithAffinity) > 0 {
@@ -547,17 +553,6 @@ func (n *NodeInfo) updateUsedPorts(pod *v1.Pod, add bool) {
 	}
 }
 
-func (n *NodeInfo) updateImageSizes() {
-	node := n.Node()
-	imageSizes := make(map[string]int64)
-	for _, image := range node.Status.Images {
-		for _, name := range image.Names {
-			imageSizes[name] = image.SizeBytes
-		}
-	}
-	n.imageSizes = imageSizes
-}
-
 // SetNode sets the overall node information.
 func (n *NodeInfo) SetNode(node *v1.Node) error {
 	n.node = node
@@ -579,7 +574,6 @@ func (n *NodeInfo) SetNode(node *v1.Node) error {
 		}
 	}
 	n.TransientInfo = newTransientSchedulerInfo()
-	n.updateImageSizes()
 	n.generation = nextGeneration()
 	return nil
 }
@@ -596,6 +590,7 @@ func (n *NodeInfo) RemoveNode(node *v1.Node) error {
 	n.memoryPressureCondition = v1.ConditionUnknown
 	n.diskPressureCondition = v1.ConditionUnknown
 	n.pidPressureCondition = v1.ConditionUnknown
+	n.imageStates = make(map[string]*ImageStateSummary)
 	n.generation = nextGeneration()
 	return nil
 }

vendor/k8s.io/kubernetes/pkg/scheduler/cache/node_tree.go

@@ -0,0 +1,187 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"fmt"
+	"sync"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilnode "k8s.io/kubernetes/pkg/util/node"
+
+	"github.com/golang/glog"
+)
+
+// NodeTree is a tree-like data structure that holds node names in each zone. Zone names are
+// keys to "NodeTree.tree" and values of "NodeTree.tree" are arrays of node names.
+type NodeTree struct {
+	tree           map[string]*nodeArray // a map from zone (region-zone) to an array of nodes in the zone.
+	zones          []string              // a list of all the zones in the tree (keys)
+	zoneIndex      int
+	exhaustedZones sets.String // set of zones that all of their nodes are returned by next()
+	NumNodes       int
+	mu             sync.RWMutex
+}
+
+// nodeArray is a struct that has nodes that are in a zone.
+// We use a slice (as opposed to a set/map) to store the nodes because iterating over the nodes is
+// a lot more frequent than searching them by name.
+type nodeArray struct {
+	nodes     []string
+	lastIndex int
+}
+
+func (na *nodeArray) next() (nodeName string, exhausted bool) {
+	if len(na.nodes) == 0 {
+		glog.Error("The nodeArray is empty. It should have been deleted from NodeTree.")
+		return "", false
+	}
+	if na.lastIndex >= len(na.nodes) {
+		return "", true
+	}
+	nodeName = na.nodes[na.lastIndex]
+	na.lastIndex++
+	return nodeName, false
+}
+
+func newNodeTree(nodes []*v1.Node) *NodeTree {
+	nt := &NodeTree{
+		tree:           make(map[string]*nodeArray),
+		exhaustedZones: sets.NewString(),
+	}
+	for _, n := range nodes {
+		nt.AddNode(n)
+	}
+	return nt
+}
+
+// AddNode adds a node and its corresponding zone to the tree. If the zone already exists, the node
+// is added to the array of nodes in that zone.
+func (nt *NodeTree) AddNode(n *v1.Node) {
+	nt.mu.Lock()
+	defer nt.mu.Unlock()
+	nt.addNode(n)
+}
+
+func (nt *NodeTree) addNode(n *v1.Node) {
+	zone := utilnode.GetZoneKey(n)
+	if na, ok := nt.tree[zone]; ok {
+		for _, nodeName := range na.nodes {
+			if nodeName == n.Name {
+				glog.Warningf("node %v already exist in the NodeTree", n.Name)
+				return
+			}
+		}
+		na.nodes = append(na.nodes, n.Name)
+	} else {
+		nt.zones = append(nt.zones, zone)
+		nt.tree[zone] = &nodeArray{nodes: []string{n.Name}, lastIndex: 0}
+	}
+	glog.V(5).Infof("Added node %v in group %v to NodeTree", n.Name, zone)
+	nt.NumNodes++
+}
+
+// RemoveNode removes a node from the NodeTree.
+func (nt *NodeTree) RemoveNode(n *v1.Node) error {
+	nt.mu.Lock()
+	defer nt.mu.Unlock()
+	return nt.removeNode(n)
+}
+
+func (nt *NodeTree) removeNode(n *v1.Node) error {
+	zone := utilnode.GetZoneKey(n)
+	if na, ok := nt.tree[zone]; ok {
+		for i, nodeName := range na.nodes {
+			if nodeName == n.Name {
+				na.nodes = append(na.nodes[:i], na.nodes[i+1:]...)
+				if len(na.nodes) == 0 {
+					nt.removeZone(zone)
+				}
+				glog.V(5).Infof("Removed node %v in group %v from NodeTree", n.Name, zone)
+				nt.NumNodes--
+				return nil
+			}
+		}
+	}
+	glog.Errorf("Node %v in group %v was not found", n.Name, zone)
+	return fmt.Errorf("node %v in group %v was not found", n.Name, zone)
+}
+
+// removeZone removes a zone from tree.
+// This function must be called while writer locks are hold.
+func (nt *NodeTree) removeZone(zone string) {
+	delete(nt.tree, zone)
+	for i, z := range nt.zones {
+		if z == zone {
+			nt.zones = append(nt.zones[:i], nt.zones[i+1:]...)
+		}
+	}
+}
+
+// UpdateNode updates a node in the NodeTree.
+func (nt *NodeTree) UpdateNode(old, new *v1.Node) {
+	var oldZone string
+	if old != nil {
+		oldZone = utilnode.GetZoneKey(old)
+	}
+	newZone := utilnode.GetZoneKey(new)
+	// If the zone ID of the node has not changed, we don't need to do anything. Name of the node
+	// cannot be changed in an update.
+	if oldZone == newZone {
+		return
+	}
+	nt.mu.Lock()
+	defer nt.mu.Unlock()
+	nt.removeNode(old) // No error checking. We ignore whether the old node exists or not.
+	nt.addNode(new)
+}
+
+func (nt *NodeTree) resetExhausted() {
+	for _, na := range nt.tree {
+		na.lastIndex = 0
+	}
+	nt.exhaustedZones = sets.NewString()
+}
+
+// Next returns the name of the next node. NodeTree iterates over zones and in each zone iterates
+// over nodes in a round robin fashion.
+func (nt *NodeTree) Next() string {
+	nt.mu.Lock()
+	defer nt.mu.Unlock()
+	if len(nt.zones) == 0 {
+		return ""
+	}
+	for {
+		if nt.zoneIndex >= len(nt.zones) {
+			nt.zoneIndex = 0
+		}
+		zone := nt.zones[nt.zoneIndex]
+		nt.zoneIndex++
+		// We do not check the set of exhausted zones before calling next() on the zone. This ensures
+		// that if more nodes are added to a zone after it is exhausted, we iterate over the new nodes.
+		nodeName, exhausted := nt.tree[zone].next()
+		if exhausted {
+			nt.exhaustedZones.Insert(zone)
+			if len(nt.exhaustedZones) == len(nt.zones) { // all zones are exhausted. we should reset.
+				nt.resetExhausted()
+			}
+		} else {
+			return nodeName
+		}
+	}
+}