DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update go dependencies

Browse source
This commit is contained in:
Manuel Alejandro de Brito Fontes 2019-03-28 20:43:46 -03:00
parent 14a9e9f3fa
commit 14f4a7b8e8
No known key found for this signature in database
GPG key ID: 786136016A8BA02A
1349 changed files with 128369 additions and 32627 deletions
Download patch file Download diff file Expand all files Collapse all files

3
vendor/k8s.io/kubernetes/.generated_files generated vendored
View file

@ -24,7 +24,4 @@ file-name types_swagger_doc_generated.go
path-prefix Godeps/
path-prefix vendor/
path-prefix api/swagger-spec/
path-prefix pkg/generated/
paths-from-repo docs/.generated_docs

2
vendor/k8s.io/kubernetes/.gitattributes generated vendored
View file

@ -6,6 +6,4 @@ test/test_owners.csv merge=union
**/generated.pb.go linguist-generated=true
**/generated.proto
**/types_swagger_doc_generated.go linguist-generated=true
docs/api-reference/** linguist-generated=true
api/swagger-spec/*.json linguist-generated=true
api/openapi-spec/*.json linguist-generated=true

2
vendor/k8s.io/kubernetes/.gitignore generated vendored
View file

@ -111,6 +111,8 @@ kubernetes.tar.gz
# TODO(thockin): uncomment this when we stop committing the generated files.
#zz_generated.*
zz_generated.openapi.go
# TODO(roycaihw): remove this when we stop committing the generated definition
!staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/zz_generated.openapi.go
zz_generated_*_test.go
# make-related metadata

1479
vendor/k8s.io/kubernetes/CHANGELOG-1.13.md generated vendored
View file

File diff suppressed because it is too large Load diff

956
vendor/k8s.io/kubernetes/CHANGELOG-1.14.md generated vendored Normal file
View file

@ -0,0 +1,956 @@
<!-- BEGIN MUNGE: GENERATED_TOC -->
- [v1.14.0-rc.1](#v1140-rc1)
- [Downloads for v1.14.0-rc.1](#downloads-for-v1140-rc1)
- [Client Binaries](#client-binaries)
- [Server Binaries](#server-binaries)
- [Node Binaries](#node-binaries)
- [Changelog since v1.14.0-beta.2](#changelog-since-v1140-beta2)
- [Action Required](#action-required)
- [Other notable changes](#other-notable-changes)
- [v1.14.0-beta.2](#v1140-beta2)
- [Downloads for v1.14.0-beta.2](#downloads-for-v1140-beta2)
- [Client Binaries](#client-binaries-1)
- [Server Binaries](#server-binaries-1)
- [Node Binaries](#node-binaries-1)
- [Changelog since v1.14.0-beta.1](#changelog-since-v1140-beta1)
- [Action Required](#action-required-1)
- [Other notable changes](#other-notable-changes-1)
- [v1.14.0-beta.1](#v1140-beta1)
- [Downloads for v1.14.0-beta.1](#downloads-for-v1140-beta1)
- [Client Binaries](#client-binaries-2)
- [Server Binaries](#server-binaries-2)
- [Node Binaries](#node-binaries-2)
- [Changelog since v1.14.0-alpha.3](#changelog-since-v1140-alpha3)
- [Action Required](#action-required-2)
- [Other notable changes](#other-notable-changes-2)
- [v1.14.0-alpha.3](#v1140-alpha3)
- [Downloads for v1.14.0-alpha.3](#downloads-for-v1140-alpha3)
- [Client Binaries](#client-binaries-3)
- [Server Binaries](#server-binaries-3)
- [Node Binaries](#node-binaries-3)
- [Changelog since v1.14.0-alpha.2](#changelog-since-v1140-alpha2)
- [Action Required](#action-required-3)
- [Other notable changes](#other-notable-changes-3)
- [v1.14.0-alpha.2](#v1140-alpha2)
- [Downloads for v1.14.0-alpha.2](#downloads-for-v1140-alpha2)
- [Client Binaries](#client-binaries-4)
- [Server Binaries](#server-binaries-4)
- [Node Binaries](#node-binaries-4)
- [Changelog since v1.14.0-alpha.1](#changelog-since-v1140-alpha1)
- [Action Required](#action-required-4)
- [Other notable changes](#other-notable-changes-4)
- [v1.14.0-alpha.1](#v1140-alpha1)
- [Downloads for v1.14.0-alpha.1](#downloads-for-v1140-alpha1)
- [Client Binaries](#client-binaries-5)
- [Server Binaries](#server-binaries-5)
- [Node Binaries](#node-binaries-5)
- [Changelog since v1.13.0](#changelog-since-v1130)
- [Action Required](#action-required-5)
- [Other notable changes](#other-notable-changes-5)
<!-- END MUNGE: GENERATED_TOC -->
<!-- NEW RELEASE NOTES ENTRY -->
# v1.14.0-rc.1
[Documentation](https://docs.k8s.io)
## Downloads for v1.14.0-rc.1
filename | sha512 hash
-------- | -----------
[kubernetes.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes.tar.gz) | `5cb5e8b14b301864063fd7531ab3b755fea054f540c55ecce70ac49fb59193488575eb42ba89c8b4a44f6f2d005602ffc50ac286354a16df27637dd2e05f90f0`
[kubernetes-src.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-src.tar.gz) | `395424c2bcdb5e242995b18e8d6e5c00002ce2cb5a3964c28da0a4a181fada73ffceaccedb1fa9799be9b3c4fb5b451010cba65af4d7385c25c8c8f0298218fc`
### Client Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-darwin-386.tar.gz) | `ea304f3f8188db30cddd5423b25dc434b8f05315103f773619a65f83bee872581d83d5498a5f36a3064815e68998746cf661802eab36bfe96217253bac7e751c`
[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-darwin-amd64.tar.gz) | `17e106b63067429b9228a4879a7350c01ae98650ef2e6fcc23d00415c2e3a7c340abd5bcfc4b976f3d737d9268159ef5e5e7b08757371daad637e721a2ffb4a9`
[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-linux-386.tar.gz) | `406323ea4cbd524807e73b9a2f4eb0a813730b262402c224e5076080b84452137521e5782056e39bf6017bda8ef9e797ed497d51a653ed6822357f43d86bb0ae`
[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-linux-amd64.tar.gz) | `293747816e5da30c53ca29f27479fae880404edd5fca413af165e52cc7ea7ff26312bb3916896eceea75af6b232647268324da76d2d30fa2a4a688557427f7c7`
[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-linux-arm.tar.gz) | `98b6749c367282048ecd0c5b70ae8b7dafad82c599e359cbbe782a530bbd7bdb84a198577251d6aedbc39fb4e0297bd929d7e988eb557556f523073227375b09`
[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-linux-arm64.tar.gz) | `0a650c53946ebd9c38705df36efabbf1fbe3da1acdf418cc4ab881530e6a9089b45ef36bd4a89467106561e2165a00cc2c47f791d9ba422f36544bfa4b1e3b68`
[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-linux-ppc64le.tar.gz) | `807263f316f46c9334ec4dad84895cfe2b942ac4ed9aacf3ec8a63193e0d0a6ecaf573d00dfea278d1e552fe91e9474f534ea6798700dbdc84f1d9556ecfda3e`
[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-linux-s390x.tar.gz) | `5589562ce0da49f4987388e3e2b6fcf29e92859fae65fb57cdd61bba20ff574c7f3e07f4fa26bb63789f927690ce19710803e0d2e3324bd2c80ddc4925ea973d`
[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-windows-386.tar.gz) | `101245cd70221b443150be046e5b5a2c6c83334085bb17f603f59bb68c5960f353d57b6761acf052449f4fb057a5525978cdd7995d06134ab98f8628c23aac0d`
[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-client-windows-amd64.tar.gz) | `b1c3802f60cc70ebc1258cb8fc4ffa1154ecee8fda473b033be4f9d1b187354cfd75d085a1ad45a35fcc42ad640c4fecaa2496cb48fad649a774f40a5150825f`
### Server Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-server-linux-amd64.tar.gz) | `8e09465aab0a1d1ec39afc98af17de9b5de99b763c0c5feb2dc824f2bbae25edb690e9d162fd44f5155bea392bd229f544850fe19e767a8b342050f4bfafc2cc`
[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-server-linux-arm.tar.gz) | `9409c368e1f9f26e633b7df5d6c90435394931d48a21f4ad1548d172b18ac462a859019cc66dba4df69d3b10702820c9a3e1bd5c469646b1db581e52e79e035d`
[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-server-linux-arm64.tar.gz) | `3893290dc11ca71746fee77a44a607ad9e02036bab56b7fc3be247b71b2cf5b3f639fa41317a713abbe9a997abf80c7113ff1155482d0cc04a318ec8beccf869`
[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-server-linux-ppc64le.tar.gz) | `8ac1e70cb39aeb5b1fe92c3ebba428db2036be739f462cc2f876f17dc71a01ba263b5611a15d95e5934e2fc7aa92042bc9b2fe65cf459263be90f5fbb5d83a15`
[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-server-linux-s390x.tar.gz) | `80576cb6cc3a69c4fc0a0358dee5772ecb38437c534a3454c9613426417d4af3c527a0809cce4d46653a7b001c58033b06326c80c498d17387569d22d3ca9b22`
### Node Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-node-linux-amd64.tar.gz) | `71cd5dd6075a2eea851231a5a855e58b3f479d83358defafd068dd1d09e5b2c426a8a046ee621de91e17d7ecd67465911b93549088bf27a41c6e6b77d692a8c7`
[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-node-linux-arm.tar.gz) | `48babf4a52013c2bc69049167579ba1bc70c769b782a2704c9dfcf44a6a8a72f07e0789af347135ee4797f2bf1a216c348a9a4a26be71855e95e8387bf4e2aac`
[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-node-linux-arm64.tar.gz) | `cc19a52beaa7440ca7581e85d1e10137e93c2decdb7d7d7919e7fdcded63e4d94b3434513ca881dae844dd1eb1e2fe98ea5332fad5ffced846f729894ecf0ed1`
[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-node-linux-ppc64le.tar.gz) | `241bf20ae366384efa0fb3798e07e1cdd3d4ea7ba91c146ef7761fb0b93a8514a0dd91f9eb47999ae263d6793e0577c2bbecf88548bcae06cedb437331d6d3bc`
[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-node-linux-s390x.tar.gz) | `37ce3e021073c6c10cf240fb2c3f9a7ab35ece3c0b4a9fecbbdf790eb348b168d179824f3a8eb57d56f962b64f8a6a71925152c087f5bfe43b004cbfae65674d`
[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-rc.1/kubernetes-node-windows-amd64.tar.gz) | `3248db12c274171f094ef41d6a59523aec35655ba35d151576ff2d2089c269f345e2c0646d585d7c04c440c6b4e7379c499bc8f51fcc8b91388e3dc4d1b6a899`
## Changelog since v1.14.0-beta.2
### Action Required
* ACTION REQUIRED: Health check (liveness & readiness) probes using an HTTPGetAction will no longer follow redirects to different hostnames from the original probe request. Instead, these non-local redirects will be treated as a Success (the documented behavior). In this case an event with reason "ProbeWarning" will be generated, indicating that the redirect was ignored. If you were previously relying on the redirect to run health checks against different endpoints, you will need to perform the healthcheck logic outside the Kubelet, for instance by proxying the external endpoint rather than redirecting to it. ([#75416](https://github.com/kubernetes/kubernetes/pull/75416), [@tallclair](https://github.com/tallclair))
### Other notable changes
* Restores --username and --password flags to kubectl ([#75451](https://github.com/kubernetes/kubernetes/pull/75451), [@liggitt](https://github.com/liggitt))
* fix race condition issue for smb mount on windows ([#75371](https://github.com/kubernetes/kubernetes/pull/75371), [@andyzhangx](https://github.com/andyzhangx))
* UDP Service conntrack entries for ExternalIPs are now correctly cleared when endpoints are added ([#75265](https://github.com/kubernetes/kubernetes/pull/75265), [@JacobTanenbaum](https://github.com/JacobTanenbaum))
* kubeadm: the kubeadm init output now provides join control-plane example only when the preconditions for joining a control plane are satisfied ([#75420](https://github.com/kubernetes/kubernetes/pull/75420), [@fabriziopandini](https://github.com/fabriziopandini))
* Fix dockershim panic issues when deleting docker images. ([#75367](https://github.com/kubernetes/kubernetes/pull/75367), [@feiskyer](https://github.com/feiskyer))
* kubeadm: Allow certain certs/keys to be missing on the secret when transferring secrets using `--experimental-upload-certs` feature ([#75415](https://github.com/kubernetes/kubernetes/pull/75415), [@ereslibre](https://github.com/ereslibre))
* Update to use go1.12.1 ([#75413](https://github.com/kubernetes/kubernetes/pull/75413), [@BenTheElder](https://github.com/BenTheElder))
* Update CSI version to 1.1 ([#75391](https://github.com/kubernetes/kubernetes/pull/75391), [@gnufied](https://github.com/gnufied))
* Ensure ownership when deleting a load balancer security group ([#74311](https://github.com/kubernetes/kubernetes/pull/74311), [@hpedrorodrigues](https://github.com/hpedrorodrigues))
* kubelet: updated logic of verifying a static critical pod. ([#75144](https://github.com/kubernetes/kubernetes/pull/75144), [@Huang-Wei](https://github.com/Huang-Wei))
* Allow disable outbound SNAT when Azure standard load balancer is used together with outbound rules. ([#75282](https://github.com/kubernetes/kubernetes/pull/75282), [@feiskyer](https://github.com/feiskyer))
* Add ResourceVersion as a precondition for delete in order to ensure a delete fails if an unobserved change happens to an object. ([#74040](https://github.com/kubernetes/kubernetes/pull/74040), [@ajatprabha](https://github.com/ajatprabha))
* Services of type=LoadBalancer which have no endpoints will now immediately ICMP reject connections, rather than time out. ([#74394](https://github.com/kubernetes/kubernetes/pull/74394), [@thockin](https://github.com/thockin))
* Ensure Azure load balancer cleaned up on 404 or 403 when deleting LoadBalancer services. ([#75256](https://github.com/kubernetes/kubernetes/pull/75256), [@feiskyer](https://github.com/feiskyer))
# v1.14.0-beta.2
[Documentation](https://docs.k8s.io)
## Downloads for v1.14.0-beta.2
filename | sha512 hash
-------- | -----------
[kubernetes.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes.tar.gz) | `c1d5f2615c3319fc167c577f40f385abe6652bf4fd3bdb04617b36029dc3000b190c18b4b3a29827da75c680979697d61fffb45b86ba6226f880b98b2f308f4f`
[kubernetes-src.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-src.tar.gz) | `0a8d8ed208bc0bf424060126c76fcd8dbbd53a9b9695647314a4097f7013f548b76850438933760ff76835867676cddddf65134ad79f977ecdb98632fc2edda3`
### Client Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-darwin-386.tar.gz) | `c919d030255c5d3879926d8aaa53939cd5aa37084799748452166ca6668bd1d10edf063d633682cddafaaed43dd1b991f4ad09139c5e4f519bd69f581b3fe0aa`
[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-darwin-amd64.tar.gz) | `ec14d4a1d720890065211544b099be17315265534cfd20435194dc842cc807c20b5fae78f5b95ba7d05f3d921d522017f50861760d195ce1bf5b1acfdb2dbb29`
[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-386.tar.gz) | `6cee12be5b855600ee80f15d1e0511099941b099bd5b252549abdc2a65c077f10ca4d53ab9804a0ce8d51f3b9cbab829cb551733cd4aed37c0d91238b82a8fe4`
[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-amd64.tar.gz) | `27d8cd48c1f3259055965b85a6b973ecb5b8a36894f94c232d773f89539e28e6c270bfe35427c70b4ad4800e42c869851981cf88f586b7d488efa538e6c88126`
[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-arm.tar.gz) | `7f98230569c61fdf2b141e519f042b2e27ff37aeda746dc30bb7ce226b5d6b0c0bb85c6070b9ffc8d38c2441feb5bd8736c67708a59552e86a2c30cc02ecdece`
[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-arm64.tar.gz) | `159da67010af38d87c5318b7ad594120afd6a9b780d11d6e607e7214862cd6514b00da673cce72574771dbc780ab435dbba0a3267f051a20155c05ee0729ded0`
[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-ppc64le.tar.gz) | `d8ca7871d3d40947db69061284cb31c4d072d4da56fbb11a4485f6853f041835d9605cfc5dcea88d58c7f484cce13dbca485e80891c845291b9b28c574df310b`
[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-linux-s390x.tar.gz) | `1c58db90b6e09b8d8f956a00263cb20271b8403f7fb6c5b20d76cca9ed973c35d2f5c910a6d42980ec9159480682d3786a59e9e05ce356a7e3b4181c848ad122`
[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-windows-386.tar.gz) | `00fb87dd4899208dd6607c22828f3985ebfd5e1f97cb24e3b2c69c249a4887d5c26c603b3bb4c21f9e2b737c917ddf95a1818d9de5c9ec97d3f5faa0c3dada52`
[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-client-windows-amd64.tar.gz) | `7afdf637d62dde480162ad1521360b2bc78e0d4d20f6e6201e2f19b55b8e9bbd69c1ce8d03101c750ef389c65a1bc0a94dfc9a2d501d6840fd31eacbd3582028`
### Server Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-amd64.tar.gz) | `2ac3c4910cd36f02a62304d78fe144b821edf445c522028e6b57d2dc3bcc7355159a58815d5a6991b3f2c33bb0ef23e07134c8bbf93b34be7452f80c9a9e6edf`
[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-arm.tar.gz) | `db06b5f1a83ca4ec82428ab771eae2858b188dc23780fb9b146494c06aa6175421090b200c58b670e2d4253a7e0d4b07172a632e0754c35ccdee7e264c636f17`
[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-arm64.tar.gz) | `b40e1745d1ecfcc95f3a750990244fa128381d6d74246798a62aecb8cec9c77cdcd470e79334eb5c670e1e3a288080b4e26a080c64481ba608e3156c72df474b`
[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-ppc64le.tar.gz) | `c84297fd2b18b6bdade5a135a3da929e286bbba5c8dd66778091bad4eea1ac4b97a32ab3b146a88f0716bebcce9a4a85a7cd421cb185a3df864dcfa77312b3a9`
[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-server-linux-s390x.tar.gz) | `39c8f6e7f52bec155b11652b4e80c2c52acf8754dbdf80a9d5bab5370d1debf4f4783c1a6968d41822b00ff744c72947df6cbc4623578e7679b9ce9a98f64ff3`
### Node Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-amd64.tar.gz) | `bf19ae7140836aea1b6f414532eff886e3b91e0746b9224ce46e60e0b83fa90a8c3df1ff8e01ff340a1e1874ce15da28e98224024ed3139589474e89befa19f2`
[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-arm.tar.gz) | `4b8194340a8675107da3969845173fb34ed2b0a38745ec0ac395ebf2116ee84d55be6e22ff84fafbaf4ca60a05f6debf6e95957a2261ac8a587eea32e5803fd5`
[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-arm64.tar.gz) | `8494dae5225f3b543afd575003fe0f30eb0f3cf9bc9dfbae72d6bda8f17c5446165433c28842a114af66ac3ae8fca9f92d780d1eb93e9bcb6b5dcc4fe8cd2a7f`
[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-ppc64le.tar.gz) | `0bd41d95f0a76c1b057a8913a8b2cffbf6d48c47aef1d9beed0de205b8010e8071e8f527eeaa003730ef97a017083278cb2036cf22a1abfb2f4669b935823cdc`
[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-linux-s390x.tar.gz) | `169b16512df42a6cb5b000a3d6d6da5ae48a733c5d11b034eaec6b3816b86ec97b92e4075872900188bc296427037299841224e552ccf079097d5cf333627cbc`
[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.2/kubernetes-node-windows-amd64.tar.gz) | `ccfebde5176cb28529552889250bf706add04df3c3f9aee5b8bdd9ebdb5bce334add8351cdbeebb9bd4b17b31d994b7739d2c494aa4c64bcb3ceba1a6ef53447`
## Changelog since v1.14.0-beta.1
### Action Required
* ACTION REQUIRED: The node.k8s.io API group and runtimeclasses.node.k8s.io resource have been migrated to a built-in API. If you were using RuntimeClasses (an default-disabled alpha feature, as of Kubernetes v1.12), then you must recreate all RuntimeClasses after upgrading, and the runtimeclasses.node.k8s.io CRD should be manually deleted. RuntimeClasses can no longer be created without a defined handler. ([#74433](https://github.com/kubernetes/kubernetes/pull/74433), [@tallclair](https://github.com/tallclair))
* Introduce a RuntimeClass v1beta1 API. This new beta API renames `runtimeHandler` to `handler`, makes it a required field, and cuts out the spec (handler is a top-level field).
* Transition CSINodeInfo and CSIDriver alpha CRDs to in-tree CSINode and CSIDriver core storage v1beta1 APIs. ([#74283](https://github.com/kubernetes/kubernetes/pull/74283), [@xing-yang](https://github.com/xing-yang))
* ACTION REQUIRED: the alpha CRDs are no longer used and drivers will need to be updated to use the beta APIs.
* The support for `_` in the CSI driver name will be dropped as the CSI Spec does not allow that.
### Other notable changes
* Support collecting pod logs under /var/log/pods/NAMESPACE_NAME_UID to stackdriver with `k8s_pod` resource type. ([#74502](https://github.com/kubernetes/kubernetes/pull/74502), [@Random-Liu](https://github.com/Random-Liu))
* --make-symlinks for hyperkube was marked hidden for a while, This flag is now deprecated and will be removed in a future release. ([#74975](https://github.com/kubernetes/kubernetes/pull/74975), [@dims](https://github.com/dims))
* fix smb unmount issue on Windows ([#75087](https://github.com/kubernetes/kubernetes/pull/75087), [@andyzhangx](https://github.com/andyzhangx))
* Kubelet no longer watches configmaps and secrets for terminated pods, in worst scenario causing it to not be able to send other requests to kube-apiserver ([#74809](https://github.com/kubernetes/kubernetes/pull/74809), [@oxddr](https://github.com/oxddr))
* - Fixes a bug concerning Quobyte volumes where user mappings only worked if the hosts Kubernetes plugin mount was provided via an external configuration using the _allow-usermapping-in-volumename_ option. ([#74520](https://github.com/kubernetes/kubernetes/pull/74520), [@casusbelli](https://github.com/casusbelli))
* Change CRI pod log directory from `/var/log/pods/UID` to `/var/log/pods/NAMESPACE_NAME_UID`. ([#74441](https://github.com/kubernetes/kubernetes/pull/74441), [@Random-Liu](https://github.com/Random-Liu))
* It is recommended to drain the node before upgrade, or reboot the node after upgrade.
* Promote RuntimeClass to beta, and enable by default. ([#75003](https://github.com/kubernetes/kubernetes/pull/75003), [@tallclair](https://github.com/tallclair))
* New "dry_run" metric label (indicating the value of the dryRun query parameter) into the metrics: ([#74997](https://github.com/kubernetes/kubernetes/pull/74997), [@jennybuckley](https://github.com/jennybuckley))
* apiserver_request_total
* apiserver_request_duration_seconds
* New "APPLY" value for the "verb" metric label which indicates a PATCH with "Content-Type: apply-patch+yaml". This value is experimental and will only be present if the ServerSideApply alpha feature is enabled.
* GCE: bump COS image version to cos-beta-73-11647-64-0 ([#75149](https://github.com/kubernetes/kubernetes/pull/75149), [@yguo0905](https://github.com/yguo0905))
* - Add duration metric for CRD webhook converters ([#74376](https://github.com/kubernetes/kubernetes/pull/74376), [@mbohlool](https://github.com/mbohlool))
* Alpha support for ephemeral CSI inline volumes that are embedded in pod specs. ([#74086](https://github.com/kubernetes/kubernetes/pull/74086), [@vladimirvivien](https://github.com/vladimirvivien))
* Add support for node side CSI volume expansion ([#74863](https://github.com/kubernetes/kubernetes/pull/74863), [@gnufied](https://github.com/gnufied))
* - Add mechanism for Admission Webhooks to specify which version of AdmissionReview they support ([#74998](https://github.com/kubernetes/kubernetes/pull/74998), [@mbohlool](https://github.com/mbohlool))
* - Add mechanism for CRD Conversion Webhooks to specify which version of ConversionReview they support
* Add a new kubelet endpoint for serving first-class resource metrics ([#73946](https://github.com/kubernetes/kubernetes/pull/73946), [@dashpole](https://github.com/dashpole))
* Deprecate AWS, Azure, GCE and Cinder specific volume limit predicates. ([#74544](https://github.com/kubernetes/kubernetes/pull/74544), [@gnufied](https://github.com/gnufied))
* PodReadinessGate feature is now GA. The feature gate will not allow disabling it. ([#74434](https://github.com/kubernetes/kubernetes/pull/74434), [@freehan](https://github.com/freehan))
* If CSINodeInfo and CSIMigration feature flags are active in the cluster, Kubelet will post NotReady until CSINode is initialized with basic volume plugin mechanism information for well-known drivers ([#74835](https://github.com/kubernetes/kubernetes/pull/74835), [@davidz627](https://github.com/davidz627))
* Add network stats for Windows nodes and containers ([#74788](https://github.com/kubernetes/kubernetes/pull/74788), [@feiskyer](https://github.com/feiskyer))
* kubeadm: when calling "reset" on a control-plane node, remove the APIEndpoint information for this node from the ClusterStatus in the kubeadm ConfigMap. ([#75082](https://github.com/kubernetes/kubernetes/pull/75082), [@neolit123](https://github.com/neolit123))
* kube-apiserver now serves OpenAPI specs for registered CRDs with defined ([#71192](https://github.com/kubernetes/kubernetes/pull/71192), [@roycaihw](https://github.com/roycaihw))
* validation schemata as an alpha feature, to be enabled via the "CustomResourcePublishOpenAPI" feature gate. Kubectl will validate client-side using those. Note that in
* future, client-side validation in 1.14 kubectl against a 1.15 cluster will reject
* unknown fields for CRDs with validation schema defined.
* Fix kubelet start failure issue on Azure Stack due to InstanceMetadata setting ([#74936](https://github.com/kubernetes/kubernetes/pull/74936), [@rjaini](https://github.com/rjaini))
* add subcommand `kubectl create cronjob` ([#71651](https://github.com/kubernetes/kubernetes/pull/71651), [@Pingan2017](https://github.com/Pingan2017))
* The CSIBlockVolume feature gate is now beta, and defaults to enabled. ([#74909](https://github.com/kubernetes/kubernetes/pull/74909), [@bswartz](https://github.com/bswartz))
* Pre-existing log files are now opened with O_APPEND, instead of O_TRUNC. This helps prevent losing logs when components crash-loop, and also enables external log rotation utilities to truncate log files in-place without components extending log files to their pre-truncation sizes on subsequent writes. ([#74837](https://github.com/kubernetes/kubernetes/pull/74837), [@mtaufen](https://github.com/mtaufen))
* the test/e2e/e2e.test binary can test arbitrary storage drivers, see the `-storage.testdriver` parameter ([#72836](https://github.com/kubernetes/kubernetes/pull/72836), [@pohly](https://github.com/pohly))
* Fix panic in kubectl cp command ([#75037](https://github.com/kubernetes/kubernetes/pull/75037), [@soltysh](https://github.com/soltysh))
* iscsi modules haven't even been loaded /sys/class/iscsi_host directory won't exist ([#74787](https://github.com/kubernetes/kubernetes/pull/74787), [@jianglingxia](https://github.com/jianglingxia))
* the fluentd addon daemonset will now target all nodes. ([#74424](https://github.com/kubernetes/kubernetes/pull/74424), [@liggitt](https://github.com/liggitt))
* setting `ENABLE_METADATA_CONCEALMENT=true` in kube-up will now set a `cloud.google.com/metadata-proxy-ready=true` label on new nodes. In v1.16, the metadata proxy add-on will switch to using that label as a node selector.
* setting `KUBE_PROXY_DAEMONSET=true` in kube-up will now set a `node.kubernetes.io/kube-proxy-ds-ready=true` label on new nodes. In v1.16, the kube-proxy daemonset add-on will switch to using that label as a node selector.
* In 1.16, the masq-agent daemonset add-on will switch to using `node.kubernetes.io/masq-agent-ds-ready` as a node selector.
* - Kubelet: replace `du` and `find` with a golang implementation ([#74675](https://github.com/kubernetes/kubernetes/pull/74675), [@dashpole](https://github.com/dashpole))
* - Kubelet: periodically update machine info to support hot-add/remove
* kubeadm: add certificate-key and skip-certificate-key-print flags to kubeadm init ([#74671](https://github.com/kubernetes/kubernetes/pull/74671), [@yagonobre](https://github.com/yagonobre))
* Admission webhooks rules can now limit scope to only match namespaced, or only cluster-scoped resources with a `scope: "Cluster" | "Namespaced" | "*"` field. ([#74477](https://github.com/kubernetes/kubernetes/pull/74477), [@liggitt](https://github.com/liggitt))
* The CSIPersistentVolume and KubeletPluginWatcher feature gates cannot be disabled, and will be removed in Kubernetes v1.16 ([#74830](https://github.com/kubernetes/kubernetes/pull/74830), [@msau42](https://github.com/msau42))
* Kubelet won't evict a static pod with priority `system-node-critical` upon resource pressure. ([#74222](https://github.com/kubernetes/kubernetes/pull/74222), [@Huang-Wei](https://github.com/Huang-Wei))
* Fixes panic if a kubelet is run against an older kube-apiserver ([#74529](https://github.com/kubernetes/kubernetes/pull/74529), [@liggitt](https://github.com/liggitt))
* The resource group name in Azure providerID is not converted to lower cases. ([#74882](https://github.com/kubernetes/kubernetes/pull/74882), [@feiskyer](https://github.com/feiskyer))
* Remove the out-of-tree PersistentVolumeLabel controller because it cannot run without Initializers (removed in v1.14). If you are using AWS EBS, GCE PD, Azure Disk, Cinder Disk or vSphere volumes and rely on zone labels, then enable the `PersistentVolumeLabel` admission controller in the `kube-apiserver` in the `--enable-admission-plugins` flag. ([#74615](https://github.com/kubernetes/kubernetes/pull/74615), [@andrewsykim](https://github.com/andrewsykim))
* kubeadm: improved RequiredIPVSKernelModulesAvailable warning message ([#74033](https://github.com/kubernetes/kubernetes/pull/74033), [@bart0sh](https://github.com/bart0sh))
* Add `nullable` support to CustomResourceDefinition OpenAPI validation schemata. ([#74804](https://github.com/kubernetes/kubernetes/pull/74804), [@sttts](https://github.com/sttts))
* Fix kube-apiserver not to create default/kubernetes service endpoints before it reports readiness via the /healthz and therefore is ready to serve requests. Also early during startup old endpoints are remove which might be left over from a previously crashed kube-apiserver. ([#74668](https://github.com/kubernetes/kubernetes/pull/74668), [@sttts](https://github.com/sttts))
* kubeadm: fix a bug where standard kubeconfig paths were searched even if the user provided /etc/kubernetes/admin.conf explicitly for commands that accept --kubeconfig, like kubeadm token. ([#71874](https://github.com/kubernetes/kubernetes/pull/71874), [@neolit123](https://github.com/neolit123))
* kubeadm: use the default kubeconfig (/etc/kubernetes/admin.conf) for "kubeadm reset" and "kubeadm upgrade" commands.
* Increase api server client certificate expiration histogram resolution to accommodate short-lived (< 6h) client certificates. ([#74806](https://github.com/kubernetes/kubernetes/pull/74806), [@mxinden](https://github.com/mxinden))
* Default RBAC policy no longer grants access to discovery and permission-checking APIs (used by `kubectl auth can-i`) to *unauthenticated* users. Upgraded clusters preserve prior behavior, but cluster administrators wishing to grant unauthenticated users access in new clusters will need to explicitly opt-in to expose the discovery and/or permission-checking APIs: ([#73807](https://github.com/kubernetes/kubernetes/pull/73807), [@dekkagaijin](https://github.com/dekkagaijin))
* `kubectl create clusterrolebinding anonymous-discovery --clusterrole=system:discovery --group=system:unauthenticated`
* `kubectl create clusterrolebinding anonymous-access-review --clusterrole=system:basic-user --group=system:unauthenticated`
* The PersistentLocalVolumes feature is GA. The feature gate cannot be disabled and will be removed in Kubernetes 1.17 ([#74769](https://github.com/kubernetes/kubernetes/pull/74769), [@msau42](https://github.com/msau42))
* kubelet: resolved hang/timeout issues when running large numbers of pods with unique configmap/secret references by reverting to 1.11 configmap/secret lookup behavior ([#74755](https://github.com/kubernetes/kubernetes/pull/74755), [@liggitt](https://github.com/liggitt))
* Convert `latency`/`latencies` in metrics name to `duration`. ([#74418](https://github.com/kubernetes/kubernetes/pull/74418), [@danielqsj](https://github.com/danielqsj))
* The following metrics are changed and mark previous metrics as deprecated:
* `rest_client_request_latency_seconds` -> `rest_client_request_duration_seconds`
* `apiserver_proxy_tunnel_sync_latency_secs` -> `apiserver_proxy_tunnel_sync_duration_seconds`
* `scheduler_scheduling_latency_seconds` -> `scheduler_scheduling_duration_seconds `
* Fix help message for --container-runtime-endpoint: only unix socket is support on Linux. ([#74712](https://github.com/kubernetes/kubernetes/pull/74712), [@feiskyer](https://github.com/feiskyer))
* Update to use golang 1.12 ([#74632](https://github.com/kubernetes/kubernetes/pull/74632), [@cblecker](https://github.com/cblecker))
* The `RunAsGroup` feature has been promoted to beta and enabled by default. PodSpec and PodSecurityPolicy objects can be used to control the primary GID of containers on supported container runtimes. ([#73007](https://github.com/kubernetes/kubernetes/pull/73007), [@krmayankk](https://github.com/krmayankk))
* fix Azure Container Registry anonymous repo image pull error ([#74715](https://github.com/kubernetes/kubernetes/pull/74715), [@andyzhangx](https://github.com/andyzhangx))
* Adds the same information to an init container as a standard container in a pod when using PodPresets. ([#71479](https://github.com/kubernetes/kubernetes/pull/71479), [@soggiest](https://github.com/soggiest))
* fix the flake in scheduling_queue_test.go ([#74611](https://github.com/kubernetes/kubernetes/pull/74611), [@denkensk](https://github.com/denkensk))
* The kube-apiserver OpenAPI definitions with the prefix "io.k8s.kubernetes.pkg" (deprecated since 1.9) have been removed. ([#74596](https://github.com/kubernetes/kubernetes/pull/74596), [@sttts](https://github.com/sttts))
* kube-conformance image will now run ginkgo with the --dryRun flag if the container is run with the environment variable E2E_DRYRUN set. ([#74731](https://github.com/kubernetes/kubernetes/pull/74731), [@johnSchnake](https://github.com/johnSchnake))
* The deprecated `MountPropagation` feature gate has been removed, and the feature is now unconditionally enabled. ([#74720](https://github.com/kubernetes/kubernetes/pull/74720), [@bertinatto](https://github.com/bertinatto))
* Introduce dynamic volume provisioning shim for CSI migration ([#73653](https://github.com/kubernetes/kubernetes/pull/73653), [@ddebroy](https://github.com/ddebroy))
* Fix --help flag parsing ([#74682](https://github.com/kubernetes/kubernetes/pull/74682), [@soltysh](https://github.com/soltysh))
* This PR removes the following metrics: ([#74636](https://github.com/kubernetes/kubernetes/pull/74636), [@logicalhan](https://github.com/logicalhan))
* reflector_items_per_list
* reflector_items_per_watch
* reflector_last_resource_version
* reflector_list_duration_seconds
* reflector_lists_total
* reflector_short_watches_total
* reflector_watch_duration_seconds
* reflector_watches_total
* While this is a backwards-incompatible change, it would have been impossible to setup reliable monitoring around these metrics since the labels were not stable.
* Add a configuration field to shorten the timeout of validating/mutating admission webhook call. The timeout value must be between 1 and 30 seconds. Default to 30 seconds when unspecified. ([#74562](https://github.com/kubernetes/kubernetes/pull/74562), [@roycaihw](https://github.com/roycaihw))
* client-go: PortForwarder.GetPorts() now contain correct local port if no local port was initially specified when setting up the port forwarder ([#73676](https://github.com/kubernetes/kubernetes/pull/73676), [@martin-helmich](https://github.com/martin-helmich))
* # Apply resources from a directory containing kustomization.yaml ([#74140](https://github.com/kubernetes/kubernetes/pull/74140), [@Liujingfang1](https://github.com/Liujingfang1))
* kubectl apply -k dir
* # Delete resources from a directory containing kustomization.yaml.
* kubectl delete -k dir
* # List resources from a directory containing kustomization.yaml
* kubectl get -k dir
* kubeadm: Allow to download certificate secrets uploaded by `init` or `upload-certs` phase, allowing to transfer certificate secrets (certificates and keys) from the cluster to other master machines when creating HA deployments. ([#74168](https://github.com/kubernetes/kubernetes/pull/74168), [@ereslibre](https://github.com/ereslibre))
* Fixes an issue with missing apiVersion/kind in object data sent to admission webhooks ([#74448](https://github.com/kubernetes/kubernetes/pull/74448), [@liggitt](https://github.com/liggitt))
* client-go: the deprecated versionless API group accessors (like `clientset.Apps()` have been removed). Use an explicit version instead (like `clientset.AppsV1()`) ([#74422](https://github.com/kubernetes/kubernetes/pull/74422), [@liggitt](https://github.com/liggitt))
* The `--quiet` option to `kubectl run` now suppresses resource deletion messages emitted when the `--rm` option is specified. ([#73266](https://github.com/kubernetes/kubernetes/pull/73266), [@awh](https://github.com/awh))
* Add Custom Resource support to "kubectl autoscale" ([#72678](https://github.com/kubernetes/kubernetes/pull/72678), [@rmohr](https://github.com/rmohr))
* Image garbage collection no longer fails for images with only one tag but more than one repository associated. ([#70647](https://github.com/kubernetes/kubernetes/pull/70647), [@corvus-ch](https://github.com/corvus-ch))
* - Fix liveness probe in fluentd-gcp cluster addon ([#74522](https://github.com/kubernetes/kubernetes/pull/74522), [@Pluies](https://github.com/Pluies))
* The new test ``[sig-network] DNS should provide /etc/hosts entries for the cluster [LinuxOnly] [Conformance]`` will validate the host entries set in the ``/etc/hosts`` file (pod's FQDN and hostname), which should be managed by Kubelet. ([#72729](https://github.com/kubernetes/kubernetes/pull/72729), [@bclau](https://github.com/bclau))
* The test has the tag ``[LinuxOnly]`` because individual files cannot be mounted in Windows Containers, which means that it cannot pass using Windows nodes.
# v1.14.0-beta.1
[Documentation](https://docs.k8s.io)
## Downloads for v1.14.0-beta.1
filename | sha512 hash
-------- | -----------
[kubernetes.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes.tar.gz) | `065014c751635f8c077fbcc105df578594baf8afd8b8339004909198e1bd68d0a7ce3644ed5d54e5964d1306aa650f22a5ce83063415240f4dffd6706c1cc33b`
[kubernetes-src.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-src.tar.gz) | `244c19d9fad21ae154ee78fc94888dc60bcfcf3ec72bdc28a82e77c572cbc969d2abbf20397ef9564a35585c08dfe179b105fc25efac973e0a13d78ee2ff8f42`
### Client Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-darwin-386.tar.gz) | `78a61a1e922daa39a9f7dd61b8bad87d202b537bda59f90ae8aae941c0ff412e3d328530af9dd9f22462cbd67254e7ce1556defe48bb10bd6a94d4302464fe8e`
[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-darwin-amd64.tar.gz) | `5a1d66fd90a5dc07b95b7617b5583595e0a4a664e7005f6281f846c85b21e28692b2e2d55b7c40c7b8cbb96b6b8cd6c832f340c7cc67579641beabac033014ad`
[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-linux-386.tar.gz) | `30991ce0776bae7551b98a811e3ccb5104b0859805c41a216db7d5779cddfb36ef3c5e658ea2adeaf67f8e6f181768850b09a0e8320c2983d34664156ea638ac`
[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-linux-amd64.tar.gz) | `5501a9c3a95e43f0c691b05043546f2c497d50a6ad88b88219842d61be83d7dcb8871ff9fd2447c02bc842c4c962f298a3ce2e0618fae70e8aa391c9407626e5`
[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-linux-arm.tar.gz) | `4bf341a2f943719d006f4cacc26fdf4d021560d37d49c8d9c4620d294142041155a88dc721d9373a8617e1baa904c02b4545f379ffe87c6ab20e5459a5d3c2de`
[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-linux-arm64.tar.gz) | `1c0c660618947b053404ab8ef40bdec0a06d54a1f9edc585a7259806f878327d9ae54100bba98e5b7f44f5db4303276d189792e68517603520a49868c07e684c`
[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-linux-ppc64le.tar.gz) | `9d2c3f4bfbad03b41859670f9cfda6596f51a89077fd4da2f74490f71b5de10e459954e897d2a1ba3a217c62caaf1be74424e5bf6a5609868dc4f069ac06c94f`
[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-linux-s390x.tar.gz) | `9be5259caf39ff3c4d0f024d616bff50ac417d42a87c56b6877fbdf5aaadecee05631ac8ddd6dca20d52ae4ca7e1227a1fd5e882be9821f3711e144b84d41099`
[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-windows-386.tar.gz) | `5b1e75f532d9a4d656cd1c5ec48a19d01c4ca731c6c3d986ebbc48a9f1e1d61d6c5603145808fb929117cf2202bc75f31ec7661a50edf24c5af6b3419ebcc0ab`
[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-client-windows-amd64.tar.gz) | `a501d3c0be55d5a73214a83d1f48b39a49d4a84f5f9988fa34cf66ef6bc78d3f3e06c40dc3d59538c3bff07aa2128ee814d9825c31f8b9abbf045e1ebb581bed`
### Server Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-server-linux-amd64.tar.gz) | `aed44ec5bc6bdaf41c20824a9841ee541bfd23362966f9326eca2cafbd03eca69325877784b8c9b058963dc5b8ba656b9da446513fd5eeed8133a783769acd16`
[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-server-linux-arm.tar.gz) | `e751cb675013183a70a8817dca0b5c456a1ccf075244b411317e813b5164ae710460a53e81191ded9d2ccaf284ae00304bfbe1b3d219b2a8d57761d733293409`
[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-server-linux-arm64.tar.gz) | `3aabb1a9bd07413d0740adfb638b0e5ca4cd4a58eda244c5ebc1ea01780e0b2863806c35792a0590069ef0cccc2665198afed1984d1c49f0726b75d4216609a4`
[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-server-linux-ppc64le.tar.gz) | `3c57e2b47b3b9ddc1039684840886877947141b1e4d31f909793678544fe92e10aa82a207936f0bcb3c657044c7b875f34f41f086ccb7f97a154499d01266f73`
[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-server-linux-s390x.tar.gz) | `fe418cfafc63cbccc1898086296e52026df27ff498753089792175ce0d41d889ca50a4eb5104a84b78a0a25d524dcdf5ec5eec8aa213d58178ae38411cfb58af`
### Node Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-node-linux-amd64.tar.gz) | `83b63d7e9d18fe35564105fd70629af9ba8f20112933b3ace92a48887702862e013ac3b3e144cfd44c8ead7c766df584766749af9d33a9aaa3808e370d3ea359`
[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-node-linux-arm.tar.gz) | `e4ac000be25ced9b308ec698da9702bd10a0a7183bfea9648500ab9d0879989c54328cb4fb51be545831d18676067d6b53e55cc49b14f55f35032a66dfa28806`
[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-node-linux-arm64.tar.gz) | `35fa5ec8a7fc001fe33abd89f8a446ce0ea2a011db27dc8ff544c2b199b065b19372afe95e3616f538347243d7599f29fb5cea1a46a3fcfdfe4a0f2a346683b2`
[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-node-linux-ppc64le.tar.gz) | `a5b938e9cdc39fc292269af4c3961a17b9bcdaf85b3c58db680f2d1a4fb088eb648efc268dfe3325aed96f6a7cdcff070dc7a3200cc169010ff2b402fae1a26a`
[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-node-linux-s390x.tar.gz) | `d8e2b5a945f881ddb0a25576a614d564ccef0ad4e93c84b30cdc57888e81e04932d798415a1a50cdfc6d2f857e1d027e2034ba9c40b5d8ed0009cfbdc8915e0f`
[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-beta.1/kubernetes-node-windows-amd64.tar.gz) | `426774466800ca11cae16821c7d92917b2ce138fcb3f6dca813ec0f060649d6a50187982aaac31fbb081604f5bd2c317616b6f211beb139f53192269194f012e`
## Changelog since v1.14.0-alpha.3
### Action Required
* Added cadvisor metric labels `pod` and `container` where `pod_name` and `container_name` are present to match instrumentation guidelines. ([#69099](https://github.com/kubernetes/kubernetes/pull/69099), [@ehashman](https://github.com/ehashman))
* Action required: any Prometheus queries that match `pod_name` and `container_name` labels (e.g. cadvisor or kubelet probe metrics) should be updated to use `pod` and `container` instead. `pod_name` and `container_name` labels will be present alongside `pod` and `container` labels for one transitional release and removed in the future.
### Other notable changes
* fix parse devicePath issue on Azure Disk ([#74499](https://github.com/kubernetes/kubernetes/pull/74499), [@andyzhangx](https://github.com/andyzhangx))
* fix issue: fail to detach azure disk when there is server side error ([#74398](https://github.com/kubernetes/kubernetes/pull/74398), [@andyzhangx](https://github.com/andyzhangx))
* Allow Cinder volume limit to be configured from node too ([#74542](https://github.com/kubernetes/kubernetes/pull/74542), [@gnufied](https://github.com/gnufied))
* Fix subnet annotation checking for Azure internal loadbalancer ([#74498](https://github.com/kubernetes/kubernetes/pull/74498), [@feiskyer](https://github.com/feiskyer))
* Allow the kubelet to pass Windows GMSA credentials down to Docker ([#73726](https://github.com/kubernetes/kubernetes/pull/73726), [@wk8](https://github.com/wk8))
* PriorityClass in scheduling.k8s.io/v1beta1 and scheduling.k8s.io/v1alpha1 are deprecated by PriorityClass scheduling.k8s.io/v1 and will not be served starting in v1.17. ([#74465](https://github.com/kubernetes/kubernetes/pull/74465), [@bsalamat](https://github.com/bsalamat))
* kubeadm: fixed nil pointer dereference caused by a bug in url parsing ([#74454](https://github.com/kubernetes/kubernetes/pull/74454), [@bart0sh](https://github.com/bart0sh))
* Fix the unexpected NotReady status when Node's iops is full if the runtime is dockershim. ([#74389](https://github.com/kubernetes/kubernetes/pull/74389), [@answer1991](https://github.com/answer1991))
* Split up the mondo `kubernetes-test` tarball into `kubernetes-test-portable` and `kubernetes-test-{OS}-{ARCH}` tarballs. ([#74065](https://github.com/kubernetes/kubernetes/pull/74065), [@ixdy](https://github.com/ixdy))
* Move fluentd-elasticsearch addon images to community controlled location ([#73819](https://github.com/kubernetes/kubernetes/pull/73819), [@coffeepac](https://github.com/coffeepac))
* The PriorityClass API has been promoted to `scheduling.k8s.io/v1` with no changes. The `scheduling.k8s.io/v1beta1` version is now deprecated and will stop being served by default in v1.17. ([#73555](https://github.com/kubernetes/kubernetes/pull/73555), [@bsalamat](https://github.com/bsalamat))
* fix get azure accounts timeout issue when there is no out-bound IP ([#74191](https://github.com/kubernetes/kubernetes/pull/74191), [@andyzhangx](https://github.com/andyzhangx))
* fix mixed protocol issue for azure load balancer ([#74200](https://github.com/kubernetes/kubernetes/pull/74200), [@andyzhangx](https://github.com/andyzhangx))
* Don't update the Pod object after each scheduling attempt by adding a timestamp to the scheduling queue. ([#73700](https://github.com/kubernetes/kubernetes/pull/73700), [@denkensk](https://github.com/denkensk))
* kubeadm: remove local etcd members from the etcd cluster when kubeadm reset ([#74112](https://github.com/kubernetes/kubernetes/pull/74112), [@pytimer](https://github.com/pytimer))
* Fix keymutex issues which may crash in some platforms. ([#74348](https://github.com/kubernetes/kubernetes/pull/74348), [@danielqsj](https://github.com/danielqsj))
* Fixed scanning of failed iSCSI targets. ([#74306](https://github.com/kubernetes/kubernetes/pull/74306), [@jsafrane](https://github.com/jsafrane))
* kubeadm: Do not fail preflight checks when running on >= 5.0 Linux kernel ([#74355](https://github.com/kubernetes/kubernetes/pull/74355), [@brb](https://github.com/brb))
* Reduces the cache TTL for negative responses to 5s minimum. ([#74093](https://github.com/kubernetes/kubernetes/pull/74093), [@blakebarnett](https://github.com/blakebarnett))
* The Ingress API is now available via `networking.k8s.io/v1beta1`. `extensions/v1beta1` Ingress objects are deprecated and will no longer be served in v1.18. ([#74057](https://github.com/kubernetes/kubernetes/pull/74057), [@liggitt](https://github.com/liggitt))
* kubelet's --containerized flag will no longer be supported and will be removed in a future release ([#74267](https://github.com/kubernetes/kubernetes/pull/74267), [@dims](https://github.com/dims))
* Optimize scheduler cache snapshot algorithm to improve scheduling throughput. ([#74041](https://github.com/kubernetes/kubernetes/pull/74041), [@bsalamat](https://github.com/bsalamat))
* Extends the VolumeSubpathEnvExpansion alpha feature to support environment variable expansion ([#71351](https://github.com/kubernetes/kubernetes/pull/71351), [@kevtaylor](https://github.com/kevtaylor))
* Implements subPathExpr field for expanding environment variables into a subPath
* The fields subPathExpr and subPath are mutually exclusive
* Note: This is a breaking change from the previous version of this alpha feature
* Added kube-proxy support for overlay networking and DSR in Windows and new flags for network-name, source-vip, and enable-dsr. ([#70896](https://github.com/kubernetes/kubernetes/pull/70896), [@ksubrmnn](https://github.com/ksubrmnn))
* StorageOS volume plugin updated to fix an issue where volume mount succeeds even if request to mount via StorageOS API fails. ([#69782](https://github.com/kubernetes/kubernetes/pull/69782), [@darkowlzz](https://github.com/darkowlzz))
* kubeadm: Allow to upload certificates required to join a new control-plane to kubeadm-certs secret using the flag `--experimental-upload-certs` on `init` or upload-certs phase. ([#73907](https://github.com/kubernetes/kubernetes/pull/73907), [@yagonobre](https://github.com/yagonobre))
* export query parameter is deprecated and will be removed in a future release ([#73783](https://github.com/kubernetes/kubernetes/pull/73783), [@deads2k](https://github.com/deads2k))
* e2e storage tests run faster and are easier to read ([#72434](https://github.com/kubernetes/kubernetes/pull/72434), [@pohly](https://github.com/pohly))
* kubectl: fix a bug where "describe" cannot obtain the event messages for a static pod ([#74156](https://github.com/kubernetes/kubernetes/pull/74156), [@gaorong](https://github.com/gaorong))
* windows: Ensure graceful termination when being run as windows service ([#73292](https://github.com/kubernetes/kubernetes/pull/73292), [@steffengy](https://github.com/steffengy))
* CoreDNS adds readinessProbe which prevents loadbalancing to unready pods, and also allows rolling updates to work as expected. ([#74137](https://github.com/kubernetes/kubernetes/pull/74137), [@rajansandeep](https://github.com/rajansandeep))
* Fixes use of webhook admission plugins with multi-version custom resources ([#74154](https://github.com/kubernetes/kubernetes/pull/74154), [@mbohlool](https://github.com/mbohlool))
* kubeadm no longer allows using v1alpha3 configs for anything else than converting them to v1beta1. ([#74025](https://github.com/kubernetes/kubernetes/pull/74025), [@rosti](https://github.com/rosti))
* Change kubelet metrics to conform metrics guidelines. ([#72470](https://github.com/kubernetes/kubernetes/pull/72470), [@danielqsj](https://github.com/danielqsj))
* The following metrics are deprecated, and will be removed in a future release:
* `kubelet_pod_worker_latency_microseconds`
* `kubelet_pod_start_latency_microseconds`
* `kubelet_cgroup_manager_latency_microseconds`
* `kubelet_pod_worker_start_latency_microseconds`
* `kubelet_pleg_relist_latency_microseconds`
* `kubelet_pleg_relist_interval_microseconds`
* `kubelet_eviction_stats_age_microseconds`
* `kubelet_runtime_operations`
* `kubelet_runtime_operations_latency_microseconds`
* `kubelet_runtime_operations_errors`
* `kubelet_device_plugin_registration_count`
* `kubelet_device_plugin_alloc_latency_microseconds`
* Please convert to the following metrics:
* `kubelet_pod_worker_duration_seconds`
* `kubelet_pod_start_duration_seconds`
* `kubelet_cgroup_manager_duration_seconds`
* `kubelet_pod_worker_start_duration_seconds`
* `kubelet_pleg_relist_duration_seconds`
* `kubelet_pleg_relist_interval_seconds`
* `kubelet_eviction_stats_age_seconds`
* `kubelet_runtime_operations_total`
* `kubelet_runtime_operations_duration_seconds`
* `kubelet_runtime_operations_errors_total`
* `kubelet_device_plugin_registration_total`
* `kubelet_device_plugin_alloc_duration_seconds`
* This change ensures that volumes get provisioned based on the zone information provided in allowedTopologies. ([#72731](https://github.com/kubernetes/kubernetes/pull/72731), [@skarthiksrinivas](https://github.com/skarthiksrinivas))
* Storage class spec:
* kind: StorageClass
* apiVersion: storage.k8s.io/v1
* metadata:
* name: fastpolicy1
* provisioner: kubernetes.io/vsphere-volume
* parameters:
* diskformat: zeroedthick
* storagePolicyName: vSAN Default Storage Policy
* allowedTopologies:
* - matchLabelExpressions:
* - key: failure-domain.beta.kubernetes.io/zone
* values:
* - zone1
* PV creation Logs:
* I0109 11:17:52.321372 1 vsphere.go:1147] Starting to create a vSphere volume with volumeOptions: &{CapacityKB:1048576 Tags:map[kubernetes.io/created-for/pvc/namespace:default kubernetes.io/created-for/pvc/name:pvcsc-1-policy kubernetes.io/created-for/pv/name:pvc-34650c12-1400-11e9-aef4-005056804cc9] Name:kubernetes-dynamic-pvc-34650c12-1400-11e9-aef4-005056804cc9 DiskFormat:zeroedthick Datastore: VSANStorageProfileData: StoragePolicyName:vSAN Default Storage Policy StoragePolicyID: SCSIControllerType: Zone:[zone1]}
* ...
* I0109 11:17:59.430113 1 vsphere.go:1334] The canonical volume path for the newly created vSphere volume is "[vsanDatastore] 98db185c-6683-d8c7-bc55-0200435ec5da/kubernetes-dynamic-pvc-34650c12-1400-11e9-aef4-005056804cc9.vmdk"
* Ran regression tests (no zone) and they passed.
* vSphere cloud provider correctly retrieves the VM's UUID when running on Windows ([#71147](https://github.com/kubernetes/kubernetes/pull/71147), [@benmoss](https://github.com/benmoss))
* Re-issue Allocate grpc calls before starting a container that requests device-plugin resources if the cached state is missing. ([#73824](https://github.com/kubernetes/kubernetes/pull/73824), [@jiayingz](https://github.com/jiayingz))
* [CRI] Add a new field called `runtime_handler` into PodSandbox and PodSandboxStatus to track the RuntimeClass information of a pod. ([#73833](https://github.com/kubernetes/kubernetes/pull/73833), [@haiyanmeng](https://github.com/haiyanmeng))
* kubelet: OS and Arch information is now recorded in `kubernetes.io/os` and `kubernetes.io/arch` labels on Node objects. The previous labels (`beta.kubernetes.io/os` and `beta.kubernetes.io/arch`) are still recorded, but are deprecated and targeted for removal in 1.18. ([#73333](https://github.com/kubernetes/kubernetes/pull/73333), [@yujuhong](https://github.com/yujuhong))
* This change applies zone labels to vSphere Volumes automatically. The zone labels are visible on the PV: ([#72687](https://github.com/kubernetes/kubernetes/pull/72687), [@subramanian-neelakantan](https://github.com/subramanian-neelakantan))
* $ kubectl get pv --show-labels
* NAME CAPACITY ACCESSMODES STATUS CLAIM REASON AGE LABELS
* pv-abc 5Gi RWO Bound default/claim1 46s failure-domain.beta.kubernetes.io/region=VC1,failure-domain.beta.kubernetes.io/zone=cluster-1
* fix smb remount issue on Windows ([#73661](https://github.com/kubernetes/kubernetes/pull/73661), [@andyzhangx](https://github.com/andyzhangx))
* Kubelet now tries to stop containers in unknown state once before restart or remove. ([#73802](https://github.com/kubernetes/kubernetes/pull/73802), [@Random-Liu](https://github.com/Random-Liu))
* Deprecate --export flag from kubectl get command. ([#73787](https://github.com/kubernetes/kubernetes/pull/73787), [@soltysh](https://github.com/soltysh))
* Breaking changes in client-go: ([#72214](https://github.com/kubernetes/kubernetes/pull/72214), [@caesarxuchao](https://github.com/caesarxuchao))
* The disk-cached discovery client is moved from k8s.io/client-go/discovery to k8s.io/client-go/discovery/cached/disk.
* The memory-cached discovery client is moved from k8s.io/client-go/discovery/cached to k8s.io/client-go/discovery/cached/memory.
* kubelet now accepts `pid=<number>` in the `--system-reserved` and `--kube-reserved` options to ensure that the specified number of process IDs will be reserved for the system as a whole and for Kubernetes system daemons respectively. Please reference `Kube Reserved` and `System Reserved` in `Reserve Compute Resources for System Daemons` in the Kubernetes documentation for general discussion of resource reservation. To utilize this functionality, you must set the feature gate `SupportNodePidsLimit=true` ([#73651](https://github.com/kubernetes/kubernetes/pull/73651), [@RobertKrawitz](https://github.com/RobertKrawitz))
* The apiserver, including both the kube-apiserver and apiservers built with the generic apiserver library, will now return 413 RequestEntityTooLarge error if a json patch contains more than 10,000 operations. ([#74000](https://github.com/kubernetes/kubernetes/pull/74000), [@caesarxuchao](https://github.com/caesarxuchao))
* kubeadm: allow the usage of --kubeconfig-dir and --config flags on kubeadm init ([#73998](https://github.com/kubernetes/kubernetes/pull/73998), [@yagonobre](https://github.com/yagonobre))
* when pleg channel is full, discard events and record its count ([#72709](https://github.com/kubernetes/kubernetes/pull/72709), [@changyaowei](https://github.com/changyaowei))
* Is ->It in line 6 ([#73898](https://github.com/kubernetes/kubernetes/pull/73898), [@xiezongzhe](https://github.com/xiezongzhe))
* Events reported for container creation, start, and stop now report the container name in the message and are more consistently formatted. ([#73892](https://github.com/kubernetes/kubernetes/pull/73892), [@smarterclayton](https://github.com/smarterclayton))
* `kubectl auth reconcile` now outputs details about what changes are being made ([#71564](https://github.com/kubernetes/kubernetes/pull/71564), [@liggitt](https://github.com/liggitt))
* kubeadm: fix a bug in the underlying library for diff related to characters like '%' ([#73941](https://github.com/kubernetes/kubernetes/pull/73941), [@neolit123](https://github.com/neolit123))
* kube-apiserver: a request body of a CREATE/UPDATE/PATCH/DELETE resource operation larger than 100 MB will return a 413 "request entity too large" error. ([#73805](https://github.com/kubernetes/kubernetes/pull/73805), [@caesarxuchao](https://github.com/caesarxuchao))
* Custom apiservers built with the latest apiserver library will have the 100MB limit on the body of resource requests as well. The limit can be altered via ServerRunOptions.MaxRequestBodyBytes.
* The body size limit does not apply to subresources like pods/proxy that proxy request content to another server.
* Kustomize is developed in its own repo https://github.com/kubernetes-sigs/kustomize ([#73033](https://github.com/kubernetes/kubernetes/pull/73033), [@Liujingfang1](https://github.com/Liujingfang1))
* This PR added a new subcommand `kustomize` in kubectl.
* kubectl kustomize <somedir> has the same effect as kustomize build <somedir>
* To build API resources from somedir with a kustomization.yaml file
* kubectl kustomize <somedir>
* This command can be piped to apply or delete
* kubectl kustomize <somedir> | kubectl apply -f -
* kubectl kustomize <somedir> | kubectl delete -f -
* kubeadm: all master components are now exclusively relying on the `PriorityClassName` pod spec for annotating them as cluster critical components. Since `scheduler.alpha.kubernetes.io/critical-pod` annotation is no longer supported by Kubernetes 1.14 this annotation is no longer added to master components. ([#73857](https://github.com/kubernetes/kubernetes/pull/73857), [@ereslibre](https://github.com/ereslibre))
* Speedup kubectl by >10 when calling out to kube-apiserver for discovery information. ([#73345](https://github.com/kubernetes/kubernetes/pull/73345), [@sttts](https://github.com/sttts))
* kubeadm no longer dumps backtrace if it fails to remove the running containers on reset. ([#73951](https://github.com/kubernetes/kubernetes/pull/73951), [@rosti](https://github.com/rosti))
# v1.14.0-alpha.3
[Documentation](https://docs.k8s.io)
## Downloads for v1.14.0-alpha.3
filename | sha512 hash
-------- | -----------
[kubernetes.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes.tar.gz) | `5060dcf689dad4e19da5029eb8fc3060a4b2bad988fddff438d0703a45c02481bcfbc15f45d2855f4fd5e9eb43847400ebb25dce19e24f0e0e194a7f57176ce5`
[kubernetes-src.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-src.tar.gz) | `754c948b5d25b01f211866d473257be5fb576b4b97703eb6fc08679d6525e1f53195a450f3f47b77fabb92bf058583b66230959197b5bcf72528e54ccb349c07`
### Client Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-darwin-386.tar.gz) | `5bd74dfc86bacf89d6b05d541e13bf390216039a42cc90fef2b248820acd84f56a445ec66d52497ff77e1af47455f285c993cd1d44cc3050996189bd328ea2be`
[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-darwin-amd64.tar.gz) | `34e16661d66d337083583dfb478756ec8cc664d7cfc2dd1817bf1da03cdc380668be9df9f178b5fd5ccab5014e6686f83b9fee6192fbf77d2298d397e872a893`
[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-linux-386.tar.gz) | `15f99e85bcc95f7b8e1b4c6ecc23de36e89a54108003db926e97ec2e7253f363f6ed85e39a47305dbccf596f72e88edd7bcda6d528919da9c0b81541f58506d4`
[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-linux-amd64.tar.gz) | `2e61cf9b776150c4f1830d068ffee9701cb04979152ed6b62fc1bf53163e6194029a4f75536e7fda71c3dfce1de285f425bde342a4efdd1f7bf973f105750ac4`
[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-linux-arm.tar.gz) | `67fb3805bb1b4a77f6603fbde9bd1d26e179de1a594c85618aa7b17be6abc510a9a0cd499ef4fe974574cf73b364da641121f21864c8472d713eec76e4c52bca`
[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-linux-arm64.tar.gz) | `28930dc384b51051081a52874bc4d6dafa3c992dfa214b977ef711de2c2bc3f90bdaa6243bded1e750997fec04b8ffb910db21c266e47e09426c4dbaf916a64d`
[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-linux-ppc64le.tar.gz) | `f59eda797a57961d52fe67ba8b25a3a10267f9ce46029ed2140ef4b02615ba9944bd83d7a6e7874c7268a09a3422858b9b0c31f861941ef8be126c594fc3a7cc`
[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-linux-s390x.tar.gz) | `c56bfb64e55cf95251157a8229a3e94310b2c46bb1c1250050893873e3112578978c1f8e29fa56fac63e2aa8a6382523ac34baf6dd523fe0919f8d702521a564`
[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-windows-386.tar.gz) | `e49a00fbe600892dc5eed0bc21bac64806da65280c818ca79b5e8adbed7fd5ecebb6b647cb9b89ac862257995145b2397996122eefb3c8d127d857c89c29c9ae`
[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-client-windows-amd64.tar.gz) | `797e20969ed4935adcbc80ccbcd72ec5aa697e70b0d071eceefc6dbacea69aff9f6660e7eefad6661ace0afb66067c4ffaa4f6bc82e8b081b57811ab0abde218`
### Server Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-server-linux-amd64.tar.gz) | `eddfc9afd7337475c3865443170d1425dcf4a87d981555871a69bcf132e73d99b1ffa08a00490b30c60232f47bbeca4ad6253cf7e1dad44797b4af044dbdbef4`
[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-server-linux-arm.tar.gz) | `dc85cd3a039cc0516beb19018c8378f3b7b88fa2edb8fa1476305e89eb7c64fef2d938bd48fd257ea8e690f7d84a69e9784a42aabed35e83ea7362c60773ba67`
[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-server-linux-arm64.tar.gz) | `d7c3a72abaa4c3e3243f8b4b3a8adb8be2758e0f883423ea62d2c61b2081464a8976ad43ea0640a7e453aa4d389e3ea2d6d1baedf3b50e1171eca6e49cd087fe`
[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-server-linux-ppc64le.tar.gz) | `b268a94eb056eea8bdf4d5739dec430f75a6a6b3c18e30df68d970c3566b3e4a638b3577f6219596ae54eac740628a7ebfecb0772645e6d960f790235e1d62c7`
[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-server-linux-s390x.tar.gz) | `f4cfd8d2faacdd1f0065f9e0f4f8d0db7bd8f438f812f70a07f4cb5272ae9bed3ec876b3cbaf2f2a71e65e4de725e1dc0829b43f60f43c9e43656ac928657d5e`
### Node Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-node-linux-amd64.tar.gz) | `7040ee3c032ec4fe14530c3e47ee53d731acb947b06e2d560cbcd0e7e513142c0f300302059aaef03e24311946a9c59b576948eec9b520e2367f28fc4f80226c`
[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-node-linux-arm.tar.gz) | `3d32e5243d1c65bce573cfb0f60d643ef3fc684a15551dbc8c3d5435e6854ff104c46c77b0b8708d9c661d52f7865a197ea758f0c17e1ed991993674929ea75e`
[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-node-linux-arm64.tar.gz) | `d3a17027fa1c057528422b35e32260f5b7c7246400df595f0ebda5d150456d4388129b1ead4229f98f2b461ff9e85382a7da0d682541844a3c06f0aebe0469b6`
[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-node-linux-ppc64le.tar.gz) | `89ed1f5093b49ab9d58d7a70089e881bf388f3316cb2607fa18e3bf072aff3d27aabe99124334774e63decb67349eb82f33ea509b56a72a51e1443c3352b4558`
[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-node-linux-s390x.tar.gz) | `755a60824a9b8c4090a791d332e410692708ecece90e37388f58eb2c7ddddea6b859fefcc5a53ec3d275fee0a355086f4446ae8e85482a668d248cca9f5e503c`
[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.3/kubernetes-node-windows-amd64.tar.gz) | `c71d8055d89e535771f345e0f673da021915a7a82c75951855ba2574a4250c8a57d0636b4ec9bba209edde8edef30098c6dec2f80403cd46139bb88d814c3751`
## Changelog since v1.14.0-alpha.2
### Action Required
* The --storage-versions flag of kube-apiserver is removed. The storage versions will always be the default value built-in the kube-apiserver binary. ([#67678](https://github.com/kubernetes/kubernetes/pull/67678), [@caesarxuchao](https://github.com/caesarxuchao))
### Other notable changes
* fix [#73264](https://github.com/kubernetes/kubernetes/pull/73264) cpuPeriod was not reset, but used as set via flag, although it was disabled via alpha gate ([#73342](https://github.com/kubernetes/kubernetes/pull/73342), [@szuecs](https://github.com/szuecs))
* Update kubelet CLI summary documentation and generated Webpage ([#73256](https://github.com/kubernetes/kubernetes/pull/73256), [@deitch](https://github.com/deitch))
* Considerably reduced the CPU load in kube-apiserver while aggregating OpenAPI specifications from aggregated API servers. ([#71223](https://github.com/kubernetes/kubernetes/pull/71223), [@sttts](https://github.com/sttts))
* kubeadm: add a preflight check that throws a warning if the cgroup driver for Docker on Linux is not "systemd" as per the k8s.io CRI installation guide. ([#73837](https://github.com/kubernetes/kubernetes/pull/73837), [@neolit123](https://github.com/neolit123))
* Kubelet: add usageNanoCores from CRI stats provider ([#73659](https://github.com/kubernetes/kubernetes/pull/73659), [@feiskyer](https://github.com/feiskyer))
* Fix watch to not send the same set of events multiple times causing watcher to go back in time ([#73845](https://github.com/kubernetes/kubernetes/pull/73845), [@wojtek-t](https://github.com/wojtek-t))
* `system:kube-controller-manager` and `system:kube-scheduler` users are now permitted to perform delegated authentication/authorization checks by default RBAC policy ([#72491](https://github.com/kubernetes/kubernetes/pull/72491), [@liggitt](https://github.com/liggitt))
* Prevent AWS Network Load Balancer security groups ingress rules to be deleted by ensuring target groups are tagged. ([#73594](https://github.com/kubernetes/kubernetes/pull/73594), [@masterzen](https://github.com/masterzen))
* Set a low oom_score_adj for containers in pods with system-critical priorities ([#73758](https://github.com/kubernetes/kubernetes/pull/73758), [@sjenning](https://github.com/sjenning))
* Ensure directories on volumes are group-executable when using fsGroup ([#73533](https://github.com/kubernetes/kubernetes/pull/73533), [@mxey](https://github.com/mxey))
* kube-apiserver now only aggregates openapi schemas from `/openapi/v2` endpoints of aggregated API servers. The fallback to aggregate from `/swagger.json` has been removed. Ensure aggregated API servers provide schema information via `/openapi/v2` (available since v1.10). ([#73441](https://github.com/kubernetes/kubernetes/pull/73441), [@roycaihw](https://github.com/roycaihw))
* Change docker metrics to conform metrics guidelines and using histogram for better aggregation. ([#72323](https://github.com/kubernetes/kubernetes/pull/72323), [@danielqsj](https://github.com/danielqsj))
* The following metrics are deprecated, and will be removed in a future release:
* `docker_operations`
* `docker_operations_latency_microseconds`
* `docker_operations_errors`
* `docker_operations_timeout`
* `network_plugin_operations_latency_microseconds`
* Please convert to the following metrics:
* `docker_operations_total`
* `docker_operations_latency_seconds`
* `docker_operations_errors_total`
* `docker_operations_timeout_total`
* `network_plugin_operations_latency_seconds`
* `kubectl delete --all-namespaces` is a recognized flag. ([#73716](https://github.com/kubernetes/kubernetes/pull/73716), [@deads2k](https://github.com/deads2k))
* MAC Address filter has been fixed in vSphere Cloud Provider, it no longer ignores `00:1c:14` and `00:05:69` prefixes ([#73721](https://github.com/kubernetes/kubernetes/pull/73721), [@frapposelli](https://github.com/frapposelli))
* Add kubelet_node_name metrics. ([#72910](https://github.com/kubernetes/kubernetes/pull/72910), [@danielqsj](https://github.com/danielqsj))
* The HugePages feature gate has graduated to GA, and can no longer be disabled. The feature gate will be removed in v1.16 ([#72785](https://github.com/kubernetes/kubernetes/pull/72785), [@derekwaynecarr](https://github.com/derekwaynecarr))
* Fix a bug that aggregated openapi spec may override swagger securityDefinitions and swagger info in kube-apiserver ([#73484](https://github.com/kubernetes/kubernetes/pull/73484), [@roycaihw](https://github.com/roycaihw))
* Fixes a bug that prevented deletion of dynamically provisioned volumes in Quobyte backends. ([#68925](https://github.com/kubernetes/kubernetes/pull/68925), [@casusbelli](https://github.com/casusbelli))
* error messages returned in authentication webhook status responses are now correctly included in the apiserver log ([#73595](https://github.com/kubernetes/kubernetes/pull/73595), [@liggitt](https://github.com/liggitt))
* kubeadm: `kubeadm alpha preflight` and `kubeadm alpha preflight node` are removed; you can now use `kubeadm join phase preflight` ([#73718](https://github.com/kubernetes/kubernetes/pull/73718), [@fabriziopandini](https://github.com/fabriziopandini))
* kube-apiserver: the deprecated `repair-malformed-updates` has been removed ([#73663](https://github.com/kubernetes/kubernetes/pull/73663), [@danielqsj](https://github.com/danielqsj))
* e2e.test now rejects unknown --provider values instead of merely warning about them. An empty provider name is not accepted anymore and was replaced by "skeleton" (= a provider with no special behavior). ([#73402](https://github.com/kubernetes/kubernetes/pull/73402), [@pohly](https://github.com/pohly))
* Updated AWS SDK to v1.16.26 for ECR PrivateLink support ([#73435](https://github.com/kubernetes/kubernetes/pull/73435), [@micahhausler](https://github.com/micahhausler))
* Expand kubectl wait to work with more types of selectors. ([#71746](https://github.com/kubernetes/kubernetes/pull/71746), [@rctl](https://github.com/rctl))
* The CustomPodDNS feature gate has graduated to GA, and can no longer be disabled. The feature gate will be removed in v1.16 ([#72832](https://github.com/kubernetes/kubernetes/pull/72832), [@MrHohn](https://github.com/MrHohn))
* The `rules` field in RBAC Role and ClusterRole objects is now correctly reported as optional in the openapi schema. ([#73250](https://github.com/kubernetes/kubernetes/pull/73250), [@liggitt](https://github.com/liggitt))
* AWS ELB health checks will now use HTTPS/SSL protocol for HTTPS/SSL backends. ([#70309](https://github.com/kubernetes/kubernetes/pull/70309), [@2rs2ts](https://github.com/2rs2ts))
* kubeadm reset: fixed crash caused by absence of a configuration file ([#73636](https://github.com/kubernetes/kubernetes/pull/73636), [@bart0sh](https://github.com/bart0sh))
* CoreDNS is now version 1.3.1 ([#73610](https://github.com/kubernetes/kubernetes/pull/73610), [@rajansandeep](https://github.com/rajansandeep))
* - A new `k8s_external` plugin that allows external zones to point to Kubernetes in-cluster services.
* - CoreDNS now checks if a zone transfer is allowed. Also allow a TTL of 0 to avoid caching in the cache plugin.
* - TTL is also applied to negative responses (NXDOMAIN, etc).
* Missing directories listed in a user's PATH are no longer considered errors and are instead logged by the "kubectl plugin list" command when listing available plugins. ([#73542](https://github.com/kubernetes/kubernetes/pull/73542), [@juanvallejo](https://github.com/juanvallejo))
* remove kubelet flag '--experimental-fail-swap-on' (deprecated in v1.8) ([#69552](https://github.com/kubernetes/kubernetes/pull/69552), [@Pingan2017](https://github.com/Pingan2017))
* Introduced support for Windows nodes into the cluster bringup scripts for GCE. ([#73442](https://github.com/kubernetes/kubernetes/pull/73442), [@pjh](https://github.com/pjh))
* Now users could get object info like: ([#73063](https://github.com/kubernetes/kubernetes/pull/73063), [@WanLinghao](https://github.com/WanLinghao))
* a. kubectl get pod test-pod -o custom-columns=CONTAINER:.spec.containers[0:3].name
* b. kubectl get pod test-pod -o custom-columns=CONTAINER:.spec.containers[-2:].name
* scheduler: use incremental scheduling cycle in PriorityQueue to put all in-flight unschedulable pods back to active queue if we received move request ([#73309](https://github.com/kubernetes/kubernetes/pull/73309), [@cofyc](https://github.com/cofyc))
* fixes an error processing watch events when running skewed apiservers ([#73482](https://github.com/kubernetes/kubernetes/pull/73482), [@liggitt](https://github.com/liggitt))
* Prometheus metrics for crd_autoregister, crd_finalizer and crd_naming_condition_controller are exported. ([#71767](https://github.com/kubernetes/kubernetes/pull/71767), [@roycaihw](https://github.com/roycaihw))
* Adds deleting pods created by DaemonSet assigned to not existing nodes. ([#73401](https://github.com/kubernetes/kubernetes/pull/73401), [@krzysztof-jastrzebski](https://github.com/krzysztof-jastrzebski))
* Graduate Pod Priority and Preemption to GA. ([#73498](https://github.com/kubernetes/kubernetes/pull/73498), [@bsalamat](https://github.com/bsalamat))
* Adds configuration for AWS endpoint fine control: ([#72245](https://github.com/kubernetes/kubernetes/pull/72245), [@ampsingram](https://github.com/ampsingram))
* OverrideEndpoints bool Set to true to allow custom endpoints
* ServiceDelimiter string Delimiter to use to separate overridden services (multiple services) Defaults to "&"
* ServicenameDelimiter string Delimiter to use to separate servicename from its configuration parameters Defaults "|"
* OverrideSeparator string Delimiter to use to separate region of occurrence, url and signing region for each override Defaults to ","
* ServiceOverrides string example: s3|region1, https://s3.foo.bar, some signing_region & ec2|region2, https://ec2.foo.bar, signing_region
* The CoreDNS configuration now has the forward plugin for proxy in the default configuration instead of the proxy plugin. ([#73267](https://github.com/kubernetes/kubernetes/pull/73267), [@rajansandeep](https://github.com/rajansandeep))
* Fixed a bug that caused PV allocation on non-English vSphere installations to fail ([#73115](https://github.com/kubernetes/kubernetes/pull/73115), [@alvaroaleman](https://github.com/alvaroaleman))
# v1.14.0-alpha.2
[Documentation](https://docs.k8s.io)
## Downloads for v1.14.0-alpha.2
filename | sha512 hash
-------- | -----------
[kubernetes.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes.tar.gz) | `1330e4421b61f6b1e6e4dee276d4742754bd3dd4493508d67ebb4445065277c619c4da8b4835febf0b2cdcf9e75fce96de1c1d99998904bae2bb794a453693f2`
[kubernetes-src.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-src.tar.gz) | `352c043bebf13a616441c920f3eec80d3f02f111d8488c31aa903e1483bce6d1fbe7472208f64730142960c8f778ab921ef7b654540a3ec09e53bd7e644521bd`
### Client Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-darwin-386.tar.gz) | `ee5aba4efce323167e6d897a2ff6962a240e466333bcae9390be2c8521c6da50ac2cb6139510b693aad49d6393b97a2118ed1fe4f999dd08bdca6d875d25f804`
[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-darwin-amd64.tar.gz) | `4b5c0b340322956a8d096c595124a765ac318d0eb460d6320218f2470e22d88221a0a9f1f93d5f3075f1c36b18c7041ee2fcb32e0f9c94d9f79bc3fd3005e68e`
[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-linux-386.tar.gz) | `7a5bfe68dd58c8478746a410872b615daf8abb9a78754140fb4d014a0c9177a87859ac046f56f5743fb97a9881abc2cf48c3e51aa02c8a86a754bf2cc59edb54`
[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-linux-amd64.tar.gz) | `c3139f58070241f2da815f701af3c0bd0ea4fdec1fe54bb859bd11237ac9b75ecb01b62ac1c7a459a4dd79696412c6d2f8cbd492fd062a790ceadd3dcc9b07fd`
[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-linux-arm.tar.gz) | `9d96d2e1e11aa61e2c3a5f4f27c18866feae9833b6ee70b15f5cdb5f992849dc1f79821af856b467487092a21a447231fb9c4de6ee6f17defed3cfa16d35b4c6`
[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-linux-arm64.tar.gz) | `7b4dd825cf9f217c18b28976a3faa94f0bd4868e541e5be7d57cd770e2b163c6daddf12e5f9ad51d92abde794a444f2a20bf582a30f03c39e60186d356030a2d`
[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-linux-ppc64le.tar.gz) | `490638e250c24b6bad8b67358fd7890f7a2f6456ae8ffe537c28bb5b3ce7abc591e6fecbddd6744f0f6c0e24b9f44c31f7ca1f7ebfc3c0d17a96fe8cf27b8548`
[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-linux-s390x.tar.gz) | `9dd8c3361eda15dd1594066c55b79cb9a34578c225b2b48647cd5b34619cf23106b845ee25b80d979f8b69e8733148842177500dc48989177b6944677f071f1c`
[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-windows-386.tar.gz) | `d624b8aead053201765b713d337528be82a71328ee3dd569f556868ceeb4904e64584892a016d247608fc4521c00ead7aed5d973b1206caa2d00406532d5b8b4`
[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-client-windows-amd64.tar.gz) | `a1cf8c67984dd4eb4610fa05d27fe9e9e4123159f933e3986e9db835b9cf136962168f0003071001e01e2c1831804ba0a366f2495741aa60a41587a69c09cb62`
### Server Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-server-linux-amd64.tar.gz) | `b93982b56371994c540cd11e6bc21808279340617164992c10f30d8e6ae4d5e270e41c1edc0625d3458a18944ec7aa8c273acbbcd718d60b6cacbc24220c42ac`
[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-server-linux-arm.tar.gz) | `bfd76c6b26e5927166d776f6110b97ee36c1d63ad39e2d18899f3e428ebb0f9615bb677ac8e9bcc1864c72a40efd71e1314fe6d137f9c6e54f720270929e3f46`
[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-server-linux-arm64.tar.gz) | `6721dec0df9466cd6c056160c73d598296cebb0af9259eb21b693abb8708901bc8bc30e11815e14d00d6eb12b8bb90b699e3119b922da855e2c411bdf229d6e5`
[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-server-linux-ppc64le.tar.gz) | `f8cd307db8141d989ae1218dd2b438bc9cee017d533b1451d2345f9689c451fdb080acd1b9b2f535ed04017e44b81a0585072e7d58a9d201a0ec28fd09df0a6f`
[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-server-linux-s390x.tar.gz) | `de7514bbd87a1b363e1bc7787f37d5ea10faac4afe7c5163c23c4df16781aa77570ec553bc4f4b6094166c1fcfc3c431f13e51ffa32f7ea2849e76ec0151ea35`
### Node Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-node-linux-amd64.tar.gz) | `8c37fd2fe6232d2c148e23df021b8b5347136263399932bcdff0c7a0186f3145de9ede4936b14de7484cc6db9241517d79b5306c380ed374396882900b63e912`
[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-node-linux-arm.tar.gz) | `389e4e77ab9e62968a25b8f4e146a2c3fbb3db2e60e051922edf6395c26cc5380e5a77bf67022339d6ebfe9abd714636d77510bbc42924b4265fdb245fae08c9`
[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-node-linux-arm64.tar.gz) | `7efc32dfeefcef7f860913c25431bd891a435e92cb8d5a95f8deca1a82aa899a007d4b19134493694a4bccb5564867488634a780c128f0cf82c61d98afa889f5`
[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-node-linux-ppc64le.tar.gz) | `da30c03bca4b81d810a7df006db02333dea87e336d6cdca9c93392e01c7e43bf4902c969efa7fa53e8a70a0e863b403ec26b87bd38226b8b9f98777ddb0051a0`
[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-node-linux-s390x.tar.gz) | `cce43b7f0350b9e5a77ea703225adb9714ef022d176db5b99a0327937d19021d7a8e93ef1169389fd53b895bb98725d23c7565ef80afdd17596c26daf41eeeac`
[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.2/kubernetes-node-windows-amd64.tar.gz) | `d3accf522d80cbfb3d03e9eaa60a09767ba11e88a8a5b44a629192a7c6916b1fb3440f022a5ffc4ea78f3595f254a42f028dd428d117360091cd0c747ec39eb5`
## Changelog since v1.14.0-alpha.1
### Action Required
* Promote ValidateProxyRedirects to Beta, and enable by default. This feature restricts redirect following from the apiserver to same-host redirects. ([#72552](https://github.com/kubernetes/kubernetes/pull/72552), [@tallclair](https://github.com/tallclair))
* ACTION REQUIRED: If nodes are configured to respond to CRI streaming requests on a different host interface than what the apiserver makes requests on (only the case if not using the built-in dockershim & setting the kubelet flag `--redirect-container-streaming=true`), then these requests will be broken. In that case, the feature can be temporarily disabled until the node configuration is corrected. We suggest setting `--redirect-container-streaming=false` on the kubelet to avoid issues.
### Other notable changes
* Added alpha field storageVersionHash to the discovery document for each resource. Its value must be treated as opaque by clients. Only equality comparison on the value is valid. ([#73191](https://github.com/kubernetes/kubernetes/pull/73191), [@caesarxuchao](https://github.com/caesarxuchao))
* Fix admission metrics in seconds. ([#72343](https://github.com/kubernetes/kubernetes/pull/72343), [@danielqsj](https://github.com/danielqsj))
* Add metrics `*_admission_latencies_milliseconds` and `*_admission_latencies_milliseconds_summary` for backward compatible, but will be removed in a future release.
* Pod eviction now honors graceful deletion by default if no delete options are provided in the eviction request ([#72730](https://github.com/kubernetes/kubernetes/pull/72730), [@liggitt](https://github.com/liggitt))
* Update to go1.11.5 ([#73326](https://github.com/kubernetes/kubernetes/pull/73326), [@ixdy](https://github.com/ixdy))
* Change proxy metrics to conform metrics guidelines. ([#72334](https://github.com/kubernetes/kubernetes/pull/72334), [@danielqsj](https://github.com/danielqsj))
* The metrics `sync_proxy_rules_latency_microseconds` is deprecated, and will be removed in a future release, please convert to metrics`sync_proxy_rules_latency_seconds`.
* Add network stats for Windows nodes and pods. ([#70121](https://github.com/kubernetes/kubernetes/pull/70121), [@feiskyer](https://github.com/feiskyer))
* kubeadm: When certificates are present joining a new control plane make sure that they match at least the required SANs ([#73093](https://github.com/kubernetes/kubernetes/pull/73093), [@ereslibre](https://github.com/ereslibre))
* A new `TaintNodesByCondition` admission plugin taints newly created Node objects as "not ready", to fix a race condition that could cause pods to be scheduled on new nodes before their taints were updated to accurately reflect their reported conditions. This admission plugin is enabled by default if the `TaintNodesByCondition` feature is enabled. ([#73097](https://github.com/kubernetes/kubernetes/pull/73097), [@bsalamat](https://github.com/bsalamat))
* kube-addon-manager was updated to v9.0, and now uses kubectl v1.13.2 and prunes workload resources via the apps/v1 API ([#72978](https://github.com/kubernetes/kubernetes/pull/72978), [@liggitt](https://github.com/liggitt))
* When a watch is closed by an HTTP2 load balancer and we are told to go away, skip printing the message to stderr by default. ([#73277](https://github.com/kubernetes/kubernetes/pull/73277), [@smarterclayton](https://github.com/smarterclayton))
* If you are running the cloud-controller-manager and you have the `pvlabel.kubernetes.io` alpha Initializer enabled, you must now enable PersistentVolume labeling using the `PersistentVolumeLabel` admission controller instead. You can do this by adding `PersistentVolumeLabel` in the `--enable-admission-plugins` kube-apiserver flag. ([#73102](https://github.com/kubernetes/kubernetes/pull/73102), [@andrewsykim](https://github.com/andrewsykim))
* The alpha Initializers feature, `admissionregistration.k8s.io/v1alpha1` API version, `Initializers` admission plugin, and use of the `metadata.initializers` API field have been removed. Discontinue use of the alpha feature and delete any existing `InitializerConfiguration` API objects before upgrading. The `metadata.initializers` field will be removed in a future release. ([#72972](https://github.com/kubernetes/kubernetes/pull/72972), [@liggitt](https://github.com/liggitt))
* Scale max-inflight limits together with master VM sizes. ([#73268](https://github.com/kubernetes/kubernetes/pull/73268), [@wojtek-t](https://github.com/wojtek-t))
* kubectl supports copying files with wild card ([#72641](https://github.com/kubernetes/kubernetes/pull/72641), [@dixudx](https://github.com/dixudx))
* kubeadm: add back `--cert-dir` option for `kubeadm init phase certs sa` ([#73239](https://github.com/kubernetes/kubernetes/pull/73239), [@mattkelly](https://github.com/mattkelly))
* Remove deprecated args '--show-all' ([#69255](https://github.com/kubernetes/kubernetes/pull/69255), [@Pingan2017](https://github.com/Pingan2017))
* As per deprecation policy in https://kubernetes.io/docs/reference/using-api/deprecation-policy/ ([#73001](https://github.com/kubernetes/kubernetes/pull/73001), [@shivnagarajan](https://github.com/shivnagarajan))
* the taints "node.alpha.kubernetes.io/notReady" and "node.alpha.kubernetes.io/unreachable". are no
* longer supported or adjusted. These uses should be replaced with "node.kubernetes.io/not-ready"
* and "node.kubernetes.io/unreachable" respectively instead.
* The /swagger.json and /swagger-2.0.0.pb-v1 schema documents, deprecated since v1.10, have been removed in favor of `/openapi/v2` ([#73148](https://github.com/kubernetes/kubernetes/pull/73148), [@liggitt](https://github.com/liggitt))
* CoreDNS is only officially supported on Linux at this time. As such, when kubeadm is used to deploy this component into your kubernetes cluster, it will be restricted (using nodeSelectors) to run only on nodes with that operating system. This ensures that in clusters which include Windows nodes, the scheduler will not ever attempt to place CoreDNS pods on these machines, reducing setup latency and enhancing initial cluster stability. ([#69940](https://github.com/kubernetes/kubernetes/pull/69940), [@MarcPow](https://github.com/MarcPow))
* kubeadm now attempts to detect an installed CRI by its usual domain socket, so that --cri-socket can be omitted from the command line if Docker is not used and there is a single CRI installed. ([#69366](https://github.com/kubernetes/kubernetes/pull/69366), [@rosti](https://github.com/rosti))
* scheduler: makes pod less racing so as to be put back into activeQ properly ([#73078](https://github.com/kubernetes/kubernetes/pull/73078), [@Huang-Wei](https://github.com/Huang-Wei))
* jsonpath expressions containing `[start:end:step]` slice are now evaluated correctly ([#73149](https://github.com/kubernetes/kubernetes/pull/73149), [@liggitt](https://github.com/liggitt))
* metadata.deletionTimestamp is no longer moved into the future when issuing repeated DELETE requests against a resource containing a finalizer. ([#73138](https://github.com/kubernetes/kubernetes/pull/73138), [@liggitt](https://github.com/liggitt))
* The "kubectl api-resources" command will no longer fail to display any resources on a single failure ([#73035](https://github.com/kubernetes/kubernetes/pull/73035), [@juanvallejo](https://github.com/juanvallejo))
* e2e tests that require SSH may be used against clusters that have nodes without external IP addresses by setting the environment variable `KUBE_SSH_BASTION` to the `host:port` of a machine that is allowed to SSH to those nodes. The same private key that the test would use is used for the bastion host. The test connects to the bastion and then tunnels another SSH connection to the node. ([#72286](https://github.com/kubernetes/kubernetes/pull/72286), [@smarterclayton](https://github.com/smarterclayton))
* kubeadm: explicitly wait for `etcd` to have grown when joining a new control plane ([#72984](https://github.com/kubernetes/kubernetes/pull/72984), [@ereslibre](https://github.com/ereslibre))
* Install CSINodeInfo and CSIDriver CRDs in the local cluster. ([#72584](https://github.com/kubernetes/kubernetes/pull/72584), [@xing-yang](https://github.com/xing-yang))
* kubectl loads config file once and uses persistent client config ([#71117](https://github.com/kubernetes/kubernetes/pull/71117), [@dixudx](https://github.com/dixudx))
* remove stale OutOfDisk condition from kubelet side ([#72507](https://github.com/kubernetes/kubernetes/pull/72507), [@dixudx](https://github.com/dixudx))
* Node OS/arch labels are promoted to GA ([#73048](https://github.com/kubernetes/kubernetes/pull/73048), [@yujuhong](https://github.com/yujuhong))
* Fix graceful apiserver shutdown to not drop outgoing bytes before the process terminates. ([#72970](https://github.com/kubernetes/kubernetes/pull/72970), [@sttts](https://github.com/sttts))
* Change apiserver metrics to conform metrics guidelines. ([#72336](https://github.com/kubernetes/kubernetes/pull/72336), [@danielqsj](https://github.com/danielqsj))
* The following metrics are deprecated, and will be removed in a future release:
* `apiserver_request_count`
* `apiserver_request_latencies`
* `apiserver_request_latencies_summary`
* `apiserver_dropped_requests`
* `etcd_helper_cache_hit_count`
* `etcd_helper_cache_miss_count`
* `etcd_helper_cache_entry_count`
* `etcd_request_cache_get_latencies_summary`
* `etcd_request_cache_add_latencies_summary`
* `etcd_request_latencies_summary`
* `transformation_latencies_microseconds `
* `data_key_generation_latencies_microseconds`
* Please convert to the following metrics:
* `apiserver_request_total`
* `apiserver_request_latency_seconds`
* `apiserver_dropped_requests_total`
* `etcd_helper_cache_hit_total`
* `etcd_helper_cache_miss_total`
* `etcd_helper_cache_entry_total`
* `etcd_request_cache_get_latency_seconds`
* `etcd_request_cache_add_latency_seconds`
* `etcd_request_latency_seconds`
* `transformation_latencies_seconds`
* `data_key_generation_latencies_seconds`
* acquire lock before operating unschedulablepodsmap ([#73022](https://github.com/kubernetes/kubernetes/pull/73022), [@denkensk](https://github.com/denkensk))
* Print `SizeLimit` of `EmptyDir` in `kubectl describe pod` outputs. ([#69279](https://github.com/kubernetes/kubernetes/pull/69279), [@dtaniwaki](https://github.com/dtaniwaki))
* add goroutine to move unschedulable pods to activeq if they are not retried for more than 1 minute ([#72558](https://github.com/kubernetes/kubernetes/pull/72558), [@denkensk](https://github.com/denkensk))
* PidPressure evicts pods from lowest priority to highest priority ([#72844](https://github.com/kubernetes/kubernetes/pull/72844), [@dashpole](https://github.com/dashpole))
* Reduce GCE log rotation check from 1 hour to every 5 minutes. Rotation policy is unchanged (new day starts, log file size > 100MB). ([#72062](https://github.com/kubernetes/kubernetes/pull/72062), [@jpbetz](https://github.com/jpbetz))
* Add support for max attach limit for Cinder ([#72980](https://github.com/kubernetes/kubernetes/pull/72980), [@gnufied](https://github.com/gnufied))
* Fixes the setting of NodeAddresses when using the vSphere CloudProvider and nodes that have multiple IP addresses. ([#70805](https://github.com/kubernetes/kubernetes/pull/70805), [@danwinship](https://github.com/danwinship))
* kubeadm: pull images when joining a new control plane instance ([#72870](https://github.com/kubernetes/kubernetes/pull/72870), [@MalloZup](https://github.com/MalloZup))
* Enable mTLS encription between etcd and kube-apiserver in GCE ([#70144](https://github.com/kubernetes/kubernetes/pull/70144), [@wenjiaswe](https://github.com/wenjiaswe))
* The `/swaggerapi/*` schema docs, deprecated since 1.7, have been removed in favor of the /openapi/v2 schema docs. ([#72924](https://github.com/kubernetes/kubernetes/pull/72924), [@liggitt](https://github.com/liggitt))
* Allow users to use Docker 18.09 with kubeadm ([#72823](https://github.com/kubernetes/kubernetes/pull/72823), [@dims](https://github.com/dims))
# v1.14.0-alpha.1
[Documentation](https://docs.k8s.io)
## Downloads for v1.14.0-alpha.1
filename | sha512 hash
-------- | -----------
[kubernetes.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes.tar.gz) | `fac80e5674e547d00987516fb2eca6ea9947529307566be6a12932e3c9e430e8ad094afae748f31e9574838d98052423e3634a067f1456f7c13f6b27bfa63bcc`
[kubernetes-src.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-src.tar.gz) | `d1b5b2c15cb0daa076606f4ccf887724b0166dee0320f2a61d16ab4689931ab0cf5dac4c499aea3d434eb96d589d2b3effe0037e2244978d4290bd19b9a3edea`
### Client Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-darwin-386.tar.gz) | `307c426e4abaf81648af393ddd641c225d87b02d8662d1309fe3528f14ed91b2470f6b46dc8ce0459cf196e2cec906f7eb972bf4c9a96cbd570e206f5a059dca`
[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-darwin-amd64.tar.gz) | `8daa85f3e8feaea0d55f20f850038dd113f0f08b62eef944b08a9109d4e69f323a8fcf20c12790c78386b454148bcc9a0cdf106ba3393620709d185c291887fa`
[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-linux-386.tar.gz) | `28d73c299cb9859fdfeb3e4869a7a9c77f5679309c2613bd2c72d92dafd5faad0653a7377616190edd29cb8fa1aff104daba98f398e72f3447a132f208dde756`
[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-linux-amd64.tar.gz) | `eb923e13026f80b743a57100d4f94995f322ab6f107c34ffd9aa74b5a6c6a4a410aff8921a4f675ace7db2ff8158a90874b8f56d3142ad2cbe615c11ec2d4535`
[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-linux-arm.tar.gz) | `279b0d0c560900021abea4bbfc25aeca7389f0b37d80022dc3335147344663424e7ba6a0abecb2dca1d2facb4163e26080750736a9a1932d67422f88b0940679`
[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-linux-arm64.tar.gz) | `d69d28361b9c9e16f3e6804ccda92d55ee743e63aba7fded04edf1f7202b1fa96c235e36ab2ca17df99b4aede80b92150790885bdb7f5b4d7956af3c269dd83c`
[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-linux-ppc64le.tar.gz) | `ca6ebb87df98bf179c94f54a4e8ae2ef2ea534b1bc5014331f937aa9d4c0442d5423651457871ef5c51f481ba8a3f449d69ef7e42e49c1b313f66cff3d44926f`
[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-linux-s390x.tar.gz) | `13fa2058ceba66d8da5ba5982aa302cdd1c61d15253183ab97739229584a178f057f7979b49a035cb2355197dbb388d1642939e2c002b10e23263127030022ab`
[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-windows-386.tar.gz) | `42ba4bba477e2958aab674a0fbf888bd5401fa5fbc39466b6cad0fc97e249ac949042c513bf176957bcb336a906e612d9c6790215e78c280225351236ec96993`
[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-client-windows-amd64.tar.gz) | `d5f339fe4d37c61babc97208446d1859423b7679f34040f72e9138b72a18d982e66732d1f4b4f3443700f9cbe96bfc0e12eaec0a8a373fb903b49efdafcbae04`
### Server Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-server-linux-amd64.tar.gz) | `bcbcbd3ac4419e54e894d1e595f883e61fcf9db0353a30d794a9e5030cde8957abe8124fa5265e8c52fbc93f07cfe79b2493f791dc225468bf927b7ab4694087`
[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-server-linux-arm.tar.gz) | `fda4ea9168555f724659601b06737dea6ec95574569df4ef7e4ab6c2cca3327623ef310bf34f792767f00ee8069b9dd83564835d43daf973087be816be40010b`
[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-server-linux-arm64.tar.gz) | `c142857711ec698844cd61188e70b5ab185ba2c8828cf5563a2f42958489e2ae4dbb2c1626271d4f5582167bb363e55ed03afb15e7e86cd414e0dc049fe384c0`
[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-server-linux-ppc64le.tar.gz) | `524a40c5717b24c5a3b2491c4c61cf3038ba5ae7f343797a1b56a5906d6a0a3eb57e9ae78590c28ac3d441d9d1bb480a0c264a07e009a4365503ad2357614aa8`
[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-server-linux-s390x.tar.gz) | `ef943fe326b05ece57f2e409ab1cc5fe863f5effa591abae17181c84a5eb4061e9f394ffcc8ee6ebb3f5165b183bab747a8cef540cbb1436343e8180cec037e0`
### Node Binaries
filename | sha512 hash
-------- | -----------
[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-node-linux-amd64.tar.gz) | `396f7588e9131dd1b99d101c8bb94fb7e67ab067327ee58dab5a6e24887d8fbb6fc78fe50804abb0ab2f626034881d4280b3f678a1fd8b34891762bf2172b268`
[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-node-linux-arm.tar.gz) | `b75c1550438da0b66582d6de90436ee3c44e41e67f74947d93ee9a07ed2b7757762f3f2b05bd7b5589d7e1ea2eb3616b2ef4fe59a9fbe9d8e7cb8f0c9d3dd158`
[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-node-linux-arm64.tar.gz) | `b6c46f9250b5565fa178ecc99ffedc6724b0bfffb73acc7d3da2c678af71008a264502cc4a48a6e7452bd0a60d77194141bbc2ea9af49176ea66e27d874b77ac`
[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-node-linux-ppc64le.tar.gz) | `8d505c61a59bc9fc53d6f219d6434ddd962ba383654c46e16d413cee0ad6bd26f276a9860ad3680349bcfacb361e75de07fc44f7d14c054c47b6bd0eae63615f`
[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-node-linux-s390x.tar.gz) | `83b6cf0fb348faa93fa40ec2a947b202b3a5a2081c3896ae39618f947a57b431bc774fbe3a5437719f50f002de252438dc16bac6f632c11140f55d5051094ae6`
[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.0-alpha.1/kubernetes-node-windows-amd64.tar.gz) | `43471680533685c534023787cd40431b67041bab43e93dea457283ee0f08a8fa02ee9ade3737d8e64d1d3255a281af9a107cb61f9e4d9c99dee188c82a075580`
## Changelog since v1.13.0
### Action Required
* action required ([#68753](https://github.com/kubernetes/kubernetes/pull/68753), [@johnSchnake](https://github.com/johnSchnake))
* If you are running E2E tests which require SSH keys and you utilize environment variables to override their location, you may need to modify the environment variable set. On all providers the environment variable override can now be either an absolute path to the key or a relative path (relative to ~/.ssh). Specifically the changes are:
* - Created new GCE_SSH_KEY allowing specification of SSH keys for gce, gke, and kubemark.
* - AWS_SSH_KEY, previously assumed to be an absolute path can now be either relative or absolute
* - LOCAL_SSH_KEY (for local and vsphere providers) was previously assumed to be a filename relative to ~/.ssh but can now also be an absolute path
* - KUBE_SSH_KEY (for skeleton provider) was previously assumed to be a filename relative to ~/.ssh but can now also be an absolute path
### Other notable changes
* Connections from Pods to Services with 0 endpoints will now ICMP reject immediately, rather than blackhole and timeout. ([#72534](https://github.com/kubernetes/kubernetes/pull/72534), [@thockin](https://github.com/thockin))
* Improve efficiency of preemption logic in clusters with many pending pods. ([#72895](https://github.com/kubernetes/kubernetes/pull/72895), [@bsalamat](https://github.com/bsalamat))
* Change scheduler metrics to conform metrics guidelines. ([#72332](https://github.com/kubernetes/kubernetes/pull/72332), [@danielqsj](https://github.com/danielqsj))
* The following metrics are deprecated, and will be removed in a future release:
* `e2e_scheduling_latency_microseconds`
* `scheduling_algorithm_latency_microseconds`
* `scheduling_algorithm_predicate_evaluation`
* `scheduling_algorithm_priority_evaluation`
* `scheduling_algorithm_preemption_evaluation`
* `binding_latency_microseconds`
* Please convert to the following metrics:
* `e2e_scheduling_latency_seconds`
* `scheduling_algorithm_latency_seconds`
* `scheduling_algorithm_predicate_evaluation_seconds`
* `scheduling_algorithm_priority_evaluation_seconds`
* `scheduling_algorithm_preemption_evaluation_seconds`
* `binding_latency_seconds`
* Fix SelectorSpreadPriority scheduler to match all selectors when distributing pods. ([#72801](https://github.com/kubernetes/kubernetes/pull/72801), [@Ramyak](https://github.com/Ramyak))
* Add bootstrap service account & cluster roles for node-lifecycle-controller, cloud-node-lifecycle-controller, and cloud-node-controller. ([#72764](https://github.com/kubernetes/kubernetes/pull/72764), [@andrewsykim](https://github.com/andrewsykim))
* Fixes spurious 0-length API responses. ([#72856](https://github.com/kubernetes/kubernetes/pull/72856), [@liggitt](https://github.com/liggitt))
* Updates Fluentd to 1.3.2 & added filter_parser ([#71180](https://github.com/kubernetes/kubernetes/pull/71180), [@monotek](https://github.com/monotek))
* The leaderelection package allows the lease holder to release its lease when the calling context is cancelled. This allows ([#71490](https://github.com/kubernetes/kubernetes/pull/71490), [@smarterclayton](https://github.com/smarterclayton))
* faster handoff when a leader-elected process is gracefully terminated.
* Make volume binder resilient to races between main schedule loop and async binding operation ([#72045](https://github.com/kubernetes/kubernetes/pull/72045), [@cofyc](https://github.com/cofyc))
* Bump minimum docker API version to 1.26 (1.13.1) ([#72831](https://github.com/kubernetes/kubernetes/pull/72831), [@yujuhong](https://github.com/yujuhong))
* If the `TokenRequestProjection` feature gate is disabled, projected serviceAccountToken volume sources are now dropped at object creation time, or at object update time if the existing object did not have a projected serviceAccountToken volume source. Previously, these would result in validation errors. ([#72714](https://github.com/kubernetes/kubernetes/pull/72714), [@mourya007](https://github.com/mourya007))
* Add `metrics-port` to kube-proxy cmd flags. ([#72682](https://github.com/kubernetes/kubernetes/pull/72682), [@whypro](https://github.com/whypro))
* kubectl: fixed an issue with "too old resource version" errors continuously appearing when calling `kubectl delete` ([#72825](https://github.com/kubernetes/kubernetes/pull/72825), [@liggitt](https://github.com/liggitt))
* [Breaking change, client-go]: The WaitFor function returns, probably an ErrWaitTimeout, when the done channel is closed, even if the `WaitFunc` doesn't handle the done channel. ([#72364](https://github.com/kubernetes/kubernetes/pull/72364), [@kdada](https://github.com/kdada))
* removes newline from json output for windows nodes [#72657](https://github.com/kubernetes/kubernetes/pull/72657) ([#72659](https://github.com/kubernetes/kubernetes/pull/72659), [@jsturtevant](https://github.com/jsturtevant))
* The DenyEscalatingExec and DenyExecOnPrivileged admission plugins are deprecated and will be removed in v1.18. Use of `PodSecurityPolicy` or a custom admission plugin to limit creation of pods is recommended instead. ([#72737](https://github.com/kubernetes/kubernetes/pull/72737), [@liggitt](https://github.com/liggitt))
* Fix `describe statefulset` not printing number of desired replicas correctly ([#72781](https://github.com/kubernetes/kubernetes/pull/72781), [@tghartland](https://github.com/tghartland))
* Fix kube-proxy PodSecurityPolicy binding on GCE & GKE. This was only an issue when running kube-proxy as a DaemonSet, with PodSecurityPolicy enabled. ([#72761](https://github.com/kubernetes/kubernetes/pull/72761), [@tallclair](https://github.com/tallclair))
* Drops `status.Conditions` of new `PersistentVolume` objects if it was not set on the old object during `PrepareForUpdate`. ([#72739](https://github.com/kubernetes/kubernetes/pull/72739), [@rajathagasthya](https://github.com/rajathagasthya))
* kubelet: fixes cadvisor internal error when "--container-runtime-endpoint" is set to "unix:///var/run/crio/crio.sock". ([#72340](https://github.com/kubernetes/kubernetes/pull/72340), [@makocchi-git](https://github.com/makocchi-git))
* The `spec.SecurityContext.Sysctls` field is now dropped during creation of `Pod` objects unless the `Sysctls` feature gate is enabled. ([#72752](https://github.com/kubernetes/kubernetes/pull/72752), [@rajathagasthya](https://github.com/rajathagasthya))
* The `spec.AllowedUnsafeSysctls` and `spec.ForbiddenSysctls` fields are now dropped during creation of `PodSecurityPolicy` objects unless the `Sysctls` feature gate is enabled.
* kubeadm: fixed storing of front-proxy certificate in secrets required by kube-controller-manager selfhosting pivoting ([#72727](https://github.com/kubernetes/kubernetes/pull/72727), [@bart0sh](https://github.com/bart0sh))
* Administrator is able to configure max pids for a pod on a node. ([#72076](https://github.com/kubernetes/kubernetes/pull/72076), [@derekwaynecarr](https://github.com/derekwaynecarr))
* Move users of `factory.NewConfigFactory` to `scheduler.New`. ([#71875](https://github.com/kubernetes/kubernetes/pull/71875), [@wgliang](https://github.com/wgliang))
* The `spec.SecurityContext.ShareProcessNamespace` field is now dropped during creation of `Pod` objects unless the `PodShareProcessNamespace ` feature gate is enabled. ([#72698](https://github.com/kubernetes/kubernetes/pull/72698), [@rajathagasthya](https://github.com/rajathagasthya))
* kube-apiserver: When configuring integration with external KMS Providers, users can supply timeout value (i.e. how long should kube-apiserver wait before giving up on a call to KMS). ([@immutableT](https://github.com/immutableT) ) ([#72540](https://github.com/kubernetes/kubernetes/pull/72540), [@immutableT](https://github.com/immutableT))
* The `spec.readinessGates` field is now dropped during creation of `Pod` objects unless the `PodReadinessGates` feature gate is enabled. ([#72695](https://github.com/kubernetes/kubernetes/pull/72695), [@rajathagasthya](https://github.com/rajathagasthya))
* The `spec.dataSource` field is now dropped during creation of PersistentVolumeClaim objects unless the `VolumeSnapshotDataSource` feature gate is enabled. ([#72666](https://github.com/kubernetes/kubernetes/pull/72666), [@rajathagasthya](https://github.com/rajathagasthya))
* Stop kubelet logging a warning to override hostname if there's no change detected. ([#71560](https://github.com/kubernetes/kubernetes/pull/71560), [@KashifSaadat](https://github.com/KashifSaadat))
* client-go: fake clients now properly return NotFound errors when attempting to patch non-existent objects ([#70886](https://github.com/kubernetes/kubernetes/pull/70886), [@bouk](https://github.com/bouk))
* kubectl: fixes a bug determining the correct namespace while running in a pod when the `--context` flag is explicitly specified, and the referenced context specifies the namespace `default` ([#72529](https://github.com/kubernetes/kubernetes/pull/72529), [@liggitt](https://github.com/liggitt))
* Fix scheduling starvation of pods in cluster with large number of unschedulable pods. ([#72619](https://github.com/kubernetes/kubernetes/pull/72619), [@everpeace](https://github.com/everpeace))
* If the AppArmor feature gate is disabled, AppArmor-specific annotations in pod and pod templates are dropped when the object is created, and during update of objects that do not already contain AppArmor annotations, rather than triggering a validation error. ([#72655](https://github.com/kubernetes/kubernetes/pull/72655), [@liggitt](https://github.com/liggitt))
* client-go: shortens refresh period for token files to 1 minute to ensure auto-rotated projected service account tokens are read frequently enough. ([#72437](https://github.com/kubernetes/kubernetes/pull/72437), [@liggitt](https://github.com/liggitt))
* Multiple tests which previously failed due to lack of external IP addresses defined on the nodes should now be passable. ([#68792](https://github.com/kubernetes/kubernetes/pull/68792), [@johnSchnake](https://github.com/johnSchnake))
* kubeadm: fixed incorrect controller manager pod mutations during selfhosting pivoting ([#72518](https://github.com/kubernetes/kubernetes/pull/72518), [@bart0sh](https://github.com/bart0sh))
* Increase Azure default maximumLoadBalancerRuleCount to 250. ([#72621](https://github.com/kubernetes/kubernetes/pull/72621), [@feiskyer](https://github.com/feiskyer))
* RuntimeClass is now printed with extra `RUNTIME-HANDLER` column. ([#72446](https://github.com/kubernetes/kubernetes/pull/72446), [@Huang-Wei](https://github.com/Huang-Wei))
* Updates the kubernetes dashboard add-on to v1.10.1. Skipping dashboard login is no longer enabled by default. ([#72495](https://github.com/kubernetes/kubernetes/pull/72495), [@liggitt](https://github.com/liggitt))
* [GCP] Remove confusing error log entry form fluentd scalers. ([#72243](https://github.com/kubernetes/kubernetes/pull/72243), [@cezarygerard](https://github.com/cezarygerard))
* change azure disk host cache to ReadOnly by default ([#72229](https://github.com/kubernetes/kubernetes/pull/72229), [@andyzhangx](https://github.com/andyzhangx))
* Nodes deleted in the cloud provider with Ready condition `Unknown` should also be deleted on the API server. ([#72559](https://github.com/kubernetes/kubernetes/pull/72559), [@andrewsykim](https://github.com/andrewsykim))
* `kubectl apply --prune` now uses the apps/v1 API to prune workload resources ([#72352](https://github.com/kubernetes/kubernetes/pull/72352), [@liggitt](https://github.com/liggitt))
* Fixes a bug in HPA controller so HPAs are always updated every resyncPeriod (15 seconds). ([#72373](https://github.com/kubernetes/kubernetes/pull/72373), [@krzysztof-jastrzebski](https://github.com/krzysztof-jastrzebski))
* IPVS: "ExternalTrafficPolicy: Local" now works with LoadBalancer services using loadBalancerIP ([#72432](https://github.com/kubernetes/kubernetes/pull/72432), [@lbernail](https://github.com/lbernail))
* Fixes issue with cleaning up stale NFS subpath mounts ([#71804](https://github.com/kubernetes/kubernetes/pull/71804), [@msau42](https://github.com/msau42))
* Modify the scheduling result struct and improve logging for successful binding. ([#71926](https://github.com/kubernetes/kubernetes/pull/71926), [@wgliang](https://github.com/wgliang))
* Run one etcd storage compaction per default interval of 5min. Do not run one for each resource and each CRD. This fixes the compaction log spam and reduces load on etcd. ([#68557](https://github.com/kubernetes/kubernetes/pull/68557), [@sttts](https://github.com/sttts))
* kube-apiserver: `--runtime-config` can once again be used to enable/disable serving specific resources in the `extensions/v1beta1` API group. Note that specific resource enablement/disablement is only allowed for the `extensions/v1beta1` API group for legacy reasons. Attempts to enable/disable individual resources in other API groups will print a warning, and will return an error in future releases. ([#72249](https://github.com/kubernetes/kubernetes/pull/72249), [@liggitt](https://github.com/liggitt))
* kubeadm: fixed storing of etcd certificates in secrets required by kube-apiserver selfhosting pivoting ([#72478](https://github.com/kubernetes/kubernetes/pull/72478), [@bart0sh](https://github.com/bart0sh))
* kubeadm: remove the deprecated "--address" flag for controller-manager and scheduler. ([#71973](https://github.com/kubernetes/kubernetes/pull/71973), [@MalloZup](https://github.com/MalloZup))
* kube-apiserver: improves performance of requests made with service account token authentication ([#71816](https://github.com/kubernetes/kubernetes/pull/71816), [@liggitt](https://github.com/liggitt))
* Use prometheus conventions for workqueue metrics. ([#71300](https://github.com/kubernetes/kubernetes/pull/71300), [@danielqsj](https://github.com/danielqsj))
* It is now deprecated to use the following metrics:
* `{WorkQueueName}_depth`
* `{WorkQueueName}_adds`
* `{WorkQueueName}_queue_latency`
* `{WorkQueueName}_work_duration`
* `{WorkQueueName}_unfinished_work_seconds`
* `{WorkQueueName}_longest_running_processor_microseconds`
* `{WorkQueueName}_retries`
* Please convert to the following metrics:
* `workqueue_depth`
* `workqueue_adds_total`
* `workqueue_queue_latency_seconds`
* `workqueue_work_duration_seconds`
* `workqueue_unfinished_work_seconds`
* `workqueue_longest_running_processor_seconds`
* `workqueue_retries_total`
* Fix inability to use k8s with dockerd having default IPC mode set to private. ([#70826](https://github.com/kubernetes/kubernetes/pull/70826), [@kolyshkin](https://github.com/kolyshkin))
* Fix a race condition in the scheduler preemption logic that could cause nominatedNodeName of a pod not to be considered in one or more scheduling cycles. ([#72259](https://github.com/kubernetes/kubernetes/pull/72259), [@bsalamat](https://github.com/bsalamat))
* Fix registration for scheduling framework plugins with the default plugin set ([#72396](https://github.com/kubernetes/kubernetes/pull/72396), [@y-taka-23](https://github.com/y-taka-23))
* The GA VolumeScheduling feature gate can no longer be disabled and will be removed in a future release ([#72382](https://github.com/kubernetes/kubernetes/pull/72382), [@liggitt](https://github.com/liggitt))
* Fix race condition introduced by graceful termination which can lead to a deadlock in kube-proxy ([#72361](https://github.com/kubernetes/kubernetes/pull/72361), [@lbernail](https://github.com/lbernail))
* Fixes issue where subpath volume content was deleted during orphaned pod cleanup for Local volumes that are directories (and not mount points) on the root filesystem. ([#72291](https://github.com/kubernetes/kubernetes/pull/72291), [@msau42](https://github.com/msau42))
* Fixes `kubectl create secret docker-registry` compatibility ([#72344](https://github.com/kubernetes/kubernetes/pull/72344), [@liggitt](https://github.com/liggitt))
* Add-on manifests now use the apps/v1 API for DaemonSets, Deployments, and ReplicaSets ([#72203](https://github.com/kubernetes/kubernetes/pull/72203), [@liggitt](https://github.com/liggitt))
* "kubectl wait" command now supports the "--all" flag to select all resources in the namespace of the specified resource types. ([#70599](https://github.com/kubernetes/kubernetes/pull/70599), [@caesarxuchao](https://github.com/caesarxuchao))
* `deployments/rollback` is now passed through validation/admission controllers ([#72271](https://github.com/kubernetes/kubernetes/pull/72271), [@jhrv](https://github.com/jhrv))
* The `Lease` API type in the `coordination.k8s.io` API group is promoted to `v1` ([#72239](https://github.com/kubernetes/kubernetes/pull/72239), [@wojtek-t](https://github.com/wojtek-t))
* Move compatibility_test.go to pkg/scheduler/api ([#72014](https://github.com/kubernetes/kubernetes/pull/72014), [@huynq0911](https://github.com/huynq0911))
* New Azure cloud provider option 'cloudProviderBackoffMode' has been added to reduce Azure API retries. Candidate values are: ([#70866](https://github.com/kubernetes/kubernetes/pull/70866), [@feiskyer](https://github.com/feiskyer))
* default (or empty string): keep same with before.
* v2: only backoff retry with Azure SDK with fixed exponent 2.
* Set percentage of nodes scored in each cycle dynamically based on the cluster size. ([#72140](https://github.com/kubernetes/kubernetes/pull/72140), [@wgliang](https://github.com/wgliang))
* Fix AAD support for Azure sovereign cloud in kubectl ([#72143](https://github.com/kubernetes/kubernetes/pull/72143), [@karataliu](https://github.com/karataliu))
* Make kube-proxy service abstraction optional. ([#71355](https://github.com/kubernetes/kubernetes/pull/71355), [@bradhoekstra](https://github.com/bradhoekstra))
* Add the 'service.kubernetes.io/service-proxy-name' label to a Service to disable the kube-proxy service proxy implementation.
* kubectl: `-A` can now be used as a shortcut for `--all-namespaces` ([#72006](https://github.com/kubernetes/kubernetes/pull/72006), [@soltysh](https://github.com/soltysh))
* discovery.CachedDiscoveryInterface implementation returned by NewMemCacheClient has changed semantics of Invalidate method -- the cache refresh is now deferred to the first cache lookup. ([#70994](https://github.com/kubernetes/kubernetes/pull/70994), [@mborsz](https://github.com/mborsz))
* Fix device mountable volume names in DSW to prevent races in device mountable plugin, e.g. local. ([#71509](https://github.com/kubernetes/kubernetes/pull/71509), [@cofyc](https://github.com/cofyc))
* Enable customize in kubectl: kubectl will be able to recognize directories with kustomization.YAML ([#70875](https://github.com/kubernetes/kubernetes/pull/70875), [@Liujingfang1](https://github.com/Liujingfang1))
* Stably sort controllerrevisions. This can prevent pods of statefulsets from continually rolling. ([#66882](https://github.com/kubernetes/kubernetes/pull/66882), [@ryanmcnamara](https://github.com/ryanmcnamara))
* Update to use go1.11.4. ([#72084](https://github.com/kubernetes/kubernetes/pull/72084), [@ixdy](https://github.com/ixdy))
* fixes an issue deleting pods containing subpath volume mounts with the VolumeSubpath feature disabled ([#70490](https://github.com/kubernetes/kubernetes/pull/70490), [@liggitt](https://github.com/liggitt))
* Clean up old eclass code ([#71399](https://github.com/kubernetes/kubernetes/pull/71399), [@resouer](https://github.com/resouer))
* Fix a race condition in which kubeadm only waits for the kubelets kubeconfig file when it has performed the TLS bootstrap, but wasn't waiting for certificates to be present in the filesystem ([#72030](https://github.com/kubernetes/kubernetes/pull/72030), [@ereslibre](https://github.com/ereslibre))
* In addition to restricting GCE metadata requests to known APIs, the metadata-proxy now restricts query strings to known parameters. ([#71094](https://github.com/kubernetes/kubernetes/pull/71094), [@dekkagaijin](https://github.com/dekkagaijin))
* kubeadm: fix a possible panic when joining a new control plane node in HA scenarios ([#72123](https://github.com/kubernetes/kubernetes/pull/72123), [@anitgandhi](https://github.com/anitgandhi))
* fix race condition when attach azure disk in vmss ([#71992](https://github.com/kubernetes/kubernetes/pull/71992), [@andyzhangx](https://github.com/andyzhangx))
* Update to use go1.11.3 with fix for CVE-2018-16875 ([#72035](https://github.com/kubernetes/kubernetes/pull/72035), [@seemethere](https://github.com/seemethere))
* kubeadm: fix a bug when syncing etcd endpoints ([#71945](https://github.com/kubernetes/kubernetes/pull/71945), [@pytimer](https://github.com/pytimer))
* fix kubelet log flushing issue in azure disk ([#71990](https://github.com/kubernetes/kubernetes/pull/71990), [@andyzhangx](https://github.com/andyzhangx))
* Disable proxy to loopback and linklocal ([#71980](https://github.com/kubernetes/kubernetes/pull/71980), [@micahhausler](https://github.com/micahhausler))
* Fix overlapping filenames in diff if multiple resources have the same name. ([#71923](https://github.com/kubernetes/kubernetes/pull/71923), [@apelisse](https://github.com/apelisse))
* fix issue: vm sku restriction policy does not work in azure disk attach/detach ([#71941](https://github.com/kubernetes/kubernetes/pull/71941), [@andyzhangx](https://github.com/andyzhangx))
* kubeadm: Create /var/lib/etcd with correct permissions (0700) by default. ([#71885](https://github.com/kubernetes/kubernetes/pull/71885), [@dims](https://github.com/dims))
* Scheduler only activates unschedulable pods if node's scheduling related properties change. ([#71551](https://github.com/kubernetes/kubernetes/pull/71551), [@mlmhl](https://github.com/mlmhl))
* kube-proxy in IPVS mode will stop initiating connections to terminating pods for services with sessionAffinity set. ([#71834](https://github.com/kubernetes/kubernetes/pull/71834), [@lbernail](https://github.com/lbernail))
* kubeadm: improve hostport parsing error messages ([#71258](https://github.com/kubernetes/kubernetes/pull/71258), [@bart0sh](https://github.com/bart0sh))
* Support graceful termination with IPVS when deleting a service ([#71895](https://github.com/kubernetes/kubernetes/pull/71895), [@lbernail](https://github.com/lbernail))
* Include CRD for BGPConfigurations, needed for calico 2.x to 3.x upgrade. ([#71868](https://github.com/kubernetes/kubernetes/pull/71868), [@satyasm](https://github.com/satyasm))
* apply: fix detection of non-dry-run enabled servers ([#71854](https://github.com/kubernetes/kubernetes/pull/71854), [@apelisse](https://github.com/apelisse))
* Clear UDP conntrack entry on endpoint changes when using nodeport ([#71573](https://github.com/kubernetes/kubernetes/pull/71573), [@JacobTanenbaum](https://github.com/JacobTanenbaum))
* Add successful and failed history limits to cronjob describe ([#71844](https://github.com/kubernetes/kubernetes/pull/71844), [@soltysh](https://github.com/soltysh))
* kube-controller-manager: fixed issue display help for the deprecated insecure --port flag ([#71601](https://github.com/kubernetes/kubernetes/pull/71601), [@liggitt](https://github.com/liggitt))
* kubectl: fixes regression in --sort-by behavior ([#71805](https://github.com/kubernetes/kubernetes/pull/71805), [@liggitt](https://github.com/liggitt))
* Fixes pod deletion when cleaning old cronjobs ([#71801](https://github.com/kubernetes/kubernetes/pull/71801), [@soltysh](https://github.com/soltysh))
* kubeadm: use kubeconfig flag instead of kubeconfig-dir on init phase bootstrap-token ([#71803](https://github.com/kubernetes/kubernetes/pull/71803), [@yagonobre](https://github.com/yagonobre))
* kube-scheduler: restores ability to run without authentication configuration lookup permissions ([#71755](https://github.com/kubernetes/kubernetes/pull/71755), [@liggitt](https://github.com/liggitt))
* Add aggregator_unavailable_apiservice_{count,gauge} metrics in the kube-aggregator. ([#71380](https://github.com/kubernetes/kubernetes/pull/71380), [@sttts](https://github.com/sttts))
* Fixes apiserver nil pointer panics when requesting v2beta1 autoscaling object metrics ([#71744](https://github.com/kubernetes/kubernetes/pull/71744), [@yue9944882](https://github.com/yue9944882))
* Only use the first IP address got from instance metadata. This is because Azure CNI would set up a list of IP addresses in instance metadata, while only the first one is the Node's IP. ([#71736](https://github.com/kubernetes/kubernetes/pull/71736), [@feiskyer](https://github.com/feiskyer))
* client-go: restores behavior of populating the BearerToken field in rest.Config objects constructed from kubeconfig files containing tokenFile config, or from in-cluster configuration. An additional BearerTokenFile field is now populated to enable constructed clients to periodically refresh tokens. ([#71713](https://github.com/kubernetes/kubernetes/pull/71713), [@liggitt](https://github.com/liggitt))
* kubeadm: remove deprecated kubeadm config print-defaults command ([#71467](https://github.com/kubernetes/kubernetes/pull/71467), [@rosti](https://github.com/rosti))
* hack/local-up-cluster.sh now enables kubelet authentication/authorization by default (they can be disabled with KUBELET_AUTHENTICATION_WEBHOOK=false and KUBELET_AUTHORIZATION_WEBHOOK=false ([#71690](https://github.com/kubernetes/kubernetes/pull/71690), [@liggitt](https://github.com/liggitt))
* Fixes an issue where Azure VMSS instances not existing in Azure were not being deleted by the Cloud Controller Manager. ([#71597](https://github.com/kubernetes/kubernetes/pull/71597), [@marc-sensenich](https://github.com/marc-sensenich))
* kubeadm reset correcty unmounts mount points inside /var/lib/kubelet ([#71663](https://github.com/kubernetes/kubernetes/pull/71663), [@bart0sh](https://github.com/bart0sh))
* Upgrade default etcd server to 3.3.10 ([#71615](https://github.com/kubernetes/kubernetes/pull/71615), [@jpbetz](https://github.com/jpbetz))
* When creating a service with annotation: service.beta.kubernetes.io/load-balancer-source-ranges containing multiple source ranges and service.beta.kubernetes.io/azure-shared-securityrule: "false", the NSG rules will be collapsed. ([#71484](https://github.com/kubernetes/kubernetes/pull/71484), [@ritazh](https://github.com/ritazh))
* disable node's proxy use of http probe ([#68663](https://github.com/kubernetes/kubernetes/pull/68663), [@WanLinghao](https://github.com/WanLinghao))
* Bumps version of kubernetes-cni to 0.6.0 ([#71629](https://github.com/kubernetes/kubernetes/pull/71629), [@mauilion](https://github.com/mauilion))
* On GCI, NPD starts to monitor kubelet, docker, containerd crashlooping, read-only filesystem and corrupt docker overlay2 issues. ([#71522](https://github.com/kubernetes/kubernetes/pull/71522), [@wangzhen127](https://github.com/wangzhen127))
* When a kubelet is using --bootstrap-kubeconfig and certificate rotation, it no longer waits for bootstrap to succeed before launching static pods. ([#71174](https://github.com/kubernetes/kubernetes/pull/71174), [@smarterclayton](https://github.com/smarterclayton))
* Add an plugin interfaces for "reserve" and "prebind" extension points of the scheduling framework. ([#70227](https://github.com/kubernetes/kubernetes/pull/70227), [@bsalamat](https://github.com/bsalamat))
* Fix scheduling starvation of pods in cluster with large number of unschedulable pods. ([#71488](https://github.com/kubernetes/kubernetes/pull/71488), [@bsalamat](https://github.com/bsalamat))
* Reduce CSI log and event spam. ([#71581](https://github.com/kubernetes/kubernetes/pull/71581), [@saad-ali](https://github.com/saad-ali))
* Add conntrack as a dependency of kubelet and kubeadm when building rpms and debs. Both require conntrack to handle cleanup of connections. ([#71540](https://github.com/kubernetes/kubernetes/pull/71540), [@mauilion](https://github.com/mauilion))
* UDP connections now support graceful termination in IPVS mode ([#71515](https://github.com/kubernetes/kubernetes/pull/71515), [@lbernail](https://github.com/lbernail))
* Log etcd client errors. The verbosity is set with the usual `-v` flag. ([#71318](https://github.com/kubernetes/kubernetes/pull/71318), [@sttts](https://github.com/sttts))
* The `DefaultFeatureGate` package variable now only exposes readonly feature gate methods. Methods for mutating feature gates have moved into a `MutableFeatureGate` interface and are accessible via the `DefaultMutableFeatureGate` package variable. Only top-level commands and options setup should access `DefaultMutableFeatureGate`. ([#71302](https://github.com/kubernetes/kubernetes/pull/71302), [@liggitt](https://github.com/liggitt))
* `node.kubernetes.io/pid-pressure` toleration is added for DaemonSet pods, and `node.kubernetes.io/out-of-disk` isn't added any more even if it's a critical pod. ([#67036](https://github.com/kubernetes/kubernetes/pull/67036), [@Huang-Wei](https://github.com/Huang-Wei))
* Update k8s.io/utils to allow for asynchronous process control ([#71047](https://github.com/kubernetes/kubernetes/pull/71047), [@hoegaarden](https://github.com/hoegaarden))
* Fixes possible panic during volume detach, if corresponding volume plugin became non-attachable ([#71471](https://github.com/kubernetes/kubernetes/pull/71471), [@mshaverdo](https://github.com/mshaverdo))
* Fix cloud-controller-manager crash when using AWS provider and PersistentVolume initializing controller ([#70432](https://github.com/kubernetes/kubernetes/pull/70432), [@mvladev](https://github.com/mvladev))
* Fixes an issue where Portworx volumes cannot be mounted if 9001 port is already in use on the host and users remap 9001 to another port. ([#70392](https://github.com/kubernetes/kubernetes/pull/70392), [@harsh-px](https://github.com/harsh-px))
* Fix `SubPath` printing of `VolumeMounts`. ([#70127](https://github.com/kubernetes/kubernetes/pull/70127), [@dtaniwaki](https://github.com/dtaniwaki))
* Fixes incorrect paths (missing first letter) when copying files from pods to ([#69885](https://github.com/kubernetes/kubernetes/pull/69885), [@clickyotomy](https://github.com/clickyotomy))
* local in `kubectl cp'.
* Fix AWS NLB security group updates where valid security group ports were incorrectly removed ([#68422](https://github.com/kubernetes/kubernetes/pull/68422), [@kellycampbell](https://github.com/kellycampbell))
* when updating a service or when node changes occur.

3
vendor/k8s.io/kubernetes/CHANGELOG.md generated vendored
View file

@ -3,10 +3,11 @@
## Current release:
- [CHANGELOG-1.12.md](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.12.md)
- [CHANGELOG-1.13.md](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md)
## Older releases:
- [CHANGELOG-1.12.md](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.12.md)
- [CHANGELOG-1.11.md](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md)
- [CHANGELOG-1.10.md](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.10.md)
- [CHANGELOG-1.9.md](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.9.md)

4
vendor/k8s.io/kubernetes/OWNERS generated vendored
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
filters:
".*":
reviewers:
@ -7,6 +9,7 @@ filters:
- lavalamp
- smarterclayton
- thockin
- liggitt
approvers:
- bgrant0607
- brendandburns
@ -17,6 +20,7 @@ filters:
- smarterclayton
- thockin
- wojtek-t
- liggitt
# Bazel build infrastructure changes often touch files throughout the tree
"\\.bzl$":

100
vendor/k8s.io/kubernetes/OWNERS_ALIASES generated vendored
View file

@ -102,12 +102,13 @@ aliases:
- tallclair
sig-auth-policy-reviewers:
- deads2k
- hongchaodeng
- hongchaodeng
- jianhuiz
- liggitt
- mbohlool
- pweil-
- tallclair
- krmayankk
sig-auth-serviceaccounts-approvers:
- deads2k
@ -119,18 +120,18 @@ aliases:
- enj
- liggitt
- mikedanese
- WanLinghao
sig-storage-reviewers:
- saad-ali
- childsb
sig-scheduling-maintainers:
- bsalamat
- davidopp
- k82cn
- timothysc
- wojtek-t
- aveshagarwal
- ravisantoshgudimetla
- Huang-Wei
sig-scheduling:
- bsalamat
- k82cn
@ -138,6 +139,8 @@ aliases:
- ravisantoshgudimetla
- misterikkit
- Huang-Wei
- wgliang
sig-cli-maintainers:
- adohe
- brendandburns
@ -168,14 +171,18 @@ aliases:
- smarterclayton
- soltysh
sig-testing-reviewers:
- bentheelder
- cblecker
- fejta
- krzyzacy
- ixdy
- rmmh
- spiffxp
sig-testing-approvers:
- bentheelder
- cblecker
- fejta
- krzyzacy
- ixdy
- rmmh
- spiffxp
sig-node-reviewers:
- Random-Liu
@ -236,81 +243,6 @@ aliases:
- DirectXMan12
- MaciekPytel
- mwielgus
milestone-maintainers:
- lavalamp
- deads2k
- michelleN
- mattfarina
- prydonius
- bgrant0607
- jdumars
- liggitt
- deads2k
- mwielgus
- directxman12
- justinsb
- kris-nova
- chrislovecnm
- mfburnett
- slack
- colemickens
- foxish
- AdoHe
- lukemarsden
- jbeda
- roberthbailey
- zehicle
- jdumars
- grodrigues3
- Phillels
- devin-donnelly
- jaredbhatti
- csbell
- quinton-hoole
- piosz
- fabxc
- thockin
- dcbw
- caseydavenport
- dchen1107
- derekwaynecarr
- zen
- marcoceppi
- dghubble
- idvoretskyi
- xsgordon
- apsinha
- idvoretskyi
- calebamiles
- calebamiles
- wojtek-t
- countspongebob
- jbeda
- davidopp
- timothysc
- pmorie
- arschles
- vaikas-google
- duglin
- saad-ali
- childsb
- spiffxp
- fejta
- timothysc
- danielromlein
- floreks
- michmike
- abgworrall
- krzyzacy
- steveperry-53
- radhikpac
- jpbetz
- cmluciano
- bsalamat
- m1093782566
- tallclair
- feiskyer
- mikedanese
api-approvers:
- erictune
- lavalamp
@ -356,11 +288,14 @@ aliases:
- jsafrane
- jbeda
dep-approvers:
- apelisse
- BenTheElder
- cblecker
- dims
- thockin
- sttts
- soltysh
feature-approvers:
- AdoHe # CLI
- bgrant0607 # Architecture
- brancz # Instrumentation
- bsalamat # Scheduling
@ -391,6 +326,7 @@ aliases:
- quinton-hoole # Multicluster
- roberthbailey # Cluster Lifecycle
- saad-ali # Storage
- seans3 # CLI
- soltysh # CLI
- tallclair # Auth
- thockin # Network

4
vendor/k8s.io/kubernetes/SECURITY_CONTACTS generated vendored
View file

@ -1,10 +1,10 @@
# Defined below are the security contacts for this repo.
#
# They are the contact point for the Product Security Team to reach out
# They are the contact point for the Product Security Committee to reach out
# to for triaging and handling of incoming issues.
#
# The below names agree to abide by the
# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy)
# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy)
# and will be removed and replaced if they violate that agreement.
#
# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE

18
vendor/k8s.io/kubernetes/SUPPORT.md generated vendored
View file

@ -1,27 +1,27 @@
## Support for deploying and using Kubernetes
## Support for deploying and using Kubernetes
Welcome to Kubernetes! We use GitHub for tracking bugs and feature requests.
This isn't the right place to get support for using Kubernetes, but the following
Welcome to Kubernetes! We use GitHub for tracking bugs and feature requests.
This isn't the right place to get support for using Kubernetes, but the following
resources are available below, thanks for understanding.
### Stack Overflow
The Kubernetes Community is active on Stack Overflow, you can post your questions there:
The Kubernetes Community is active on Stack Overflow, you can post your questions there:
* [Kubernetes on Stack Overflow](http://stackoverflow.com/questions/tagged/kubernetes)
* [Kubernetes on Stack Overflow](https://stackoverflow.com/questions/tagged/kubernetes)
* Here are some tips for [about how to ask good questions](http://stackoverflow.com/help/how-to-ask).
* Here are some tips for [about how to ask good questions](https://stackoverflow.com/help/how-to-ask).
* Don't forget to check to see [what's on topic](http://stackoverflow.com/help/on-topic).
### Documentation
### Documentation
* [User Documentation](https://kubernetes.io/docs/)
* [User Documentation](https://kubernetes.io/docs/)
* [Troubleshooting Guide](https://kubernetes.io/docs/tasks/debug-application-cluster/troubleshooting/)
### Real-time Chat
* [Slack](https://kubernetes.slack.com) ([registration](http://slack.k8s.io)):
The `#kubernetes-users` and `#kubernetes-novice` channels are usual places where
The `#kubernetes-users` and `#kubernetes-novice` channels are usual places where
people offer support.
### Forum

3
vendor/k8s.io/kubernetes/pkg/.import-restrictions generated vendored
View file

@ -6,7 +6,8 @@
""
],
"ForbiddenPrefixes": [
"k8s.io/kubernetes/cmd"
"k8s.io/kubernetes/cmd",
"github.com/ghodss/yaml"
]
}
]

3
vendor/k8s.io/kubernetes/pkg/BUILD generated vendored
View file

@ -19,11 +19,11 @@ filegroup(
"//pkg/api/podsecuritypolicy:all-srcs",
"//pkg/api/ref:all-srcs",
"//pkg/api/resource:all-srcs",
"//pkg/api/resourcequota:all-srcs",
"//pkg/api/service:all-srcs",
"//pkg/api/testapi:all-srcs",
"//pkg/api/testing:all-srcs",
"//pkg/api/v1/endpoints:all-srcs",
"//pkg/api/v1/node:all-srcs",
"//pkg/api/v1/persistentvolume:all-srcs",
"//pkg/api/v1/pod:all-srcs",
"//pkg/api/v1/resource:all-srcs",
@ -44,6 +44,7 @@ filegroup(
"//pkg/apis/extensions:all-srcs",
"//pkg/apis/imagepolicy:all-srcs",
"//pkg/apis/networking:all-srcs",
"//pkg/apis/node:all-srcs",
"//pkg/apis/policy:all-srcs",
"//pkg/apis/rbac:all-srcs",
"//pkg/apis/scheduling:all-srcs",

2
vendor/k8s.io/kubernetes/pkg/OWNERS generated vendored
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- brendandburns
- dchen1107

1
vendor/k8s.io/kubernetes/pkg/api/OWNERS generated vendored
View file

@ -1,4 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
# Disable inheritance as this is an api owners file
options:
no_parent_owners: true

2
vendor/k8s.io/kubernetes/pkg/api/v1/OWNERS generated vendored Executable file → Normal file
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- thockin
- lavalamp

4
vendor/k8s.io/kubernetes/pkg/api/v1/pod/util.go generated vendored
View file

@ -120,6 +120,10 @@ func VisitPodSecretNames(pod *v1.Pod, visitor Visitor) bool {
if source.StorageOS.SecretRef != nil && !visitor(source.StorageOS.SecretRef.Name) {
return false
}
case source.CSI != nil:
if source.CSI.NodePublishSecretRef != nil && !visitor(source.CSI.NodePublishSecretRef.Name) {
return false
}
}
}
return true

16
vendor/k8s.io/kubernetes/pkg/customers.yaml generated vendored
View file

@ -1,16 +0,0 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
name: standard-rdi-hdd-pool
parameters:
adminId: sam
adminSecretName: ceph-sec-admin-rdi
adminSecretNamespace: sam-system
monitors: 10.253.101.170:6789,10.253.101.154:6789,10.253.101.186:6789
pool: sam
userId: sam
userSecretName: ceph-sec-user-rdi
provisioner: kubernetes.io/rbd
reclaimPolicy: Retain
volumeBindingMode: Immediate

11
vendor/k8s.io/kubernetes/pkg/kubelet/BUILD generated vendored
View file

@ -103,7 +103,6 @@ go_library(
"//pkg/security/podsecuritypolicy/sysctl:go_default_library",
"//pkg/securitycontext:go_default_library",
"//pkg/util/dbus:go_default_library",
"//pkg/util/file:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/util/node:go_default_library",
@ -113,6 +112,7 @@ go_library(
"//pkg/volume:go_default_library",
"//pkg/volume/csi:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//pkg/volume/util/types:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
"//pkg/volume/validation:go_default_library",
@ -132,7 +132,6 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
@ -140,9 +139,7 @@ go_library(
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/util/certificate:go_default_library",
"//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
"//staging/src/k8s.io/client-go/util/integer:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
"//third_party/forked/golang/expansion:go_default_library",
"//vendor/github.com/golang/groupcache/lru:go_default_library",
"//vendor/github.com/google/cadvisor/events:go_default_library",
@ -150,6 +147,8 @@ go_library(
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/integer:go_default_library",
"//vendor/k8s.io/utils/path:go_default_library",
],
)
@ -167,7 +166,6 @@ go_test(
"kubelet_test.go",
"kubelet_volumes_linux_test.go",
"kubelet_volumes_test.go",
"main_test.go",
"oom_watcher_test.go",
"pod_container_deletor_test.go",
"pod_workers_test.go",
@ -210,7 +208,7 @@ go_test(
"//pkg/kubelet/util/sliceutils:go_default_library",
"//pkg/kubelet/volumemanager:go_default_library",
"//pkg/scheduler/api:go_default_library",
"//pkg/scheduler/cache:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/util/taints:go_default_library",
"//pkg/version:go_default_library",
@ -221,6 +219,7 @@ go_test(
"//pkg/volume/host_path:go_default_library",
"//pkg/volume/testing:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

2
vendor/k8s.io/kubernetes/pkg/kubelet/OWNERS generated vendored
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- Random-Liu
- dchen1107

2
vendor/k8s.io/kubernetes/pkg/kubelet/apis/BUILD generated vendored
View file

@ -14,6 +14,7 @@ go_library(
],
importpath = "k8s.io/kubernetes/pkg/kubelet/apis",
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:windows": [
@ -43,6 +44,7 @@ filegroup(
"//pkg/kubelet/apis/pluginregistration/v1alpha1:all-srcs",
"//pkg/kubelet/apis/pluginregistration/v1beta1:all-srcs",
"//pkg/kubelet/apis/podresources:all-srcs",
"//pkg/kubelet/apis/resourcemetrics/v1alpha1:all-srcs",
"//pkg/kubelet/apis/stats/v1alpha1:all-srcs",
],
tags = ["automanaged"],

693
vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go generated vendored
View file

@ -736,7 +736,8 @@ type PodSandboxConfig struct {
// operation. The runtime may also use this information to improve UX, such
// as by constructing a readable name.
Metadata *PodSandboxMetadata `protobuf:"bytes,1,opt,name=metadata" json:"metadata,omitempty"`
// Hostname of the sandbox.
// Hostname of the sandbox. Hostname could only be empty when the pod
// network namespace is NODE.
Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3" json:"hostname,omitempty"`
// Path to the directory on the host in which container log files are
// stored.
@ -848,7 +849,7 @@ type RunPodSandboxRequest struct {
// If the runtime handler is unknown, this request should be rejected. An
// empty string should select the default handler, equivalent to the
// behavior before this feature was added.
// See https://git.k8s.io/community/keps/sig-node/0014-runtime-class.md
// See https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md
RuntimeHandler string `protobuf:"bytes,2,opt,name=runtime_handler,json=runtimeHandler,proto3" json:"runtime_handler,omitempty"`
}
@ -1029,6 +1030,8 @@ type PodSandboxStatus struct {
// MUST be identical to that of the corresponding PodSandboxConfig used to
// instantiate the pod sandbox this status represents.
Annotations map[string]string `protobuf:"bytes,8,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// runtime configuration used for this PodSandbox.
RuntimeHandler string `protobuf:"bytes,9,opt,name=runtime_handler,json=runtimeHandler,proto3" json:"runtime_handler,omitempty"`
}
func (m *PodSandboxStatus) Reset() { *m = PodSandboxStatus{} }
@ -1091,6 +1094,13 @@ func (m *PodSandboxStatus) GetAnnotations() map[string]string {
return nil
}
func (m *PodSandboxStatus) GetRuntimeHandler() string {
if m != nil {
return m.RuntimeHandler
}
return ""
}
type PodSandboxStatusResponse struct {
// Status of the PodSandbox.
Status *PodSandboxStatus `protobuf:"bytes,1,opt,name=status" json:"status,omitempty"`
@ -1207,6 +1217,8 @@ type PodSandbox struct {
// MUST be identical to that of the corresponding PodSandboxConfig used to
// instantiate this PodSandbox.
Annotations map[string]string `protobuf:"bytes,6,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// runtime configuration used for this PodSandbox.
RuntimeHandler string `protobuf:"bytes,7,opt,name=runtime_handler,json=runtimeHandler,proto3" json:"runtime_handler,omitempty"`
}
func (m *PodSandbox) Reset() { *m = PodSandbox{} }
@ -1255,6 +1267,13 @@ func (m *PodSandbox) GetAnnotations() map[string]string {
return nil
}
func (m *PodSandbox) GetRuntimeHandler() string {
if m != nil {
return m.RuntimeHandler
}
return ""
}
type ListPodSandboxResponse struct {
// List of PodSandboxes.
Items []*PodSandbox `protobuf:"bytes,1,rep,name=items" json:"items,omitempty"`
@ -3550,7 +3569,7 @@ type ContainerStats struct {
Cpu *CpuUsage `protobuf:"bytes,2,opt,name=cpu" json:"cpu,omitempty"`
// Memory usage gathered from the container.
Memory *MemoryUsage `protobuf:"bytes,3,opt,name=memory" json:"memory,omitempty"`
// Usage of the writeable layer.
// Usage of the writable layer.
WritableLayer *FilesystemUsage `protobuf:"bytes,4,opt,name=writable_layer,json=writableLayer" json:"writable_layer,omitempty"`
}
@ -5774,6 +5793,12 @@ func (m *PodSandboxStatus) MarshalTo(dAtA []byte) (int, error) {
i += copy(dAtA[i:], v)
}
}
if len(m.RuntimeHandler) > 0 {
dAtA[i] = 0x4a
i++
i = encodeVarintApi(dAtA, i, uint64(len(m.RuntimeHandler)))
i += copy(dAtA[i:], m.RuntimeHandler)
}
return i, nil
}
@ -5999,6 +6024,12 @@ func (m *PodSandbox) MarshalTo(dAtA []byte) (int, error) {
i += copy(dAtA[i:], v)
}
}
if len(m.RuntimeHandler) > 0 {
dAtA[i] = 0x3a
i++
i = encodeVarintApi(dAtA, i, uint64(len(m.RuntimeHandler)))
i += copy(dAtA[i:], m.RuntimeHandler)
}
return i, nil
}
@ -9446,6 +9477,10 @@ func (m *PodSandboxStatus) Size() (n int) {
n += mapEntrySize + 1 + sovApi(uint64(mapEntrySize))
}
}
l = len(m.RuntimeHandler)
if l > 0 {
n += 1 + l + sovApi(uint64(l))
}
return n
}
@ -9541,6 +9576,10 @@ func (m *PodSandbox) Size() (n int) {
n += mapEntrySize + 1 + sovApi(uint64(mapEntrySize))
}
}
l = len(m.RuntimeHandler)
if l > 0 {
n += 1 + l + sovApi(uint64(l))
}
return n
}
@ -11127,6 +11166,7 @@ func (this *PodSandboxStatus) String() string {
`Linux:` + strings.Replace(fmt.Sprintf("%v", this.Linux), "LinuxPodSandboxStatus", "LinuxPodSandboxStatus", 1) + `,`,
`Labels:` + mapStringForLabels + `,`,
`Annotations:` + mapStringForAnnotations + `,`,
`RuntimeHandler:` + fmt.Sprintf("%v", this.RuntimeHandler) + `,`,
`}`,
}, "")
return s
@ -11225,6 +11265,7 @@ func (this *PodSandbox) String() string {
`CreatedAt:` + fmt.Sprintf("%v", this.CreatedAt) + `,`,
`Labels:` + mapStringForLabels + `,`,
`Annotations:` + mapStringForAnnotations + `,`,
`RuntimeHandler:` + fmt.Sprintf("%v", this.RuntimeHandler) + `,`,
`}`,
}, "")
return s
@ -15490,6 +15531,35 @@ func (m *PodSandboxStatus) Unmarshal(dAtA []byte) error {
}
m.Annotations[mapkey] = mapvalue
iNdEx = postIndex
case 9:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field RuntimeHandler", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowApi
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthApi
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.RuntimeHandler = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipApi(dAtA[iNdEx:])
@ -16459,6 +16529,35 @@ func (m *PodSandbox) Unmarshal(dAtA []byte) error {
}
m.Annotations[mapkey] = mapvalue
iNdEx = postIndex
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field RuntimeHandler", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowApi
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthApi
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.RuntimeHandler = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipApi(dAtA[iNdEx:])
@ -26861,300 +26960,300 @@ var (
func init() { proto.RegisterFile("api.proto", fileDescriptorApi) }
var fileDescriptorApi = []byte{
// 4708 bytes of a gzipped FileDescriptorProto
// 4720 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5c, 0xcd, 0x6f, 0x1b, 0x49,
0x76, 0x57, 0x93, 0xfa, 0x20, 0x1f, 0x45, 0x89, 0x2a, 0xcb, 0x16, 0x4d, 0x8f, 0x35, 0x56, 0xcf,
0xf8, 0x73, 0x66, 0xe4, 0xb1, 0x66, 0xd7, 0x13, 0xdb, 0xb3, 0xb6, 0x69, 0x49, 0xb6, 0x99, 0xb5,
0x29, 0xa6, 0x29, 0xcd, 0xc7, 0xce, 0x00, 0xbd, 0x2d, 0x76, 0x89, 0xea, 0x35, 0xd9, 0xd5, 0xd3,
0xdd, 0xb4, 0xad, 0x04, 0x08, 0x16, 0x08, 0xb2, 0x87, 0x00, 0x01, 0x72, 0xce, 0x71, 0x73, 0xc8,
0x21, 0xb7, 0x00, 0x41, 0x0e, 0x39, 0x6d, 0xb0, 0x87, 0xbd, 0x04, 0xc8, 0x69, 0x91, 0x8f, 0x4b,
0x66, 0x92, 0x5c, 0x72, 0x08, 0xf2, 0x07, 0xe4, 0x10, 0xd4, 0x57, 0xb3, 0x3f, 0xf9, 0xe1, 0xf1,
0xee, 0x4c, 0x4e, 0xea, 0x7a, 0xfd, 0xde, 0xab, 0x57, 0xaf, 0x5e, 0xbf, 0x7a, 0xf5, 0xab, 0xa2,
0xa0, 0x68, 0x38, 0xd6, 0xa6, 0xe3, 0x12, 0x9f, 0xa0, 0x8a, 0x3b, 0xb0, 0x7d, 0xab, 0x8f, 0x37,
0x9f, 0xdf, 0x30, 0x7a, 0xce, 0xb1, 0xb1, 0x55, 0x7b, 0xaf, 0x6b, 0xf9, 0xc7, 0x83, 0xc3, 0xcd,
0x0e, 0xe9, 0x5f, 0xef, 0x92, 0x2e, 0xb9, 0xce, 0x18, 0x0f, 0x07, 0x47, 0xac, 0xc5, 0x1a, 0xec,
0x89, 0x2b, 0x50, 0xaf, 0xc1, 0xd2, 0xc7, 0xd8, 0xf5, 0x2c, 0x62, 0x6b, 0xf8, 0xcb, 0x01, 0xf6,
0x7c, 0x54, 0x85, 0x85, 0xe7, 0x9c, 0x52, 0x55, 0x2e, 0x28, 0x57, 0x8a, 0x9a, 0x6c, 0xaa, 0x7f,
0xa9, 0xc0, 0x72, 0xc0, 0xec, 0x39, 0xc4, 0xf6, 0x70, 0x36, 0x37, 0xda, 0x80, 0x45, 0x61, 0x9c,
0x6e, 0x1b, 0x7d, 0x5c, 0xcd, 0xb1, 0xd7, 0x25, 0x41, 0x6b, 0x1a, 0x7d, 0x8c, 0x2e, 0xc3, 0xb2,
0x64, 0x91, 0x4a, 0xf2, 0x8c, 0x6b, 0x49, 0x90, 0x45, 0x6f, 0x68, 0x13, 0x4e, 0x49, 0x46, 0xc3,
0xb1, 0x02, 0xe6, 0x59, 0xc6, 0xbc, 0x22, 0x5e, 0xd5, 0x1d, 0x4b, 0xf0, 0xab, 0x9f, 0x43, 0x71,
0xa7, 0xd9, 0xde, 0x26, 0xf6, 0x91, 0xd5, 0xa5, 0x26, 0x7a, 0xd8, 0xa5, 0x32, 0x55, 0xe5, 0x42,
0x9e, 0x9a, 0x28, 0x9a, 0xa8, 0x06, 0x05, 0x0f, 0x1b, 0x6e, 0xe7, 0x18, 0x7b, 0xd5, 0x1c, 0x7b,
0x15, 0xb4, 0xa9, 0x14, 0x71, 0x7c, 0x8b, 0xd8, 0x5e, 0x35, 0xcf, 0xa5, 0x44, 0x53, 0xfd, 0xb9,
0x02, 0xa5, 0x16, 0x71, 0xfd, 0xa7, 0x86, 0xe3, 0x58, 0x76, 0x17, 0xdd, 0x84, 0x02, 0xf3, 0x65,
0x87, 0xf4, 0x98, 0x0f, 0x96, 0xb6, 0x6a, 0x9b, 0xf1, 0x69, 0xd9, 0x6c, 0x09, 0x0e, 0x2d, 0xe0,
0x45, 0x17, 0x61, 0xa9, 0x43, 0x6c, 0xdf, 0xb0, 0x6c, 0xec, 0xea, 0x0e, 0x71, 0x7d, 0xe6, 0xa2,
0x39, 0xad, 0x1c, 0x50, 0x69, 0x2f, 0xe8, 0x1c, 0x14, 0x8f, 0x89, 0xe7, 0x73, 0x8e, 0x3c, 0xe3,
0x28, 0x50, 0x02, 0x7b, 0xb9, 0x06, 0x0b, 0xec, 0xa5, 0xe5, 0x08, 0x67, 0xcc, 0xd3, 0x66, 0xc3,
0x51, 0x7f, 0xad, 0xc0, 0xdc, 0x53, 0x32, 0xb0, 0xfd, 0x58, 0x37, 0x86, 0x7f, 0x2c, 0x26, 0x2a,
0xd4, 0x8d, 0xe1, 0x1f, 0x0f, 0xbb, 0xa1, 0x1c, 0x7c, 0xae, 0x78, 0x37, 0xf4, 0x65, 0x0d, 0x0a,
0x2e, 0x36, 0x4c, 0x62, 0xf7, 0x4e, 0x98, 0x09, 0x05, 0x2d, 0x68, 0xd3, 0x49, 0xf4, 0x70, 0xcf,
0xb2, 0x07, 0x2f, 0x75, 0x17, 0xf7, 0x8c, 0x43, 0xdc, 0x63, 0xa6, 0x14, 0xb4, 0x25, 0x41, 0xd6,
0x38, 0x15, 0xed, 0x40, 0xc9, 0x71, 0x89, 0x63, 0x74, 0x0d, 0xea, 0xc7, 0xea, 0x1c, 0x73, 0x95,
0x9a, 0x74, 0x15, 0x33, 0xbb, 0x35, 0xe4, 0xd4, 0xc2, 0x62, 0xea, 0x5f, 0x2b, 0xb0, 0x4c, 0x83,
0xc7, 0x73, 0x8c, 0x0e, 0xde, 0x63, 0x53, 0x82, 0x6e, 0xc1, 0x82, 0x8d, 0xfd, 0x17, 0xc4, 0x7d,
0x26, 0x26, 0xe0, 0xcd, 0xa4, 0xd6, 0x40, 0xe6, 0x29, 0x31, 0xb1, 0x26, 0xf9, 0xd1, 0x0d, 0xc8,
0x3b, 0x96, 0xc9, 0x06, 0x3c, 0x81, 0x18, 0xe5, 0xa5, 0x22, 0x96, 0xd3, 0x61, 0x7e, 0x98, 0x44,
0xc4, 0x72, 0x3a, 0xaa, 0x0a, 0xd0, 0xb0, 0xfd, 0x9b, 0xdf, 0xfb, 0xd8, 0xe8, 0x0d, 0x30, 0x5a,
0x85, 0xb9, 0xe7, 0xf4, 0x81, 0x19, 0x9b, 0xd7, 0x78, 0x43, 0xfd, 0x2a, 0x0f, 0xe7, 0x9e, 0x50,
0x7f, 0xb5, 0x0d, 0xdb, 0x3c, 0x24, 0x2f, 0xdb, 0xb8, 0x33, 0x70, 0x2d, 0xff, 0x64, 0x9b, 0xd8,
0x3e, 0x7e, 0xe9, 0xa3, 0x26, 0xac, 0xd8, 0x52, 0xb3, 0x2e, 0x43, 0x93, 0x6a, 0x28, 0x6d, 0x6d,
0x8c, 0x30, 0x82, 0xbb, 0x48, 0xab, 0xd8, 0x51, 0x82, 0x87, 0x1e, 0x0f, 0xe7, 0x4d, 0x6a, 0xcb,
0x31, 0x6d, 0x29, 0x43, 0x6a, 0xef, 0x32, 0xcb, 0x84, 0x2e, 0x39, 0xb1, 0x52, 0xd3, 0x47, 0x40,
0xbf, 0x6a, 0xdd, 0xf0, 0xf4, 0x81, 0x87, 0x5d, 0xe6, 0x98, 0xd2, 0xd6, 0x1b, 0x49, 0x2d, 0x43,
0x17, 0x68, 0x45, 0x77, 0x60, 0xd7, 0xbd, 0x03, 0x0f, 0xbb, 0x2c, 0x09, 0x88, 0x58, 0xd2, 0x5d,
0x42, 0xfc, 0x23, 0x4f, 0xc6, 0x8f, 0x24, 0x6b, 0x8c, 0x8a, 0xae, 0xc3, 0x29, 0x6f, 0xe0, 0x38,
0x3d, 0xdc, 0xc7, 0xb6, 0x6f, 0xf4, 0xf4, 0xae, 0x4b, 0x06, 0x8e, 0x57, 0x9d, 0xbb, 0x90, 0xbf,
0x92, 0xd7, 0x50, 0xf8, 0xd5, 0x23, 0xf6, 0x06, 0xad, 0x03, 0x38, 0xae, 0xf5, 0xdc, 0xea, 0xe1,
0x2e, 0x36, 0xab, 0xf3, 0x4c, 0x69, 0x88, 0x82, 0xde, 0x87, 0x55, 0x0f, 0x77, 0x3a, 0xa4, 0xef,
0xe8, 0x8e, 0x4b, 0x8e, 0xac, 0x1e, 0xe6, 0xd1, 0xbf, 0xc0, 0xa2, 0x1f, 0x89, 0x77, 0x2d, 0xfe,
0x8a, 0x7d, 0x07, 0x77, 0x59, 0x4e, 0xa3, 0x23, 0x65, 0x9d, 0x57, 0x0b, 0x13, 0x0c, 0x15, 0xd8,
0x50, 0x99, 0x49, 0xea, 0xcf, 0x73, 0x70, 0x9a, 0x79, 0xb2, 0x45, 0x4c, 0x31, 0xcd, 0x22, 0x49,
0xbd, 0x05, 0xe5, 0x0e, 0xd3, 0xa9, 0x3b, 0x86, 0x8b, 0x6d, 0x5f, 0x7c, 0xa4, 0x8b, 0x9c, 0xd8,
0x62, 0x34, 0xf4, 0x29, 0x54, 0x3c, 0x11, 0x15, 0x7a, 0x87, 0x87, 0x85, 0x98, 0xb3, 0xf7, 0x92,
0x26, 0x8c, 0x88, 0x25, 0x6d, 0xd9, 0x4b, 0x04, 0xd7, 0x82, 0x77, 0xe2, 0x75, 0xfc, 0x1e, 0xcf,
0x76, 0xa5, 0xad, 0xef, 0x65, 0x28, 0x8c, 0x1b, 0xbe, 0xd9, 0xe6, 0x62, 0xbb, 0xb6, 0xef, 0x9e,
0x68, 0x52, 0x49, 0xed, 0x36, 0x2c, 0x86, 0x5f, 0xa0, 0x0a, 0xe4, 0x9f, 0xe1, 0x13, 0x31, 0x28,
0xfa, 0x38, 0xfc, 0x08, 0x78, 0xae, 0xe1, 0x8d, 0xdb, 0xb9, 0xdf, 0x51, 0x54, 0x17, 0xd0, 0xb0,
0x97, 0xa7, 0xd8, 0x37, 0x4c, 0xc3, 0x37, 0x10, 0x82, 0x59, 0xb6, 0x8c, 0x70, 0x15, 0xec, 0x99,
0x6a, 0x1d, 0x88, 0x8f, 0xb7, 0xa8, 0xd1, 0x47, 0xf4, 0x06, 0x14, 0x83, 0x40, 0x17, 0x6b, 0xc9,
0x90, 0x40, 0x73, 0xba, 0xe1, 0xfb, 0xb8, 0xef, 0xf8, 0x2c, 0xc4, 0xca, 0x9a, 0x6c, 0xaa, 0xff,
0x3d, 0x0b, 0x95, 0xc4, 0x9c, 0xdc, 0x87, 0x42, 0x5f, 0x74, 0x2f, 0x3e, 0xb4, 0xb7, 0x53, 0x12,
0x7b, 0xc2, 0x54, 0x2d, 0x90, 0xa2, 0x79, 0x93, 0xe6, 0xd0, 0xd0, 0xfa, 0x17, 0xb4, 0xe9, 0x8c,
0xf7, 0x48, 0x57, 0x37, 0x2d, 0x17, 0x77, 0x7c, 0xe2, 0x9e, 0x08, 0x73, 0x17, 0x7b, 0xa4, 0xbb,
0x23, 0x69, 0xe8, 0x36, 0x80, 0x69, 0x7b, 0x74, 0xb2, 0x8f, 0xac, 0x2e, 0x33, 0xba, 0xb4, 0x75,
0x2e, 0x69, 0x44, 0xb0, 0xd8, 0x69, 0x45, 0xd3, 0xf6, 0x84, 0xf9, 0x0f, 0xa0, 0x4c, 0xd7, 0x0c,
0xbd, 0xcf, 0xd7, 0x29, 0xfe, 0xa5, 0x94, 0xb6, 0xce, 0xa7, 0x8d, 0x21, 0x58, 0xcd, 0xb4, 0x45,
0x67, 0xd8, 0xf0, 0xd0, 0x43, 0x98, 0x67, 0xc9, 0xdb, 0xab, 0xce, 0x33, 0xe1, 0xcd, 0x51, 0x0e,
0x10, 0x11, 0xf1, 0x84, 0x09, 0xf0, 0x80, 0x10, 0xd2, 0xe8, 0x00, 0x4a, 0x86, 0x6d, 0x13, 0xdf,
0xe0, 0x89, 0x66, 0x81, 0x29, 0xfb, 0x60, 0x02, 0x65, 0xf5, 0xa1, 0x14, 0xd7, 0x18, 0xd6, 0x83,
0x7e, 0x00, 0x73, 0x2c, 0x13, 0x89, 0x0f, 0xf1, 0xf2, 0x84, 0x41, 0xab, 0x71, 0xa9, 0xda, 0x2d,
0x28, 0x85, 0x8c, 0x9d, 0x26, 0x48, 0x6b, 0x77, 0xa1, 0x12, 0x37, 0x6d, 0xaa, 0x20, 0xff, 0x03,
0x58, 0xd5, 0x06, 0xf6, 0xd0, 0x30, 0x59, 0x7d, 0xdd, 0x86, 0x79, 0x31, 0xd9, 0x3c, 0xe2, 0xd4,
0xf1, 0x3e, 0xd2, 0x84, 0x44, 0xb8, 0x9c, 0x3a, 0x36, 0x6c, 0xb3, 0x87, 0x5d, 0xd1, 0xaf, 0x2c,
0xa7, 0x1e, 0x73, 0xaa, 0xfa, 0x03, 0x38, 0x1d, 0xeb, 0x5c, 0x54, 0x73, 0x6f, 0xc3, 0x92, 0x43,
0x4c, 0xdd, 0xe3, 0x64, 0xdd, 0x32, 0x65, 0x1a, 0x72, 0x02, 0xde, 0x86, 0x49, 0xc5, 0xdb, 0x3e,
0x71, 0x92, 0xc6, 0x4f, 0x26, 0x5e, 0x85, 0x33, 0x71, 0x71, 0xde, 0xbd, 0x7a, 0x0f, 0xd6, 0x34,
0xdc, 0x27, 0xcf, 0xf1, 0xab, 0xaa, 0xae, 0x41, 0x35, 0xa9, 0x40, 0x28, 0xff, 0x0c, 0xd6, 0x86,
0xd4, 0xb6, 0x6f, 0xf8, 0x03, 0x6f, 0x2a, 0xe5, 0xa2, 0xd4, 0x3d, 0x24, 0x1e, 0x9f, 0xce, 0x82,
0x26, 0x9b, 0xea, 0xd5, 0xb0, 0xea, 0x26, 0xaf, 0x2c, 0x78, 0x0f, 0x68, 0x09, 0x72, 0x96, 0x23,
0xd4, 0xe5, 0x2c, 0x47, 0x7d, 0x0c, 0xc5, 0x60, 0x69, 0x46, 0x77, 0x86, 0x35, 0x66, 0x6e, 0xd2,
0x85, 0x3c, 0x28, 0x43, 0xf7, 0x13, 0x4b, 0x89, 0xe8, 0xf2, 0x0e, 0x40, 0x90, 0xf2, 0x64, 0x85,
0x70, 0x6e, 0x84, 0x62, 0x2d, 0xc4, 0xae, 0xfe, 0x4b, 0x24, 0x11, 0x86, 0x06, 0x61, 0x06, 0x83,
0x30, 0x23, 0x89, 0x31, 0xf7, 0x4a, 0x89, 0xf1, 0x43, 0x98, 0xf3, 0x7c, 0xc3, 0xc7, 0xa2, 0x8a,
0xda, 0x18, 0x25, 0x4e, 0x8d, 0xc0, 0x1a, 0xe7, 0x47, 0xe7, 0x01, 0x3a, 0x2e, 0x36, 0x7c, 0x6c,
0xea, 0x06, 0xcf, 0xe2, 0x79, 0xad, 0x28, 0x28, 0x75, 0x1f, 0x6d, 0x0f, 0x2b, 0xc1, 0x39, 0x66,
0xd8, 0xd5, 0x51, 0x9a, 0x23, 0x53, 0x35, 0xac, 0x09, 0x83, 0xac, 0x32, 0x3f, 0x61, 0x56, 0x11,
0x0a, 0xb8, 0x54, 0x28, 0x67, 0x2e, 0x8c, 0xcf, 0x99, 0x5c, 0x74, 0x92, 0x9c, 0x59, 0x18, 0x9f,
0x33, 0x85, 0xb2, 0x91, 0x39, 0xf3, 0xdb, 0x4c, 0x7a, 0xff, 0xac, 0x40, 0x35, 0xf9, 0x0d, 0x8a,
0xdc, 0x73, 0x1b, 0xe6, 0x3d, 0x46, 0x99, 0x24, 0xf3, 0x09, 0x59, 0x21, 0x81, 0x1e, 0xc3, 0xac,
0x65, 0x1f, 0x11, 0xb6, 0x89, 0x4b, 0xad, 0x5d, 0xb2, 0x7a, 0xdd, 0x6c, 0xd8, 0x47, 0x84, 0x3b,
0x89, 0x69, 0xa8, 0x7d, 0x08, 0xc5, 0x80, 0x34, 0xd5, 0xd8, 0xf6, 0x60, 0x35, 0x16, 0xb2, 0xbc,
0xd8, 0x0f, 0x22, 0x5d, 0x99, 0x2e, 0xd2, 0xd5, 0x9f, 0xe6, 0xc2, 0x5f, 0xe2, 0x43, 0xab, 0xe7,
0x63, 0x37, 0xf1, 0x25, 0x7e, 0x24, 0xb5, 0xf3, 0xcf, 0xf0, 0xd2, 0x58, 0xed, 0xbc, 0x26, 0x15,
0x1f, 0xd3, 0x17, 0xb0, 0xc4, 0x62, 0x4d, 0xf7, 0x70, 0x8f, 0x15, 0x1c, 0xa2, 0xf8, 0xfb, 0xfe,
0x28, 0x35, 0xdc, 0x12, 0x1e, 0xb1, 0x6d, 0x21, 0xc7, 0x3d, 0x58, 0xee, 0x85, 0x69, 0xb5, 0xfb,
0x80, 0x92, 0x4c, 0x53, 0xf9, 0xb4, 0x4d, 0x53, 0x1c, 0xdd, 0xe9, 0xa6, 0xac, 0x92, 0x47, 0xcc,
0x8c, 0x49, 0x62, 0x85, 0x1b, 0xac, 0x09, 0x09, 0xf5, 0x97, 0x79, 0x80, 0xe1, 0xcb, 0xff, 0x47,
0xb9, 0xed, 0x7e, 0x90, 0x57, 0x78, 0x21, 0x77, 0x65, 0x94, 0xe2, 0xd4, 0x8c, 0xb2, 0x17, 0xcd,
0x28, 0xbc, 0xa4, 0x7b, 0x6f, 0xa4, 0x9a, 0xef, 0x6c, 0x2e, 0x79, 0x02, 0x67, 0xe2, 0xb1, 0x21,
0x12, 0xc9, 0x16, 0xcc, 0x59, 0x3e, 0xee, 0x73, 0xb4, 0x27, 0x75, 0x77, 0x16, 0x12, 0xe2, 0xac,
0xea, 0x06, 0x14, 0x1b, 0x7d, 0xa3, 0x8b, 0xdb, 0x0e, 0xee, 0xd0, 0x4e, 0x2d, 0xda, 0x10, 0x86,
0xf0, 0x86, 0xba, 0x05, 0x85, 0x1f, 0xe2, 0x13, 0xfe, 0x51, 0x4f, 0x68, 0xa8, 0xfa, 0xa7, 0x39,
0x58, 0x63, 0x6b, 0xc5, 0xb6, 0xc4, 0x5a, 0x34, 0xec, 0x91, 0x81, 0xdb, 0xc1, 0x1e, 0x9b, 0x6d,
0x67, 0xa0, 0x3b, 0xd8, 0xb5, 0x88, 0x29, 0xa0, 0x80, 0x62, 0xc7, 0x19, 0xb4, 0x18, 0x01, 0x9d,
0x03, 0xda, 0xd0, 0xbf, 0x1c, 0x10, 0x11, 0x88, 0x79, 0xad, 0xd0, 0x71, 0x06, 0xbf, 0x47, 0xdb,
0x52, 0xd6, 0x3b, 0x36, 0x5c, 0xec, 0xb1, 0x38, 0xe3, 0xb2, 0x6d, 0x46, 0x40, 0x37, 0xe0, 0x74,
0x1f, 0xf7, 0x89, 0x7b, 0xa2, 0xf7, 0xac, 0xbe, 0xe5, 0xeb, 0x96, 0xad, 0x1f, 0x9e, 0xf8, 0xd8,
0x13, 0x31, 0x85, 0xf8, 0xcb, 0x27, 0xf4, 0x5d, 0xc3, 0x7e, 0x40, 0xdf, 0x20, 0x15, 0xca, 0x84,
0xf4, 0x75, 0xaf, 0x43, 0x5c, 0xac, 0x1b, 0xe6, 0x4f, 0xd8, 0xf2, 0x99, 0xd7, 0x4a, 0x84, 0xf4,
0xdb, 0x94, 0x56, 0x37, 0x7f, 0x82, 0xde, 0x84, 0x52, 0xc7, 0x19, 0x78, 0xd8, 0xd7, 0xe9, 0x1f,
0xb6, 0x3a, 0x16, 0x35, 0xe0, 0xa4, 0x6d, 0x67, 0xe0, 0x85, 0x18, 0xfa, 0xd4, 0xff, 0x0b, 0x61,
0x86, 0xa7, 0xd4, 0xcd, 0x06, 0x94, 0x23, 0x50, 0x02, 0xdd, 0xd5, 0x31, 0xcc, 0x40, 0xec, 0xea,
0xe8, 0x33, 0xa5, 0xb9, 0xa4, 0x27, 0x3d, 0xc9, 0x9e, 0x29, 0xcd, 0x3f, 0x71, 0xe4, 0x96, 0x8e,
0x3d, 0x53, 0x97, 0xf7, 0xf0, 0x73, 0x01, 0x37, 0x15, 0x35, 0xde, 0x50, 0x4d, 0x80, 0x6d, 0xc3,
0x31, 0x0e, 0xad, 0x9e, 0xe5, 0x9f, 0xa0, 0xab, 0x50, 0x31, 0x4c, 0x53, 0xef, 0x48, 0x8a, 0x85,
0x25, 0x08, 0xb8, 0x6c, 0x98, 0xe6, 0x76, 0x88, 0x8c, 0xde, 0x81, 0x15, 0xd3, 0x25, 0x4e, 0x94,
0x97, 0xa3, 0x82, 0x15, 0xfa, 0x22, 0xcc, 0xac, 0xfe, 0xc7, 0x1c, 0x9c, 0x8f, 0x4e, 0x6c, 0x1c,
0xae, 0xb9, 0x0f, 0x8b, 0xb1, 0x5e, 0x33, 0xa0, 0x82, 0xa1, 0xb5, 0x5a, 0x44, 0x22, 0x06, 0x5f,
0xe4, 0x12, 0xf0, 0x45, 0x2a, 0x20, 0x94, 0x7f, 0xad, 0x80, 0xd0, 0xec, 0x6b, 0x01, 0x84, 0xe6,
0xa6, 0x03, 0x84, 0x2e, 0xb1, 0x6d, 0x8c, 0x94, 0x66, 0x7b, 0x67, 0x1e, 0x6a, 0xe5, 0x80, 0xc7,
0x96, 0xe8, 0x71, 0x0c, 0x38, 0x5a, 0x98, 0x06, 0x38, 0x2a, 0x64, 0x02, 0x47, 0x34, 0x6a, 0x1c,
0xc7, 0x70, 0xfb, 0xc4, 0x95, 0xc8, 0x50, 0xb5, 0xc8, 0x4c, 0x58, 0x96, 0x74, 0x81, 0x0a, 0x65,
0x62, 0x48, 0x90, 0x89, 0x21, 0x5d, 0x80, 0x45, 0x9b, 0xe8, 0x36, 0x7e, 0xa1, 0xd3, 0xb9, 0xf4,
0xaa, 0x25, 0x3e, 0xb1, 0x36, 0x69, 0xe2, 0x17, 0x2d, 0x4a, 0x49, 0xa0, 0x4c, 0x8b, 0xd3, 0xa1,
0x4c, 0x68, 0x03, 0x16, 0xfb, 0x86, 0xf7, 0x0c, 0x9b, 0xcc, 0x14, 0xaf, 0x5a, 0x66, 0x41, 0x5c,
0xe2, 0x34, 0x6a, 0x83, 0x87, 0x2e, 0x42, 0xe0, 0x24, 0xc1, 0xb4, 0xc4, 0x98, 0xca, 0x92, 0xca,
0xd8, 0xd4, 0xbf, 0x53, 0x60, 0x35, 0x1a, 0xe6, 0x02, 0x5b, 0x78, 0x04, 0x45, 0x57, 0x66, 0x32,
0x11, 0xda, 0x57, 0x33, 0xca, 0xe4, 0x64, 0xea, 0xd3, 0x86, 0xb2, 0xe8, 0x47, 0x99, 0x90, 0xd6,
0xf5, 0x71, 0xfa, 0xc6, 0x81, 0x5a, 0x6a, 0x03, 0xde, 0xfc, 0xc4, 0xb2, 0x4d, 0xf2, 0xc2, 0xcb,
0xfc, 0x4a, 0x53, 0x62, 0x4d, 0x49, 0x89, 0x35, 0xf5, 0x17, 0x0a, 0x9c, 0x89, 0xeb, 0x12, 0xae,
0x68, 0x24, 0x5d, 0xf1, 0x4e, 0xd2, 0xf4, 0xb8, 0x70, 0xaa, 0x33, 0xbe, 0xc8, 0x74, 0xc6, 0x8d,
0xf1, 0x1a, 0xc7, 0xba, 0xe3, 0xaf, 0x14, 0x38, 0x9b, 0x69, 0x46, 0x6c, 0x49, 0x51, 0xe2, 0x4b,
0x8a, 0x58, 0x8e, 0x3a, 0x64, 0x60, 0xfb, 0xa1, 0xe5, 0x68, 0x9b, 0x1d, 0x31, 0xf0, 0xbc, 0xaf,
0xf7, 0x8d, 0x97, 0x56, 0x7f, 0xd0, 0x17, 0xeb, 0x11, 0x55, 0xf7, 0x94, 0x53, 0x5e, 0x61, 0x41,
0x52, 0xeb, 0xb0, 0x12, 0x58, 0x39, 0x12, 0x04, 0x0c, 0x81, 0x7a, 0xb9, 0x28, 0xa8, 0x67, 0xc3,
0xfc, 0x0e, 0x7e, 0x6e, 0x75, 0xf0, 0x6b, 0x39, 0x03, 0xb9, 0x00, 0x25, 0x07, 0xbb, 0x7d, 0xcb,
0xf3, 0x82, 0x44, 0x5b, 0xd4, 0xc2, 0x24, 0xf5, 0x3f, 0xe7, 0x61, 0x39, 0x1e, 0x1d, 0xf7, 0x12,
0x18, 0xe2, 0x5b, 0x29, 0x4b, 0x40, 0x7c, 0xa0, 0xa1, 0x6a, 0xf2, 0x86, 0x2c, 0x46, 0x72, 0x59,
0x1b, 0xf9, 0xa0, 0x70, 0x11, 0x95, 0x0a, 0xf5, 0x48, 0x87, 0xf4, 0xfb, 0x86, 0x6d, 0xca, 0xa3,
0x2b, 0xd1, 0xa4, 0xfe, 0x33, 0xdc, 0x2e, 0x75, 0x3b, 0x25, 0xb3, 0x67, 0x3a, 0x79, 0x74, 0xd7,
0x6b, 0xd9, 0x0c, 0x8b, 0x64, 0xc9, 0xba, 0xa8, 0x81, 0x20, 0xed, 0x58, 0x2e, 0xda, 0x84, 0x59,
0x6c, 0x3f, 0x97, 0xe5, 0x62, 0xca, 0xd9, 0x96, 0x2c, 0x8b, 0x34, 0xc6, 0x87, 0xae, 0xc3, 0x7c,
0x9f, 0x86, 0x85, 0xdc, 0xff, 0xae, 0x65, 0x1c, 0xf1, 0x68, 0x82, 0x0d, 0x6d, 0xc1, 0x82, 0xc9,
0xe6, 0x49, 0x6e, 0x72, 0xab, 0x29, 0x08, 0x27, 0x63, 0xd0, 0x24, 0x23, 0xda, 0x0d, 0x8a, 0xe1,
0x62, 0x56, 0x15, 0x1b, 0x9b, 0x8a, 0xd4, 0x8a, 0x78, 0x3f, 0x5a, 0x11, 0x03, 0xd3, 0xb5, 0x35,
0x5e, 0xd7, 0x68, 0x58, 0xf2, 0x2c, 0x14, 0x7a, 0xa4, 0xcb, 0xc3, 0xa8, 0xc4, 0x4f, 0x45, 0x7b,
0xa4, 0xcb, 0xa2, 0x68, 0x95, 0x6e, 0x0e, 0x4c, 0xcb, 0x66, 0x49, 0xbd, 0xa0, 0xf1, 0x06, 0xfd,
0xf8, 0xd8, 0x83, 0x4e, 0xec, 0x0e, 0xae, 0x96, 0xd9, 0xab, 0x22, 0xa3, 0xec, 0xd9, 0x1d, 0x56,
0x6e, 0xfa, 0xfe, 0x49, 0x75, 0x89, 0xd1, 0xe9, 0x23, 0xdd, 0xf7, 0x71, 0x88, 0x62, 0x39, 0x6b,
0xdf, 0x97, 0x96, 0xb6, 0x25, 0x42, 0xf1, 0x00, 0x16, 0x5e, 0xf0, 0x44, 0x50, 0xad, 0x30, 0xf9,
0x2b, 0xe3, 0xd3, 0x8b, 0xd0, 0x20, 0x05, 0xbf, 0xcd, 0xd2, 0xff, 0x97, 0x0a, 0x9c, 0xd9, 0x66,
0xdb, 0xa2, 0x50, 0x1e, 0x9b, 0x06, 0xc9, 0xbb, 0x15, 0x80, 0xac, 0x99, 0xb0, 0x5b, 0x7c, 0xdc,
0x12, 0x63, 0x6d, 0xc0, 0x92, 0x54, 0x2e, 0x54, 0xe4, 0x27, 0xc6, 0x69, 0xcb, 0x5e, 0xb8, 0xa9,
0x7e, 0x04, 0x6b, 0x89, 0x51, 0x88, 0x2d, 0xcc, 0x06, 0x2c, 0x0e, 0xf3, 0x55, 0x30, 0x88, 0x52,
0x40, 0x6b, 0x98, 0xea, 0x6d, 0x38, 0xdd, 0xf6, 0x0d, 0xd7, 0x4f, 0xb8, 0x60, 0x02, 0x59, 0x86,
0xc0, 0x46, 0x65, 0x05, 0x48, 0xda, 0x86, 0xd5, 0xb6, 0x4f, 0x9c, 0x57, 0x50, 0x4a, 0xb3, 0x0e,
0x1d, 0x3f, 0x19, 0xc8, 0xf5, 0x41, 0x36, 0xd5, 0x35, 0x8e, 0x17, 0x27, 0x7b, 0xbb, 0x03, 0x67,
0x38, 0x5c, 0xfb, 0x2a, 0x83, 0x38, 0x2b, 0xc1, 0xe2, 0xa4, 0xde, 0xa7, 0x70, 0x6a, 0xb8, 0x2c,
0x0e, 0xa1, 0x98, 0x9b, 0x51, 0x28, 0xe6, 0xc2, 0x88, 0x59, 0x8f, 0x20, 0x31, 0x7f, 0x91, 0x0b,
0xe5, 0xf5, 0x0c, 0x20, 0xe6, 0x4e, 0x14, 0x88, 0xb9, 0x38, 0x4e, 0x77, 0x04, 0x87, 0x49, 0x46,
0x6d, 0x3e, 0x25, 0x6a, 0x3f, 0x4f, 0xa0, 0x35, 0xb3, 0x59, 0x70, 0x57, 0xcc, 0xda, 0xdf, 0x0a,
0x58, 0xa3, 0x71, 0xb0, 0x26, 0xe8, 0x3a, 0x40, 0xd7, 0x6f, 0xc5, 0xc0, 0x9a, 0x8d, 0xb1, 0xf6,
0x06, 0x58, 0xcd, 0xdf, 0xcc, 0x42, 0x31, 0x78, 0x97, 0xf0, 0x79, 0xd2, 0x6d, 0xb9, 0x14, 0xb7,
0x85, 0x57, 0xe0, 0xfc, 0x37, 0x5a, 0x81, 0x67, 0x27, 0x5e, 0x81, 0xcf, 0x41, 0x91, 0x3d, 0xe8,
0x2e, 0x3e, 0x12, 0x2b, 0x6a, 0x81, 0x11, 0x34, 0x7c, 0x34, 0x0c, 0xc3, 0xf9, 0xa9, 0xc2, 0x30,
0x06, 0x0f, 0x2d, 0xc4, 0xe1, 0xa1, 0x7b, 0xc1, 0x8a, 0xc8, 0x17, 0xd1, 0xcb, 0x23, 0xf4, 0xa6,
0xae, 0x85, 0xcd, 0xe8, 0x5a, 0xc8, 0xd7, 0xd5, 0x77, 0x47, 0x69, 0xf9, 0xce, 0x82, 0x43, 0x07,
0x1c, 0x1c, 0x0a, 0xc7, 0xa2, 0xc8, 0xac, 0x77, 0x00, 0x82, 0x24, 0x22, 0x11, 0xa2, 0x73, 0x23,
0xc6, 0xa8, 0x85, 0xd8, 0xa9, 0xda, 0xc8, 0xd4, 0x0c, 0x4f, 0x90, 0x26, 0xcb, 0x8f, 0x19, 0xc7,
0x47, 0xff, 0x3b, 0x17, 0xca, 0x2f, 0x19, 0x47, 0x2e, 0xf7, 0x12, 0xb0, 0xe4, 0x94, 0x51, 0x7c,
0x33, 0x8a, 0x4a, 0xbe, 0x62, 0xd4, 0x25, 0x40, 0x49, 0x56, 0xb9, 0x18, 0xae, 0x78, 0xcd, 0x41,
0xa3, 0xa2, 0xa0, 0xd4, 0xd9, 0xce, 0xe0, 0xc8, 0xb2, 0x2d, 0xef, 0x98, 0xbf, 0x9f, 0xe7, 0x3b,
0x03, 0x49, 0xaa, 0xb3, 0xdb, 0x4d, 0xf8, 0xa5, 0xe5, 0xeb, 0x1d, 0x62, 0x62, 0x16, 0xd3, 0x73,
0x5a, 0x81, 0x12, 0xb6, 0x89, 0x89, 0x87, 0x5f, 0x5e, 0xe1, 0xd5, 0xbe, 0xbc, 0x62, 0xec, 0xcb,
0x3b, 0x03, 0xf3, 0x2e, 0x36, 0x3c, 0x62, 0x8b, 0xed, 0xb9, 0x68, 0xd1, 0xa9, 0xe9, 0x63, 0xcf,
0xa3, 0x3d, 0x89, 0x72, 0x4d, 0x34, 0x43, 0x65, 0xe6, 0xe2, 0xd8, 0x32, 0x73, 0xc4, 0x51, 0x4e,
0xac, 0xcc, 0x2c, 0x8f, 0x2d, 0x33, 0x27, 0x39, 0xc9, 0x09, 0x15, 0xda, 0x4b, 0x93, 0x15, 0xda,
0xe1, 0xba, 0x74, 0x39, 0x52, 0x97, 0x7e, 0x9b, 0x1f, 0xeb, 0xaf, 0x15, 0x58, 0x4b, 0x7c, 0x56,
0xe2, 0x73, 0xbd, 0x15, 0x3b, 0x14, 0xda, 0x18, 0xeb, 0xb3, 0xe0, 0x4c, 0xe8, 0x51, 0xe4, 0x4c,
0xe8, 0x83, 0xf1, 0x82, 0xaf, 0xfd, 0x48, 0xe8, 0x8f, 0x15, 0x78, 0xf3, 0xc0, 0x31, 0x63, 0x15,
0x9e, 0xd8, 0xf6, 0x4f, 0x9e, 0x38, 0xee, 0xc9, 0x5a, 0x3f, 0x37, 0x2d, 0xce, 0xc2, 0xe5, 0x54,
0x15, 0x2e, 0x64, 0x9b, 0x21, 0x4a, 0xa6, 0x1f, 0xc3, 0xf2, 0xee, 0x4b, 0xdc, 0x69, 0x9f, 0xd8,
0x9d, 0x29, 0x4c, 0xab, 0x40, 0xbe, 0xd3, 0x37, 0x05, 0x4a, 0x4a, 0x1f, 0xc3, 0x55, 0x60, 0x3e,
0x5a, 0x05, 0xea, 0x50, 0x19, 0xf6, 0x20, 0xa6, 0xf7, 0x0c, 0x9d, 0x5e, 0x93, 0x32, 0x53, 0xe5,
0x8b, 0x9a, 0x68, 0x09, 0x3a, 0x76, 0xf9, 0x05, 0x06, 0x4e, 0xc7, 0xae, 0x1b, 0xcd, 0x16, 0xf9,
0x68, 0xb6, 0x50, 0xff, 0x5c, 0x81, 0x12, 0xed, 0xe1, 0x1b, 0xd9, 0x2f, 0xb6, 0x5a, 0xf9, 0xe1,
0x56, 0x2b, 0xd8, 0xb1, 0xcd, 0x86, 0x77, 0x6c, 0x43, 0xcb, 0xe7, 0x18, 0x39, 0x69, 0xf9, 0x7c,
0x40, 0xc7, 0xae, 0xab, 0x5e, 0x80, 0x45, 0x6e, 0x9b, 0x18, 0x79, 0x05, 0xf2, 0x03, 0xb7, 0x27,
0xe3, 0x68, 0xe0, 0xf6, 0xd4, 0x3f, 0x51, 0xa0, 0x5c, 0xf7, 0x7d, 0xa3, 0x73, 0x3c, 0xc5, 0x00,
0x02, 0xe3, 0x72, 0x61, 0xe3, 0x92, 0x83, 0x18, 0x9a, 0x3b, 0x9b, 0x61, 0xee, 0x5c, 0xc4, 0x5c,
0x15, 0x96, 0xa4, 0x2d, 0x99, 0x06, 0x37, 0x01, 0xb5, 0x88, 0xeb, 0x3f, 0x24, 0xee, 0x0b, 0xc3,
0x35, 0xa7, 0xdb, 0x81, 0x21, 0x98, 0x15, 0x37, 0x5e, 0xf3, 0x57, 0xe6, 0x34, 0xf6, 0xac, 0x5e,
0x86, 0x53, 0x11, 0x7d, 0x99, 0x1d, 0xdf, 0x87, 0x12, 0xcb, 0xfb, 0xa2, 0x14, 0xbf, 0x11, 0x3e,
0xae, 0x99, 0x68, 0x95, 0x50, 0x7f, 0x17, 0x56, 0x68, 0x7d, 0xc0, 0xe8, 0xc1, 0xa7, 0xf8, 0xfd,
0x58, 0x9d, 0x7a, 0x3e, 0x43, 0x51, 0xac, 0x46, 0xfd, 0x5b, 0x05, 0xe6, 0x18, 0x3d, 0xb1, 0x66,
0x9f, 0x83, 0xa2, 0x8b, 0x1d, 0xa2, 0xfb, 0x46, 0x37, 0xb8, 0x5f, 0x4c, 0x09, 0xfb, 0x46, 0xd7,
0x63, 0xd7, 0xa3, 0xe9, 0x4b, 0xd3, 0xea, 0x62, 0xcf, 0x97, 0x97, 0x8c, 0x4b, 0x94, 0xb6, 0xc3,
0x49, 0xd4, 0x49, 0x9e, 0xf5, 0xfb, 0xbc, 0xee, 0x9c, 0xd5, 0xd8, 0x33, 0xda, 0xe4, 0x57, 0xde,
0x26, 0x81, 0xd4, 0xd9, 0x85, 0xb8, 0x1a, 0x14, 0x62, 0x28, 0x7a, 0xd0, 0x56, 0x77, 0x01, 0x85,
0xbd, 0x20, 0xfc, 0x7d, 0x1d, 0xe6, 0x99, 0x93, 0x64, 0x75, 0xb4, 0x96, 0xe1, 0x06, 0x4d, 0xb0,
0xa9, 0x06, 0x20, 0xee, 0xe0, 0x48, 0x45, 0x34, 0xfd, 0xac, 0x8c, 0xa8, 0x90, 0xfe, 0x5e, 0x81,
0x53, 0x91, 0x3e, 0x84, 0xad, 0xef, 0x45, 0x3b, 0xc9, 0x34, 0x55, 0x74, 0xb0, 0x1d, 0x59, 0x12,
0xae, 0x67, 0x99, 0xf4, 0x1b, 0x5a, 0x0e, 0xfe, 0x41, 0x01, 0xa8, 0x0f, 0xfc, 0x63, 0x81, 0x0c,
0x86, 0x67, 0x46, 0x89, 0xce, 0x0c, 0x7d, 0xe7, 0x18, 0x9e, 0xf7, 0x82, 0xb8, 0x72, 0x4f, 0x13,
0xb4, 0x19, 0x86, 0x37, 0xf0, 0x8f, 0xe5, 0x51, 0x18, 0x7d, 0x46, 0x17, 0x61, 0x89, 0xdf, 0x69,
0xd7, 0x0d, 0xd3, 0x74, 0xb1, 0xe7, 0x89, 0x33, 0xb1, 0x32, 0xa7, 0xd6, 0x39, 0x91, 0xb2, 0x59,
0x26, 0xb6, 0x7d, 0xcb, 0x3f, 0xd1, 0x7d, 0xf2, 0x0c, 0xdb, 0x62, 0x6f, 0x52, 0x96, 0xd4, 0x7d,
0x4a, 0xe4, 0x87, 0x03, 0x5d, 0xcb, 0xf3, 0x5d, 0xc9, 0x26, 0xcf, 0x5f, 0x04, 0x95, 0xb1, 0xd1,
0x49, 0xa9, 0xb4, 0x06, 0xbd, 0x1e, 0x77, 0xf1, 0xab, 0x4f, 0xfb, 0xfb, 0x62, 0x40, 0xb9, 0xac,
0x98, 0x1e, 0x3a, 0x4d, 0x0c, 0xf7, 0x35, 0x82, 0x30, 0xef, 0xc3, 0x4a, 0x68, 0x0c, 0x22, 0xac,
0x22, 0x45, 0xa4, 0x12, 0x2d, 0x22, 0xd5, 0x47, 0x80, 0x38, 0xee, 0xf0, 0x0d, 0xc7, 0xad, 0x9e,
0x86, 0x53, 0x11, 0x45, 0x62, 0x25, 0xbe, 0x06, 0x65, 0x71, 0x2f, 0x49, 0x04, 0xca, 0x59, 0x28,
0xd0, 0x8c, 0xda, 0xb1, 0x4c, 0x79, 0x4e, 0xba, 0xe0, 0x10, 0x73, 0xdb, 0x32, 0x5d, 0xf5, 0x13,
0x28, 0x6b, 0xbc, 0x1f, 0xc1, 0xfb, 0x10, 0x96, 0xc4, 0x2d, 0x26, 0x3d, 0x72, 0x8d, 0x30, 0xed,
0x9a, 0x7a, 0xb8, 0x13, 0xad, 0x6c, 0x87, 0x9b, 0xaa, 0x09, 0x35, 0x5e, 0x32, 0x44, 0xd4, 0xcb,
0xc1, 0x3e, 0x04, 0x79, 0xa3, 0x70, 0x6c, 0x2f, 0x51, 0xf9, 0xb2, 0x1b, 0x6e, 0xaa, 0xe7, 0xe1,
0x5c, 0x6a, 0x2f, 0xc2, 0x13, 0x0e, 0x54, 0x86, 0x2f, 0x4c, 0x4b, 0x1e, 0x18, 0xb3, 0x83, 0x60,
0x25, 0x74, 0x10, 0x7c, 0x26, 0x28, 0x12, 0x73, 0x72, 0x11, 0x63, 0x15, 0xe0, 0xb0, 0xdc, 0xcf,
0x67, 0x95, 0xfb, 0xb3, 0x91, 0x72, 0x5f, 0x6d, 0x07, 0xfe, 0x14, 0xdb, 0xb0, 0x07, 0x6c, 0xbb,
0xc8, 0xfb, 0x96, 0x09, 0x51, 0x1d, 0x35, 0x4a, 0xce, 0xaa, 0x85, 0xa4, 0xd4, 0xab, 0x50, 0x8e,
0xa6, 0xc6, 0x50, 0x9e, 0x53, 0x12, 0x79, 0x6e, 0x29, 0x96, 0xe2, 0x3e, 0x8c, 0x55, 0xc0, 0xd9,
0x3e, 0x8e, 0xd5, 0xbf, 0x77, 0x23, 0xc9, 0xee, 0x5a, 0xca, 0x19, 0xee, 0x6f, 0x28, 0xcf, 0xad,
0x8a, 0xf5, 0xe0, 0xa1, 0x47, 0xe5, 0xc5, 0xa0, 0xd5, 0xb7, 0xa0, 0x74, 0x90, 0xf5, 0x1b, 0x88,
0x59, 0x79, 0x5f, 0xe2, 0x26, 0xac, 0x3e, 0xb4, 0x7a, 0xd8, 0x3b, 0xf1, 0x7c, 0xdc, 0x6f, 0xb0,
0xa4, 0x74, 0x64, 0x61, 0x17, 0xad, 0x03, 0xb0, 0x2d, 0x8c, 0x43, 0xac, 0xe0, 0x6a, 0x7c, 0x88,
0xa2, 0xfe, 0x97, 0x02, 0xcb, 0x43, 0xc1, 0x03, 0xb6, 0x75, 0x7b, 0x03, 0x8a, 0x74, 0xbc, 0x9e,
0x6f, 0xf4, 0x1d, 0x79, 0x9e, 0x15, 0x10, 0xd0, 0x1d, 0x98, 0x3b, 0xf2, 0x24, 0x64, 0x94, 0x0a,
0xa0, 0xa7, 0x19, 0xa2, 0xcd, 0x1e, 0x79, 0x0d, 0x13, 0x7d, 0x04, 0x30, 0xf0, 0xb0, 0x29, 0xce,
0xb0, 0xf2, 0x59, 0xd5, 0xc2, 0x41, 0xf8, 0x7c, 0x9b, 0x0a, 0xf0, 0xab, 0x16, 0x77, 0xa1, 0x64,
0xd9, 0xc4, 0xc4, 0xec, 0xcc, 0xd1, 0x14, 0xa8, 0xd2, 0x18, 0x71, 0xe0, 0x12, 0x07, 0x1e, 0x36,
0x55, 0x2c, 0xd6, 0x42, 0xe9, 0x5f, 0x11, 0x28, 0x4d, 0x58, 0xe1, 0x49, 0xeb, 0x28, 0x30, 0x5c,
0x46, 0xec, 0xc6, 0xa8, 0xd1, 0x31, 0x6f, 0x69, 0x15, 0x4b, 0x94, 0x36, 0x52, 0x54, 0xbd, 0x0d,
0xa7, 0x23, 0x3b, 0xa4, 0x29, 0xb6, 0x2c, 0x6a, 0x2b, 0x06, 0x94, 0x0c, 0xc3, 0x59, 0xc0, 0x10,
0x32, 0x9a, 0xc7, 0xc1, 0x10, 0x1e, 0x87, 0x21, 0x3c, 0xf5, 0x73, 0x38, 0x1b, 0x41, 0x74, 0x22,
0x16, 0xdd, 0x8d, 0x55, 0x6e, 0x97, 0xc6, 0x69, 0x8d, 0x95, 0x70, 0xff, 0xa3, 0xc0, 0x6a, 0x1a,
0xc3, 0x2b, 0x22, 0x8e, 0x3f, 0xce, 0xb8, 0x56, 0x77, 0x6b, 0x32, 0xb3, 0x7e, 0x2b, 0x68, 0xed,
0x3e, 0xd4, 0xd2, 0xfc, 0x99, 0x9c, 0xa5, 0xfc, 0x34, 0xb3, 0xf4, 0xb3, 0x7c, 0x08, 0x79, 0xaf,
0xfb, 0xbe, 0x6b, 0x1d, 0x0e, 0x68, 0xc8, 0xbf, 0x76, 0x34, 0xab, 0x11, 0xe0, 0x32, 0xdc, 0xb5,
0x37, 0x46, 0x88, 0x0f, 0xed, 0x48, 0xc5, 0x66, 0x3e, 0x8d, 0x62, 0x33, 0x1c, 0x53, 0xbf, 0x39,
0x99, 0xbe, 0xef, 0x2c, 0x00, 0xfa, 0xb3, 0x1c, 0x2c, 0x45, 0xa7, 0x08, 0xed, 0x02, 0x18, 0x81,
0xe5, 0xe2, 0x43, 0xb9, 0x38, 0xd1, 0x30, 0xb5, 0x90, 0x20, 0x7a, 0x17, 0xf2, 0x1d, 0x67, 0x20,
0x66, 0x2d, 0xe5, 0x30, 0x78, 0xdb, 0x19, 0xf0, 0x8c, 0x42, 0xd9, 0xe8, 0x9e, 0x8a, 0x9f, 0xed,
0x67, 0x67, 0xc9, 0xa7, 0xec, 0x3d, 0x97, 0x11, 0xcc, 0xe8, 0x31, 0x2c, 0xbd, 0x70, 0x2d, 0xdf,
0x38, 0xec, 0x61, 0xbd, 0x67, 0x9c, 0x60, 0x57, 0x64, 0xc9, 0x09, 0x12, 0x59, 0x59, 0x0a, 0x3e,
0xa1, 0x72, 0xea, 0x1f, 0x42, 0x41, 0x5a, 0x34, 0x66, 0x45, 0xd8, 0x87, 0xb5, 0x01, 0x65, 0xd3,
0xd9, 0x15, 0x38, 0xdb, 0xb0, 0x89, 0xee, 0x61, 0xba, 0x8c, 0xcb, 0xcb, 0xf9, 0x63, 0x52, 0xf4,
0x2a, 0x93, 0xde, 0x26, 0x2e, 0x6e, 0x1a, 0x36, 0x69, 0x73, 0x51, 0xf5, 0x39, 0x94, 0x42, 0x03,
0x1c, 0x63, 0x42, 0x03, 0x56, 0xe4, 0x51, 0xbc, 0x87, 0x7d, 0xb1, 0xbc, 0x4c, 0xd4, 0xf9, 0xb2,
0x90, 0x6b, 0x63, 0x9f, 0x5f, 0x9f, 0xb8, 0x0b, 0x67, 0x35, 0x4c, 0x1c, 0x6c, 0x07, 0xf3, 0xf9,
0x84, 0x74, 0xa7, 0xc8, 0xe0, 0x6f, 0x40, 0x2d, 0x4d, 0x9e, 0xe7, 0x87, 0x6b, 0x97, 0xa0, 0x20,
0x7f, 0xd0, 0x8a, 0x16, 0x20, 0xbf, 0xbf, 0xdd, 0xaa, 0xcc, 0xd0, 0x87, 0x83, 0x9d, 0x56, 0x45,
0x41, 0x05, 0x98, 0x6d, 0x6f, 0xef, 0xb7, 0x2a, 0xb9, 0x6b, 0x7d, 0xa8, 0xc4, 0x7f, 0xcd, 0x89,
0xd6, 0xe0, 0x54, 0x4b, 0xdb, 0x6b, 0xd5, 0x1f, 0xd5, 0xf7, 0x1b, 0x7b, 0x4d, 0xbd, 0xa5, 0x35,
0x3e, 0xae, 0xef, 0xef, 0x56, 0x66, 0xd0, 0x06, 0x9c, 0x0f, 0xbf, 0x78, 0xbc, 0xd7, 0xde, 0xd7,
0xf7, 0xf7, 0xf4, 0xed, 0xbd, 0xe6, 0x7e, 0xbd, 0xd1, 0xdc, 0xd5, 0x2a, 0x0a, 0x3a, 0x0f, 0x67,
0xc3, 0x2c, 0x0f, 0x1a, 0x3b, 0x0d, 0x6d, 0x77, 0x9b, 0x3e, 0xd7, 0x9f, 0x54, 0x72, 0xd7, 0x6e,
0x40, 0x39, 0xf2, 0xe3, 0x4b, 0x6a, 0x52, 0x6b, 0x6f, 0xa7, 0x32, 0x83, 0xca, 0x50, 0x0c, 0xeb,
0x29, 0xc0, 0x6c, 0x73, 0x6f, 0x67, 0xb7, 0x92, 0xbb, 0x76, 0x1b, 0x96, 0x63, 0xb7, 0x71, 0xd1,
0x0a, 0x94, 0xdb, 0xf5, 0xe6, 0xce, 0x83, 0xbd, 0x4f, 0x75, 0x6d, 0xb7, 0xbe, 0xf3, 0x59, 0x65,
0x06, 0xad, 0x42, 0x45, 0x92, 0x9a, 0x7b, 0xfb, 0x9c, 0xaa, 0x5c, 0x7b, 0x16, 0xfb, 0xc6, 0x30,
0x3a, 0x0d, 0x2b, 0x41, 0x37, 0xfa, 0xb6, 0xb6, 0x5b, 0xdf, 0xdf, 0xa5, 0xbd, 0x47, 0xc8, 0xda,
0x41, 0xb3, 0xd9, 0x68, 0x3e, 0xaa, 0x28, 0x54, 0xeb, 0x90, 0xbc, 0xfb, 0x69, 0x83, 0x32, 0xe7,
0xa2, 0xcc, 0x07, 0xcd, 0x1f, 0x36, 0xf7, 0x3e, 0x69, 0x56, 0xf2, 0x5b, 0xbf, 0x58, 0x81, 0x25,
0x59, 0xe8, 0x61, 0x97, 0xdd, 0x6a, 0x69, 0xc1, 0x82, 0xfc, 0x81, 0x74, 0x4a, 0x86, 0x8e, 0xfe,
0xac, 0xbb, 0xb6, 0x31, 0x82, 0x43, 0xd4, 0xdb, 0x33, 0xe8, 0x90, 0xd5, 0xbf, 0xa1, 0xdb, 0xd1,
0x97, 0x52, 0xab, 0xcd, 0xc4, 0x85, 0xec, 0xda, 0xe5, 0xb1, 0x7c, 0x41, 0x1f, 0x98, 0x96, 0xb8,
0xe1, 0x9f, 0xff, 0xa0, 0xcb, 0x69, 0xb5, 0x69, 0xca, 0xef, 0x8b, 0x6a, 0x57, 0xc6, 0x33, 0x06,
0xdd, 0x3c, 0x83, 0x4a, 0xfc, 0xa7, 0x40, 0x28, 0x05, 0x3a, 0xcd, 0xf8, 0xbd, 0x51, 0xed, 0xda,
0x24, 0xac, 0xe1, 0xce, 0x12, 0x3f, 0x9a, 0xb9, 0x3a, 0xc9, 0xaf, 0x10, 0x32, 0x3b, 0xcb, 0xfa,
0xc1, 0x02, 0x77, 0x60, 0xf4, 0xe6, 0x33, 0x4a, 0xfd, 0x85, 0x4a, 0xca, 0xbd, 0xf9, 0x34, 0x07,
0xa6, 0x5f, 0xa2, 0x56, 0x67, 0xd0, 0x31, 0x2c, 0xc7, 0xae, 0x27, 0xa0, 0x14, 0xf1, 0xf4, 0x7b,
0x18, 0xb5, 0xab, 0x13, 0x70, 0x46, 0x23, 0x22, 0x7c, 0x1d, 0x21, 0x3d, 0x22, 0x52, 0x2e, 0x3b,
0xa4, 0x47, 0x44, 0xea, 0xcd, 0x06, 0x16, 0xdc, 0x91, 0x6b, 0x08, 0x69, 0xc1, 0x9d, 0x76, 0xf9,
0xa1, 0x76, 0x79, 0x2c, 0x5f, 0xd8, 0x69, 0xb1, 0x4b, 0x09, 0x69, 0x4e, 0x4b, 0xbf, 0xf4, 0x50,
0xbb, 0x3a, 0x01, 0x67, 0x3c, 0x0a, 0x86, 0x47, 0x9c, 0x59, 0x51, 0x90, 0x38, 0x90, 0xcf, 0x8a,
0x82, 0xe4, 0x69, 0xa9, 0x88, 0x82, 0xd8, 0xd1, 0xe4, 0x95, 0x09, 0x8e, 0x52, 0xb2, 0xa3, 0x20,
0xfd, 0xd0, 0x45, 0x9d, 0x41, 0x7f, 0xa4, 0x40, 0x35, 0xeb, 0x98, 0x02, 0xa5, 0xd4, 0x77, 0x63,
0x4e, 0x56, 0x6a, 0x5b, 0xd3, 0x88, 0x04, 0x56, 0x7c, 0x09, 0x28, 0xb9, 0xee, 0xa1, 0x77, 0xd2,
0x66, 0x26, 0x63, 0x75, 0xad, 0xbd, 0x3b, 0x19, 0x73, 0xd0, 0x65, 0x1b, 0x0a, 0xf2, 0x60, 0x04,
0xa5, 0x64, 0xe9, 0xd8, 0xb1, 0x4c, 0x4d, 0x1d, 0xc5, 0x12, 0x28, 0x7d, 0x04, 0xb3, 0x94, 0x8a,
0xce, 0xa7, 0x73, 0x4b, 0x65, 0xeb, 0x59, 0xaf, 0x03, 0x45, 0x4f, 0x61, 0x9e, 0x9f, 0x04, 0xa0,
0x14, 0xe4, 0x21, 0x72, 0x5e, 0x51, 0xbb, 0x90, 0xcd, 0x10, 0xa8, 0xfb, 0x82, 0xff, 0xef, 0x0c,
0x01, 0xf2, 0xa3, 0xb7, 0xd3, 0x7f, 0x8c, 0x1c, 0x3d, 0x53, 0xa8, 0x5d, 0x1c, 0xc3, 0x15, 0xfe,
0x28, 0x62, 0x55, 0xef, 0xe5, 0xb1, 0x5b, 0x97, 0xec, 0x8f, 0x22, 0x7d, 0x73, 0xc4, 0x83, 0x24,
0xb9, 0x79, 0x4a, 0x0b, 0x92, 0xcc, 0x2d, 0x6b, 0x5a, 0x90, 0x64, 0xef, 0xc7, 0xd4, 0x19, 0xe4,
0xc3, 0xa9, 0x14, 0xa8, 0x0c, 0xbd, 0x9b, 0x15, 0xe4, 0x69, 0xb8, 0x5d, 0xed, 0xbd, 0x09, 0xb9,
0xc3, 0x93, 0x2f, 0x3e, 0xfa, 0x37, 0xb3, 0xf1, 0xa3, 0xcc, 0xc9, 0x8f, 0x7f, 0xe2, 0x5b, 0xff,
0x9a, 0x87, 0x45, 0x0e, 0x83, 0x8a, 0x0a, 0xe6, 0x33, 0x80, 0xe1, 0x09, 0x04, 0x7a, 0x2b, 0xdd,
0x27, 0x91, 0x53, 0x9a, 0xda, 0xdb, 0xa3, 0x99, 0xc2, 0x81, 0x16, 0x42, 0xf3, 0xd3, 0x02, 0x2d,
0x79, 0x68, 0x91, 0x16, 0x68, 0x29, 0x47, 0x02, 0xea, 0x0c, 0xfa, 0x18, 0x8a, 0x01, 0x6c, 0x8c,
0xd2, 0x60, 0xe7, 0x18, 0x2e, 0x5e, 0x7b, 0x6b, 0x24, 0x4f, 0xd8, 0xea, 0x10, 0x26, 0x9c, 0x66,
0x75, 0x12, 0x7b, 0x4e, 0xb3, 0x3a, 0x0d, 0x58, 0x1e, 0xfa, 0x84, 0x23, 0x47, 0x99, 0x3e, 0x89,
0x00, 0x77, 0x99, 0x3e, 0x89, 0xc2, 0x4f, 0xea, 0xcc, 0x83, 0x4b, 0xbf, 0xfa, 0x6a, 0x5d, 0xf9,
0xa7, 0xaf, 0xd6, 0x67, 0x7e, 0xfa, 0xf5, 0xba, 0xf2, 0xab, 0xaf, 0xd7, 0x95, 0x7f, 0xfc, 0x7a,
0x5d, 0xf9, 0xb7, 0xaf, 0xd7, 0x95, 0x3f, 0xfb, 0xf7, 0xf5, 0x99, 0x1f, 0x15, 0xa4, 0xf4, 0xe1,
0x3c, 0xfb, 0x0f, 0x38, 0x1f, 0xfc, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x15, 0x30, 0x49, 0xcc,
0xc7, 0x48, 0x00, 0x00,
0xf8, 0x73, 0xc7, 0xf2, 0x58, 0xb3, 0xeb, 0x89, 0xed, 0x59, 0xdb, 0xb4, 0x24, 0xdb, 0xcc, 0xda,
0x14, 0xd3, 0x94, 0xe6, 0x63, 0x67, 0x80, 0xde, 0x16, 0xbb, 0x44, 0xf5, 0x9a, 0xec, 0xea, 0xe9,
0x6e, 0xda, 0x56, 0x02, 0x04, 0x0b, 0x2c, 0xb2, 0x87, 0x00, 0x01, 0x72, 0xce, 0x71, 0x73, 0xc8,
0x21, 0xb7, 0x00, 0x41, 0x0e, 0x39, 0x6d, 0x90, 0xc3, 0x5e, 0x02, 0xe4, 0xb4, 0x48, 0x90, 0x4b,
0x66, 0x92, 0x5c, 0x02, 0x24, 0xc8, 0x1f, 0x90, 0x43, 0x50, 0x5f, 0xcd, 0xfe, 0xe4, 0x87, 0xc7,
0xbb, 0xb3, 0x39, 0xa9, 0xfb, 0xf5, 0x7b, 0xaf, 0x5e, 0xbd, 0x7a, 0xf5, 0xea, 0xd5, 0xaf, 0x8a,
0x82, 0xa2, 0xe1, 0x58, 0x9b, 0x8e, 0x4b, 0x7c, 0x82, 0x2a, 0xee, 0xc0, 0xf6, 0xad, 0x3e, 0xde,
0x7c, 0x71, 0xd3, 0xe8, 0x39, 0xc7, 0xc6, 0x56, 0xed, 0x7a, 0xd7, 0xf2, 0x8f, 0x07, 0x87, 0x9b,
0x1d, 0xd2, 0xbf, 0xd1, 0x25, 0x5d, 0x72, 0x83, 0x31, 0x1e, 0x0e, 0x8e, 0xd8, 0x1b, 0x7b, 0x61,
0x4f, 0x5c, 0x81, 0x7a, 0x0d, 0x96, 0x3e, 0xc6, 0xae, 0x67, 0x11, 0x5b, 0xc3, 0x5f, 0x0e, 0xb0,
0xe7, 0xa3, 0x2a, 0x2c, 0xbc, 0xe0, 0x94, 0xaa, 0x72, 0x41, 0xb9, 0x52, 0xd4, 0xe4, 0xab, 0xfa,
0x17, 0x0a, 0x2c, 0x07, 0xcc, 0x9e, 0x43, 0x6c, 0x0f, 0x67, 0x73, 0xa3, 0x0d, 0x58, 0x14, 0xc6,
0xe9, 0xb6, 0xd1, 0xc7, 0xd5, 0x1c, 0xfb, 0x5c, 0x12, 0xb4, 0xa6, 0xd1, 0xc7, 0xe8, 0x32, 0x2c,
0x4b, 0x16, 0xa9, 0x24, 0xcf, 0xb8, 0x96, 0x04, 0x59, 0xb4, 0x86, 0x36, 0xe1, 0x94, 0x64, 0x34,
0x1c, 0x2b, 0x60, 0x9e, 0x65, 0xcc, 0x2b, 0xe2, 0x53, 0xdd, 0xb1, 0x04, 0xbf, 0xfa, 0x39, 0x14,
0x77, 0x9a, 0xed, 0x6d, 0x62, 0x1f, 0x59, 0x5d, 0x6a, 0xa2, 0x87, 0x5d, 0x2a, 0x53, 0x55, 0x2e,
0xe4, 0xa9, 0x89, 0xe2, 0x15, 0xd5, 0xa0, 0xe0, 0x61, 0xc3, 0xed, 0x1c, 0x63, 0xaf, 0x9a, 0x63,
0x9f, 0x82, 0x77, 0x2a, 0x45, 0x1c, 0xdf, 0x22, 0xb6, 0x57, 0xcd, 0x73, 0x29, 0xf1, 0xaa, 0xfe,
0x5c, 0x81, 0x52, 0x8b, 0xb8, 0xfe, 0x33, 0xc3, 0x71, 0x2c, 0xbb, 0x8b, 0x6e, 0x41, 0x81, 0xf9,
0xb2, 0x43, 0x7a, 0xcc, 0x07, 0x4b, 0x5b, 0xb5, 0xcd, 0xf8, 0xb0, 0x6c, 0xb6, 0x04, 0x87, 0x16,
0xf0, 0xa2, 0x8b, 0xb0, 0xd4, 0x21, 0xb6, 0x6f, 0x58, 0x36, 0x76, 0x75, 0x87, 0xb8, 0x3e, 0x73,
0xd1, 0x9c, 0x56, 0x0e, 0xa8, 0xb4, 0x15, 0x74, 0x0e, 0x8a, 0xc7, 0xc4, 0xf3, 0x39, 0x47, 0x9e,
0x71, 0x14, 0x28, 0x81, 0x7d, 0x5c, 0x83, 0x05, 0xf6, 0xd1, 0x72, 0x84, 0x33, 0xe6, 0xe9, 0x6b,
0xc3, 0x51, 0x7f, 0xa5, 0xc0, 0xdc, 0x33, 0x32, 0xb0, 0xfd, 0x58, 0x33, 0x86, 0x7f, 0x2c, 0x06,
0x2a, 0xd4, 0x8c, 0xe1, 0x1f, 0x0f, 0x9b, 0xa1, 0x1c, 0x7c, 0xac, 0x78, 0x33, 0xf4, 0x63, 0x0d,
0x0a, 0x2e, 0x36, 0x4c, 0x62, 0xf7, 0x4e, 0x98, 0x09, 0x05, 0x2d, 0x78, 0xa7, 0x83, 0xe8, 0xe1,
0x9e, 0x65, 0x0f, 0x5e, 0xe9, 0x2e, 0xee, 0x19, 0x87, 0xb8, 0xc7, 0x4c, 0x29, 0x68, 0x4b, 0x82,
0xac, 0x71, 0x2a, 0xda, 0x81, 0x92, 0xe3, 0x12, 0xc7, 0xe8, 0x1a, 0xd4, 0x8f, 0xd5, 0x39, 0xe6,
0x2a, 0x35, 0xe9, 0x2a, 0x66, 0x76, 0x6b, 0xc8, 0xa9, 0x85, 0xc5, 0xd4, 0xbf, 0x52, 0x60, 0x99,
0x06, 0x8f, 0xe7, 0x18, 0x1d, 0xbc, 0xc7, 0x86, 0x04, 0xdd, 0x86, 0x05, 0x1b, 0xfb, 0x2f, 0x89,
0xfb, 0x5c, 0x0c, 0xc0, 0xdb, 0x49, 0xad, 0x81, 0xcc, 0x33, 0x62, 0x62, 0x4d, 0xf2, 0xa3, 0x9b,
0x90, 0x77, 0x2c, 0x93, 0x75, 0x78, 0x02, 0x31, 0xca, 0x4b, 0x45, 0x2c, 0xa7, 0xc3, 0xfc, 0x30,
0x89, 0x88, 0xe5, 0x74, 0x54, 0x15, 0xa0, 0x61, 0xfb, 0xb7, 0xbe, 0xfb, 0xb1, 0xd1, 0x1b, 0x60,
0xb4, 0x0a, 0x73, 0x2f, 0xe8, 0x03, 0x33, 0x36, 0xaf, 0xf1, 0x17, 0xf5, 0xab, 0x3c, 0x9c, 0x7b,
0x4a, 0xfd, 0xd5, 0x36, 0x6c, 0xf3, 0x90, 0xbc, 0x6a, 0xe3, 0xce, 0xc0, 0xb5, 0xfc, 0x93, 0x6d,
0x62, 0xfb, 0xf8, 0x95, 0x8f, 0x9a, 0xb0, 0x62, 0x4b, 0xcd, 0xba, 0x0c, 0x4d, 0xaa, 0xa1, 0xb4,
0xb5, 0x31, 0xc2, 0x08, 0xee, 0x22, 0xad, 0x62, 0x47, 0x09, 0x1e, 0x7a, 0x32, 0x1c, 0x37, 0xa9,
0x2d, 0xc7, 0xb4, 0xa5, 0x74, 0xa9, 0xbd, 0xcb, 0x2c, 0x13, 0xba, 0xe4, 0xc0, 0x4a, 0x4d, 0x1f,
0x01, 0x9d, 0xd5, 0xba, 0xe1, 0xe9, 0x03, 0x0f, 0xbb, 0xcc, 0x31, 0xa5, 0xad, 0xb7, 0x92, 0x5a,
0x86, 0x2e, 0xd0, 0x8a, 0xee, 0xc0, 0xae, 0x7b, 0x07, 0x1e, 0x76, 0x59, 0x12, 0x10, 0xb1, 0xa4,
0xbb, 0x84, 0xf8, 0x47, 0x9e, 0x8c, 0x1f, 0x49, 0xd6, 0x18, 0x15, 0xdd, 0x80, 0x53, 0xde, 0xc0,
0x71, 0x7a, 0xb8, 0x8f, 0x6d, 0xdf, 0xe8, 0xe9, 0x5d, 0x97, 0x0c, 0x1c, 0xaf, 0x3a, 0x77, 0x21,
0x7f, 0x25, 0xaf, 0xa1, 0xf0, 0xa7, 0xc7, 0xec, 0x0b, 0x5a, 0x07, 0x70, 0x5c, 0xeb, 0x85, 0xd5,
0xc3, 0x5d, 0x6c, 0x56, 0xe7, 0x99, 0xd2, 0x10, 0x05, 0xbd, 0x0f, 0xab, 0x1e, 0xee, 0x74, 0x48,
0xdf, 0xd1, 0x1d, 0x97, 0x1c, 0x59, 0x3d, 0xcc, 0xa3, 0x7f, 0x81, 0x45, 0x3f, 0x12, 0xdf, 0x5a,
0xfc, 0x13, 0x9b, 0x07, 0xf7, 0x58, 0x4e, 0xa3, 0x3d, 0x65, 0x8d, 0x57, 0x0b, 0x13, 0x74, 0x15,
0x58, 0x57, 0x99, 0x49, 0xea, 0xcf, 0x73, 0x70, 0x9a, 0x79, 0xb2, 0x45, 0x4c, 0x31, 0xcc, 0x22,
0x49, 0xbd, 0x03, 0xe5, 0x0e, 0xd3, 0xa9, 0x3b, 0x86, 0x8b, 0x6d, 0x5f, 0x4c, 0xd2, 0x45, 0x4e,
0x6c, 0x31, 0x1a, 0xfa, 0x14, 0x2a, 0x9e, 0x88, 0x0a, 0xbd, 0xc3, 0xc3, 0x42, 0x8c, 0xd9, 0xf5,
0xa4, 0x09, 0x23, 0x62, 0x49, 0x5b, 0xf6, 0x12, 0xc1, 0xb5, 0xe0, 0x9d, 0x78, 0x1d, 0xbf, 0xc7,
0xb3, 0x5d, 0x69, 0xeb, 0xbb, 0x19, 0x0a, 0xe3, 0x86, 0x6f, 0xb6, 0xb9, 0xd8, 0xae, 0xed, 0xbb,
0x27, 0x9a, 0x54, 0x52, 0xbb, 0x03, 0x8b, 0xe1, 0x0f, 0xa8, 0x02, 0xf9, 0xe7, 0xf8, 0x44, 0x74,
0x8a, 0x3e, 0x0e, 0x27, 0x01, 0xcf, 0x35, 0xfc, 0xe5, 0x4e, 0xee, 0x77, 0x14, 0xd5, 0x05, 0x34,
0x6c, 0xe5, 0x19, 0xf6, 0x0d, 0xd3, 0xf0, 0x0d, 0x84, 0x60, 0x96, 0x2d, 0x23, 0x5c, 0x05, 0x7b,
0xa6, 0x5a, 0x07, 0x62, 0xf2, 0x16, 0x35, 0xfa, 0x88, 0xde, 0x82, 0x62, 0x10, 0xe8, 0x62, 0x2d,
0x19, 0x12, 0x68, 0x4e, 0x37, 0x7c, 0x1f, 0xf7, 0x1d, 0x9f, 0x85, 0x58, 0x59, 0x93, 0xaf, 0xea,
0x7f, 0xcf, 0x42, 0x25, 0x31, 0x26, 0x0f, 0xa0, 0xd0, 0x17, 0xcd, 0x8b, 0x89, 0xf6, 0x6e, 0x4a,
0x62, 0x4f, 0x98, 0xaa, 0x05, 0x52, 0x34, 0x6f, 0xd2, 0x1c, 0x1a, 0x5a, 0xff, 0x82, 0x77, 0x3a,
0xe2, 0x3d, 0xd2, 0xd5, 0x4d, 0xcb, 0xc5, 0x1d, 0x9f, 0xb8, 0x27, 0xc2, 0xdc, 0xc5, 0x1e, 0xe9,
0xee, 0x48, 0x1a, 0xba, 0x03, 0x60, 0xda, 0x1e, 0x1d, 0xec, 0x23, 0xab, 0xcb, 0x8c, 0x2e, 0x6d,
0x9d, 0x4b, 0x1a, 0x11, 0x2c, 0x76, 0x5a, 0xd1, 0xb4, 0x3d, 0x61, 0xfe, 0x43, 0x28, 0xd3, 0x35,
0x43, 0xef, 0xf3, 0x75, 0x8a, 0xcf, 0x94, 0xd2, 0xd6, 0xf9, 0xb4, 0x3e, 0x04, 0xab, 0x99, 0xb6,
0xe8, 0x0c, 0x5f, 0x3c, 0xf4, 0x08, 0xe6, 0x59, 0xf2, 0xf6, 0xaa, 0xf3, 0x4c, 0x78, 0x73, 0x94,
0x03, 0x44, 0x44, 0x3c, 0x65, 0x02, 0x3c, 0x20, 0x84, 0x34, 0x3a, 0x80, 0x92, 0x61, 0xdb, 0xc4,
0x37, 0x78, 0xa2, 0x59, 0x60, 0xca, 0x3e, 0x98, 0x40, 0x59, 0x7d, 0x28, 0xc5, 0x35, 0x86, 0xf5,
0xa0, 0xef, 0xc3, 0x1c, 0xcb, 0x44, 0x62, 0x22, 0x5e, 0x9e, 0x30, 0x68, 0x35, 0x2e, 0x55, 0xbb,
0x0d, 0xa5, 0x90, 0xb1, 0xd3, 0x04, 0x69, 0xed, 0x1e, 0x54, 0xe2, 0xa6, 0x4d, 0x15, 0xe4, 0x7f,
0x00, 0xab, 0xda, 0xc0, 0x1e, 0x1a, 0x26, 0xab, 0xaf, 0x3b, 0x30, 0x2f, 0x06, 0x9b, 0x47, 0x9c,
0x3a, 0xde, 0x47, 0x9a, 0x90, 0x08, 0x97, 0x53, 0xc7, 0x86, 0x6d, 0xf6, 0xb0, 0x2b, 0xda, 0x95,
0xe5, 0xd4, 0x13, 0x4e, 0x55, 0xbf, 0x0f, 0xa7, 0x63, 0x8d, 0x8b, 0x6a, 0xee, 0x5d, 0x58, 0x72,
0x88, 0xa9, 0x7b, 0x9c, 0xac, 0x5b, 0xa6, 0x4c, 0x43, 0x4e, 0xc0, 0xdb, 0x30, 0xa9, 0x78, 0xdb,
0x27, 0x4e, 0xd2, 0xf8, 0xc9, 0xc4, 0xab, 0x70, 0x26, 0x2e, 0xce, 0x9b, 0x57, 0xef, 0xc3, 0x9a,
0x86, 0xfb, 0xe4, 0x05, 0x7e, 0x5d, 0xd5, 0x35, 0xa8, 0x26, 0x15, 0x08, 0xe5, 0x9f, 0xc1, 0xda,
0x90, 0xda, 0xf6, 0x0d, 0x7f, 0xe0, 0x4d, 0xa5, 0x5c, 0x94, 0xba, 0x87, 0xc4, 0xe3, 0xc3, 0x59,
0xd0, 0xe4, 0xab, 0x7a, 0x35, 0xac, 0xba, 0xc9, 0x2b, 0x0b, 0xde, 0x02, 0x5a, 0x82, 0x9c, 0xe5,
0x08, 0x75, 0x39, 0xcb, 0x51, 0x9f, 0x40, 0x31, 0x58, 0x9a, 0xd1, 0xdd, 0x61, 0x8d, 0x99, 0x9b,
0x74, 0x21, 0x0f, 0xca, 0xd0, 0xfd, 0xc4, 0x52, 0x22, 0x9a, 0xbc, 0x0b, 0x10, 0xa4, 0x3c, 0x59,
0x21, 0x9c, 0x1b, 0xa1, 0x58, 0x0b, 0xb1, 0xab, 0x3f, 0x9d, 0x0b, 0x27, 0xc2, 0x50, 0x27, 0xcc,
0xa0, 0x13, 0x66, 0x24, 0x31, 0xe6, 0x5e, 0x2b, 0x31, 0x7e, 0x08, 0x73, 0x9e, 0x6f, 0xf8, 0x58,
0x54, 0x51, 0x1b, 0xa3, 0xc4, 0xa9, 0x11, 0x58, 0xe3, 0xfc, 0xe8, 0x3c, 0x40, 0xc7, 0xc5, 0x86,
0x8f, 0x4d, 0xdd, 0xe0, 0x59, 0x3c, 0xaf, 0x15, 0x05, 0xa5, 0xee, 0xa3, 0xed, 0x61, 0x25, 0x38,
0xc7, 0x0c, 0xbb, 0x3a, 0x4a, 0x73, 0x64, 0xa8, 0x86, 0x35, 0x61, 0x90, 0x55, 0xe6, 0x27, 0xcc,
0x2a, 0x42, 0x01, 0x97, 0x0a, 0xe5, 0xcc, 0x85, 0xf1, 0x39, 0x93, 0x8b, 0x4e, 0x92, 0x33, 0x0b,
0xe3, 0x73, 0xa6, 0x50, 0x36, 0x3a, 0x67, 0xa6, 0x64, 0x89, 0x62, 0x5a, 0x96, 0xf8, 0x36, 0xb3,
0xe3, 0x3f, 0x2b, 0x50, 0x4d, 0x4e, 0x56, 0x91, 0xa4, 0xee, 0xc0, 0xbc, 0xc7, 0x28, 0x93, 0xa4,
0x48, 0x21, 0x2b, 0x24, 0xd0, 0x13, 0x98, 0xb5, 0xec, 0x23, 0xc2, 0x76, 0x7b, 0xa9, 0x45, 0x4e,
0x56, 0xab, 0x9b, 0x0d, 0xfb, 0x88, 0x70, 0x6f, 0x32, 0x0d, 0xb5, 0x0f, 0xa1, 0x18, 0x90, 0xa6,
0xea, 0xdb, 0x1e, 0xac, 0xc6, 0x62, 0x9b, 0xef, 0x0a, 0x82, 0x29, 0xa1, 0x4c, 0x37, 0x25, 0xd4,
0x9f, 0xe4, 0xc2, 0x53, 0xf6, 0x91, 0xd5, 0xf3, 0xb1, 0x9b, 0x98, 0xb2, 0x1f, 0x49, 0xed, 0x7c,
0xbe, 0x5e, 0x1a, 0xab, 0x9d, 0x17, 0xaf, 0x62, 0xd6, 0x7d, 0x01, 0x4b, 0x2c, 0x28, 0x75, 0x0f,
0xf7, 0x58, 0x65, 0x22, 0xaa, 0xc4, 0xef, 0x8d, 0x52, 0xc3, 0x2d, 0xe1, 0xa1, 0xdd, 0x16, 0x72,
0xdc, 0x83, 0xe5, 0x5e, 0x98, 0x56, 0x7b, 0x00, 0x28, 0xc9, 0x34, 0x95, 0x4f, 0xdb, 0x34, 0x17,
0xd2, 0x2d, 0x71, 0xca, 0x72, 0x7a, 0xc4, 0xcc, 0x98, 0x24, 0x56, 0xb8, 0xc1, 0x9a, 0x90, 0x50,
0xff, 0x2b, 0x0f, 0x30, 0xfc, 0xf8, 0xff, 0x28, 0x09, 0x3e, 0x08, 0x12, 0x10, 0xaf, 0xf8, 0xae,
0x8c, 0x52, 0x9c, 0x9a, 0x7a, 0xf6, 0xa2, 0xa9, 0x87, 0xd7, 0x7e, 0xd7, 0x47, 0xaa, 0x99, 0x3a,
0xe9, 0x2c, 0xfc, 0xb6, 0x25, 0x9d, 0xa7, 0x70, 0x26, 0x1e, 0x44, 0x22, 0xe3, 0x6c, 0xc1, 0x9c,
0xe5, 0xe3, 0x3e, 0xc7, 0x8f, 0x52, 0xf7, 0x7b, 0x21, 0x21, 0xce, 0xaa, 0x6e, 0x40, 0xb1, 0xd1,
0x37, 0xba, 0xb8, 0xed, 0xe0, 0x0e, 0x6d, 0xd4, 0xa2, 0x2f, 0xc2, 0x10, 0xfe, 0xa2, 0x6e, 0x41,
0xe1, 0x07, 0xf8, 0x84, 0xcf, 0xfe, 0x09, 0x0d, 0x55, 0xff, 0x24, 0x07, 0x6b, 0x6c, 0xf5, 0xd9,
0x96, 0xe8, 0x8d, 0x86, 0x3d, 0x32, 0x70, 0x3b, 0xd8, 0x63, 0x61, 0xe1, 0x0c, 0x74, 0x07, 0xbb,
0x16, 0x31, 0x05, 0xb8, 0x50, 0xec, 0x38, 0x83, 0x16, 0x23, 0xa0, 0x73, 0x40, 0x5f, 0xf4, 0x2f,
0x07, 0x44, 0x44, 0x6c, 0x5e, 0x2b, 0x74, 0x9c, 0xc1, 0xef, 0xd1, 0x77, 0x29, 0xeb, 0x1d, 0x1b,
0x2e, 0xf6, 0x58, 0x40, 0x72, 0xd9, 0x36, 0x23, 0xa0, 0x9b, 0x70, 0xba, 0x8f, 0xfb, 0xc4, 0x3d,
0xd1, 0x7b, 0x56, 0xdf, 0xf2, 0x75, 0xcb, 0xd6, 0x0f, 0x4f, 0x7c, 0xec, 0x89, 0xe0, 0x43, 0xfc,
0xe3, 0x53, 0xfa, 0xad, 0x61, 0x3f, 0xa4, 0x5f, 0x90, 0x0a, 0x65, 0x42, 0xfa, 0xba, 0xd7, 0x21,
0x2e, 0xd6, 0x0d, 0xf3, 0xc7, 0x6c, 0x41, 0xce, 0x6b, 0x25, 0x42, 0xfa, 0x6d, 0x4a, 0xab, 0x9b,
0x3f, 0x46, 0x6f, 0x43, 0xa9, 0xe3, 0x0c, 0x3c, 0xec, 0xeb, 0xf4, 0x0f, 0x5b, 0x6f, 0x8b, 0x1a,
0x70, 0xd2, 0xb6, 0x33, 0xf0, 0x42, 0x0c, 0x7d, 0xea, 0xff, 0x85, 0x30, 0xc3, 0x33, 0xea, 0x66,
0x03, 0xca, 0x11, 0x70, 0x82, 0xee, 0x13, 0x19, 0x0a, 0x21, 0xf6, 0x89, 0xf4, 0x99, 0xd2, 0x5c,
0xd2, 0x93, 0x9e, 0x64, 0xcf, 0x94, 0xe6, 0x9f, 0x38, 0x72, 0x93, 0xc8, 0x9e, 0xa9, 0xcb, 0x7b,
0xf8, 0x85, 0x00, 0xb0, 0x8a, 0x1a, 0x7f, 0x51, 0x4d, 0x80, 0x6d, 0xc3, 0x31, 0x0e, 0xad, 0x9e,
0xe5, 0x9f, 0xa0, 0xab, 0x50, 0x31, 0x4c, 0x53, 0xef, 0x48, 0x8a, 0x85, 0x25, 0xac, 0xb8, 0x6c,
0x98, 0xe6, 0x76, 0x88, 0x8c, 0xbe, 0x03, 0x2b, 0xa6, 0x4b, 0x9c, 0x28, 0x2f, 0xc7, 0x19, 0x2b,
0xf4, 0x43, 0x98, 0x59, 0xfd, 0xf7, 0x39, 0x38, 0x1f, 0x1d, 0xd8, 0x38, 0x00, 0xf4, 0x00, 0x16,
0x63, 0xad, 0x66, 0x80, 0x0f, 0x43, 0x6b, 0xb5, 0x88, 0x44, 0x0c, 0x10, 0xc9, 0x25, 0x00, 0x91,
0x54, 0x88, 0x29, 0xff, 0x46, 0x21, 0xa6, 0xd9, 0x37, 0x02, 0x31, 0xcd, 0x4d, 0x07, 0x31, 0x5d,
0x62, 0xd9, 0x47, 0x4a, 0xb3, 0xdd, 0x38, 0x0f, 0xb5, 0x72, 0xc0, 0x63, 0x4b, 0x3c, 0x3a, 0x06,
0x45, 0x2d, 0x4c, 0x03, 0x45, 0x15, 0x32, 0xa1, 0x28, 0x1a, 0x35, 0x8e, 0x63, 0xb8, 0x7d, 0xe2,
0x4a, 0xac, 0x49, 0x54, 0x5d, 0xcb, 0x92, 0x2e, 0x70, 0xa6, 0x4c, 0x54, 0x0a, 0x32, 0x51, 0xa9,
0x0b, 0xb0, 0x68, 0x13, 0xdd, 0xc6, 0x2f, 0x75, 0x3a, 0x96, 0x5e, 0xb5, 0xc4, 0x07, 0xd6, 0x26,
0x4d, 0xfc, 0xb2, 0x45, 0x29, 0x09, 0xdc, 0x6a, 0x71, 0x3a, 0xdc, 0x0a, 0x6d, 0xc0, 0x62, 0xdf,
0xf0, 0x9e, 0x63, 0x93, 0x99, 0xe2, 0x55, 0xcb, 0x2c, 0x88, 0x4b, 0x9c, 0x46, 0x6d, 0xf0, 0xd0,
0x45, 0x08, 0x9c, 0x24, 0x98, 0x96, 0x18, 0x53, 0x59, 0x52, 0x19, 0x9b, 0xfa, 0xb7, 0x0a, 0xac,
0x46, 0xc3, 0x5c, 0xa0, 0x15, 0x8f, 0xa1, 0xe8, 0xca, 0x4c, 0x26, 0x42, 0xfb, 0x6a, 0x46, 0xe1,
0x9d, 0x4c, 0x7d, 0xda, 0x50, 0x16, 0xfd, 0x30, 0x13, 0x24, 0xbb, 0x31, 0x4e, 0xdf, 0x38, 0x98,
0x4c, 0x6d, 0xc0, 0xdb, 0x9f, 0x58, 0xb6, 0x49, 0x5e, 0x7a, 0x99, 0xb3, 0x34, 0x25, 0xd6, 0x94,
0x94, 0x58, 0x53, 0x7f, 0xa1, 0xc0, 0x99, 0xb8, 0x2e, 0xe1, 0x8a, 0x46, 0xd2, 0x15, 0xdf, 0x49,
0x9a, 0x1e, 0x17, 0x4e, 0x75, 0xc6, 0x17, 0x99, 0xce, 0xb8, 0x39, 0x5e, 0xe3, 0x58, 0x77, 0xfc,
0xa5, 0x02, 0x67, 0x33, 0xcd, 0x88, 0x2d, 0x29, 0x4a, 0x7c, 0x49, 0x11, 0xcb, 0x51, 0x87, 0x0c,
0x6c, 0x3f, 0xb4, 0x1c, 0x6d, 0xb3, 0x43, 0x0b, 0x9e, 0xf7, 0xf5, 0xbe, 0xf1, 0xca, 0xea, 0x0f,
0xfa, 0x62, 0x3d, 0xa2, 0xea, 0x9e, 0x71, 0xca, 0x6b, 0x2c, 0x48, 0x6a, 0x1d, 0x56, 0x02, 0x2b,
0x47, 0xc2, 0x8a, 0x21, 0x98, 0x30, 0x17, 0x85, 0x09, 0x6d, 0x98, 0xdf, 0xc1, 0x2f, 0xac, 0x0e,
0x7e, 0x23, 0xa7, 0x2a, 0x17, 0xa0, 0xe4, 0x60, 0xb7, 0x6f, 0x79, 0x5e, 0x90, 0x68, 0x8b, 0x5a,
0x98, 0xa4, 0xfe, 0xc7, 0x3c, 0x2c, 0xc7, 0xa3, 0xe3, 0x7e, 0x02, 0x95, 0x7c, 0x27, 0x65, 0x09,
0x88, 0x77, 0x34, 0x54, 0x76, 0xde, 0x94, 0xc5, 0x48, 0x2e, 0x0b, 0x1a, 0x08, 0x0a, 0x17, 0x51,
0xa9, 0x50, 0x8f, 0x74, 0x48, 0xbf, 0x6f, 0xd8, 0xa6, 0x3c, 0x0c, 0x13, 0xaf, 0xd4, 0x7f, 0x86,
0xdb, 0xa5, 0x6e, 0xa7, 0x64, 0xf6, 0x4c, 0x07, 0x8f, 0xee, 0xa3, 0x2d, 0x9b, 0xa1, 0x9b, 0x2c,
0x59, 0x17, 0x35, 0x10, 0xa4, 0x1d, 0xcb, 0x45, 0x9b, 0x30, 0x8b, 0xed, 0x17, 0xb2, 0xae, 0x4c,
0x39, 0x2d, 0x93, 0x65, 0x91, 0xc6, 0xf8, 0xd0, 0x0d, 0x98, 0xef, 0xd3, 0xb0, 0x90, 0x3b, 0xea,
0xb5, 0x8c, 0x43, 0x23, 0x4d, 0xb0, 0xa1, 0x2d, 0x58, 0x30, 0xd9, 0x38, 0xc9, 0x6d, 0x73, 0x35,
0x05, 0x33, 0x65, 0x0c, 0x9a, 0x64, 0x44, 0xbb, 0x41, 0xd5, 0x5c, 0xcc, 0x2a, 0x77, 0x63, 0x43,
0x91, 0x5a, 0x3a, 0xef, 0x47, 0x4b, 0x67, 0x60, 0xba, 0xb6, 0xc6, 0xeb, 0x1a, 0x5d, 0x3f, 0x9f,
0x85, 0x42, 0x8f, 0x74, 0x79, 0x18, 0x95, 0xf8, 0x39, 0x6b, 0x8f, 0x74, 0x59, 0x14, 0xad, 0xd2,
0x5d, 0x84, 0x69, 0xd9, 0x2c, 0xa9, 0x17, 0x34, 0xfe, 0x42, 0x27, 0x1f, 0x7b, 0xd0, 0x89, 0xdd,
0xc1, 0xd5, 0x32, 0xfb, 0x54, 0x64, 0x94, 0x3d, 0xbb, 0xc3, 0xca, 0x4d, 0xdf, 0x3f, 0xa9, 0x2e,
0x31, 0x3a, 0x7d, 0xa4, 0x1b, 0x44, 0x0e, 0x7a, 0x2c, 0x67, 0x6d, 0x10, 0xd3, 0xd2, 0xb6, 0xc4,
0x3c, 0x1e, 0xc2, 0xc2, 0x4b, 0x9e, 0x08, 0xaa, 0x15, 0x26, 0x7f, 0x65, 0x7c, 0x7a, 0x11, 0x1a,
0xa4, 0xe0, 0xb7, 0x59, 0xfa, 0xff, 0xbd, 0x02, 0x67, 0xb6, 0xd9, 0xfe, 0x29, 0x94, 0xc7, 0xa6,
0xc1, 0x06, 0x6f, 0x07, 0xb0, 0x6d, 0x26, 0x90, 0x17, 0xef, 0xb7, 0x44, 0x6d, 0x1b, 0xb0, 0x24,
0x95, 0x0b, 0x15, 0xf9, 0x89, 0x91, 0xdf, 0xb2, 0x17, 0x7e, 0x55, 0x3f, 0x82, 0xb5, 0x44, 0x2f,
0xc4, 0x16, 0x66, 0x03, 0x16, 0x87, 0xf9, 0x2a, 0xe8, 0x44, 0x29, 0xa0, 0x35, 0x4c, 0xf5, 0x0e,
0x9c, 0x6e, 0xfb, 0x86, 0xeb, 0x27, 0x5c, 0x30, 0x81, 0x2c, 0xc3, 0x74, 0xa3, 0xb2, 0x02, 0x76,
0x6d, 0xc3, 0x6a, 0xdb, 0x27, 0xce, 0x6b, 0x28, 0xa5, 0x59, 0x87, 0xf6, 0x9f, 0x0c, 0xe4, 0xfa,
0x20, 0x5f, 0xd5, 0x35, 0x8e, 0x40, 0x27, 0x5b, 0xbb, 0x0b, 0x67, 0x38, 0x00, 0xfc, 0x3a, 0x9d,
0x38, 0x2b, 0xe1, 0xe7, 0xa4, 0xde, 0x67, 0x70, 0x6a, 0xb8, 0x2c, 0x0e, 0x31, 0x9b, 0x5b, 0x51,
0xcc, 0xe6, 0xc2, 0x88, 0x51, 0x8f, 0x40, 0x36, 0x7f, 0x9e, 0x0b, 0xe5, 0xf5, 0x0c, 0xc4, 0xe6,
0x6e, 0x14, 0xb1, 0xb9, 0x38, 0x4e, 0x77, 0x04, 0xb0, 0x49, 0x46, 0x6d, 0x3e, 0x25, 0x6a, 0x3f,
0x4f, 0xc0, 0x3a, 0xb3, 0x59, 0xb8, 0x58, 0xcc, 0xda, 0xdf, 0x08, 0xaa, 0xa3, 0x71, 0x54, 0x27,
0x68, 0x3a, 0xc0, 0xeb, 0x6f, 0xc7, 0x50, 0x9d, 0x8d, 0xb1, 0xf6, 0x06, 0xa0, 0xce, 0x5f, 0xcf,
0x42, 0x31, 0xf8, 0x96, 0xf0, 0x79, 0xd2, 0x6d, 0xb9, 0x14, 0xb7, 0x85, 0x57, 0xe0, 0xfc, 0x37,
0x5a, 0x81, 0x67, 0x27, 0x5e, 0x81, 0xcf, 0x41, 0x91, 0x3d, 0xe8, 0x2e, 0x3e, 0x12, 0x2b, 0x6a,
0x81, 0x11, 0x34, 0x7c, 0x34, 0x0c, 0xc3, 0xf9, 0xa9, 0xc2, 0x30, 0x86, 0x23, 0x2d, 0xc4, 0x71,
0xa4, 0xfb, 0xc1, 0x8a, 0xc8, 0x17, 0xd1, 0xcb, 0x23, 0xf4, 0xa6, 0xae, 0x85, 0xcd, 0xe8, 0x5a,
0xc8, 0xd7, 0xd5, 0xf7, 0x46, 0x69, 0x19, 0xb9, 0x0a, 0x7e, 0x9b, 0x2b, 0xc4, 0x01, 0x07, 0x87,
0xc2, 0xb1, 0x28, 0x32, 0xeb, 0x5d, 0x80, 0x20, 0x89, 0x48, 0x84, 0xe8, 0xdc, 0x88, 0x3e, 0x6a,
0x21, 0x76, 0xaa, 0x36, 0x32, 0x34, 0xc3, 0x33, 0xa9, 0xc9, 0xf2, 0x63, 0xc6, 0x81, 0xd4, 0xff,
0xce, 0x85, 0xf2, 0x4b, 0xc6, 0x21, 0xce, 0xfd, 0x04, 0x7e, 0x39, 0x65, 0x14, 0xdf, 0x8a, 0xc2,
0x97, 0xaf, 0x19, 0x75, 0x09, 0xf4, 0x92, 0x55, 0x2e, 0x86, 0x2b, 0x3e, 0x73, 0xd0, 0xa8, 0x28,
0x28, 0x75, 0xb6, 0x33, 0x38, 0xb2, 0x6c, 0xcb, 0x3b, 0xe6, 0xdf, 0xe7, 0xf9, 0xce, 0x40, 0x92,
0xea, 0xec, 0xbe, 0x14, 0x7e, 0x65, 0xf9, 0x7a, 0x87, 0x98, 0x98, 0xc5, 0xf4, 0x9c, 0x56, 0xa0,
0x84, 0x6d, 0x62, 0xe2, 0xe1, 0xcc, 0x2b, 0xbc, 0xde, 0xcc, 0x2b, 0xc6, 0x66, 0xde, 0x19, 0x98,
0x77, 0xb1, 0xe1, 0x11, 0x5b, 0x6c, 0xcf, 0xc5, 0x1b, 0x1d, 0x9a, 0x3e, 0xf6, 0x3c, 0xda, 0x92,
0x28, 0xd7, 0xc4, 0x6b, 0xa8, 0xcc, 0x5c, 0x1c, 0x5b, 0x66, 0x8e, 0x38, 0x1c, 0x8a, 0x95, 0x99,
0xe5, 0xb1, 0x65, 0xe6, 0x44, 0x67, 0x43, 0xc3, 0x42, 0x7b, 0x69, 0xb2, 0x42, 0x3b, 0x5c, 0x97,
0x2e, 0x47, 0xea, 0xd2, 0x6f, 0x73, 0xb2, 0xfe, 0x4a, 0x81, 0xb5, 0xc4, 0xb4, 0x12, 0xd3, 0xf5,
0x76, 0xec, 0xf4, 0x68, 0x63, 0xac, 0xcf, 0x82, 0xc3, 0xa3, 0xc7, 0x91, 0xc3, 0xa3, 0x0f, 0xc6,
0x0b, 0xbe, 0xf1, 0xb3, 0xa3, 0x3f, 0x52, 0xe0, 0xed, 0x03, 0xc7, 0x8c, 0x55, 0x78, 0x62, 0xdb,
0x3f, 0x79, 0xe2, 0xb8, 0x2f, 0x6b, 0xfd, 0xdc, 0xb4, 0x38, 0x0b, 0x97, 0x53, 0x55, 0xb8, 0x90,
0x6d, 0x86, 0x28, 0x99, 0x7e, 0x04, 0xcb, 0xbb, 0xaf, 0x70, 0xa7, 0x7d, 0x62, 0x77, 0xa6, 0x30,
0xad, 0x02, 0xf9, 0x4e, 0xdf, 0x14, 0x28, 0x29, 0x7d, 0x0c, 0x57, 0x81, 0xf9, 0x68, 0x15, 0xa8,
0x43, 0x65, 0xd8, 0x82, 0x18, 0xde, 0x33, 0x74, 0x78, 0x4d, 0xca, 0x4c, 0x95, 0x2f, 0x6a, 0xe2,
0x4d, 0xd0, 0xb1, 0xcb, 0xaf, 0x44, 0x70, 0x3a, 0x76, 0xdd, 0x68, 0xb6, 0xc8, 0x47, 0xb3, 0x85,
0xfa, 0x67, 0x0a, 0x94, 0x68, 0x0b, 0xdf, 0xc8, 0x7e, 0xb1, 0xd5, 0xca, 0x0f, 0xb7, 0x5a, 0xc1,
0x8e, 0x6d, 0x36, 0xbc, 0x63, 0x1b, 0x5a, 0x3e, 0xc7, 0xc8, 0x49, 0xcb, 0xe7, 0x03, 0x3a, 0x76,
0x5d, 0xf5, 0x02, 0x2c, 0x72, 0xdb, 0x44, 0xcf, 0x2b, 0x90, 0x1f, 0xb8, 0x3d, 0x19, 0x47, 0x03,
0xb7, 0xa7, 0xfe, 0xb1, 0x02, 0xe5, 0xba, 0xef, 0x1b, 0x9d, 0xe3, 0x29, 0x3a, 0x10, 0x18, 0x97,
0x0b, 0x1b, 0x97, 0xec, 0xc4, 0xd0, 0xdc, 0xd9, 0x0c, 0x73, 0xe7, 0x22, 0xe6, 0xaa, 0xb0, 0x24,
0x6d, 0xc9, 0x34, 0xb8, 0x09, 0xa8, 0x45, 0x5c, 0xff, 0x11, 0x71, 0x5f, 0x1a, 0xae, 0x39, 0xdd,
0x0e, 0x0c, 0xc1, 0xac, 0xb8, 0x43, 0x9b, 0xbf, 0x32, 0xa7, 0xb1, 0x67, 0xf5, 0x32, 0x9c, 0x8a,
0xe8, 0xcb, 0x6c, 0xf8, 0x01, 0x94, 0x58, 0xde, 0x17, 0xa5, 0xf8, 0xcd, 0xf0, 0x71, 0xcd, 0x44,
0xab, 0x84, 0xfa, 0xbb, 0xb0, 0x42, 0xeb, 0x03, 0x46, 0x0f, 0xa6, 0xe2, 0xf7, 0x62, 0x75, 0xea,
0xf9, 0x0c, 0x45, 0xb1, 0x1a, 0xf5, 0x6f, 0x14, 0x98, 0x63, 0xf4, 0xc4, 0x9a, 0x7d, 0x0e, 0x8a,
0x2e, 0x76, 0x88, 0xee, 0x1b, 0xdd, 0xe0, 0xc6, 0x32, 0x25, 0xec, 0x1b, 0x5d, 0x8f, 0x5d, 0xb8,
0xa6, 0x1f, 0x4d, 0xab, 0x8b, 0x3d, 0x5f, 0x5e, 0x5b, 0x2e, 0x51, 0xda, 0x0e, 0x27, 0x51, 0x27,
0x79, 0xd6, 0xef, 0xf3, 0xba, 0x73, 0x56, 0x63, 0xcf, 0x68, 0x93, 0x5f, 0xa2, 0x9b, 0x04, 0x52,
0x67, 0x57, 0xec, 0x6a, 0x50, 0x88, 0xa1, 0xe8, 0xc1, 0xbb, 0xba, 0x0b, 0x28, 0xec, 0x05, 0xe1,
0xef, 0x1b, 0x30, 0xcf, 0x9c, 0x24, 0xab, 0xa3, 0xb5, 0x0c, 0x37, 0x68, 0x82, 0x4d, 0x35, 0x00,
0x71, 0x07, 0x47, 0x2a, 0xa2, 0xe9, 0x47, 0x65, 0x44, 0x85, 0xf4, 0x77, 0x0a, 0x9c, 0x8a, 0xb4,
0x21, 0x6c, 0xbd, 0x1e, 0x6d, 0x24, 0xd3, 0x54, 0xd1, 0xc0, 0x76, 0x64, 0x49, 0xb8, 0x91, 0x65,
0xd2, 0xaf, 0x69, 0x39, 0xf8, 0x07, 0x05, 0xa0, 0x3e, 0xf0, 0x8f, 0x05, 0x32, 0x18, 0x1e, 0x19,
0x25, 0x3a, 0x32, 0xf4, 0x9b, 0x63, 0x78, 0xde, 0x4b, 0xe2, 0xca, 0x3d, 0x4d, 0xf0, 0xce, 0x30,
0xbc, 0x81, 0x7f, 0x2c, 0x8f, 0xc2, 0xe8, 0x33, 0xba, 0x08, 0x4b, 0xfc, 0x96, 0xbc, 0x6e, 0x98,
0xa6, 0x8b, 0x3d, 0x4f, 0x9c, 0x89, 0x95, 0x39, 0xb5, 0xce, 0x89, 0x94, 0xcd, 0x32, 0xb1, 0xed,
0x5b, 0xfe, 0x89, 0xee, 0x93, 0xe7, 0xd8, 0x16, 0x7b, 0x93, 0xb2, 0xa4, 0xee, 0x53, 0x22, 0x3f,
0x1c, 0xe8, 0x5a, 0x9e, 0xef, 0x4a, 0x36, 0x79, 0xfe, 0x22, 0xa8, 0x8c, 0x8d, 0x0e, 0x4a, 0xa5,
0x35, 0xe8, 0xf5, 0xb8, 0x8b, 0x5f, 0x7f, 0xd8, 0xdf, 0x17, 0x1d, 0xca, 0x65, 0xc5, 0xf4, 0xd0,
0x69, 0xa2, 0xbb, 0x6f, 0x10, 0x84, 0x79, 0x1f, 0x56, 0x42, 0x7d, 0x10, 0x61, 0x15, 0x29, 0x22,
0x95, 0x68, 0x11, 0xa9, 0x3e, 0x06, 0xc4, 0x71, 0x87, 0x6f, 0xd8, 0x6f, 0xf5, 0x34, 0x9c, 0x8a,
0x28, 0x12, 0x2b, 0xf1, 0x35, 0x28, 0x8b, 0x9b, 0x4e, 0x22, 0x50, 0xce, 0x42, 0x81, 0x66, 0xd4,
0x8e, 0x65, 0xca, 0x73, 0xd2, 0x05, 0x87, 0x98, 0xdb, 0x96, 0xe9, 0xaa, 0x9f, 0x40, 0x59, 0xe3,
0xed, 0x08, 0xde, 0x47, 0xb0, 0x24, 0xee, 0x45, 0xe9, 0x91, 0x8b, 0x89, 0x69, 0x17, 0xdf, 0xc3,
0x8d, 0x68, 0x65, 0x3b, 0xfc, 0xaa, 0x9a, 0x50, 0xe3, 0x25, 0x43, 0x44, 0xbd, 0xec, 0xec, 0x23,
0x90, 0x17, 0x01, 0xc6, 0xb6, 0x12, 0x95, 0x2f, 0xbb, 0xe1, 0x57, 0xf5, 0x3c, 0x9c, 0x4b, 0x6d,
0x45, 0x78, 0xc2, 0x81, 0xca, 0xf0, 0x83, 0x69, 0xc9, 0x03, 0x63, 0x76, 0x10, 0xac, 0x84, 0x0e,
0x82, 0xcf, 0x04, 0x45, 0x62, 0x4e, 0x2e, 0x62, 0xac, 0x02, 0x1c, 0x96, 0xfb, 0xf9, 0xac, 0x72,
0x7f, 0x36, 0x52, 0xee, 0xab, 0xed, 0xc0, 0x9f, 0x62, 0x1b, 0xf6, 0x90, 0x6d, 0x17, 0x79, 0xdb,
0x32, 0x21, 0xaa, 0xa3, 0x7a, 0xc9, 0x59, 0xb5, 0x90, 0x94, 0x7a, 0x15, 0xca, 0xd1, 0xd4, 0x18,
0xca, 0x73, 0x4a, 0x22, 0xcf, 0x2d, 0xc5, 0x52, 0xdc, 0x87, 0xb1, 0x0a, 0x38, 0xdb, 0xc7, 0xb1,
0xfa, 0xf7, 0x5e, 0x24, 0xd9, 0x5d, 0x4b, 0x39, 0xc3, 0xfd, 0x35, 0xe5, 0xb9, 0x55, 0xb1, 0x1e,
0x3c, 0xf2, 0xa8, 0xbc, 0xe8, 0xb4, 0xfa, 0x0e, 0x94, 0x0e, 0xb2, 0x7e, 0x55, 0x31, 0x2b, 0xef,
0x4b, 0xdc, 0x82, 0xd5, 0x47, 0x56, 0x0f, 0x7b, 0x27, 0x9e, 0x8f, 0xfb, 0x0d, 0x96, 0x94, 0x8e,
0x2c, 0xec, 0xa2, 0x75, 0x00, 0xb6, 0x85, 0x71, 0x88, 0x15, 0x5c, 0xb6, 0x0f, 0x51, 0xd4, 0xff,
0x54, 0x60, 0x79, 0x28, 0x78, 0xc0, 0xb6, 0x6e, 0x6f, 0x41, 0x91, 0xf6, 0xd7, 0xf3, 0x8d, 0xbe,
0x23, 0xcf, 0xb3, 0x02, 0x02, 0xba, 0x0b, 0x73, 0x47, 0x9e, 0x84, 0x8c, 0x52, 0x01, 0xf4, 0x34,
0x43, 0xb4, 0xd9, 0x23, 0xaf, 0x61, 0xa2, 0x8f, 0x00, 0x06, 0x1e, 0x36, 0xc5, 0x19, 0x56, 0x3e,
0xab, 0x5a, 0x38, 0x08, 0x9f, 0x6f, 0x53, 0x01, 0x7e, 0xd5, 0xe2, 0x1e, 0x94, 0x2c, 0x9b, 0x98,
0x98, 0x9d, 0x39, 0x9a, 0x02, 0x55, 0x1a, 0x23, 0x0e, 0x5c, 0xe2, 0xc0, 0xc3, 0xa6, 0x8a, 0xc5,
0x5a, 0x28, 0xfd, 0x2b, 0x02, 0xa5, 0x09, 0x2b, 0x3c, 0x69, 0x1d, 0x05, 0x86, 0xcb, 0x88, 0xdd,
0x18, 0xd5, 0x3b, 0xe6, 0x2d, 0xad, 0x62, 0x89, 0xd2, 0x46, 0x8a, 0xaa, 0x77, 0xe0, 0x74, 0x64,
0x87, 0x34, 0xc5, 0x96, 0x45, 0x6d, 0xc5, 0x80, 0x92, 0x61, 0x38, 0x0b, 0x18, 0x42, 0x46, 0xf3,
0x38, 0x18, 0xc2, 0xe3, 0x30, 0x84, 0xa7, 0x7e, 0x0e, 0x67, 0x23, 0x88, 0x4e, 0xc4, 0xa2, 0x7b,
0xb1, 0xca, 0xed, 0xd2, 0x38, 0xad, 0xb1, 0x12, 0xee, 0x7f, 0x14, 0x58, 0x4d, 0x63, 0x78, 0x4d,
0xc4, 0xf1, 0x47, 0x19, 0xf7, 0xef, 0x6e, 0x4f, 0x66, 0xd6, 0x6f, 0x04, 0xad, 0xdd, 0x87, 0x5a,
0x9a, 0x3f, 0x93, 0xa3, 0x94, 0x9f, 0x66, 0x94, 0x7e, 0x96, 0x0f, 0x21, 0xef, 0x75, 0xdf, 0x77,
0xad, 0xc3, 0x01, 0x0d, 0xf9, 0x37, 0x8e, 0x66, 0x35, 0x02, 0x5c, 0x86, 0xbb, 0xf6, 0xe6, 0x08,
0xf1, 0xa1, 0x1d, 0xa9, 0xd8, 0xcc, 0xa7, 0x51, 0x6c, 0x86, 0x63, 0xea, 0xb7, 0x26, 0xd3, 0xf7,
0x5b, 0x0b, 0x80, 0xfe, 0x2c, 0x07, 0x4b, 0xd1, 0x21, 0x42, 0xbb, 0x00, 0x46, 0x60, 0xb9, 0x98,
0x28, 0x17, 0x27, 0xea, 0xa6, 0x16, 0x12, 0x44, 0xef, 0x41, 0xbe, 0xe3, 0x0c, 0xc4, 0xa8, 0xa5,
0x1c, 0x06, 0x6f, 0x3b, 0x03, 0x9e, 0x51, 0x28, 0x1b, 0xdd, 0x53, 0xf1, 0xb3, 0xfd, 0xec, 0x2c,
0xf9, 0x8c, 0x7d, 0xe7, 0x32, 0x82, 0x19, 0x3d, 0x81, 0xa5, 0x97, 0xae, 0xe5, 0x1b, 0x87, 0x3d,
0xac, 0xf7, 0x8c, 0x13, 0xec, 0x8a, 0x2c, 0x39, 0x41, 0x22, 0x2b, 0x4b, 0xc1, 0xa7, 0x54, 0x4e,
0xfd, 0x43, 0x28, 0x48, 0x8b, 0xc6, 0xac, 0x08, 0xfb, 0xb0, 0x36, 0xa0, 0x6c, 0x3a, 0xbb, 0x02,
0x67, 0x1b, 0x36, 0xd1, 0x3d, 0x4c, 0x97, 0x71, 0x79, 0xdd, 0x7f, 0x4c, 0x8a, 0x5e, 0x65, 0xd2,
0xdb, 0xc4, 0xc5, 0x4d, 0xc3, 0x26, 0x6d, 0x2e, 0xaa, 0xbe, 0x80, 0x52, 0xa8, 0x83, 0x63, 0x4c,
0x68, 0xc0, 0x8a, 0x3c, 0x8a, 0xf7, 0xb0, 0x2f, 0x96, 0x97, 0x89, 0x1a, 0x5f, 0x16, 0x72, 0x6d,
0xec, 0xf3, 0xeb, 0x13, 0xf7, 0xe0, 0xac, 0x86, 0x89, 0x83, 0xed, 0x60, 0x3c, 0x9f, 0x92, 0xee,
0x14, 0x19, 0xfc, 0x2d, 0xa8, 0xa5, 0xc9, 0xf3, 0xfc, 0x70, 0xed, 0x12, 0x14, 0xe4, 0x4f, 0x64,
0xd1, 0x02, 0xe4, 0xf7, 0xb7, 0x5b, 0x95, 0x19, 0xfa, 0x70, 0xb0, 0xd3, 0xaa, 0x28, 0xa8, 0x00,
0xb3, 0xed, 0xed, 0xfd, 0x56, 0x25, 0x77, 0xad, 0x0f, 0x95, 0xf8, 0xef, 0x43, 0xd1, 0x1a, 0x9c,
0x6a, 0x69, 0x7b, 0xad, 0xfa, 0xe3, 0xfa, 0x7e, 0x63, 0xaf, 0xa9, 0xb7, 0xb4, 0xc6, 0xc7, 0xf5,
0xfd, 0xdd, 0xca, 0x0c, 0xda, 0x80, 0xf3, 0xe1, 0x0f, 0x4f, 0xf6, 0xda, 0xfb, 0xfa, 0xfe, 0x9e,
0xbe, 0xbd, 0xd7, 0xdc, 0xaf, 0x37, 0x9a, 0xbb, 0x5a, 0x45, 0x41, 0xe7, 0xe1, 0x6c, 0x98, 0xe5,
0x61, 0x63, 0xa7, 0xa1, 0xed, 0x6e, 0xd3, 0xe7, 0xfa, 0xd3, 0x4a, 0xee, 0xda, 0x4d, 0x28, 0x47,
0x7e, 0xce, 0x49, 0x4d, 0x6a, 0xed, 0xed, 0x54, 0x66, 0x50, 0x19, 0x8a, 0x61, 0x3d, 0x05, 0x98,
0x6d, 0xee, 0xed, 0xec, 0x56, 0x72, 0xd7, 0xee, 0xc0, 0x72, 0xec, 0xda, 0x2e, 0x5a, 0x81, 0x72,
0xbb, 0xde, 0xdc, 0x79, 0xb8, 0xf7, 0xa9, 0xae, 0xed, 0xd6, 0x77, 0x3e, 0xab, 0xcc, 0xa0, 0x55,
0xa8, 0x48, 0x52, 0x73, 0x6f, 0x9f, 0x53, 0x95, 0x6b, 0xcf, 0x63, 0x73, 0x0c, 0xa3, 0xd3, 0xb0,
0x12, 0x34, 0xa3, 0x6f, 0x6b, 0xbb, 0xf5, 0xfd, 0x5d, 0xda, 0x7a, 0x84, 0xac, 0x1d, 0x34, 0x9b,
0x8d, 0xe6, 0xe3, 0x8a, 0x42, 0xb5, 0x0e, 0xc9, 0xbb, 0x9f, 0x36, 0x28, 0x73, 0x2e, 0xca, 0x7c,
0xd0, 0xfc, 0x41, 0x73, 0xef, 0x93, 0x66, 0x25, 0xbf, 0xf5, 0x8b, 0x15, 0x58, 0x92, 0x85, 0x1e,
0x76, 0xd9, 0xad, 0x96, 0x16, 0x2c, 0xc8, 0x9f, 0x5c, 0xa7, 0x64, 0xe8, 0xe8, 0x0f, 0xc5, 0x6b,
0x1b, 0x23, 0x38, 0x44, 0xbd, 0x3d, 0x83, 0x0e, 0x59, 0xfd, 0x1b, 0xba, 0x46, 0x7d, 0x29, 0xb5,
0xda, 0x4c, 0xdc, 0xdc, 0xae, 0x5d, 0x1e, 0xcb, 0x17, 0xb4, 0x81, 0x69, 0x89, 0x1b, 0xfe, 0x41,
0x11, 0xba, 0x9c, 0x56, 0x9b, 0xa6, 0xfc, 0x62, 0xa9, 0x76, 0x65, 0x3c, 0x63, 0xd0, 0xcc, 0x73,
0xa8, 0xc4, 0x7f, 0x5c, 0x84, 0x52, 0xa0, 0xd3, 0x8c, 0x5f, 0x30, 0xd5, 0xae, 0x4d, 0xc2, 0x1a,
0x6e, 0x2c, 0xf1, 0x33, 0x9c, 0xab, 0x93, 0xfc, 0x5c, 0x21, 0xb3, 0xb1, 0xac, 0x5f, 0x36, 0x70,
0x07, 0x46, 0x6f, 0x3e, 0xa3, 0xd4, 0xdf, 0xbc, 0xa4, 0x5c, 0xb0, 0x4f, 0x73, 0x60, 0xfa, 0x25,
0x6a, 0x75, 0x06, 0x1d, 0xc3, 0x72, 0xec, 0x7a, 0x02, 0x4a, 0x11, 0x4f, 0xbf, 0x87, 0x51, 0xbb,
0x3a, 0x01, 0x67, 0x34, 0x22, 0xc2, 0xd7, 0x11, 0xd2, 0x23, 0x22, 0xe5, 0xb2, 0x43, 0x7a, 0x44,
0xa4, 0xde, 0x6c, 0x60, 0xc1, 0x1d, 0xb9, 0x86, 0x90, 0x16, 0xdc, 0x69, 0x97, 0x1f, 0x6a, 0x97,
0xc7, 0xf2, 0x85, 0x9d, 0x16, 0xbb, 0x94, 0x90, 0xe6, 0xb4, 0xf4, 0x4b, 0x0f, 0xb5, 0xab, 0x13,
0x70, 0xc6, 0xa3, 0x60, 0x78, 0xc4, 0x99, 0x15, 0x05, 0x89, 0x03, 0xf9, 0xac, 0x28, 0x48, 0x9e,
0x96, 0x8a, 0x28, 0x88, 0x1d, 0x4d, 0x5e, 0x99, 0xe0, 0x28, 0x25, 0x3b, 0x0a, 0xd2, 0x0f, 0x5d,
0xd4, 0x19, 0xf4, 0x53, 0x05, 0xaa, 0x59, 0xc7, 0x14, 0x28, 0xa5, 0xbe, 0x1b, 0x73, 0xb2, 0x52,
0xdb, 0x9a, 0x46, 0x24, 0xb0, 0xe2, 0x4b, 0x40, 0xc9, 0x75, 0x0f, 0x7d, 0x27, 0x6d, 0x64, 0x32,
0x56, 0xd7, 0xda, 0x7b, 0x93, 0x31, 0x07, 0x4d, 0xb6, 0xa1, 0x20, 0x0f, 0x46, 0x50, 0x4a, 0x96,
0x8e, 0x1d, 0xcb, 0xd4, 0xd4, 0x51, 0x2c, 0x81, 0xd2, 0xc7, 0x30, 0x4b, 0xa9, 0xe8, 0x7c, 0x3a,
0xb7, 0x54, 0xb6, 0x9e, 0xf5, 0x39, 0x50, 0xf4, 0x0c, 0xe6, 0xf9, 0x49, 0x00, 0x4a, 0x41, 0x1e,
0x22, 0xe7, 0x15, 0xb5, 0x0b, 0xd9, 0x0c, 0x81, 0xba, 0x2f, 0xf8, 0x7f, 0xe3, 0x10, 0x20, 0x3f,
0x7a, 0x37, 0xfd, 0xe7, 0xcd, 0xd1, 0x33, 0x85, 0xda, 0xc5, 0x31, 0x5c, 0xe1, 0x49, 0x11, 0xab,
0x7a, 0x2f, 0x8f, 0xdd, 0xba, 0x64, 0x4f, 0x8a, 0xf4, 0xcd, 0x11, 0x0f, 0x92, 0xe4, 0xe6, 0x29,
0x2d, 0x48, 0x32, 0xb7, 0xac, 0x69, 0x41, 0x92, 0xbd, 0x1f, 0x53, 0x67, 0x90, 0x0f, 0xa7, 0x52,
0xa0, 0x32, 0xf4, 0x5e, 0x56, 0x90, 0xa7, 0xe1, 0x76, 0xb5, 0xeb, 0x13, 0x72, 0x87, 0x07, 0x5f,
0x4c, 0xfa, 0xb7, 0xb3, 0xf1, 0xa3, 0xcc, 0xc1, 0x8f, 0x4f, 0xf1, 0xad, 0x7f, 0xc9, 0xc3, 0x22,
0x87, 0x41, 0x45, 0x05, 0xf3, 0x19, 0xc0, 0xf0, 0x04, 0x02, 0xbd, 0x93, 0xee, 0x93, 0xc8, 0x29,
0x4d, 0xed, 0xdd, 0xd1, 0x4c, 0xe1, 0x40, 0x0b, 0xa1, 0xf9, 0x69, 0x81, 0x96, 0x3c, 0xb4, 0x48,
0x0b, 0xb4, 0x94, 0x23, 0x01, 0x75, 0x06, 0x7d, 0x0c, 0xc5, 0x00, 0x36, 0x46, 0x69, 0xb0, 0x73,
0x0c, 0x17, 0xaf, 0xbd, 0x33, 0x92, 0x27, 0x6c, 0x75, 0x08, 0x13, 0x4e, 0xb3, 0x3a, 0x89, 0x3d,
0xa7, 0x59, 0x9d, 0x06, 0x2c, 0x0f, 0x7d, 0xc2, 0x91, 0xa3, 0x4c, 0x9f, 0x44, 0x80, 0xbb, 0x4c,
0x9f, 0x44, 0xe1, 0x27, 0x75, 0xe6, 0xe1, 0xa5, 0x5f, 0x7e, 0xb5, 0xae, 0xfc, 0xd3, 0x57, 0xeb,
0x33, 0x3f, 0xf9, 0x7a, 0x5d, 0xf9, 0xe5, 0xd7, 0xeb, 0xca, 0x3f, 0x7e, 0xbd, 0xae, 0xfc, 0xeb,
0xd7, 0xeb, 0xca, 0x9f, 0xfe, 0xdb, 0xfa, 0xcc, 0x0f, 0x0b, 0x52, 0xfa, 0x70, 0x9e, 0xfd, 0x4f,
0x9d, 0x0f, 0xfe, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x11, 0x3f, 0x6e, 0x39, 0x19, 0x49, 0x00, 0x00,
}

27
vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto generated vendored
View file

@ -1,3 +1,19 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// To regenerate api.pb.go run hack/update-generated-runtime.sh
syntax = 'proto3';
@ -297,7 +313,8 @@ message PodSandboxConfig {
// operation. The runtime may also use this information to improve UX, such
// as by constructing a readable name.
PodSandboxMetadata metadata = 1;
// Hostname of the sandbox.
// Hostname of the sandbox. Hostname could only be empty when the pod
// network namespace is NODE.
string hostname = 2;
// Path to the directory on the host in which container log files are
// stored.
@ -349,7 +366,7 @@ message RunPodSandboxRequest {
// If the runtime handler is unknown, this request should be rejected. An
// empty string should select the default handler, equivalent to the
// behavior before this feature was added.
// See https://git.k8s.io/community/keps/sig-node/0014-runtime-class.md
// See https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md
string runtime_handler = 2;
}
@ -423,6 +440,8 @@ message PodSandboxStatus {
// MUST be identical to that of the corresponding PodSandboxConfig used to
// instantiate the pod sandbox this status represents.
map<string, string> annotations = 8;
// runtime configuration used for this PodSandbox.
string runtime_handler = 9;
}
message PodSandboxStatusResponse {
@ -477,6 +496,8 @@ message PodSandbox {
// MUST be identical to that of the corresponding PodSandboxConfig used to
// instantiate this PodSandbox.
map<string, string> annotations = 6;
// runtime configuration used for this PodSandbox.
string runtime_handler = 7;
}
message ListPodSandboxResponse {
@ -1221,7 +1242,7 @@ message ContainerStats {
CpuUsage cpu = 2;
// Memory usage gathered from the container.
MemoryUsage memory = 3;
// Usage of the writeable layer.
// Usage of the writable layer.
FilesystemUsage writable_layer = 4;
}

2
vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/services.go generated vendored
View file

@ -111,7 +111,7 @@ type ImageManagerService interface {
// ImageStatus returns the status of the image.
ImageStatus(image *runtimeapi.ImageSpec) (*runtimeapi.Image, error)
// PullImage pulls an image with the authentication config.
PullImage(image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig) (string, error)
PullImage(image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error)
// RemoveImage removes the image.
RemoveImage(image *runtimeapi.ImageSpec) error
// ImageFsInfo returns information of the filesystem that is used to store images.

42
vendor/k8s.io/kubernetes/pkg/kubelet/apis/well_known_labels.go generated vendored
View file

@ -19,17 +19,14 @@ package apis
import (
"strings"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
)
const (
LabelHostname = "kubernetes.io/hostname"
LabelZoneFailureDomain = "failure-domain.beta.kubernetes.io/zone"
LabelMultiZoneDelimiter = "__"
LabelZoneRegion = "failure-domain.beta.kubernetes.io/region"
LabelInstanceType = "beta.kubernetes.io/instance-type"
// The OS/Arch labels are promoted to GA in 1.14. kubelet applies both beta
// and GA labels to ensure backward compatibility.
// TODO: stop applying the beta OS/Arch labels in Kubernetes 1.18.
LabelOS = "beta.kubernetes.io/os"
LabelArch = "beta.kubernetes.io/arch"
@ -38,40 +35,27 @@ const (
labelZoneFailureDomainGA = "failure-domain.kubernetes.io/zone"
labelZoneRegionGA = "failure-domain.kubernetes.io/region"
labelInstanceTypeGA = "kubernetes.io/instance-type"
labelOSGA = "kubernetes.io/os"
labelArchGA = "kubernetes.io/arch"
// LabelNamespaceSuffixKubelet is an allowed label namespace suffix kubelets can self-set ([*.]kubelet.kubernetes.io/*)
LabelNamespaceSuffixKubelet = "kubelet.kubernetes.io"
// LabelNamespaceSuffixNode is an allowed label namespace suffix kubelets can self-set ([*.]node.kubernetes.io/*)
LabelNamespaceSuffixNode = "node.kubernetes.io"
// LabelNamespaceNodeRestriction is a forbidden label namespace that kubelets may not self-set when the NodeRestriction admission plugin is enabled
LabelNamespaceNodeRestriction = "node-restriction.kubernetes.io"
)
// When the --failure-domains scheduler flag is not specified,
// DefaultFailureDomains defines the set of label keys used when TopologyKey is empty in PreferredDuringScheduling anti-affinity.
var DefaultFailureDomains string = LabelHostname + "," + LabelZoneFailureDomain + "," + LabelZoneRegion
var kubeletLabels = sets.NewString(
LabelHostname,
LabelZoneFailureDomain,
LabelZoneRegion,
LabelInstanceType,
v1.LabelHostname,
v1.LabelZoneFailureDomain,
v1.LabelZoneRegion,
v1.LabelInstanceType,
v1.LabelOSStable,
v1.LabelArchStable,
LabelOS,
LabelArch,
labelZoneFailureDomainGA,
labelZoneRegionGA,
labelInstanceTypeGA,
labelOSGA,
labelArchGA,
)
var kubeletLabelNamespaces = sets.NewString(
LabelNamespaceSuffixKubelet,
LabelNamespaceSuffixNode,
v1.LabelNamespaceSuffixKubelet,
v1.LabelNamespaceSuffixNode,
)
// KubeletLabels returns the list of label keys kubelets are allowed to set on their own Node objects

1
vendor/k8s.io/kubernetes/pkg/kubelet/container/BUILD generated vendored
View file

@ -29,6 +29,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/tools/reference:go_default_library",
"//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library",

20
vendor/k8s.io/kubernetes/pkg/kubelet/container/helpers.go generated vendored
View file

@ -27,6 +27,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/record"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/kubelet/util/format"
@ -130,9 +131,22 @@ func ExpandContainerCommandOnlyStatic(containerCommand []string, envs []v1.EnvVa
return command
}
func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (expandedSubpath string) {
mapping := expansion.MappingFuncFor(EnvVarsToMap(envs))
return expansion.Expand(mount.SubPath, mapping)
func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (string, error) {
envmap := EnvVarsToMap(envs)
missingKeys := sets.NewString()
expanded := expansion.Expand(mount.SubPathExpr, func(key string) string {
value, ok := envmap[key]
if !ok || len(value) == 0 {
missingKeys.Insert(key)
}
return value
})
if len(missingKeys) > 0 {
return "", fmt.Errorf("missing value for %s", strings.Join(missingKeys.List(), ", "))
}
return expanded, nil
}
func ExpandContainerCommandAndArgs(container *v1.Container, envs []EnvVar) (command []string, args []string) {

26
vendor/k8s.io/kubernetes/pkg/kubelet/container/runtime.go generated vendored
View file

@ -88,7 +88,7 @@ type Runtime interface {
// TODO: Revisit this method and make it cleaner.
GarbageCollect(gcPolicy ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error
// Syncs the running pod into the desired pod.
SyncPod(pod *v1.Pod, apiPodStatus v1.PodStatus, podStatus *PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) PodSyncResult
SyncPod(pod *v1.Pod, podStatus *PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) PodSyncResult
// KillPod kills all the containers of a pod. Pod may be nil, running pod must not be.
// TODO(random-liu): Return PodSyncResult in KillPod.
// gracePeriodOverride if specified allows the caller to override the pod default grace period.
@ -98,17 +98,6 @@ type Runtime interface {
// GetPodStatus retrieves the status of the pod, including the
// information of all containers in the pod that are visible in Runtime.
GetPodStatus(uid types.UID, name, namespace string) (*PodStatus, error)
// Returns the filesystem path of the pod's network namespace; if the
// runtime does not handle namespace creation itself, or cannot return
// the network namespace path, it should return an error.
// TODO: Change ContainerID to a Pod ID since the namespace is shared
// by all containers in the pod.
GetNetNS(containerID ContainerID) (string, error)
// Returns the container ID that represents the Pod, as passed to network
// plugins. For example, if the runtime uses an infra container, returns
// the infra container's ContainerID.
// TODO: Change ContainerID to a Pod ID, see GetNetNS()
GetPodContainerID(*Pod) (ContainerID, error)
// TODO(vmarmol): Unify pod and containerID args.
// GetContainerLogs returns logs of a specific container. By
// default, it returns a snapshot of the container log. Set 'follow' to true to
@ -137,7 +126,7 @@ type StreamingRuntime interface {
type ImageService interface {
// PullImage pulls an image from the network to local storage using the supplied
// secrets if necessary. It returns a reference (digest or ID) to the pulled image.
PullImage(image ImageSpec, pullSecrets []v1.Secret) (string, error)
PullImage(image ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error)
// GetImageRef gets the reference (digest or ID) of the image which has already been in
// the local storage. It returns ("", nil) if the image isn't in the local storage.
GetImageRef(image ImageSpec) (string, error)
@ -155,7 +144,7 @@ type ContainerAttacher interface {
type ContainerCommandRunner interface {
// RunInContainer synchronously executes the command in the container, and returns the output.
// If the command completes with a non-0 exit code, a pkg/util/exec.ExitError will be returned.
// If the command completes with a non-0 exit code, a k8s.io/utils/exec.ExitError will be returned.
RunInContainer(id ContainerID, cmd []string, timeout time.Duration) ([]byte, error)
}
@ -254,13 +243,6 @@ const (
ContainerStateUnknown ContainerState = "unknown"
)
type ContainerType string
const (
ContainerTypeInit ContainerType = "INIT"
ContainerTypeRegular ContainerType = "REGULAR"
)
// Container provides the runtime information for a container, such as ID, hash,
// state of the container.
type Container struct {
@ -289,7 +271,7 @@ type PodStatus struct {
ID types.UID
// Name of the pod.
Name string
// Namspace of the pod.
// Namespace of the pod.
Namespace string
// IP of the pod.
IP string

6
vendor/k8s.io/kubernetes/pkg/kubelet/errors.go generated vendored
View file

@ -16,6 +16,12 @@ limitations under the License.
package kubelet
import "errors"
const (
NetworkNotReadyErrorMsg = "network is not ready"
)
var (
ErrNetworkUnknown = errors.New("network state unknown")
)

94
vendor/k8s.io/kubernetes/pkg/kubelet/kubelet.go generated vendored
View file

@ -43,7 +43,6 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
corelisters "k8s.io/client-go/listers/core/v1"
@ -51,9 +50,7 @@ import (
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/certificate"
"k8s.io/client-go/util/flowcontrol"
"k8s.io/client-go/util/integer"
cloudprovider "k8s.io/cloud-provider"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/klog"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/features"
@ -115,7 +112,9 @@ import (
"k8s.io/kubernetes/pkg/util/oom"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/csi"
"k8s.io/kubernetes/pkg/volume/util/subpath"
utilexec "k8s.io/utils/exec"
"k8s.io/utils/integer"
)
const (
@ -248,13 +247,12 @@ type Dependencies struct {
HeartbeatClient clientset.Interface
OnHeartbeatFailure func()
KubeClient clientset.Interface
CSIClient csiclientset.Interface
DynamicKubeClient dynamic.Interface
Mounter mount.Interface
OOMAdjuster *oom.OOMAdjuster
OSInterface kubecontainer.OSInterface
PodConfig *config.PodConfig
Recorder record.EventRecorder
Subpather subpath.Interface
VolumePlugins []volume.VolumePlugin
DynamicPluginProber volume.DynamicPluginProber
TLSOptions *server.TLSOptions
@ -491,7 +489,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
hostnameOverridden: len(hostnameOverride) > 0,
nodeName: nodeName,
kubeClient: kubeDeps.KubeClient,
csiClient: kubeDeps.CSIClient,
heartbeatClient: kubeDeps.HeartbeatClient,
onRepeatedHeartbeatFailure: kubeDeps.OnHeartbeatFailure,
rootDirectory: rootDirectory,
@ -519,6 +516,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
cgroupsPerQOS: kubeCfg.CgroupsPerQOS,
cgroupRoot: kubeCfg.CgroupRoot,
mounter: kubeDeps.Mounter,
subpather: kubeDeps.Subpather,
maxPods: int(kubeCfg.MaxPods),
podsPerCore: int(kubeCfg.PodsPerCore),
syncLoopMonitor: atomic.Value{},
@ -590,6 +588,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
// podManager is also responsible for keeping secretManager and configMapManager contents up-to-date.
klet.podManager = kubepod.NewBasicPodManager(kubepod.NewBasicMirrorClient(klet.kubeClient), secretManager, configMapManager, checkpointManager)
klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
if remoteRuntimeEndpoint != "" {
// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
if remoteImageEndpoint == "" {
@ -609,10 +609,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.resourceAnalyzer = serverstats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration)
if containerRuntime == "rkt" {
klog.Fatalln("rktnetes has been deprecated in favor of rktlet. Please see https://github.com/kubernetes-incubator/rktlet for more information.")
}
// if left at nil, that means it is unneeded
var legacyLogProvider kuberuntime.LegacyLogProvider
@ -660,8 +656,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
klet.runtimeService = runtimeService
if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.DynamicKubeClient != nil {
klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.DynamicKubeClient)
if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) {
klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient)
}
runtime, err := kuberuntime.NewKubeGenericRuntimeManager(
@ -705,7 +701,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.resourceAnalyzer,
klet.podManager,
klet.runtimeCache,
klet.containerRuntime)
klet.containerRuntime,
klet.statusManager)
} else {
klet.StatsProvider = stats.NewCRIStatsProvider(
klet.cadvisor,
@ -714,7 +711,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.runtimeCache,
runtimeService,
imageService,
stats.NewLogMetricsService())
stats.NewLogMetricsService(),
kubecontainer.RealOS{})
}
klet.pleg = pleg.NewGenericPLEG(klet.containerRuntime, plegChannelCapacity, plegRelistPeriod, klet.podCache, clock.RealClock{})
@ -754,8 +752,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.containerLogManager = logs.NewStubContainerLogManager()
}
klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
if kubeCfg.ServerTLSBootstrap && kubeDeps.TLSOptions != nil && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory)
if err != nil {
@ -779,10 +775,9 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
tokenManager := token.NewManager(kubeDeps.KubeClient)
if !utilfeature.DefaultFeatureGate.Enabled(features.MountPropagation) {
return nil, fmt.Errorf("mount propagation feature gate has been deprecated and will be removed in 1.14")
}
// NewInitializedVolumePluginMgr intializes some storageErrors on the Kubelet runtimeState (in csi_plugin.go init)
// which affects node ready status. This function must be called before Kubelet is initialized so that the Node
// ReadyState is accurate with the storage state.
klet.volumePluginMgr, err =
NewInitializedVolumePluginMgr(klet, secretManager, configMapManager, tokenManager, kubeDeps.VolumePlugins, kubeDeps.DynamicPluginProber)
if err != nil {
@ -827,7 +822,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.podKillingCh = make(chan *kubecontainer.PodPair, podKillingChannelCapacity)
// setup eviction manager
evictionManager, evictionAdmitHandler := eviction.NewManager(klet.resourceAnalyzer, evictionConfig, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.imageManager, klet.containerGC, kubeDeps.Recorder, nodeRef, klet.clock)
evictionManager, evictionAdmitHandler := eviction.NewManager(klet.resourceAnalyzer, evictionConfig, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.podManager.GetMirrorPodByPod, klet.imageManager, klet.containerGC, kubeDeps.Recorder, nodeRef, klet.clock)
klet.evictionManager = evictionManager
klet.admitHandlers.AddPodAdmitHandler(evictionAdmitHandler)
@ -902,7 +897,6 @@ type Kubelet struct {
nodeName types.NodeName
runtimeCache kubecontainer.RuntimeCache
kubeClient clientset.Interface
csiClient csiclientset.Interface
heartbeatClient clientset.Interface
iptClient utilipt.Interface
rootDirectory string
@ -1064,15 +1058,15 @@ type Kubelet struct {
lastStatusReportTime time.Time
// syncNodeStatusMux is a lock on updating the node status, because this path is not thread-safe.
// This lock is used by Kublet.syncNodeStatus function and shouldn't be used anywhere else.
// This lock is used by Kubelet.syncNodeStatus function and shouldn't be used anywhere else.
syncNodeStatusMux sync.Mutex
// updatePodCIDRMux is a lock on updating pod CIDR, because this path is not thread-safe.
// This lock is used by Kublet.syncNodeStatus function and shouldn't be used anywhere else.
// This lock is used by Kubelet.syncNodeStatus function and shouldn't be used anywhere else.
updatePodCIDRMux sync.Mutex
// updateRuntimeMux is a lock on updating runtime, because this path is not thread-safe.
// This lock is used by Kublet.updateRuntimeUp function and shouldn't be used anywhere else.
// This lock is used by Kubelet.updateRuntimeUp function and shouldn't be used anywhere else.
updateRuntimeMux sync.Mutex
// nodeLeaseController claims and renews the node lease for this Kubelet
@ -1102,6 +1096,9 @@ type Kubelet struct {
// Mounter to use for volumes.
mounter mount.Interface
// subpather to execute subpath actions
subpather subpath.Interface
// Manager of non-Runtime containers.
containerManager cm.ContainerManager
@ -1221,29 +1218,6 @@ type Kubelet struct {
runtimeClassManager *runtimeclass.Manager
}
func allGlobalUnicastIPs() ([]net.IP, error) {
interfaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("could not list network interfaces: %v", err)
}
var ips []net.IP
for _, i := range interfaces {
addresses, err := i.Addrs()
if err != nil {
return nil, fmt.Errorf("could not list the addresses for network interface %v: %v", i, err)
}
for _, address := range addresses {
switch v := address.(type) {
case *net.IPNet:
if v.IP.IsGlobalUnicast() {
ips = append(ips, v.IP)
}
}
}
}
return ips, nil
}
// setupDataDirs creates:
// 1. the root directory
// 2. the pods directory
@ -1329,6 +1303,7 @@ func (kl *Kubelet) initializeModules() error {
collectors.NewVolumeStatsCollector(kl),
collectors.NewLogMetricsCollector(kl.StatsProvider.ListPodStats),
)
metrics.SetNodeName(kl.nodeName)
// Setup filesystem directories.
if err := kl.setupDataDirs(); err != nil {
@ -1452,7 +1427,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
// Start syncing RuntimeClasses if enabled.
if kl.runtimeClassManager != nil {
go kl.runtimeClassManager.Run(wait.NeverStop)
kl.runtimeClassManager.Start(wait.NeverStop)
}
// Start the pod lifecycle event generator.
@ -1525,7 +1500,8 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
if !firstSeenTime.IsZero() {
// This is the first time we are syncing the pod. Record the latency
// since kubelet first saw the pod if firstSeenTime is set.
metrics.PodWorkerStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime))
metrics.PodWorkerStartDuration.Observe(metrics.SinceInSeconds(firstSeenTime))
metrics.DeprecatedPodWorkerStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime))
} else {
klog.V(3).Infof("First seen time not recorded for pod %q", pod.UID)
}
@ -1542,7 +1518,8 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
existingStatus, ok := kl.statusManager.GetPodStatus(pod.UID)
if !ok || existingStatus.Phase == v1.PodPending && apiPodStatus.Phase == v1.PodRunning &&
!firstSeenTime.IsZero() {
metrics.PodStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime))
metrics.PodStartDuration.Observe(metrics.SinceInSeconds(firstSeenTime))
metrics.DeprecatedPodStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime))
}
runnable := kl.canRunPod(pod)
@ -1585,9 +1562,9 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
}
// If the network plugin is not ready, only start the pod if it uses the host network
if rs := kl.runtimeState.networkErrors(); len(rs) != 0 && !kubecontainer.IsHostNetworkPod(pod) {
kl.recorder.Eventf(pod, v1.EventTypeWarning, events.NetworkNotReady, "%s: %v", NetworkNotReadyErrorMsg, rs)
return fmt.Errorf("%s: %v", NetworkNotReadyErrorMsg, rs)
if err := kl.runtimeState.networkErrors(); err != nil && !kubecontainer.IsHostNetworkPod(pod) {
kl.recorder.Eventf(pod, v1.EventTypeWarning, events.NetworkNotReady, "%s: %v", NetworkNotReadyErrorMsg, err)
return fmt.Errorf("%s: %v", NetworkNotReadyErrorMsg, err)
}
// Create Cgroups for the pod and apply resource parameters
@ -1686,7 +1663,7 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
pullSecrets := kl.getPullSecretsForPod(pod)
// Call the container runtime's SyncPod callback
result := kl.containerRuntime.SyncPod(pod, apiPodStatus, podStatus, pullSecrets, kl.backOff)
result := kl.containerRuntime.SyncPod(pod, podStatus, pullSecrets, kl.backOff)
kl.reasonCache.Update(pod.UID, result)
if err := result.Error(); err != nil {
// Do not return error if the only failures were pods in backoff
@ -1842,8 +1819,8 @@ func (kl *Kubelet) syncLoop(updates <-chan kubetypes.PodUpdate, handler SyncHand
)
duration := base
for {
if rs := kl.runtimeState.runtimeErrors(); len(rs) != 0 {
klog.Infof("skipping pod synchronization - %v", rs)
if err := kl.runtimeState.runtimeErrors(); err != nil {
klog.Infof("skipping pod synchronization - %v", err)
// exponential backoff
time.Sleep(duration)
duration = time.Duration(math.Min(float64(max), factor*float64(duration)))
@ -2021,7 +1998,8 @@ func (kl *Kubelet) dispatchWork(pod *v1.Pod, syncType kubetypes.SyncPodType, mir
UpdateType: syncType,
OnCompleteFunc: func(err error) {
if err != nil {
metrics.PodWorkerLatency.WithLabelValues(syncType.String()).Observe(metrics.SinceInMicroseconds(start))
metrics.PodWorkerDuration.WithLabelValues(syncType.String()).Observe(metrics.SinceInSeconds(start))
metrics.DeprecatedPodWorkerLatency.WithLabelValues(syncType.String()).Observe(metrics.SinceInMicroseconds(start))
}
},
})

12
vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_getters.go generated vendored
View file

@ -25,14 +25,14 @@ import (
cadvisorapiv1 "github.com/google/cadvisor/info/v1"
"k8s.io/klog"
v1 "k8s.io/api/core/v1"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
utilfile "k8s.io/kubernetes/pkg/util/file"
"k8s.io/kubernetes/pkg/util/mount"
utilnode "k8s.io/kubernetes/pkg/util/node"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
utilpath "k8s.io/utils/path"
)
// getRootDir returns the full path to the directory under which kubelet can
@ -278,7 +278,7 @@ func (kl *Kubelet) getPodVolumePathListFromDisk(podUID types.UID) ([]string, err
volumes := []string{}
podVolDir := kl.getPodVolumesDir(podUID)
if pathExists, pathErr := volumeutil.PathExists(podVolDir); pathErr != nil {
if pathExists, pathErr := mount.PathExists(podVolDir); pathErr != nil {
return volumes, fmt.Errorf("Error checking if path %q exists: %v", podVolDir, pathErr)
} else if !pathExists {
klog.Warningf("Path %q does not exist", podVolDir)
@ -293,7 +293,7 @@ func (kl *Kubelet) getPodVolumePathListFromDisk(podUID types.UID) ([]string, err
for _, volumePluginDir := range volumePluginDirs {
volumePluginName := volumePluginDir.Name()
volumePluginPath := filepath.Join(podVolDir, volumePluginName)
volumeDirs, err := utilfile.ReadDirNoStat(volumePluginPath)
volumeDirs, err := utilpath.ReadDirNoStat(volumePluginPath)
if err != nil {
return volumes, fmt.Errorf("Could not read directory %s: %v", volumePluginPath, err)
}
@ -327,7 +327,7 @@ func (kl *Kubelet) getMountedVolumePathListFromDisk(podUID types.UID) ([]string,
func (kl *Kubelet) podVolumeSubpathsDirExists(podUID types.UID) (bool, error) {
podVolDir := kl.getPodVolumeSubpathsDir(podUID)
if pathExists, pathErr := volumeutil.PathExists(podVolDir); pathErr != nil {
if pathExists, pathErr := mount.PathExists(podVolDir); pathErr != nil {
return true, fmt.Errorf("Error checking if path %q exists: %v", podVolDir, pathErr)
} else if !pathExists {
return false, nil

51
vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_node_status.go generated vendored
View file

@ -134,6 +134,7 @@ func (kl *Kubelet) reconcileExtendedResource(initialNode, node *v1.Node) bool {
requiresUpdate := false
for k := range node.Status.Capacity {
if v1helper.IsExtendedResourceName(k) {
klog.Infof("Zero out resource %s capacity in existing node.", k)
node.Status.Capacity[k] = *resource.NewQuantity(int64(0), resource.DecimalSI)
node.Status.Allocatable[k] = *resource.NewQuantity(int64(0), resource.DecimalSI)
requiresUpdate = true
@ -145,10 +146,12 @@ func (kl *Kubelet) reconcileExtendedResource(initialNode, node *v1.Node) bool {
// updateDefaultLabels will set the default labels on the node
func (kl *Kubelet) updateDefaultLabels(initialNode, existingNode *v1.Node) bool {
defaultLabels := []string{
kubeletapis.LabelHostname,
kubeletapis.LabelZoneFailureDomain,
kubeletapis.LabelZoneRegion,
kubeletapis.LabelInstanceType,
v1.LabelHostname,
v1.LabelZoneFailureDomain,
v1.LabelZoneRegion,
v1.LabelInstanceType,
v1.LabelOSStable,
v1.LabelArchStable,
kubeletapis.LabelOS,
kubeletapis.LabelArch,
}
@ -213,9 +216,11 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
ObjectMeta: metav1.ObjectMeta{
Name: string(kl.nodeName),
Labels: map[string]string{
kubeletapis.LabelHostname: kl.hostname,
kubeletapis.LabelOS: goruntime.GOOS,
kubeletapis.LabelArch: goruntime.GOARCH,
v1.LabelHostname: kl.hostname,
v1.LabelOSStable: goruntime.GOOS,
v1.LabelArchStable: goruntime.GOARCH,
kubeletapis.LabelOS: goruntime.GOOS,
kubeletapis.LabelArch: goruntime.GOARCH,
},
},
Spec: v1.NodeSpec{
@ -323,8 +328,8 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
return nil, err
}
if instanceType != "" {
klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelInstanceType, instanceType)
node.ObjectMeta.Labels[kubeletapis.LabelInstanceType] = instanceType
klog.Infof("Adding node label from cloud provider: %s=%s", v1.LabelInstanceType, instanceType)
node.ObjectMeta.Labels[v1.LabelInstanceType] = instanceType
}
// If the cloud has zone information, label the node with the zone information
zones, ok := kl.cloud.Zones()
@ -334,12 +339,12 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
return nil, fmt.Errorf("failed to get zone from cloud provider: %v", err)
}
if zone.FailureDomain != "" {
klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneFailureDomain, zone.FailureDomain)
node.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain] = zone.FailureDomain
klog.Infof("Adding node label from cloud provider: %s=%s", v1.LabelZoneFailureDomain, zone.FailureDomain)
node.ObjectMeta.Labels[v1.LabelZoneFailureDomain] = zone.FailureDomain
}
if zone.Region != "" {
klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneRegion, zone.Region)
node.ObjectMeta.Labels[kubeletapis.LabelZoneRegion] = zone.Region
klog.Infof("Adding node label from cloud provider: %s=%s", v1.LabelZoneRegion, zone.Region)
node.ObjectMeta.Labels[v1.LabelZoneRegion] = zone.Region
}
}
}
@ -423,6 +428,23 @@ func (kl *Kubelet) tryUpdateNodeStatus(tryNumber int) error {
now := kl.clock.Now()
if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) && now.Before(kl.lastStatusReportTime.Add(kl.nodeStatusReportFrequency)) {
if !podCIDRChanged && !nodeStatusHasChanged(&originalNode.Status, &node.Status) {
// We must mark the volumes as ReportedInUse in volume manager's dsw even
// if no changes were made to the node status (no volumes were added or removed
// from the VolumesInUse list).
//
// The reason is that on a kubelet restart, the volume manager's dsw is
// repopulated and the volume ReportedInUse is initialized to false, while the
// VolumesInUse list from the Node object still contains the state from the
// previous kubelet instantiation.
//
// Once the volumes are added to the dsw, the ReportedInUse field needs to be
// synced from the VolumesInUse list in the Node.Status.
//
// The MarkVolumesAsReportedInUse() call cannot be performed in dsw directly
// because it does not have access to the Node object.
// This also cannot be populated on node status manager init because the volume
// may not have been added to dsw at that time.
kl.volumeManager.MarkVolumesAsReportedInUse(node.Status.VolumesInUse)
return nil
}
}
@ -522,8 +544,9 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error {
nodestatus.MemoryPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderMemoryPressure, kl.recordNodeStatusEvent),
nodestatus.DiskPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderDiskPressure, kl.recordNodeStatusEvent),
nodestatus.PIDPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderPIDPressure, kl.recordNodeStatusEvent),
nodestatus.ReadyCondition(kl.clock.Now, kl.runtimeState.runtimeErrors, kl.runtimeState.networkErrors, validateHostFunc, kl.containerManager.Status, kl.recordNodeStatusEvent),
nodestatus.ReadyCondition(kl.clock.Now, kl.runtimeState.runtimeErrors, kl.runtimeState.networkErrors, kl.runtimeState.storageErrors, validateHostFunc, kl.containerManager.Status, kl.recordNodeStatusEvent),
nodestatus.VolumesInUse(kl.volumeManager.ReconcilerStatesHasBeenSynced, kl.volumeManager.GetVolumesInUse),
nodestatus.RemoveOutOfDiskCondition(),
// TODO(mtaufen): I decided not to move this setter for now, since all it does is send an event
// and record state back to the Kubelet runtime object. In the future, I'd like to isolate
// these side-effects by decoupling the decisions to send events and partial status recording

49
vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods.go generated vendored
View file

@ -61,6 +61,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format"
mountutil "k8s.io/kubernetes/pkg/util/mount"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
"k8s.io/kubernetes/third_party/forked/golang/expansion"
@ -127,7 +128,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol
}
// makeMounts determines the mount points for the given container.
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, subpather subpath.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
// Kubernetes only mounts on /etc/hosts if:
// - container is not an infrastructure (pause) container
// - container is not already mounting on /etc/hosts
@ -159,27 +160,40 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
if err != nil {
return nil, cleanupAction, err
}
if mount.SubPath != "" {
subPath := mount.SubPath
if mount.SubPathExpr != "" {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
}
// Expand subpath variables
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
mount.SubPath = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
return nil, cleanupAction, fmt.Errorf("volume subpath expansion is disabled")
}
if filepath.IsAbs(mount.SubPath) {
return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
}
subPath, err = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
err = volumevalidation.ValidatePathNoBacksteps(mount.SubPath)
if err != nil {
return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
return nil, cleanupAction, err
}
}
if subPath != "" {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
}
if filepath.IsAbs(subPath) {
return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", subPath)
}
err = volumevalidation.ValidatePathNoBacksteps(subPath)
if err != nil {
return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", subPath, err)
}
volumePath := hostPath
hostPath = filepath.Join(volumePath, mount.SubPath)
hostPath = filepath.Join(volumePath, subPath)
if subPathExists, err := mounter.ExistsPath(hostPath); err != nil {
klog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath)
@ -193,13 +207,13 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
if err != nil {
return nil, cleanupAction, err
}
if err := mounter.SafeMakeDir(mount.SubPath, volumePath, perm); err != nil {
if err := subpather.SafeMakeDir(subPath, volumePath, perm); err != nil {
// Don't pass detailed error back to the user because it could give information about host filesystem
klog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name)
}
}
hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{
hostPath, cleanupAction, err = subpather.PrepareSafeSubpath(subpath.Subpath{
VolumeMountIndex: i,
Path: hostPath,
VolumeName: vol.InnerVolumeSpecName,
@ -384,7 +398,6 @@ func truncatePodHostnameIfNeeded(podName, hostname string) (string, error) {
// GeneratePodHostNameAndDomain creates a hostname and domain name for a pod,
// given that pod's spec and annotations or returns an error.
func (kl *Kubelet) GeneratePodHostNameAndDomain(pod *v1.Pod) (string, string, error) {
// TODO(vmarmol): Handle better.
clusterDomain := kl.dnsConfigurer.ClusterDomain
hostname := pod.Name
@ -452,7 +465,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
}
opts.Envs = append(opts.Envs, envs...)
mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, opts.Envs)
mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, kl.subpather, opts.Envs)
if err != nil {
return nil, cleanupAction, err
}
@ -533,6 +546,10 @@ func (kl *Kubelet) getServiceEnvVarMap(ns string, enableServiceLinks bool) (map[
// Make the environment variables for a pod in the given namespace.
func (kl *Kubelet) makeEnvironmentVariables(pod *v1.Pod, container *v1.Container, podIP string) ([]kubecontainer.EnvVar, error) {
if pod.Spec.EnableServiceLinks == nil {
return nil, fmt.Errorf("nil pod.spec.enableServiceLinks encountered, cannot construct envvars")
}
var result []kubecontainer.EnvVar
// Note: These are added to the docker Config, but are not included in the checksum computed
// by kubecontainer.HashContainer(...). That way, we can still determine whether an
@ -852,7 +869,7 @@ func (kl *Kubelet) podIsTerminated(pod *v1.Pod) bool {
return status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded || (pod.DeletionTimestamp != nil && notRunning(status.ContainerStatuses))
}
// IsPodTerminated returns trus if the pod with the provided UID is in a terminated state ("Failed" or "Succeeded")
// IsPodTerminated returns true if the pod with the provided UID is in a terminated state ("Failed" or "Succeeded")
// or if the pod has been deleted or removed
func (kl *Kubelet) IsPodTerminated(uid types.UID) bool {
pod, podFound := kl.podManager.GetPodByUID(uid)

42
vendor/k8s.io/kubernetes/pkg/kubelet/runtime.go generated vendored
View file

@ -17,9 +17,12 @@ limitations under the License.
package kubelet
import (
"errors"
"fmt"
"sync"
"time"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
)
type runtimeState struct {
@ -27,6 +30,7 @@ type runtimeState struct {
lastBaseRuntimeSync time.Time
baseRuntimeSyncThreshold time.Duration
networkError error
storageError error
cidr string
healthChecks []*healthCheck
}
@ -58,6 +62,12 @@ func (s *runtimeState) setNetworkState(err error) {
s.networkError = err
}
func (s *runtimeState) setStorageState(err error) {
s.Lock()
defer s.Unlock()
s.storageError = err
}
func (s *runtimeState) setPodCIDR(cidr string) {
s.Lock()
defer s.Unlock()
@ -70,32 +80,42 @@ func (s *runtimeState) podCIDR() string {
return s.cidr
}
func (s *runtimeState) runtimeErrors() []string {
func (s *runtimeState) runtimeErrors() error {
s.RLock()
defer s.RUnlock()
var ret []string
errs := []error{}
if s.lastBaseRuntimeSync.IsZero() {
ret = append(ret, "container runtime status check may not have completed yet")
errs = append(errs, errors.New("container runtime status check may not have completed yet."))
} else if !s.lastBaseRuntimeSync.Add(s.baseRuntimeSyncThreshold).After(time.Now()) {
ret = append(ret, "container runtime is down")
errs = append(errs, errors.New("container runtime is down."))
}
for _, hc := range s.healthChecks {
if ok, err := hc.fn(); !ok {
ret = append(ret, fmt.Sprintf("%s is not healthy: %v", hc.name, err))
errs = append(errs, fmt.Errorf("%s is not healthy: %v.", hc.name, err))
}
}
return ret
return utilerrors.NewAggregate(errs)
}
func (s *runtimeState) networkErrors() []string {
func (s *runtimeState) networkErrors() error {
s.RLock()
defer s.RUnlock()
var ret []string
errs := []error{}
if s.networkError != nil {
ret = append(ret, s.networkError.Error())
errs = append(errs, s.networkError)
}
return ret
return utilerrors.NewAggregate(errs)
}
func (s *runtimeState) storageErrors() error {
s.RLock()
defer s.RUnlock()
errs := []error{}
if s.storageError != nil {
errs = append(errs, s.storageError)
}
return utilerrors.NewAggregate(errs)
}
func newRuntimeState(
@ -104,6 +124,6 @@ func newRuntimeState(
return &runtimeState{
lastBaseRuntimeSync: time.Time{},
baseRuntimeSyncThreshold: runtimeSyncThreshold,
networkError: fmt.Errorf("network state unknown"),
networkError: ErrNetworkUnknown,
}
}

1
vendor/k8s.io/kubernetes/pkg/kubelet/util/sliceutils/sliceutils.go generated vendored
View file

@ -21,6 +21,7 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
)
// StringInSlice returns true if s is in list
func StringInSlice(s string, list []string) bool {
for _, v := range list {
if v == s {

11
vendor/k8s.io/kubernetes/pkg/kubelet/volume_host.go generated vendored
View file

@ -30,7 +30,6 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/record"
cloudprovider "k8s.io/cloud-provider"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/configmap"
"k8s.io/kubernetes/pkg/kubelet/container"
@ -40,6 +39,7 @@ import (
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
// NewInitializedVolumePluginMgr returns a new instance of
@ -80,6 +80,7 @@ func NewInitializedVolumePluginMgr(
// Compile-time check to ensure kubeletVolumeHost implements the VolumeHost interface
var _ volume.VolumeHost = &kubeletVolumeHost{}
var _ volume.KubeletVolumeHost = &kubeletVolumeHost{}
func (kvh *kubeletVolumeHost) GetPluginDir(pluginName string) string {
return kvh.kubelet.getPluginDir(pluginName)
@ -94,6 +95,10 @@ type kubeletVolumeHost struct {
mountPodManager mountpod.Manager
}
func (kvh *kubeletVolumeHost) SetKubeletError(err error) {
kvh.kubelet.runtimeState.setStorageState(err)
}
func (kvh *kubeletVolumeHost) GetVolumeDevicePluginDir(pluginName string) string {
return kvh.kubelet.getVolumeDevicePluginDir(pluginName)
}
@ -122,8 +127,8 @@ func (kvh *kubeletVolumeHost) GetKubeClient() clientset.Interface {
return kvh.kubelet.kubeClient
}
func (kvh *kubeletVolumeHost) GetCSIClient() csiclientset.Interface {
return kvh.kubelet.csiClient
func (kvh *kubeletVolumeHost) GetSubpather() subpath.Interface {
return kvh.kubelet.subpather
}
func (kvh *kubeletVolumeHost) NewWrapperMounter(

7
vendor/k8s.io/kubernetes/pkg/util/BUILD generated vendored
View file

@ -20,7 +20,6 @@ filegroup(
"//pkg/util/dbus:all-srcs",
"//pkg/util/ebtables:all-srcs",
"//pkg/util/env:all-srcs",
"//pkg/util/file:all-srcs",
"//pkg/util/filesystem:all-srcs",
"//pkg/util/flag:all-srcs",
"//pkg/util/flock:all-srcs",
@ -28,25 +27,22 @@ filegroup(
"//pkg/util/hash:all-srcs",
"//pkg/util/initsystem:all-srcs",
"//pkg/util/interrupt:all-srcs",
"//pkg/util/io:all-srcs",
"//pkg/util/ipconfig:all-srcs",
"//pkg/util/ipset:all-srcs",
"//pkg/util/iptables:all-srcs",
"//pkg/util/ipvs:all-srcs",
"//pkg/util/keymutex:all-srcs",
"//pkg/util/labels:all-srcs",
"//pkg/util/maps:all-srcs",
"//pkg/util/metrics:all-srcs",
"//pkg/util/mount:all-srcs",
"//pkg/util/net:all-srcs",
"//pkg/util/netsh:all-srcs",
"//pkg/util/node:all-srcs",
"//pkg/util/normalizer:all-srcs",
"//pkg/util/nsenter:all-srcs",
"//pkg/util/oom:all-srcs",
"//pkg/util/parsers:all-srcs",
"//pkg/util/pod:all-srcs",
"//pkg/util/procfs:all-srcs",
"//pkg/util/prometheusclientgo:all-srcs",
"//pkg/util/reflector/prometheus:all-srcs",
"//pkg/util/removeall:all-srcs",
"//pkg/util/resizefs:all-srcs",
@ -54,7 +50,6 @@ filegroup(
"//pkg/util/rlimit:all-srcs",
"//pkg/util/selinux:all-srcs",
"//pkg/util/slice:all-srcs",
"//pkg/util/strings:all-srcs",
"//pkg/util/sysctl:all-srcs",
"//pkg/util/system:all-srcs",
"//pkg/util/tail:all-srcs",

36
vendor/k8s.io/kubernetes/pkg/util/file/BUILD generated vendored
View file

@ -1,36 +0,0 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = ["file.go"],
importpath = "k8s.io/kubernetes/pkg/util/file",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
go_test(
name = "go_default_test",
srcs = ["file_test.go"],
embed = [":go_default_library"],
deps = [
"//vendor/github.com/spf13/afero:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
)

57
vendor/k8s.io/kubernetes/pkg/util/file/file.go generated vendored
View file

@ -1,57 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package file
import (
"os"
)
// FileExists checks if specified file exists.
func FileExists(filename string) (bool, error) {
if _, err := os.Stat(filename); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, err
}
return true, nil
}
// FileOrSymlinkExists checks if specified file or symlink exists.
func FileOrSymlinkExists(filename string) (bool, error) {
if _, err := os.Lstat(filename); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, err
}
return true, nil
}
// ReadDirNoStat returns a string of files/directories contained
// in dirname without calling lstat on them.
func ReadDirNoStat(dirname string) ([]string, error) {
if dirname == "" {
dirname = "."
}
f, err := os.Open(dirname)
if err != nil {
return nil, err
}
defer f.Close()
return f.Readdirnames(-1)
}

25
vendor/k8s.io/kubernetes/pkg/util/io/BUILD generated vendored
View file

@ -1,25 +0,0 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["consistentread.go"],
importpath = "k8s.io/kubernetes/pkg/util/io",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

45
vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go generated vendored
View file

@ -1,45 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package io
import (
"bytes"
"fmt"
"io/ioutil"
)
// ConsistentRead repeatedly reads a file until it gets the same content twice.
// This is useful when reading files in /proc that are larger than page size
// and kernel may modify them between individual read() syscalls.
func ConsistentRead(filename string, attempts int) ([]byte, error) {
oldContent, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
for i := 0; i < attempts; i++ {
newContent, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
if bytes.Compare(oldContent, newContent) == 0 {
return newContent, nil
}
// Files are different, continue reading
oldContent = newContent
}
return nil, fmt.Errorf("could not get consistent content of %s after %d attempts", filename, attempts)
}

37
vendor/k8s.io/kubernetes/pkg/util/mount/BUILD generated vendored
View file

@ -9,7 +9,9 @@ go_library(
"exec_mount_unsupported.go",
"fake.go",
"mount.go",
"mount_helper.go",
"mount_helper_common.go",
"mount_helper_unix.go",
"mount_helper_windows.go",
"mount_linux.go",
"mount_unsupported.go",
"mount_windows.go",
@ -23,42 +25,43 @@ go_library(
"//vendor/k8s.io/utils/exec:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/file:go_default_library",
"//pkg/util/io:go_default_library",
"//pkg/util/nsenter:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/utils/io:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
"//vendor/k8s.io/utils/path:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//pkg/util/file:go_default_library",
"//pkg/util/nsenter:go_default_library",
"//vendor/k8s.io/utils/keymutex:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
"//vendor/k8s.io/utils/path:go_default_library",
],
"//conditions:default": [],
}),
@ -80,10 +83,8 @@ go_test(
"//vendor/k8s.io/utils/exec/testing:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/nsenter:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/github.com/stretchr/testify/assert:go_default_library",

2
vendor/k8s.io/kubernetes/pkg/util/mount/OWNERS generated vendored
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- jingxu97
- saad-ali

12
vendor/k8s.io/kubernetes/pkg/util/mount/exec_mount.go generated vendored
View file

@ -144,18 +144,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
return m.wrappedMounter.EvalHostSymlinks(pathname)
}
func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return m.wrappedMounter.PrepareSafeSubpath(subPath)
}
func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return m.wrappedMounter.CleanSubPaths(podDir, volumeName)
}
func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
}
func (m *execMounter) GetMountRefs(pathname string) ([]string, error) {
return m.wrappedMounter.GetMountRefs(pathname)
}

14
vendor/k8s.io/kubernetes/pkg/util/mount/exec_mount_unsupported.go generated vendored
View file

@ -48,7 +48,7 @@ func (mounter *execMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
}
func (mounter *execMounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
return isNotMountPoint(mounter, dir)
}
func (mounter *execMounter) IsLikelyNotMountPoint(file string) (bool, error) {
@ -91,18 +91,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

13
vendor/k8s.io/kubernetes/pkg/util/mount/fake.go generated vendored
View file

@ -137,7 +137,7 @@ func (f *FakeMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
}
func (f *FakeMounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(f, dir)
return isNotMountPoint(f, dir)
}
func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
@ -220,17 +220,6 @@ func (f *FakeMounter) EvalHostSymlinks(pathname string) (string, error) {
return pathname, nil
}
func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) {
realpath, err := filepath.EvalSymlinks(pathname)
if err != nil {

40
vendor/k8s.io/kubernetes/pkg/util/mount/mount.go generated vendored
View file

@ -84,35 +84,12 @@ type Interface interface {
// MakeDir creates a new directory.
// Will operate in the host mount namespace if kubelet is running in a container
MakeDir(pathname string) error
// SafeMakeDir creates subdir within given base. It makes sure that the
// created directory does not escape given base directory mis-using
// symlinks. Note that the function makes sure that it creates the directory
// somewhere under the base, nothing else. E.g. if the directory already
// exists, it may exist outside of the base due to symlinks.
// This method should be used if the directory to create is inside volume
// that's under user control. User must not be able to use symlinks to
// escape the volume to create directories somewhere else.
SafeMakeDir(subdir string, base string, perm os.FileMode) error
// Will operate in the host mount namespace if kubelet is running in a container.
// Error is returned on any other error than "file not found".
ExistsPath(pathname string) (bool, error)
// EvalHostSymlinks returns the path name after evaluating symlinks.
// Will operate in the host mount namespace if kubelet is running in a container.
EvalHostSymlinks(pathname string) (string, error)
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory.
CleanSubPaths(podDir string, volumeName string) error
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
// that's 1) inside given volumePath and 2) immutable after this call.
//
// newHostPath - location of prepared subPath. It should be used instead of
// hostName when running the container.
// cleanupAction - action to run when the container is running or it failed to start.
//
// CleanupAction must be called immediately after the container with given
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
// when the pod finishes.
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
// GetMountRefs finds all mount references to the path, returns a
// list of paths. Path could be a mountpoint path, device or a normal
// directory (for bind mount).
@ -143,7 +120,7 @@ type Subpath struct {
// Exec executes command where mount utilities are. This can be either the host,
// container where kubelet runs or even a remote pod with mount utilities.
// Usual pkg/util/exec interface is not used because kubelet.RunInContainer does
// Usual k8s.io/utils/exec interface is not used because kubelet.RunInContainer does
// not provide stdin/stdout/stderr streams.
type Exec interface {
// Run executes a command and returns its stdout + stderr combined in one
@ -245,12 +222,9 @@ func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, e
return device, refCount, nil
}
// IsNotMountPoint determines if a directory is a mountpoint.
// It should return ErrNotExist when the directory does not exist.
// This method uses the List() of all mountpoints
// It is more extensive than IsLikelyNotMountPoint
// and it detects bind mounts in linux
func IsNotMountPoint(mounter Interface, file string) (bool, error) {
// isNotMountPoint implements Mounter.IsNotMountPoint and is shared by mounter
// implementations.
func isNotMountPoint(mounter Interface, file string) (bool, error) {
// IsLikelyNotMountPoint provides a quick check
// to determine whether file IS A mountpoint
notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file)
@ -358,15 +332,15 @@ func PathWithinBase(fullPath, basePath string) bool {
if err != nil {
return false
}
if startsWithBackstep(rel) {
if StartsWithBackstep(rel) {
// Needed to escape the base path
return false
}
return true
}
// startsWithBackstep checks if the given path starts with a backstep segment
func startsWithBackstep(rel string) bool {
// StartsWithBackstep checks if the given path starts with a backstep segment
func StartsWithBackstep(rel string) bool {
// normalize to / and check for ../
return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
}

23
vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper.go → vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper_common.go generated vendored
View file

@ -19,7 +19,6 @@ package mount
import (
"fmt"
"os"
"syscall"
"k8s.io/klog"
)
@ -56,7 +55,7 @@ func doCleanupMountPoint(mountPath string, mounter Interface, extensiveMountPoin
var notMnt bool
var err error
if extensiveMountPointCheck {
notMnt, err = IsNotMountPoint(mounter, mountPath)
notMnt, err = mounter.IsNotMountPoint(mountPath)
} else {
notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
}
@ -102,23 +101,3 @@ func PathExists(path string) (bool, error) {
return false, err
}
}
// IsCorruptedMnt return true if err is about corrupted mount point
func IsCorruptedMnt(err error) bool {
if err == nil {
return false
}
var underlyingError error
switch pe := err.(type) {
case nil:
return false
case *os.PathError:
underlyingError = pe.Err
case *os.LinkError:
underlyingError = pe.Err
case *os.SyscallError:
underlyingError = pe.Err
}
return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO
}

44
vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper_unix.go generated vendored Normal file
View file

@ -0,0 +1,44 @@
// +build !windows
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
import (
"os"
"syscall"
)
// IsCorruptedMnt return true if err is about corrupted mount point
func IsCorruptedMnt(err error) bool {
if err == nil {
return false
}
var underlyingError error
switch pe := err.(type) {
case nil:
return false
case *os.PathError:
underlyingError = pe.Err
case *os.LinkError:
underlyingError = pe.Err
case *os.SyscallError:
underlyingError = pe.Err
}
return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES
}

68
vendor/k8s.io/kubernetes/pkg/util/mount/mount_helper_windows.go generated vendored Normal file
View file

@ -0,0 +1,68 @@
// +build windows
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
import (
"os"
"syscall"
"k8s.io/klog"
)
// following failure codes are from https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1300-1699-
// ERROR_BAD_NETPATH = 53
// ERROR_NETWORK_BUSY = 54
// ERROR_UNEXP_NET_ERR = 59
// ERROR_NETNAME_DELETED = 64
// ERROR_NETWORK_ACCESS_DENIED = 65
// ERROR_BAD_DEV_TYPE = 66
// ERROR_BAD_NET_NAME = 67
// ERROR_SESSION_CREDENTIAL_CONFLICT = 1219
// ERROR_LOGON_FAILURE = 1326
var errorNoList = [...]int{53, 54, 59, 64, 65, 66, 67, 1219, 1326}
// IsCorruptedMnt return true if err is about corrupted mount point
func IsCorruptedMnt(err error) bool {
if err == nil {
return false
}
var underlyingError error
switch pe := err.(type) {
case nil:
return false
case *os.PathError:
underlyingError = pe.Err
case *os.LinkError:
underlyingError = pe.Err
case *os.SyscallError:
underlyingError = pe.Err
}
if ee, ok := underlyingError.(syscall.Errno); ok {
for _, errno := range errorNoList {
if int(ee) == errno {
klog.Warningf("IsCorruptedMnt failed with error: %v, error code: %v", err, errno)
return true
}
}
}
return false
}

534
vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux.go generated vendored
View file

@ -21,7 +21,6 @@ package mount
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
@ -33,9 +32,9 @@ import (
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
utilfile "k8s.io/kubernetes/pkg/util/file"
utilio "k8s.io/kubernetes/pkg/util/io"
utilexec "k8s.io/utils/exec"
utilio "k8s.io/utils/io"
utilpath "k8s.io/utils/path"
)
const (
@ -53,14 +52,6 @@ const (
fsckErrorsCorrected = 1
// 'fsck' found errors but exited without correcting them
fsckErrorsUncorrected = 4
// place for subpath mounts
// TODO: pass in directory using kubelet_getters instead
containerSubPathDirectoryName = "volume-subpaths"
// syscall.Openat flags used to traverse directories not following symlinks
nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
// flags for getting file descriptor without following the symlink
openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
)
// Mounter provides the default implementation of mount.Interface
@ -229,7 +220,7 @@ func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
}
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
return isNotMountPoint(mounter, dir)
}
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
@ -417,7 +408,7 @@ func (mounter *Mounter) MakeFile(pathname string) error {
}
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
return utilfile.FileExists(pathname)
return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}
func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
@ -726,287 +717,15 @@ func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) {
return false, nil
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
// This implementation is shared between Linux and NsEnterMounter
func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
if !PathWithinBase(subpath.Path, subpath.VolumePath) {
return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
}
fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
if err != nil {
return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
}
return fd, nil
}
// prepareSubpathTarget creates target for bind-mount of subpath. It returns
// "true" when the target already exists and something is mounted there.
// Given Subpath must have all paths with already resolved symlinks and with
// paths relevant to kubelet (when it runs in a container).
// This function is called also by NsEnterMounter. It works because
// /var/lib/kubelet is mounted from the host into the container with Kubelet as
// /var/lib/kubelet too.
func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) {
// Early check for already bind-mounted subpath.
bindPathTarget := getSubpathBindTarget(subpath)
notMount, err := IsNotMountPoint(mounter, bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
return true, bindPathTarget, nil
}
// bindPathTarget is in /var/lib/kubelet and thus reachable without any
// translation even to containerized kubelet.
bindParent := filepath.Dir(bindPathTarget)
err = os.MkdirAll(bindParent, 0750)
if err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
}
t, err := os.Lstat(subpath.Path)
if err != nil {
return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindPathTarget>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
return false, bindPathTarget, nil
}
func getSubpathBindTarget(subpath Subpath) string {
// containerName is DNS label, i.e. safe as a directory name.
return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
}
func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs on the host:
// - safely open the subpath
// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
// Evaluate all symlinks here once for all subsequent functions.
newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
newPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
subpath.VolumePath = newVolumePath
subpath.Path = newPath
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
kubeletPid := os.Getpid()
mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
// Do the bind mount
options := []string{"bind"}
klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
// This implementation is shared between Linux and NsEnterMounter
func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error {
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
containerDirs, err := ioutil.ReadDir(subPathDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("error reading %s: %s", subPathDir, err)
}
for _, containerDir := range containerDirs {
if !containerDir.IsDir() {
klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
continue
}
klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
if path == fullContainerDirPath {
// Skip top level directory
return nil
}
// pass through errors and let doCleanSubPath handle them
if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
return err
}
return nil
})
if err != nil {
return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
}
// Whole container has been processed, remove its directory.
if err := os.Remove(fullContainerDirPath); err != nil {
return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
}
klog.V(5).Infof("Removed %s", fullContainerDirPath)
}
// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
if err := os.Remove(subPathDir); err != nil {
return fmt.Errorf("error deleting %s: %s", subPathDir, err)
}
klog.V(5).Infof("Removed %s", subPathDir)
// Remove entire subpath directory if it's the last one
podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
}
klog.V(5).Infof("Removed %s", podSubPathDir)
return nil
}
// doCleanSubPath tears down the single subpath bind mount
func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error {
// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil {
return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
}
klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
return nil
}
// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
func cleanSubPath(mounter Interface, subpath Subpath) error {
containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
// Clean subdir bindmount
if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
return err
}
// Recusively remove directories if empty
if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
return err
}
return nil
}
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error {
if !PathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
}
for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
s, err := os.Stat(curDir)
if err != nil {
if os.IsNotExist(err) {
klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
continue
}
return fmt.Errorf("error stat %q: %v", curDir, err)
}
if !s.IsDir() {
return fmt.Errorf("path %q not a directory", curDir)
}
err = os.Remove(curDir)
if os.IsExist(err) {
klog.V(5).Infof("Directory %q not empty, not removing", curDir)
break
} else if err != nil {
return fmt.Errorf("error removing directory %q: %v", curDir, err)
}
klog.V(5).Infof("Removed directory %q", curDir)
}
return nil
}
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
if _, err := os.Stat(pathname); os.IsNotExist(err) {
pathExists, pathErr := PathExists(pathname)
if !pathExists {
return []string{}, nil
} else if err != nil {
return nil, err
} else if IsCorruptedMnt(pathErr) {
klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", pathname)
return []string{}, nil
} else if pathErr != nil {
return nil, fmt.Errorf("error checking path %s: %v", pathname, pathErr)
}
realpath, err := filepath.EvalSymlinks(pathname)
if err != nil {
@ -1049,237 +768,6 @@ func getMode(pathname string) (os.FileMode, error) {
return info.Mode(), nil
}
// This implementation is shared between Linux and NsEnterMounter. Both pathname
// and base must be either already resolved symlinks or thet will be resolved in
// kubelet's mount namespace (in case it runs containerized).
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err)
}
if !PathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
parentFD, err := doSafeOpen(fullExistingPath, base)
if err != nil {
return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
}
childFD := -1
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
}
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink.
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
if err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
// Dive into the created directory
childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
if err != nil {
return fmt.Errorf("cannot open %s: %s", currentPath, err)
}
// We can be sure that childFD is safe to use. It could be changed
// by user after Mkdirat() and before Openat(), however:
// - it could not be changed to symlink - we use nofollowFlags
// - it could be changed to a file (or device, pipe, socket, ...)
// but either subsequent Mkdirat() fails or we mount this file
// to user's container. Security is no violated in both cases
// and user either gets error or the file that it can already access.
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
parentFD = childFD
childFD = -1
}
// Everything was created. mkdirat(..., perm) above was affected by current
// umask and we must apply the right permissions to the last directory
// (that's the one that will be available to the container as subpath)
// so user can read/write it. This is the behavior of previous code.
// TODO: chmod all created directories, not just the last one.
// parentFD is the last created directory.
// Translate perm (os.FileMode) to uint32 that fchmod() expects
kernelPerm := uint32(perm & os.ModePerm)
if perm&os.ModeSetgid > 0 {
kernelPerm |= syscall.S_ISGID
}
if perm&os.ModeSetuid > 0 {
kernelPerm |= syscall.S_ISUID
}
if perm&os.ModeSticky > 0 {
kernelPerm |= syscall.S_ISVTX
}
if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
return fmt.Errorf("chmod %q failed: %s", currentPath, err)
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
dirs := strings.Split(rel, string(filepath.Separator))
// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
// This should be faster than looping through all dirs and calling os.Stat()
// on each of them, as the symlinks are resolved only once with OpenAt().
currentPath := base
fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
if err != nil {
return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
}
defer func() {
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
}()
for i, dir := range dirs {
// Using O_PATH here will prevent hangs in case user replaces directory with
// fifo
childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
if err != nil {
if os.IsNotExist(err) {
return currentPath, dirs[i:], nil
}
return base, nil, err
}
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
fd = childFD
currentPath = filepath.Join(currentPath, dir)
}
return pathname, []string{}, nil
}
// This implementation is shared between Linux and NsEnterMounter
// Open path and return its fd.
// Symlinks are disallowed (pathname must already resolve symlinks),
// and the path must be within the base directory.
func doSafeOpen(pathname string, base string) (int, error) {
pathname = filepath.Clean(pathname)
base = filepath.Clean(base)
// Calculate segments to follow
subpath, err := filepath.Rel(base, pathname)
if err != nil {
return -1, err
}
segments := strings.Split(subpath, string(filepath.Separator))
// Assumption: base is the only directory that we have under control.
// Base dir is not allowed to be a symlink.
parentFD, err := syscall.Open(base, nofollowFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
}
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
}
}
}()
childFD := -1
defer func() {
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := base
// Follow the segments one by one using openat() to make
// sure the user cannot change already existing directories into symlinks.
for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg)
if !PathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
}
klog.V(5).Infof("Opening path %s", currentPath)
childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
}
var deviceStat unix.Stat_t
err := unix.Fstat(childFD, &deviceStat)
if err != nil {
return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
}
fileFmt := deviceStat.Mode & syscall.S_IFMT
if fileFmt == syscall.S_IFLNK {
return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
}
// Close parentFD
if err = syscall.Close(parentFD); err != nil {
return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
}
// Set child to new parent
parentFD = childFD
childFD = -1
}
// We made it to the end, return this fd, don't close it
finalFD := parentFD
parentFD = -1
return finalFD, nil
}
// searchMountPoints finds all mount references to the source, returns a list of
// mountpoints.
// This function assumes source cannot be device.

14
vendor/k8s.io/kubernetes/pkg/util/mount/mount_unsupported.go generated vendored
View file

@ -55,7 +55,7 @@ func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
}
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
return isNotMountPoint(mounter, dir)
}
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
@ -110,18 +110,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return "", unsupportedErr
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, unsupportedErr
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return unsupportedErr
}
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return unsupportedErr
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

342
vendor/k8s.io/kubernetes/pkg/util/mount/mount_windows.go generated vendored
View file

@ -26,11 +26,11 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"k8s.io/klog"
"k8s.io/utils/keymutex"
utilfile "k8s.io/kubernetes/pkg/util/file"
utilpath "k8s.io/utils/path"
)
// Mounter provides the default implementation of mount.Interface
@ -49,12 +49,16 @@ func New(mounterPath string) Interface {
}
}
// Mount : mounts source to target as NTFS with given options.
// acquire lock for smb mount
var getSMBMountMutex = keymutex.NewHashed(0)
// Mount : mounts source to target with given options.
// currently only supports cifs(smb), bind mount(for disk)
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
target = normalizeWindowsPath(target)
if source == "tmpfs" {
klog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options)
klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, options)
return os.MkdirAll(target, 0755)
}
@ -63,9 +67,9 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
return err
}
klog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
options, source, target, fstype)
bindSource := ""
bindSource := source
// tell it's going to mount azure disk or azure file according to options
if bind, _, _ := isBind(options); bind {
@ -73,31 +77,32 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
bindSource = normalizeWindowsPath(source)
} else {
if len(options) < 2 {
klog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
options, len(options), source, target)
return nil
}
// currently only cifs mount is supported
if strings.ToLower(fstype) != "cifs" {
return fmt.Errorf("azureMount: only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options)
return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options)
}
bindSource = source
// lock smb mount for the same source
getSMBMountMutex.LockKey(source)
defer getSMBMountMutex.UnlockKey(source)
// use PowerShell Environment Variables to store user input string to prevent command line injection
// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
cmdLine := fmt.Sprintf(`$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`)
cmd := exec.Command("powershell", "/c", cmdLine)
cmd.Env = append(os.Environ(),
fmt.Sprintf("smbuser=%s", options[0]),
fmt.Sprintf("smbpassword=%s", options[1]),
fmt.Sprintf("smbremotepath=%s", source))
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("azureMount: SmbGlobalMapping failed: %v, only SMB mount is supported now, output: %q", err, string(output))
if output, err := newSMBMapping(options[0], options[1], source); err != nil {
if isSMBMappingExist(source) {
klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source)
if output, err := removeSMBMapping(source); err != nil {
return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output)
}
if output, err := newSMBMapping(options[0], options[1], source); err != nil {
return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output)
}
} else {
return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output)
}
}
}
@ -109,6 +114,44 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
return nil
}
// do the SMB mount with username, password, remotepath
// return (output, error)
func newSMBMapping(username, password, remotepath string) (string, error) {
if username == "" || password == "" || remotepath == "" {
return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, password, remotepath)
}
// use PowerShell Environment Variables to store user input string to prevent command line injection
// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`
cmd := exec.Command("powershell", "/c", cmdLine)
cmd.Env = append(os.Environ(),
fmt.Sprintf("smbuser=%s", username),
fmt.Sprintf("smbpassword=%s", password),
fmt.Sprintf("smbremotepath=%s", remotepath))
output, err := cmd.CombinedOutput()
return string(output), err
}
// check whether remotepath is already mounted
func isSMBMappingExist(remotepath string) bool {
cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`)
cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
_, err := cmd.CombinedOutput()
return err == nil
}
// remove SMB mapping
func removeSMBMapping(remotepath string) (string, error) {
cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`)
cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
output, err := cmd.CombinedOutput()
return string(output), err
}
// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
klog.V(4).Infof("azureMount: Unmount target (%q)", target)
@ -132,7 +175,7 @@ func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
// IsNotMountPoint determines if a directory is a mountpoint.
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
return isNotMountPoint(mounter, dir)
}
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
@ -235,7 +278,7 @@ func (mounter *Mounter) MakeFile(pathname string) error {
// ExistsPath checks whether the path exists
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
return utilfile.FileExists(pathname)
return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}
// EvalHostSymlinks returns the path name after evaluating symlinks
@ -243,123 +286,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return filepath.EvalSymlinks(pathname)
}
// check whether hostPath is within volume path
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(hostPath) == 0 {
return []uintptr{}, nil
}
finalSubPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
}
finalVolumePath, err := filepath.EvalSymlinks(volumePath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
}
return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
}
// lock all intermediate subPath directories and check they are all within volumePath
// volumePath & subPath should not contain any symlink, otherwise it will return error
func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(subPath) == 0 {
return []uintptr{}, nil
}
// get relative path to volumePath
relSubPath, err := filepath.Rel(volumePath, subPath)
if err != nil {
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
}
if startsWithBackstep(relSubPath) {
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
}
if relSubPath == "." {
// volumePath and subPath are equal
return []uintptr{}, nil
}
fileHandles := []uintptr{}
var errorResult error
currentFullPath := volumePath
dirs := strings.Split(relSubPath, string(os.PathSeparator))
for _, dir := range dirs {
// lock intermediate subPath directory first
currentFullPath = filepath.Join(currentFullPath, dir)
handle, err := lockPath(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
break
}
fileHandles = append(fileHandles, handle)
// make sure intermediate subPath directory does not contain symlink any more
stat, err := os.Lstat(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
break
}
if stat.Mode()&os.ModeSymlink != 0 {
errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
break
}
if !PathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break
}
}
return fileHandles, errorResult
}
// unlockPath unlock directories
func unlockPath(fileHandles []uintptr) {
if fileHandles != nil {
for _, handle := range fileHandles {
syscall.CloseHandle(syscall.Handle(handle))
}
}
}
// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
func lockPath(path string) (uintptr, error) {
if len(path) == 0 {
return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return uintptr(syscall.InvalidHandle), err
}
access := uint32(syscall.GENERIC_READ)
sharemode := uint32(syscall.FILE_SHARE_READ)
createmode := uint32(syscall.OPEN_EXISTING)
flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
return uintptr(fd), err
}
// Lock all directories in subPath and check they're not symlinks.
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
// Unlock the directories when the container starts
cleanupAction = func() {
unlockPath(handles)
}
return subPath.Path, cleanupAction, err
}
// No bind-mounts for subpaths are necessary on Windows
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
// Try to mount the disk
klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
@ -460,10 +386,15 @@ func getAllParentLinks(path string) ([]string, error) {
// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
if _, err := os.Stat(normalizeWindowsPath(pathname)); os.IsNotExist(err) {
windowsPath := normalizeWindowsPath(pathname)
pathExists, pathErr := PathExists(windowsPath)
if !pathExists {
return []string{}, nil
} else if err != nil {
return nil, err
} else if IsCorruptedMnt(pathErr) {
klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath)
return []string{}, nil
} else if pathErr != nil {
return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr)
}
return []string{pathname}, nil
}
@ -486,126 +417,3 @@ func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
}
return info.Mode(), nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
if len(toCreate) == 0 {
return nil
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
}
fullBasePath, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err)
}
if !PathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
defer unlockPath(fileHandles)
if err != nil {
return err
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink by lock that directory immediately
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
if err := os.Mkdir(currentPath, perm); err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
handle, err := lockPath(currentPath)
if err != nil {
return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
}
defer syscall.CloseHandle(syscall.Handle(handle))
// make sure newly created directory does not contain symlink after lock
stat, err := os.Lstat(currentPath)
if err != nil {
return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
}
if stat.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
}
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
if startsWithBackstep(rel) {
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
}
if rel == "." {
// base and pathname are equal
return pathname, []string{}, nil
}
dirs := strings.Split(rel, string(filepath.Separator))
parent := base
currentPath := base
for i, dir := range dirs {
parent = currentPath
currentPath = filepath.Join(parent, dir)
if _, err := os.Lstat(currentPath); err != nil {
if os.IsNotExist(err) {
return parent, dirs[i:], nil
}
return base, nil, err
}
}
return pathname, []string{}, nil
}

153
vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount.go generated vendored
View file

@ -23,12 +23,10 @@ import (
"os"
"path/filepath"
"strings"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
utilfile "k8s.io/kubernetes/pkg/util/file"
"k8s.io/kubernetes/pkg/util/nsenter"
"k8s.io/utils/nsenter"
utilpath "k8s.io/utils/path"
)
const (
@ -145,7 +143,7 @@ func (*NsenterMounter) List() ([]MountPoint, error) {
}
func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(m, dir)
return isNotMountPoint(m, dir)
}
func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
@ -291,65 +289,19 @@ func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) {
return false, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return utilfile.FileExists(kubeletpath)
return utilpath.Exists(utilpath.CheckFollowSymlink, kubeletpath)
}
func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return mounter.ne.EvalSymlinks(pathname, true)
}
func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(mounter.rootDir)
if PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
// create subpaths for paths in /var/lib/kubelet.
return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
}
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
}
func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
exists, err := mounter.ExistsPath(pathname)
if err != nil {
return nil, err
}
if !exists {
pathExists, pathErr := PathExists(pathname)
if !pathExists || IsCorruptedMnt(pathErr) {
return []string{}, nil
} else if pathErr != nil {
return nil, fmt.Errorf("Error checking path %s: %v", pathname, pathErr)
}
hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
@ -358,95 +310,6 @@ func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return searchMountPoints(hostpath, hostProcMountinfoPath)
}
func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
// - check that we mounted the right thing by comparing device ID and inode
// of the subpath (via safely opened fd) and the target (that's under our
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Leap of faith: optimistically expect that nobody has modified previously
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
// Check that the bind-mount target is the same inode and device as the
// source that we keept open, i.e. we mounted the right thing.
err = checkDeviceInode(fd, bindPathTarget)
if err != nil {
return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// checkDeviceInode checks that opened file and path represent the same file.
func checkDeviceInode(fd int, path string) error {
var srcStat, dstStat unix.Stat_t
err := unix.Fstat(fd, &srcStat)
if err != nil {
return fmt.Errorf("error running fstat on subpath FD: %v", err)
}
err = unix.Stat(path, &dstStat)
if err != nil {
return fmt.Errorf("error running fstat on %s: %v", path, err)
}
if srcStat.Dev != dstStat.Dev {
return fmt.Errorf("different device number")
}
if srcStat.Ino != dstStat.Ino {
return fmt.Errorf("different inode")
}
return nil
}
func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {

16
vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount_unsupported.go generated vendored
View file

@ -22,7 +22,7 @@ import (
"errors"
"os"
"k8s.io/kubernetes/pkg/util/nsenter"
"k8s.io/utils/nsenter"
)
type NsenterMounter struct{}
@ -46,7 +46,7 @@ func (*NsenterMounter) List() ([]MountPoint, error) {
}
func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(m, dir)
return isNotMountPoint(m, dir)
}
func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
@ -93,18 +93,6 @@ func (*NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

76
vendor/k8s.io/kubernetes/pkg/util/nsenter/BUILD generated vendored
View file

@ -1,76 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"exec.go",
"exec_unsupported.go",
"nsenter.go",
"nsenter_unsupported.go",
],
importpath = "k8s.io/kubernetes/pkg/util/nsenter",
visibility = ["//visibility:public"],
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = ["nsenter_test.go"],
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"//conditions:default": [],
}),
)

8
vendor/k8s.io/kubernetes/pkg/util/nsenter/OWNERS generated vendored
View file

@ -1,8 +0,0 @@
reviewers:
- jsafrane
- msau42
- cofyc
approvers:
- jsafrane
- msau42
- cofyc

67
vendor/k8s.io/kubernetes/pkg/util/nsenter/exec.go generated vendored
View file

@ -1,67 +0,0 @@
// +build linux
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"fmt"
"path/filepath"
"k8s.io/klog"
"k8s.io/utils/exec"
)
// Executor wraps executor interface to be executed via nsenter
type Executor struct {
// Exec implementation
executor exec.Interface
// Path to the host's root proc path
hostProcMountNsPath string
}
// NewNsenterExecutor returns new nsenter based executor
func NewNsenterExecutor(hostRootFsPath string, executor exec.Interface) *Executor {
hostProcMountNsPath := filepath.Join(hostRootFsPath, mountNsPath)
nsExecutor := &Executor{
hostProcMountNsPath: hostProcMountNsPath,
executor: executor,
}
return nsExecutor
}
// Command returns a command wrapped with nenter
func (nsExecutor *Executor) Command(cmd string, args ...string) exec.Cmd {
fullArgs := append([]string{fmt.Sprintf("--mount=%s", nsExecutor.hostProcMountNsPath), "--"},
append([]string{cmd}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return nsExecutor.executor.Command(nsenterPath, fullArgs...)
}
// CommandContext returns a CommandContext wrapped with nsenter
func (nsExecutor *Executor) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
fullArgs := append([]string{fmt.Sprintf("--mount=%s", nsExecutor.hostProcMountNsPath), "--"},
append([]string{cmd}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return nsExecutor.executor.CommandContext(ctx, nsenterPath, fullArgs...)
}
// LookPath returns a LookPath wrapped with nsenter
func (nsExecutor *Executor) LookPath(file string) (string, error) {
return "", fmt.Errorf("not implemented, error looking up : %s", file)
}

58
vendor/k8s.io/kubernetes/pkg/util/nsenter/exec_unsupported.go generated vendored
View file

@ -1,58 +0,0 @@
// +build !linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"fmt"
"k8s.io/utils/exec"
)
// Executor wraps executor interface to be executed via nsenter
type Executor struct {
// Exec implementation
executor exec.Interface
// Path to the host's root proc path
hostProcMountNsPath string
}
// NewNsenterExecutor returns new nsenter based executor
func NewNsenterExecutor(hostRootFsPath string, executor exec.Interface) *Executor {
nsExecutor := &Executor{
hostProcMountNsPath: hostRootFsPath,
executor: executor,
}
return nsExecutor
}
// Command returns a command wrapped with nenter
func (nsExecutor *Executor) Command(cmd string, args ...string) exec.Cmd {
return nil
}
// CommandContext returns a CommandContext wrapped with nsenter
func (nsExecutor *Executor) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return nil
}
// LookPath returns a LookPath wrapped with nsenter
func (nsExecutor *Executor) LookPath(file string) (string, error) {
return "", fmt.Errorf("not implemented, error looking up : %s", file)
}

236
vendor/k8s.io/kubernetes/pkg/util/nsenter/nsenter.go generated vendored
View file

@ -1,236 +0,0 @@
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"k8s.io/utils/exec"
"k8s.io/klog"
)
const (
// DefaultHostRootFsPath is path to host's filesystem mounted into container
// with kubelet.
DefaultHostRootFsPath = "/rootfs"
// mountNsPath is the default mount namespace of the host
mountNsPath = "/proc/1/ns/mnt"
// nsenterPath is the default nsenter command
nsenterPath = "nsenter"
)
// Nsenter is part of experimental support for running the kubelet
// in a container.
//
// Nsenter requires:
//
// 1. Docker >= 1.6 due to the dependency on the slave propagation mode
// of the bind-mount of the kubelet root directory in the container.
// Docker 1.5 used a private propagation mode for bind-mounts, so mounts
// performed in the host's mount namespace do not propagate out to the
// bind-mount in this docker version.
// 2. The host's root filesystem must be available at /rootfs
// 3. The nsenter binary must be on the Kubelet process' PATH in the container's
// filesystem.
// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at
// the present, this effectively means that the kubelet is running in a
// privileged container.
// 5. The volume path used by the Kubelet must be the same inside and outside
// the container and be writable by the container (to initialize volume)
// contents. TODO: remove this requirement.
// 6. The host image must have "mount", "findmnt", "umount", "stat", "touch",
// "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin
// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin if
// systemd is installed/enabled in the operating system.
// For more information about mount propagation modes, see:
// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt
type Nsenter struct {
// a map of commands to their paths on the host filesystem
paths map[string]string
// Path to the host filesystem, typically "/rootfs". Used only for testing.
hostRootFsPath string
// Exec implementation, used only for testing
executor exec.Interface
}
// NewNsenter constructs a new instance of Nsenter
func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
ne := &Nsenter{
hostRootFsPath: hostRootFsPath,
executor: executor,
}
if err := ne.initPaths(); err != nil {
return nil, err
}
return ne, nil
}
func (ne *Nsenter) initPaths() error {
ne.paths = map[string]string{}
binaries := []string{
"mount",
"findmnt",
"umount",
"systemd-run",
"stat",
"touch",
"mkdir",
"sh",
"chmod",
"realpath",
}
// search for the required commands in other locations besides /usr/bin
for _, binary := range binaries {
// check for binary under the following directories
for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
binPath := filepath.Join(path, binary)
if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
continue
}
ne.paths[binary] = binPath
break
}
// systemd-run is optional, bailout if we don't find any of the other binaries
if ne.paths[binary] == "" && binary != "systemd-run" {
return fmt.Errorf("unable to find %v", binary)
}
}
return nil
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
append([]string{ne.AbsHostPath(cmd)}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return ne.executor.Command(nsenterPath, fullArgs...)
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *Nsenter) AbsHostPath(command string) string {
path, ok := ne.paths[command]
if !ok {
return command
}
return path
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *Nsenter) SupportsSystemd() (string, bool) {
systemdRunPath, ok := ne.paths["systemd-run"]
return systemdRunPath, ok && systemdRunPath != ""
}
// EvalSymlinks returns the path name on the host after evaluating symlinks on the
// host.
// mustExist makes EvalSymlinks to return error when the path does not
// exist. When it's false, it evaluates symlinks of the existing part and
// blindly adds the non-existing part:
// pathname: /mnt/volume/non/existing/directory
// /mnt/volume exists
// non/existing/directory does not exist
// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
// /mnt/foo/non/existing/directory.
//
// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
var args []string
if mustExist {
// "realpath -e: all components of the path must exist"
args = []string{"-e", pathname}
} else {
// "realpath -m: no path components need exist or be a directory"
args = []string{"-m", pathname}
}
outBytes, err := ne.Exec("realpath", args).CombinedOutput()
if err != nil {
klog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
return "", err
}
return strings.TrimSpace(string(outBytes)), nil
}
// KubeletPath returns the path name that can be accessed by containerized
// kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
// before calling this function
func (ne *Nsenter) KubeletPath(pathname string) string {
return filepath.Join(ne.hostRootFsPath, pathname)
}
// NewFakeNsenter returns a Nsenter that does not run "nsenter --mount=... --",
// but runs everything in the same mount namespace as the unit test binary.
// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
// This fake Nsenter is enough for most operations, e.g. to resolve symlinks,
// but it's not enough to call /bin/mount - unit tests don't run as root.
func NewFakeNsenter(rootfsPath string) (*Nsenter, error) {
executor := &fakeExec{
rootfsPath: rootfsPath,
}
// prepare /rootfs/bin, usr/bin and usr/sbin
bin := filepath.Join(rootfsPath, "bin")
if err := os.Symlink("/bin", bin); err != nil {
return nil, err
}
usr := filepath.Join(rootfsPath, "usr")
if err := os.Mkdir(usr, 0755); err != nil {
return nil, err
}
usrbin := filepath.Join(usr, "bin")
if err := os.Symlink("/usr/bin", usrbin); err != nil {
return nil, err
}
usrsbin := filepath.Join(usr, "sbin")
if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
return nil, err
}
return NewNsenter(rootfsPath, executor)
}
type fakeExec struct {
rootfsPath string
}
func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
// This will intentionaly panic if Nsenter does not provide enough arguments.
realCmd := args[2]
realArgs := args[3:]
return exec.New().Command(realCmd, realArgs...)
}
func (fakeExec) LookPath(file string) (string, error) {
return "", errors.New("not implemented")
}
func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return nil
}
var _ exec.Interface = fakeExec{}

56
vendor/k8s.io/kubernetes/pkg/util/nsenter/nsenter_unsupported.go generated vendored
View file

@ -1,56 +0,0 @@
// +build !linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"k8s.io/utils/exec"
)
const (
// DefaultHostRootFsPath is path to host's filesystem mounted into container
// with kubelet.
DefaultHostRootFsPath = "/rootfs"
)
// Nsenter is part of experimental support for running the kubelet
// in a container.
type Nsenter struct {
// a map of commands to their paths on the host filesystem
Paths map[string]string
}
// NewNsenter constructs a new instance of Nsenter
func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
return &Nsenter{}, nil
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
return nil
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *Nsenter) AbsHostPath(command string) string {
return ""
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *Nsenter) SupportsSystemd() (string, bool) {
return "", false
}

6
vendor/k8s.io/kubernetes/pkg/util/verify-util-pkg.sh generated vendored
View file

@ -21,7 +21,7 @@ set -o errexit
set -o nounset
set -o pipefail
BASH_DIR=$(dirname "${BASH_SOURCE}")
BASH_DIR=$(dirname "${BASH_SOURCE[0]}")
find_go_files() {
find . -maxdepth 1 -not \( \
@ -34,8 +34,8 @@ find_go_files() {
ret=0
pushd "${BASH_DIR}" > /dev/null
for path in `find_go_files`; do
file=$(basename $path)
for path in $(find_go_files); do
file=$(basename "$path")
echo "Found pkg/util/${file}, but should be moved into util sub-pkgs." 1>&2
ret=1
done

3
vendor/k8s.io/kubernetes/pkg/volume/BUILD generated vendored
View file

@ -21,6 +21,7 @@ go_library(
"//pkg/util/mount:go_default_library",
"//pkg/volume/util/fs:go_default_library",
"//pkg/volume/util/recyclerclient:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
@ -29,9 +30,9 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)

5
vendor/k8s.io/kubernetes/pkg/volume/OWNERS generated vendored
View file

@ -1,11 +1,13 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- saad-ali
- thockin
- jsafrane
- matchstick
- rootfs
- gnufied
- childsb
- msau42
reviewers:
- saad-ali
- jsafrane
@ -15,5 +17,6 @@ reviewers:
- gnufied
- verult
- davidz627
- humblec
labels:
- sig/storage

4
vendor/k8s.io/kubernetes/pkg/volume/noop_expandable_plugin.go generated vendored
View file

@ -48,6 +48,10 @@ func (n *noopExpandableVolumePluginInstance) CanSupport(spec *Spec) bool {
return true
}
func (n *noopExpandableVolumePluginInstance) IsMigratedToCSI() bool {
return false
}
func (n *noopExpandableVolumePluginInstance) RequiresRemount() bool {
return false
}

143
vendor/k8s.io/kubernetes/pkg/volume/plugins.go generated vendored
View file

@ -30,12 +30,13 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/validation"
clientset "k8s.io/client-go/kubernetes"
storagelisters "k8s.io/client-go/listers/storage/v1beta1"
"k8s.io/client-go/tools/record"
cloudprovider "k8s.io/cloud-provider"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume/util/recyclerclient"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
type ProbeOperation uint32
@ -45,6 +46,9 @@ type ProbeEvent struct {
Op ProbeOperation // The operation to the plugin
}
// CSIVolumePhaseType stores information about CSI volume path.
type CSIVolumePhaseType string
const (
// Common parameter which can be specified in StorageClass to specify the desired FSType
// Provisioners SHOULD implement support for this if they are block device based
@ -54,6 +58,8 @@ const (
ProbeAddOrUpdate ProbeOperation = 1 << iota
ProbeRemove
CSIVolumeStaged CSIVolumePhaseType = "staged"
CSIVolumePublished CSIVolumePhaseType = "published"
)
// VolumeOptions contains option information about a volume.
@ -84,6 +90,26 @@ type VolumeOptions struct {
Parameters map[string]string
}
// NodeResizeOptions contain options to be passed for node expansion.
type NodeResizeOptions struct {
VolumeSpec *Spec
// DevicePath - location of actual device on the node. In case of CSI
// this just could be volumeID
DevicePath string
// DeviceMountPath location where device is mounted on the node. If volume type
// is attachable - this would be global mount path otherwise
// it would be location where volume was mounted for the pod
DeviceMountPath string
NewSize resource.Quantity
OldSize resource.Quantity
// CSIVolumePhase contains volume phase on the node
CSIVolumePhase CSIVolumePhaseType
}
type DynamicPluginProber interface {
Init() error
@ -118,6 +144,10 @@ type VolumePlugin interface {
// const.
CanSupport(spec *Spec) bool
// IsMigratedToCSI tests whether a CSIDriver implements this plugin's
// functionality
IsMigratedToCSI() bool
// RequiresRemount returns true if this plugin requires mount calls to be
// reexecuted. Atomically updating volumes, like Downward API, depend on
// this to update the contents of the volume.
@ -135,10 +165,10 @@ type VolumePlugin interface {
NewUnmounter(name string, podUID types.UID) (Unmounter, error)
// ConstructVolumeSpec constructs a volume spec based on the given volume name
// and mountPath. The spec may have incomplete information due to limited
// and volumePath. The spec may have incomplete information due to limited
// information from input. This function is used by volume manager to reconstruct
// volume spec by reading the volume directories from disk
ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error)
ConstructVolumeSpec(volumeName, volumePath string) (*Spec, error)
// SupportsMountOption returns true if volume plugins supports Mount options
// Specifying mount options in a volume plugin that doesn't support
@ -184,12 +214,6 @@ type DeletableVolumePlugin interface {
NewDeleter(spec *Spec) (Deleter, error)
}
const (
// Name of a volume in external cloud that is being provisioned and thus
// should be ignored by rest of Kubernetes.
ProvisionedVolumeName = "placeholder-for-provisioning"
)
// ProvisionableVolumePlugin is an extended interface of VolumePlugin and is
// used to create volumes for the cluster.
type ProvisionableVolumePlugin interface {
@ -206,6 +230,8 @@ type AttachableVolumePlugin interface {
DeviceMountableVolumePlugin
NewAttacher() (Attacher, error)
NewDetacher() (Detacher, error)
// CanAttach tests if provided volume spec is attachable
CanAttach(spec *Spec) bool
}
// DeviceMountableVolumePlugin is an extended interface of VolumePlugin and is used
@ -218,18 +244,20 @@ type DeviceMountableVolumePlugin interface {
}
// ExpandableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that can be
// expanded
// expanded via control-plane ExpandVolumeDevice call.
type ExpandableVolumePlugin interface {
VolumePlugin
ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error)
RequiresFSResize() bool
}
// FSResizableVolumePlugin is an extension of ExpandableVolumePlugin and is used for volumes (flex)
// that require extra steps on nodes for expansion to complete
type FSResizableVolumePlugin interface {
ExpandableVolumePlugin
ExpandFS(spec *Spec, devicePath, deviceMountPath string, newSize, oldSize resource.Quantity) error
// NodeExpandableVolumePlugin is an expanded interface of VolumePlugin and is used for volumes that
// require expansion on the node via NodeExpand call.
type NodeExpandableVolumePlugin interface {
VolumePlugin
RequiresFSResize() bool
// NodeExpand expands volume on given deviceMountPath and returns true if resize is successful.
NodeExpand(resizeOptions NodeResizeOptions) (bool, error)
}
// VolumePluginWithAttachLimits is an extended interface of VolumePlugin that restricts number of
@ -275,7 +303,33 @@ type BlockVolumePlugin interface {
// The spec may have incomplete information due to limited information
// from input. This function is used by volume manager to reconstruct
// volume spec by reading the volume directories from disk.
ConstructBlockVolumeSpec(podUID types.UID, volumeName, mountPath string) (*Spec, error)
ConstructBlockVolumeSpec(podUID types.UID, volumeName, volumePath string) (*Spec, error)
}
// TODO(#14217)
// As part of the Volume Host refactor we are starting to create Volume Hosts
// for specific hosts. New methods for each specific host can be added here.
// Currently consumers will do type assertions to get the specific type of Volume
// Host; however, the end result should be that specific Volume Hosts are passed
// to the specific functions they are needed in (instead of using a catch-all
// VolumeHost interface)
// KubeletVolumeHost is a Kubelet specific interface that plugins can use to access the kubelet.
type KubeletVolumeHost interface {
// SetKubeletError lets plugins set an error on the Kubelet runtime status
// that will cause the Kubelet to post NotReady status with the error message provided
SetKubeletError(err error)
}
// AttachDetachVolumeHost is a AttachDetach Controller specific interface that plugins can use
// to access methods on the Attach Detach Controller.
type AttachDetachVolumeHost interface {
// CSINodeLister returns the informer lister for the CSINode API Object
CSINodeLister() storagelisters.CSINodeLister
// IsAttachDetachController is an interface marker to strictly tie AttachDetachVolumeHost
// to the attachDetachController
IsAttachDetachController() bool
}
// VolumeHost is an interface that plugins can use to access the kubelet.
@ -317,9 +371,6 @@ type VolumeHost interface {
// GetKubeClient returns a client interface
GetKubeClient() clientset.Interface
// GetCSIClient returns a client interface to csi.storage.k8s.io
GetCSIClient() csiclientset.Interface
// NewWrapperMounter finds an appropriate plugin with which to handle
// the provided spec. This is used to implement volume plugins which
// "wrap" other plugins. For example, the "secret" volume is
@ -367,6 +418,9 @@ type VolumeHost interface {
// Returns the event recorder of kubelet.
GetEventRecorder() record.EventRecorder
// Returns an interface that should be used to execute subpath operations
GetSubpather() subpath.Interface
}
// VolumePluginMgr tracks registered plugins.
@ -601,6 +655,37 @@ func (pm *VolumePluginMgr) FindPluginBySpec(spec *Spec) (VolumePlugin, error) {
return matches[0], nil
}
// IsPluginMigratableBySpec looks for a plugin that can support a given volume
// specification and whether that plugin is Migratable. If no plugins can
// support or more than one plugin can support it, return error.
func (pm *VolumePluginMgr) IsPluginMigratableBySpec(spec *Spec) (bool, error) {
pm.mutex.Lock()
defer pm.mutex.Unlock()
if spec == nil {
return false, fmt.Errorf("could not find if plugin is migratable because volume spec is nil")
}
matchedPluginNames := []string{}
matches := []VolumePlugin{}
for k, v := range pm.plugins {
if v.CanSupport(spec) {
matchedPluginNames = append(matchedPluginNames, k)
matches = append(matches, v)
}
}
if len(matches) == 0 {
// Not a known plugin (flex) in which case it is not migratable
return false, nil
}
if len(matches) > 1 {
return false, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ","))
}
return matches[0].IsMigratedToCSI(), nil
}
// FindPluginByName fetches a plugin by name or by legacy name. If no plugin
// is found, returns error.
func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) {
@ -788,7 +873,9 @@ func (pm *VolumePluginMgr) FindAttachablePluginBySpec(spec *Spec) (AttachableVol
return nil, err
}
if attachableVolumePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok {
return attachableVolumePlugin, nil
if attachableVolumePlugin.CanAttach(spec) {
return attachableVolumePlugin, nil
}
}
return nil, nil
}
@ -839,10 +926,10 @@ func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVol
if spec.IsKubeletExpandable() {
// for kubelet expandable volumes, return a noop plugin that
// returns success for expand on the controller
klog.Warningf("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name())
klog.V(4).Infof("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name())
return &noopExpandableVolumePluginInstance{spec}, nil
}
klog.Warningf("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err)
klog.V(4).Infof("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err)
return nil, err
}
@ -891,26 +978,26 @@ func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugi
return nil, nil
}
// FindFSResizablePluginBySpec fetches a persistent volume plugin by spec
func (pm *VolumePluginMgr) FindFSResizablePluginBySpec(spec *Spec) (FSResizableVolumePlugin, error) {
// FindNodeExpandablePluginBySpec fetches a persistent volume plugin by spec
func (pm *VolumePluginMgr) FindNodeExpandablePluginBySpec(spec *Spec) (NodeExpandableVolumePlugin, error) {
volumePlugin, err := pm.FindPluginBySpec(spec)
if err != nil {
return nil, err
}
if fsResizablePlugin, ok := volumePlugin.(FSResizableVolumePlugin); ok {
if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok {
return fsResizablePlugin, nil
}
return nil, nil
}
// FindFSResizablePluginByName fetches a persistent volume plugin by name
func (pm *VolumePluginMgr) FindFSResizablePluginByName(name string) (FSResizableVolumePlugin, error) {
// FindNodeExpandablePluginByName fetches a persistent volume plugin by name
func (pm *VolumePluginMgr) FindNodeExpandablePluginByName(name string) (NodeExpandableVolumePlugin, error) {
volumePlugin, err := pm.FindPluginByName(name)
if err != nil {
return nil, err
}
if fsResizablePlugin, ok := volumePlugin.(FSResizableVolumePlugin); ok {
if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok {
return fsResizablePlugin, nil
}

10
vendor/k8s.io/kubernetes/pkg/volume/util/BUILD generated vendored
View file

@ -9,7 +9,6 @@ go_library(
"device_util_linux.go",
"device_util_unsupported.go",
"doc.go",
"error.go",
"finalizer.go",
"io_util.go",
"metrics.go",
@ -23,10 +22,8 @@ go_library(
"//pkg/api/legacyscheme:go_default_library",
"//pkg/apis/core/v1/helper:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/util/resizefs:go_default_library",
"//pkg/util/strings:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/util/types:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
@ -43,6 +40,7 @@ go_library(
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/strings:go_default_library",
],
)
@ -52,7 +50,6 @@ go_test(
"atomic_writer_test.go",
"attach_limit_test.go",
"device_util_linux_test.go",
"main_test.go",
"nested_volumes_test.go",
"resize_util_test.go",
"util_test.go",
@ -61,8 +58,6 @@ go_test(
deps = [
"//pkg/apis/core/install:go_default_library",
"//pkg/apis/core/v1/helper:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/slice:go_default_library",
"//pkg/volume:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
@ -70,8 +65,6 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/client-go/util/testing:go_default_library",
@ -95,6 +88,7 @@ filegroup(
"//pkg/volume/util/nestedpendingoperations:all-srcs",
"//pkg/volume/util/operationexecutor:all-srcs",
"//pkg/volume/util/recyclerclient:all-srcs",
"//pkg/volume/util/subpath:all-srcs",
"//pkg/volume/util/types:all-srcs",
"//pkg/volume/util/volumepathhandler:all-srcs",
],

2
vendor/k8s.io/kubernetes/pkg/volume/util/OWNERS generated vendored Executable file → Normal file
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- saad-ali
reviewers:

7
vendor/k8s.io/kubernetes/pkg/volume/util/attach_limit.go generated vendored
View file

@ -40,6 +40,13 @@ const (
// GCEVolumeLimitKey stores resource name that will store volume limits for GCE node
GCEVolumeLimitKey = "attachable-volumes-gce-pd"
// CinderVolumeLimitKey contains Volume limit key for Cinder
CinderVolumeLimitKey = "attachable-volumes-cinder"
// DefaultMaxCinderVolumes defines the maximum number of PD Volumes for Cinder
// For Openstack we are keeping this to a high enough value so as depending on backend
// cluster admins can configure it.
DefaultMaxCinderVolumes = 256
// CSIAttachLimitPrefix defines prefix used for CSI volumes
CSIAttachLimitPrefix = "attachable-volumes-csi-"

25
vendor/k8s.io/kubernetes/pkg/volume/util/device_util_linux.go generated vendored
View file

@ -21,10 +21,12 @@ package util
import (
"errors"
"fmt"
"k8s.io/klog"
"os"
"path"
"strconv"
"strings"
"k8s.io/klog"
)
// FindMultipathDeviceForDevice given a device name like /dev/sdx, find the devicemapper parent
@ -98,6 +100,9 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (
sysPath := "/sys/class/iscsi_host"
hostDirs, err := io.ReadDir(sysPath)
if err != nil {
if os.IsNotExist(err) {
return portalHostMap, nil
}
return nil, err
}
for _, hostDir := range hostDirs {
@ -135,7 +140,8 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (
targetNamePath := sessionPath + "/iscsi_session/" + sessionName + "/targetname"
targetName, err := io.ReadFile(targetNamePath)
if err != nil {
return nil, err
klog.Infof("Failed to process session %s, assuming this session is unavailable: %s", sessionName, err)
continue
}
// Ignore hosts that don't matchthe target we were looking for.
@ -147,7 +153,8 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (
// for the iSCSI connection.
dirs2, err := io.ReadDir(sessionPath)
if err != nil {
return nil, err
klog.Infof("Failed to process session %s, assuming this session is unavailable: %s", sessionName, err)
continue
}
for _, dir2 := range dirs2 {
// Skip over files that aren't the connection
@ -164,25 +171,29 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (
addrPath := connectionPath + "/address"
addr, err := io.ReadFile(addrPath)
if err != nil {
return nil, err
klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
continue
}
portPath := connectionPath + "/port"
port, err := io.ReadFile(portPath)
if err != nil {
return nil, err
klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
continue
}
persistentAddrPath := connectionPath + "/persistent_address"
persistentAddr, err := io.ReadFile(persistentAddrPath)
if err != nil {
return nil, err
klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
continue
}
persistentPortPath := connectionPath + "/persistent_port"
persistentPort, err := io.ReadFile(persistentPortPath)
if err != nil {
return nil, err
klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
continue
}
// Add entries to the map for both the current and persistent portals

42
vendor/k8s.io/kubernetes/pkg/volume/util/error.go generated vendored
View file

@ -1,42 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
k8stypes "k8s.io/apimachinery/pkg/types"
)
// DanglingAttachError indicates volume is attached to a different node
// than we expected.
type DanglingAttachError struct {
msg string
CurrentNode k8stypes.NodeName
DevicePath string
}
func (err *DanglingAttachError) Error() string {
return err.msg
}
// NewDanglingError create a new dangling error
func NewDanglingError(msg string, node k8stypes.NodeName, devicePath string) error {
return &DanglingAttachError{
msg: msg,
CurrentNode: node,
DevicePath: devicePath,
}
}

100
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/BUILD generated vendored Normal file
View file

@ -0,0 +1,100 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"subpath.go",
"subpath_linux.go",
"subpath_nsenter.go",
"subpath_unsupported.go",
"subpath_windows.go",
],
importpath = "k8s.io/kubernetes/pkg/volume/util/subpath",
visibility = ["//visibility:public"],
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/mount:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"//conditions:default": [],
}),
)
go_test(
name = "go_default_test",
srcs = [
"subpath_linux_test.go",
"subpath_nsenter_test.go",
"subpath_windows_test.go",
],
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/mount:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

13
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/OWNERS generated vendored Normal file
View file

@ -0,0 +1,13 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- jingxu97
- saad-ali
- jsafrane
- msau42
- andyzhangx
approvers:
- jingxu97
- saad-ali
- jsafrane

92
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath.go generated vendored Normal file
View file

@ -0,0 +1,92 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import "os"
// Interface defines the set of methods all subpathers must implement
type Interface interface {
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory.
CleanSubPaths(poodDir string, volumeName string) error
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
// that's 1) inside given volumePath and 2) immutable after this call.
//
// newHostPath - location of prepared subPath. It should be used instead of
// hostName when running the container.
// cleanupAction - action to run when the container is running or it failed to start.
//
// CleanupAction must be called immediately after the container with given
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
// when the pod finishes.
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
// SafeMakeDir creates subdir within given base. It makes sure that the
// created directory does not escape given base directory mis-using
// symlinks. Note that the function makes sure that it creates the directory
// somewhere under the base, nothing else. E.g. if the directory already
// exists, it may exist outside of the base due to symlinks.
// This method should be used if the directory to create is inside volume
// that's under user control. User must not be able to use symlinks to
// escape the volume to create directories somewhere else.
SafeMakeDir(subdir string, base string, perm os.FileMode) error
}
// Subpath defines the attributes of a subpath
type Subpath struct {
// index of the VolumeMount for this container
VolumeMountIndex int
// Full path to the subpath directory on the host
Path string
// name of the volume that is a valid directory name.
VolumeName string
// Full path to the volume path
VolumePath string
// Path to the pod's directory, including pod UID
PodDir string
// Name of the container
ContainerName string
}
// Compile time-check for all implementers of subpath interface
var _ Interface = &subpath{}
var _ Interface = &FakeSubpath{}
// FakeSubpath is a subpather implementation for testing
type FakeSubpath struct{}
// PrepareSafeSubpath is a fake implementation of PrepareSafeSubpath. Always returns
// newHostPath == subPath.Path
func (fs *FakeSubpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
// CleanSubPaths is a fake implementation of CleanSubPaths. It is a noop
func (fs *FakeSubpath) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
// SafeMakeDir is a fake implementation of SafeMakeDir. It is a noop
func (fs *FakeSubpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}

563
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go generated vendored Normal file
View file

@ -0,0 +1,563 @@
// +build linux
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/mount"
)
const (
// place for subpath mounts
// TODO: pass in directory using kubelet_getters instead
containerSubPathDirectoryName = "volume-subpaths"
// syscall.Openat flags used to traverse directories not following symlinks
nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
// flags for getting file descriptor without following the symlink
openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
)
type subpath struct {
mounter mount.Interface
}
// New returns a subpath.Interface for the current system
func New(mounter mount.Interface) Interface {
return &subpath{
mounter: mounter,
}
}
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(sp.mounter, podDir, volumeName)
}
func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(sp.mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
// This implementation is shared between Linux and NsEnter
func safeOpenSubPath(mounter mount.Interface, subpath Subpath) (int, error) {
if !mount.PathWithinBase(subpath.Path, subpath.VolumePath) {
return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
}
fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
if err != nil {
return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
}
return fd, nil
}
// prepareSubpathTarget creates target for bind-mount of subpath. It returns
// "true" when the target already exists and something is mounted there.
// Given Subpath must have all paths with already resolved symlinks and with
// paths relevant to kubelet (when it runs in a container).
// This function is called also by NsEnterMounter. It works because
// /var/lib/kubelet is mounted from the host into the container with Kubelet as
// /var/lib/kubelet too.
func prepareSubpathTarget(mounter mount.Interface, subpath Subpath) (bool, string, error) {
// Early check for already bind-mounted subpath.
bindPathTarget := getSubpathBindTarget(subpath)
notMount, err := mounter.IsNotMountPoint(bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
return true, bindPathTarget, nil
}
// bindPathTarget is in /var/lib/kubelet and thus reachable without any
// translation even to containerized kubelet.
bindParent := filepath.Dir(bindPathTarget)
err = os.MkdirAll(bindParent, 0750)
if err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
}
t, err := os.Lstat(subpath.Path)
if err != nil {
return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindPathTarget>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
return false, bindPathTarget, nil
}
func getSubpathBindTarget(subpath Subpath) string {
// containerName is DNS label, i.e. safe as a directory name.
return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
}
func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs on the host:
// - safely open the subpath
// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
// Evaluate all symlinks here once for all subsequent functions.
newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
newPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
subpath.VolumePath = newVolumePath
subpath.Path = newPath
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
kubeletPid := os.Getpid()
mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
// Do the bind mount
options := []string{"bind"}
klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// This implementation is shared between Linux and NsEnter
func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string) error {
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
containerDirs, err := ioutil.ReadDir(subPathDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("error reading %s: %s", subPathDir, err)
}
for _, containerDir := range containerDirs {
if !containerDir.IsDir() {
klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
continue
}
klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
if path == fullContainerDirPath {
// Skip top level directory
return nil
}
// pass through errors and let doCleanSubPath handle them
if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
return err
}
return nil
})
if err != nil {
return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
}
// Whole container has been processed, remove its directory.
if err := os.Remove(fullContainerDirPath); err != nil {
return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
}
klog.V(5).Infof("Removed %s", fullContainerDirPath)
}
// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
if err := os.Remove(subPathDir); err != nil {
return fmt.Errorf("error deleting %s: %s", subPathDir, err)
}
klog.V(5).Infof("Removed %s", subPathDir)
// Remove entire subpath directory if it's the last one
podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
}
klog.V(5).Infof("Removed %s", podSubPathDir)
return nil
}
// doCleanSubPath tears down the single subpath bind mount
func doCleanSubPath(mounter mount.Interface, fullContainerDirPath, subPathIndex string) error {
// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
if err := mount.CleanupMountPoint(fullSubPath, mounter, true); err != nil {
return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
}
klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
return nil
}
// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
func cleanSubPath(mounter mount.Interface, subpath Subpath) error {
containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
// Clean subdir bindmount
if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
return err
}
// Recusively remove directories if empty
if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
return err
}
return nil
}
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error {
if !mount.PathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
}
for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
s, err := os.Stat(curDir)
if err != nil {
if os.IsNotExist(err) {
klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
continue
}
return fmt.Errorf("error stat %q: %v", curDir, err)
}
if !s.IsDir() {
return fmt.Errorf("path %q not a directory", curDir)
}
err = os.Remove(curDir)
if os.IsExist(err) {
klog.V(5).Infof("Directory %q not empty, not removing", curDir)
break
} else if err != nil {
return fmt.Errorf("error removing directory %q: %v", curDir, err)
}
klog.V(5).Infof("Removed directory %q", curDir)
}
return nil
}
// This implementation is shared between Linux and NsEnterMounter. Both pathname
// and base must be either already resolved symlinks or thet will be resolved in
// kubelet's mount namespace (in case it runs containerized).
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !mount.PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err)
}
if !mount.PathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
parentFD, err := doSafeOpen(fullExistingPath, base)
if err != nil {
return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
}
childFD := -1
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
}
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink.
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
if err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
// Dive into the created directory
childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
if err != nil {
return fmt.Errorf("cannot open %s: %s", currentPath, err)
}
// We can be sure that childFD is safe to use. It could be changed
// by user after Mkdirat() and before Openat(), however:
// - it could not be changed to symlink - we use nofollowFlags
// - it could be changed to a file (or device, pipe, socket, ...)
// but either subsequent Mkdirat() fails or we mount this file
// to user's container. Security is no violated in both cases
// and user either gets error or the file that it can already access.
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
parentFD = childFD
childFD = -1
}
// Everything was created. mkdirat(..., perm) above was affected by current
// umask and we must apply the right permissions to the last directory
// (that's the one that will be available to the container as subpath)
// so user can read/write it. This is the behavior of previous code.
// TODO: chmod all created directories, not just the last one.
// parentFD is the last created directory.
// Translate perm (os.FileMode) to uint32 that fchmod() expects
kernelPerm := uint32(perm & os.ModePerm)
if perm&os.ModeSetgid > 0 {
kernelPerm |= syscall.S_ISGID
}
if perm&os.ModeSetuid > 0 {
kernelPerm |= syscall.S_ISUID
}
if perm&os.ModeSticky > 0 {
kernelPerm |= syscall.S_ISVTX
}
if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
return fmt.Errorf("chmod %q failed: %s", currentPath, err)
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
dirs := strings.Split(rel, string(filepath.Separator))
// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
// This should be faster than looping through all dirs and calling os.Stat()
// on each of them, as the symlinks are resolved only once with OpenAt().
currentPath := base
fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
if err != nil {
return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
}
defer func() {
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
}()
for i, dir := range dirs {
// Using O_PATH here will prevent hangs in case user replaces directory with
// fifo
childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
if err != nil {
if os.IsNotExist(err) {
return currentPath, dirs[i:], nil
}
return base, nil, err
}
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
fd = childFD
currentPath = filepath.Join(currentPath, dir)
}
return pathname, []string{}, nil
}
// This implementation is shared between Linux and NsEnterMounter
// Open path and return its fd.
// Symlinks are disallowed (pathname must already resolve symlinks),
// and the path must be within the base directory.
func doSafeOpen(pathname string, base string) (int, error) {
pathname = filepath.Clean(pathname)
base = filepath.Clean(base)
// Calculate segments to follow
subpath, err := filepath.Rel(base, pathname)
if err != nil {
return -1, err
}
segments := strings.Split(subpath, string(filepath.Separator))
// Assumption: base is the only directory that we have under control.
// Base dir is not allowed to be a symlink.
parentFD, err := syscall.Open(base, nofollowFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
}
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
}
}
}()
childFD := -1
defer func() {
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := base
// Follow the segments one by one using openat() to make
// sure the user cannot change already existing directories into symlinks.
for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg)
if !mount.PathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
}
klog.V(5).Infof("Opening path %s", currentPath)
childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
}
var deviceStat unix.Stat_t
err := unix.Fstat(childFD, &deviceStat)
if err != nil {
return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
}
fileFmt := deviceStat.Mode & syscall.S_IFMT
if fileFmt == syscall.S_IFLNK {
return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
}
// Close parentFD
if err = syscall.Close(parentFD); err != nil {
return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
}
// Set child to new parent
parentFD = childFD
childFD = -1
}
// We made it to the end, return this fd, don't close it
finalFD := parentFD
parentFD = -1
return finalFD, nil
}

186
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_nsenter.go generated vendored Normal file
View file

@ -0,0 +1,186 @@
// +build linux
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"fmt"
"os"
"path/filepath"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
"k8s.io/utils/nsenter"
"k8s.io/kubernetes/pkg/util/mount"
)
type subpathNSE struct {
mounter mount.Interface
ne *nsenter.Nsenter
rootDir string
}
// Compile time-check for all implementers of subpath interface
var _ Interface = &subpathNSE{}
// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter
// It is only valid on Linux systems
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return &subpathNSE{
mounter: mounter,
ne: ne,
rootDir: rootDir,
}
}
func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(sp.mounter, podDir, volumeName)
}
func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = sp.doNsEnterBindSubPath(subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(sp.rootDir)
if mount.PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
// create subpaths for paths in /var/lib/kubelet.
return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
}
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := sp.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
}
func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
// - check that we mounted the right thing by comparing device ID and inode
// of the subpath (via safely opened fd) and the target (that's under our
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(sp.mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Leap of faith: optimistically expect that nobody has modified previously
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
// Check that the bind-mount target is the same inode and device as the
// source that we keept open, i.e. we mounted the right thing.
err = checkDeviceInode(fd, bindPathTarget)
if err != nil {
return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// checkDeviceInode checks that opened file and path represent the same file.
func checkDeviceInode(fd int, path string) error {
var srcStat, dstStat unix.Stat_t
err := unix.Fstat(fd, &srcStat)
if err != nil {
return fmt.Errorf("error running fstat on subpath FD: %v", err)
}
err = unix.Stat(path, &dstStat)
if err != nil {
return fmt.Errorf("error running fstat on %s: %v", path, err)
}
if srcStat.Dev != dstStat.Dev {
return fmt.Errorf("different device number")
}
if srcStat.Ino != dstStat.Ino {
return fmt.Errorf("different inode")
}
return nil
}

54
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_unsupported.go generated vendored Normal file
View file

@ -0,0 +1,54 @@
// +build !linux,!windows
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"errors"
"os"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/utils/nsenter"
)
type subpath struct{}
var errUnsupported = errors.New("util/subpath on this platform is not supported")
// New returns a subpath.Interface for the current system.
func New(mount.Interface) Interface {
return &subpath{}
}
// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
// OS choices. however, NSEnter is only valid on Linux
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return nil
}
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, errUnsupported
}
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return errUnsupported
}
func (sp *subpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return errUnsupported
}

284
vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_windows.go generated vendored Normal file
View file

@ -0,0 +1,284 @@
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/utils/nsenter"
)
type subpath struct{}
// New returns a subpath.Interface for the current system
func New(mount.Interface) Interface {
return &subpath{}
}
// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
// OS choices. however, NSEnter is only valid on Linux
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return nil
}
// check whether hostPath is within volume path
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(hostPath) == 0 {
return []uintptr{}, nil
}
finalSubPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
}
finalVolumePath, err := filepath.EvalSymlinks(volumePath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
}
return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
}
// lock all intermediate subPath directories and check they are all within volumePath
// volumePath & subPath should not contain any symlink, otherwise it will return error
func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(subPath) == 0 {
return []uintptr{}, nil
}
// get relative path to volumePath
relSubPath, err := filepath.Rel(volumePath, subPath)
if err != nil {
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
}
if mount.StartsWithBackstep(relSubPath) {
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
}
if relSubPath == "." {
// volumePath and subPath are equal
return []uintptr{}, nil
}
fileHandles := []uintptr{}
var errorResult error
currentFullPath := volumePath
dirs := strings.Split(relSubPath, string(os.PathSeparator))
for _, dir := range dirs {
// lock intermediate subPath directory first
currentFullPath = filepath.Join(currentFullPath, dir)
handle, err := lockPath(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
break
}
fileHandles = append(fileHandles, handle)
// make sure intermediate subPath directory does not contain symlink any more
stat, err := os.Lstat(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
break
}
if stat.Mode()&os.ModeSymlink != 0 {
errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
break
}
if !mount.PathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break
}
}
return fileHandles, errorResult
}
// unlockPath unlock directories
func unlockPath(fileHandles []uintptr) {
if fileHandles != nil {
for _, handle := range fileHandles {
syscall.CloseHandle(syscall.Handle(handle))
}
}
}
// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
func lockPath(path string) (uintptr, error) {
if len(path) == 0 {
return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return uintptr(syscall.InvalidHandle), err
}
access := uint32(syscall.GENERIC_READ)
sharemode := uint32(syscall.FILE_SHARE_READ)
createmode := uint32(syscall.OPEN_EXISTING)
flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
return uintptr(fd), err
}
// Lock all directories in subPath and check they're not symlinks.
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
// Unlock the directories when the container starts
cleanupAction = func() {
unlockPath(handles)
}
return subPath.Path, cleanupAction, err
}
// No bind-mounts for subpaths are necessary on Windows
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !mount.PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
if len(toCreate) == 0 {
return nil
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
}
fullBasePath, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err)
}
if !mount.PathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
defer unlockPath(fileHandles)
if err != nil {
return err
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink by lock that directory immediately
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
if err := os.Mkdir(currentPath, perm); err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
handle, err := lockPath(currentPath)
if err != nil {
return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
}
defer syscall.CloseHandle(syscall.Handle(handle))
// make sure newly created directory does not contain symlink after lock
stat, err := os.Lstat(currentPath)
if err != nil {
return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
}
if stat.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
}
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
if mount.StartsWithBackstep(rel) {
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
}
if rel == "." {
// base and pathname are equal
return pathname, []string{}, nil
}
dirs := strings.Split(rel, string(filepath.Separator))
parent := base
currentPath := base
for i, dir := range dirs {
parent = currentPath
currentPath = filepath.Join(parent, dir)
if _, err := os.Lstat(currentPath); err != nil {
if os.IsNotExist(err) {
return parent, dirs[i:], nil
}
return base, nil, err
}
}
return pathname, []string{}, nil
}

430
vendor/k8s.io/kubernetes/pkg/volume/util/util.go generated vendored
View file

@ -22,13 +22,16 @@ import (
"os"
"path"
"path/filepath"
"reflect"
"strings"
v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
utypes "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
@ -36,29 +39,14 @@ import (
"k8s.io/kubernetes/pkg/api/legacyscheme"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
"k8s.io/kubernetes/pkg/features"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
"k8s.io/kubernetes/pkg/util/mount"
utilstrings "k8s.io/kubernetes/pkg/util/strings"
"k8s.io/kubernetes/pkg/volume"
"reflect"
"hash/fnv"
"math/rand"
"strconv"
"k8s.io/apimachinery/pkg/api/resource"
utypes "k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/volume/util/types"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
utilstrings "k8s.io/utils/strings"
)
const (
// GB - GigaByte size
GB = 1000 * 1000 * 1000
// GIB - GibiByte size
GIB = 1024 * 1024 * 1024
readyFileName = "ready"
// ControllerManagedAttachAnnotation is the key of the annotation on Node
@ -79,16 +67,6 @@ const (
VolumeDynamicallyCreatedByKey = "kubernetes.io/createdby"
)
// VolumeZoneConfig contains config information about zonal volume.
type VolumeZoneConfig struct {
ZonePresent bool
ZonesPresent bool
ReplicaZoneFromNodePresent bool
Zone string
Zones string
ReplicaZoneFromNode string
}
// IsReady checks for the existence of a regular file
// called 'ready' in the given directory and returns
// true if that file exists.
@ -125,35 +103,6 @@ func SetReady(dir string) {
file.Close()
}
// UnmountPath is a common unmount routine that unmounts the given path and
// deletes the remaining directory if successful.
// TODO: Remove this function and change callers to call mount pkg directly
func UnmountPath(mountPath string, mounter mount.Interface) error {
return mount.CleanupMountPoint(mountPath, mounter, false /* extensiveMountPointCheck */)
}
// UnmountMountPoint is a common unmount routine that unmounts the given path and
// deletes the remaining directory if successful.
// if extensiveMountPointCheck is true
// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
// IsNotMountPoint is more expensive but properly handles bind mounts.
// TODO: Change callers to call mount pkg directly
func UnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMountPointCheck bool) error {
return mount.CleanupMountPoint(mountPath, mounter, extensiveMountPointCheck)
}
// PathExists returns true if the specified path exists.
// TODO: Change callers to call mount pkg directly
func PathExists(path string) (bool, error) {
return mount.PathExists(path)
}
// IsCorruptedMnt return true if err is about corrupted mount point
// TODO: Change callers to call mount pkg directly
func IsCorruptedMnt(err error) bool {
return mount.IsCorruptedMnt(err)
}
// GetSecretForPod locates secret by name in the pod's namespace and returns secret map
func GetSecretForPod(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (map[string]string, error) {
secret := make(map[string]string)
@ -249,174 +198,6 @@ func LoadPodFromFile(filePath string) (*v1.Pod, error) {
return pod, nil
}
// SelectZoneForVolume is a wrapper around SelectZonesForVolume
// to select a single zone for a volume based on parameters
func SelectZoneForVolume(zoneParameterPresent, zonesParameterPresent bool, zoneParameter string, zonesParameter, zonesWithNodes sets.String, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, pvcName string) (string, error) {
zones, err := SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent, zoneParameter, zonesParameter, zonesWithNodes, node, allowedTopologies, pvcName, 1)
if err != nil {
return "", err
}
zone, ok := zones.PopAny()
if !ok {
return "", fmt.Errorf("could not determine a zone to provision volume in")
}
return zone, nil
}
// SelectZonesForVolume selects zones for a volume based on several factors:
// node.zone, allowedTopologies, zone/zones parameters from storageclass,
// zones with active nodes from the cluster. The number of zones = replicas.
func SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent bool, zoneParameter string, zonesParameter, zonesWithNodes sets.String, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, pvcName string, numReplicas uint32) (sets.String, error) {
if zoneParameterPresent && zonesParameterPresent {
return nil, fmt.Errorf("both zone and zones StorageClass parameters must not be used at the same time")
}
var zoneFromNode string
// pick one zone from node if present
if node != nil {
// VolumeScheduling implicit since node is not nil
if zoneParameterPresent || zonesParameterPresent {
return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if VolumeBindingMode is set to WaitForFirstConsumer. Please specify allowedTopologies in StorageClass for constraining zones")
}
// pick node's zone for one of the replicas
var ok bool
zoneFromNode, ok = node.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain]
if !ok {
return nil, fmt.Errorf("%s Label for node missing", kubeletapis.LabelZoneFailureDomain)
}
// if single replica volume and node with zone found, return immediately
if numReplicas == 1 {
return sets.NewString(zoneFromNode), nil
}
}
// pick zone from allowedZones if specified
allowedZones, err := ZonesFromAllowedTopologies(allowedTopologies)
if err != nil {
return nil, err
}
if (len(allowedTopologies) > 0) && (allowedZones.Len() == 0) {
return nil, fmt.Errorf("no matchLabelExpressions with %s key found in allowedTopologies. Please specify matchLabelExpressions with %s key", kubeletapis.LabelZoneFailureDomain, kubeletapis.LabelZoneFailureDomain)
}
if allowedZones.Len() > 0 {
// VolumeScheduling implicit since allowedZones present
if zoneParameterPresent || zonesParameterPresent {
return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if allowedTopologies specified")
}
// scheduler will guarantee if node != null above, zoneFromNode is member of allowedZones.
// so if zoneFromNode != "", we can safely assume it is part of allowedZones.
zones, err := chooseZonesForVolumeIncludingZone(allowedZones, pvcName, zoneFromNode, numReplicas)
if err != nil {
return nil, fmt.Errorf("cannot process zones in allowedTopologies: %v", err)
}
return zones, nil
}
// pick zone from parameters if present
if zoneParameterPresent {
if numReplicas > 1 {
return nil, fmt.Errorf("zone cannot be specified if desired number of replicas for pv is greather than 1. Please specify zones or allowedTopologies to specify desired zones")
}
return sets.NewString(zoneParameter), nil
}
if zonesParameterPresent {
if uint32(zonesParameter.Len()) < numReplicas {
return nil, fmt.Errorf("not enough zones found in zones parameter to provision a volume with %d replicas. Found %d zones, need %d zones", numReplicas, zonesParameter.Len(), numReplicas)
}
// directly choose from zones parameter; no zone from node need to be considered
return ChooseZonesForVolume(zonesParameter, pvcName, numReplicas), nil
}
// pick zone from zones with nodes
if zonesWithNodes.Len() > 0 {
// If node != null (and thus zoneFromNode != ""), zoneFromNode will be member of zonesWithNodes
zones, err := chooseZonesForVolumeIncludingZone(zonesWithNodes, pvcName, zoneFromNode, numReplicas)
if err != nil {
return nil, fmt.Errorf("cannot process zones where nodes exist in the cluster: %v", err)
}
return zones, nil
}
return nil, fmt.Errorf("cannot determine zones to provision volume in")
}
// ZonesFromAllowedTopologies returns a list of zones specified in allowedTopologies
func ZonesFromAllowedTopologies(allowedTopologies []v1.TopologySelectorTerm) (sets.String, error) {
zones := make(sets.String)
for _, term := range allowedTopologies {
for _, exp := range term.MatchLabelExpressions {
if exp.Key == kubeletapis.LabelZoneFailureDomain {
for _, value := range exp.Values {
zones.Insert(value)
}
} else {
return nil, fmt.Errorf("unsupported key found in matchLabelExpressions: %s", exp.Key)
}
}
}
return zones, nil
}
// ZonesSetToLabelValue converts zones set to label value
func ZonesSetToLabelValue(strSet sets.String) string {
return strings.Join(strSet.UnsortedList(), kubeletapis.LabelMultiZoneDelimiter)
}
// ZonesToSet converts a string containing a comma separated list of zones to set
func ZonesToSet(zonesString string) (sets.String, error) {
zones, err := stringToSet(zonesString, ",")
if err != nil {
return nil, fmt.Errorf("error parsing zones %s, must be strings separated by commas: %v", zonesString, err)
}
return zones, nil
}
// LabelZonesToSet converts a PV label value from string containing a delimited list of zones to set
func LabelZonesToSet(labelZonesValue string) (sets.String, error) {
return stringToSet(labelZonesValue, kubeletapis.LabelMultiZoneDelimiter)
}
// StringToSet converts a string containing list separated by specified delimiter to a set
func stringToSet(str, delimiter string) (sets.String, error) {
zonesSlice := strings.Split(str, delimiter)
zonesSet := make(sets.String)
for _, zone := range zonesSlice {
trimmedZone := strings.TrimSpace(zone)
if trimmedZone == "" {
return make(sets.String), fmt.Errorf(
"%q separated list (%q) must not contain an empty string",
delimiter,
str)
}
zonesSet.Insert(trimmedZone)
}
return zonesSet, nil
}
// LabelZonesToList converts a PV label value from string containing a delimited list of zones to list
func LabelZonesToList(labelZonesValue string) ([]string, error) {
return stringToList(labelZonesValue, kubeletapis.LabelMultiZoneDelimiter)
}
// StringToList converts a string containing list separated by specified delimiter to a list
func stringToList(str, delimiter string) ([]string, error) {
zonesSlice := make([]string, 0)
for _, zone := range strings.Split(str, delimiter) {
trimmedZone := strings.TrimSpace(zone)
if trimmedZone == "" {
return nil, fmt.Errorf(
"%q separated list (%q) must not contain an empty string",
delimiter,
str)
}
zonesSlice = append(zonesSlice, trimmedZone)
}
return zonesSlice, nil
}
// CalculateTimeoutForVolume calculates time for a Recycler pod to complete a
// recycle operation. The calculation and return value is either the
// minimumTimeout or the timeoutIncrement per Gi of storage size, whichever is
@ -433,57 +214,6 @@ func CalculateTimeoutForVolume(minimumTimeout, timeoutIncrement int, pv *v1.Pers
return timeout
}
// RoundUpSize calculates how many allocation units are needed to accommodate
// a volume of given size. E.g. when user wants 1500MiB volume, while AWS EBS
// allocates volumes in gibibyte-sized chunks,
// RoundUpSize(1500 * 1024*1024, 1024*1024*1024) returns '2'
// (2 GiB is the smallest allocatable volume that can hold 1500MiB)
func RoundUpSize(volumeSizeBytes int64, allocationUnitBytes int64) int64 {
roundedUp := volumeSizeBytes / allocationUnitBytes
if volumeSizeBytes%allocationUnitBytes > 0 {
roundedUp++
}
return roundedUp
}
// RoundUpToGB rounds up given quantity to chunks of GB
func RoundUpToGB(size resource.Quantity) int64 {
requestBytes := size.Value()
return RoundUpSize(requestBytes, GB)
}
// RoundUpToGiB rounds up given quantity upto chunks of GiB
func RoundUpToGiB(size resource.Quantity) int64 {
requestBytes := size.Value()
return RoundUpSize(requestBytes, GIB)
}
// RoundUpSizeInt calculates how many allocation units are needed to accommodate
// a volume of given size. It returns an int instead of an int64 and an error if
// there's overflow
func RoundUpSizeInt(volumeSizeBytes int64, allocationUnitBytes int64) (int, error) {
roundedUp := RoundUpSize(volumeSizeBytes, allocationUnitBytes)
roundedUpInt := int(roundedUp)
if int64(roundedUpInt) != roundedUp {
return 0, fmt.Errorf("capacity %v is too great, casting results in integer overflow", roundedUp)
}
return roundedUpInt, nil
}
// RoundUpToGBInt rounds up given quantity to chunks of GB. It returns an
// int instead of an int64 and an error if there's overflow
func RoundUpToGBInt(size resource.Quantity) (int, error) {
requestBytes := size.Value()
return RoundUpSizeInt(requestBytes, GB)
}
// RoundUpToGiBInt rounds up given quantity upto chunks of GiB. It returns an
// int instead of an int64 and an error if there's overflow
func RoundUpToGiBInt(size resource.Quantity) (int, error) {
requestBytes := size.Value()
return RoundUpSizeInt(requestBytes, GIB)
}
// GenerateVolumeName returns a PV name with clusterName prefix. The function
// should be used to generate a name of GCE PD or Cinder volume. It basically
// adds "<clusterName>-dynamic-" before the PV name, making sure the resulting
@ -509,148 +239,6 @@ func GetPath(mounter volume.Mounter) (string, error) {
return path, nil
}
// ChooseZoneForVolume implements our heuristics for choosing a zone for volume creation based on the volume name
// Volumes are generally round-robin-ed across all active zones, using the hash of the PVC Name.
// However, if the PVCName ends with `-<integer>`, we will hash the prefix, and then add the integer to the hash.
// This means that a StatefulSet's volumes (`claimname-statefulsetname-id`) will spread across available zones,
// assuming the id values are consecutive.
func ChooseZoneForVolume(zones sets.String, pvcName string) string {
// No zones available, return empty string.
if zones.Len() == 0 {
return ""
}
// We create the volume in a zone determined by the name
// Eventually the scheduler will coordinate placement into an available zone
hash, index := getPVCNameHashAndIndexOffset(pvcName)
// Zones.List returns zones in a consistent order (sorted)
// We do have a potential failure case where volumes will not be properly spread,
// if the set of zones changes during StatefulSet volume creation. However, this is
// probably relatively unlikely because we expect the set of zones to be essentially
// static for clusters.
// Hopefully we can address this problem if/when we do full scheduler integration of
// PVC placement (which could also e.g. avoid putting volumes in overloaded or
// unhealthy zones)
zoneSlice := zones.List()
zone := zoneSlice[(hash+index)%uint32(len(zoneSlice))]
klog.V(2).Infof("Creating volume for PVC %q; chose zone=%q from zones=%q", pvcName, zone, zoneSlice)
return zone
}
// chooseZonesForVolumeIncludingZone is a wrapper around ChooseZonesForVolume that ensures zoneToInclude is chosen
// zoneToInclude can either be empty in which case it is ignored. If non-empty, zoneToInclude is expected to be member of zones.
// numReplicas is expected to be > 0 and <= zones.Len()
func chooseZonesForVolumeIncludingZone(zones sets.String, pvcName, zoneToInclude string, numReplicas uint32) (sets.String, error) {
if numReplicas == 0 {
return nil, fmt.Errorf("invalid number of replicas passed")
}
if uint32(zones.Len()) < numReplicas {
return nil, fmt.Errorf("not enough zones found to provision a volume with %d replicas. Need at least %d distinct zones for a volume with %d replicas", numReplicas, numReplicas, numReplicas)
}
if zoneToInclude != "" && !zones.Has(zoneToInclude) {
return nil, fmt.Errorf("zone to be included: %s needs to be member of set: %v", zoneToInclude, zones)
}
if uint32(zones.Len()) == numReplicas {
return zones, nil
}
if zoneToInclude != "" {
zones.Delete(zoneToInclude)
numReplicas = numReplicas - 1
}
zonesChosen := ChooseZonesForVolume(zones, pvcName, numReplicas)
if zoneToInclude != "" {
zonesChosen.Insert(zoneToInclude)
}
return zonesChosen, nil
}
// ChooseZonesForVolume is identical to ChooseZoneForVolume, but selects a multiple zones, for multi-zone disks.
func ChooseZonesForVolume(zones sets.String, pvcName string, numZones uint32) sets.String {
// No zones available, return empty set.
replicaZones := sets.NewString()
if zones.Len() == 0 {
return replicaZones
}
// We create the volume in a zone determined by the name
// Eventually the scheduler will coordinate placement into an available zone
hash, index := getPVCNameHashAndIndexOffset(pvcName)
// Zones.List returns zones in a consistent order (sorted)
// We do have a potential failure case where volumes will not be properly spread,
// if the set of zones changes during StatefulSet volume creation. However, this is
// probably relatively unlikely because we expect the set of zones to be essentially
// static for clusters.
// Hopefully we can address this problem if/when we do full scheduler integration of
// PVC placement (which could also e.g. avoid putting volumes in overloaded or
// unhealthy zones)
zoneSlice := zones.List()
startingIndex := index * numZones
for index = startingIndex; index < startingIndex+numZones; index++ {
zone := zoneSlice[(hash+index)%uint32(len(zoneSlice))]
replicaZones.Insert(zone)
}
klog.V(2).Infof("Creating volume for replicated PVC %q; chosen zones=%q from zones=%q",
pvcName, replicaZones.UnsortedList(), zoneSlice)
return replicaZones
}
func getPVCNameHashAndIndexOffset(pvcName string) (hash uint32, index uint32) {
if pvcName == "" {
// We should always be called with a name; this shouldn't happen
klog.Warningf("No name defined during volume create; choosing random zone")
hash = rand.Uint32()
} else {
hashString := pvcName
// Heuristic to make sure that volumes in a StatefulSet are spread across zones
// StatefulSet PVCs are (currently) named ClaimName-StatefulSetName-Id,
// where Id is an integer index.
// Note though that if a StatefulSet pod has multiple claims, we need them to be
// in the same zone, because otherwise the pod will be unable to mount both volumes,
// and will be unschedulable. So we hash _only_ the "StatefulSetName" portion when
// it looks like `ClaimName-StatefulSetName-Id`.
// We continue to round-robin volume names that look like `Name-Id` also; this is a useful
// feature for users that are creating statefulset-like functionality without using statefulsets.
lastDash := strings.LastIndexByte(pvcName, '-')
if lastDash != -1 {
statefulsetIDString := pvcName[lastDash+1:]
statefulsetID, err := strconv.ParseUint(statefulsetIDString, 10, 32)
if err == nil {
// Offset by the statefulsetID, so we round-robin across zones
index = uint32(statefulsetID)
// We still hash the volume name, but only the prefix
hashString = pvcName[:lastDash]
// In the special case where it looks like `ClaimName-StatefulSetName-Id`,
// hash only the StatefulSetName, so that different claims on the same StatefulSet
// member end up in the same zone.
// Note that StatefulSetName (and ClaimName) might themselves both have dashes.
// We actually just take the portion after the final - of ClaimName-StatefulSetName.
// For our purposes it doesn't much matter (just suboptimal spreading).
lastDash := strings.LastIndexByte(hashString, '-')
if lastDash != -1 {
hashString = hashString[lastDash+1:]
}
klog.V(2).Infof("Detected StatefulSet-style volume name %q; index=%d", pvcName, index)
}
}
// We hash the (base) volume name, so we don't bias towards the first N zones
h := fnv.New32()
h.Write([]byte(hashString))
hash = h.Sum32()
}
return hash, index
}
// UnmountViaEmptyDir delegates the tear down operation for secret, configmap, git_repo and downwardapi
// to empty_dir
func UnmountViaEmptyDir(dir string, host volume.VolumeHost, volName string, volSpec volume.Spec, podUID utypes.UID) error {
@ -699,16 +287,6 @@ func JoinMountOptions(userOptions []string, systemOptions []string) []string {
return allMountOptions.List()
}
// ValidateZone returns:
// - an error in case zone is an empty string or contains only any combination of spaces and tab characters
// - nil otherwise
func ValidateZone(zone string) error {
if strings.TrimSpace(zone) == "" {
return fmt.Errorf("the provided %q zone is not valid, it's an empty string or contains only spaces and tab characters", zone)
}
return nil
}
// AccessModesContains returns whether the requested mode is contained by modes
func AccessModesContains(modes []v1.PersistentVolumeAccessMode, mode v1.PersistentVolumeAccessMode) bool {
for _, m := range modes {

25
vendor/k8s.io/kubernetes/pkg/volume/volume.go generated vendored
View file

@ -260,28 +260,3 @@ type DeviceUnmounter interface {
// unmounted.
UnmountDevice(deviceMountPath string) error
}
// NewDeletedVolumeInUseError returns a new instance of DeletedVolumeInUseError
// error.
func NewDeletedVolumeInUseError(message string) error {
return deletedVolumeInUseError(message)
}
type deletedVolumeInUseError string
var _ error = deletedVolumeInUseError("")
// IsDeletedVolumeInUse returns true if an error returned from Delete() is
// deletedVolumeInUseError
func IsDeletedVolumeInUse(err error) bool {
switch err.(type) {
case deletedVolumeInUseError:
return true
default:
return false
}
}
func (err deletedVolumeInUseError) Error() string {
return string(err)
}

6
vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go generated vendored
View file

@ -28,8 +28,9 @@ import (
)
const (
rwMask = os.FileMode(0660)
roMask = os.FileMode(0440)
rwMask = os.FileMode(0660)
roMask = os.FileMode(0440)
execMask = os.FileMode(0110)
)
// SetVolumeOwnership modifies the given volume to be owned by
@ -79,6 +80,7 @@ func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error {
if info.IsDir() {
mask |= os.ModeSetgid
mask |= execMask
}
err = os.Chmod(path, info.Mode()|mask)

2
vendor/k8s.io/kubernetes/third_party/OWNERS generated vendored
View file

@ -1,3 +1,5 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- lavalamp
- smarterclayton