Replace godep with dep

vendor/k8s.io/apiserver/pkg/storage/BUILD

@@ -0,0 +1,95 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cacher_whitebox_test.go",
+        "selection_predicate_test.go",
+        "time_budget_test.go",
+        "util_test.go",
+        "watch_cache_test.go",
+    ],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cacher.go",
+        "doc.go",
+        "errors.go",
+        "interfaces.go",
+        "selection_predicate.go",
+        "time_budget.go",
+        "util.go",
+        "watch_cache.go",
+    ],
+    deps = [
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/validation/path:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/features:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/trace:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/errors:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd3:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/names:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/testing:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/tests:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/value:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/OWNERS

@@ -0,0 +1,30 @@
+approvers:
+- lavalamp
+- liggitt
+- timothysc
+- wojtek-t
+- xiang90
+reviewers:
+- lavalamp
+- smarterclayton
+- wojtek-t
+- deads2k
+- caesarxuchao
+- mikedanese
+- liggitt
+- ncdc
+- tallclair
+- timothysc
+- hongchaodeng
+- krousey
+- fgrzadkowski
+- xiang90
+- mml
+- ingvagabund
+- resouer
+- mbohlool
+- lixiaobing10051267
+- mqliang
+- feihujiang
+- rrati
+- enj

vendor/k8s.io/apiserver/pkg/storage/cacher.go

@@ -0,0 +1,988 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"net/http"
+	"reflect"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utiltrace "k8s.io/apiserver/pkg/util/trace"
+	"k8s.io/client-go/tools/cache"
+)
+
+// CacherConfig contains the configuration for a given Cache.
+type CacherConfig struct {
+	// Maximum size of the history cached in memory.
+	CacheCapacity int
+
+	// An underlying storage.Interface.
+	Storage Interface
+
+	// An underlying storage.Versioner.
+	Versioner Versioner
+
+	Copier runtime.ObjectCopier
+
+	// The Cache will be caching objects of a given Type and assumes that they
+	// are all stored under ResourcePrefix directory in the underlying database.
+	Type           interface{}
+	ResourcePrefix string
+
+	// KeyFunc is used to get a key in the underlying storage for a given object.
+	KeyFunc func(runtime.Object) (string, error)
+
+	// GetAttrsFunc is used to get object labels, fields, and the uninitialized bool
+	GetAttrsFunc func(runtime.Object) (label labels.Set, field fields.Set, uninitialized bool, err error)
+
+	// TriggerPublisherFunc is used for optimizing amount of watchers that
+	// needs to process an incoming event.
+	TriggerPublisherFunc TriggerPublisherFunc
+
+	// NewList is a function that creates new empty object storing a list of
+	// objects of type Type.
+	NewListFunc func() runtime.Object
+
+	Codec runtime.Codec
+}
+
+type watchersMap map[int]*cacheWatcher
+
+func (wm watchersMap) addWatcher(w *cacheWatcher, number int) {
+	wm[number] = w
+}
+
+func (wm watchersMap) deleteWatcher(number int) {
+	delete(wm, number)
+}
+
+func (wm watchersMap) terminateAll() {
+	for key, watcher := range wm {
+		delete(wm, key)
+		watcher.stop()
+	}
+}
+
+type indexedWatchers struct {
+	allWatchers   watchersMap
+	valueWatchers map[string]watchersMap
+}
+
+func (i *indexedWatchers) addWatcher(w *cacheWatcher, number int, value string, supported bool) {
+	if supported {
+		if _, ok := i.valueWatchers[value]; !ok {
+			i.valueWatchers[value] = watchersMap{}
+		}
+		i.valueWatchers[value].addWatcher(w, number)
+	} else {
+		i.allWatchers.addWatcher(w, number)
+	}
+}
+
+func (i *indexedWatchers) deleteWatcher(number int, value string, supported bool) {
+	if supported {
+		i.valueWatchers[value].deleteWatcher(number)
+		if len(i.valueWatchers[value]) == 0 {
+			delete(i.valueWatchers, value)
+		}
+	} else {
+		i.allWatchers.deleteWatcher(number)
+	}
+}
+
+func (i *indexedWatchers) terminateAll(objectType reflect.Type) {
+	if len(i.allWatchers) > 0 || len(i.valueWatchers) > 0 {
+		glog.Warningf("Terminating all watchers from cacher %v", objectType)
+	}
+	i.allWatchers.terminateAll()
+	for index, watchers := range i.valueWatchers {
+		watchers.terminateAll()
+		delete(i.valueWatchers, index)
+	}
+}
+
+type watchFilterFunc func(key string, l labels.Set, f fields.Set, uninitialized bool) bool
+
+// Cacher is responsible for serving WATCH and LIST requests for a given
+// resource from its internal cache and updating its cache in the background
+// based on the underlying storage contents.
+// Cacher implements storage.Interface (although most of the calls are just
+// delegated to the underlying storage).
+type Cacher struct {
+	// HighWaterMarks for performance debugging.
+	// Important: Since HighWaterMark is using sync/atomic, it has to be at the top of the struct due to a bug on 32-bit platforms
+	// See: https://golang.org/pkg/sync/atomic/ for more information
+	incomingHWM HighWaterMark
+	// Incoming events that should be dispatched to watchers.
+	incoming chan watchCacheEvent
+
+	sync.RWMutex
+
+	// Before accessing the cacher's cache, wait for the ready to be ok.
+	// This is necessary to prevent users from accessing structures that are
+	// uninitialized or are being repopulated right now.
+	// ready needs to be set to false when the cacher is paused or stopped.
+	// ready needs to be set to true when the cacher is ready to use after
+	// initialization.
+	ready *ready
+
+	// Underlying storage.Interface.
+	storage Interface
+
+	copier runtime.ObjectCopier
+
+	// Expected type of objects in the underlying cache.
+	objectType reflect.Type
+
+	// "sliding window" of recent changes of objects and the current state.
+	watchCache *watchCache
+	reflector  *cache.Reflector
+
+	// Versioner is used to handle resource versions.
+	versioner Versioner
+
+	// triggerFunc is used for optimizing amount of watchers that needs to process
+	// an incoming event.
+	triggerFunc TriggerPublisherFunc
+	// watchers is mapping from the value of trigger function that a
+	// watcher is interested into the watchers
+	watcherIdx int
+	watchers   indexedWatchers
+
+	// Defines a time budget that can be spend on waiting for not-ready watchers
+	// while dispatching event before shutting them down.
+	dispatchTimeoutBudget *timeBudget
+
+	// Handling graceful termination.
+	stopLock sync.RWMutex
+	stopped  bool
+	stopCh   chan struct{}
+	stopWg   sync.WaitGroup
+}
+
+// Create a new Cacher responsible for servicing WATCH and LIST requests from
+// its internal cache and updating its cache in the background based on the
+// given configuration.
+func NewCacherFromConfig(config CacherConfig) *Cacher {
+	watchCache := newWatchCache(config.CacheCapacity, config.KeyFunc, config.GetAttrsFunc)
+	listerWatcher := newCacherListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc)
+	reflectorName := "storage/cacher.go:" + config.ResourcePrefix
+
+	// Give this error when it is constructed rather than when you get the
+	// first watch item, because it's much easier to track down that way.
+	if obj, ok := config.Type.(runtime.Object); ok {
+		if err := runtime.CheckCodec(config.Codec, obj); err != nil {
+			panic("storage codec doesn't seem to match given type: " + err.Error())
+		}
+	}
+
+	stopCh := make(chan struct{})
+	cacher := &Cacher{
+		ready:       newReady(),
+		storage:     config.Storage,
+		copier:      config.Copier,
+		objectType:  reflect.TypeOf(config.Type),
+		watchCache:  watchCache,
+		reflector:   cache.NewNamedReflector(reflectorName, listerWatcher, config.Type, watchCache, 0),
+		versioner:   config.Versioner,
+		triggerFunc: config.TriggerPublisherFunc,
+		watcherIdx:  0,
+		watchers: indexedWatchers{
+			allWatchers:   make(map[int]*cacheWatcher),
+			valueWatchers: make(map[string]watchersMap),
+		},
+		// TODO: Figure out the correct value for the buffer size.
+		incoming:              make(chan watchCacheEvent, 100),
+		dispatchTimeoutBudget: newTimeBudget(stopCh),
+		// We need to (potentially) stop both:
+		// - wait.Until go-routine
+		// - reflector.ListAndWatch
+		// and there are no guarantees on the order that they will stop.
+		// So we will be simply closing the channel, and synchronizing on the WaitGroup.
+		stopCh: stopCh,
+	}
+	watchCache.SetOnEvent(cacher.processEvent)
+	go cacher.dispatchEvents()
+
+	cacher.stopWg.Add(1)
+	go func() {
+		defer cacher.stopWg.Done()
+		wait.Until(
+			func() {
+				if !cacher.isStopped() {
+					cacher.startCaching(stopCh)
+				}
+			}, time.Second, stopCh,
+		)
+	}()
+	return cacher
+}
+
+func (c *Cacher) startCaching(stopChannel <-chan struct{}) {
+	// The 'usable' lock is always 'RLock'able when it is safe to use the cache.
+	// It is safe to use the cache after a successful list until a disconnection.
+	// We start with usable (write) locked. The below OnReplace function will
+	// unlock it after a successful list. The below defer will then re-lock
+	// it when this function exits (always due to disconnection), only if
+	// we actually got a successful list. This cycle will repeat as needed.
+	successfulList := false
+	c.watchCache.SetOnReplace(func() {
+		successfulList = true
+		c.ready.set(true)
+	})
+	defer func() {
+		if successfulList {
+			c.ready.set(false)
+		}
+	}()
+
+	c.terminateAllWatchers()
+	// Note that since onReplace may be not called due to errors, we explicitly
+	// need to retry it on errors under lock.
+	// Also note that startCaching is called in a loop, so there's no need
+	// to have another loop here.
+	if err := c.reflector.ListAndWatch(stopChannel); err != nil {
+		glog.Errorf("unexpected ListAndWatch error: %v", err)
+	}
+}
+
+// Implements storage.Interface.
+func (c *Cacher) Versioner() Versioner {
+	return c.storage.Versioner()
+}
+
+// Implements storage.Interface.
+func (c *Cacher) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error {
+	return c.storage.Create(ctx, key, obj, out, ttl)
+}
+
+// Implements storage.Interface.
+func (c *Cacher) Delete(ctx context.Context, key string, out runtime.Object, preconditions *Preconditions) error {
+	return c.storage.Delete(ctx, key, out, preconditions)
+}
+
+// Implements storage.Interface.
+func (c *Cacher) Watch(ctx context.Context, key string, resourceVersion string, pred SelectionPredicate) (watch.Interface, error) {
+	watchRV, err := ParseWatchResourceVersion(resourceVersion)
+	if err != nil {
+		return nil, err
+	}
+
+	c.ready.wait()
+
+	// We explicitly use thread unsafe version and do locking ourself to ensure that
+	// no new events will be processed in the meantime. The watchCache will be unlocked
+	// on return from this function.
+	// Note that we cannot do it under Cacher lock, to avoid a deadlock, since the
+	// underlying watchCache is calling processEvent under its lock.
+	c.watchCache.RLock()
+	defer c.watchCache.RUnlock()
+	initEvents, err := c.watchCache.GetAllEventsSinceThreadUnsafe(watchRV)
+	if err != nil {
+		// To match the uncached watch implementation, once we have passed authn/authz/admission,
+		// and successfully parsed a resource version, other errors must fail with a watch event of type ERROR,
+		// rather than a directly returned error.
+		return newErrWatcher(err), nil
+	}
+
+	triggerValue, triggerSupported := "", false
+	// TODO: Currently we assume that in a given Cacher object, any <predicate> that is
+	// passed here is aware of exactly the same trigger (at most one).
+	// Thus, either 0 or 1 values will be returned.
+	if matchValues := pred.MatcherIndex(); len(matchValues) > 0 {
+		triggerValue, triggerSupported = matchValues[0].Value, true
+	}
+
+	// If there is triggerFunc defined, but triggerSupported is false,
+	// we can't narrow the amount of events significantly at this point.
+	//
+	// That said, currently triggerFunc is defined only for Pods and Nodes,
+	// and there is only constant number of watchers for which triggerSupported
+	// is false (excluding those issues explicitly by users).
+	// Thus, to reduce the risk of those watchers blocking all watchers of a
+	// given resource in the system, we increase the sizes of buffers for them.
+	chanSize := 10
+	if c.triggerFunc != nil && !triggerSupported {
+		// TODO: We should tune this value and ideally make it dependent on the
+		// number of objects of a given type and/or their churn.
+		chanSize = 1000
+	}
+
+	c.Lock()
+	defer c.Unlock()
+	forget := forgetWatcher(c, c.watcherIdx, triggerValue, triggerSupported)
+	watcher := newCacheWatcher(c.copier, watchRV, chanSize, initEvents, watchFilterFunction(key, pred), forget)
+
+	c.watchers.addWatcher(watcher, c.watcherIdx, triggerValue, triggerSupported)
+	c.watcherIdx++
+	return watcher, nil
+}
+
+// Implements storage.Interface.
+func (c *Cacher) WatchList(ctx context.Context, key string, resourceVersion string, pred SelectionPredicate) (watch.Interface, error) {
+	return c.Watch(ctx, key, resourceVersion, pred)
+}
+
+// Implements storage.Interface.
+func (c *Cacher) Get(ctx context.Context, key string, resourceVersion string, objPtr runtime.Object, ignoreNotFound bool) error {
+	if resourceVersion == "" {
+		// If resourceVersion is not specified, serve it from underlying
+		// storage (for backward compatibility).
+		return c.storage.Get(ctx, key, resourceVersion, objPtr, ignoreNotFound)
+	}
+
+	// If resourceVersion is specified, serve it from cache.
+	// It's guaranteed that the returned value is at least that
+	// fresh as the given resourceVersion.
+	getRV, err := ParseListResourceVersion(resourceVersion)
+	if err != nil {
+		return err
+	}
+
+	if getRV == 0 && !c.ready.check() {
+		// If Cacher is not yet initialized and we don't require any specific
+		// minimal resource version, simply forward the request to storage.
+		return c.storage.Get(ctx, key, resourceVersion, objPtr, ignoreNotFound)
+	}
+
+	// Do not create a trace - it's not for free and there are tons
+	// of Get requests. We can add it if it will be really needed.
+	c.ready.wait()
+
+	objVal, err := conversion.EnforcePtr(objPtr)
+	if err != nil {
+		return err
+	}
+
+	obj, exists, readResourceVersion, err := c.watchCache.WaitUntilFreshAndGet(getRV, key, nil)
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		elem, ok := obj.(*storeElement)
+		if !ok {
+			return fmt.Errorf("non *storeElement returned from storage: %v", obj)
+		}
+		objVal.Set(reflect.ValueOf(elem.Object).Elem())
+	} else {
+		objVal.Set(reflect.Zero(objVal.Type()))
+		if !ignoreNotFound {
+			return NewKeyNotFoundError(key, int64(readResourceVersion))
+		}
+	}
+	return nil
+}
+
+// Implements storage.Interface.
+func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion string, pred SelectionPredicate, listObj runtime.Object) error {
+	pagingEnabled := utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
+	if resourceVersion == "" || (pagingEnabled && (len(pred.Continue) > 0 || pred.Limit > 0)) {
+		// If resourceVersion is not specified, serve it from underlying
+		// storage (for backward compatibility). If a continuation or limit is
+		// requested, serve it from the underlying storage as well.
+		return c.storage.GetToList(ctx, key, resourceVersion, pred, listObj)
+	}
+
+	// If resourceVersion is specified, serve it from cache.
+	// It's guaranteed that the returned value is at least that
+	// fresh as the given resourceVersion.
+	listRV, err := ParseListResourceVersion(resourceVersion)
+	if err != nil {
+		return err
+	}
+
+	if listRV == 0 && !c.ready.check() {
+		// If Cacher is not yet initialized and we don't require any specific
+		// minimal resource version, simply forward the request to storage.
+		return c.storage.GetToList(ctx, key, resourceVersion, pred, listObj)
+	}
+
+	trace := utiltrace.New(fmt.Sprintf("cacher %v: List", c.objectType.String()))
+	defer trace.LogIfLong(500 * time.Millisecond)
+
+	c.ready.wait()
+	trace.Step("Ready")
+
+	// List elements with at least 'listRV' from cache.
+	listPtr, err := meta.GetItemsPtr(listObj)
+	if err != nil {
+		return err
+	}
+	listVal, err := conversion.EnforcePtr(listPtr)
+	if err != nil || listVal.Kind() != reflect.Slice {
+		return fmt.Errorf("need a pointer to slice, got %v", listVal.Kind())
+	}
+	filter := filterFunction(key, pred)
+
+	obj, exists, readResourceVersion, err := c.watchCache.WaitUntilFreshAndGet(listRV, key, trace)
+	if err != nil {
+		return err
+	}
+	trace.Step("Got from cache")
+
+	if exists {
+		elem, ok := obj.(*storeElement)
+		if !ok {
+			return fmt.Errorf("non *storeElement returned from storage: %v", obj)
+		}
+		if filter(elem.Key, elem.Object) {
+			listVal.Set(reflect.Append(listVal, reflect.ValueOf(elem.Object).Elem()))
+		}
+	}
+	if c.versioner != nil {
+		if err := c.versioner.UpdateList(listObj, readResourceVersion, ""); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Implements storage.Interface.
+func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, pred SelectionPredicate, listObj runtime.Object) error {
+	pagingEnabled := utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
+	if resourceVersion == "" || (pagingEnabled && (len(pred.Continue) > 0 || pred.Limit > 0)) {
+		// If resourceVersion is not specified, serve it from underlying
+		// storage (for backward compatibility). If a continuation or limit is
+		// requested, serve it from the underlying storage as well.
+		return c.storage.List(ctx, key, resourceVersion, pred, listObj)
+	}
+
+	// If resourceVersion is specified, serve it from cache.
+	// It's guaranteed that the returned value is at least that
+	// fresh as the given resourceVersion.
+	listRV, err := ParseListResourceVersion(resourceVersion)
+	if err != nil {
+		return err
+	}
+
+	if listRV == 0 && !c.ready.check() {
+		// If Cacher is not yet initialized and we don't require any specific
+		// minimal resource version, simply forward the request to storage.
+		return c.storage.List(ctx, key, resourceVersion, pred, listObj)
+	}
+
+	trace := utiltrace.New(fmt.Sprintf("cacher %v: List", c.objectType.String()))
+	defer trace.LogIfLong(500 * time.Millisecond)
+
+	c.ready.wait()
+	trace.Step("Ready")
+
+	// List elements with at least 'listRV' from cache.
+	listPtr, err := meta.GetItemsPtr(listObj)
+	if err != nil {
+		return err
+	}
+	listVal, err := conversion.EnforcePtr(listPtr)
+	if err != nil || listVal.Kind() != reflect.Slice {
+		return fmt.Errorf("need a pointer to slice, got %v", listVal.Kind())
+	}
+	filter := filterFunction(key, pred)
+
+	objs, readResourceVersion, err := c.watchCache.WaitUntilFreshAndList(listRV, trace)
+	if err != nil {
+		return err
+	}
+	trace.Step(fmt.Sprintf("Listed %d items from cache", len(objs)))
+	if len(objs) > listVal.Cap() && pred.Label.Empty() && pred.Field.Empty() {
+		// Resize the slice appropriately, since we already know that none
+		// of the elements will be filtered out.
+		listVal.Set(reflect.MakeSlice(reflect.SliceOf(c.objectType.Elem()), 0, len(objs)))
+		trace.Step("Resized result")
+	}
+	for _, obj := range objs {
+		elem, ok := obj.(*storeElement)
+		if !ok {
+			return fmt.Errorf("non *storeElement returned from storage: %v", obj)
+		}
+		if filter(elem.Key, elem.Object) {
+			listVal.Set(reflect.Append(listVal, reflect.ValueOf(elem.Object).Elem()))
+		}
+	}
+	trace.Step(fmt.Sprintf("Filtered %d items", listVal.Len()))
+	if c.versioner != nil {
+		if err := c.versioner.UpdateList(listObj, readResourceVersion, ""); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Implements storage.Interface.
+func (c *Cacher) GuaranteedUpdate(
+	ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
+	preconditions *Preconditions, tryUpdate UpdateFunc, _ ...runtime.Object) error {
+	// Ignore the suggestion and try to pass down the current version of the object
+	// read from cache.
+	if elem, exists, err := c.watchCache.GetByKey(key); err != nil {
+		glog.Errorf("GetByKey returned error: %v", err)
+	} else if exists {
+		currObj := elem.(*storeElement).Object.DeepCopyObject()
+		return c.storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate, currObj)
+	}
+	// If we couldn't get the object, fallback to no-suggestion.
+	return c.storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate)
+}
+
+func (c *Cacher) triggerValues(event *watchCacheEvent) ([]string, bool) {
+	// TODO: Currently we assume that in a given Cacher object, its <c.triggerFunc>
+	// is aware of exactly the same trigger (at most one). Thus calling:
+	//   c.triggerFunc(<some object>)
+	// can return only 0 or 1 values.
+	// That means, that triggerValues itself may return up to 2 different values.
+	if c.triggerFunc == nil {
+		return nil, false
+	}
+	result := make([]string, 0, 2)
+	matchValues := c.triggerFunc(event.Object)
+	if len(matchValues) > 0 {
+		result = append(result, matchValues[0].Value)
+	}
+	if event.PrevObject == nil {
+		return result, len(result) > 0
+	}
+	prevMatchValues := c.triggerFunc(event.PrevObject)
+	if len(prevMatchValues) > 0 {
+		if len(result) == 0 || result[0] != prevMatchValues[0].Value {
+			result = append(result, prevMatchValues[0].Value)
+		}
+	}
+	return result, len(result) > 0
+}
+
+func (c *Cacher) processEvent(event *watchCacheEvent) {
+	if curLen := int64(len(c.incoming)); c.incomingHWM.Update(curLen) {
+		// Monitor if this gets backed up, and how much.
+		glog.V(1).Infof("cacher (%v): %v objects queued in incoming channel.", c.objectType.String(), curLen)
+	}
+	c.incoming <- *event
+}
+
+func (c *Cacher) dispatchEvents() {
+	for {
+		select {
+		case event, ok := <-c.incoming:
+			if !ok {
+				return
+			}
+			c.dispatchEvent(&event)
+		case <-c.stopCh:
+			return
+		}
+	}
+}
+
+func (c *Cacher) dispatchEvent(event *watchCacheEvent) {
+	triggerValues, supported := c.triggerValues(event)
+
+	c.Lock()
+	defer c.Unlock()
+	// Iterate over "allWatchers" no matter what the trigger function is.
+	for _, watcher := range c.watchers.allWatchers {
+		watcher.add(event, c.dispatchTimeoutBudget)
+	}
+	if supported {
+		// Iterate over watchers interested in the given values of the trigger.
+		for _, triggerValue := range triggerValues {
+			for _, watcher := range c.watchers.valueWatchers[triggerValue] {
+				watcher.add(event, c.dispatchTimeoutBudget)
+			}
+		}
+	} else {
+		// supported equal to false generally means that trigger function
+		// is not defined (or not aware of any indexes). In this case,
+		// watchers filters should generally also don't generate any
+		// trigger values, but can cause problems in case of some
+		// misconfiguration. Thus we paranoidly leave this branch.
+
+		// Iterate over watchers interested in exact values for all values.
+		for _, watchers := range c.watchers.valueWatchers {
+			for _, watcher := range watchers {
+				watcher.add(event, c.dispatchTimeoutBudget)
+			}
+		}
+	}
+}
+
+func (c *Cacher) terminateAllWatchers() {
+	c.Lock()
+	defer c.Unlock()
+	c.watchers.terminateAll(c.objectType)
+}
+
+func (c *Cacher) isStopped() bool {
+	c.stopLock.RLock()
+	defer c.stopLock.RUnlock()
+	return c.stopped
+}
+
+func (c *Cacher) Stop() {
+	c.stopLock.Lock()
+	c.stopped = true
+	c.stopLock.Unlock()
+	close(c.stopCh)
+	c.stopWg.Wait()
+}
+
+func forgetWatcher(c *Cacher, index int, triggerValue string, triggerSupported bool) func(bool) {
+	return func(lock bool) {
+		if lock {
+			c.Lock()
+			defer c.Unlock()
+		} else {
+			// false is currently passed only if we are forcing watcher to close due
+			// to its unresponsiveness and blocking other watchers.
+			// TODO: Get this information in cleaner way.
+			glog.V(1).Infof("Forcing watcher close due to unresponsiveness: %v", c.objectType.String())
+		}
+		// It's possible that the watcher is already not in the structure (e.g. in case of
+		// simulaneous Stop() and terminateAllWatchers(), but it doesn't break anything.
+		c.watchers.deleteWatcher(index, triggerValue, triggerSupported)
+	}
+}
+
+func filterFunction(key string, p SelectionPredicate) func(string, runtime.Object) bool {
+	f := SimpleFilter(p)
+	filterFunc := func(objKey string, obj runtime.Object) bool {
+		if !hasPathPrefix(objKey, key) {
+			return false
+		}
+		return f(obj)
+	}
+	return filterFunc
+}
+
+func watchFilterFunction(key string, p SelectionPredicate) watchFilterFunc {
+	filterFunc := func(objKey string, label labels.Set, field fields.Set, uninitialized bool) bool {
+		if !hasPathPrefix(objKey, key) {
+			return false
+		}
+		return p.MatchesObjectAttributes(label, field, uninitialized)
+	}
+	return filterFunc
+}
+
+// Returns resource version to which the underlying cache is synced.
+func (c *Cacher) LastSyncResourceVersion() (uint64, error) {
+	c.ready.wait()
+
+	resourceVersion := c.reflector.LastSyncResourceVersion()
+	if resourceVersion == "" {
+		return 0, nil
+	}
+
+	return strconv.ParseUint(resourceVersion, 10, 64)
+}
+
+// cacherListerWatcher opaques storage.Interface to expose cache.ListerWatcher.
+type cacherListerWatcher struct {
+	storage        Interface
+	resourcePrefix string
+	newListFunc    func() runtime.Object
+}
+
+func newCacherListerWatcher(storage Interface, resourcePrefix string, newListFunc func() runtime.Object) cache.ListerWatcher {
+	return &cacherListerWatcher{
+		storage:        storage,
+		resourcePrefix: resourcePrefix,
+		newListFunc:    newListFunc,
+	}
+}
+
+// Implements cache.ListerWatcher interface.
+func (lw *cacherListerWatcher) List(options metav1.ListOptions) (runtime.Object, error) {
+	list := lw.newListFunc()
+	if err := lw.storage.List(context.TODO(), lw.resourcePrefix, "", Everything, list); err != nil {
+		return nil, err
+	}
+	return list, nil
+}
+
+// Implements cache.ListerWatcher interface.
+func (lw *cacherListerWatcher) Watch(options metav1.ListOptions) (watch.Interface, error) {
+	return lw.storage.WatchList(context.TODO(), lw.resourcePrefix, options.ResourceVersion, Everything)
+}
+
+// cacherWatch implements watch.Interface to return a single error
+type errWatcher struct {
+	result chan watch.Event
+}
+
+func newErrWatcher(err error) *errWatcher {
+	// Create an error event
+	errEvent := watch.Event{Type: watch.Error}
+	switch err := err.(type) {
+	case runtime.Object:
+		errEvent.Object = err
+	case *errors.StatusError:
+		errEvent.Object = &err.ErrStatus
+	default:
+		errEvent.Object = &metav1.Status{
+			Status:  metav1.StatusFailure,
+			Message: err.Error(),
+			Reason:  metav1.StatusReasonInternalError,
+			Code:    http.StatusInternalServerError,
+		}
+	}
+
+	// Create a watcher with room for a single event, populate it, and close the channel
+	watcher := &errWatcher{result: make(chan watch.Event, 1)}
+	watcher.result <- errEvent
+	close(watcher.result)
+
+	return watcher
+}
+
+// Implements watch.Interface.
+func (c *errWatcher) ResultChan() <-chan watch.Event {
+	return c.result
+}
+
+// Implements watch.Interface.
+func (c *errWatcher) Stop() {
+	// no-op
+}
+
+// cacherWatch implements watch.Interface
+type cacheWatcher struct {
+	sync.Mutex
+	copier  runtime.ObjectCopier
+	input   chan *watchCacheEvent
+	result  chan watch.Event
+	done    chan struct{}
+	filter  watchFilterFunc
+	stopped bool
+	forget  func(bool)
+}
+
+func newCacheWatcher(copier runtime.ObjectCopier, resourceVersion uint64, chanSize int, initEvents []*watchCacheEvent, filter watchFilterFunc, forget func(bool)) *cacheWatcher {
+	watcher := &cacheWatcher{
+		copier:  copier,
+		input:   make(chan *watchCacheEvent, chanSize),
+		result:  make(chan watch.Event, chanSize),
+		done:    make(chan struct{}),
+		filter:  filter,
+		stopped: false,
+		forget:  forget,
+	}
+	go watcher.process(initEvents, resourceVersion)
+	return watcher
+}
+
+// Implements watch.Interface.
+func (c *cacheWatcher) ResultChan() <-chan watch.Event {
+	return c.result
+}
+
+// Implements watch.Interface.
+func (c *cacheWatcher) Stop() {
+	c.forget(true)
+	c.stop()
+}
+
+func (c *cacheWatcher) stop() {
+	c.Lock()
+	defer c.Unlock()
+	if !c.stopped {
+		c.stopped = true
+		close(c.done)
+		close(c.input)
+	}
+}
+
+var timerPool sync.Pool
+
+func (c *cacheWatcher) add(event *watchCacheEvent, budget *timeBudget) {
+	// Try to send the event immediately, without blocking.
+	select {
+	case c.input <- event:
+		return
+	default:
+	}
+
+	// OK, block sending, but only for up to <timeout>.
+	// cacheWatcher.add is called very often, so arrange
+	// to reuse timers instead of constantly allocating.
+	startTime := time.Now()
+	timeout := budget.takeAvailable()
+
+	t, ok := timerPool.Get().(*time.Timer)
+	if ok {
+		t.Reset(timeout)
+	} else {
+		t = time.NewTimer(timeout)
+	}
+	defer timerPool.Put(t)
+
+	select {
+	case c.input <- event:
+		stopped := t.Stop()
+		if !stopped {
+			// Consume triggered (but not yet received) timer event
+			// so that future reuse does not get a spurious timeout.
+			<-t.C
+		}
+	case <-t.C:
+		// This means that we couldn't send event to that watcher.
+		// Since we don't want to block on it infinitely,
+		// we simply terminate it.
+		c.forget(false)
+		c.stop()
+	}
+
+	budget.returnUnused(timeout - time.Since(startTime))
+}
+
+// NOTE: sendWatchCacheEvent is assumed to not modify <event> !!!
+func (c *cacheWatcher) sendWatchCacheEvent(event *watchCacheEvent) {
+	curObjPasses := event.Type != watch.Deleted && c.filter(event.Key, event.ObjLabels, event.ObjFields, event.ObjUninitialized)
+	oldObjPasses := false
+	if event.PrevObject != nil {
+		oldObjPasses = c.filter(event.Key, event.PrevObjLabels, event.PrevObjFields, event.PrevObjUninitialized)
+	}
+	if !curObjPasses && !oldObjPasses {
+		// Watcher is not interested in that object.
+		return
+	}
+
+	var watchEvent watch.Event
+	switch {
+	case curObjPasses && !oldObjPasses:
+		watchEvent = watch.Event{Type: watch.Added, Object: event.Object.DeepCopyObject()}
+	case curObjPasses && oldObjPasses:
+		watchEvent = watch.Event{Type: watch.Modified, Object: event.Object.DeepCopyObject()}
+	case !curObjPasses && oldObjPasses:
+		watchEvent = watch.Event{Type: watch.Deleted, Object: event.PrevObject.DeepCopyObject()}
+	}
+
+	// We need to ensure that if we put event X to the c.result, all
+	// previous events were already put into it before, no matter whether
+	// c.done is close or not.
+	// Thus we cannot simply select from c.done and c.result and this
+	// would give us non-determinism.
+	// At the same time, we don't want to block infinitely on putting
+	// to c.result, when c.done is already closed.
+
+	// This ensures that with c.done already close, we at most once go
+	// into the next select after this. With that, no matter which
+	// statement we choose there, we will deliver only consecutive
+	// events.
+	select {
+	case <-c.done:
+		return
+	default:
+	}
+
+	select {
+	case c.result <- watchEvent:
+	case <-c.done:
+	}
+}
+
+func (c *cacheWatcher) process(initEvents []*watchCacheEvent, resourceVersion uint64) {
+	defer utilruntime.HandleCrash()
+
+	// Check how long we are processing initEvents.
+	// As long as these are not processed, we are not processing
+	// any incoming events, so if it takes long, we may actually
+	// block all watchers for some time.
+	// TODO: From the logs it seems that there happens processing
+	// times even up to 1s which is very long. However, this doesn't
+	// depend that much on the number of initEvents. E.g. from the
+	// 2000-node Kubemark run we have logs like this, e.g.:
+	// ... processing 13862 initEvents took 66.808689ms
+	// ... processing 14040 initEvents took 993.532539ms
+	// We should understand what is blocking us in those cases (e.g.
+	// is it lack of CPU, network, or sth else) and potentially
+	// consider increase size of result buffer in those cases.
+	const initProcessThreshold = 500 * time.Millisecond
+	startTime := time.Now()
+	for _, event := range initEvents {
+		c.sendWatchCacheEvent(event)
+	}
+	processingTime := time.Since(startTime)
+	if processingTime > initProcessThreshold {
+		objType := "<null>"
+		if len(initEvents) > 0 {
+			objType = reflect.TypeOf(initEvents[0].Object).String()
+		}
+		glog.V(2).Infof("processing %d initEvents of %s took %v", len(initEvents), objType, processingTime)
+	}
+
+	defer close(c.result)
+	defer c.Stop()
+	for {
+		event, ok := <-c.input
+		if !ok {
+			return
+		}
+		// only send events newer than resourceVersion
+		if event.ResourceVersion > resourceVersion {
+			c.sendWatchCacheEvent(event)
+		}
+	}
+}
+
+type ready struct {
+	ok bool
+	c  *sync.Cond
+}
+
+func newReady() *ready {
+	return &ready{c: sync.NewCond(&sync.Mutex{})}
+}
+
+func (r *ready) wait() {
+	r.c.L.Lock()
+	for !r.ok {
+		r.c.Wait()
+	}
+	r.c.L.Unlock()
+}
+
+// TODO: Make check() function more sophisticated, in particular
+// allow it to behave as "waitWithTimeout".
+func (r *ready) check() bool {
+	r.c.L.Lock()
+	defer r.c.L.Unlock()
+	return r.ok
+}
+
+func (r *ready) set(ok bool) {
+	r.c.L.Lock()
+	defer r.c.L.Unlock()
+	r.ok = ok
+	r.c.Broadcast()
+}

vendor/k8s.io/apiserver/pkg/storage/cacher_whitebox_test.go

@@ -0,0 +1,178 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+// verifies the cacheWatcher.process goroutine is properly cleaned up even if
+// the writes to cacheWatcher.result channel is blocked.
+func TestCacheWatcherCleanupNotBlockedByResult(t *testing.T) {
+	var lock sync.RWMutex
+	count := 0
+	filter := func(string, labels.Set, fields.Set, bool) bool { return true }
+	forget := func(bool) {
+		lock.Lock()
+		defer lock.Unlock()
+		count++
+	}
+	initEvents := []*watchCacheEvent{
+		{Object: &v1.Pod{}},
+		{Object: &v1.Pod{}},
+	}
+	// set the size of the buffer of w.result to 0, so that the writes to
+	// w.result is blocked.
+	w := newCacheWatcher(scheme.Scheme, 0, 0, initEvents, filter, forget)
+	w.Stop()
+	if err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) {
+		lock.RLock()
+		defer lock.RUnlock()
+		return count == 2, nil
+	}); err != nil {
+		t.Fatalf("expected forget() to be called twice, because sendWatchCacheEvent should not be blocked by the result channel: %v", err)
+	}
+}
+
+func TestCacheWatcherHandlesFiltering(t *testing.T) {
+	filter := func(_ string, _ labels.Set, field fields.Set, _ bool) bool {
+		return field["spec.nodeName"] == "host"
+	}
+	forget := func(bool) {}
+
+	testCases := []struct {
+		events   []*watchCacheEvent
+		expected []watch.Event
+	}{
+		// properly handle starting with the filter, then being deleted, then re-added
+		{
+			events: []*watchCacheEvent{
+				{
+					Type:      watch.Added,
+					Object:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}},
+					ObjFields: fields.Set{"spec.nodeName": "host"},
+				},
+				{
+					Type:          watch.Modified,
+					PrevObject:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}},
+					PrevObjFields: fields.Set{"spec.nodeName": "host"},
+					Object:        &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}},
+					ObjFields:     fields.Set{"spec.nodeName": ""},
+				},
+				{
+					Type:          watch.Modified,
+					PrevObject:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}},
+					PrevObjFields: fields.Set{"spec.nodeName": ""},
+					Object:        &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}},
+					ObjFields:     fields.Set{"spec.nodeName": "host"},
+				},
+			},
+			expected: []watch.Event{
+				{Type: watch.Added, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}}},
+				{Type: watch.Deleted, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}}},
+				{Type: watch.Added, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}}},
+			},
+		},
+		// properly handle ignoring changes prior to the filter, then getting added, then deleted
+		{
+			events: []*watchCacheEvent{
+				{
+					Type:      watch.Added,
+					Object:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}},
+					ObjFields: fields.Set{"spec.nodeName": ""},
+				},
+				{
+					Type:          watch.Modified,
+					PrevObject:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}},
+					PrevObjFields: fields.Set{"spec.nodeName": ""},
+					Object:        &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}},
+					ObjFields:     fields.Set{"spec.nodeName": ""},
+				},
+				{
+					Type:          watch.Modified,
+					PrevObject:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}},
+					PrevObjFields: fields.Set{"spec.nodeName": ""},
+					Object:        &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}},
+					ObjFields:     fields.Set{"spec.nodeName": "host"},
+				},
+				{
+					Type:          watch.Modified,
+					PrevObject:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}},
+					PrevObjFields: fields.Set{"spec.nodeName": "host"},
+					Object:        &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "4"}},
+					ObjFields:     fields.Set{"spec.nodeName": "host"},
+				},
+				{
+					Type:          watch.Modified,
+					PrevObject:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "4"}},
+					PrevObjFields: fields.Set{"spec.nodeName": "host"},
+					Object:        &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "5"}},
+					ObjFields:     fields.Set{"spec.nodeName": ""},
+				},
+				{
+					Type:          watch.Modified,
+					PrevObject:    &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "5"}},
+					PrevObjFields: fields.Set{"spec.nodeName": ""},
+					Object:        &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "6"}},
+					ObjFields:     fields.Set{"spec.nodeName": ""},
+				},
+			},
+			expected: []watch.Event{
+				{Type: watch.Added, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}}},
+				{Type: watch.Modified, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "4"}}},
+				{Type: watch.Deleted, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "4"}}},
+			},
+		},
+	}
+
+TestCase:
+	for i, testCase := range testCases {
+		// set the size of the buffer of w.result to 0, so that the writes to
+		// w.result is blocked.
+		for j := range testCase.events {
+			testCase.events[j].ResourceVersion = uint64(j) + 1
+		}
+		w := newCacheWatcher(scheme.Scheme, 0, 0, testCase.events, filter, forget)
+		ch := w.ResultChan()
+		for j, event := range testCase.expected {
+			e := <-ch
+			if !reflect.DeepEqual(event, e) {
+				t.Errorf("%d: unexpected event %d: %s", i, j, diff.ObjectReflectDiff(event, e))
+				break TestCase
+			}
+		}
+		select {
+		case obj, ok := <-ch:
+			t.Errorf("%d: unexpected excess event: %#v %t", i, obj, ok)
+			break TestCase
+		default:
+		}
+		w.Stop()
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Interfaces for database-related operations.
+package storage // import "k8s.io/apiserver/pkg/storage"

vendor/k8s.io/apiserver/pkg/storage/errors.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+const (
+	ErrCodeKeyNotFound int = iota + 1
+	ErrCodeKeyExists
+	ErrCodeResourceVersionConflicts
+	ErrCodeInvalidObj
+	ErrCodeUnreachable
+)
+
+var errCodeToMessage = map[int]string{
+	ErrCodeKeyNotFound:              "key not found",
+	ErrCodeKeyExists:                "key exists",
+	ErrCodeResourceVersionConflicts: "resource version conflicts",
+	ErrCodeInvalidObj:               "invalid object",
+	ErrCodeUnreachable:              "server unreachable",
+}
+
+func NewKeyNotFoundError(key string, rv int64) *StorageError {
+	return &StorageError{
+		Code:            ErrCodeKeyNotFound,
+		Key:             key,
+		ResourceVersion: rv,
+	}
+}
+
+func NewKeyExistsError(key string, rv int64) *StorageError {
+	return &StorageError{
+		Code:            ErrCodeKeyExists,
+		Key:             key,
+		ResourceVersion: rv,
+	}
+}
+
+func NewResourceVersionConflictsError(key string, rv int64) *StorageError {
+	return &StorageError{
+		Code:            ErrCodeResourceVersionConflicts,
+		Key:             key,
+		ResourceVersion: rv,
+	}
+}
+
+func NewUnreachableError(key string, rv int64) *StorageError {
+	return &StorageError{
+		Code:            ErrCodeUnreachable,
+		Key:             key,
+		ResourceVersion: rv,
+	}
+}
+
+func NewInvalidObjError(key, msg string) *StorageError {
+	return &StorageError{
+		Code:               ErrCodeInvalidObj,
+		Key:                key,
+		AdditionalErrorMsg: msg,
+	}
+}
+
+type StorageError struct {
+	Code               int
+	Key                string
+	ResourceVersion    int64
+	AdditionalErrorMsg string
+}
+
+func (e *StorageError) Error() string {
+	return fmt.Sprintf("StorageError: %s, Code: %d, Key: %s, ResourceVersion: %d, AdditionalErrorMsg: %s",
+		errCodeToMessage[e.Code], e.Code, e.Key, e.ResourceVersion, e.AdditionalErrorMsg)
+}
+
+// IsNotFound returns true if and only if err is "key" not found error.
+func IsNotFound(err error) bool {
+	return isErrCode(err, ErrCodeKeyNotFound)
+}
+
+// IsNodeExist returns true if and only if err is an node already exist error.
+func IsNodeExist(err error) bool {
+	return isErrCode(err, ErrCodeKeyExists)
+}
+
+// IsUnreachable returns true if and only if err indicates the server could not be reached.
+func IsUnreachable(err error) bool {
+	return isErrCode(err, ErrCodeUnreachable)
+}
+
+// IsConflict returns true if and only if err is a write conflict.
+func IsConflict(err error) bool {
+	return isErrCode(err, ErrCodeResourceVersionConflicts)
+}
+
+// IsInvalidObj returns true if and only if err is invalid error
+func IsInvalidObj(err error) bool {
+	return isErrCode(err, ErrCodeInvalidObj)
+}
+
+func isErrCode(err error, code int) bool {
+	if err == nil {
+		return false
+	}
+	if e, ok := err.(*StorageError); ok {
+		return e.Code == code
+	}
+	return false
+}
+
+// InvalidError is generated when an error caused by invalid API object occurs
+// in the storage package.
+type InvalidError struct {
+	Errs field.ErrorList
+}
+
+func (e InvalidError) Error() string {
+	return e.Errs.ToAggregate().Error()
+}
+
+// IsInvalidError returns true if and only if err is an InvalidError.
+func IsInvalidError(err error) bool {
+	_, ok := err.(InvalidError)
+	return ok
+}
+
+func NewInvalidError(errors field.ErrorList) InvalidError {
+	return InvalidError{errors}
+}
+
+// InternalError is generated when an error occurs in the storage package, i.e.,
+// not from the underlying storage backend (e.g., etcd).
+type InternalError struct {
+	Reason string
+}
+
+func (e InternalError) Error() string {
+	return e.Reason
+}
+
+// IsInternalError returns true if and only if err is an InternalError.
+func IsInternalError(err error) bool {
+	_, ok := err.(InternalError)
+	return ok
+}
+
+func NewInternalError(reason string) InternalError {
+	return InternalError{reason}
+}
+
+func NewInternalErrorf(format string, a ...interface{}) InternalError {
+	return InternalError{fmt.Sprintf(format, a)}
+}

vendor/k8s.io/apiserver/pkg/storage/errors/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "storage.go",
+    ],
+    deps = [
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/errors/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package etcd provides conversion of etcd errors to API errors.
+package storage // import "k8s.io/apiserver/pkg/storage/errors"

vendor/k8s.io/apiserver/pkg/storage/errors/storage.go

@@ -0,0 +1,116 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/storage"
+)
+
+// InterpretListError converts a generic error on a retrieval
+// operation into the appropriate API error.
+func InterpretListError(err error, qualifiedResource schema.GroupResource) error {
+	switch {
+	case storage.IsNotFound(err):
+		return errors.NewNotFound(qualifiedResource, "")
+	case storage.IsUnreachable(err):
+		return errors.NewServerTimeout(qualifiedResource, "list", 2) // TODO: make configurable or handled at a higher level
+	case storage.IsInternalError(err):
+		return errors.NewInternalError(err)
+	default:
+		return err
+	}
+}
+
+// InterpretGetError converts a generic error on a retrieval
+// operation into the appropriate API error.
+func InterpretGetError(err error, qualifiedResource schema.GroupResource, name string) error {
+	switch {
+	case storage.IsNotFound(err):
+		return errors.NewNotFound(qualifiedResource, name)
+	case storage.IsUnreachable(err):
+		return errors.NewServerTimeout(qualifiedResource, "get", 2) // TODO: make configurable or handled at a higher level
+	case storage.IsInternalError(err):
+		return errors.NewInternalError(err)
+	default:
+		return err
+	}
+}
+
+// InterpretCreateError converts a generic error on a create
+// operation into the appropriate API error.
+func InterpretCreateError(err error, qualifiedResource schema.GroupResource, name string) error {
+	switch {
+	case storage.IsNodeExist(err):
+		return errors.NewAlreadyExists(qualifiedResource, name)
+	case storage.IsUnreachable(err):
+		return errors.NewServerTimeout(qualifiedResource, "create", 2) // TODO: make configurable or handled at a higher level
+	case storage.IsInternalError(err):
+		return errors.NewInternalError(err)
+	default:
+		return err
+	}
+}
+
+// InterpretUpdateError converts a generic error on an update
+// operation into the appropriate API error.
+func InterpretUpdateError(err error, qualifiedResource schema.GroupResource, name string) error {
+	switch {
+	case storage.IsConflict(err), storage.IsNodeExist(err), storage.IsInvalidObj(err):
+		return errors.NewConflict(qualifiedResource, name, err)
+	case storage.IsUnreachable(err):
+		return errors.NewServerTimeout(qualifiedResource, "update", 2) // TODO: make configurable or handled at a higher level
+	case storage.IsNotFound(err):
+		return errors.NewNotFound(qualifiedResource, name)
+	case storage.IsInternalError(err):
+		return errors.NewInternalError(err)
+	default:
+		return err
+	}
+}
+
+// InterpretDeleteError converts a generic error on a delete
+// operation into the appropriate API error.
+func InterpretDeleteError(err error, qualifiedResource schema.GroupResource, name string) error {
+	switch {
+	case storage.IsNotFound(err):
+		return errors.NewNotFound(qualifiedResource, name)
+	case storage.IsUnreachable(err):
+		return errors.NewServerTimeout(qualifiedResource, "delete", 2) // TODO: make configurable or handled at a higher level
+	case storage.IsConflict(err), storage.IsNodeExist(err), storage.IsInvalidObj(err):
+		return errors.NewConflict(qualifiedResource, name, err)
+	case storage.IsInternalError(err):
+		return errors.NewInternalError(err)
+	default:
+		return err
+	}
+}
+
+// InterpretWatchError converts a generic error on a watch
+// operation into the appropriate API error.
+func InterpretWatchError(err error, resource schema.GroupResource, name string) error {
+	switch {
+	case storage.IsInvalidError(err):
+		invalidError, _ := err.(storage.InvalidError)
+		return errors.NewInvalid(schema.GroupKind{Group: resource.Group, Kind: resource.Resource}, name, invalidError.Errs)
+	case storage.IsInternalError(err):
+		return errors.NewInternalError(err)
+	default:
+		return err
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/BUILD

@@ -0,0 +1,84 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "api_object_versioner_test.go",
+        "etcd_helper_test.go",
+        "etcd_watcher_test.go",
+    ],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/coreos/etcd/client:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/testing:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/etcdtest:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/testing:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/tests:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api_object_versioner.go",
+        "doc.go",
+        "etcd_helper.go",
+        "etcd_watcher.go",
+    ],
+    deps = [
+        "//vendor/github.com/coreos/etcd/client:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/cache:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/metrics:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/util:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd/etcdtest:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd/metrics:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd/util:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/etcd/OWNERS

@@ -0,0 +1,25 @@
+reviewers:
+- lavalamp
+- smarterclayton
+- wojtek-t
+- deads2k
+- derekwaynecarr
+- caesarxuchao
+- mikedanese
+- liggitt
+- davidopp
+- pmorie
+- luxas
+- janetkuo
+- roberthbailey
+- tallclair
+- timothysc
+- dims
+- hongchaodeng
+- krousey
+- fgrzadkowski
+- resouer
+- pweil-
+- mqliang
+- feihujiang
+- enj

vendor/k8s.io/apiserver/pkg/storage/etcd/api_object_versioner.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd
+
+import (
+	"strconv"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/storage"
+)
+
+// APIObjectVersioner implements versioning and extracting etcd node information
+// for objects that have an embedded ObjectMeta or ListMeta field.
+type APIObjectVersioner struct{}
+
+// UpdateObject implements Versioner
+func (a APIObjectVersioner) UpdateObject(obj runtime.Object, resourceVersion uint64) error {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	versionString := ""
+	if resourceVersion != 0 {
+		versionString = strconv.FormatUint(resourceVersion, 10)
+	}
+	accessor.SetResourceVersion(versionString)
+	return nil
+}
+
+// UpdateList implements Versioner
+func (a APIObjectVersioner) UpdateList(obj runtime.Object, resourceVersion uint64, nextKey string) error {
+	listAccessor, err := meta.ListAccessor(obj)
+	if err != nil || listAccessor == nil {
+		return err
+	}
+	versionString := ""
+	if resourceVersion != 0 {
+		versionString = strconv.FormatUint(resourceVersion, 10)
+	}
+	listAccessor.SetResourceVersion(versionString)
+	listAccessor.SetContinue(nextKey)
+	return nil
+}
+
+// PrepareObjectForStorage clears resource version and self link prior to writing to etcd.
+func (a APIObjectVersioner) PrepareObjectForStorage(obj runtime.Object) error {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	accessor.SetResourceVersion("")
+	accessor.SetSelfLink("")
+	return nil
+}
+
+// ObjectResourceVersion implements Versioner
+func (a APIObjectVersioner) ObjectResourceVersion(obj runtime.Object) (uint64, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return 0, err
+	}
+	version := accessor.GetResourceVersion()
+	if len(version) == 0 {
+		return 0, nil
+	}
+	return strconv.ParseUint(version, 10, 64)
+}
+
+// APIObjectVersioner implements Versioner
+var Versioner storage.Versioner = APIObjectVersioner{}
+
+// CompareResourceVersion compares etcd resource versions.  Outside this API they are all strings,
+// but etcd resource versions are special, they're actually ints, so we can easily compare them.
+func (a APIObjectVersioner) CompareResourceVersion(lhs, rhs runtime.Object) int {
+	lhsVersion, err := Versioner.ObjectResourceVersion(lhs)
+	if err != nil {
+		// coder error
+		panic(err)
+	}
+	rhsVersion, err := Versioner.ObjectResourceVersion(rhs)
+	if err != nil {
+		// coder error
+		panic(err)
+	}
+
+	if lhsVersion == rhsVersion {
+		return 0
+	}
+	if lhsVersion < rhsVersion {
+		return -1
+	}
+
+	return 1
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/api_object_versioner_test.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd
+
+import (
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	storagetesting "k8s.io/apiserver/pkg/storage/testing"
+)
+
+func TestObjectVersioner(t *testing.T) {
+	v := APIObjectVersioner{}
+	if ver, err := v.ObjectResourceVersion(&storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "5"}}); err != nil || ver != 5 {
+		t.Errorf("unexpected version: %d %v", ver, err)
+	}
+	if ver, err := v.ObjectResourceVersion(&storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "a"}}); err == nil || ver != 0 {
+		t.Errorf("unexpected version: %d %v", ver, err)
+	}
+	obj := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "a"}}
+	if err := v.UpdateObject(obj, 5); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if obj.ResourceVersion != "5" || obj.DeletionTimestamp != nil {
+		t.Errorf("unexpected resource version: %#v", obj)
+	}
+}
+
+func TestCompareResourceVersion(t *testing.T) {
+	five := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "5"}}
+	six := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "6"}}
+
+	versioner := APIObjectVersioner{}
+
+	if e, a := -1, versioner.CompareResourceVersion(five, six); e != a {
+		t.Errorf("expected %v got %v", e, a)
+	}
+	if e, a := 1, versioner.CompareResourceVersion(six, five); e != a {
+		t.Errorf("expected %v got %v", e, a)
+	}
+	if e, a := 0, versioner.CompareResourceVersion(six, six); e != a {
+		t.Errorf("expected %v got %v", e, a)
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/doc.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd // import "k8s.io/apiserver/pkg/storage/etcd"

vendor/k8s.io/apiserver/pkg/storage/etcd/etcd_helper.go

@@ -0,0 +1,651 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd
+
+import (
+	"errors"
+	"fmt"
+	"path"
+	"reflect"
+	"time"
+
+	etcd "github.com/coreos/etcd/client"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilcache "k8s.io/apimachinery/pkg/util/cache"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/etcd/metrics"
+	etcdutil "k8s.io/apiserver/pkg/storage/etcd/util"
+	utiltrace "k8s.io/apiserver/pkg/util/trace"
+)
+
+// ValueTransformer allows a string value to be transformed before being read from or written to the underlying store. The methods
+// must be able to undo the transformation caused by the other.
+type ValueTransformer interface {
+	// TransformStringFromStorage may transform the provided string from its underlying storage representation or return an error.
+	// Stale is true if the object on disk is stale and a write to etcd should be issued, even if the contents of the object
+	// have not changed.
+	TransformStringFromStorage(string) (value string, stale bool, err error)
+	// TransformStringToStorage may transform the provided string into the appropriate form in storage or return an error.
+	TransformStringToStorage(string) (value string, err error)
+}
+
+type identityTransformer struct{}
+
+func (identityTransformer) TransformStringFromStorage(s string) (string, bool, error) {
+	return s, false, nil
+}
+func (identityTransformer) TransformStringToStorage(s string) (string, error) { return s, nil }
+
+// IdentityTransformer performs no transformation on the provided values.
+var IdentityTransformer ValueTransformer = identityTransformer{}
+
+// Creates a new storage interface from the client
+// TODO: deprecate in favor of storage.Config abstraction over time
+func NewEtcdStorage(client etcd.Client, codec runtime.Codec, prefix string, quorum bool, cacheSize int, copier runtime.ObjectCopier, transformer ValueTransformer) storage.Interface {
+	return &etcdHelper{
+		etcdMembersAPI: etcd.NewMembersAPI(client),
+		etcdKeysAPI:    etcd.NewKeysAPI(client),
+		codec:          codec,
+		versioner:      APIObjectVersioner{},
+		copier:         copier,
+		transformer:    transformer,
+		pathPrefix:     path.Join("/", prefix),
+		quorum:         quorum,
+		cache:          utilcache.NewCache(cacheSize),
+	}
+}
+
+// etcdHelper is the reference implementation of storage.Interface.
+type etcdHelper struct {
+	etcdMembersAPI etcd.MembersAPI
+	etcdKeysAPI    etcd.KeysAPI
+	codec          runtime.Codec
+	copier         runtime.ObjectCopier
+	transformer    ValueTransformer
+	// Note that versioner is required for etcdHelper to work correctly.
+	// The public constructors (NewStorage & NewEtcdStorage) are setting it
+	// correctly, so be careful when manipulating with it manually.
+	// optional, has to be set to perform any atomic operations
+	versioner storage.Versioner
+	// prefix for all etcd keys
+	pathPrefix string
+	// if true,  perform quorum read
+	quorum bool
+
+	// We cache objects stored in etcd. For keys we use Node.ModifiedIndex which is equivalent
+	// to resourceVersion.
+	// This depends on etcd's indexes being globally unique across all objects/types. This will
+	// have to revisited if we decide to do things like multiple etcd clusters, or etcd will
+	// support multi-object transaction that will result in many objects with the same index.
+	// Number of entries stored in the cache is controlled by maxEtcdCacheEntries constant.
+	// TODO: Measure how much this cache helps after the conversion code is optimized.
+	cache utilcache.Cache
+}
+
+func init() {
+	metrics.Register()
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) Versioner() storage.Versioner {
+	return h.versioner
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error {
+	trace := utiltrace.New("etcdHelper::Create " + getTypeName(obj))
+	defer trace.LogIfLong(250 * time.Millisecond)
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	key = path.Join(h.pathPrefix, key)
+	data, err := runtime.Encode(h.codec, obj)
+	trace.Step("Object encoded")
+	if err != nil {
+		return err
+	}
+	if version, err := h.versioner.ObjectResourceVersion(obj); err == nil && version != 0 {
+		return errors.New("resourceVersion may not be set on objects to be created")
+	}
+	if err := h.versioner.PrepareObjectForStorage(obj); err != nil {
+		return fmt.Errorf("PrepareObjectForStorage returned an error: %v", err)
+	}
+	trace.Step("Version checked")
+
+	startTime := time.Now()
+	opts := etcd.SetOptions{
+		TTL:       time.Duration(ttl) * time.Second,
+		PrevExist: etcd.PrevNoExist,
+	}
+
+	newBody, err := h.transformer.TransformStringToStorage(string(data))
+	if err != nil {
+		return storage.NewInternalError(err.Error())
+	}
+
+	response, err := h.etcdKeysAPI.Set(ctx, key, newBody, &opts)
+	trace.Step("Object created")
+	metrics.RecordEtcdRequestLatency("create", getTypeName(obj), startTime)
+	if err != nil {
+		return toStorageErr(err, key, 0)
+	}
+	if out != nil {
+		if _, err := conversion.EnforcePtr(out); err != nil {
+			panic("unable to convert output object to pointer")
+		}
+		_, _, _, err = h.extractObj(response, err, out, false, false)
+	}
+	return err
+}
+
+func checkPreconditions(key string, preconditions *storage.Preconditions, out runtime.Object) error {
+	if preconditions == nil {
+		return nil
+	}
+	objMeta, err := meta.Accessor(out)
+	if err != nil {
+		return storage.NewInternalErrorf("can't enforce preconditions %v on un-introspectable object %v, got error: %v", *preconditions, out, err)
+	}
+	if preconditions.UID != nil && *preconditions.UID != objMeta.GetUID() {
+		errMsg := fmt.Sprintf("Precondition failed: UID in precondition: %v, UID in object meta: %v", preconditions.UID, objMeta.GetUID())
+		return storage.NewInvalidObjError(key, errMsg)
+	}
+	return nil
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions) error {
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	key = path.Join(h.pathPrefix, key)
+	v, err := conversion.EnforcePtr(out)
+	if err != nil {
+		panic("unable to convert output object to pointer")
+	}
+
+	if preconditions == nil {
+		startTime := time.Now()
+		response, err := h.etcdKeysAPI.Delete(ctx, key, nil)
+		metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime)
+		if !etcdutil.IsEtcdNotFound(err) {
+			// if the object that existed prior to the delete is returned by etcd, update the out object.
+			if err != nil || response.PrevNode != nil {
+				_, _, _, err = h.extractObj(response, err, out, false, true)
+			}
+		}
+		return toStorageErr(err, key, 0)
+	}
+
+	// Check the preconditions match.
+	obj := reflect.New(v.Type()).Interface().(runtime.Object)
+	for {
+		_, node, res, _, err := h.bodyAndExtractObj(ctx, key, obj, false)
+		if err != nil {
+			return toStorageErr(err, key, 0)
+		}
+		if err := checkPreconditions(key, preconditions, obj); err != nil {
+			return toStorageErr(err, key, 0)
+		}
+		index := uint64(0)
+		if node != nil {
+			index = node.ModifiedIndex
+		} else if res != nil {
+			index = res.Index
+		}
+		opt := etcd.DeleteOptions{PrevIndex: index}
+		startTime := time.Now()
+		response, err := h.etcdKeysAPI.Delete(ctx, key, &opt)
+		metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime)
+		if !etcdutil.IsEtcdTestFailed(err) {
+			if !etcdutil.IsEtcdNotFound(err) {
+				// if the object that existed prior to the delete is returned by etcd, update the out object.
+				if err != nil || response.PrevNode != nil {
+					_, _, _, err = h.extractObj(response, err, out, false, true)
+				}
+			}
+			return toStorageErr(err, key, 0)
+		}
+
+		glog.V(4).Infof("deletion of %s failed because of a conflict, going to retry", key)
+	}
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) Watch(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate) (watch.Interface, error) {
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	watchRV, err := storage.ParseWatchResourceVersion(resourceVersion)
+	if err != nil {
+		return nil, err
+	}
+	key = path.Join(h.pathPrefix, key)
+	w := newEtcdWatcher(false, h.quorum, nil, storage.SimpleFilter(pred), h.codec, h.versioner, nil, h.transformer, h)
+	go w.etcdWatch(ctx, h.etcdKeysAPI, key, watchRV)
+	return w, nil
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) WatchList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate) (watch.Interface, error) {
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	watchRV, err := storage.ParseWatchResourceVersion(resourceVersion)
+	if err != nil {
+		return nil, err
+	}
+	key = path.Join(h.pathPrefix, key)
+	w := newEtcdWatcher(true, h.quorum, exceptKey(key), storage.SimpleFilter(pred), h.codec, h.versioner, nil, h.transformer, h)
+	go w.etcdWatch(ctx, h.etcdKeysAPI, key, watchRV)
+	return w, nil
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) Get(ctx context.Context, key string, resourceVersion string, objPtr runtime.Object, ignoreNotFound bool) error {
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	key = path.Join(h.pathPrefix, key)
+	_, _, _, _, err := h.bodyAndExtractObj(ctx, key, objPtr, ignoreNotFound)
+	return err
+}
+
+// bodyAndExtractObj performs the normal Get path to etcd, returning the parsed node and response for additional information
+// about the response, like the current etcd index and the ttl.
+func (h *etcdHelper) bodyAndExtractObj(ctx context.Context, key string, objPtr runtime.Object, ignoreNotFound bool) (body string, node *etcd.Node, res *etcd.Response, stale bool, err error) {
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	startTime := time.Now()
+
+	opts := &etcd.GetOptions{
+		Quorum: h.quorum,
+	}
+
+	response, err := h.etcdKeysAPI.Get(ctx, key, opts)
+	metrics.RecordEtcdRequestLatency("get", getTypeName(objPtr), startTime)
+	if err != nil && !etcdutil.IsEtcdNotFound(err) {
+		return "", nil, nil, false, toStorageErr(err, key, 0)
+	}
+	body, node, stale, err = h.extractObj(response, err, objPtr, ignoreNotFound, false)
+	return body, node, response, stale, toStorageErr(err, key, 0)
+}
+
+func (h *etcdHelper) extractObj(response *etcd.Response, inErr error, objPtr runtime.Object, ignoreNotFound, prevNode bool) (body string, node *etcd.Node, stale bool, err error) {
+	if response != nil {
+		if prevNode {
+			node = response.PrevNode
+		} else {
+			node = response.Node
+		}
+	}
+	if inErr != nil || node == nil || len(node.Value) == 0 {
+		if ignoreNotFound {
+			v, err := conversion.EnforcePtr(objPtr)
+			if err != nil {
+				return "", nil, false, err
+			}
+			v.Set(reflect.Zero(v.Type()))
+			return "", nil, false, nil
+		} else if inErr != nil {
+			return "", nil, false, inErr
+		}
+		return "", nil, false, fmt.Errorf("unable to locate a value on the response: %#v", response)
+	}
+
+	body, stale, err = h.transformer.TransformStringFromStorage(node.Value)
+	if err != nil {
+		return body, nil, stale, storage.NewInternalError(err.Error())
+	}
+	out, gvk, err := h.codec.Decode([]byte(body), nil, objPtr)
+	if err != nil {
+		return body, nil, stale, err
+	}
+	if out != objPtr {
+		return body, nil, stale, fmt.Errorf("unable to decode object %s into %v", gvk.String(), reflect.TypeOf(objPtr))
+	}
+	// being unable to set the version does not prevent the object from being extracted
+	_ = h.versioner.UpdateObject(objPtr, node.ModifiedIndex)
+	return body, node, stale, err
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) GetToList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	trace := utiltrace.New("GetToList " + getTypeName(listObj))
+	listPtr, err := meta.GetItemsPtr(listObj)
+	if err != nil {
+		return err
+	}
+	key = path.Join(h.pathPrefix, key)
+	startTime := time.Now()
+	trace.Step("About to read etcd node")
+
+	opts := &etcd.GetOptions{
+		Quorum: h.quorum,
+	}
+	response, err := h.etcdKeysAPI.Get(ctx, key, opts)
+	trace.Step("Etcd node read")
+	metrics.RecordEtcdRequestLatency("get", getTypeName(listPtr), startTime)
+	if err != nil {
+		if etcdutil.IsEtcdNotFound(err) {
+			return nil
+		}
+		return toStorageErr(err, key, 0)
+	}
+
+	nodes := make([]*etcd.Node, 0)
+	nodes = append(nodes, response.Node)
+
+	if err := h.decodeNodeList(nodes, storage.SimpleFilter(pred), listPtr); err != nil {
+		return err
+	}
+	trace.Step("Object decoded")
+	if err := h.versioner.UpdateList(listObj, response.Index, ""); err != nil {
+		return err
+	}
+	return nil
+}
+
+// decodeNodeList walks the tree of each node in the list and decodes into the specified object
+func (h *etcdHelper) decodeNodeList(nodes []*etcd.Node, filter storage.FilterFunc, slicePtr interface{}) error {
+	trace := utiltrace.New("decodeNodeList " + getTypeName(slicePtr))
+	defer trace.LogIfLong(400 * time.Millisecond)
+	v, err := conversion.EnforcePtr(slicePtr)
+	if err != nil || v.Kind() != reflect.Slice {
+		// This should not happen at runtime.
+		panic("need ptr to slice")
+	}
+	for _, node := range nodes {
+		if node.Dir {
+			// IMPORTANT: do not log each key as a discrete step in the trace log
+			// as it produces an immense amount of log spam when there is a large
+			// amount of content in the list.
+			if err := h.decodeNodeList(node.Nodes, filter, slicePtr); err != nil {
+				return err
+			}
+			continue
+		}
+		if obj, found := h.getFromCache(node.ModifiedIndex, filter); found {
+			// obj != nil iff it matches the filter function.
+			if obj != nil {
+				v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
+			}
+		} else {
+			body, _, err := h.transformer.TransformStringFromStorage(node.Value)
+			if err != nil {
+				// omit items from lists and watches that cannot be transformed, but log the error
+				utilruntime.HandleError(fmt.Errorf("unable to transform key %q: %v", node.Key, err))
+				continue
+			}
+
+			obj, _, err := h.codec.Decode([]byte(body), nil, reflect.New(v.Type().Elem()).Interface().(runtime.Object))
+			if err != nil {
+				return err
+			}
+			// being unable to set the version does not prevent the object from being extracted
+			_ = h.versioner.UpdateObject(obj, node.ModifiedIndex)
+			if filter(obj) {
+				v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
+			}
+			if node.ModifiedIndex != 0 {
+				h.addToCache(node.ModifiedIndex, obj)
+			}
+		}
+	}
+	trace.Step(fmt.Sprintf("Decoded %v nodes", len(nodes)))
+	return nil
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) List(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	trace := utiltrace.New("List " + getTypeName(listObj))
+	defer trace.LogIfLong(400 * time.Millisecond)
+	listPtr, err := meta.GetItemsPtr(listObj)
+	if err != nil {
+		return err
+	}
+	key = path.Join(h.pathPrefix, key)
+	startTime := time.Now()
+	trace.Step("About to list etcd node")
+	nodes, index, err := h.listEtcdNode(ctx, key)
+	trace.Step("Etcd node listed")
+	metrics.RecordEtcdRequestLatency("list", getTypeName(listPtr), startTime)
+	if err != nil {
+		return err
+	}
+	if err := h.decodeNodeList(nodes, storage.SimpleFilter(pred), listPtr); err != nil {
+		return err
+	}
+	trace.Step("Node list decoded")
+	if err := h.versioner.UpdateList(listObj, index, ""); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (h *etcdHelper) listEtcdNode(ctx context.Context, key string) ([]*etcd.Node, uint64, error) {
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	opts := etcd.GetOptions{
+		Recursive: true,
+		Sort:      true,
+		Quorum:    h.quorum,
+	}
+	result, err := h.etcdKeysAPI.Get(ctx, key, &opts)
+	if err != nil {
+		var index uint64
+		if etcdError, ok := err.(etcd.Error); ok {
+			index = etcdError.Index
+		}
+		nodes := make([]*etcd.Node, 0)
+		if etcdutil.IsEtcdNotFound(err) {
+			return nodes, index, nil
+		} else {
+			return nodes, index, toStorageErr(err, key, 0)
+		}
+	}
+	return result.Node.Nodes, result.Index, nil
+}
+
+// Implements storage.Interface.
+func (h *etcdHelper) GuaranteedUpdate(
+	ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
+	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, _ ...runtime.Object) error {
+	// Ignore the suggestion about current object.
+	if ctx == nil {
+		glog.Errorf("Context is nil")
+	}
+	v, err := conversion.EnforcePtr(ptrToType)
+	if err != nil {
+		// Panic is appropriate, because this is a programming error.
+		panic("need ptr to type")
+	}
+	key = path.Join(h.pathPrefix, key)
+	for {
+		obj := reflect.New(v.Type()).Interface().(runtime.Object)
+		origBody, node, res, stale, err := h.bodyAndExtractObj(ctx, key, obj, ignoreNotFound)
+		if err != nil {
+			return toStorageErr(err, key, 0)
+		}
+		if err := checkPreconditions(key, preconditions, obj); err != nil {
+			return toStorageErr(err, key, 0)
+		}
+		meta := storage.ResponseMeta{}
+		if node != nil {
+			meta.TTL = node.TTL
+			meta.ResourceVersion = node.ModifiedIndex
+		}
+		// Get the object to be written by calling tryUpdate.
+		ret, newTTL, err := tryUpdate(obj, meta)
+		if err != nil {
+			return toStorageErr(err, key, 0)
+		}
+
+		index := uint64(0)
+		ttl := uint64(0)
+		if node != nil {
+			index = node.ModifiedIndex
+			if node.TTL != 0 {
+				ttl = uint64(node.TTL)
+			}
+			if node.Expiration != nil && ttl == 0 {
+				ttl = 1
+			}
+		} else if res != nil {
+			index = res.Index
+		}
+
+		if newTTL != nil {
+			if ttl != 0 && *newTTL == 0 {
+				// TODO: remove this after we have verified this is no longer an issue
+				glog.V(4).Infof("GuaranteedUpdate is clearing TTL for %q, may not be intentional", key)
+			}
+			ttl = *newTTL
+		}
+
+		// Since update object may have a resourceVersion set, we need to clear it here.
+		if err := h.versioner.PrepareObjectForStorage(ret); err != nil {
+			return errors.New("resourceVersion cannot be set on objects store in etcd")
+		}
+
+		newBodyData, err := runtime.Encode(h.codec, ret)
+		if err != nil {
+			return err
+		}
+		newBody := string(newBodyData)
+		data, err := h.transformer.TransformStringToStorage(newBody)
+		if err != nil {
+			return storage.NewInternalError(err.Error())
+		}
+
+		// First time this key has been used, try creating new value.
+		if index == 0 {
+			startTime := time.Now()
+			opts := etcd.SetOptions{
+				TTL:       time.Duration(ttl) * time.Second,
+				PrevExist: etcd.PrevNoExist,
+			}
+			response, err := h.etcdKeysAPI.Set(ctx, key, data, &opts)
+			metrics.RecordEtcdRequestLatency("create", getTypeName(ptrToType), startTime)
+			if etcdutil.IsEtcdNodeExist(err) {
+				continue
+			}
+			_, _, _, err = h.extractObj(response, err, ptrToType, false, false)
+			return toStorageErr(err, key, 0)
+		}
+
+		// If we don't send an update, we simply return the currently existing
+		// version of the object. However, the value transformer may indicate that
+		// the on disk representation has changed and that we must commit an update.
+		if newBody == origBody && !stale {
+			_, _, _, err := h.extractObj(res, nil, ptrToType, ignoreNotFound, false)
+			return err
+		}
+
+		startTime := time.Now()
+		// Swap origBody with data, if origBody is the latest etcd data.
+		opts := etcd.SetOptions{
+			PrevIndex: index,
+			TTL:       time.Duration(ttl) * time.Second,
+		}
+		response, err := h.etcdKeysAPI.Set(ctx, key, data, &opts)
+		metrics.RecordEtcdRequestLatency("compareAndSwap", getTypeName(ptrToType), startTime)
+		if etcdutil.IsEtcdTestFailed(err) {
+			// Try again.
+			continue
+		}
+		_, _, _, err = h.extractObj(response, err, ptrToType, false, false)
+		return toStorageErr(err, key, int64(index))
+	}
+}
+
+// etcdCache defines interface used for caching objects stored in etcd. Objects are keyed by
+// their Node.ModifiedIndex, which is unique across all types.
+// All implementations must be thread-safe.
+type etcdCache interface {
+	getFromCache(index uint64, filter storage.FilterFunc) (runtime.Object, bool)
+	addToCache(index uint64, obj runtime.Object)
+}
+
+func getTypeName(obj interface{}) string {
+	return reflect.TypeOf(obj).String()
+}
+
+func (h *etcdHelper) getFromCache(index uint64, filter storage.FilterFunc) (runtime.Object, bool) {
+	startTime := time.Now()
+	defer func() {
+		metrics.ObserveGetCache(startTime)
+	}()
+	obj, found := h.cache.Get(index)
+	if found {
+		if !filter(obj.(runtime.Object)) {
+			return nil, true
+		}
+		// We should not return the object itself to avoid polluting the cache if someone
+		// modifies returned values.
+		objCopy := obj.(runtime.Object).DeepCopyObject()
+		metrics.ObserveCacheHit()
+		return objCopy.(runtime.Object), true
+	}
+	metrics.ObserveCacheMiss()
+	return nil, false
+}
+
+func (h *etcdHelper) addToCache(index uint64, obj runtime.Object) {
+	startTime := time.Now()
+	defer func() {
+		metrics.ObserveAddCache(startTime)
+	}()
+	objCopy := obj.DeepCopyObject()
+	isOverwrite := h.cache.Add(index, objCopy)
+	if !isOverwrite {
+		metrics.ObserveNewEntry()
+	}
+}
+
+func toStorageErr(err error, key string, rv int64) error {
+	if err == nil {
+		return nil
+	}
+	switch {
+	case etcdutil.IsEtcdNotFound(err):
+		return storage.NewKeyNotFoundError(key, rv)
+	case etcdutil.IsEtcdNodeExist(err):
+		return storage.NewKeyExistsError(key, rv)
+	case etcdutil.IsEtcdTestFailed(err):
+		return storage.NewResourceVersionConflictsError(key, rv)
+	case etcdutil.IsEtcdUnreachable(err):
+		return storage.NewUnreachableError(key, rv)
+	default:
+		return err
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/etcd_helper_test.go

@@ -0,0 +1,695 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd
+
+import (
+	"fmt"
+	"path"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	etcd "github.com/coreos/etcd/client"
+	"golang.org/x/net/context"
+	apitesting "k8s.io/apimachinery/pkg/api/testing"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/apiserver/pkg/apis/example"
+	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/etcd/etcdtest"
+	etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
+	storagetesting "k8s.io/apiserver/pkg/storage/testing"
+	storagetests "k8s.io/apiserver/pkg/storage/tests"
+)
+
+// prefixTransformer adds and verifies that all data has the correct prefix on its way in and out.
+type prefixTransformer struct {
+	prefix string
+	stale  bool
+	err    error
+}
+
+func (p prefixTransformer) TransformStringFromStorage(s string) (string, bool, error) {
+	if !strings.HasPrefix(s, p.prefix) {
+		return "", false, fmt.Errorf("value does not have expected prefix: %s", s)
+	}
+	return strings.TrimPrefix(s, p.prefix), p.stale, p.err
+}
+func (p prefixTransformer) TransformStringToStorage(s string) (string, error) {
+	if len(s) > 0 {
+		return p.prefix + s, p.err
+	}
+	return s, p.err
+}
+
+func defaultPrefix(s string) string {
+	return "test!" + s
+}
+
+func defaultPrefixValue(value []byte) string {
+	return defaultPrefix(string(value))
+}
+
+func testScheme(t *testing.T) (*runtime.Scheme, serializer.CodecFactory) {
+	scheme := runtime.NewScheme()
+	scheme.Log(t)
+	scheme.AddKnownTypes(schema.GroupVersion{Version: runtime.APIVersionInternal}, &storagetesting.TestResource{})
+	example.AddToScheme(scheme)
+	examplev1.AddToScheme(scheme)
+	if err := scheme.AddConversionFuncs(
+		func(in *storagetesting.TestResource, out *storagetesting.TestResource, s conversion.Scope) error {
+			*out = *in
+			return nil
+		},
+		func(in, out *time.Time, s conversion.Scope) error {
+			*out = *in
+			return nil
+		},
+	); err != nil {
+		panic(err)
+	}
+	codecs := serializer.NewCodecFactory(scheme)
+	return scheme, codecs
+}
+
+func newEtcdHelper(client etcd.Client, scheme *runtime.Scheme, codec runtime.Codec, prefix string) etcdHelper {
+	return *NewEtcdStorage(client, codec, prefix, false, etcdtest.DeserializationCacheSize, scheme, prefixTransformer{prefix: "test!"}).(*etcdHelper)
+}
+
+// Returns an encoded version of example.Pod with the given name.
+func getEncodedPod(name string, codec runtime.Codec) string {
+	pod, _ := runtime.Encode(codec, &examplev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Name: name},
+	})
+	return string(pod)
+}
+
+func createObj(t *testing.T, helper etcdHelper, name string, obj, out runtime.Object, ttl uint64) error {
+	err := helper.Create(context.TODO(), name, obj, out, ttl)
+	if err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+	return err
+}
+
+func createPodList(t *testing.T, helper etcdHelper, list *example.PodList) error {
+	for i := range list.Items {
+		returnedObj := &example.Pod{}
+		err := createObj(t, helper, list.Items[i].Name, &list.Items[i], returnedObj, 0)
+		if err != nil {
+			return err
+		}
+		list.Items[i] = *returnedObj
+	}
+	return nil
+}
+
+func TestList(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	list := example.PodList{
+		Items: []example.Pod{
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "bar"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "baz"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+		},
+	}
+
+	createPodList(t, helper, &list)
+	var got example.PodList
+	// TODO: a sorted filter function could be applied such implied
+	// ordering on the returned list doesn't matter.
+	err := helper.List(context.TODO(), "/", "", storage.Everything, &got)
+	if err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+
+	if e, a := list.Items, got.Items; !reflect.DeepEqual(e, a) {
+		t.Errorf("Expected %#v, got %#v", e, a)
+	}
+}
+
+func TestTransformationFailure(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	pods := []example.Pod{
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "bar"},
+			Spec:       storagetests.DeepEqualSafePodSpec(),
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "baz"},
+			Spec:       storagetests.DeepEqualSafePodSpec(),
+		},
+	}
+	createPodList(t, helper, &example.PodList{Items: pods[:1]})
+
+	// create a second resource with an invalid prefix
+	oldTransformer := helper.transformer
+	helper.transformer = prefixTransformer{prefix: "otherprefix!"}
+	createPodList(t, helper, &example.PodList{Items: pods[1:]})
+	helper.transformer = oldTransformer
+
+	// only the first item is returned, and no error
+	var got example.PodList
+	if err := helper.List(context.TODO(), "/", "", storage.Everything, &got); err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+	if e, a := pods[:1], got.Items; !reflect.DeepEqual(e, a) {
+		t.Errorf("Unexpected: %s", diff.ObjectReflectDiff(e, a))
+	}
+
+	// Get should fail
+	if err := helper.Get(context.TODO(), "/baz", "", &example.Pod{}, false); !storage.IsInternalError(err) {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	// GuaranteedUpdate should return an error
+	if err := helper.GuaranteedUpdate(context.TODO(), "/baz", &example.Pod{}, false, nil, func(input runtime.Object, res storage.ResponseMeta) (output runtime.Object, ttl *uint64, err error) {
+		return input, nil, nil
+	}, &pods[1]); !storage.IsInternalError(err) {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	// Delete succeeds but reports an error because we cannot access the body
+	if err := helper.Delete(context.TODO(), "/baz", &example.Pod{}, nil); !storage.IsInternalError(err) {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	if err := helper.Get(context.TODO(), "/baz", "", &example.Pod{}, false); !storage.IsNotFound(err) {
+		t.Errorf("Unexpected error: %v", err)
+	}
+}
+
+func TestListFiltered(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	list := example.PodList{
+		Items: []example.Pod{
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "bar"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "baz"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+		},
+	}
+
+	createPodList(t, helper, &list)
+	// List only "bar" pod
+	p := storage.SelectionPredicate{
+		Label: labels.Everything(),
+		Field: fields.SelectorFromSet(fields.Set{"metadata.name": "bar"}),
+		GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+			pod := obj.(*example.Pod)
+			return labels.Set(pod.Labels), fields.Set{"metadata.name": pod.Name}, pod.Initializers != nil, nil
+		},
+	}
+	var got example.PodList
+	err := helper.List(context.TODO(), "/", "", p, &got)
+	if err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+	// Check to make certain that the filter function only returns "bar"
+	if e, a := list.Items[0], got.Items[0]; !reflect.DeepEqual(e, a) {
+		t.Errorf("Expected %#v, got %#v", e, a)
+	}
+}
+
+// TestListAcrossDirectories ensures that the client excludes directories and flattens tree-response - simulates cross-namespace query
+func TestListAcrossDirectories(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	defer server.Terminate(t)
+
+	roothelper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+	helper1 := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix()+"/dir1")
+	helper2 := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix()+"/dir2")
+
+	list := example.PodList{
+		Items: []example.Pod{
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "baz"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "bar"},
+				Spec:       storagetests.DeepEqualSafePodSpec(),
+			},
+		},
+	}
+
+	returnedObj := &example.Pod{}
+	// create the 1st 2 elements in one directory
+	createObj(t, helper1, list.Items[0].Name, &list.Items[0], returnedObj, 0)
+	list.Items[0] = *returnedObj
+	createObj(t, helper1, list.Items[1].Name, &list.Items[1], returnedObj, 0)
+	list.Items[1] = *returnedObj
+	// create the last element in the other directory
+	createObj(t, helper2, list.Items[2].Name, &list.Items[2], returnedObj, 0)
+	list.Items[2] = *returnedObj
+
+	var got example.PodList
+	err := roothelper.List(context.TODO(), "/", "", storage.Everything, &got)
+	if err != nil {
+		t.Errorf("Unexpected error %v", err)
+	}
+	if e, a := list.Items, got.Items; !reflect.DeepEqual(e, a) {
+		t.Errorf("Expected %#v, got %#v", e, a)
+	}
+}
+
+func TestGet(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	key := "/some/key"
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+	expect := example.Pod{
+		ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+		Spec:       storagetests.DeepEqualSafePodSpec(),
+	}
+	var got example.Pod
+	if err := helper.Create(context.TODO(), key, &expect, &got, 0); err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+	expect = got
+	if err := helper.Get(context.TODO(), key, "", &got, false); err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+	if !reflect.DeepEqual(got, expect) {
+		t.Errorf("Wanted %#v, got %#v", expect, got)
+	}
+}
+
+func TestGetNotFoundErr(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, schema.GroupVersion{Version: "v1"})
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	boguskey := "/some/boguskey"
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	var got example.Pod
+	err := helper.Get(context.TODO(), boguskey, "", &got, false)
+	if !storage.IsNotFound(err) {
+		t.Errorf("Unexpected reponse on key=%v, err=%v", boguskey, err)
+	}
+}
+
+func TestCreate(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+	returnedObj := &example.Pod{}
+	err := helper.Create(context.TODO(), "/some/key", obj, returnedObj, 5)
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+	_, err = runtime.Encode(codec, obj)
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+	err = helper.Get(context.TODO(), "/some/key", "", returnedObj, false)
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+	_, err = runtime.Encode(codec, returnedObj)
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+	if obj.Name != returnedObj.Name {
+		t.Errorf("Wanted %v, got %v", obj.Name, returnedObj.Name)
+	}
+}
+
+func TestCreateNilOutParam(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+	err := helper.Create(context.TODO(), "/some/key", obj, nil, 5)
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+}
+
+func TestGuaranteedUpdate(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, schema.GroupVersion{Version: runtime.APIVersionInternal})
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	key := "/some/key"
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	obj := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Value: 1}
+	err := helper.GuaranteedUpdate(context.TODO(), key, &storagetesting.TestResource{}, true, nil, storage.SimpleUpdate(func(in runtime.Object) (runtime.Object, error) {
+		return obj, nil
+	}))
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+
+	// Update an existing node.
+	callbackCalled := false
+	objUpdate := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Value: 2}
+	err = helper.GuaranteedUpdate(context.TODO(), key, &storagetesting.TestResource{}, true, nil, storage.SimpleUpdate(func(in runtime.Object) (runtime.Object, error) {
+		callbackCalled = true
+
+		if in.(*storagetesting.TestResource).Value != 1 {
+			t.Errorf("Callback input was not current set value")
+		}
+
+		return objUpdate, nil
+	}))
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	objCheck := &storagetesting.TestResource{}
+	err = helper.Get(context.TODO(), key, "", objCheck, false)
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+	if objCheck.Value != 2 {
+		t.Errorf("Value should have been 2 but got %v", objCheck.Value)
+	}
+
+	if !callbackCalled {
+		t.Errorf("tryUpdate callback should have been called.")
+	}
+}
+
+func TestGuaranteedUpdateNoChange(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, schema.GroupVersion{Version: runtime.APIVersionInternal})
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	key := "/some/key"
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	obj := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Value: 1}
+	original := &storagetesting.TestResource{}
+	err := helper.GuaranteedUpdate(context.TODO(), key, original, true, nil, storage.SimpleUpdate(func(in runtime.Object) (runtime.Object, error) {
+		return obj, nil
+	}))
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+
+	// Update an existing node with the same data
+	callbackCalled := false
+	objUpdate := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: original.ResourceVersion}, Value: 1}
+	result := &storagetesting.TestResource{}
+	err = helper.GuaranteedUpdate(context.TODO(), key, result, true, nil, storage.SimpleUpdate(func(in runtime.Object) (runtime.Object, error) {
+		callbackCalled = true
+		return objUpdate, nil
+	}))
+	if err != nil {
+		t.Fatalf("Unexpected error %#v", err)
+	}
+	if !callbackCalled {
+		t.Errorf("tryUpdate callback should have been called.")
+	}
+	if result.ResourceVersion != original.ResourceVersion {
+		t.Fatalf("updated the object resource version")
+	}
+
+	// Update an existing node with the same data but return stale
+	helper.transformer = prefixTransformer{prefix: "test!", stale: true}
+	callbackCalled = false
+	result = &storagetesting.TestResource{}
+	objUpdate = &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Value: 1}
+	err = helper.GuaranteedUpdate(context.TODO(), key, &storagetesting.TestResource{}, true, nil, storage.SimpleUpdate(func(in runtime.Object) (runtime.Object, error) {
+		callbackCalled = true
+		return objUpdate, nil
+	}))
+	if err != nil {
+		t.Fatalf("Unexpected error %#v", err)
+	}
+	if !callbackCalled {
+		t.Errorf("tryUpdate callback should have been called.")
+	}
+	if result.ResourceVersion == original.ResourceVersion {
+		t.Errorf("did not update the object resource version")
+	}
+}
+
+func TestGuaranteedUpdateKeyNotFound(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, schema.GroupVersion{Version: runtime.APIVersionInternal})
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	key := "/some/key"
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	// Create a new node.
+	obj := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Value: 1}
+
+	f := storage.SimpleUpdate(func(in runtime.Object) (runtime.Object, error) {
+		return obj, nil
+	})
+
+	ignoreNotFound := false
+	err := helper.GuaranteedUpdate(context.TODO(), key, &storagetesting.TestResource{}, ignoreNotFound, nil, f)
+	if err == nil {
+		t.Errorf("Expected error for key not found.")
+	}
+
+	ignoreNotFound = true
+	err = helper.GuaranteedUpdate(context.TODO(), key, &storagetesting.TestResource{}, ignoreNotFound, nil, f)
+	if err != nil {
+		t.Errorf("Unexpected error %v.", err)
+	}
+}
+
+func TestGuaranteedUpdate_CreateCollision(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, schema.GroupVersion{Version: runtime.APIVersionInternal})
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	key := "/some/key"
+	helper := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	const concurrency = 10
+	var wgDone sync.WaitGroup
+	var wgForceCollision sync.WaitGroup
+	wgDone.Add(concurrency)
+	wgForceCollision.Add(concurrency)
+
+	for i := 0; i < concurrency; i++ {
+		// Increment storagetesting.TestResource.Value by 1
+		go func() {
+			defer wgDone.Done()
+
+			firstCall := true
+			err := helper.GuaranteedUpdate(context.TODO(), key, &storagetesting.TestResource{}, true, nil, storage.SimpleUpdate(func(in runtime.Object) (runtime.Object, error) {
+				defer func() { firstCall = false }()
+
+				if firstCall {
+					// Force collision by joining all concurrent GuaranteedUpdate operations here.
+					wgForceCollision.Done()
+					wgForceCollision.Wait()
+				}
+
+				currValue := in.(*storagetesting.TestResource).Value
+				obj := &storagetesting.TestResource{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Value: currValue + 1}
+				return obj, nil
+			}))
+			if err != nil {
+				t.Errorf("Unexpected error %#v", err)
+			}
+		}()
+	}
+	wgDone.Wait()
+
+	stored := &storagetesting.TestResource{}
+	err := helper.Get(context.TODO(), key, "", stored, false)
+	if err != nil {
+		t.Errorf("Unexpected error %#v", stored)
+	}
+	if stored.Value != concurrency {
+		t.Errorf("Some of the writes were lost. Stored value: %d", stored.Value)
+	}
+}
+
+func TestGuaranteedUpdateUIDMismatch(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	prefix := path.Join("/", etcdtest.PathPrefix())
+	helper := newEtcdHelper(server.Client, scheme, codec, prefix)
+
+	obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "A"}}
+	podPtr := &example.Pod{}
+	err := helper.Create(context.TODO(), "/some/key", obj, podPtr, 0)
+	if err != nil {
+		t.Fatalf("Unexpected error %#v", err)
+	}
+	err = helper.GuaranteedUpdate(context.TODO(), "/some/key", podPtr, true, storage.NewUIDPreconditions("B"), storage.SimpleUpdate(func(in runtime.Object) (runtime.Object, error) {
+		return obj, nil
+	}))
+	if !storage.IsInvalidObj(err) {
+		t.Fatalf("Expect a Test Failed (write conflict) error, got: %v", err)
+	}
+}
+
+func TestDeleteUIDMismatch(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	prefix := path.Join("/", etcdtest.PathPrefix())
+	helper := newEtcdHelper(server.Client, scheme, codec, prefix)
+
+	obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "A"}}
+	podPtr := &example.Pod{}
+	err := helper.Create(context.TODO(), "/some/key", obj, podPtr, 0)
+	if err != nil {
+		t.Fatalf("Unexpected error %#v", err)
+	}
+	err = helper.Delete(context.TODO(), "/some/key", obj, storage.NewUIDPreconditions("B"))
+	if !storage.IsInvalidObj(err) {
+		t.Fatalf("Expect a Test Failed (write conflict) error, got: %v", err)
+	}
+}
+
+type getFunc func(ctx context.Context, key string, opts *etcd.GetOptions) (*etcd.Response, error)
+
+type fakeDeleteKeysAPI struct {
+	etcd.KeysAPI
+	fakeGetFunc getFunc
+	getCount    int
+	// The fakeGetFunc will be called fakeGetCap times before the KeysAPI's Get will be called.
+	fakeGetCap int
+}
+
+func (f *fakeDeleteKeysAPI) Get(ctx context.Context, key string, opts *etcd.GetOptions) (*etcd.Response, error) {
+	f.getCount++
+	if f.getCount < f.fakeGetCap {
+		return f.fakeGetFunc(ctx, key, opts)
+	}
+	return f.KeysAPI.Get(ctx, key, opts)
+}
+
+// This is to emulate the case where another party updates the object when
+// etcdHelper.Delete has verified the preconditions, but hasn't carried out the
+// deletion yet. Etcd will fail the deletion and report the conflict. etcdHelper
+// should retry until there is no conflict.
+func TestDeleteWithRetry(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	prefix := path.Join("/", etcdtest.PathPrefix())
+
+	obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "A"}}
+	// fakeGet returns a large ModifiedIndex to emulate the case that another
+	// party has updated the object.
+	fakeGet := func(ctx context.Context, key string, opts *etcd.GetOptions) (*etcd.Response, error) {
+		data, _ := runtime.Encode(codec, obj)
+		return &etcd.Response{Node: &etcd.Node{Value: defaultPrefixValue(data), ModifiedIndex: 99}}, nil
+	}
+	expectedRetries := 3
+	helper := newEtcdHelper(server.Client, scheme, codec, prefix)
+	fake := &fakeDeleteKeysAPI{KeysAPI: helper.etcdKeysAPI, fakeGetCap: expectedRetries, fakeGetFunc: fakeGet}
+	helper.etcdKeysAPI = fake
+
+	returnedObj := &example.Pod{}
+	err := helper.Create(context.TODO(), "/some/key", obj, returnedObj, 0)
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+
+	err = helper.Delete(context.TODO(), "/some/key", obj, storage.NewUIDPreconditions("A"))
+	if err != nil {
+		t.Errorf("Unexpected error %#v", err)
+	}
+	if fake.getCount != expectedRetries {
+		t.Errorf("Expect %d retries, got %d", expectedRetries, fake.getCount)
+	}
+	err = helper.Get(context.TODO(), "/some/key", "", obj, false)
+	if !storage.IsNotFound(err) {
+		t.Errorf("Expect an NotFound error, got %v", err)
+	}
+}
+
+func TestPrefix(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+
+	testcases := map[string]string{
+		"custom/prefix":     "/custom/prefix",
+		"/custom//prefix//": "/custom/prefix",
+		"/registry":         "/registry",
+	}
+	for configuredPrefix, effectivePrefix := range testcases {
+		helper := newEtcdHelper(server.Client, scheme, codec, configuredPrefix)
+		if helper.pathPrefix != effectivePrefix {
+			t.Errorf("configured prefix of %s, expected effective prefix of %s, got %s", configuredPrefix, effectivePrefix, helper.pathPrefix)
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/etcd_watcher.go

@@ -0,0 +1,497 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd
+
+import (
+	"fmt"
+	"net/http"
+	"reflect"
+	"sync"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/storage"
+	etcdutil "k8s.io/apiserver/pkg/storage/etcd/util"
+
+	etcd "github.com/coreos/etcd/client"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+)
+
+// Etcd watch event actions
+const (
+	EtcdCreate = "create"
+	EtcdGet    = "get"
+	EtcdSet    = "set"
+	EtcdCAS    = "compareAndSwap"
+	EtcdDelete = "delete"
+	EtcdCAD    = "compareAndDelete"
+	EtcdExpire = "expire"
+)
+
+// TransformFunc attempts to convert an object to another object for use with a watcher.
+type TransformFunc func(runtime.Object) (runtime.Object, error)
+
+// includeFunc returns true if the given key should be considered part of a watch
+type includeFunc func(key string) bool
+
+// exceptKey is an includeFunc that returns false when the provided key matches the watched key
+func exceptKey(except string) includeFunc {
+	return func(key string) bool {
+		return key != except
+	}
+}
+
+// etcdWatcher converts a native etcd watch to a watch.Interface.
+type etcdWatcher struct {
+	// HighWaterMarks for performance debugging.
+	// Important: Since HighWaterMark is using sync/atomic, it has to be at the top of the struct due to a bug on 32-bit platforms
+	// See: https://golang.org/pkg/sync/atomic/ for more information
+	incomingHWM storage.HighWaterMark
+	outgoingHWM storage.HighWaterMark
+
+	encoding runtime.Codec
+	// Note that versioner is required for etcdWatcher to work correctly.
+	// There is no public constructor of it, so be careful when manipulating
+	// with it manually.
+	versioner        storage.Versioner
+	transform        TransformFunc
+	valueTransformer ValueTransformer
+
+	list    bool // If we're doing a recursive watch, should be true.
+	quorum  bool // If we enable quorum, shoule be true
+	include includeFunc
+	filter  storage.FilterFunc
+
+	etcdIncoming  chan *etcd.Response
+	etcdError     chan error
+	ctx           context.Context
+	cancel        context.CancelFunc
+	etcdCallEnded chan struct{}
+
+	outgoing chan watch.Event
+	userStop chan struct{}
+	stopped  bool
+	stopLock sync.Mutex
+	// wg is used to avoid calls to etcd after Stop(), and to make sure
+	// that the translate goroutine is not leaked.
+	wg sync.WaitGroup
+
+	// Injectable for testing. Send the event down the outgoing channel.
+	emit func(watch.Event)
+
+	cache etcdCache
+}
+
+// watchWaitDuration is the amount of time to wait for an error from watch.
+const watchWaitDuration = 100 * time.Millisecond
+
+// newEtcdWatcher returns a new etcdWatcher; if list is true, watch sub-nodes.
+// The versioner must be able to handle the objects that transform creates.
+func newEtcdWatcher(
+	list bool, quorum bool, include includeFunc, filter storage.FilterFunc,
+	encoding runtime.Codec, versioner storage.Versioner, transform TransformFunc,
+	valueTransformer ValueTransformer,
+	cache etcdCache) *etcdWatcher {
+	w := &etcdWatcher{
+		encoding:         encoding,
+		versioner:        versioner,
+		transform:        transform,
+		valueTransformer: valueTransformer,
+
+		list:    list,
+		quorum:  quorum,
+		include: include,
+		filter:  filter,
+		// Buffer this channel, so that the etcd client is not forced
+		// to context switch with every object it gets, and so that a
+		// long time spent decoding an object won't block the *next*
+		// object. Basically, we see a lot of "401 window exceeded"
+		// errors from etcd, and that's due to the client not streaming
+		// results but rather getting them one at a time. So we really
+		// want to never block the etcd client, if possible. The 100 is
+		// mostly arbitrary--we know it goes as high as 50, though.
+		// There's a V(2) log message that prints the length so we can
+		// monitor how much of this buffer is actually used.
+		etcdIncoming: make(chan *etcd.Response, 100),
+		etcdError:    make(chan error, 1),
+		// Similarly to etcdIncomming, we don't want to force context
+		// switch on every new incoming object.
+		outgoing: make(chan watch.Event, 100),
+		userStop: make(chan struct{}),
+		stopped:  false,
+		wg:       sync.WaitGroup{},
+		cache:    cache,
+		ctx:      nil,
+		cancel:   nil,
+	}
+	w.emit = func(e watch.Event) {
+		if curLen := int64(len(w.outgoing)); w.outgoingHWM.Update(curLen) {
+			// Monitor if this gets backed up, and how much.
+			glog.V(1).Infof("watch (%v): %v objects queued in outgoing channel.", reflect.TypeOf(e.Object).String(), curLen)
+		}
+		// Give up on user stop, without this we leak a lot of goroutines in tests.
+		select {
+		case w.outgoing <- e:
+		case <-w.userStop:
+		}
+	}
+	// translate will call done. We need to Add() here because otherwise,
+	// if Stop() gets called before translate gets started, there'd be a
+	// problem.
+	w.wg.Add(1)
+	go w.translate()
+	return w
+}
+
+// etcdWatch calls etcd's Watch function, and handles any errors. Meant to be called
+// as a goroutine.
+func (w *etcdWatcher) etcdWatch(ctx context.Context, client etcd.KeysAPI, key string, resourceVersion uint64) {
+	defer utilruntime.HandleCrash()
+	defer close(w.etcdError)
+	defer close(w.etcdIncoming)
+
+	// All calls to etcd are coming from this function - once it is finished
+	// no other call to etcd should be generated by this watcher.
+	done := func() {}
+
+	// We need to be prepared, that Stop() can be called at any time.
+	// It can potentially also be called, even before this function is called.
+	// If that is the case, we simply skip all the code here.
+	// See #18928 for more details.
+	var watcher etcd.Watcher
+	returned := func() bool {
+		w.stopLock.Lock()
+		defer w.stopLock.Unlock()
+		if w.stopped {
+			// Watcher has already been stopped - don't event initiate it here.
+			return true
+		}
+		w.wg.Add(1)
+		done = w.wg.Done
+		// Perform initialization of watcher under lock - we want to avoid situation when
+		// Stop() is called in the meantime (which in tests can cause etcd termination and
+		// strange behavior here).
+		if resourceVersion == 0 {
+			latest, err := etcdGetInitialWatchState(ctx, client, key, w.list, w.quorum, w.etcdIncoming)
+			if err != nil {
+				w.etcdError <- err
+				return true
+			}
+			resourceVersion = latest
+		}
+
+		opts := etcd.WatcherOptions{
+			Recursive:  w.list,
+			AfterIndex: resourceVersion,
+		}
+		watcher = client.Watcher(key, &opts)
+		w.ctx, w.cancel = context.WithCancel(ctx)
+		return false
+	}()
+	defer done()
+	if returned {
+		return
+	}
+
+	for {
+		resp, err := watcher.Next(w.ctx)
+		if err != nil {
+			w.etcdError <- err
+			return
+		}
+		w.etcdIncoming <- resp
+	}
+}
+
+// etcdGetInitialWatchState turns an etcd Get request into a watch equivalent
+func etcdGetInitialWatchState(ctx context.Context, client etcd.KeysAPI, key string, recursive bool, quorum bool, incoming chan<- *etcd.Response) (resourceVersion uint64, err error) {
+	opts := etcd.GetOptions{
+		Recursive: recursive,
+		Sort:      false,
+		Quorum:    quorum,
+	}
+	resp, err := client.Get(ctx, key, &opts)
+	if err != nil {
+		if !etcdutil.IsEtcdNotFound(err) {
+			utilruntime.HandleError(fmt.Errorf("watch was unable to retrieve the current index for the provided key (%q): %v", key, err))
+			return resourceVersion, toStorageErr(err, key, 0)
+		}
+		if etcdError, ok := err.(etcd.Error); ok {
+			resourceVersion = etcdError.Index
+		}
+		return resourceVersion, nil
+	}
+	resourceVersion = resp.Index
+	convertRecursiveResponse(resp.Node, resp, incoming)
+	return
+}
+
+// convertRecursiveResponse turns a recursive get response from etcd into individual response objects
+// by copying the original response.  This emulates the behavior of a recursive watch.
+func convertRecursiveResponse(node *etcd.Node, response *etcd.Response, incoming chan<- *etcd.Response) {
+	if node.Dir {
+		for i := range node.Nodes {
+			convertRecursiveResponse(node.Nodes[i], response, incoming)
+		}
+		return
+	}
+	copied := *response
+	copied.Action = "get"
+	copied.Node = node
+	incoming <- &copied
+}
+
+// translate pulls stuff from etcd, converts, and pushes out the outgoing channel. Meant to be
+// called as a goroutine.
+func (w *etcdWatcher) translate() {
+	defer w.wg.Done()
+	defer close(w.outgoing)
+	defer utilruntime.HandleCrash()
+
+	for {
+		select {
+		case err := <-w.etcdError:
+			if err != nil {
+				var status *metav1.Status
+				switch {
+				case etcdutil.IsEtcdWatchExpired(err):
+					status = &metav1.Status{
+						Status:  metav1.StatusFailure,
+						Message: err.Error(),
+						Code:    http.StatusGone, // Gone
+						Reason:  metav1.StatusReasonExpired,
+					}
+				// TODO: need to generate errors using api/errors which has a circular dependency on this package
+				//   no other way to inject errors
+				// case etcdutil.IsEtcdUnreachable(err):
+				//   status = errors.NewServerTimeout(...)
+				default:
+					status = &metav1.Status{
+						Status:  metav1.StatusFailure,
+						Message: err.Error(),
+						Code:    http.StatusInternalServerError,
+						Reason:  metav1.StatusReasonInternalError,
+					}
+				}
+				w.emit(watch.Event{
+					Type:   watch.Error,
+					Object: status,
+				})
+			}
+			return
+		case <-w.userStop:
+			return
+		case res, ok := <-w.etcdIncoming:
+			if ok {
+				if curLen := int64(len(w.etcdIncoming)); w.incomingHWM.Update(curLen) {
+					// Monitor if this gets backed up, and how much.
+					glog.V(1).Infof("watch: %v objects queued in incoming channel.", curLen)
+				}
+				w.sendResult(res)
+			}
+			// If !ok, don't return here-- must wait for etcdError channel
+			// to give an error or be closed.
+		}
+	}
+}
+
+// decodeObject extracts an object from the provided etcd node or returns an error.
+func (w *etcdWatcher) decodeObject(node *etcd.Node) (runtime.Object, error) {
+	if obj, found := w.cache.getFromCache(node.ModifiedIndex, storage.SimpleFilter(storage.Everything)); found {
+		return obj, nil
+	}
+
+	body, _, err := w.valueTransformer.TransformStringFromStorage(node.Value)
+	if err != nil {
+		return nil, err
+	}
+
+	obj, err := runtime.Decode(w.encoding, []byte(body))
+	if err != nil {
+		return nil, err
+	}
+
+	// ensure resource version is set on the object we load from etcd
+	if err := w.versioner.UpdateObject(obj, node.ModifiedIndex); err != nil {
+		utilruntime.HandleError(fmt.Errorf("failure to version api object (%d) %#v: %v", node.ModifiedIndex, obj, err))
+	}
+
+	// perform any necessary transformation
+	if w.transform != nil {
+		obj, err = w.transform(obj)
+		if err != nil {
+			utilruntime.HandleError(fmt.Errorf("failure to transform api object %#v: %v", obj, err))
+			return nil, err
+		}
+	}
+
+	if node.ModifiedIndex != 0 {
+		w.cache.addToCache(node.ModifiedIndex, obj)
+	}
+	return obj, nil
+}
+
+func (w *etcdWatcher) sendAdd(res *etcd.Response) {
+	if res.Node == nil {
+		utilruntime.HandleError(fmt.Errorf("unexpected nil node: %#v", res))
+		return
+	}
+	if w.include != nil && !w.include(res.Node.Key) {
+		return
+	}
+	obj, err := w.decodeObject(res.Node)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("failure to decode api object: %v\n'%v' from %#v %#v", err, string(res.Node.Value), res, res.Node))
+		// TODO: expose an error through watch.Interface?
+		// Ignore this value. If we stop the watch on a bad value, a client that uses
+		// the resourceVersion to resume will never be able to get past a bad value.
+		return
+	}
+	if !w.filter(obj) {
+		return
+	}
+	action := watch.Added
+	w.emit(watch.Event{
+		Type:   action,
+		Object: obj,
+	})
+}
+
+func (w *etcdWatcher) sendModify(res *etcd.Response) {
+	if res.Node == nil {
+		glog.Errorf("unexpected nil node: %#v", res)
+		return
+	}
+	if w.include != nil && !w.include(res.Node.Key) {
+		return
+	}
+	curObj, err := w.decodeObject(res.Node)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("failure to decode api object: %v\n'%v' from %#v %#v", err, string(res.Node.Value), res, res.Node))
+		// TODO: expose an error through watch.Interface?
+		// Ignore this value. If we stop the watch on a bad value, a client that uses
+		// the resourceVersion to resume will never be able to get past a bad value.
+		return
+	}
+	curObjPasses := w.filter(curObj)
+	oldObjPasses := false
+	var oldObj runtime.Object
+	if res.PrevNode != nil && res.PrevNode.Value != "" {
+		// Ignore problems reading the old object.
+		if oldObj, err = w.decodeObject(res.PrevNode); err == nil {
+			if err := w.versioner.UpdateObject(oldObj, res.Node.ModifiedIndex); err != nil {
+				utilruntime.HandleError(fmt.Errorf("failure to version api object (%d) %#v: %v", res.Node.ModifiedIndex, oldObj, err))
+			}
+			oldObjPasses = w.filter(oldObj)
+		}
+	}
+	// Some changes to an object may cause it to start or stop matching a filter.
+	// We need to report those as adds/deletes. So we have to check both the previous
+	// and current value of the object.
+	switch {
+	case curObjPasses && oldObjPasses:
+		w.emit(watch.Event{
+			Type:   watch.Modified,
+			Object: curObj,
+		})
+	case curObjPasses && !oldObjPasses:
+		w.emit(watch.Event{
+			Type:   watch.Added,
+			Object: curObj,
+		})
+	case !curObjPasses && oldObjPasses:
+		w.emit(watch.Event{
+			Type:   watch.Deleted,
+			Object: oldObj,
+		})
+	}
+	// Do nothing if neither new nor old object passed the filter.
+}
+
+func (w *etcdWatcher) sendDelete(res *etcd.Response) {
+	if res.PrevNode == nil {
+		utilruntime.HandleError(fmt.Errorf("unexpected nil prev node: %#v", res))
+		return
+	}
+	if w.include != nil && !w.include(res.PrevNode.Key) {
+		return
+	}
+	node := *res.PrevNode
+	if res.Node != nil {
+		// Note that this sends the *old* object with the etcd index for the time at
+		// which it gets deleted. This will allow users to restart the watch at the right
+		// index.
+		node.ModifiedIndex = res.Node.ModifiedIndex
+	}
+	obj, err := w.decodeObject(&node)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("failure to decode api object: %v\nfrom %#v %#v", err, res, res.Node))
+		// TODO: expose an error through watch.Interface?
+		// Ignore this value. If we stop the watch on a bad value, a client that uses
+		// the resourceVersion to resume will never be able to get past a bad value.
+		return
+	}
+	if !w.filter(obj) {
+		return
+	}
+	w.emit(watch.Event{
+		Type:   watch.Deleted,
+		Object: obj,
+	})
+}
+
+func (w *etcdWatcher) sendResult(res *etcd.Response) {
+	switch res.Action {
+	case EtcdCreate, EtcdGet:
+		// "Get" will only happen in watch 0 case, where we explicitly want ADDED event
+		// for initial state.
+		w.sendAdd(res)
+	case EtcdSet, EtcdCAS:
+		w.sendModify(res)
+	case EtcdDelete, EtcdExpire, EtcdCAD:
+		w.sendDelete(res)
+	default:
+		utilruntime.HandleError(fmt.Errorf("unknown action: %v", res.Action))
+	}
+}
+
+// ResultChan implements watch.Interface.
+func (w *etcdWatcher) ResultChan() <-chan watch.Event {
+	return w.outgoing
+}
+
+// Stop implements watch.Interface.
+func (w *etcdWatcher) Stop() {
+	w.stopLock.Lock()
+	if w.cancel != nil {
+		w.cancel()
+		w.cancel = nil
+	}
+	if !w.stopped {
+		w.stopped = true
+		close(w.userStop)
+	}
+	w.stopLock.Unlock()
+
+	// Wait until all calls to etcd are finished and no other
+	// will be issued.
+	w.wg.Wait()
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/etcd_watcher_test.go

@@ -0,0 +1,557 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd
+
+import (
+	rt "runtime"
+	"testing"
+
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apitesting "k8s.io/apimachinery/pkg/api/testing"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/apis/example"
+	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/etcd/etcdtest"
+	etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
+
+	etcd "github.com/coreos/etcd/client"
+	"golang.org/x/net/context"
+)
+
+var versioner = APIObjectVersioner{}
+
+// Implements etcdCache interface as empty methods (i.e. does not cache any objects)
+type fakeEtcdCache struct{}
+
+func (f *fakeEtcdCache) getFromCache(index uint64, filter storage.FilterFunc) (runtime.Object, bool) {
+	return nil, false
+}
+
+func (f *fakeEtcdCache) addToCache(index uint64, obj runtime.Object) {
+}
+
+var _ etcdCache = &fakeEtcdCache{}
+
+func TestWatchInterpretations(t *testing.T) {
+	_, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	// Declare some pods to make the test cases compact.
+	podFoo := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+	podBar := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}
+	podBaz := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "baz"}}
+
+	// All of these test cases will be run with the firstLetterIsB Filter.
+	table := map[string]struct {
+		actions       []string // Run this test item for every action here.
+		prevNodeValue string
+		nodeValue     string
+		expectEmit    bool
+		expectType    watch.EventType
+		expectObject  runtime.Object
+	}{
+		"create": {
+			actions:      []string{"create", "get"},
+			nodeValue:    runtime.EncodeOrDie(codec, podBar),
+			expectEmit:   true,
+			expectType:   watch.Added,
+			expectObject: podBar,
+		},
+		"create but filter blocks": {
+			actions:    []string{"create", "get"},
+			nodeValue:  runtime.EncodeOrDie(codec, podFoo),
+			expectEmit: false,
+		},
+		"delete": {
+			actions:       []string{"delete"},
+			prevNodeValue: runtime.EncodeOrDie(codec, podBar),
+			expectEmit:    true,
+			expectType:    watch.Deleted,
+			expectObject:  podBar,
+		},
+		"delete but filter blocks": {
+			actions:    []string{"delete"},
+			nodeValue:  runtime.EncodeOrDie(codec, podFoo),
+			expectEmit: false,
+		},
+		"modify appears to create 1": {
+			actions:      []string{"set", "compareAndSwap"},
+			nodeValue:    runtime.EncodeOrDie(codec, podBar),
+			expectEmit:   true,
+			expectType:   watch.Added,
+			expectObject: podBar,
+		},
+		"modify appears to create 2": {
+			actions:       []string{"set", "compareAndSwap"},
+			prevNodeValue: runtime.EncodeOrDie(codec, podFoo),
+			nodeValue:     runtime.EncodeOrDie(codec, podBar),
+			expectEmit:    true,
+			expectType:    watch.Added,
+			expectObject:  podBar,
+		},
+		"modify appears to delete": {
+			actions:       []string{"set", "compareAndSwap"},
+			prevNodeValue: runtime.EncodeOrDie(codec, podBar),
+			nodeValue:     runtime.EncodeOrDie(codec, podFoo),
+			expectEmit:    true,
+			expectType:    watch.Deleted,
+			expectObject:  podBar, // Should return last state that passed the filter!
+		},
+		"modify modifies": {
+			actions:       []string{"set", "compareAndSwap"},
+			prevNodeValue: runtime.EncodeOrDie(codec, podBar),
+			nodeValue:     runtime.EncodeOrDie(codec, podBaz),
+			expectEmit:    true,
+			expectType:    watch.Modified,
+			expectObject:  podBaz,
+		},
+		"modify ignores": {
+			actions:    []string{"set", "compareAndSwap"},
+			nodeValue:  runtime.EncodeOrDie(codec, podFoo),
+			expectEmit: false,
+		},
+	}
+	firstLetterIsB := func(obj runtime.Object) bool {
+		return obj.(*example.Pod).Name[0] == 'b'
+	}
+	for name, item := range table {
+		for _, action := range item.actions {
+			w := newEtcdWatcher(true, false, nil, firstLetterIsB, codec, versioner, nil, prefixTransformer{prefix: "test!"}, &fakeEtcdCache{})
+			emitCalled := false
+			w.emit = func(event watch.Event) {
+				emitCalled = true
+				if !item.expectEmit {
+					return
+				}
+				if e, a := item.expectType, event.Type; e != a {
+					t.Errorf("'%v - %v': expected %v, got %v", name, action, e, a)
+				}
+				if e, a := item.expectObject, event.Object; !apiequality.Semantic.DeepDerivative(e, a) {
+					t.Errorf("'%v - %v': expected %v, got %v", name, action, e, a)
+				}
+			}
+
+			var n, pn *etcd.Node
+			if item.nodeValue != "" {
+				n = &etcd.Node{Value: defaultPrefix(item.nodeValue)}
+			}
+			if item.prevNodeValue != "" {
+				pn = &etcd.Node{Value: defaultPrefix(item.prevNodeValue)}
+			}
+
+			w.sendResult(&etcd.Response{
+				Action:   action,
+				Node:     n,
+				PrevNode: pn,
+			})
+
+			if e, a := item.expectEmit, emitCalled; e != a {
+				t.Errorf("'%v - %v': expected %v, got %v", name, action, e, a)
+			}
+			w.Stop()
+		}
+	}
+}
+
+func TestWatchInterpretation_ResponseNotSet(t *testing.T) {
+	_, codecs := testScheme(t)
+	codec := codecs.LegacyCodec(schema.GroupVersion{Version: "v1"})
+	w := newEtcdWatcher(false, false, nil, storage.SimpleFilter(storage.Everything), codec, versioner, nil, prefixTransformer{prefix: "test!"}, &fakeEtcdCache{})
+	w.emit = func(e watch.Event) {
+		t.Errorf("Unexpected emit: %v", e)
+	}
+
+	w.sendResult(&etcd.Response{
+		Action: "update",
+	})
+	w.Stop()
+}
+
+func TestWatchInterpretation_ResponseNoNode(t *testing.T) {
+	_, codecs := testScheme(t)
+	codec := codecs.LegacyCodec(schema.GroupVersion{Version: "v1"})
+	actions := []string{"create", "set", "compareAndSwap", "delete"}
+	for _, action := range actions {
+		w := newEtcdWatcher(false, false, nil, storage.SimpleFilter(storage.Everything), codec, versioner, nil, prefixTransformer{prefix: "test!"}, &fakeEtcdCache{})
+		w.emit = func(e watch.Event) {
+			t.Errorf("Unexpected emit: %v", e)
+		}
+		w.sendResult(&etcd.Response{
+			Action: action,
+		})
+		w.Stop()
+	}
+}
+
+func TestWatchInterpretation_ResponseBadData(t *testing.T) {
+	_, codecs := testScheme(t)
+	codec := codecs.LegacyCodec(schema.GroupVersion{Version: "v1"})
+	actions := []string{"create", "set", "compareAndSwap", "delete"}
+	for _, action := range actions {
+		w := newEtcdWatcher(false, false, nil, storage.SimpleFilter(storage.Everything), codec, versioner, nil, prefixTransformer{prefix: "test!"}, &fakeEtcdCache{})
+		w.emit = func(e watch.Event) {
+			t.Errorf("Unexpected emit: %v", e)
+		}
+		w.sendResult(&etcd.Response{
+			Action: action,
+			Node: &etcd.Node{
+				Value: defaultPrefix("foobar"),
+			},
+		})
+		w.sendResult(&etcd.Response{
+			Action: action,
+			PrevNode: &etcd.Node{
+				Value: defaultPrefix("foobar"),
+			},
+		})
+		w.Stop()
+	}
+}
+
+func TestSendResultDeleteEventHaveLatestIndex(t *testing.T) {
+	_, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	filter := func(obj runtime.Object) bool {
+		return obj.(*example.Pod).Name != "bar"
+	}
+	w := newEtcdWatcher(false, false, nil, filter, codec, versioner, nil, prefixTransformer{prefix: "test!"}, &fakeEtcdCache{})
+
+	eventChan := make(chan watch.Event, 1)
+	w.emit = func(e watch.Event) {
+		eventChan <- e
+	}
+
+	fooPod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+	barPod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}
+	fooBytes, err := runtime.Encode(codec, fooPod)
+	if err != nil {
+		t.Fatalf("Encode failed: %v", err)
+	}
+	barBytes, err := runtime.Encode(codec, barPod)
+	if err != nil {
+		t.Fatalf("Encode failed: %v", err)
+	}
+
+	tests := []struct {
+		response *etcd.Response
+		expRV    string
+	}{{ // Delete event
+		response: &etcd.Response{
+			Action: EtcdDelete,
+			Node: &etcd.Node{
+				ModifiedIndex: 2,
+			},
+			PrevNode: &etcd.Node{
+				Value:         defaultPrefixValue(fooBytes),
+				ModifiedIndex: 1,
+			},
+		},
+		expRV: "2",
+	}, { // Modify event with uninterested data
+		response: &etcd.Response{
+			Action: EtcdSet,
+			Node: &etcd.Node{
+				Value:         defaultPrefixValue(barBytes),
+				ModifiedIndex: 2,
+			},
+			PrevNode: &etcd.Node{
+				Value:         defaultPrefixValue(fooBytes),
+				ModifiedIndex: 1,
+			},
+		},
+		expRV: "2",
+	}}
+
+	for i, tt := range tests {
+		w.sendResult(tt.response)
+		ev := <-eventChan
+		if ev.Type != watch.Deleted {
+			t.Errorf("#%d: event type want=Deleted, get=%s", i, ev.Type)
+			return
+		}
+		rv := ev.Object.(*example.Pod).ResourceVersion
+		if rv != tt.expRV {
+			t.Errorf("#%d: resource version want=%s, get=%s", i, tt.expRV, rv)
+		}
+	}
+	w.Stop()
+}
+
+func TestWatch(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	key := "/some/key"
+	h := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	watching, err := h.Watch(context.TODO(), key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	// watching is explicitly closed below.
+
+	// Test normal case
+	pod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+	returnObj := &example.Pod{}
+	err = h.Create(context.TODO(), key, pod, returnObj, 0)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	event := <-watching.ResultChan()
+	if e, a := watch.Added, event.Type; e != a {
+		t.Errorf("Expected %v, got %v", e, a)
+	}
+	if e, a := pod, event.Object; !apiequality.Semantic.DeepDerivative(e, a) {
+		t.Errorf("Expected %v, got %v", e, a)
+	}
+
+	watching.Stop()
+
+	// There is a race in etcdWatcher so that after calling Stop() one of
+	// two things can happen:
+	// - ResultChan() may be closed (triggered by closing userStop channel)
+	// - an Error "context cancelled" may be emitted (triggered by cancelling request
+	//   to etcd and putting that error to etcdError channel)
+	// We need to be prepared for both here.
+	event, open := <-watching.ResultChan()
+	if open && event.Type != watch.Error {
+		t.Errorf("Unexpected event from stopped watcher: %#v", event)
+	}
+}
+
+func TestWatchEtcdState(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	key := "/somekey/foo"
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+
+	h := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+	watching, err := h.Watch(context.TODO(), key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer watching.Stop()
+
+	pod := &example.Pod{
+		ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+	}
+
+	err = h.Create(context.TODO(), key, pod, pod, 0)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	event := <-watching.ResultChan()
+	if event.Type != watch.Added {
+		t.Errorf("Unexpected event %#v", event)
+	}
+
+	pod.ResourceVersion = ""
+	pod.Status = example.PodStatus{
+		Phase: example.PodPhase("Running"),
+	}
+
+	// CAS the previous value
+	updateFn := func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) {
+		return pod.DeepCopyObject(), nil, nil
+	}
+	err = h.GuaranteedUpdate(context.TODO(), key, &example.Pod{}, false, nil, updateFn)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	event = <-watching.ResultChan()
+	if event.Type != watch.Modified {
+		t.Errorf("Unexpected event %#v", event)
+	}
+
+	if e, a := pod, event.Object; !apiequality.Semantic.DeepDerivative(e, a) {
+		t.Errorf("Unexpected error: expected %#v, got %#v", e, a)
+	}
+}
+
+func TestWatchFromZeroIndex(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	pod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+
+	key := "/somekey/foo"
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+
+	h := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	// set before the watch and verify events
+	err := h.Create(context.TODO(), key, pod, pod, 0)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	pod.ResourceVersion = ""
+
+	watching, err := h.Watch(context.TODO(), key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	// The create trigger ADDED event when watching from 0
+	event := <-watching.ResultChan()
+	watching.Stop()
+	if event.Type != watch.Added {
+		t.Errorf("Unexpected event %#v", event)
+	}
+
+	// check for concatenation on watch event with CAS
+	updateFn := func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) {
+		pod := input.(*example.Pod)
+		pod.Name = "bar"
+		return pod, nil, nil
+	}
+	err = h.GuaranteedUpdate(context.TODO(), key, &example.Pod{}, false, nil, updateFn)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	watching, err = h.Watch(context.TODO(), key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer watching.Stop()
+
+	// because we watch from 0, first event that we receive will always be ADDED
+	event = <-watching.ResultChan()
+	if event.Type != watch.Added {
+		t.Errorf("Unexpected event %#v", event)
+	}
+
+	pod.Name = "baz"
+	updateFn = func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) {
+		pod := input.(*example.Pod)
+		pod.Name = "baz"
+		return pod, nil, nil
+	}
+	err = h.GuaranteedUpdate(context.TODO(), key, &example.Pod{}, false, nil, updateFn)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	event = <-watching.ResultChan()
+	if event.Type != watch.Modified {
+		t.Errorf("Unexpected event %#v", event)
+	}
+
+	if e, a := pod, event.Object; a == nil || !apiequality.Semantic.DeepDerivative(e, a) {
+		t.Errorf("Unexpected error: expected %#v, got %#v", e, a)
+	}
+}
+
+func TestWatchListFromZeroIndex(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	prefix := "/some/key"
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	h := newEtcdHelper(server.Client, scheme, codec, prefix)
+
+	watching, err := h.WatchList(context.TODO(), "/", "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer watching.Stop()
+
+	// creates foo which should trigger the WatchList for "/"
+	pod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+	err = h.Create(context.TODO(), pod.Name, pod, pod, 0)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	event, _ := <-watching.ResultChan()
+	if event.Type != watch.Added {
+		t.Errorf("Unexpected event %#v", event)
+	}
+
+	if e, a := pod, event.Object; !apiequality.Semantic.DeepDerivative(e, a) {
+		t.Errorf("Unexpected error: expected %v, got %v", e, a)
+	}
+}
+
+func TestWatchListIgnoresRootKey(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+	pod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+	key := "/some/key"
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	h := newEtcdHelper(server.Client, scheme, codec, key)
+
+	watching, err := h.WatchList(context.TODO(), key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer watching.Stop()
+
+	// creates key/foo which should trigger the WatchList for "key"
+	err = h.Create(context.TODO(), key, pod, pod, 0)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	// force context switch to ensure watches would catch and notify.
+	rt.Gosched()
+
+	select {
+	case event, _ := <-watching.ResultChan():
+		t.Fatalf("Unexpected event: %#v", event)
+	default:
+		// fall through, expected behavior
+	}
+}
+
+func TestWatchPurposefulShutdown(t *testing.T) {
+	scheme, codecs := testScheme(t)
+	codec := codecs.LegacyCodec(schema.GroupVersion{Version: "v1"})
+	server := etcdtesting.NewEtcdTestClientServer(t)
+	defer server.Terminate(t)
+	key := "/some/key"
+	h := newEtcdHelper(server.Client, scheme, codec, etcdtest.PathPrefix())
+
+	// Test purposeful shutdown
+	watching, err := h.Watch(context.TODO(), key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	watching.Stop()
+	rt.Gosched()
+
+	// There is a race in etcdWatcher so that after calling Stop() one of
+	// two things can happen:
+	// - ResultChan() may be closed (triggered by closing userStop channel)
+	// - an Error "context cancelled" may be emitted (triggered by cancelling request
+	//   to etcd and putting that error to etcdError channel)
+	// We need to be prepared for both here.
+	event, open := <-watching.ResultChan()
+	if open && event.Type != watch.Error {
+		t.Errorf("Unexpected event from stopped watcher: %#v", event)
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/etcdtest/BUILD

@@ -0,0 +1,27 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "etcdtest.go",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/etcd/etcdtest/doc.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcdtest // import "k8s.io/apiserver/pkg/storage/etcd/etcdtest"

vendor/k8s.io/apiserver/pkg/storage/etcd/etcdtest/etcdtest.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcdtest
+
+import (
+	"os"
+	"path"
+)
+
+// Cache size to use for tests.
+const DeserializationCacheSize = 150
+
+// Returns the prefix set via the ETCD_PREFIX environment variable (if any).
+func PathPrefix() string {
+	pref := os.Getenv("ETCD_PREFIX")
+	if pref == "" {
+		pref = "registry"
+	}
+	return path.Join("/", pref)
+}
+
+// Adds the ETCD_PREFIX to the provided key
+func AddPrefix(in string) string {
+	return path.Join(PathPrefix(), in)
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/metrics/BUILD

@@ -0,0 +1,25 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["metrics.go"],
+    deps = ["//vendor/github.com/prometheus/client_golang/prometheus:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/etcd/metrics/metrics.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	cacheHitCounterOpts = prometheus.CounterOpts{
+		Name: "etcd_helper_cache_hit_count",
+		Help: "Counter of etcd helper cache hits.",
+	}
+	cacheHitCounter      = prometheus.NewCounter(cacheHitCounterOpts)
+	cacheMissCounterOpts = prometheus.CounterOpts{
+		Name: "etcd_helper_cache_miss_count",
+		Help: "Counter of etcd helper cache miss.",
+	}
+	cacheMissCounter      = prometheus.NewCounter(cacheMissCounterOpts)
+	cacheEntryCounterOpts = prometheus.CounterOpts{
+		Name: "etcd_helper_cache_entry_count",
+		Help: "Counter of etcd helper cache entries. This can be different from etcd_helper_cache_miss_count " +
+			"because two concurrent threads can miss the cache and generate the same entry twice.",
+	}
+	cacheEntryCounter = prometheus.NewCounter(cacheEntryCounterOpts)
+	cacheGetLatency   = prometheus.NewSummary(
+		prometheus.SummaryOpts{
+			Name: "etcd_request_cache_get_latencies_summary",
+			Help: "Latency in microseconds of getting an object from etcd cache",
+		},
+	)
+	cacheAddLatency = prometheus.NewSummary(
+		prometheus.SummaryOpts{
+			Name: "etcd_request_cache_add_latencies_summary",
+			Help: "Latency in microseconds of adding an object to etcd cache",
+		},
+	)
+	etcdRequestLatenciesSummary = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "etcd_request_latencies_summary",
+			Help: "Etcd request latency summary in microseconds for each operation and object type.",
+		},
+		[]string{"operation", "type"},
+	)
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	// Register the metrics.
+	registerMetrics.Do(func() {
+		prometheus.MustRegister(cacheHitCounter)
+		prometheus.MustRegister(cacheMissCounter)
+		prometheus.MustRegister(cacheEntryCounter)
+		prometheus.MustRegister(cacheAddLatency)
+		prometheus.MustRegister(cacheGetLatency)
+		prometheus.MustRegister(etcdRequestLatenciesSummary)
+	})
+}
+
+func RecordEtcdRequestLatency(verb, resource string, startTime time.Time) {
+	etcdRequestLatenciesSummary.WithLabelValues(verb, resource).Observe(float64(time.Since(startTime) / time.Microsecond))
+}
+
+func ObserveGetCache(startTime time.Time) {
+	cacheGetLatency.Observe(float64(time.Since(startTime) / time.Microsecond))
+}
+
+func ObserveAddCache(startTime time.Time) {
+	cacheAddLatency.Observe(float64(time.Since(startTime) / time.Microsecond))
+}
+
+func ObserveCacheHit() {
+	cacheHitCounter.Inc()
+}
+
+func ObserveCacheMiss() {
+	cacheMissCounter.Inc()
+}
+
+func ObserveNewEntry() {
+	cacheEntryCounter.Inc()
+}
+
+func Reset() {
+	cacheHitCounter = prometheus.NewCounter(cacheHitCounterOpts)
+	cacheMissCounter = prometheus.NewCounter(cacheMissCounterOpts)
+	cacheEntryCounter = prometheus.NewCounter(cacheEntryCounterOpts)
+	// TODO: Reset cacheAddLatency.
+	// TODO: Reset cacheGetLatency.
+	etcdRequestLatenciesSummary.Reset()
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/testing/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["utils.go"],
+    deps = [
+        "//vendor/github.com/coreos/etcd/client:go_default_library",
+        "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
+        "//vendor/github.com/coreos/etcd/etcdserver:go_default_library",
+        "//vendor/github.com/coreos/etcd/etcdserver/api/v2http:go_default_library",
+        "//vendor/github.com/coreos/etcd/integration:go_default_library",
+        "//vendor/github.com/coreos/etcd/pkg/testutil:go_default_library",
+        "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
+        "//vendor/github.com/coreos/etcd/pkg/types:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/etcdtest:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/testing/testingcert:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/testingcert:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/etcd/testing/testingcert/BUILD

@@ -0,0 +1,24 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["certificates.go"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/etcd/testing/testingcert/certificates.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testingcert
+
+// You can use cfssl tool to generate certificates, please refer
+// https://github.com/coreos/etcd/tree/master/hack/tls-setup for more details.
+//
+// ca-config.json:
+//   expiry was changed from 1 year to 100 years (876000h)
+// ca-csr.json:
+//   ca expiry was set to 100 years (876000h) ("ca":{"expiry":"876000h"})
+//   key was changed from ecdsa,384 to rsa,2048
+// req-csr.json:
+//   key was changed from ecdsa,384 to rsa,2048
+//   hosts were changed to "localhost","127.0.0.1"
+const CAFileContent = `
+-----BEGIN CERTIFICATE-----
+MIIEUDCCAzigAwIBAgIUKfV5+qwlw3JneAPdJS7JCO8xIlYwDQYJKoZIhvcNAQEL
+BQAwgawxCzAJBgNVBAYTAlVTMSowKAYDVQQKEyFIb25lc3QgQWNobWVkJ3MgVXNl
+ZCBDZXJ0aWZpY2F0ZXMxKTAnBgNVBAsTIEhhc3RpbHktR2VuZXJhdGVkIFZhbHVl
+cyBEaXZpc29uMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRMwEQYDVQQIEwpDYWxp
+Zm9ybmlhMRkwFwYDVQQDExBBdXRvZ2VuZXJhdGVkIENBMCAXDTE2MDMxMjIzMTQw
+MFoYDzIxMTYwMjE3MjMxNDAwWjCBrDELMAkGA1UEBhMCVVMxKjAoBgNVBAoTIUhv
+bmVzdCBBY2htZWQncyBVc2VkIENlcnRpZmljYXRlczEpMCcGA1UECxMgSGFzdGls
+eS1HZW5lcmF0ZWQgVmFsdWVzIERpdmlzb24xFjAUBgNVBAcTDVNhbiBGcmFuY2lz
+Y28xEzARBgNVBAgTCkNhbGlmb3JuaWExGTAXBgNVBAMTEEF1dG9nZW5lcmF0ZWQg
+Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDP+acpr1USrObZFu+6
+v+Bk6rYw+sWynP373cNUUiHfnZ3D7f9yJsDscV0Mo4R8DddqkxawrA5fK2Fm2Z9G
+vvY5par4/JbwRIEkXmeM4e52Mqv0Yuoz62O+0jQvRawnCCJMcKuo+ijHMjmm0AF1
+JdhTpTgvUwEP9WtY9JVTkfMCnDqZiqOU5D+d4YWUtkKqgQNvbZRs6wGubhMCZe8X
+m+3bK8YAsWWtoFgr7plxXk4D8MLh+PqJ3oJjfxfW5A9dHbnSEmdZ3vrYwrKgyfNf
+bvHE5qQmiSZUbUaCw3mKfaEMCNesPT46nBHxhAWc5aiL1tOXzvV5Uze7A7huPoI9
+a3etAgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEC
+MB0GA1UdDgQWBBQYc0xXQ6VNjFvIOqWfXorxx9rKRzAfBgNVHSMEGDAWgBQYc0xX
+Q6VNjFvIOqWfXorxx9rKRzANBgkqhkiG9w0BAQsFAAOCAQEAaKyHDWYVjEyEKTXJ
+qS9r46ehL5FZlWD2ZytBP8aHE307l9AfQ+DFWldCNaqMXLZozsresVaSzSOI6UUD
+lCIQLDpPyxbpR320u8mC08+lhhwR/YRkrEqKHk56Wl4OaqoyWmguqYU9p0DiQeTU
+sZsxOwG7cyEEvvs+XmZ/vBLBOr59xyjwn4seQqzwZj3VYeiKLw40iQt1yT442rcP
+CfdlE9wTEONvWT+kBGMt0JlalXH3jFvlfcGQdDfRmDeTJtA+uIbvJhwJuGCNHHAc
+xqC+4mAGBPN/dMPXpjayHD5dOXIKLfrNpqse6jImYlY9zduvwIHRDK/zvqTyPlNZ
+uR84Nw==
+-----END CERTIFICATE-----
+`
+const CertFileContent = `
+-----BEGIN CERTIFICATE-----
+MIIELzCCAxegAwIBAgIUcjkJA3cmHeoBQggaKZmfKebFL9cwDQYJKoZIhvcNAQEL
+BQAwgawxCzAJBgNVBAYTAlVTMSowKAYDVQQKEyFIb25lc3QgQWNobWVkJ3MgVXNl
+ZCBDZXJ0aWZpY2F0ZXMxKTAnBgNVBAsTIEhhc3RpbHktR2VuZXJhdGVkIFZhbHVl
+cyBEaXZpc29uMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRMwEQYDVQQIEwpDYWxp
+Zm9ybmlhMRkwFwYDVQQDExBBdXRvZ2VuZXJhdGVkIENBMCAXDTE2MDMxMjIzMTQw
+MFoYDzIxMTYwMjE3MjMxNDAwWjBVMRYwFAYDVQQKEw1hdXRvZ2VuZXJhdGVkMRUw
+EwYDVQQLEwxldGNkIGNsdXN0ZXIxFTATBgNVBAcTDHRoZSBpbnRlcm5ldDENMAsG
+A1UEAxMEZXRjZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOiW5A65
+hWGbnwceoZHM0+OexU4cPF/FpP+7BOK5i7ymSWAqfKfNuio2TB1lAErC1oX7bgTX
+ieP10uz3FYWQNrlDn0I4KSA888rFPtx8GwoxH/52fGlE80BUV9PNeOVP+mYza0ih
+oFj2+PhXVL/JZbx9P/2RLSNbEnq+OPk8AN82SkNtpFzanwtpb3f+kt73878KNoQu
+xYZaCF1sK45Kn7mjKSDu/b3xUbTrNwnyVAGOdLzI7CCWOu+ECoZYAH4ZNHHakbyY
+eWQ7U9leocEOPlqxsQAKodaCYjuAaOFIcz8/W81q+3qNw/6GbZ4znjRKQ3OtIPZ4
+JH1iNofCudWDp+0CAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFMJE
+43qLCWhyZAE/wxNneSJw7aUVMB8GA1UdIwQYMBaAFBhzTFdDpU2MW8g6pZ9eivHH
+2spHMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOC
+AQEAuELC8tbmpyKlA4HLSDHOUquypNyiE6ftBIifJtp8bvBd+jiv4Pr8oVGxHoqq
+48X7lamvDirLV5gmK0CxO+EXkIUHhULzPyYPynqsR7KZlk1PWghqsF65nwqcjS3b
+tykLttD1AUDIozYvujVYBKXGxb6jcGM1rBF1XtslciFZ5qQnj6dTUujo9/xBA2ql
+kOKiVXBNU8KFzq4c20RzHFLfWkbc30Q4XG4dTDVBeGupnFQRkZ0y2dSSU82QcLA/
+HgAyQSO7+csN13r84zbmDuRpUgo6eTXzJ+77G19KDkEL7XEtlw2jB2L6/o+3RGtw
+JLOpEsgi7hsvOYCuTA3Krw52Mw==
+-----END CERTIFICATE-----
+`
+const KeyFileContent = `
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA6JbkDrmFYZufBx6hkczT457FThw8X8Wk/7sE4rmLvKZJYCp8
+p826KjZMHWUASsLWhftuBNeJ4/XS7PcVhZA2uUOfQjgpIDzzysU+3HwbCjEf/nZ8
+aUTzQFRX08145U/6ZjNrSKGgWPb4+FdUv8llvH0//ZEtI1sSer44+TwA3zZKQ22k
+XNqfC2lvd/6S3vfzvwo2hC7FhloIXWwrjkqfuaMpIO79vfFRtOs3CfJUAY50vMjs
+IJY674QKhlgAfhk0cdqRvJh5ZDtT2V6hwQ4+WrGxAAqh1oJiO4Bo4UhzPz9bzWr7
+eo3D/oZtnjOeNEpDc60g9ngkfWI2h8K51YOn7QIDAQABAoIBAQCj88Fc08++x0kp
+ZqEzunPebsvcTLEOPa8aiUVfYLWszHbKsAhg7Pb+zHmI+upiyMcZeOvLw/eyVlVR
+rrZgCRFaNN2texMaY3zigXnXSDBzVb+cyv7V4cGqpgmnBp7i3ia/Jh3I/A2gyK8l
+t8HI03nAjXWvE0gDNS5okXBt16sxq6ZWyzHHVbN3UYtCDxnyh2Ibck4b+K8I8Bn1
+mwMsSqPXJS1UQ3U5UqcaMs7WOEGx+xmaPJTWm5Lb//BkakGuBTQj+7wotyXQYG5U
+uZdPPcFRk6cqgjzUeKVUtGkdmfgHSTdIwZowkKibB4rdrudsRnSwfeB+83Jp9JwG
+JPrGvsbNAoGBAPULIO+vVBZXVpUEAhvNSXtmOi/hAbQhOuix8iwHbJ5EbrWaDn4B
+Reb2cw/fZGgGG4jtAOXdiY8R1XGGP8+RPZ5El10ZWnNrKQfpZ27gK/5yeq5dfGBG
+4JLUpcrT180FJo00rgiQYJnHCk1fWrnzXNV6K08ZZHGr6yv4S/jbq/7vAoGBAPL9
+NTN/UWXWFlSHVcb2dFHcvIiPwRj9KwhuMu90b/CilBbSJ1av13xtf2ar5zkrEtWH
+CB3q3wBaklQP9MfOqEWGZeOUcd9AbYWtxHjHmP5fJA9RjErjlTtqGkusNtZJbchU
+UWfT/Tl9pREpCvJ/8iawc1hx7sHHKzYwnDnMaQbjAoGAfJdd9cBltr5NjZLuJ4in
+dhCyQSncnePPegUQJwbXWVleGQPtnm+zRQ3Fzyo8eQ+x7Frk+/s6N/5PUlt6EmW8
+uL4TYAjGDq1LvXQVXTCp7cPzULjDxogDI2Tvr0MrFFksEtvYKQ6Pr2CeglybWrS8
+XOazIpK8mXdaKY8jwbKfrw0CgYAFnfrb3OaZzxAnFhXSiqH3vn2RPpl9JWUYRcvh
+ozRvQKLhwCvuohP+KV3XlsO6m5dM3lk+r85F6NIXJWNINyvGp6u1ThovygJ+I502
+GY8c2kAwJndyx74MaJCBDVMbMwlZpzFWkBz7dj8ZnXRGVNTZNh0Ef2XAjwUdtJP3
+9hS7dwKBgQDCzq0RIxFyy3F5baGHWLVICxmhNExQ2+Vebh+DvsPKtnz6OrWdRbGX
+wgGVLrn53s6eCblnXLtKr/Li+t7fS8IkQkvu5guOvI9VeVUmZhFET3GVmUxu+JTb
+iQY4uBgaf8Fgay4dkOfjvlOpFDR4E7UbJpg8/cFKTrpwgOiUVyFVdQ==
+-----END RSA PRIVATE KEY-----
+`

vendor/k8s.io/apiserver/pkg/storage/etcd/testing/utils.go

@@ -0,0 +1,328 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/storage/etcd/etcdtest"
+	"k8s.io/apiserver/pkg/storage/etcd/testing/testingcert"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+
+	etcd "github.com/coreos/etcd/client"
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/etcdserver"
+	"github.com/coreos/etcd/etcdserver/api/v2http"
+	"github.com/coreos/etcd/integration"
+	"github.com/coreos/etcd/pkg/testutil"
+	"github.com/coreos/etcd/pkg/transport"
+	"github.com/coreos/etcd/pkg/types"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+)
+
+// EtcdTestServer encapsulates the datastructures needed to start local instance for testing
+type EtcdTestServer struct {
+	// The following are lumped etcd2 test server params
+	// TODO: Deprecate in a post 1.5 release
+	etcdserver.ServerConfig
+	PeerListeners, ClientListeners []net.Listener
+	Client                         etcd.Client
+
+	CertificatesDir string
+	CertFile        string
+	KeyFile         string
+	CAFile          string
+
+	raftHandler http.Handler
+	s           *etcdserver.EtcdServer
+	hss         []*httptest.Server
+
+	// The following are lumped etcd3 test server params
+	v3Cluster *integration.ClusterV3
+	V3Client  *clientv3.Client
+}
+
+// newLocalListener opens a port localhost using any port
+func newLocalListener(t *testing.T) net.Listener {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return l
+}
+
+// newSecuredLocalListener opens a port localhost using any port
+// with SSL enable
+func newSecuredLocalListener(t *testing.T, certFile, keyFile, caFile string) net.Listener {
+	var l net.Listener
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tlsInfo := transport.TLSInfo{
+		CertFile: certFile,
+		KeyFile:  keyFile,
+		CAFile:   caFile,
+	}
+	tlscfg, err := tlsInfo.ServerConfig()
+	if err != nil {
+		t.Fatalf("unexpected serverConfig error: %v", err)
+	}
+	l, err = transport.NewKeepAliveListener(l, "https", tlscfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return l
+}
+
+func newHttpTransport(t *testing.T, certFile, keyFile, caFile string) etcd.CancelableTransport {
+	tlsInfo := transport.TLSInfo{
+		CertFile: certFile,
+		KeyFile:  keyFile,
+		CAFile:   caFile,
+	}
+	tr, err := transport.NewTransport(tlsInfo, time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tr
+}
+
+// configureTestCluster will set the params to start an etcd server
+func configureTestCluster(t *testing.T, name string, https bool) *EtcdTestServer {
+	var err error
+	m := &EtcdTestServer{}
+
+	pln := newLocalListener(t)
+	m.PeerListeners = []net.Listener{pln}
+	m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Allow test launches to control where etcd data goes, for space or performance reasons
+	baseDir := os.Getenv("TEST_ETCD_DIR")
+	if len(baseDir) == 0 {
+		baseDir = os.TempDir()
+	}
+
+	if https {
+		m.CertificatesDir, err = ioutil.TempDir(baseDir, "etcd_certificates")
+		if err != nil {
+			t.Fatal(err)
+		}
+		m.CertFile = path.Join(m.CertificatesDir, "etcdcert.pem")
+		if err = ioutil.WriteFile(m.CertFile, []byte(testingcert.CertFileContent), 0644); err != nil {
+			t.Fatal(err)
+		}
+		m.KeyFile = path.Join(m.CertificatesDir, "etcdkey.pem")
+		if err = ioutil.WriteFile(m.KeyFile, []byte(testingcert.KeyFileContent), 0644); err != nil {
+			t.Fatal(err)
+		}
+		m.CAFile = path.Join(m.CertificatesDir, "ca.pem")
+		if err = ioutil.WriteFile(m.CAFile, []byte(testingcert.CAFileContent), 0644); err != nil {
+			t.Fatal(err)
+		}
+
+		cln := newSecuredLocalListener(t, m.CertFile, m.KeyFile, m.CAFile)
+		m.ClientListeners = []net.Listener{cln}
+		m.ClientURLs, err = types.NewURLs([]string{"https://" + cln.Addr().String()})
+		if err != nil {
+			t.Fatal(err)
+		}
+	} else {
+		cln := newLocalListener(t)
+		m.ClientListeners = []net.Listener{cln}
+		m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	m.Name = name
+	m.DataDir, err = ioutil.TempDir(baseDir, "etcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String())
+	m.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m.InitialClusterToken = "TestEtcd"
+	m.NewCluster = true
+	m.ForceNewCluster = false
+	m.ElectionTicks = 10
+	m.TickMs = uint(10)
+
+	return m
+}
+
+// launch will attempt to start the etcd server
+func (m *EtcdTestServer) launch(t *testing.T) error {
+	var err error
+	if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
+		return fmt.Errorf("failed to initialize the etcd server: %v", err)
+	}
+	m.s.SyncTicker = time.Tick(500 * time.Millisecond)
+	m.s.Start()
+	m.raftHandler = &testutil.PauseableHandler{Next: v2http.NewPeerHandler(m.s)}
+	for _, ln := range m.PeerListeners {
+		hs := &httptest.Server{
+			Listener: ln,
+			Config:   &http.Server{Handler: m.raftHandler},
+		}
+		hs.Start()
+		m.hss = append(m.hss, hs)
+	}
+	for _, ln := range m.ClientListeners {
+		hs := &httptest.Server{
+			Listener: ln,
+			Config:   &http.Server{Handler: v2http.NewClientHandler(m.s, m.ServerConfig.ReqTimeout())},
+		}
+		hs.Start()
+		m.hss = append(m.hss, hs)
+	}
+	return nil
+}
+
+// waitForEtcd wait until etcd is propagated correctly
+func (m *EtcdTestServer) waitUntilUp() error {
+	membersAPI := etcd.NewMembersAPI(m.Client)
+	for start := time.Now(); time.Since(start) < wait.ForeverTestTimeout; time.Sleep(10 * time.Millisecond) {
+		members, err := membersAPI.List(context.TODO())
+		if err != nil {
+			glog.Errorf("Error when getting etcd cluster members")
+			continue
+		}
+		if len(members) == 1 && len(members[0].ClientURLs) > 0 {
+			return nil
+		}
+	}
+	return fmt.Errorf("timeout on waiting for etcd cluster")
+}
+
+// Terminate will shutdown the running etcd server
+func (m *EtcdTestServer) Terminate(t *testing.T) {
+	if m.v3Cluster != nil {
+		m.v3Cluster.Terminate(t)
+	} else {
+		m.Client = nil
+		m.s.Stop()
+		// TODO: This is a pretty ugly hack to workaround races during closing
+		// in-memory etcd server in unit tests - see #18928 for more details.
+		// We should get rid of it as soon as we have a proper fix - etcd clients
+		// have overwritten transport counting opened connections (probably by
+		// overwriting Dial function) and termination function waiting for all
+		// connections to be closed and stopping accepting new ones.
+		time.Sleep(250 * time.Millisecond)
+		for _, hs := range m.hss {
+			hs.CloseClientConnections()
+			hs.Close()
+		}
+		if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
+			t.Fatal(err)
+		}
+		if len(m.CertificatesDir) > 0 {
+			if err := os.RemoveAll(m.CertificatesDir); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+}
+
+// NewEtcdTestClientServer DEPRECATED creates a new client and server for testing
+func NewEtcdTestClientServer(t *testing.T) *EtcdTestServer {
+	server := configureTestCluster(t, "foo", true)
+	err := server.launch(t)
+	if err != nil {
+		t.Fatalf("Failed to start etcd server error=%v", err)
+		return nil
+	}
+
+	cfg := etcd.Config{
+		Endpoints: server.ClientURLs.StringSlice(),
+		Transport: newHttpTransport(t, server.CertFile, server.KeyFile, server.CAFile),
+	}
+	server.Client, err = etcd.New(cfg)
+	if err != nil {
+		server.Terminate(t)
+		t.Fatalf("Unexpected error in NewEtcdTestClientServer (%v)", err)
+		return nil
+	}
+	if err := server.waitUntilUp(); err != nil {
+		server.Terminate(t)
+		t.Fatalf("Unexpected error in waitUntilUp (%v)", err)
+		return nil
+	}
+	return server
+}
+
+// NewUnsecuredEtcdTestClientServer DEPRECATED creates a new client and server for testing
+func NewUnsecuredEtcdTestClientServer(t *testing.T) *EtcdTestServer {
+	server := configureTestCluster(t, "foo", false)
+	err := server.launch(t)
+	if err != nil {
+		t.Fatalf("Failed to start etcd server error=%v", err)
+		return nil
+	}
+	cfg := etcd.Config{
+		Endpoints: server.ClientURLs.StringSlice(),
+		Transport: newHttpTransport(t, server.CertFile, server.KeyFile, server.CAFile),
+	}
+	server.Client, err = etcd.New(cfg)
+	if err != nil {
+		t.Errorf("Unexpected error in NewUnsecuredEtcdTestClientServer (%v)", err)
+		server.Terminate(t)
+		return nil
+	}
+	if err := server.waitUntilUp(); err != nil {
+		t.Errorf("Unexpected error in waitUntilUp (%v)", err)
+		server.Terminate(t)
+		return nil
+	}
+	return server
+}
+
+// NewEtcd3TestClientServer creates a new client and server for testing
+func NewUnsecuredEtcd3TestClientServer(t *testing.T, scheme *runtime.Scheme) (*EtcdTestServer, *storagebackend.Config) {
+	server := &EtcdTestServer{
+		v3Cluster: integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}),
+	}
+	server.V3Client = server.v3Cluster.RandClient()
+	config := &storagebackend.Config{
+		Type:                     "etcd3",
+		Prefix:                   etcdtest.PathPrefix(),
+		ServerList:               server.V3Client.Endpoints(),
+		DeserializationCacheSize: etcdtest.DeserializationCacheSize,
+		Copier: scheme,
+		Paging: true,
+	}
+	return server, config
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/util/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["etcd_util_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/coreos/etcd/client:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "etcd_util.go",
+    ],
+    deps = ["//vendor/github.com/coreos/etcd/client:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/etcd/util/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package util holds generic etcd-related utility functions that any user of ectd might want to
+// use, without pulling in kubernetes-specific code.
+package util // import "k8s.io/apiserver/pkg/storage/etcd/util"

vendor/k8s.io/apiserver/pkg/storage/etcd/util/etcd_util.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	etcd "github.com/coreos/etcd/client"
+)
+
+// IsEtcdNotFound returns true if and only if err is an etcd not found error.
+func IsEtcdNotFound(err error) bool {
+	return isEtcdErrorNum(err, etcd.ErrorCodeKeyNotFound)
+}
+
+// IsEtcdNodeExist returns true if and only if err is an etcd node already exist error.
+func IsEtcdNodeExist(err error) bool {
+	return isEtcdErrorNum(err, etcd.ErrorCodeNodeExist)
+}
+
+// IsEtcdTestFailed returns true if and only if err is an etcd write conflict.
+func IsEtcdTestFailed(err error) bool {
+	return isEtcdErrorNum(err, etcd.ErrorCodeTestFailed)
+}
+
+// IsEtcdWatchExpired returns true if and only if err indicates the watch has expired.
+func IsEtcdWatchExpired(err error) bool {
+	// NOTE: This seems weird why it wouldn't be etcd.ErrorCodeWatcherCleared
+	//       I'm using the previous matching value
+	return isEtcdErrorNum(err, etcd.ErrorCodeEventIndexCleared)
+}
+
+// IsEtcdUnreachable returns true if and only if err indicates the server could not be reached.
+func IsEtcdUnreachable(err error) bool {
+	// NOTE: The logic has changed previous error code no longer applies
+	return err == etcd.ErrClusterUnavailable
+}
+
+// isEtcdErrorNum returns true if and only if err is an etcd error, whose errorCode matches errorCode
+func isEtcdErrorNum(err error, errorCode int) bool {
+	if err != nil {
+		if etcdError, ok := err.(etcd.Error); ok {
+			return etcdError.Code == errorCode
+		}
+		// NOTE: There are other error types returned
+	}
+	return false
+}
+
+// GetEtcdVersion performs a version check against the provided Etcd server,
+// returning the string response, and error (if any).
+func GetEtcdVersion(host string) (string, error) {
+	response, err := http.Get(host + "/version")
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unsuccessful response from etcd server %q: %v", host, err)
+	}
+	versionBytes, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(versionBytes), nil
+}
+
+type etcdHealth struct {
+	// Note this has to be public so the json library can modify it.
+	Health string `json:"health"`
+}
+
+func EtcdHealthCheck(data []byte) error {
+	obj := etcdHealth{}
+	if err := json.Unmarshal(data, &obj); err != nil {
+		return err
+	}
+	if obj.Health != "true" {
+		return fmt.Errorf("Unhealthy status: %s", obj.Health)
+	}
+	return nil
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/util/etcd_util_test.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	etcd "github.com/coreos/etcd/client"
+	"github.com/stretchr/testify/assert"
+)
+
+const validEtcdVersion = "etcd 2.0.9"
+
+func TestIsEtcdNotFound(t *testing.T) {
+	try := func(err error, isNotFound bool) {
+		if IsEtcdNotFound(err) != isNotFound {
+			t.Errorf("Expected %#v to return %v, but it did not", err, isNotFound)
+		}
+	}
+	try(&etcd.Error{Code: 101}, false)
+	try(nil, false)
+	try(fmt.Errorf("some other kind of error"), false)
+}
+
+func TestGetEtcdVersion_ValidVersion(t *testing.T) {
+	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, validEtcdVersion)
+	}))
+	defer testServer.Close()
+
+	var version string
+	var err error
+	if version, err = GetEtcdVersion(testServer.URL); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	assert.Equal(t, validEtcdVersion, version, "Unexpected version")
+	assert.Nil(t, err)
+}
+
+func TestGetEtcdVersion_ErrorStatus(t *testing.T) {
+	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	defer testServer.Close()
+
+	_, err := GetEtcdVersion(testServer.URL)
+	assert.NotNil(t, err)
+}
+
+func TestGetEtcdVersion_NotListening(t *testing.T) {
+	portIsOpen := func(port int) bool {
+		conn, err := net.DialTimeout("tcp", "127.0.0.1:"+strconv.Itoa(port), 1*time.Second)
+		if err == nil {
+			conn.Close()
+			return true
+		}
+		return false
+	}
+
+	port := rand.Intn((1 << 16) - 1)
+	for tried := 0; portIsOpen(port); tried++ {
+		if tried >= 10 {
+			t.Fatal("Couldn't find a closed TCP port to continue testing")
+		}
+		port++
+	}
+
+	_, err := GetEtcdVersion("http://127.0.0.1:" + strconv.Itoa(port))
+	assert.NotNil(t, err)
+}
+
+func TestEtcdHealthCheck(t *testing.T) {
+	tests := []struct {
+		data      string
+		expectErr bool
+	}{
+		{
+			data:      "{\"health\": \"true\"}",
+			expectErr: false,
+		},
+		{
+			data:      "{\"health\": \"false\"}",
+			expectErr: true,
+		},
+		{
+			data:      "invalid json",
+			expectErr: true,
+		},
+	}
+	for _, test := range tests {
+		err := EtcdHealthCheck([]byte(test.data))
+		if err != nil && !test.expectErr {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if err == nil && test.expectErr {
+			t.Error("unexpected non-error")
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/BUILD

@@ -0,0 +1,83 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "compact_test.go",
+        "store_test.go",
+        "watcher_test.go",
+    ],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
+        "//vendor/github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes:go_default_library",
+        "//vendor/github.com/coreos/etcd/integration:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/testing:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/tests:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "compact.go",
+        "errors.go",
+        "event.go",
+        "store.go",
+        "watcher.go",
+    ],
+    deps = [
+        "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
+        "//vendor/github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes:go_default_library",
+        "//vendor/github.com/coreos/etcd/mvcc/mvccpb:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/etcd3/OWNERS

@@ -0,0 +1,5 @@
+reviewers:
+- wojtek-t
+- timothysc
+- madhusudancs
+- hongchaodeng

vendor/k8s.io/apiserver/pkg/storage/etcd3/compact.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+)
+
+const (
+	compactInterval = 5 * time.Minute
+	compactRevKey   = "compact_rev_key"
+)
+
+var (
+	endpointsMapMu sync.Mutex
+	endpointsMap   map[string]struct{}
+)
+
+func init() {
+	endpointsMap = make(map[string]struct{})
+}
+
+// StartCompactor starts a compactor in the background to compact old version of keys that's not needed.
+// By default, we save the most recent 10 minutes data and compact versions > 10minutes ago.
+// It should be enough for slow watchers and to tolerate burst.
+// TODO: We might keep a longer history (12h) in the future once storage API can take advantage of past version of keys.
+func StartCompactor(ctx context.Context, client *clientv3.Client) {
+	endpointsMapMu.Lock()
+	defer endpointsMapMu.Unlock()
+
+	// In one process, we can have only one compactor for one cluster.
+	// Currently we rely on endpoints to differentiate clusters.
+	for _, ep := range client.Endpoints() {
+		if _, ok := endpointsMap[ep]; ok {
+			glog.V(4).Infof("compactor already exists for endpoints %v", client.Endpoints())
+			return
+		}
+	}
+	for _, ep := range client.Endpoints() {
+		endpointsMap[ep] = struct{}{}
+	}
+
+	go compactor(ctx, client, compactInterval)
+}
+
+// compactor periodically compacts historical versions of keys in etcd.
+// It will compact keys with versions older than given interval.
+// In other words, after compaction, it will only contain keys set during last interval.
+// Any API call for the older versions of keys will return error.
+// Interval is the time interval between each compaction. The first compaction happens after "interval".
+func compactor(ctx context.Context, client *clientv3.Client, interval time.Duration) {
+	// Technical definitions:
+	// We have a special key in etcd defined as *compactRevKey*.
+	// compactRevKey's value will be set to the string of last compacted revision.
+	// compactRevKey's version will be used as logical time for comparison. THe version is referred as compact time.
+	// Initially, because the key doesn't exist, the compact time (version) is 0.
+	//
+	// Algorithm:
+	// - Compare to see if (local compact_time) = (remote compact_time).
+	// - If yes, increment both local and remote compact_time, and do a compaction.
+	// - If not, set local to remote compact_time.
+	//
+	// Technical details/insights:
+	//
+	// The protocol here is lease based. If one compactor CAS successfully, the others would know it when they fail in
+	// CAS later and would try again in 10 minutes. If an APIServer crashed, another one would "take over" the lease.
+	//
+	// For example, in the following diagram, we have a compactor C1 doing compaction in t1, t2. Another compactor C2
+	// at t1' (t1 < t1' < t2) would CAS fail, set its known oldRev to rev at t1', and try again in t2' (t2' > t2).
+	// If C1 crashed and wouldn't compact at t2, C2 would CAS successfully at t2'.
+	//
+	//                 oldRev(t2)     curRev(t2)
+	//                                  +
+	//   oldRev        curRev           |
+	//     +             +              |
+	//     |             |              |
+	//     |             |    t1'       |     t2'
+	// +---v-------------v----^---------v------^---->
+	//     t0           t1             t2
+	//
+	// We have the guarantees:
+	// - in normal cases, the interval is 10 minutes.
+	// - in failover, the interval is >10m and <20m
+	//
+	// FAQ:
+	// - What if time is not accurate? We don't care as long as someone did the compaction. Atomicity is ensured using
+	//   etcd API.
+	// - What happened under heavy load scenarios? Initially, each apiserver will do only one compaction
+	//   every 10 minutes. This is very unlikely affecting or affected w.r.t. server load.
+
+	var compactTime int64
+	var rev int64
+	var err error
+	for {
+		select {
+		case <-time.After(interval):
+		case <-ctx.Done():
+			return
+		}
+
+		compactTime, rev, err = compact(ctx, client, compactTime, rev)
+		if err != nil {
+			glog.Errorf("etcd: endpoint (%v) compact failed: %v", client.Endpoints(), err)
+			continue
+		}
+	}
+}
+
+// compact compacts etcd store and returns current rev.
+// It will return the current compact time and global revision if no error occurred.
+// Note that CAS fail will not incur any error.
+func compact(ctx context.Context, client *clientv3.Client, t, rev int64) (int64, int64, error) {
+	resp, err := client.KV.Txn(ctx).If(
+		clientv3.Compare(clientv3.Version(compactRevKey), "=", t),
+	).Then(
+		clientv3.OpPut(compactRevKey, strconv.FormatInt(rev, 10)), // Expect side effect: increment Version
+	).Else(
+		clientv3.OpGet(compactRevKey),
+	).Commit()
+	if err != nil {
+		return t, rev, err
+	}
+
+	curRev := resp.Header.Revision
+
+	if !resp.Succeeded {
+		curTime := resp.Responses[0].GetResponseRange().Kvs[0].Version
+		return curTime, curRev, nil
+	}
+	curTime := t + 1
+
+	if rev == 0 {
+		// We don't compact on bootstrap.
+		return curTime, curRev, nil
+	}
+	if _, err = client.Compact(ctx, rev); err != nil {
+		return curTime, curRev, err
+	}
+	glog.V(4).Infof("etcd: compacted rev (%d), endpoints (%v)", rev, client.Endpoints())
+	return curTime, curRev, nil
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/compact_test.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"testing"
+
+	"github.com/coreos/etcd/clientv3"
+	etcdrpc "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	"github.com/coreos/etcd/integration"
+	"golang.org/x/net/context"
+)
+
+func TestCompact(t *testing.T) {
+	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer cluster.Terminate(t)
+	client := cluster.RandClient()
+	ctx := context.Background()
+
+	putResp, err := client.Put(ctx, "/somekey", "data")
+	if err != nil {
+		t.Fatalf("Put failed: %v", err)
+	}
+
+	putResp1, err := client.Put(ctx, "/somekey", "data2")
+	if err != nil {
+		t.Fatalf("Put failed: %v", err)
+	}
+
+	_, _, err = compact(ctx, client, 0, putResp1.Header.Revision)
+	if err != nil {
+		t.Fatalf("compact failed: %v", err)
+	}
+
+	obj, err := client.Get(ctx, "/somekey", clientv3.WithRev(putResp.Header.Revision))
+	if err != etcdrpc.ErrCompacted {
+		t.Errorf("Expecting ErrCompacted, but get=%v err=%v", obj, err)
+	}
+}
+
+// TestCompactConflict tests that two compactors (Let's use C1, C2) are trying to compact etcd cluster with the same
+// logical time.
+// - C1 compacts first. It will succeed.
+// - C2 compacts after. It will fail. But it will get latest logical time, which should be larger by one.
+func TestCompactConflict(t *testing.T) {
+	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer cluster.Terminate(t)
+	client := cluster.RandClient()
+	ctx := context.Background()
+
+	putResp, err := client.Put(ctx, "/somekey", "data")
+	if err != nil {
+		t.Fatalf("Put failed: %v", err)
+	}
+
+	// Compact first. It would do the compaction and return compact time which is incremented by 1.
+	curTime, _, err := compact(ctx, client, 0, putResp.Header.Revision)
+	if err != nil {
+		t.Fatalf("compact failed: %v", err)
+	}
+	if curTime != 1 {
+		t.Errorf("Expect current logical time = 1, get = %v", curTime)
+	}
+
+	// Compact again with the same parameters. It won't do compaction but return the latest compact time.
+	curTime2, _, err := compact(ctx, client, 0, putResp.Header.Revision)
+	if err != nil {
+		t.Fatalf("compact failed: %v", err)
+	}
+	if curTime != curTime2 {
+		t.Errorf("Unexpected curTime (%v) != curTime2 (%v)", curTime, curTime2)
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/errors.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"k8s.io/apimachinery/pkg/api/errors"
+
+	etcdrpc "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+)
+
+func interpretWatchError(err error) error {
+	switch {
+	case err == etcdrpc.ErrCompacted:
+		return errors.NewResourceExpired("The resourceVersion for the provided watch is too old.")
+	}
+	return err
+}
+
+func interpretListError(err error, paging bool) error {
+	switch {
+	case err == etcdrpc.ErrCompacted:
+		if paging {
+			return errors.NewResourceExpired("The provided from parameter is too old to display a consistent list result. You must start a new list without the from.")
+		}
+		return errors.NewResourceExpired("The resourceVersion for the provided list is too old.")
+	}
+	return err
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/event.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/mvcc/mvccpb"
+)
+
+type event struct {
+	key       string
+	value     []byte
+	prevValue []byte
+	rev       int64
+	isDeleted bool
+	isCreated bool
+}
+
+// parseKV converts a KeyValue retrieved from an initial sync() listing to a synthetic isCreated event.
+func parseKV(kv *mvccpb.KeyValue) *event {
+	return &event{
+		key:       string(kv.Key),
+		value:     kv.Value,
+		prevValue: nil,
+		rev:       kv.ModRevision,
+		isDeleted: false,
+		isCreated: true,
+	}
+}
+
+func parseEvent(e *clientv3.Event) *event {
+	ret := &event{
+		key:       string(e.Kv.Key),
+		value:     e.Kv.Value,
+		rev:       e.Kv.ModRevision,
+		isDeleted: e.Type == clientv3.EventTypeDelete,
+		isCreated: e.IsCreate(),
+	}
+	if e.PrevKv != nil {
+		ret.prevValue = e.PrevKv.Value
+	}
+	return ret
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["checks.go"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["checks_test.go"],
+    library = ":go_default_library",
+    deps = ["//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package preflight
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"time"
+)
+
+const connectionTimeout = 1 * time.Second
+
+type connection interface {
+	serverReachable(address string) bool
+	parseServerList(serverList []string) error
+	CheckEtcdServers() (bool, error)
+}
+
+// EtcdConnection holds the Etcd server list
+type EtcdConnection struct {
+	ServerList []string
+}
+
+func (EtcdConnection) serverReachable(connURL *url.URL) bool {
+	scheme := connURL.Scheme
+	if scheme == "http" || scheme == "https" || scheme == "tcp" {
+		scheme = "tcp"
+	}
+	if conn, err := net.DialTimeout(scheme, connURL.Host, connectionTimeout); err == nil {
+		defer conn.Close()
+		return true
+	}
+	return false
+}
+
+func parseServerURI(serverURI string) (*url.URL, error) {
+	connURL, err := url.Parse(serverURI)
+	if err != nil {
+		return &url.URL{}, fmt.Errorf("unable to parse etcd url: %v", err)
+	}
+	return connURL, nil
+}
+
+// CheckEtcdServers will attempt to reach all etcd servers once. If any
+// can be reached, return true.
+func (con EtcdConnection) CheckEtcdServers() (done bool, err error) {
+	// Attempt to reach every Etcd server in order
+	for _, serverURI := range con.ServerList {
+		host, err := parseServerURI(serverURI)
+		if err != nil {
+			return false, err
+		}
+		if con.serverReachable(host) {
+			return true, nil
+		}
+	}
+	return false, nil
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks_test.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package preflight
+
+import (
+	"net/url"
+	"testing"
+	"time"
+
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
+)
+
+func TestParseServerURIGood(t *testing.T) {
+	connURL, err := parseServerURI("https://127.0.0.1:2379")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	reference := "127.0.0.1:2379"
+	if connURL.Host != reference {
+		t.Fatalf("server uri was not parsed correctly, host %s was invalid", connURL.Host)
+	}
+}
+
+func TestParseServerURIGoodUnix(t *testing.T) {
+	connURL, err := parseServerURI("unix://127.0.0.1:21002112605")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	reference := "127.0.0.1:21002112605"
+	if connURL.Host != reference {
+		t.Fatalf("server uri was not parsed correctly, host %s was invalid", connURL.Host)
+	}
+}
+
+func TestParseServerURIBad(t *testing.T) {
+	_, err := parseServerURI("-invalid uri$@#%")
+	if err == nil {
+		t.Fatal("expected bad uri to raise parse error")
+	}
+}
+
+func TestEtcdConnection(t *testing.T) {
+	etcd := new(EtcdConnection)
+
+	result := etcd.serverReachable(&url.URL{Host: "-not a real network address-", Scheme: "tcp"})
+	if result {
+		t.Fatal("checkConnection should not have succeeded")
+	}
+}
+
+func TestCheckEtcdServersEmpty(t *testing.T) {
+	etcd := new(EtcdConnection)
+	result, err := etcd.CheckEtcdServers()
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if result {
+		t.Fatal("CheckEtcdServers should not have succeeded")
+	}
+}
+
+func TestCheckEtcdServersUri(t *testing.T) {
+	etcd := new(EtcdConnection)
+	etcd.ServerList = []string{"-invalid uri$@#%"}
+	result, err := etcd.CheckEtcdServers()
+	if err == nil {
+		t.Fatalf("expected bad uri to raise parse error")
+	}
+	if result {
+		t.Fatal("CheckEtcdServers should not have succeeded")
+	}
+}
+
+func TestCheckEtcdServers(t *testing.T) {
+	etcd := new(EtcdConnection)
+	etcd.ServerList = []string{""}
+	result, err := etcd.CheckEtcdServers()
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if result {
+		t.Fatal("CheckEtcdServers should not have succeeded")
+	}
+}
+
+func TestPollCheckServer(t *testing.T) {
+	err := utilwait.PollImmediate(1*time.Microsecond,
+		2*time.Microsecond,
+		EtcdConnection{ServerList: []string{""}}.CheckEtcdServers)
+	if err == nil {
+		t.Fatal("expected check to time out")
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go

@@ -0,0 +1,718 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/etcd"
+	"k8s.io/apiserver/pkg/storage/value"
+	utiltrace "k8s.io/apiserver/pkg/util/trace"
+)
+
+// authenticatedDataString satisfies the value.Context interface. It uses the key to
+// authenticate the stored data. This does not defend against reuse of previously
+// encrypted values under the same key, but will prevent an attacker from using an
+// encrypted value from a different key. A stronger authenticated data segment would
+// include the etcd3 Version field (which is incremented on each write to a key and
+// reset when the key is deleted), but an attacker with write access to etcd can
+// force deletion and recreation of keys to weaken that angle.
+type authenticatedDataString string
+
+// AuthenticatedData implements the value.Context interface.
+func (d authenticatedDataString) AuthenticatedData() []byte {
+	return []byte(string(d))
+}
+
+var _ value.Context = authenticatedDataString("")
+
+type store struct {
+	client *clientv3.Client
+	// getOpts contains additional options that should be passed
+	// to all Get() calls.
+	getOps        []clientv3.OpOption
+	codec         runtime.Codec
+	versioner     storage.Versioner
+	transformer   value.Transformer
+	pathPrefix    string
+	watcher       *watcher
+	pagingEnabled bool
+}
+
+type elemForDecode struct {
+	data []byte
+	rev  uint64
+}
+
+type objState struct {
+	obj   runtime.Object
+	meta  *storage.ResponseMeta
+	rev   int64
+	data  []byte
+	stale bool
+}
+
+// New returns an etcd3 implementation of storage.Interface.
+func New(c *clientv3.Client, codec runtime.Codec, prefix string, transformer value.Transformer, pagingEnabled bool) storage.Interface {
+	return newStore(c, true, pagingEnabled, codec, prefix, transformer)
+}
+
+// NewWithNoQuorumRead returns etcd3 implementation of storage.Interface
+// where Get operations don't require quorum read.
+func NewWithNoQuorumRead(c *clientv3.Client, codec runtime.Codec, prefix string, transformer value.Transformer, pagingEnabled bool) storage.Interface {
+	return newStore(c, false, pagingEnabled, codec, prefix, transformer)
+}
+
+func newStore(c *clientv3.Client, quorumRead, pagingEnabled bool, codec runtime.Codec, prefix string, transformer value.Transformer) *store {
+	versioner := etcd.APIObjectVersioner{}
+	result := &store{
+		client:        c,
+		codec:         codec,
+		versioner:     versioner,
+		transformer:   transformer,
+		pagingEnabled: pagingEnabled,
+		// for compatibility with etcd2 impl.
+		// no-op for default prefix of '/registry'.
+		// keeps compatibility with etcd2 impl for custom prefixes that don't start with '/'
+		pathPrefix: path.Join("/", prefix),
+		watcher:    newWatcher(c, codec, versioner, transformer),
+	}
+	if !quorumRead {
+		// In case of non-quorum reads, we can set WithSerializable()
+		// options for all Get operations.
+		result.getOps = append(result.getOps, clientv3.WithSerializable())
+	}
+	return result
+}
+
+// Versioner implements storage.Interface.Versioner.
+func (s *store) Versioner() storage.Versioner {
+	return s.versioner
+}
+
+// Get implements storage.Interface.Get.
+func (s *store) Get(ctx context.Context, key string, resourceVersion string, out runtime.Object, ignoreNotFound bool) error {
+	key = path.Join(s.pathPrefix, key)
+	getResp, err := s.client.KV.Get(ctx, key, s.getOps...)
+	if err != nil {
+		return err
+	}
+
+	if len(getResp.Kvs) == 0 {
+		if ignoreNotFound {
+			return runtime.SetZeroValue(out)
+		}
+		return storage.NewKeyNotFoundError(key, 0)
+	}
+	kv := getResp.Kvs[0]
+
+	data, _, err := s.transformer.TransformFromStorage(kv.Value, authenticatedDataString(key))
+	if err != nil {
+		return storage.NewInternalError(err.Error())
+	}
+
+	return decode(s.codec, s.versioner, data, out, kv.ModRevision)
+}
+
+// Create implements storage.Interface.Create.
+func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error {
+	if version, err := s.versioner.ObjectResourceVersion(obj); err == nil && version != 0 {
+		return errors.New("resourceVersion should not be set on objects to be created")
+	}
+	if err := s.versioner.PrepareObjectForStorage(obj); err != nil {
+		return fmt.Errorf("PrepareObjectForStorage failed: %v", err)
+	}
+	data, err := runtime.Encode(s.codec, obj)
+	if err != nil {
+		return err
+	}
+	key = path.Join(s.pathPrefix, key)
+
+	opts, err := s.ttlOpts(ctx, int64(ttl))
+	if err != nil {
+		return err
+	}
+
+	newData, err := s.transformer.TransformToStorage(data, authenticatedDataString(key))
+	if err != nil {
+		return storage.NewInternalError(err.Error())
+	}
+
+	txnResp, err := s.client.KV.Txn(ctx).If(
+		notFound(key),
+	).Then(
+		clientv3.OpPut(key, string(newData), opts...),
+	).Commit()
+	if err != nil {
+		return err
+	}
+	if !txnResp.Succeeded {
+		return storage.NewKeyExistsError(key, 0)
+	}
+
+	if out != nil {
+		putResp := txnResp.Responses[0].GetResponsePut()
+		return decode(s.codec, s.versioner, data, out, putResp.Header.Revision)
+	}
+	return nil
+}
+
+// Delete implements storage.Interface.Delete.
+func (s *store) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions) error {
+	v, err := conversion.EnforcePtr(out)
+	if err != nil {
+		panic("unable to convert output object to pointer")
+	}
+	key = path.Join(s.pathPrefix, key)
+	if preconditions == nil {
+		return s.unconditionalDelete(ctx, key, out)
+	}
+	return s.conditionalDelete(ctx, key, out, v, preconditions)
+}
+
+func (s *store) unconditionalDelete(ctx context.Context, key string, out runtime.Object) error {
+	// We need to do get and delete in single transaction in order to
+	// know the value and revision before deleting it.
+	txnResp, err := s.client.KV.Txn(ctx).If().Then(
+		clientv3.OpGet(key),
+		clientv3.OpDelete(key),
+	).Commit()
+	if err != nil {
+		return err
+	}
+	getResp := txnResp.Responses[0].GetResponseRange()
+	if len(getResp.Kvs) == 0 {
+		return storage.NewKeyNotFoundError(key, 0)
+	}
+
+	kv := getResp.Kvs[0]
+	data, _, err := s.transformer.TransformFromStorage(kv.Value, authenticatedDataString(key))
+	if err != nil {
+		return storage.NewInternalError(err.Error())
+	}
+	return decode(s.codec, s.versioner, data, out, kv.ModRevision)
+}
+
+func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.Object, v reflect.Value, preconditions *storage.Preconditions) error {
+	getResp, err := s.client.KV.Get(ctx, key)
+	if err != nil {
+		return err
+	}
+	for {
+		origState, err := s.getState(getResp, key, v, false)
+		if err != nil {
+			return err
+		}
+		if err := checkPreconditions(key, preconditions, origState.obj); err != nil {
+			return err
+		}
+		txnResp, err := s.client.KV.Txn(ctx).If(
+			clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev),
+		).Then(
+			clientv3.OpDelete(key),
+		).Else(
+			clientv3.OpGet(key),
+		).Commit()
+		if err != nil {
+			return err
+		}
+		if !txnResp.Succeeded {
+			getResp = (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange())
+			glog.V(4).Infof("deletion of %s failed because of a conflict, going to retry", key)
+			continue
+		}
+		return decode(s.codec, s.versioner, origState.data, out, origState.rev)
+	}
+}
+
+// GuaranteedUpdate implements storage.Interface.GuaranteedUpdate.
+func (s *store) GuaranteedUpdate(
+	ctx context.Context, key string, out runtime.Object, ignoreNotFound bool,
+	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion ...runtime.Object) error {
+	trace := utiltrace.New(fmt.Sprintf("GuaranteedUpdate etcd3: %s", reflect.TypeOf(out).String()))
+	defer trace.LogIfLong(500 * time.Millisecond)
+
+	v, err := conversion.EnforcePtr(out)
+	if err != nil {
+		panic("unable to convert output object to pointer")
+	}
+	key = path.Join(s.pathPrefix, key)
+
+	getCurrentState := func() (*objState, error) {
+		getResp, err := s.client.KV.Get(ctx, key, s.getOps...)
+		if err != nil {
+			return nil, err
+		}
+		return s.getState(getResp, key, v, ignoreNotFound)
+	}
+
+	var origState *objState
+	var mustCheckData bool
+	if len(suggestion) == 1 && suggestion[0] != nil {
+		origState, err = s.getStateFromObject(suggestion[0])
+		if err != nil {
+			return err
+		}
+		mustCheckData = true
+	} else {
+		origState, err = getCurrentState()
+		if err != nil {
+			return err
+		}
+	}
+	trace.Step("initial value restored")
+
+	transformContext := authenticatedDataString(key)
+	for {
+		if err := checkPreconditions(key, preconditions, origState.obj); err != nil {
+			return err
+		}
+
+		ret, ttl, err := s.updateState(origState, tryUpdate)
+		if err != nil {
+			// It's possible we were working with stale data
+			if mustCheckData && apierrors.IsConflict(err) {
+				// Actually fetch
+				origState, err = getCurrentState()
+				if err != nil {
+					return err
+				}
+				mustCheckData = false
+				// Retry
+				continue
+			}
+
+			return err
+		}
+
+		data, err := runtime.Encode(s.codec, ret)
+		if err != nil {
+			return err
+		}
+		if !origState.stale && bytes.Equal(data, origState.data) {
+			// if we skipped the original Get in this loop, we must refresh from
+			// etcd in order to be sure the data in the store is equivalent to
+			// our desired serialization
+			if mustCheckData {
+				origState, err = getCurrentState()
+				if err != nil {
+					return err
+				}
+				mustCheckData = false
+				continue
+			}
+			return decode(s.codec, s.versioner, origState.data, out, origState.rev)
+		}
+
+		newData, err := s.transformer.TransformToStorage(data, transformContext)
+		if err != nil {
+			return storage.NewInternalError(err.Error())
+		}
+
+		opts, err := s.ttlOpts(ctx, int64(ttl))
+		if err != nil {
+			return err
+		}
+		trace.Step("Transaction prepared")
+
+		txnResp, err := s.client.KV.Txn(ctx).If(
+			clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev),
+		).Then(
+			clientv3.OpPut(key, string(newData), opts...),
+		).Else(
+			clientv3.OpGet(key),
+		).Commit()
+		if err != nil {
+			return err
+		}
+		trace.Step("Transaction committed")
+		if !txnResp.Succeeded {
+			getResp := (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange())
+			glog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", key)
+			origState, err = s.getState(getResp, key, v, ignoreNotFound)
+			if err != nil {
+				return err
+			}
+			trace.Step("Retry value restored")
+			mustCheckData = false
+			continue
+		}
+		putResp := txnResp.Responses[0].GetResponsePut()
+
+		return decode(s.codec, s.versioner, data, out, putResp.Header.Revision)
+	}
+}
+
+// GetToList implements storage.Interface.GetToList.
+func (s *store) GetToList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
+	listPtr, err := meta.GetItemsPtr(listObj)
+	if err != nil {
+		return err
+	}
+	key = path.Join(s.pathPrefix, key)
+
+	getResp, err := s.client.KV.Get(ctx, key, s.getOps...)
+	if err != nil {
+		return err
+	}
+	if len(getResp.Kvs) == 0 {
+		return nil
+	}
+	data, _, err := s.transformer.TransformFromStorage(getResp.Kvs[0].Value, authenticatedDataString(key))
+	if err != nil {
+		return storage.NewInternalError(err.Error())
+	}
+	elems := []*elemForDecode{{
+		data: data,
+		rev:  uint64(getResp.Kvs[0].ModRevision),
+	}}
+	if err := decodeList(elems, storage.SimpleFilter(pred), listPtr, s.codec, s.versioner); err != nil {
+		return err
+	}
+	// update version with cluster level revision
+	return s.versioner.UpdateList(listObj, uint64(getResp.Header.Revision), "")
+}
+
+// continueToken is a simple structured object for encoding the state of a continue token.
+// TODO: if we change the version of the encoded from, we can't start encoding the new version
+// until all other servers are upgraded (i.e. we need to support rolling schema)
+// This is a public API struct and cannot change.
+type continueToken struct {
+	APIVersion      string `json:"v"`
+	ResourceVersion int64  `json:"rv"`
+	StartKey        string `json:"start"`
+}
+
+// parseFrom transforms an encoded predicate from into a versioned struct.
+// TODO: return a typed error that instructs clients that they must relist
+func decodeContinue(continueValue, keyPrefix string) (fromKey string, rv int64, err error) {
+	data, err := base64.RawURLEncoding.DecodeString(continueValue)
+	if err != nil {
+		return "", 0, fmt.Errorf("continue key is not valid: %v", err)
+	}
+	var c continueToken
+	if err := json.Unmarshal(data, &c); err != nil {
+		return "", 0, fmt.Errorf("continue key is not valid: %v", err)
+	}
+	switch c.APIVersion {
+	case "v1alpha1":
+		if c.ResourceVersion == 0 {
+			return "", 0, fmt.Errorf("continue key is not valid: incorrect encoded start resourceVersion (version v1alpha1)")
+		}
+		if len(c.StartKey) == 0 {
+			return "", 0, fmt.Errorf("continue key is not valid: encoded start key empty (version v1alpha1)")
+		}
+		// defend against path traversal attacks by clients - path.Clean will ensure that startKey cannot
+		// be at a higher level of the hierarchy, and so when we append the key prefix we will end up with
+		// continue start key that is fully qualified and cannot range over anything less specific than
+		// keyPrefix.
+		cleaned := path.Clean(c.StartKey)
+		if cleaned != c.StartKey || cleaned == "." || cleaned == "/" {
+			return "", 0, fmt.Errorf("continue key is not valid: %s", cleaned)
+		}
+		if len(cleaned) == 0 {
+			return "", 0, fmt.Errorf("continue key is not valid: encoded start key empty (version 0)")
+		}
+		return keyPrefix + cleaned, c.ResourceVersion, nil
+	default:
+		return "", 0, fmt.Errorf("continue key is not valid: server does not recognize this encoded version %q", c.APIVersion)
+	}
+}
+
+// encodeContinue returns a string representing the encoded continuation of the current query.
+func encodeContinue(key, keyPrefix string, resourceVersion int64) (string, error) {
+	nextKey := strings.TrimPrefix(key, keyPrefix)
+	if nextKey == key {
+		return "", fmt.Errorf("unable to encode next field: the key and key prefix do not match")
+	}
+	out, err := json.Marshal(&continueToken{APIVersion: "v1alpha1", ResourceVersion: resourceVersion, StartKey: nextKey})
+	if err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(out), nil
+}
+
+// List implements storage.Interface.List.
+func (s *store) List(ctx context.Context, key, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
+	listPtr, err := meta.GetItemsPtr(listObj)
+	if err != nil {
+		return err
+	}
+	key = path.Join(s.pathPrefix, key)
+	// We need to make sure the key ended with "/" so that we only get children "directories".
+	// e.g. if we have key "/a", "/a/b", "/ab", getting keys with prefix "/a" will return all three,
+	// while with prefix "/a/" will return only "/a/b" which is the correct answer.
+	if !strings.HasSuffix(key, "/") {
+		key += "/"
+	}
+	keyPrefix := key
+
+	// set the appropriate clientv3 options to filter the returned data set
+	options := make([]clientv3.OpOption, 0, 4)
+	if s.pagingEnabled && pred.Limit > 0 {
+		options = append(options, clientv3.WithLimit(pred.Limit))
+	}
+	var returnedRV int64
+	switch {
+	case s.pagingEnabled && len(pred.Continue) > 0:
+		continueKey, continueRV, err := decodeContinue(pred.Continue, keyPrefix)
+		if err != nil {
+			return err
+		}
+
+		options = append(options, clientv3.WithRange(clientv3.GetPrefixRangeEnd(key)))
+		key = continueKey
+
+		options = append(options, clientv3.WithRev(continueRV))
+		returnedRV = continueRV
+
+	case len(resourceVersion) > 0:
+		fromRV, err := strconv.ParseInt(resourceVersion, 10, 64)
+		if err != nil {
+			return fmt.Errorf("invalid resource version: %v", err)
+		}
+
+		options = append(options, clientv3.WithPrefix(), clientv3.WithRev(fromRV))
+		returnedRV = fromRV
+
+	default:
+		options = append(options, clientv3.WithPrefix())
+	}
+
+	getResp, err := s.client.KV.Get(ctx, key, options...)
+	if err != nil {
+		return interpretListError(err, len(pred.Continue) > 0)
+	}
+
+	elems := make([]*elemForDecode, 0, len(getResp.Kvs))
+	for _, kv := range getResp.Kvs {
+		data, _, err := s.transformer.TransformFromStorage(kv.Value, authenticatedDataString(kv.Key))
+		if err != nil {
+			utilruntime.HandleError(fmt.Errorf("unable to transform key %q: %v", kv.Key, err))
+			continue
+		}
+
+		elems = append(elems, &elemForDecode{
+			data: data,
+			rev:  uint64(kv.ModRevision),
+		})
+	}
+
+	if err := decodeList(elems, storage.SimpleFilter(pred), listPtr, s.codec, s.versioner); err != nil {
+		return err
+	}
+
+	// indicate to the client which resource version was returned
+	if returnedRV == 0 {
+		returnedRV = getResp.Header.Revision
+	}
+	switch {
+	case !getResp.More:
+		// no continuation
+		return s.versioner.UpdateList(listObj, uint64(returnedRV), "")
+	case len(getResp.Kvs) == 0:
+		return fmt.Errorf("no results were found, but etcd indicated there were more values")
+	default:
+		// we want to start immediately after the last key
+		// TODO: this reveals info about certain keys
+		key := string(getResp.Kvs[len(getResp.Kvs)-1].Key)
+		next, err := encodeContinue(key+"\x00", keyPrefix, returnedRV)
+		if err != nil {
+			return err
+		}
+		return s.versioner.UpdateList(listObj, uint64(returnedRV), next)
+	}
+}
+
+// Watch implements storage.Interface.Watch.
+func (s *store) Watch(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate) (watch.Interface, error) {
+	return s.watch(ctx, key, resourceVersion, pred, false)
+}
+
+// WatchList implements storage.Interface.WatchList.
+func (s *store) WatchList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate) (watch.Interface, error) {
+	return s.watch(ctx, key, resourceVersion, pred, true)
+}
+
+func (s *store) watch(ctx context.Context, key string, rv string, pred storage.SelectionPredicate, recursive bool) (watch.Interface, error) {
+	rev, err := storage.ParseWatchResourceVersion(rv)
+	if err != nil {
+		return nil, err
+	}
+	key = path.Join(s.pathPrefix, key)
+	return s.watcher.Watch(ctx, key, int64(rev), recursive, pred)
+}
+
+func (s *store) getState(getResp *clientv3.GetResponse, key string, v reflect.Value, ignoreNotFound bool) (*objState, error) {
+	state := &objState{
+		obj:  reflect.New(v.Type()).Interface().(runtime.Object),
+		meta: &storage.ResponseMeta{},
+	}
+	if len(getResp.Kvs) == 0 {
+		if !ignoreNotFound {
+			return nil, storage.NewKeyNotFoundError(key, 0)
+		}
+		if err := runtime.SetZeroValue(state.obj); err != nil {
+			return nil, err
+		}
+	} else {
+		data, stale, err := s.transformer.TransformFromStorage(getResp.Kvs[0].Value, authenticatedDataString(key))
+		if err != nil {
+			return nil, storage.NewInternalError(err.Error())
+		}
+		state.rev = getResp.Kvs[0].ModRevision
+		state.meta.ResourceVersion = uint64(state.rev)
+		state.data = data
+		state.stale = stale
+		if err := decode(s.codec, s.versioner, state.data, state.obj, state.rev); err != nil {
+			return nil, err
+		}
+	}
+	return state, nil
+}
+
+func (s *store) getStateFromObject(obj runtime.Object) (*objState, error) {
+	state := &objState{
+		obj:  obj,
+		meta: &storage.ResponseMeta{},
+	}
+
+	rv, err := s.versioner.ObjectResourceVersion(obj)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't get resource version: %v", err)
+	}
+	state.rev = int64(rv)
+	state.meta.ResourceVersion = uint64(state.rev)
+
+	// Compute the serialized form - for that we need to temporarily clean
+	// its resource version field (those are not stored in etcd).
+	if err := s.versioner.PrepareObjectForStorage(obj); err != nil {
+		return nil, fmt.Errorf("PrepareObjectForStorage failed: %v", err)
+	}
+	state.data, err = runtime.Encode(s.codec, obj)
+	if err != nil {
+		return nil, err
+	}
+	s.versioner.UpdateObject(state.obj, uint64(rv))
+	return state, nil
+}
+
+func (s *store) updateState(st *objState, userUpdate storage.UpdateFunc) (runtime.Object, uint64, error) {
+	ret, ttlPtr, err := userUpdate(st.obj, *st.meta)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	if err := s.versioner.PrepareObjectForStorage(ret); err != nil {
+		return nil, 0, fmt.Errorf("PrepareObjectForStorage failed: %v", err)
+	}
+	var ttl uint64
+	if ttlPtr != nil {
+		ttl = *ttlPtr
+	}
+	return ret, ttl, nil
+}
+
+// ttlOpts returns client options based on given ttl.
+// ttl: if ttl is non-zero, it will attach the key to a lease with ttl of roughly the same length
+func (s *store) ttlOpts(ctx context.Context, ttl int64) ([]clientv3.OpOption, error) {
+	if ttl == 0 {
+		return nil, nil
+	}
+	// TODO: one lease per ttl key is expensive. Based on current use case, we can have a long window to
+	// put keys within into same lease. We shall benchmark this and optimize the performance.
+	lcr, err := s.client.Lease.Grant(ctx, ttl)
+	if err != nil {
+		return nil, err
+	}
+	return []clientv3.OpOption{clientv3.WithLease(clientv3.LeaseID(lcr.ID))}, nil
+}
+
+// decode decodes value of bytes into object. It will also set the object resource version to rev.
+// On success, objPtr would be set to the object.
+func decode(codec runtime.Codec, versioner storage.Versioner, value []byte, objPtr runtime.Object, rev int64) error {
+	if _, err := conversion.EnforcePtr(objPtr); err != nil {
+		panic("unable to convert output object to pointer")
+	}
+	_, _, err := codec.Decode(value, nil, objPtr)
+	if err != nil {
+		return err
+	}
+	// being unable to set the version does not prevent the object from being extracted
+	versioner.UpdateObject(objPtr, uint64(rev))
+	return nil
+}
+
+// decodeList decodes a list of values into a list of objects, with resource version set to corresponding rev.
+// On success, ListPtr would be set to the list of objects.
+func decodeList(elems []*elemForDecode, filter storage.FilterFunc, listPtr interface{}, codec runtime.Codec, versioner storage.Versioner) error {
+	v, err := conversion.EnforcePtr(listPtr)
+	if err != nil || v.Kind() != reflect.Slice {
+		panic("need ptr to slice")
+	}
+	for _, elem := range elems {
+		obj, _, err := codec.Decode(elem.data, nil, reflect.New(v.Type().Elem()).Interface().(runtime.Object))
+		if err != nil {
+			return err
+		}
+		// being unable to set the version does not prevent the object from being extracted
+		versioner.UpdateObject(obj, elem.rev)
+		if filter(obj) {
+			v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
+		}
+	}
+	return nil
+}
+
+func checkPreconditions(key string, preconditions *storage.Preconditions, out runtime.Object) error {
+	if preconditions == nil {
+		return nil
+	}
+	objMeta, err := meta.Accessor(out)
+	if err != nil {
+		return storage.NewInternalErrorf("can't enforce preconditions %v on un-introspectable object %v, got error: %v", *preconditions, out, err)
+	}
+	if preconditions.UID != nil && *preconditions.UID != objMeta.GetUID() {
+		errMsg := fmt.Sprintf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *preconditions.UID, objMeta.GetUID())
+		return storage.NewInvalidObjError(key, errMsg)
+	}
+	return nil
+}
+
+func notFound(key string) clientv3.Cmp {
+	return clientv3.Compare(clientv3.ModRevision(key), "=", 0)
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/watcher.go

@@ -0,0 +1,401 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/value"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+)
+
+const (
+	// We have set a buffer in order to reduce times of context switches.
+	incomingBufSize = 100
+	outgoingBufSize = 100
+)
+
+// fatalOnDecodeError is used during testing to panic the server if watcher encounters a decoding error
+var fatalOnDecodeError = false
+
+// errTestingDecode is the only error that testingDeferOnDecodeError catches during a panic
+var errTestingDecode = errors.New("sentinel error only used during testing to indicate watch decoding error")
+
+// testingDeferOnDecodeError is used during testing to recover from a panic caused by errTestingDecode, all other values continue to panic
+func testingDeferOnDecodeError() {
+	if r := recover(); r != nil && r != errTestingDecode {
+		panic(r)
+	}
+}
+
+func init() {
+	// check to see if we are running in a test environment
+	fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR"))
+}
+
+type watcher struct {
+	client      *clientv3.Client
+	codec       runtime.Codec
+	versioner   storage.Versioner
+	transformer value.Transformer
+}
+
+// watchChan implements watch.Interface.
+type watchChan struct {
+	watcher           *watcher
+	key               string
+	initialRev        int64
+	recursive         bool
+	internalFilter    storage.FilterFunc
+	ctx               context.Context
+	cancel            context.CancelFunc
+	incomingEventChan chan *event
+	resultChan        chan watch.Event
+	errChan           chan error
+}
+
+func newWatcher(client *clientv3.Client, codec runtime.Codec, versioner storage.Versioner, transformer value.Transformer) *watcher {
+	return &watcher{
+		client:      client,
+		codec:       codec,
+		versioner:   versioner,
+		transformer: transformer,
+	}
+}
+
+// Watch watches on a key and returns a watch.Interface that transfers relevant notifications.
+// If rev is zero, it will return the existing object(s) and then start watching from
+// the maximum revision+1 from returned objects.
+// If rev is non-zero, it will watch events happened after given revision.
+// If recursive is false, it watches on given key.
+// If recursive is true, it watches any children and directories under the key, excluding the root key itself.
+// pred must be non-nil. Only if pred matches the change, it will be returned.
+func (w *watcher) Watch(ctx context.Context, key string, rev int64, recursive bool, pred storage.SelectionPredicate) (watch.Interface, error) {
+	if recursive && !strings.HasSuffix(key, "/") {
+		key += "/"
+	}
+	wc := w.createWatchChan(ctx, key, rev, recursive, pred)
+	go wc.run()
+	return wc, nil
+}
+
+func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive bool, pred storage.SelectionPredicate) *watchChan {
+	wc := &watchChan{
+		watcher:           w,
+		key:               key,
+		initialRev:        rev,
+		recursive:         recursive,
+		internalFilter:    storage.SimpleFilter(pred),
+		incomingEventChan: make(chan *event, incomingBufSize),
+		resultChan:        make(chan watch.Event, outgoingBufSize),
+		errChan:           make(chan error, 1),
+	}
+	if pred.Empty() {
+		// The filter doesn't filter out any object.
+		wc.internalFilter = nil
+	}
+	wc.ctx, wc.cancel = context.WithCancel(ctx)
+	return wc
+}
+
+func (wc *watchChan) run() {
+	watchClosedCh := make(chan struct{})
+	go wc.startWatching(watchClosedCh)
+
+	var resultChanWG sync.WaitGroup
+	resultChanWG.Add(1)
+	go wc.processEvent(&resultChanWG)
+
+	select {
+	case err := <-wc.errChan:
+		if err == context.Canceled {
+			break
+		}
+		errResult := transformErrorToEvent(err)
+		if errResult != nil {
+			// error result is guaranteed to be received by user before closing ResultChan.
+			select {
+			case wc.resultChan <- *errResult:
+			case <-wc.ctx.Done(): // user has given up all results
+			}
+		}
+	case <-watchClosedCh:
+	case <-wc.ctx.Done(): // user cancel
+	}
+
+	// We use wc.ctx to reap all goroutines. Under whatever condition, we should stop them all.
+	// It's fine to double cancel.
+	wc.cancel()
+
+	// we need to wait until resultChan wouldn't be used anymore
+	resultChanWG.Wait()
+	close(wc.resultChan)
+}
+
+func (wc *watchChan) Stop() {
+	wc.cancel()
+}
+
+func (wc *watchChan) ResultChan() <-chan watch.Event {
+	return wc.resultChan
+}
+
+// sync tries to retrieve existing data and send them to process.
+// The revision to watch will be set to the revision in response.
+// All events sent will have isCreated=true
+func (wc *watchChan) sync() error {
+	opts := []clientv3.OpOption{}
+	if wc.recursive {
+		opts = append(opts, clientv3.WithPrefix())
+	}
+	getResp, err := wc.watcher.client.Get(wc.ctx, wc.key, opts...)
+	if err != nil {
+		return err
+	}
+	wc.initialRev = getResp.Header.Revision
+	for _, kv := range getResp.Kvs {
+		wc.sendEvent(parseKV(kv))
+	}
+	return nil
+}
+
+// startWatching does:
+// - get current objects if initialRev=0; set initialRev to current rev
+// - watch on given key and send events to process.
+func (wc *watchChan) startWatching(watchClosedCh chan struct{}) {
+	if wc.initialRev == 0 {
+		if err := wc.sync(); err != nil {
+			glog.Errorf("failed to sync with latest state: %v", err)
+			wc.sendError(err)
+			return
+		}
+	}
+	opts := []clientv3.OpOption{clientv3.WithRev(wc.initialRev + 1), clientv3.WithPrevKV()}
+	if wc.recursive {
+		opts = append(opts, clientv3.WithPrefix())
+	}
+	wch := wc.watcher.client.Watch(wc.ctx, wc.key, opts...)
+	for wres := range wch {
+		if wres.Err() != nil {
+			err := wres.Err()
+			// If there is an error on server (e.g. compaction), the channel will return it before closed.
+			glog.Errorf("watch chan error: %v", err)
+			wc.sendError(err)
+			return
+		}
+		for _, e := range wres.Events {
+			wc.sendEvent(parseEvent(e))
+		}
+	}
+	// When we come to this point, it's only possible that client side ends the watch.
+	// e.g. cancel the context, close the client.
+	// If this watch chan is broken and context isn't cancelled, other goroutines will still hang.
+	// We should notify the main thread that this goroutine has exited.
+	close(watchClosedCh)
+}
+
+// processEvent processes events from etcd watcher and sends results to resultChan.
+func (wc *watchChan) processEvent(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	for {
+		select {
+		case e := <-wc.incomingEventChan:
+			res := wc.transform(e)
+			if res == nil {
+				continue
+			}
+			if len(wc.resultChan) == outgoingBufSize {
+				glog.Warningf("Fast watcher, slow processing. Number of buffered events: %d."+
+					"Probably caused by slow dispatching events to watchers", outgoingBufSize)
+			}
+			// If user couldn't receive results fast enough, we also block incoming events from watcher.
+			// Because storing events in local will cause more memory usage.
+			// The worst case would be closing the fast watcher.
+			select {
+			case wc.resultChan <- *res:
+			case <-wc.ctx.Done():
+				return
+			}
+		case <-wc.ctx.Done():
+			return
+		}
+	}
+}
+
+func (wc *watchChan) filter(obj runtime.Object) bool {
+	if wc.internalFilter == nil {
+		return true
+	}
+	return wc.internalFilter(obj)
+}
+
+func (wc *watchChan) acceptAll() bool {
+	return wc.internalFilter == nil
+}
+
+// transform transforms an event into a result for user if not filtered.
+func (wc *watchChan) transform(e *event) (res *watch.Event) {
+	curObj, oldObj, err := wc.prepareObjs(e)
+	if err != nil {
+		glog.Errorf("failed to prepare current and previous objects: %v", err)
+		wc.sendError(err)
+		return nil
+	}
+
+	switch {
+	case e.isDeleted:
+		if !wc.filter(oldObj) {
+			return nil
+		}
+		res = &watch.Event{
+			Type:   watch.Deleted,
+			Object: oldObj,
+		}
+	case e.isCreated:
+		if !wc.filter(curObj) {
+			return nil
+		}
+		res = &watch.Event{
+			Type:   watch.Added,
+			Object: curObj,
+		}
+	default:
+		if wc.acceptAll() {
+			res = &watch.Event{
+				Type:   watch.Modified,
+				Object: curObj,
+			}
+			return res
+		}
+		curObjPasses := wc.filter(curObj)
+		oldObjPasses := wc.filter(oldObj)
+		switch {
+		case curObjPasses && oldObjPasses:
+			res = &watch.Event{
+				Type:   watch.Modified,
+				Object: curObj,
+			}
+		case curObjPasses && !oldObjPasses:
+			res = &watch.Event{
+				Type:   watch.Added,
+				Object: curObj,
+			}
+		case !curObjPasses && oldObjPasses:
+			res = &watch.Event{
+				Type:   watch.Deleted,
+				Object: oldObj,
+			}
+		}
+	}
+	return res
+}
+
+func transformErrorToEvent(err error) *watch.Event {
+	err = interpretWatchError(err)
+	if _, ok := err.(apierrs.APIStatus); !ok {
+		err = apierrs.NewInternalError(err)
+	}
+	status := err.(apierrs.APIStatus).Status()
+	return &watch.Event{
+		Type:   watch.Error,
+		Object: &status,
+	}
+}
+
+func (wc *watchChan) sendError(err error) {
+	select {
+	case wc.errChan <- err:
+	case <-wc.ctx.Done():
+	}
+}
+
+func (wc *watchChan) sendEvent(e *event) {
+	if len(wc.incomingEventChan) == incomingBufSize {
+		glog.Warningf("Fast watcher, slow processing. Number of buffered events: %d."+
+			"Probably caused by slow decoding, user not receiving fast, or other processing logic",
+			incomingBufSize)
+	}
+	select {
+	case wc.incomingEventChan <- e:
+	case <-wc.ctx.Done():
+	}
+}
+
+func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) {
+	if !e.isDeleted {
+		data, _, err := wc.watcher.transformer.TransformFromStorage(e.value, authenticatedDataString(e.key))
+		if err != nil {
+			return nil, nil, err
+		}
+		curObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	// We need to decode prevValue, only if this is deletion event or
+	// the underlying filter doesn't accept all objects (otherwise we
+	// know that the filter for previous object will return true and
+	// we need the object only to compute whether it was filtered out
+	// before).
+	if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) {
+		data, _, err := wc.watcher.transformer.TransformFromStorage(e.prevValue, authenticatedDataString(e.key))
+		if err != nil {
+			return nil, nil, err
+		}
+		// Note that this sends the *old* object with the etcd revision for the time at
+		// which it gets deleted.
+		oldObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	return curObj, oldObj, nil
+}
+
+func decodeObj(codec runtime.Codec, versioner storage.Versioner, data []byte, rev int64) (_ runtime.Object, err error) {
+	obj, err := runtime.Decode(codec, []byte(data))
+	if err != nil {
+		if fatalOnDecodeError {
+			// catch watch decode error iff we caused it on
+			// purpose during a unit test
+			defer testingDeferOnDecodeError()
+			// we are running in a test environment and thus an
+			// error here is due to a coder mistake if the defer
+			// does not catch it
+			panic(err)
+		}
+		return nil, err
+	}
+	// ensure resource version is set on the object we load from etcd
+	if err := versioner.UpdateObject(obj, uint64(rev)); err != nil {
+		return nil, fmt.Errorf("failure to version api object (%d) %#v: %v", rev, obj, err)
+	}
+	return obj, nil
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go

@@ -0,0 +1,374 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/integration"
+	"golang.org/x/net/context"
+
+	apitesting "k8s.io/apimachinery/pkg/api/testing"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/apis/example"
+	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
+	"k8s.io/apiserver/pkg/storage"
+)
+
+func TestWatch(t *testing.T) {
+	testWatch(t, false)
+}
+
+func TestWatchList(t *testing.T) {
+	testWatch(t, true)
+}
+
+// It tests that
+// - first occurrence of objects should notify Add event
+// - update should trigger Modified event
+// - update that gets filtered should trigger Deleted event
+func testWatch(t *testing.T, recursive bool) {
+	ctx, store, cluster := testSetup(t)
+	defer cluster.Terminate(t)
+	podFoo := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
+	podBar := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}
+
+	tests := []struct {
+		key        string
+		pred       storage.SelectionPredicate
+		watchTests []*testWatchStruct
+	}{{ // create a key
+		key:        "/somekey-1",
+		watchTests: []*testWatchStruct{{podFoo, true, watch.Added}},
+		pred:       storage.Everything,
+	}, { // create a key but obj gets filtered. Then update it with unfiltered obj
+		key:        "/somekey-3",
+		watchTests: []*testWatchStruct{{podFoo, false, ""}, {podBar, true, watch.Added}},
+		pred: storage.SelectionPredicate{
+			Label: labels.Everything(),
+			Field: fields.ParseSelectorOrDie("metadata.name=bar"),
+			GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+				pod := obj.(*example.Pod)
+				return nil, fields.Set{"metadata.name": pod.Name}, pod.Initializers != nil, nil
+			},
+		},
+	}, { // update
+		key:        "/somekey-4",
+		watchTests: []*testWatchStruct{{podFoo, true, watch.Added}, {podBar, true, watch.Modified}},
+		pred:       storage.Everything,
+	}, { // delete because of being filtered
+		key:        "/somekey-5",
+		watchTests: []*testWatchStruct{{podFoo, true, watch.Added}, {podBar, true, watch.Deleted}},
+		pred: storage.SelectionPredicate{
+			Label: labels.Everything(),
+			Field: fields.ParseSelectorOrDie("metadata.name!=bar"),
+			GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+				pod := obj.(*example.Pod)
+				return nil, fields.Set{"metadata.name": pod.Name}, pod.Initializers != nil, nil
+			},
+		},
+	}}
+	for i, tt := range tests {
+		w, err := store.watch(ctx, tt.key, "0", tt.pred, recursive)
+		if err != nil {
+			t.Fatalf("Watch failed: %v", err)
+		}
+		var prevObj *example.Pod
+		for _, watchTest := range tt.watchTests {
+			out := &example.Pod{}
+			key := tt.key
+			if recursive {
+				key = key + "/item"
+			}
+			err := store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
+				func(runtime.Object) (runtime.Object, error) {
+					return watchTest.obj, nil
+				}))
+			if err != nil {
+				t.Fatalf("GuaranteedUpdate failed: %v", err)
+			}
+			if watchTest.expectEvent {
+				expectObj := out
+				if watchTest.watchType == watch.Deleted {
+					expectObj = prevObj
+					expectObj.ResourceVersion = out.ResourceVersion
+				}
+				testCheckResult(t, i, watchTest.watchType, w, expectObj)
+			}
+			prevObj = out
+		}
+		w.Stop()
+		testCheckStop(t, i, w)
+	}
+}
+
+func TestDeleteTriggerWatch(t *testing.T) {
+	ctx, store, cluster := testSetup(t)
+	defer cluster.Terminate(t)
+	key, storedObj := testPropogateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}})
+	w, err := store.Watch(ctx, key, storedObj.ResourceVersion, storage.Everything)
+	if err != nil {
+		t.Fatalf("Watch failed: %v", err)
+	}
+	if err := store.Delete(ctx, key, &example.Pod{}, nil); err != nil {
+		t.Fatalf("Delete failed: %v", err)
+	}
+	testCheckEventType(t, watch.Deleted, w)
+}
+
+// TestWatchFromZero tests that
+// - watch from 0 should sync up and grab the object added before
+// - watch from 0 is able to return events for objects whose previous version has been compacted
+func TestWatchFromZero(t *testing.T) {
+	ctx, store, cluster := testSetup(t)
+	defer cluster.Terminate(t)
+	key, storedObj := testPropogateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "ns"}})
+
+	w, err := store.Watch(ctx, key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Watch failed: %v", err)
+	}
+	testCheckResult(t, 0, watch.Added, w, storedObj)
+	w.Stop()
+
+	// Update
+	out := &example.Pod{}
+	err = store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
+		func(runtime.Object) (runtime.Object, error) {
+			return &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "ns", Annotations: map[string]string{"a": "1"}}}, nil
+		}))
+	if err != nil {
+		t.Fatalf("GuaranteedUpdate failed: %v", err)
+	}
+
+	// Make sure when we watch from 0 we receive an ADDED event
+	w, err = store.Watch(ctx, key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Watch failed: %v", err)
+	}
+	testCheckResult(t, 1, watch.Added, w, out)
+	w.Stop()
+
+	// Update again
+	out = &example.Pod{}
+	err = store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
+		func(runtime.Object) (runtime.Object, error) {
+			return &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "ns"}}, nil
+		}))
+	if err != nil {
+		t.Fatalf("GuaranteedUpdate failed: %v", err)
+	}
+
+	// Compact previous versions
+	revToCompact, err := strconv.Atoi(out.ResourceVersion)
+	if err != nil {
+		t.Fatalf("Error converting %q to an int: %v", storedObj.ResourceVersion, err)
+	}
+	_, err = cluster.RandClient().Compact(ctx, int64(revToCompact), clientv3.WithCompactPhysical())
+	if err != nil {
+		t.Fatalf("Error compacting: %v", err)
+	}
+
+	// Make sure we can still watch from 0 and receive an ADDED event
+	w, err = store.Watch(ctx, key, "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Watch failed: %v", err)
+	}
+	testCheckResult(t, 2, watch.Added, w, out)
+}
+
+// TestWatchFromNoneZero tests that
+// - watch from non-0 should just watch changes after given version
+func TestWatchFromNoneZero(t *testing.T) {
+	ctx, store, cluster := testSetup(t)
+	defer cluster.Terminate(t)
+	key, storedObj := testPropogateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}})
+
+	w, err := store.Watch(ctx, key, storedObj.ResourceVersion, storage.Everything)
+	if err != nil {
+		t.Fatalf("Watch failed: %v", err)
+	}
+	out := &example.Pod{}
+	store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
+		func(runtime.Object) (runtime.Object, error) {
+			return &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, err
+		}))
+	testCheckResult(t, 0, watch.Modified, w, out)
+}
+
+func TestWatchError(t *testing.T) {
+	codec := &testCodec{apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)}
+	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer cluster.Terminate(t)
+	invalidStore := newStore(cluster.RandClient(), false, true, codec, "", prefixTransformer{prefix: []byte("test!")})
+	ctx := context.Background()
+	w, err := invalidStore.Watch(ctx, "/abc", "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Watch failed: %v", err)
+	}
+	validStore := newStore(cluster.RandClient(), false, true, codec, "", prefixTransformer{prefix: []byte("test!")})
+	validStore.GuaranteedUpdate(ctx, "/abc", &example.Pod{}, true, nil, storage.SimpleUpdate(
+		func(runtime.Object) (runtime.Object, error) {
+			return &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, nil
+		}))
+	testCheckEventType(t, watch.Error, w)
+}
+
+func TestWatchContextCancel(t *testing.T) {
+	ctx, store, cluster := testSetup(t)
+	defer cluster.Terminate(t)
+	canceledCtx, cancel := context.WithCancel(ctx)
+	cancel()
+	// When we watch with a canceled context, we should detect that it's context canceled.
+	// We won't take it as error and also close the watcher.
+	w, err := store.watcher.Watch(canceledCtx, "/abc", 0, false, storage.Everything)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case _, ok := <-w.ResultChan():
+		if ok {
+			t.Error("ResultChan() should be closed")
+		}
+	case <-time.After(wait.ForeverTestTimeout):
+		t.Errorf("timeout after %v", wait.ForeverTestTimeout)
+	}
+}
+
+func TestWatchErrResultNotBlockAfterCancel(t *testing.T) {
+	origCtx, store, cluster := testSetup(t)
+	defer cluster.Terminate(t)
+	ctx, cancel := context.WithCancel(origCtx)
+	w := store.watcher.createWatchChan(ctx, "/abc", 0, false, storage.Everything)
+	// make resutlChan and errChan blocking to ensure ordering.
+	w.resultChan = make(chan watch.Event)
+	w.errChan = make(chan error)
+	// The event flow goes like:
+	// - first we send an error, it should block on resultChan.
+	// - Then we cancel ctx. The blocking on resultChan should be freed up
+	//   and run() goroutine should return.
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		w.run()
+		wg.Done()
+	}()
+	w.errChan <- fmt.Errorf("some error")
+	cancel()
+	wg.Wait()
+}
+
+func TestWatchDeleteEventObjectHaveLatestRV(t *testing.T) {
+	ctx, store, cluster := testSetup(t)
+	defer cluster.Terminate(t)
+	key, storedObj := testPropogateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}})
+
+	w, err := store.Watch(ctx, key, storedObj.ResourceVersion, storage.Everything)
+	if err != nil {
+		t.Fatalf("Watch failed: %v", err)
+	}
+	etcdW := cluster.RandClient().Watch(ctx, "/", clientv3.WithPrefix())
+
+	if err := store.Delete(ctx, key, &example.Pod{}, &storage.Preconditions{}); err != nil {
+		t.Fatalf("Delete failed: %v", err)
+	}
+
+	e := <-w.ResultChan()
+	watchedDeleteObj := e.Object.(*example.Pod)
+	var wres clientv3.WatchResponse
+	wres = <-etcdW
+
+	watchedDeleteRev, err := storage.ParseWatchResourceVersion(watchedDeleteObj.ResourceVersion)
+	if err != nil {
+		t.Fatalf("ParseWatchResourceVersion failed: %v", err)
+	}
+	if int64(watchedDeleteRev) != wres.Events[0].Kv.ModRevision {
+		t.Errorf("Object from delete event have version: %v, should be the same as etcd delete's mod rev: %d",
+			watchedDeleteRev, wres.Events[0].Kv.ModRevision)
+	}
+}
+
+type testWatchStruct struct {
+	obj         *example.Pod
+	expectEvent bool
+	watchType   watch.EventType
+}
+
+type testCodec struct {
+	runtime.Codec
+}
+
+func (c *testCodec) Decode(data []byte, defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
+	return nil, nil, errTestingDecode
+}
+
+func testCheckEventType(t *testing.T, expectEventType watch.EventType, w watch.Interface) {
+	select {
+	case res := <-w.ResultChan():
+		if res.Type != expectEventType {
+			t.Errorf("event type want=%v, get=%v", expectEventType, res.Type)
+		}
+	case <-time.After(wait.ForeverTestTimeout):
+		t.Errorf("time out after waiting %v on ResultChan", wait.ForeverTestTimeout)
+	}
+}
+
+func testCheckResult(t *testing.T, i int, expectEventType watch.EventType, w watch.Interface, expectObj *example.Pod) {
+	select {
+	case res := <-w.ResultChan():
+		if res.Type != expectEventType {
+			t.Errorf("#%d: event type want=%v, get=%v", i, expectEventType, res.Type)
+			return
+		}
+		if !reflect.DeepEqual(expectObj, res.Object) {
+			t.Errorf("#%d: obj want=\n%#v\nget=\n%#v", i, expectObj, res.Object)
+		}
+	case <-time.After(wait.ForeverTestTimeout):
+		t.Errorf("#%d: time out after waiting %v on ResultChan", i, wait.ForeverTestTimeout)
+	}
+}
+
+func testCheckStop(t *testing.T, i int, w watch.Interface) {
+	select {
+	case e, ok := <-w.ResultChan():
+		if ok {
+			var obj string
+			switch e.Object.(type) {
+			case *example.Pod:
+				obj = e.Object.(*example.Pod).Name
+			case *metav1.Status:
+				obj = e.Object.(*metav1.Status).Message
+			}
+			t.Errorf("#%d: ResultChan should have been closed. Event: %s. Object: %s", i, e.Type, obj)
+		}
+	case <-time.After(wait.ForeverTestTimeout):
+		t.Errorf("#%d: time out after waiting 1s on ResultChan", i)
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/interfaces.go

@@ -0,0 +1,189 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"golang.org/x/net/context"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+// Versioner abstracts setting and retrieving metadata fields from database response
+// onto the object ot list. It is required to maintain storage invariants - updating an
+// object twice with the same data except for the ResourceVersion and SelfLink must be
+// a no-op.
+type Versioner interface {
+	// UpdateObject sets storage metadata into an API object. Returns an error if the object
+	// cannot be updated correctly. May return nil if the requested object does not need metadata
+	// from database.
+	UpdateObject(obj runtime.Object, resourceVersion uint64) error
+	// UpdateList sets the resource version into an API list object. Returns an error if the object
+	// cannot be updated correctly. May return nil if the requested object does not need metadata
+	// from database. continueValue is optional and indicates that more results are available if
+	// the client passes that value to the server in a subsequent call.
+	UpdateList(obj runtime.Object, resourceVersion uint64, continueValue string) error
+	// PrepareObjectForStorage should set SelfLink and ResourceVersion to the empty value. Should
+	// return an error if the specified object cannot be updated.
+	PrepareObjectForStorage(obj runtime.Object) error
+	// ObjectResourceVersion returns the resource version (for persistence) of the specified object.
+	// Should return an error if the specified object does not have a persistable version.
+	ObjectResourceVersion(obj runtime.Object) (uint64, error)
+}
+
+// ResponseMeta contains information about the database metadata that is associated with
+// an object. It abstracts the actual underlying objects to prevent coupling with concrete
+// database and to improve testability.
+type ResponseMeta struct {
+	// TTL is the time to live of the node that contained the returned object. It may be
+	// zero or negative in some cases (objects may be expired after the requested
+	// expiration time due to server lag).
+	TTL int64
+	// The resource version of the node that contained the returned object.
+	ResourceVersion uint64
+}
+
+// MatchValue defines a pair (<index name>, <value for that index>).
+type MatchValue struct {
+	IndexName string
+	Value     string
+}
+
+// TriggerPublisherFunc is a function that takes an object, and returns a list of pairs
+// (<index name>, <index value for the given object>) for all indexes known
+// to that function.
+type TriggerPublisherFunc func(obj runtime.Object) []MatchValue
+
+// FilterFunc takes an API object and returns true if the object satisfies some requirements.
+// TODO: We will remove this type and use SelectionPredicate everywhere.
+type FilterFunc func(obj runtime.Object) bool
+
+// Everything accepts all objects.
+var Everything = SelectionPredicate{
+	Label: labels.Everything(),
+	Field: fields.Everything(),
+	// TODO: split this into a new top level constant?
+	IncludeUninitialized: true,
+}
+
+// Pass an UpdateFunc to Interface.GuaranteedUpdate to make an update
+// that is guaranteed to succeed.
+// See the comment for GuaranteedUpdate for more details.
+type UpdateFunc func(input runtime.Object, res ResponseMeta) (output runtime.Object, ttl *uint64, err error)
+
+// Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.
+type Preconditions struct {
+	// Specifies the target UID.
+	// +optional
+	UID *types.UID `json:"uid,omitempty"`
+}
+
+// NewUIDPreconditions returns a Preconditions with UID set.
+func NewUIDPreconditions(uid string) *Preconditions {
+	u := types.UID(uid)
+	return &Preconditions{UID: &u}
+}
+
+// Interface offers a common interface for object marshaling/unmarshaling operations and
+// hides all the storage-related operations behind it.
+type Interface interface {
+	// Returns Versioner associated with this interface.
+	Versioner() Versioner
+
+	// Create adds a new object at a key unless it already exists. 'ttl' is time-to-live
+	// in seconds (0 means forever). If no error is returned and out is not nil, out will be
+	// set to the read value from database.
+	Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error
+
+	// Delete removes the specified key and returns the value that existed at that spot.
+	// If key didn't exist, it will return NotFound storage error.
+	Delete(ctx context.Context, key string, out runtime.Object, preconditions *Preconditions) error
+
+	// Watch begins watching the specified key. Events are decoded into API objects,
+	// and any items selected by 'p' are sent down to returned watch.Interface.
+	// resourceVersion may be used to specify what version to begin watching,
+	// which should be the current resourceVersion, and no longer rv+1
+	// (e.g. reconnecting without missing any updates).
+	// If resource version is "0", this interface will get current object at given key
+	// and send it in an "ADDED" event, before watch starts.
+	Watch(ctx context.Context, key string, resourceVersion string, p SelectionPredicate) (watch.Interface, error)
+
+	// WatchList begins watching the specified key's items. Items are decoded into API
+	// objects and any item selected by 'p' are sent down to returned watch.Interface.
+	// resourceVersion may be used to specify what version to begin watching,
+	// which should be the current resourceVersion, and no longer rv+1
+	// (e.g. reconnecting without missing any updates).
+	// If resource version is "0", this interface will list current objects directory defined by key
+	// and send them in "ADDED" events, before watch starts.
+	WatchList(ctx context.Context, key string, resourceVersion string, p SelectionPredicate) (watch.Interface, error)
+
+	// Get unmarshals json found at key into objPtr. On a not found error, will either
+	// return a zero object of the requested type, or an error, depending on ignoreNotFound.
+	// Treats empty responses and nil response nodes exactly like a not found error.
+	// The returned contents may be delayed, but it is guaranteed that they will
+	// be have at least 'resourceVersion'.
+	Get(ctx context.Context, key string, resourceVersion string, objPtr runtime.Object, ignoreNotFound bool) error
+
+	// GetToList unmarshals json found at key and opaque it into *List api object
+	// (an object that satisfies the runtime.IsList definition).
+	// The returned contents may be delayed, but it is guaranteed that they will
+	// be have at least 'resourceVersion'.
+	GetToList(ctx context.Context, key string, resourceVersion string, p SelectionPredicate, listObj runtime.Object) error
+
+	// List unmarshalls jsons found at directory defined by key and opaque them
+	// into *List api object (an object that satisfies runtime.IsList definition).
+	// The returned contents may be delayed, but it is guaranteed that they will
+	// be have at least 'resourceVersion'.
+	List(ctx context.Context, key string, resourceVersion string, p SelectionPredicate, listObj runtime.Object) error
+
+	// GuaranteedUpdate keeps calling 'tryUpdate()' to update key 'key' (of type 'ptrToType')
+	// retrying the update until success if there is index conflict.
+	// Note that object passed to tryUpdate may change across invocations of tryUpdate() if
+	// other writers are simultaneously updating it, so tryUpdate() needs to take into account
+	// the current contents of the object when deciding how the update object should look.
+	// If the key doesn't exist, it will return NotFound storage error if ignoreNotFound=false
+	// or zero value in 'ptrToType' parameter otherwise.
+	// If the object to update has the same value as previous, it won't do any update
+	// but will return the object in 'ptrToType' parameter.
+	// If 'suggestion' can contain zero or one element - in such case this can be used as
+	// a suggestion about the current version of the object to avoid read operation from
+	// storage to get it.
+	//
+	// Example:
+	//
+	// s := /* implementation of Interface */
+	// err := s.GuaranteedUpdate(
+	//     "myKey", &MyType{}, true,
+	//     func(input runtime.Object, res ResponseMeta) (runtime.Object, *uint64, error) {
+	//       // Before each incovation of the user defined function, "input" is reset to
+	//       // current contents for "myKey" in database.
+	//       curr := input.(*MyType)  // Guaranteed to succeed.
+	//
+	//       // Make the modification
+	//       curr.Counter++
+	//
+	//       // Return the modified object - return an error to stop iterating. Return
+	//       // a uint64 to alter the TTL on the object, or nil to keep it the same value.
+	//       return cur, nil, nil
+	//    }
+	// })
+	GuaranteedUpdate(
+		ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
+		precondtions *Preconditions, tryUpdate UpdateFunc, suggestion ...runtime.Object) error
+}

vendor/k8s.io/apiserver/pkg/storage/names/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["generate_test.go"],
+    library = ":go_default_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["generate.go"],
+    deps = ["//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/names/generate.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package names
+
+import (
+	"fmt"
+
+	utilrand "k8s.io/apimachinery/pkg/util/rand"
+)
+
+// NameGenerator generates names for objects. Some backends may have more information
+// available to guide selection of new names and this interface hides those details.
+type NameGenerator interface {
+	// GenerateName generates a valid name from the base name, adding a random suffix to the
+	// the base. If base is valid, the returned name must also be valid. The generator is
+	// responsible for knowing the maximum valid name length.
+	GenerateName(base string) string
+}
+
+// simpleNameGenerator generates random names.
+type simpleNameGenerator struct{}
+
+// SimpleNameGenerator is a generator that returns the name plus a random suffix of five alphanumerics
+// when a name is requested. The string is guaranteed to not exceed the length of a standard Kubernetes
+// name (63 characters)
+var SimpleNameGenerator NameGenerator = simpleNameGenerator{}
+
+const (
+	// TODO: make this flexible for non-core resources with alternate naming rules.
+	maxNameLength          = 63
+	randomLength           = 5
+	maxGeneratedNameLength = maxNameLength - randomLength
+)
+
+func (simpleNameGenerator) GenerateName(base string) string {
+	if len(base) > maxGeneratedNameLength {
+		base = base[:maxGeneratedNameLength]
+	}
+	return fmt.Sprintf("%s%s", base, utilrand.String(randomLength))
+}

vendor/k8s.io/apiserver/pkg/storage/names/generate_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package names
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSimpleNameGenerator(t *testing.T) {
+	name := SimpleNameGenerator.GenerateName("foo")
+	if !strings.HasPrefix(name, "foo") || name == "foo" {
+		t.Errorf("unexpected name: %s", name)
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/selection_predicate.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// AttrFunc returns label and field sets and the uninitialized flag for List or Watch to match.
+// In any failure to parse given object, it returns error.
+type AttrFunc func(obj runtime.Object) (labels.Set, fields.Set, bool, error)
+
+// FieldMutationFunc allows the mutation of the field selection fields.  It is mutating to
+// avoid the extra allocation on this common path
+type FieldMutationFunc func(obj runtime.Object, fieldSet fields.Set) error
+
+func DefaultClusterScopedAttr(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		return nil, nil, false, err
+	}
+	fieldSet := fields.Set{
+		"metadata.name": metadata.GetName(),
+	}
+
+	return labels.Set(metadata.GetLabels()), fieldSet, metadata.GetInitializers() != nil, nil
+}
+
+func DefaultNamespaceScopedAttr(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		return nil, nil, false, err
+	}
+	fieldSet := fields.Set{
+		"metadata.name":      metadata.GetName(),
+		"metadata.namespace": metadata.GetNamespace(),
+	}
+
+	return labels.Set(metadata.GetLabels()), fieldSet, metadata.GetInitializers() != nil, nil
+}
+
+func (f AttrFunc) WithFieldMutation(fieldMutator FieldMutationFunc) AttrFunc {
+	return func(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+		labelSet, fieldSet, initialized, err := f(obj)
+		if err != nil {
+			return nil, nil, false, err
+		}
+		if err := fieldMutator(obj, fieldSet); err != nil {
+			return nil, nil, false, err
+		}
+		return labelSet, fieldSet, initialized, nil
+	}
+}
+
+// SelectionPredicate is used to represent the way to select objects from api storage.
+type SelectionPredicate struct {
+	Label                labels.Selector
+	Field                fields.Selector
+	IncludeUninitialized bool
+	GetAttrs             AttrFunc
+	IndexFields          []string
+	Limit                int64
+	Continue             string
+}
+
+// Matches returns true if the given object's labels and fields (as
+// returned by s.GetAttrs) match s.Label and s.Field. An error is
+// returned if s.GetAttrs fails.
+func (s *SelectionPredicate) Matches(obj runtime.Object) (bool, error) {
+	if s.Empty() {
+		return true, nil
+	}
+	labels, fields, uninitialized, err := s.GetAttrs(obj)
+	if err != nil {
+		return false, err
+	}
+	if !s.IncludeUninitialized && uninitialized {
+		return false, nil
+	}
+	matched := s.Label.Matches(labels)
+	if matched && s.Field != nil {
+		matched = (matched && s.Field.Matches(fields))
+	}
+	return matched, nil
+}
+
+// MatchesObjectAttributes returns true if the given labels and fields
+// match s.Label and s.Field.
+func (s *SelectionPredicate) MatchesObjectAttributes(l labels.Set, f fields.Set, uninitialized bool) bool {
+	if !s.IncludeUninitialized && uninitialized {
+		return false
+	}
+	if s.Label.Empty() && s.Field.Empty() {
+		return true
+	}
+	matched := s.Label.Matches(l)
+	if matched && s.Field != nil {
+		matched = (matched && s.Field.Matches(f))
+	}
+	return matched
+}
+
+// MatchesSingle will return (name, true) if and only if s.Field matches on the object's
+// name.
+func (s *SelectionPredicate) MatchesSingle() (string, bool) {
+	if len(s.Continue) > 0 {
+		return "", false
+	}
+	// TODO: should be namespace.name
+	if name, ok := s.Field.RequiresExactMatch("metadata.name"); ok {
+		return name, true
+	}
+	return "", false
+}
+
+// For any index defined by IndexFields, if a matcher can match only (a subset)
+// of objects that return <value> for a given index, a pair (<index name>, <value>)
+// wil be returned.
+// TODO: Consider supporting also labels.
+func (s *SelectionPredicate) MatcherIndex() []MatchValue {
+	var result []MatchValue
+	for _, field := range s.IndexFields {
+		if value, ok := s.Field.RequiresExactMatch(field); ok {
+			result = append(result, MatchValue{IndexName: field, Value: value})
+		}
+	}
+	return result
+}
+
+// Empty returns true if the predicate performs no filtering.
+func (s *SelectionPredicate) Empty() bool {
+	return s.Label.Empty() && s.Field.Empty() && s.IncludeUninitialized
+}

vendor/k8s.io/apiserver/pkg/storage/selection_predicate_test.go

@@ -0,0 +1,134 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"errors"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type Ignored struct {
+	ID string
+}
+
+type IgnoredList struct {
+	Items []Ignored
+}
+
+func (obj *Ignored) GetObjectKind() schema.ObjectKind     { return schema.EmptyObjectKind }
+func (obj *IgnoredList) GetObjectKind() schema.ObjectKind { return schema.EmptyObjectKind }
+func (obj *Ignored) DeepCopyObject() runtime.Object {
+	panic("Ignored does not support DeepCopy")
+}
+func (obj *IgnoredList) DeepCopyObject() runtime.Object {
+	panic("IgnoredList does not support DeepCopy")
+}
+
+func TestSelectionPredicate(t *testing.T) {
+	table := map[string]struct {
+		labelSelector, fieldSelector string
+		labels                       labels.Set
+		fields                       fields.Set
+		uninitialized                bool
+		err                          error
+		shouldMatch                  bool
+		matchSingleKey               string
+	}{
+		"A": {
+			labelSelector: "name=foo",
+			fieldSelector: "uid=12345",
+			labels:        labels.Set{"name": "foo"},
+			fields:        fields.Set{"uid": "12345"},
+			shouldMatch:   true,
+		},
+		"B": {
+			labelSelector: "name=foo",
+			fieldSelector: "uid=12345",
+			labels:        labels.Set{"name": "foo"},
+			fields:        fields.Set{},
+			shouldMatch:   false,
+		},
+		"C": {
+			labelSelector: "name=foo",
+			fieldSelector: "uid=12345",
+			labels:        labels.Set{},
+			fields:        fields.Set{"uid": "12345"},
+			shouldMatch:   false,
+		},
+		"D": {
+			fieldSelector:  "metadata.name=12345",
+			labels:         labels.Set{},
+			fields:         fields.Set{"metadata.name": "12345"},
+			shouldMatch:    true,
+			matchSingleKey: "12345",
+		},
+		"E": {
+			fieldSelector:  "metadata.name=12345",
+			labels:         labels.Set{},
+			fields:         fields.Set{"metadata.name": "12345"},
+			uninitialized:  true,
+			shouldMatch:    false,
+			matchSingleKey: "12345",
+		},
+		"error": {
+			labelSelector: "name=foo",
+			fieldSelector: "uid=12345",
+			err:           errors.New("maybe this is a 'wrong object type' error"),
+			shouldMatch:   false,
+		},
+	}
+
+	for name, item := range table {
+		parsedLabel, err := labels.Parse(item.labelSelector)
+		if err != nil {
+			panic(err)
+		}
+		parsedField, err := fields.ParseSelector(item.fieldSelector)
+		if err != nil {
+			panic(err)
+		}
+		sp := &SelectionPredicate{
+			Label: parsedLabel,
+			Field: parsedField,
+			GetAttrs: func(runtime.Object) (label labels.Set, field fields.Set, uninitialized bool, err error) {
+				return item.labels, item.fields, item.uninitialized, item.err
+			},
+		}
+		got, err := sp.Matches(&Ignored{})
+		if e, a := item.err, err; e != a {
+			t.Errorf("%v: expected %v, got %v", name, e, a)
+			continue
+		}
+		if e, a := item.shouldMatch, got; e != a {
+			t.Errorf("%v: expected %v, got %v", name, e, a)
+		}
+		if key := item.matchSingleKey; key != "" {
+			got, ok := sp.MatchesSingle()
+			if !ok {
+				t.Errorf("%v: expected single match", name)
+			}
+			if e, a := key, got; e != a {
+				t.Errorf("%v: expected %v, got %v", name, e, a)
+			}
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/storagebackend/BUILD

@@ -0,0 +1,31 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    deps = [
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/storagebackend/OWNERS

@@ -0,0 +1,6 @@
+reviewers:
+- lavalamp
+- smarterclayton
+- wojtek-t
+- timothysc
+- hongchaodeng

vendor/k8s.io/apiserver/pkg/storage/storagebackend/config.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storagebackend
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+const (
+	StorageTypeUnset = ""
+	StorageTypeETCD2 = "etcd2"
+	StorageTypeETCD3 = "etcd3"
+)
+
+// Config is configuration for creating a storage backend.
+type Config struct {
+	// Type defines the type of storage backend, e.g. "etcd2", etcd3". Default ("") is "etcd3".
+	Type string
+	// Prefix is the prefix to all keys passed to storage.Interface methods.
+	Prefix string
+	// ServerList is the list of storage servers to connect with.
+	ServerList []string
+	// TLS credentials
+	KeyFile  string
+	CertFile string
+	CAFile   string
+	// Quorum indicates that whether read operations should be quorum-level consistent.
+	Quorum bool
+	// Paging indicates whether the server implementation should allow paging (if it is
+	// supported). This is generally configured by feature gating, or by a specific
+	// resource type not wishing to allow paging, and is not intended for end users to
+	// set.
+	Paging bool
+	// DeserializationCacheSize is the size of cache of deserialized objects.
+	// Currently this is only supported in etcd2.
+	// We will drop the cache once using protobuf.
+	DeserializationCacheSize int
+
+	Codec  runtime.Codec
+	Copier runtime.ObjectCopier
+	// Transformer allows the value to be transformed prior to persisting into etcd.
+	Transformer value.Transformer
+}
+
+func NewDefaultConfig(prefix string, copier runtime.ObjectCopier, codec runtime.Codec) *Config {
+	return &Config{
+		Prefix: prefix,
+		// Default cache size to 0 - if unset, its size will be set based on target
+		// memory usage.
+		DeserializationCacheSize: 0,
+		Copier: copier,
+		Codec:  codec,
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["tls_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/coreos/etcd/integration:go_default_library",
+        "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/testing:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/testing/testingcert:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "etcd2.go",
+        "etcd3.go",
+        "factory.go",
+    ],
+    deps = [
+        "//vendor/github.com/coreos/etcd/client:go_default_library",
+        "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
+        "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd3:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd2.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package factory
+
+import (
+	"net"
+	"net/http"
+	"time"
+
+	etcd2client "github.com/coreos/etcd/client"
+	"github.com/coreos/etcd/pkg/transport"
+
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/etcd"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+)
+
+func newETCD2Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+	tr, err := newTransportForETCD2(c.CertFile, c.KeyFile, c.CAFile)
+	if err != nil {
+		return nil, nil, err
+	}
+	client, err := newETCD2Client(tr, c.ServerList)
+	if err != nil {
+		return nil, nil, err
+	}
+	s := etcd.NewEtcdStorage(client, c.Codec, c.Prefix, c.Quorum, c.DeserializationCacheSize, c.Copier, etcd.IdentityTransformer)
+	return s, tr.CloseIdleConnections, nil
+}
+
+func newETCD2Client(tr *http.Transport, serverList []string) (etcd2client.Client, error) {
+	cli, err := etcd2client.New(etcd2client.Config{
+		Endpoints: serverList,
+		Transport: tr,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return cli, nil
+}
+
+func newTransportForETCD2(certFile, keyFile, caFile string) (*http.Transport, error) {
+	info := transport.TLSInfo{
+		CertFile: certFile,
+		KeyFile:  keyFile,
+		CAFile:   caFile,
+	}
+	cfg, err := info.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	// Copied from etcd.DefaultTransport declaration.
+	// TODO: Determine if transport needs optimization
+	tr := utilnet.SetTransportDefaults(&http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		Dial: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout: 10 * time.Second,
+		MaxIdleConnsPerHost: 500,
+		TLSClientConfig:     cfg,
+	})
+	return tr, nil
+}

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package factory
+
+import (
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/pkg/transport"
+	"golang.org/x/net/context"
+
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/etcd3"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+	tlsInfo := transport.TLSInfo{
+		CertFile: c.CertFile,
+		KeyFile:  c.KeyFile,
+		CAFile:   c.CAFile,
+	}
+	tlsConfig, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, nil, err
+	}
+	// NOTE: Client relies on nil tlsConfig
+	// for non-secure connections, update the implicit variable
+	if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.CAFile) == 0 {
+		tlsConfig = nil
+	}
+	cfg := clientv3.Config{
+		Endpoints: c.ServerList,
+		TLS:       tlsConfig,
+	}
+	client, err := clientv3.New(cfg)
+	if err != nil {
+		return nil, nil, err
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	etcd3.StartCompactor(ctx, client)
+	destroyFunc := func() {
+		cancel()
+		client.Close()
+	}
+	transformer := c.Transformer
+	if transformer == nil {
+		transformer = value.IdentityTransformer
+	}
+	if c.Quorum {
+		return etcd3.New(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
+	}
+	return etcd3.NewWithNoQuorumRead(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
+}

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package factory
+
+import (
+	"fmt"
+
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+)
+
+// DestroyFunc is to destroy any resources used by the storage returned in Create() together.
+type DestroyFunc func()
+
+// Create creates a storage backend based on given config.
+func Create(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+	switch c.Type {
+	case storagebackend.StorageTypeETCD2:
+		return newETCD2Storage(c)
+	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
+		// TODO: We have the following features to implement:
+		// - Support secure connection by using key, cert, and CA files.
+		// - Honor "https" scheme to support secure connection in gRPC.
+		// - Support non-quorum read.
+		return newETCD3Storage(c)
+	default:
+		return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type)
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/tls_test.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package factory
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/coreos/etcd/integration"
+	"github.com/coreos/etcd/pkg/transport"
+	"golang.org/x/net/context"
+
+	apitesting "k8s.io/apimachinery/pkg/api/testing"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/apis/example"
+	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
+	"k8s.io/apiserver/pkg/storage/etcd/testing/testingcert"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+)
+
+var scheme = runtime.NewScheme()
+var codecs = serializer.NewCodecFactory(scheme)
+
+func init() {
+	metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
+	example.AddToScheme(scheme)
+	examplev1.AddToScheme(scheme)
+}
+
+func TestTLSConnection(t *testing.T) {
+	codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
+
+	certFile, keyFile, caFile := configureTLSCerts(t)
+	defer os.RemoveAll(filepath.Dir(certFile))
+
+	tlsInfo := &transport.TLSInfo{
+		CertFile: certFile,
+		KeyFile:  keyFile,
+		CAFile:   caFile,
+	}
+
+	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{
+		Size:      1,
+		ClientTLS: tlsInfo,
+	})
+	defer cluster.Terminate(t)
+
+	cfg := storagebackend.Config{
+		Type:       storagebackend.StorageTypeETCD3,
+		ServerList: []string{cluster.Members[0].GRPCAddr()},
+		CertFile:   certFile,
+		KeyFile:    keyFile,
+		CAFile:     caFile,
+		Codec:      codec,
+		Copier:     scheme,
+	}
+	storage, destroyFunc, err := newETCD3Storage(cfg)
+	defer destroyFunc()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = storage.Create(context.TODO(), "/abc", &example.Pod{}, nil, 0)
+	if err != nil {
+		t.Fatalf("Create failed: %v", err)
+	}
+}
+
+func configureTLSCerts(t *testing.T) (certFile, keyFile, caFile string) {
+	baseDir := os.TempDir()
+	tempDir, err := ioutil.TempDir(baseDir, "etcd_certificates")
+	if err != nil {
+		t.Fatal(err)
+	}
+	certFile = path.Join(tempDir, "etcdcert.pem")
+	if err := ioutil.WriteFile(certFile, []byte(testingcert.CertFileContent), 0644); err != nil {
+		t.Fatal(err)
+	}
+	keyFile = path.Join(tempDir, "etcdkey.pem")
+	if err := ioutil.WriteFile(keyFile, []byte(testingcert.KeyFileContent), 0644); err != nil {
+		t.Fatal(err)
+	}
+	caFile = path.Join(tempDir, "ca.pem")
+	if err := ioutil.WriteFile(caFile, []byte(testingcert.CAFileContent), 0644); err != nil {
+		t.Fatal(err)
+	}
+	return certFile, keyFile, caFile
+}

vendor/k8s.io/apiserver/pkg/storage/testing/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "types.go",
+        "utils.go",
+        "zz_generated.deepcopy.go",
+    ],
+    deps = [
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/testing/OWNERS

@@ -0,0 +1,10 @@
+reviewers:
+- smarterclayton
+- wojtek-t
+- liggitt
+- erictune
+- timothysc
+- soltysh
+- mml
+- feihujiang
+- enj

vendor/k8s.io/apiserver/pkg/storage/testing/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:deepcopy-gen=package
+
+package testing

vendor/k8s.io/apiserver/pkg/storage/testing/types.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type TestResource struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata"`
+	Value             int `json:"value"`
+}

vendor/k8s.io/apiserver/pkg/storage/testing/utils.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"path"
+
+	"golang.org/x/net/context"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/storage"
+)
+
+// CreateObj will create a single object using the storage interface
+func CreateObj(helper storage.Interface, name string, obj, out runtime.Object, ttl uint64) error {
+	return helper.Create(context.TODO(), name, obj, out, ttl)
+}
+
+//CreateObjList will create a list from the array of objects
+func CreateObjList(prefix string, helper storage.Interface, items []runtime.Object) error {
+	for i := range items {
+		obj := items[i]
+		meta, err := meta.Accessor(obj)
+		if err != nil {
+			return err
+		}
+		err = CreateObj(helper, path.Join(prefix, meta.GetName()), obj, obj, 0)
+		if err != nil {
+			return err
+		}
+		items[i] = obj
+	}
+	return nil
+}
+
+// CreateList will properly create a list using the storage interface
+func CreateList(prefix string, helper storage.Interface, list runtime.Object) error {
+	items, err := meta.ExtractList(list)
+	if err != nil {
+		return err
+	}
+	err = CreateObjList(prefix, helper, items)
+	if err != nil {
+		return err
+	}
+	return meta.SetList(list, items)
+}

vendor/k8s.io/apiserver/pkg/storage/testing/zz_generated.deepcopy.go

@@ -0,0 +1,66 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This file was autogenerated by deepcopy-gen. Do not edit it manually!
+
+package testing
+
+import (
+	conversion "k8s.io/apimachinery/pkg/conversion"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	reflect "reflect"
+)
+
+// GetGeneratedDeepCopyFuncs returns the generated funcs, since we aren't registering them.
+//
+// Deprecated: deepcopy registration will go away when static deepcopy is fully implemented.
+func GetGeneratedDeepCopyFuncs() []conversion.GeneratedDeepCopyFunc {
+	return []conversion.GeneratedDeepCopyFunc{
+		{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*TestResource).DeepCopyInto(out.(*TestResource))
+			return nil
+		}, InType: reflect.TypeOf(&TestResource{})},
+	}
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TestResource) DeepCopyInto(out *TestResource) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestResource.
+func (in *TestResource) DeepCopy() *TestResource {
+	if in == nil {
+		return nil
+	}
+	out := new(TestResource)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *TestResource) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	} else {
+		return nil
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/tests/BUILD

@@ -0,0 +1,55 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["cacher_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/testing:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/etcdtest:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd3:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["utils.go"],
+    deps = ["//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/apiserver/pkg/storage/tests/cacher_test.go

@@ -0,0 +1,638 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tests
+
+import (
+	"fmt"
+	"reflect"
+	goruntime "runtime"
+	"strconv"
+	"testing"
+	"time"
+
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	apitesting "k8s.io/apimachinery/pkg/api/testing"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/apis/example"
+	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
+	"k8s.io/apiserver/pkg/storage"
+	etcdstorage "k8s.io/apiserver/pkg/storage/etcd"
+	"k8s.io/apiserver/pkg/storage/etcd/etcdtest"
+	etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
+	"k8s.io/apiserver/pkg/storage/etcd3"
+	"k8s.io/apiserver/pkg/storage/value"
+
+	"golang.org/x/net/context"
+
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+)
+
+var (
+	scheme = runtime.NewScheme()
+	codecs = serializer.NewCodecFactory(scheme)
+)
+
+func init() {
+	metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
+	example.AddToScheme(scheme)
+	examplev1.AddToScheme(scheme)
+}
+
+// GetAttrs returns labels and fields of a given object for filtering purposes.
+func GetAttrs(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+	pod, ok := obj.(*example.Pod)
+	if !ok {
+		return nil, nil, false, fmt.Errorf("not a pod")
+	}
+	return labels.Set(pod.ObjectMeta.Labels), PodToSelectableFields(pod), pod.Initializers != nil, nil
+}
+
+// PodToSelectableFields returns a field set that represents the object
+// TODO: fields are not labels, and the validation rules for them do not apply.
+func PodToSelectableFields(pod *example.Pod) fields.Set {
+	// The purpose of allocation with a given number of elements is to reduce
+	// amount of allocations needed to create the fields.Set. If you add any
+	// field here or the number of object-meta related fields changes, this should
+	// be adjusted.
+	podSpecificFieldsSet := make(fields.Set, 5)
+	podSpecificFieldsSet["spec.nodeName"] = pod.Spec.NodeName
+	podSpecificFieldsSet["spec.restartPolicy"] = string(pod.Spec.RestartPolicy)
+	podSpecificFieldsSet["status.phase"] = string(pod.Status.Phase)
+	return AddObjectMetaFieldsSet(podSpecificFieldsSet, &pod.ObjectMeta, true)
+}
+
+func AddObjectMetaFieldsSet(source fields.Set, objectMeta *metav1.ObjectMeta, hasNamespaceField bool) fields.Set {
+	source["metadata.name"] = objectMeta.Name
+	if hasNamespaceField {
+		source["metadata.namespace"] = objectMeta.Namespace
+	}
+	return source
+}
+
+func newEtcdTestStorage(t *testing.T, prefix string) (*etcdtesting.EtcdTestServer, storage.Interface) {
+	server, _ := etcdtesting.NewUnsecuredEtcd3TestClientServer(t, scheme)
+	storage := etcd3.New(server.V3Client, apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion), prefix, value.IdentityTransformer, true)
+	return server, storage
+}
+
+func newTestCacher(s storage.Interface, cap int) *storage.Cacher {
+	prefix := "pods"
+	config := storage.CacherConfig{
+		CacheCapacity:  cap,
+		Storage:        s,
+		Versioner:      etcdstorage.APIObjectVersioner{},
+		Copier:         scheme,
+		Type:           &example.Pod{},
+		ResourcePrefix: prefix,
+		KeyFunc:        func(obj runtime.Object) (string, error) { return storage.NamespaceKeyFunc(prefix, obj) },
+		GetAttrsFunc:   GetAttrs,
+		NewListFunc:    func() runtime.Object { return &example.PodList{} },
+		Codec:          codecs.LegacyCodec(examplev1.SchemeGroupVersion),
+	}
+	return storage.NewCacherFromConfig(config)
+}
+
+func makeTestPod(name string) *example.Pod {
+	return &example.Pod{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: name},
+		Spec:       DeepEqualSafePodSpec(),
+	}
+}
+
+func updatePod(t *testing.T, s storage.Interface, obj, old *example.Pod) *example.Pod {
+	updateFn := func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) {
+		return obj.DeepCopyObject(), nil, nil
+	}
+	key := "pods/" + obj.Namespace + "/" + obj.Name
+	if err := s.GuaranteedUpdate(context.TODO(), key, &example.Pod{}, old == nil, nil, updateFn); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	obj.ResourceVersion = ""
+	result := &example.Pod{}
+	if err := s.Get(context.TODO(), key, "", result, false); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	return result
+}
+
+func TestGet(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	defer server.Terminate(t)
+	cacher := newTestCacher(etcdStorage, 10)
+	defer cacher.Stop()
+
+	podFoo := makeTestPod("foo")
+	fooCreated := updatePod(t, etcdStorage, podFoo, nil)
+
+	// We pass the ResourceVersion from the above Create() operation.
+	result := &example.Pod{}
+	if err := cacher.Get(context.TODO(), "pods/ns/foo", fooCreated.ResourceVersion, result, true); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if e, a := *fooCreated, *result; !reflect.DeepEqual(e, a) {
+		t.Errorf("Expected: %#v, got: %#v", e, a)
+	}
+
+	if err := cacher.Get(context.TODO(), "pods/ns/bar", fooCreated.ResourceVersion, result, true); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	emptyPod := example.Pod{}
+	if e, a := emptyPod, *result; !reflect.DeepEqual(e, a) {
+		t.Errorf("Expected: %#v, got: %#v", e, a)
+	}
+
+	if err := cacher.Get(context.TODO(), "pods/ns/bar", fooCreated.ResourceVersion, result, false); !storage.IsNotFound(err) {
+		t.Errorf("Unexpected error: %v", err)
+	}
+}
+
+func TestList(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	defer server.Terminate(t)
+	cacher := newTestCacher(etcdStorage, 10)
+	defer cacher.Stop()
+
+	podFoo := makeTestPod("foo")
+	podBar := makeTestPod("bar")
+	podBaz := makeTestPod("baz")
+
+	podFooPrime := makeTestPod("foo")
+	podFooPrime.Spec.NodeName = "fakeNode"
+
+	fooCreated := updatePod(t, etcdStorage, podFoo, nil)
+	_ = updatePod(t, etcdStorage, podBar, nil)
+	_ = updatePod(t, etcdStorage, podBaz, nil)
+
+	_ = updatePod(t, etcdStorage, podFooPrime, fooCreated)
+
+	// Create a pod in a namespace that contains "ns" as a prefix
+	// Make sure it is not returned in a watch of "ns"
+	podFooNS2 := makeTestPod("foo")
+	podFooNS2.Namespace += "2"
+	updatePod(t, etcdStorage, podFooNS2, nil)
+
+	deleted := example.Pod{}
+	if err := etcdStorage.Delete(context.TODO(), "pods/ns/bar", &deleted, nil); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	// We first List directly from etcd by passing empty resourceVersion,
+	// to get the current etcd resourceVersion.
+	rvResult := &example.PodList{}
+	if err := cacher.List(context.TODO(), "pods/ns", "", storage.Everything, rvResult); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	deletedPodRV := rvResult.ListMeta.ResourceVersion
+
+	result := &example.PodList{}
+	// We pass the current etcd ResourceVersion received from the above List() operation,
+	// since there is not easy way to get ResourceVersion of barPod deletion operation.
+	if err := cacher.List(context.TODO(), "pods/ns", deletedPodRV, storage.Everything, result); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if result.ListMeta.ResourceVersion != deletedPodRV {
+		t.Errorf("Incorrect resource version: %v", result.ListMeta.ResourceVersion)
+	}
+	if len(result.Items) != 2 {
+		t.Errorf("Unexpected list result: %d", len(result.Items))
+	}
+	keys := sets.String{}
+	for _, item := range result.Items {
+		keys.Insert(item.Name)
+	}
+	if !keys.HasAll("foo", "baz") {
+		t.Errorf("Unexpected list result: %#v", result)
+	}
+	for _, item := range result.Items {
+		// unset fields that are set by the infrastructure
+		item.ResourceVersion = ""
+		item.CreationTimestamp = metav1.Time{}
+
+		if item.Namespace != "ns" {
+			t.Errorf("Unexpected namespace: %s", item.Namespace)
+		}
+
+		var expected *example.Pod
+		switch item.Name {
+		case "foo":
+			expected = podFooPrime
+		case "baz":
+			expected = podBaz
+		default:
+			t.Errorf("Unexpected item: %v", item)
+		}
+		if e, a := *expected, item; !reflect.DeepEqual(e, a) {
+			t.Errorf("Expected: %#v, got: %#v", e, a)
+		}
+	}
+}
+
+func TestInfiniteList(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	defer server.Terminate(t)
+	cacher := newTestCacher(etcdStorage, 10)
+	defer cacher.Stop()
+
+	podFoo := makeTestPod("foo")
+	fooCreated := updatePod(t, etcdStorage, podFoo, nil)
+
+	// Set up List at fooCreated.ResourceVersion + 10
+	rv, err := storage.ParseWatchResourceVersion(fooCreated.ResourceVersion)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	listRV := strconv.Itoa(int(rv + 10))
+
+	result := &example.PodList{}
+	err = cacher.List(context.TODO(), "pods/ns", listRV, storage.Everything, result)
+	if !errors.IsTimeout(err) {
+		t.Errorf("Unexpected error: %v", err)
+	}
+}
+
+func verifyWatchEvent(t *testing.T, w watch.Interface, eventType watch.EventType, eventObject runtime.Object) {
+	_, _, line, _ := goruntime.Caller(1)
+	select {
+	case event := <-w.ResultChan():
+		if e, a := eventType, event.Type; e != a {
+			t.Logf("(called from line %d)", line)
+			t.Errorf("Expected: %s, got: %s", eventType, event.Type)
+		}
+		if e, a := eventObject, event.Object; !apiequality.Semantic.DeepDerivative(e, a) {
+			t.Logf("(called from line %d)", line)
+			t.Errorf("Expected (%s): %#v, got: %#v", eventType, e, a)
+		}
+	case <-time.After(wait.ForeverTestTimeout):
+		t.Logf("(called from line %d)", line)
+		t.Errorf("Timed out waiting for an event")
+	}
+}
+
+type injectListError struct {
+	errors int
+	storage.Interface
+}
+
+func (self *injectListError) List(ctx context.Context, key string, resourceVersion string, p storage.SelectionPredicate, listObj runtime.Object) error {
+	if self.errors > 0 {
+		self.errors--
+		return fmt.Errorf("injected error")
+	}
+	return self.Interface.List(ctx, key, resourceVersion, p, listObj)
+}
+
+func TestWatch(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	// Inject one list error to make sure we test the relist case.
+	etcdStorage = &injectListError{errors: 1, Interface: etcdStorage}
+	defer server.Terminate(t)
+	cacher := newTestCacher(etcdStorage, 3) // small capacity to trigger "too old version" error
+	defer cacher.Stop()
+
+	podFoo := makeTestPod("foo")
+	podBar := makeTestPod("bar")
+
+	podFooPrime := makeTestPod("foo")
+	podFooPrime.Spec.NodeName = "fakeNode"
+
+	podFooBis := makeTestPod("foo")
+	podFooBis.Spec.NodeName = "anotherFakeNode"
+
+	podFooNS2 := makeTestPod("foo")
+	podFooNS2.Namespace += "2"
+
+	// initialVersion is used to initate the watcher at the beginning of the world,
+	// which is not defined precisely in etcd.
+	initialVersion, err := cacher.LastSyncResourceVersion()
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	startVersion := strconv.Itoa(int(initialVersion))
+
+	// Set up Watch for object "podFoo".
+	watcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", startVersion, storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer watcher.Stop()
+
+	// Create in another namespace first to make sure events from other namespaces don't get delivered
+	updatePod(t, etcdStorage, podFooNS2, nil)
+
+	fooCreated := updatePod(t, etcdStorage, podFoo, nil)
+	_ = updatePod(t, etcdStorage, podBar, nil)
+	fooUpdated := updatePod(t, etcdStorage, podFooPrime, fooCreated)
+
+	verifyWatchEvent(t, watcher, watch.Added, podFoo)
+	verifyWatchEvent(t, watcher, watch.Modified, podFooPrime)
+
+	// Check whether we get too-old error via the watch channel
+	tooOldWatcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", "1", storage.Everything)
+	if err != nil {
+		t.Fatalf("Expected no direct error, got %v", err)
+	}
+	defer tooOldWatcher.Stop()
+	// Ensure we get a "Gone" error
+	expectedGoneError := errors.NewGone("").ErrStatus
+	verifyWatchEvent(t, tooOldWatcher, watch.Error, &expectedGoneError)
+
+	initialWatcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", fooCreated.ResourceVersion, storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer initialWatcher.Stop()
+
+	verifyWatchEvent(t, initialWatcher, watch.Modified, podFooPrime)
+
+	// Now test watch from "now".
+	nowWatcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer nowWatcher.Stop()
+
+	verifyWatchEvent(t, nowWatcher, watch.Added, podFooPrime)
+
+	_ = updatePod(t, etcdStorage, podFooBis, fooUpdated)
+
+	verifyWatchEvent(t, nowWatcher, watch.Modified, podFooBis)
+}
+
+func TestWatcherTimeout(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	defer server.Terminate(t)
+	cacher := newTestCacher(etcdStorage, 10)
+	defer cacher.Stop()
+
+	// initialVersion is used to initate the watcher at the beginning of the world,
+	// which is not defined precisely in etcd.
+	initialVersion, err := cacher.LastSyncResourceVersion()
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	startVersion := strconv.Itoa(int(initialVersion))
+
+	// Create a number of watchers that will not be reading any result.
+	nonReadingWatchers := 50
+	for i := 0; i < nonReadingWatchers; i++ {
+		watcher, err := cacher.WatchList(context.TODO(), "pods/ns", startVersion, storage.Everything)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+		defer watcher.Stop()
+	}
+
+	// Create a second watcher that will be reading result.
+	readingWatcher, err := cacher.WatchList(context.TODO(), "pods/ns", startVersion, storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer readingWatcher.Stop()
+
+	startTime := time.Now()
+	for i := 1; i <= 22; i++ {
+		pod := makeTestPod(strconv.Itoa(i))
+		_ = updatePod(t, etcdStorage, pod, nil)
+		verifyWatchEvent(t, readingWatcher, watch.Added, pod)
+	}
+	if time.Since(startTime) > time.Duration(250*nonReadingWatchers)*time.Millisecond {
+		t.Errorf("waiting for events took too long: %v", time.Since(startTime))
+	}
+}
+
+func TestFiltering(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	defer server.Terminate(t)
+	cacher := newTestCacher(etcdStorage, 10)
+	defer cacher.Stop()
+
+	// Ensure that the cacher is initialized, before creating any pods,
+	// so that we are sure that all events will be present in cacher.
+	syncWatcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", "0", storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	syncWatcher.Stop()
+
+	podFoo := makeTestPod("foo")
+	podFoo.Labels = map[string]string{"filter": "foo"}
+	podFooFiltered := makeTestPod("foo")
+	podFooPrime := makeTestPod("foo")
+	podFooPrime.Labels = map[string]string{"filter": "foo"}
+	podFooPrime.Spec.NodeName = "fakeNode"
+
+	podFooNS2 := makeTestPod("foo")
+	podFooNS2.Namespace += "2"
+	podFooNS2.Labels = map[string]string{"filter": "foo"}
+
+	// Create in another namespace first to make sure events from other namespaces don't get delivered
+	updatePod(t, etcdStorage, podFooNS2, nil)
+
+	fooCreated := updatePod(t, etcdStorage, podFoo, nil)
+	fooFiltered := updatePod(t, etcdStorage, podFooFiltered, fooCreated)
+	fooUnfiltered := updatePod(t, etcdStorage, podFoo, fooFiltered)
+	_ = updatePod(t, etcdStorage, podFooPrime, fooUnfiltered)
+
+	deleted := example.Pod{}
+	if err := etcdStorage.Delete(context.TODO(), "pods/ns/foo", &deleted, nil); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	// Set up Watch for object "podFoo" with label filter set.
+	pred := storage.SelectionPredicate{
+		Label: labels.SelectorFromSet(labels.Set{"filter": "foo"}),
+		Field: fields.Everything(),
+		GetAttrs: func(obj runtime.Object) (label labels.Set, field fields.Set, uninitialized bool, err error) {
+			metadata, err := meta.Accessor(obj)
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+			return labels.Set(metadata.GetLabels()), nil, metadata.GetInitializers() != nil, nil
+		},
+	}
+	watcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", fooCreated.ResourceVersion, pred)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer watcher.Stop()
+
+	verifyWatchEvent(t, watcher, watch.Deleted, podFooFiltered)
+	verifyWatchEvent(t, watcher, watch.Added, podFoo)
+	verifyWatchEvent(t, watcher, watch.Modified, podFooPrime)
+	verifyWatchEvent(t, watcher, watch.Deleted, podFooPrime)
+}
+
+func TestStartingResourceVersion(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	defer server.Terminate(t)
+	cacher := newTestCacher(etcdStorage, 10)
+	defer cacher.Stop()
+
+	// add 1 object
+	podFoo := makeTestPod("foo")
+	fooCreated := updatePod(t, etcdStorage, podFoo, nil)
+
+	// Set up Watch starting at fooCreated.ResourceVersion + 10
+	rv, err := storage.ParseWatchResourceVersion(fooCreated.ResourceVersion)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	rv += 10
+	startVersion := strconv.Itoa(int(rv))
+
+	watcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", startVersion, storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer watcher.Stop()
+
+	lastFoo := fooCreated
+	for i := 0; i < 11; i++ {
+		podFooForUpdate := makeTestPod("foo")
+		podFooForUpdate.Labels = map[string]string{"foo": strconv.Itoa(i)}
+		lastFoo = updatePod(t, etcdStorage, podFooForUpdate, lastFoo)
+	}
+
+	select {
+	case e := <-watcher.ResultChan():
+		pod := e.Object.(*example.Pod)
+		podRV, err := storage.ParseWatchResourceVersion(pod.ResourceVersion)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		// event should have at least rv + 1, since we're starting the watch at rv
+		if podRV <= rv {
+			t.Errorf("expected event with resourceVersion of at least %d, got %d", rv+1, podRV)
+		}
+	case <-time.After(wait.ForeverTestTimeout):
+		t.Errorf("timed out waiting for event")
+	}
+}
+
+func TestEmptyWatchEventCache(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	defer server.Terminate(t)
+
+	// add a few objects
+	updatePod(t, etcdStorage, makeTestPod("pod1"), nil)
+	updatePod(t, etcdStorage, makeTestPod("pod2"), nil)
+	updatePod(t, etcdStorage, makeTestPod("pod3"), nil)
+	updatePod(t, etcdStorage, makeTestPod("pod4"), nil)
+	updatePod(t, etcdStorage, makeTestPod("pod5"), nil)
+
+	fooCreated := updatePod(t, etcdStorage, makeTestPod("foo"), nil)
+
+	// get rv of last pod created
+	rv, err := storage.ParseWatchResourceVersion(fooCreated.ResourceVersion)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	cacher := newTestCacher(etcdStorage, 10)
+	defer cacher.Stop()
+
+	// We now have a cacher with an empty cache of watch events and a resourceVersion of rv.
+	// It should support establishing watches from rv and higher, but not older.
+
+	{
+		watcher, err := cacher.Watch(context.TODO(), "pods/ns", strconv.Itoa(int(rv-1)), storage.Everything)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+		defer watcher.Stop()
+		expectedGoneError := errors.NewGone("").ErrStatus
+		verifyWatchEvent(t, watcher, watch.Error, &expectedGoneError)
+	}
+
+	{
+		watcher, err := cacher.Watch(context.TODO(), "pods/ns", strconv.Itoa(int(rv+1)), storage.Everything)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+		defer watcher.Stop()
+		select {
+		case e := <-watcher.ResultChan():
+			t.Errorf("unexpected event %#v", e)
+		case <-time.After(3 * time.Second):
+			// watch from rv+1 remained established successfully
+		}
+	}
+
+	{
+		watcher, err := cacher.Watch(context.TODO(), "pods/ns", strconv.Itoa(int(rv)), storage.Everything)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+		defer watcher.Stop()
+		select {
+		case e := <-watcher.ResultChan():
+			t.Errorf("unexpected event %#v", e)
+		case <-time.After(3 * time.Second):
+			// watch from rv remained established successfully
+		}
+	}
+}
+
+func TestRandomWatchDeliver(t *testing.T) {
+	server, etcdStorage := newEtcdTestStorage(t, etcdtest.PathPrefix())
+	defer server.Terminate(t)
+	cacher := newTestCacher(etcdStorage, 10)
+	defer cacher.Stop()
+
+	fooCreated := updatePod(t, etcdStorage, makeTestPod("foo"), nil)
+	rv, err := storage.ParseWatchResourceVersion(fooCreated.ResourceVersion)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	startVersion := strconv.Itoa(int(rv))
+
+	watcher, err := cacher.WatchList(context.TODO(), "pods/ns", startVersion, storage.Everything)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	// Now we can create exactly 21 events that should be delivered
+	// to the watcher, before it will completely block cacher and as
+	// a result will be dropped.
+	for i := 0; i < 21; i++ {
+		updatePod(t, etcdStorage, makeTestPod(fmt.Sprintf("foo-%d", i)), nil)
+	}
+
+	// Now stop the watcher and check if the consecutive events are being delivered.
+	watcher.Stop()
+
+	watched := 0
+	for {
+		event, ok := <-watcher.ResultChan()
+		if !ok {
+			break
+		}
+		if a, e := event.Object.(*example.Pod).Name, fmt.Sprintf("foo-%d", watched); e != a {
+			t.Errorf("Unexpected object watched: %s, expected %s", a, e)
+		}
+		watched++
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/tests/utils.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tests
+
+import (
+	"k8s.io/apiserver/pkg/apis/example"
+)
+
+func DeepEqualSafePodSpec() example.PodSpec {
+	grace := int64(30)
+	return example.PodSpec{
+		RestartPolicy:                 "Always",
+		TerminationGracePeriodSeconds: &grace,
+		SchedulerName:                 "default-scheduler",
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/time_budget.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"sync"
+	"time"
+)
+
+const (
+	refreshPerSecond = 50 * time.Millisecond
+	maxBudget        = 100 * time.Millisecond
+)
+
+// timeBudget implements a budget of time that you can use and is
+// periodically being refreshed. The pattern to use it is:
+//   budget := newTimeBudget(...)
+//   ...
+//   timeout := budget.takeAvailable()
+//   // Now you can spend at most timeout on doing stuff
+//   ...
+//   // If you didn't use all timeout, return what you didn't use
+//   budget.returnUnused(<unused part of timeout>)
+//
+// NOTE: It's not recommended to be used concurrently from multiple threads -
+// if first user takes the whole timeout, the second one will get 0 timeout
+// even though the first one may return something later.
+type timeBudget struct {
+	sync.Mutex
+	budget time.Duration
+
+	refresh   time.Duration
+	maxBudget time.Duration
+}
+
+func newTimeBudget(stopCh <-chan struct{}) *timeBudget {
+	result := &timeBudget{
+		budget:    time.Duration(0),
+		refresh:   refreshPerSecond,
+		maxBudget: maxBudget,
+	}
+	go result.periodicallyRefresh(stopCh)
+	return result
+}
+
+func (t *timeBudget) periodicallyRefresh(stopCh <-chan struct{}) {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			t.Lock()
+			if t.budget = t.budget + t.refresh; t.budget > t.maxBudget {
+				t.budget = t.maxBudget
+			}
+			t.Unlock()
+		case <-stopCh:
+			return
+		}
+	}
+}
+
+func (t *timeBudget) takeAvailable() time.Duration {
+	t.Lock()
+	defer t.Unlock()
+	result := t.budget
+	t.budget = time.Duration(0)
+	return result
+}
+
+func (t *timeBudget) returnUnused(unused time.Duration) {
+	t.Lock()
+	defer t.Unlock()
+	if unused < 0 {
+		// We used more than allowed.
+		return
+	}
+	if t.budget = t.budget + unused; t.budget > t.maxBudget {
+		t.budget = t.maxBudget
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/time_budget_test.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"testing"
+	"time"
+)
+
+func TestTimeBudget(t *testing.T) {
+	budget := &timeBudget{
+		budget:    time.Duration(0),
+		maxBudget: time.Duration(200),
+	}
+	if res := budget.takeAvailable(); res != time.Duration(0) {
+		t.Errorf("Expected: %v, got: %v", time.Duration(0), res)
+	}
+	budget.budget = time.Duration(100)
+	if res := budget.takeAvailable(); res != time.Duration(100) {
+		t.Errorf("Expected: %v, got: %v", time.Duration(100), res)
+	}
+	if res := budget.takeAvailable(); res != time.Duration(0) {
+		t.Errorf("Expected: %v, got: %v", time.Duration(0), res)
+	}
+	budget.returnUnused(time.Duration(50))
+	if res := budget.takeAvailable(); res != time.Duration(50) {
+		t.Errorf("Expected: %v, got: %v", time.Duration(50), res)
+	}
+	budget.budget = time.Duration(100)
+	budget.returnUnused(-time.Duration(50))
+	if res := budget.takeAvailable(); res != time.Duration(100) {
+		t.Errorf("Expected: %v, got: %v", time.Duration(100), res)
+	}
+	// test overflow.
+	budget.returnUnused(time.Duration(500))
+	if res := budget.takeAvailable(); res != time.Duration(200) {
+		t.Errorf("Expected: %v, got: %v", time.Duration(200), res)
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/util.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"sync/atomic"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/api/validation/path"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+type SimpleUpdateFunc func(runtime.Object) (runtime.Object, error)
+
+// SimpleUpdateFunc converts SimpleUpdateFunc into UpdateFunc
+func SimpleUpdate(fn SimpleUpdateFunc) UpdateFunc {
+	return func(input runtime.Object, _ ResponseMeta) (runtime.Object, *uint64, error) {
+		out, err := fn(input)
+		return out, nil, err
+	}
+}
+
+// SimpleFilter converts a selection predicate into a FilterFunc.
+// It ignores any error from Matches().
+func SimpleFilter(p SelectionPredicate) FilterFunc {
+	return func(obj runtime.Object) bool {
+		matches, err := p.Matches(obj)
+		if err != nil {
+			glog.Errorf("invalid object for matching. Obj: %v. Err: %v", obj, err)
+			return false
+		}
+		return matches
+	}
+}
+
+func EverythingFunc(runtime.Object) bool {
+	return true
+}
+
+func NoTriggerFunc() []MatchValue {
+	return nil
+}
+
+func NoTriggerPublisher(runtime.Object) []MatchValue {
+	return nil
+}
+
+// ParseWatchResourceVersion takes a resource version argument and converts it to
+// the etcd version we should pass to helper.Watch(). Because resourceVersion is
+// an opaque value, the default watch behavior for non-zero watch is to watch
+// the next value (if you pass "1", you will see updates from "2" onwards).
+func ParseWatchResourceVersion(resourceVersion string) (uint64, error) {
+	if resourceVersion == "" || resourceVersion == "0" {
+		return 0, nil
+	}
+	version, err := strconv.ParseUint(resourceVersion, 10, 64)
+	if err != nil {
+		return 0, NewInvalidError(field.ErrorList{
+			// Validation errors are supposed to return version-specific field
+			// paths, but this is probably close enough.
+			field.Invalid(field.NewPath("resourceVersion"), resourceVersion, err.Error()),
+		})
+	}
+	return version, nil
+}
+
+// ParseListResourceVersion takes a resource version argument and converts it to
+// the etcd version.
+func ParseListResourceVersion(resourceVersion string) (uint64, error) {
+	if resourceVersion == "" {
+		return 0, nil
+	}
+	version, err := strconv.ParseUint(resourceVersion, 10, 64)
+	return version, err
+}
+
+func NamespaceKeyFunc(prefix string, obj runtime.Object) (string, error) {
+	meta, err := meta.Accessor(obj)
+	if err != nil {
+		return "", err
+	}
+	name := meta.GetName()
+	if msgs := path.IsValidPathSegmentName(name); len(msgs) != 0 {
+		return "", fmt.Errorf("invalid name: %v", msgs)
+	}
+	return prefix + "/" + meta.GetNamespace() + "/" + name, nil
+}
+
+func NoNamespaceKeyFunc(prefix string, obj runtime.Object) (string, error) {
+	meta, err := meta.Accessor(obj)
+	if err != nil {
+		return "", err
+	}
+	name := meta.GetName()
+	if msgs := path.IsValidPathSegmentName(name); len(msgs) != 0 {
+		return "", fmt.Errorf("invalid name: %v", msgs)
+	}
+	return prefix + "/" + name, nil
+}
+
+// hasPathPrefix returns true if the string matches pathPrefix exactly, or if is prefixed with pathPrefix at a path segment boundary
+func hasPathPrefix(s, pathPrefix string) bool {
+	// Short circuit if s doesn't contain the prefix at all
+	if !strings.HasPrefix(s, pathPrefix) {
+		return false
+	}
+
+	pathPrefixLength := len(pathPrefix)
+
+	if len(s) == pathPrefixLength {
+		// Exact match
+		return true
+	}
+	if strings.HasSuffix(pathPrefix, "/") {
+		// pathPrefix already ensured a path segment boundary
+		return true
+	}
+	if s[pathPrefixLength:pathPrefixLength+1] == "/" {
+		// The next character in s is a path segment boundary
+		// Check this instead of normalizing pathPrefix to avoid allocating on every call
+		return true
+	}
+	return false
+}
+
+// HighWaterMark is a thread-safe object for tracking the maximum value seen
+// for some quantity.
+type HighWaterMark int64
+
+// Update returns true if and only if 'current' is the highest value ever seen.
+func (hwm *HighWaterMark) Update(current int64) bool {
+	for {
+		old := atomic.LoadInt64((*int64)(hwm))
+		if current <= old {
+			return false
+		}
+		if atomic.CompareAndSwapInt64((*int64)(hwm), old, current) {
+			return true
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/util_test.go

@@ -0,0 +1,136 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"math/rand"
+	"sync"
+	"testing"
+)
+
+func TestEtcdParseWatchResourceVersion(t *testing.T) {
+	testCases := []struct {
+		Version       string
+		ExpectVersion uint64
+		Err           bool
+	}{
+		{Version: "", ExpectVersion: 0},
+		{Version: "a", Err: true},
+		{Version: " ", Err: true},
+		{Version: "1", ExpectVersion: 1},
+		{Version: "10", ExpectVersion: 10},
+	}
+	for _, testCase := range testCases {
+		version, err := ParseWatchResourceVersion(testCase.Version)
+		switch {
+		case testCase.Err:
+			if err == nil {
+				t.Errorf("%s: unexpected non-error", testCase.Version)
+				continue
+			}
+			if !IsInvalidError(err) {
+				t.Errorf("%s: unexpected error: %v", testCase.Version, err)
+				continue
+			}
+		case !testCase.Err && err != nil:
+			t.Errorf("%s: unexpected error: %v", testCase.Version, err)
+			continue
+		}
+		if version != testCase.ExpectVersion {
+			t.Errorf("%s: expected version %d but was %d", testCase.Version, testCase.ExpectVersion, version)
+		}
+	}
+}
+
+func TestHasPathPrefix(t *testing.T) {
+	validTestcases := []struct {
+		s      string
+		prefix string
+	}{
+		// Exact matches
+		{"", ""},
+		{"a", "a"},
+		{"a/", "a/"},
+		{"a/../", "a/../"},
+
+		// Path prefix matches
+		{"a/b", "a"},
+		{"a/b", "a/"},
+		{"中文/", "中文"},
+	}
+	for i, tc := range validTestcases {
+		if !hasPathPrefix(tc.s, tc.prefix) {
+			t.Errorf(`%d: Expected hasPathPrefix("%s","%s") to be true`, i, tc.s, tc.prefix)
+		}
+	}
+
+	invalidTestcases := []struct {
+		s      string
+		prefix string
+	}{
+		// Mismatch
+		{"a", "b"},
+
+		// Dir requirement
+		{"a", "a/"},
+
+		// Prefix mismatch
+		{"ns2", "ns"},
+		{"ns2", "ns/"},
+		{"中文文", "中文"},
+
+		// Ensure no normalization is applied
+		{"a/c/../b/", "a/b/"},
+		{"a/", "a/b/.."},
+	}
+	for i, tc := range invalidTestcases {
+		if hasPathPrefix(tc.s, tc.prefix) {
+			t.Errorf(`%d: Expected hasPathPrefix("%s","%s") to be false`, i, tc.s, tc.prefix)
+		}
+	}
+}
+
+func TestHighWaterMark(t *testing.T) {
+	var h HighWaterMark
+
+	for i := int64(10); i < 20; i++ {
+		if !h.Update(i) {
+			t.Errorf("unexpected false for %v", i)
+		}
+		if h.Update(i - 1) {
+			t.Errorf("unexpected true for %v", i-1)
+		}
+	}
+
+	m := int64(0)
+	wg := sync.WaitGroup{}
+	for i := 0; i < 300; i++ {
+		wg.Add(1)
+		v := rand.Int63()
+		go func(v int64) {
+			defer wg.Done()
+			h.Update(v)
+		}(v)
+		if v > m {
+			m = v
+		}
+	}
+	wg.Wait()
+	if m != int64(h) {
+		t.Errorf("unexpected value, wanted %v, got %v", m, int64(h))
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes_test.go

@@ -0,0 +1,372 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aes
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"reflect"
+	"testing"
+
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+func TestGCMDataStable(t *testing.T) {
+	block, err := aes.NewCipher([]byte("0123456789abcdef"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	aead, err := cipher.NewGCM(block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// IMPORTANT: If you must fix this test, then all previously encrypted data from previously compiled versions is broken unless you hardcode the nonce size to 12
+	if aead.NonceSize() != 12 {
+		t.Fatalf("The underlying Golang crypto size has changed, old version of AES on disk will not be readable unless the AES implementation is changed to hardcode nonce size.")
+	}
+}
+
+func TestGCMKeyRotation(t *testing.T) {
+	testErr := fmt.Errorf("test error")
+	block1, err := aes.NewCipher([]byte("abcdefghijklmnop"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	block2, err := aes.NewCipher([]byte("0123456789abcdef"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	context := value.DefaultContext([]byte("authenticated_data"))
+
+	p := value.NewPrefixTransformers(testErr,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)},
+	)
+	out, err := p.TransformToStorage([]byte("firstvalue"), context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.HasPrefix(out, []byte("first:")) {
+		t.Fatalf("unexpected prefix: %q", out)
+	}
+	from, stale, err := p.TransformFromStorage(out, context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stale || !bytes.Equal([]byte("firstvalue"), from) {
+		t.Fatalf("unexpected data: %t %q", stale, from)
+	}
+
+	// verify changing the context fails storage
+	from, stale, err = p.TransformFromStorage(out, value.DefaultContext([]byte("incorrect_context")))
+	if err == nil {
+		t.Fatalf("expected unauthenticated data")
+	}
+
+	// reverse the order, use the second key
+	p = value.NewPrefixTransformers(testErr,
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)},
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)},
+	)
+	from, stale, err = p.TransformFromStorage(out, context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !stale || !bytes.Equal([]byte("firstvalue"), from) {
+		t.Fatalf("unexpected data: %t %q", stale, from)
+	}
+}
+
+func TestCBCKeyRotation(t *testing.T) {
+	testErr := fmt.Errorf("test error")
+	block1, err := aes.NewCipher([]byte("abcdefghijklmnop"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	block2, err := aes.NewCipher([]byte("0123456789abcdef"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	context := value.DefaultContext([]byte("authenticated_data"))
+
+	p := value.NewPrefixTransformers(testErr,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewCBCTransformer(block1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewCBCTransformer(block2)},
+	)
+	out, err := p.TransformToStorage([]byte("firstvalue"), context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.HasPrefix(out, []byte("first:")) {
+		t.Fatalf("unexpected prefix: %q", out)
+	}
+	from, stale, err := p.TransformFromStorage(out, context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stale || !bytes.Equal([]byte("firstvalue"), from) {
+		t.Fatalf("unexpected data: %t %q", stale, from)
+	}
+
+	// verify changing the context fails storage
+	from, stale, err = p.TransformFromStorage(out, value.DefaultContext([]byte("incorrect_context")))
+	if err != nil {
+		t.Fatalf("CBC mode does not support authentication: %v", err)
+	}
+
+	// reverse the order, use the second key
+	p = value.NewPrefixTransformers(testErr,
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewCBCTransformer(block2)},
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewCBCTransformer(block1)},
+	)
+	from, stale, err = p.TransformFromStorage(out, context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !stale || !bytes.Equal([]byte("firstvalue"), from) {
+		t.Fatalf("unexpected data: %t %q", stale, from)
+	}
+}
+
+func BenchmarkGCMRead_16_1024(b *testing.B)        { benchmarkGCMRead(b, 16, 1024, false) }
+func BenchmarkGCMRead_32_1024(b *testing.B)        { benchmarkGCMRead(b, 32, 1024, false) }
+func BenchmarkGCMRead_32_16384(b *testing.B)       { benchmarkGCMRead(b, 32, 16384, false) }
+func BenchmarkGCMRead_32_16384_Stale(b *testing.B) { benchmarkGCMRead(b, 32, 16384, true) }
+
+func BenchmarkGCMWrite_16_1024(b *testing.B)  { benchmarkGCMWrite(b, 16, 1024) }
+func BenchmarkGCMWrite_32_1024(b *testing.B)  { benchmarkGCMWrite(b, 32, 1024) }
+func BenchmarkGCMWrite_32_16384(b *testing.B) { benchmarkGCMWrite(b, 32, 16384) }
+
+func benchmarkGCMRead(b *testing.B, keyLength int, valueLength int, expectStale bool) {
+	block1, err := aes.NewCipher(bytes.Repeat([]byte("a"), keyLength))
+	if err != nil {
+		b.Fatal(err)
+	}
+	block2, err := aes.NewCipher(bytes.Repeat([]byte("b"), keyLength))
+	if err != nil {
+		b.Fatal(err)
+	}
+	p := value.NewPrefixTransformers(nil,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)},
+	)
+
+	context := value.DefaultContext([]byte("authenticated_data"))
+	v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16)
+
+	out, err := p.TransformToStorage(v, context)
+	if err != nil {
+		b.Fatal(err)
+	}
+	// reverse the key order if expecting stale
+	if expectStale {
+		p = value.NewPrefixTransformers(nil,
+			value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)},
+			value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)},
+		)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		from, stale, err := p.TransformFromStorage(out, context)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if expectStale != stale {
+			b.Fatalf("unexpected data: %q, expect stale %t but got %t", from, expectStale, stale)
+		}
+	}
+	b.StopTimer()
+}
+
+func benchmarkGCMWrite(b *testing.B, keyLength int, valueLength int) {
+	block1, err := aes.NewCipher(bytes.Repeat([]byte("a"), keyLength))
+	if err != nil {
+		b.Fatal(err)
+	}
+	block2, err := aes.NewCipher(bytes.Repeat([]byte("b"), keyLength))
+	if err != nil {
+		b.Fatal(err)
+	}
+	p := value.NewPrefixTransformers(nil,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)},
+	)
+
+	context := value.DefaultContext([]byte("authenticated_data"))
+	v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := p.TransformToStorage(v, context)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.StopTimer()
+}
+
+func BenchmarkCBCRead_32_1024(b *testing.B)        { benchmarkCBCRead(b, 32, 1024, false) }
+func BenchmarkCBCRead_32_16384(b *testing.B)       { benchmarkCBCRead(b, 32, 16384, false) }
+func BenchmarkCBCRead_32_16384_Stale(b *testing.B) { benchmarkCBCRead(b, 32, 16384, true) }
+
+func BenchmarkCBCWrite_32_1024(b *testing.B)  { benchmarkCBCWrite(b, 32, 1024) }
+func BenchmarkCBCWrite_32_16384(b *testing.B) { benchmarkCBCWrite(b, 32, 16384) }
+
+func benchmarkCBCRead(b *testing.B, keyLength int, valueLength int, expectStale bool) {
+	block1, err := aes.NewCipher(bytes.Repeat([]byte("a"), keyLength))
+	if err != nil {
+		b.Fatal(err)
+	}
+	block2, err := aes.NewCipher(bytes.Repeat([]byte("b"), keyLength))
+	if err != nil {
+		b.Fatal(err)
+	}
+	p := value.NewPrefixTransformers(nil,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewCBCTransformer(block1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewCBCTransformer(block2)},
+	)
+
+	context := value.DefaultContext([]byte("authenticated_data"))
+	v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16)
+
+	out, err := p.TransformToStorage(v, context)
+	if err != nil {
+		b.Fatal(err)
+	}
+	// reverse the key order if expecting stale
+	if expectStale {
+		p = value.NewPrefixTransformers(nil,
+			value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewCBCTransformer(block2)},
+			value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewCBCTransformer(block1)},
+		)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		from, stale, err := p.TransformFromStorage(out, context)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if expectStale != stale {
+			b.Fatalf("unexpected data: %q, expect stale %t but got %t", from, expectStale, stale)
+		}
+	}
+	b.StopTimer()
+}
+
+func benchmarkCBCWrite(b *testing.B, keyLength int, valueLength int) {
+	block1, err := aes.NewCipher(bytes.Repeat([]byte("a"), keyLength))
+	if err != nil {
+		b.Fatal(err)
+	}
+	block2, err := aes.NewCipher(bytes.Repeat([]byte("b"), keyLength))
+	if err != nil {
+		b.Fatal(err)
+	}
+	p := value.NewPrefixTransformers(nil,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewCBCTransformer(block1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewCBCTransformer(block2)},
+	)
+
+	context := value.DefaultContext([]byte("authenticated_data"))
+	v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := p.TransformToStorage(v, context)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.StopTimer()
+}
+
+func TestRoundTrip(t *testing.T) {
+	lengths := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 128, 1024}
+
+	aes16block, err := aes.NewCipher([]byte(bytes.Repeat([]byte("a"), 16)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	aes24block, err := aes.NewCipher([]byte(bytes.Repeat([]byte("b"), 24)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	aes32block, err := aes.NewCipher([]byte(bytes.Repeat([]byte("c"), 32)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name    string
+		context value.Context
+		t       value.Transformer
+	}{
+		{name: "GCM 16 byte key", t: NewGCMTransformer(aes16block)},
+		{name: "GCM 24 byte key", t: NewGCMTransformer(aes24block)},
+		{name: "GCM 32 byte key", t: NewGCMTransformer(aes32block)},
+		{name: "CBC 32 byte key", t: NewCBCTransformer(aes32block)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			context := tt.context
+			if context == nil {
+				context = value.DefaultContext("")
+			}
+			for _, l := range lengths {
+				data := make([]byte, l)
+				if _, err := io.ReadFull(rand.Reader, data); err != nil {
+					t.Fatalf("unable to read sufficient random bytes: %v", err)
+				}
+				original := append([]byte{}, data...)
+
+				ciphertext, err := tt.t.TransformToStorage(data, context)
+				if err != nil {
+					t.Errorf("TransformToStorage error = %v", err)
+					continue
+				}
+
+				result, stale, err := tt.t.TransformFromStorage(ciphertext, context)
+				if err != nil {
+					t.Errorf("TransformFromStorage error = %v", err)
+					continue
+				}
+				if stale {
+					t.Errorf("unexpected stale output")
+					continue
+				}
+
+				switch {
+				case l == 0:
+					if len(result) != 0 {
+						t.Errorf("Round trip failed len=%d\noriginal:\n%s\nresult:\n%s", l, hex.Dump(original), hex.Dump(result))
+					}
+				case !reflect.DeepEqual(original, result):
+					t.Errorf("Round trip failed len=%d\noriginal:\n%s\nresult:\n%s", l, hex.Dump(original), hex.Dump(result))
+				}
+			}
+		})
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope_test.go

@@ -0,0 +1,203 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package envelope
+
+import (
+	"bytes"
+	"crypto/aes"
+	"encoding/base64"
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+
+	"k8s.io/apiserver/pkg/storage/value"
+	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
+)
+
+const (
+	testText              = "abcdefghijklmnopqrstuvwxyz"
+	testContextText       = "0123456789"
+	testEnvelopeCacheSize = 10
+)
+
+// testEnvelopeService is a mock Envelope service which can be used to simulate remote Envelope services
+// for testing of Envelope based encryption providers.
+type testEnvelopeService struct {
+	disabled   bool
+	keyVersion string
+}
+
+func (t *testEnvelopeService) Decrypt(data string) ([]byte, error) {
+	if t.disabled {
+		return nil, fmt.Errorf("Envelope service was disabled")
+	}
+	dataChunks := strings.SplitN(data, ":", 2)
+	if len(dataChunks) != 2 {
+		return nil, fmt.Errorf("invalid data encountered for decryption: %s. Missing key version", data)
+	}
+	return base64.StdEncoding.DecodeString(dataChunks[1])
+}
+
+func (t *testEnvelopeService) Encrypt(data []byte) (string, error) {
+	if t.disabled {
+		return "", fmt.Errorf("Envelope service was disabled")
+	}
+	return t.keyVersion + ":" + base64.StdEncoding.EncodeToString(data), nil
+}
+
+func (t *testEnvelopeService) SetDisabledStatus(status bool) {
+	t.disabled = status
+}
+
+func (t *testEnvelopeService) Rotate() {
+	i, _ := strconv.Atoi(t.keyVersion)
+	t.keyVersion = strconv.FormatInt(int64(i+1), 10)
+}
+
+func newTestEnvelopeService() *testEnvelopeService {
+	return &testEnvelopeService{
+		keyVersion: "1",
+	}
+}
+
+// Throw error if Envelope transformer tries to contact Envelope without hitting cache.
+func TestEnvelopeCaching(t *testing.T) {
+	envelopeService := newTestEnvelopeService()
+	envelopeTransformer, err := NewEnvelopeTransformer(envelopeService, testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
+	if err != nil {
+		t.Fatalf("failed to initialize envelope transformer: %v", err)
+	}
+	context := value.DefaultContext([]byte(testContextText))
+	originalText := []byte(testText)
+
+	transformedData, err := envelopeTransformer.TransformToStorage(originalText, context)
+	if err != nil {
+		t.Fatalf("envelopeTransformer: error while transforming data to storage: %s", err)
+	}
+	untransformedData, _, err := envelopeTransformer.TransformFromStorage(transformedData, context)
+	if err != nil {
+		t.Fatalf("could not decrypt Envelope transformer's encrypted data even once: %v", err)
+	}
+	if bytes.Compare(untransformedData, originalText) != 0 {
+		t.Fatalf("envelopeTransformer transformed data incorrectly. Expected: %v, got %v", originalText, untransformedData)
+	}
+
+	envelopeService.SetDisabledStatus(true)
+	// Subsequent read for the same data should work fine due to caching.
+	untransformedData, _, err = envelopeTransformer.TransformFromStorage(transformedData, context)
+	if err != nil {
+		t.Fatalf("could not decrypt Envelope transformer's encrypted data using just cache: %v", err)
+	}
+	if bytes.Compare(untransformedData, originalText) != 0 {
+		t.Fatalf("envelopeTransformer transformed data incorrectly using cache. Expected: %v, got %v", originalText, untransformedData)
+	}
+}
+
+// Makes Envelope transformer hit cache limit, throws error if it misbehaves.
+func TestEnvelopeCacheLimit(t *testing.T) {
+	envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
+	if err != nil {
+		t.Fatalf("failed to initialize envelope transformer: %v", err)
+	}
+	context := value.DefaultContext([]byte(testContextText))
+
+	transformedOutputs := map[int][]byte{}
+
+	// Overwrite lots of entries in the map
+	for i := 0; i < 2*testEnvelopeCacheSize; i++ {
+		numberText := []byte(strconv.Itoa(i))
+
+		res, err := envelopeTransformer.TransformToStorage(numberText, context)
+		transformedOutputs[i] = res
+		if err != nil {
+			t.Fatalf("envelopeTransformer: error while transforming data (%v) to storage: %s", numberText, err)
+		}
+	}
+
+	// Try reading all the data now, ensuring cache misses don't cause a concern.
+	for i := 0; i < 2*testEnvelopeCacheSize; i++ {
+		numberText := []byte(strconv.Itoa(i))
+
+		output, _, err := envelopeTransformer.TransformFromStorage(transformedOutputs[i], context)
+		if err != nil {
+			t.Fatalf("envelopeTransformer: error while transforming data (%v) from storage: %s", transformedOutputs[i], err)
+		}
+
+		if bytes.Compare(numberText, output) != 0 {
+			t.Fatalf("envelopeTransformer transformed data incorrectly using cache. Expected: %v, got %v", numberText, output)
+		}
+	}
+}
+
+func BenchmarkEnvelopeCBCRead(b *testing.B) {
+	envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewCBCTransformer)
+	if err != nil {
+		b.Fatalf("failed to initialize envelope transformer: %v", err)
+	}
+	benchmarkRead(b, envelopeTransformer, 1024)
+}
+
+func BenchmarkAESCBCRead(b *testing.B) {
+	block, err := aes.NewCipher(bytes.Repeat([]byte("a"), 32))
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	aesCBCTransformer := aestransformer.NewCBCTransformer(block)
+	benchmarkRead(b, aesCBCTransformer, 1024)
+}
+
+func BenchmarkEnvelopeGCMRead(b *testing.B) {
+	envelopeTransformer, err := NewEnvelopeTransformer(newTestEnvelopeService(), testEnvelopeCacheSize, aestransformer.NewGCMTransformer)
+	if err != nil {
+		b.Fatalf("failed to initialize envelope transformer: %v", err)
+	}
+	benchmarkRead(b, envelopeTransformer, 1024)
+}
+
+func BenchmarkAESGCMRead(b *testing.B) {
+	block, err := aes.NewCipher(bytes.Repeat([]byte("a"), 32))
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	aesGCMTransformer := aestransformer.NewGCMTransformer(block)
+	benchmarkRead(b, aesGCMTransformer, 1024)
+}
+
+func benchmarkRead(b *testing.B, transformer value.Transformer, valueLength int) {
+	context := value.DefaultContext([]byte(testContextText))
+	v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16)
+
+	out, err := transformer.TransformToStorage(v, context)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		from, stale, err := transformer.TransformFromStorage(out, context)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if stale {
+			b.Fatalf("unexpected data: %t %q", stale, from)
+		}
+	}
+	b.StopTimer()
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/secretbox/secretbox_test.go

@@ -0,0 +1,190 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package secretbox
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"reflect"
+	"testing"
+
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+var (
+	key1 = [32]byte{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}
+	key2 = [32]byte{0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02}
+)
+
+func TestSecretboxKeyRotation(t *testing.T) {
+	testErr := fmt.Errorf("test error")
+	context := value.DefaultContext([]byte("authenticated_data"))
+
+	p := value.NewPrefixTransformers(testErr,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewSecretboxTransformer(key1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewSecretboxTransformer(key2)},
+	)
+	out, err := p.TransformToStorage([]byte("firstvalue"), context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.HasPrefix(out, []byte("first:")) {
+		t.Fatalf("unexpected prefix: %q", out)
+	}
+	from, stale, err := p.TransformFromStorage(out, context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stale || !bytes.Equal([]byte("firstvalue"), from) {
+		t.Fatalf("unexpected data: %t %q", stale, from)
+	}
+
+	// verify changing the context does not fails storage
+	// Secretbox is not currently an authenticating store
+	from, stale, err = p.TransformFromStorage(out, value.DefaultContext([]byte("incorrect_context")))
+	if err != nil {
+		t.Fatalf("secretbox is not authenticated")
+	}
+
+	// reverse the order, use the second key
+	p = value.NewPrefixTransformers(testErr,
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewSecretboxTransformer(key2)},
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewSecretboxTransformer(key1)},
+	)
+	from, stale, err = p.TransformFromStorage(out, context)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !stale || !bytes.Equal([]byte("firstvalue"), from) {
+		t.Fatalf("unexpected data: %t %q", stale, from)
+	}
+}
+
+func BenchmarkSecretboxRead_32_1024(b *testing.B)        { benchmarkSecretboxRead(b, 32, 1024, false) }
+func BenchmarkSecretboxRead_32_16384(b *testing.B)       { benchmarkSecretboxRead(b, 32, 16384, false) }
+func BenchmarkSecretboxRead_32_16384_Stale(b *testing.B) { benchmarkSecretboxRead(b, 32, 16384, true) }
+
+func BenchmarkSecretboxWrite_32_1024(b *testing.B)  { benchmarkSecretboxWrite(b, 32, 1024) }
+func BenchmarkSecretboxWrite_32_16384(b *testing.B) { benchmarkSecretboxWrite(b, 32, 16384) }
+
+func benchmarkSecretboxRead(b *testing.B, keyLength int, valueLength int, expectStale bool) {
+	p := value.NewPrefixTransformers(nil,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewSecretboxTransformer(key1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewSecretboxTransformer(key2)},
+	)
+
+	context := value.DefaultContext([]byte("authenticated_data"))
+	v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16)
+
+	out, err := p.TransformToStorage(v, context)
+	if err != nil {
+		b.Fatal(err)
+	}
+	// reverse the key order if expecting stale
+	if expectStale {
+		p = value.NewPrefixTransformers(nil,
+			value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewSecretboxTransformer(key2)},
+			value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewSecretboxTransformer(key1)},
+		)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		from, stale, err := p.TransformFromStorage(out, context)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if expectStale != stale {
+			b.Fatalf("unexpected data: %q, expect stale %t but got %t", from, expectStale, stale)
+		}
+	}
+	b.StopTimer()
+}
+
+func benchmarkSecretboxWrite(b *testing.B, keyLength int, valueLength int) {
+	p := value.NewPrefixTransformers(nil,
+		value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewSecretboxTransformer(key1)},
+		value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewSecretboxTransformer(key2)},
+	)
+
+	context := value.DefaultContext([]byte("authenticated_data"))
+	v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := p.TransformToStorage(v, context)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.StopTimer()
+}
+
+func TestRoundTrip(t *testing.T) {
+	lengths := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 128, 1024}
+
+	tests := []struct {
+		name    string
+		context value.Context
+		t       value.Transformer
+	}{
+		{name: "Secretbox 32 byte key", t: NewSecretboxTransformer(key1)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			context := tt.context
+			if context == nil {
+				context = value.DefaultContext("")
+			}
+			for _, l := range lengths {
+				data := make([]byte, l)
+				if _, err := io.ReadFull(rand.Reader, data); err != nil {
+					t.Fatalf("unable to read sufficient random bytes: %v", err)
+				}
+				original := append([]byte{}, data...)
+
+				ciphertext, err := tt.t.TransformToStorage(data, context)
+				if err != nil {
+					t.Errorf("TransformToStorage error = %v", err)
+					continue
+				}
+
+				result, stale, err := tt.t.TransformFromStorage(ciphertext, context)
+				if err != nil {
+					t.Errorf("TransformFromStorage error = %v", err)
+					continue
+				}
+				if stale {
+					t.Errorf("unexpected stale output")
+					continue
+				}
+
+				switch {
+				case l == 0:
+					if len(result) != 0 {
+						t.Errorf("Round trip failed len=%d\noriginal:\n%s\nresult:\n%s", l, hex.Dump(original), hex.Dump(result))
+					}
+				case !reflect.DeepEqual(original, result):
+					t.Errorf("Round trip failed len=%d\noriginal:\n%s\nresult:\n%s", l, hex.Dump(original), hex.Dump(result))
+				}
+			}
+		})
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/value/transformer_test.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package value
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+)
+
+type testTransformer struct {
+	from, to                 []byte
+	err                      error
+	stale                    bool
+	receivedFrom, receivedTo []byte
+}
+
+func (t *testTransformer) TransformFromStorage(from []byte, context Context) (data []byte, stale bool, err error) {
+	t.receivedFrom = from
+	return t.from, t.stale, t.err
+}
+
+func (t *testTransformer) TransformToStorage(to []byte, context Context) (data []byte, err error) {
+	t.receivedTo = to
+	return t.to, t.err
+}
+
+func TestPrefixFrom(t *testing.T) {
+	testErr := fmt.Errorf("test error")
+	transformErr := fmt.Errorf("test error")
+	transformers := []PrefixTransformer{
+		{Prefix: []byte("first:"), Transformer: &testTransformer{from: []byte("value1")}},
+		{Prefix: []byte("second:"), Transformer: &testTransformer{from: []byte("value2")}},
+		{Prefix: []byte("fails:"), Transformer: &testTransformer{err: transformErr}},
+		{Prefix: []byte("stale:"), Transformer: &testTransformer{from: []byte("value3"), stale: true}},
+	}
+	p := NewPrefixTransformers(testErr, transformers...)
+
+	testCases := []struct {
+		input  []byte
+		expect []byte
+		stale  bool
+		err    error
+		match  int
+	}{
+		{[]byte("first:value"), []byte("value1"), false, nil, 0},
+		{[]byte("second:value"), []byte("value2"), true, nil, 1},
+		{[]byte("third:value"), nil, false, testErr, -1},
+		{[]byte("fails:value"), nil, true, transformErr, 2},
+		{[]byte("stale:value"), []byte("value3"), true, nil, 3},
+	}
+	for i, test := range testCases {
+		got, stale, err := p.TransformFromStorage(test.input, nil)
+		if err != test.err || stale != test.stale || !bytes.Equal(got, test.expect) {
+			t.Errorf("%d: unexpected out: %q %t %#v", i, string(got), stale, err)
+			continue
+		}
+		if test.match != -1 && !bytes.Equal([]byte("value"), transformers[test.match].Transformer.(*testTransformer).receivedFrom) {
+			t.Errorf("%d: unexpected value received by transformer: %s", i, transformers[test.match].Transformer.(*testTransformer).receivedFrom)
+		}
+	}
+}
+
+func TestPrefixTo(t *testing.T) {
+	testErr := fmt.Errorf("test error")
+	transformErr := fmt.Errorf("test error")
+	testCases := []struct {
+		transformers []PrefixTransformer
+		expect       []byte
+		err          error
+	}{
+		{[]PrefixTransformer{{Prefix: []byte("first:"), Transformer: &testTransformer{to: []byte("value1")}}}, []byte("first:value1"), nil},
+		{[]PrefixTransformer{{Prefix: []byte("second:"), Transformer: &testTransformer{to: []byte("value2")}}}, []byte("second:value2"), nil},
+		{[]PrefixTransformer{{Prefix: []byte("fails:"), Transformer: &testTransformer{err: transformErr}}}, nil, transformErr},
+	}
+	for i, test := range testCases {
+		p := NewPrefixTransformers(testErr, test.transformers...)
+		got, err := p.TransformToStorage([]byte("value"), nil)
+		if err != test.err || !bytes.Equal(got, test.expect) {
+			t.Errorf("%d: unexpected out: %q %#v", i, string(got), err)
+			continue
+		}
+		if !bytes.Equal([]byte("value"), test.transformers[0].Transformer.(*testTransformer).receivedTo) {
+			t.Errorf("%d: unexpected value received by transformer: %s", i, test.transformers[0].Transformer.(*testTransformer).receivedTo)
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/watch_cache.go

@@ -0,0 +1,476 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apimachinery/pkg/watch"
+	utiltrace "k8s.io/apiserver/pkg/util/trace"
+	"k8s.io/client-go/tools/cache"
+)
+
+const (
+	// blockTimeout determines how long we're willing to block the request
+	// to wait for a given resource version to be propagated to cache,
+	// before terminating request and returning Timeout error with retry
+	// after suggestion.
+	blockTimeout = 3 * time.Second
+)
+
+// watchCacheEvent is a single "watch event" that is send to users of
+// watchCache. Additionally to a typical "watch.Event" it contains
+// the previous value of the object to enable proper filtering in the
+// upper layers.
+type watchCacheEvent struct {
+	Type                 watch.EventType
+	Object               runtime.Object
+	ObjLabels            labels.Set
+	ObjFields            fields.Set
+	ObjUninitialized     bool
+	PrevObject           runtime.Object
+	PrevObjLabels        labels.Set
+	PrevObjFields        fields.Set
+	PrevObjUninitialized bool
+	Key                  string
+	ResourceVersion      uint64
+}
+
+// Computing a key of an object is generally non-trivial (it performs
+// e.g. validation underneath). To avoid computing it multiple times
+// (to serve the event in different List/Watch requests), in the
+// underlying store we are keeping pair (key, object).
+type storeElement struct {
+	Key    string
+	Object runtime.Object
+}
+
+func storeElementKey(obj interface{}) (string, error) {
+	elem, ok := obj.(*storeElement)
+	if !ok {
+		return "", fmt.Errorf("not a storeElement: %v", obj)
+	}
+	return elem.Key, nil
+}
+
+// watchCacheElement is a single "watch event" stored in a cache.
+// It contains the resource version of the object and the object
+// itself.
+type watchCacheElement struct {
+	resourceVersion uint64
+	watchCacheEvent *watchCacheEvent
+}
+
+// watchCache implements a Store interface.
+// However, it depends on the elements implementing runtime.Object interface.
+//
+// watchCache is a "sliding window" (with a limited capacity) of objects
+// observed from a watch.
+type watchCache struct {
+	sync.RWMutex
+
+	// Condition on which lists are waiting for the fresh enough
+	// resource version.
+	cond *sync.Cond
+
+	// Maximum size of history window.
+	capacity int
+
+	// keyFunc is used to get a key in the underlying storage for a given object.
+	keyFunc func(runtime.Object) (string, error)
+
+	// getAttrsFunc is used to get labels and fields of an object.
+	getAttrsFunc func(runtime.Object) (labels.Set, fields.Set, bool, error)
+
+	// cache is used a cyclic buffer - its first element (with the smallest
+	// resourceVersion) is defined by startIndex, its last element is defined
+	// by endIndex (if cache is full it will be startIndex + capacity).
+	// Both startIndex and endIndex can be greater than buffer capacity -
+	// you should always apply modulo capacity to get an index in cache array.
+	cache      []watchCacheElement
+	startIndex int
+	endIndex   int
+
+	// store will effectively support LIST operation from the "end of cache
+	// history" i.e. from the moment just after the newest cached watched event.
+	// It is necessary to effectively allow clients to start watching at now.
+	// NOTE: We assume that <store> is thread-safe.
+	store cache.Store
+
+	// ResourceVersion up to which the watchCache is propagated.
+	resourceVersion uint64
+
+	// This handler is run at the end of every successful Replace() method.
+	onReplace func()
+
+	// This handler is run at the end of every Add/Update/Delete method
+	// and additionally gets the previous value of the object.
+	onEvent func(*watchCacheEvent)
+
+	// for testing timeouts.
+	clock clock.Clock
+}
+
+func newWatchCache(
+	capacity int,
+	keyFunc func(runtime.Object) (string, error),
+	getAttrsFunc func(runtime.Object) (labels.Set, fields.Set, bool, error)) *watchCache {
+	wc := &watchCache{
+		capacity:        capacity,
+		keyFunc:         keyFunc,
+		getAttrsFunc:    getAttrsFunc,
+		cache:           make([]watchCacheElement, capacity),
+		startIndex:      0,
+		endIndex:        0,
+		store:           cache.NewStore(storeElementKey),
+		resourceVersion: 0,
+		clock:           clock.RealClock{},
+	}
+	wc.cond = sync.NewCond(wc.RLocker())
+	return wc
+}
+
+// Add takes runtime.Object as an argument.
+func (w *watchCache) Add(obj interface{}) error {
+	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
+	if err != nil {
+		return err
+	}
+	event := watch.Event{Type: watch.Added, Object: object}
+
+	f := func(elem *storeElement) error { return w.store.Add(elem) }
+	return w.processEvent(event, resourceVersion, f)
+}
+
+// Update takes runtime.Object as an argument.
+func (w *watchCache) Update(obj interface{}) error {
+	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
+	if err != nil {
+		return err
+	}
+	event := watch.Event{Type: watch.Modified, Object: object}
+
+	f := func(elem *storeElement) error { return w.store.Update(elem) }
+	return w.processEvent(event, resourceVersion, f)
+}
+
+// Delete takes runtime.Object as an argument.
+func (w *watchCache) Delete(obj interface{}) error {
+	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
+	if err != nil {
+		return err
+	}
+	event := watch.Event{Type: watch.Deleted, Object: object}
+
+	f := func(elem *storeElement) error { return w.store.Delete(elem) }
+	return w.processEvent(event, resourceVersion, f)
+}
+
+func objectToVersionedRuntimeObject(obj interface{}) (runtime.Object, uint64, error) {
+	object, ok := obj.(runtime.Object)
+	if !ok {
+		return nil, 0, fmt.Errorf("obj does not implement runtime.Object interface: %v", obj)
+	}
+	meta, err := meta.Accessor(object)
+	if err != nil {
+		return nil, 0, err
+	}
+	resourceVersion, err := parseResourceVersion(meta.GetResourceVersion())
+	if err != nil {
+		return nil, 0, err
+	}
+	return object, resourceVersion, nil
+}
+
+func parseResourceVersion(resourceVersion string) (uint64, error) {
+	if resourceVersion == "" {
+		return 0, nil
+	}
+	// Use bitsize being the size of int on the machine.
+	return strconv.ParseUint(resourceVersion, 10, 0)
+}
+
+func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, updateFunc func(*storeElement) error) error {
+	key, err := w.keyFunc(event.Object)
+	if err != nil {
+		return fmt.Errorf("couldn't compute key: %v", err)
+	}
+	elem := &storeElement{Key: key, Object: event.Object}
+
+	// TODO: We should consider moving this lock below after the watchCacheEvent
+	// is created. In such situation, the only problematic scenario is Replace(
+	// happening after getting object from store and before acquiring a lock.
+	// Maybe introduce another lock for this purpose.
+	w.Lock()
+	defer w.Unlock()
+	previous, exists, err := w.store.Get(elem)
+	if err != nil {
+		return err
+	}
+	objLabels, objFields, objUninitialized, err := w.getAttrsFunc(event.Object)
+	if err != nil {
+		return err
+	}
+	var prevObject runtime.Object
+	var prevObjLabels labels.Set
+	var prevObjFields fields.Set
+	var prevObjUninitialized bool
+	if exists {
+		prevObject = previous.(*storeElement).Object
+		prevObjLabels, prevObjFields, prevObjUninitialized, err = w.getAttrsFunc(prevObject)
+		if err != nil {
+			return err
+		}
+	}
+	watchCacheEvent := &watchCacheEvent{
+		Type:                 event.Type,
+		Object:               event.Object,
+		ObjLabels:            objLabels,
+		ObjFields:            objFields,
+		ObjUninitialized:     objUninitialized,
+		PrevObject:           prevObject,
+		PrevObjLabels:        prevObjLabels,
+		PrevObjFields:        prevObjFields,
+		PrevObjUninitialized: prevObjUninitialized,
+		Key:                  key,
+		ResourceVersion:      resourceVersion,
+	}
+	if w.onEvent != nil {
+		w.onEvent(watchCacheEvent)
+	}
+	w.updateCache(resourceVersion, watchCacheEvent)
+	w.resourceVersion = resourceVersion
+	w.cond.Broadcast()
+	return updateFunc(elem)
+}
+
+// Assumes that lock is already held for write.
+func (w *watchCache) updateCache(resourceVersion uint64, event *watchCacheEvent) {
+	if w.endIndex == w.startIndex+w.capacity {
+		// Cache is full - remove the oldest element.
+		w.startIndex++
+	}
+	w.cache[w.endIndex%w.capacity] = watchCacheElement{resourceVersion, event}
+	w.endIndex++
+}
+
+// List returns list of pointers to <storeElement> objects.
+func (w *watchCache) List() []interface{} {
+	return w.store.List()
+}
+
+// waitUntilFreshAndBlock waits until cache is at least as fresh as given <resourceVersion>.
+// NOTE: This function acquired lock and doesn't release it.
+// You HAVE TO explicitly call w.RUnlock() after this function.
+func (w *watchCache) waitUntilFreshAndBlock(resourceVersion uint64, trace *utiltrace.Trace) error {
+	startTime := w.clock.Now()
+	go func() {
+		// Wake us up when the time limit has expired.  The docs
+		// promise that time.After (well, NewTimer, which it calls)
+		// will wait *at least* the duration given. Since this go
+		// routine starts sometime after we record the start time, and
+		// it will wake up the loop below sometime after the broadcast,
+		// we don't need to worry about waking it up before the time
+		// has expired accidentally.
+		<-w.clock.After(blockTimeout)
+		w.cond.Broadcast()
+	}()
+
+	w.RLock()
+	if trace != nil {
+		trace.Step("watchCache locked acquired")
+	}
+	for w.resourceVersion < resourceVersion {
+		if w.clock.Since(startTime) >= blockTimeout {
+			// Timeout with retry after 1 second.
+			return errors.NewTimeoutError(fmt.Sprintf("Too large resource version: %v, current: %v", resourceVersion, w.resourceVersion), 1)
+		}
+		w.cond.Wait()
+	}
+	if trace != nil {
+		trace.Step("watchCache fresh enough")
+	}
+	return nil
+}
+
+// WaitUntilFreshAndList returns list of pointers to <storeElement> objects.
+func (w *watchCache) WaitUntilFreshAndList(resourceVersion uint64, trace *utiltrace.Trace) ([]interface{}, uint64, error) {
+	err := w.waitUntilFreshAndBlock(resourceVersion, trace)
+	defer w.RUnlock()
+	if err != nil {
+		return nil, 0, err
+	}
+	return w.store.List(), w.resourceVersion, nil
+}
+
+// WaitUntilFreshAndGet returns a pointers to <storeElement> object.
+func (w *watchCache) WaitUntilFreshAndGet(resourceVersion uint64, key string, trace *utiltrace.Trace) (interface{}, bool, uint64, error) {
+	err := w.waitUntilFreshAndBlock(resourceVersion, trace)
+	defer w.RUnlock()
+	if err != nil {
+		return nil, false, 0, err
+	}
+	value, exists, err := w.store.GetByKey(key)
+	return value, exists, w.resourceVersion, err
+}
+
+func (w *watchCache) ListKeys() []string {
+	return w.store.ListKeys()
+}
+
+// Get takes runtime.Object as a parameter. However, it returns
+// pointer to <storeElement>.
+func (w *watchCache) Get(obj interface{}) (interface{}, bool, error) {
+	object, ok := obj.(runtime.Object)
+	if !ok {
+		return nil, false, fmt.Errorf("obj does not implement runtime.Object interface: %v", obj)
+	}
+	key, err := w.keyFunc(object)
+	if err != nil {
+		return nil, false, fmt.Errorf("couldn't compute key: %v", err)
+	}
+
+	return w.store.Get(&storeElement{Key: key, Object: object})
+}
+
+// GetByKey returns pointer to <storeElement>.
+func (w *watchCache) GetByKey(key string) (interface{}, bool, error) {
+	return w.store.GetByKey(key)
+}
+
+// Replace takes slice of runtime.Object as a paramater.
+func (w *watchCache) Replace(objs []interface{}, resourceVersion string) error {
+	version, err := parseResourceVersion(resourceVersion)
+	if err != nil {
+		return err
+	}
+
+	toReplace := make([]interface{}, 0, len(objs))
+	for _, obj := range objs {
+		object, ok := obj.(runtime.Object)
+		if !ok {
+			return fmt.Errorf("didn't get runtime.Object for replace: %#v", obj)
+		}
+		key, err := w.keyFunc(object)
+		if err != nil {
+			return fmt.Errorf("couldn't compute key: %v", err)
+		}
+		toReplace = append(toReplace, &storeElement{Key: key, Object: object})
+	}
+
+	w.Lock()
+	defer w.Unlock()
+
+	w.startIndex = 0
+	w.endIndex = 0
+	if err := w.store.Replace(toReplace, resourceVersion); err != nil {
+		return err
+	}
+	w.resourceVersion = version
+	if w.onReplace != nil {
+		w.onReplace()
+	}
+	w.cond.Broadcast()
+	return nil
+}
+
+func (w *watchCache) SetOnReplace(onReplace func()) {
+	w.Lock()
+	defer w.Unlock()
+	w.onReplace = onReplace
+}
+
+func (w *watchCache) SetOnEvent(onEvent func(*watchCacheEvent)) {
+	w.Lock()
+	defer w.Unlock()
+	w.onEvent = onEvent
+}
+
+func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]*watchCacheEvent, error) {
+	size := w.endIndex - w.startIndex
+	// if we have no watch events in our cache, the oldest one we can successfully deliver to a watcher
+	// is the *next* event we'll receive, which will be at least one greater than our current resourceVersion
+	oldest := w.resourceVersion + 1
+	if size > 0 {
+		oldest = w.cache[w.startIndex%w.capacity].resourceVersion
+	}
+	if resourceVersion == 0 {
+		// resourceVersion = 0 means that we don't require any specific starting point
+		// and we would like to start watching from ~now.
+		// However, to keep backward compatibility, we additionally need to return the
+		// current state and only then start watching from that point.
+		//
+		// TODO: In v2 api, we should stop returning the current state - #13969.
+		allItems := w.store.List()
+		result := make([]*watchCacheEvent, len(allItems))
+		for i, item := range allItems {
+			elem, ok := item.(*storeElement)
+			if !ok {
+				return nil, fmt.Errorf("not a storeElement: %v", elem)
+			}
+			objLabels, objFields, objUninitialized, err := w.getAttrsFunc(elem.Object)
+			if err != nil {
+				return nil, err
+			}
+			result[i] = &watchCacheEvent{
+				Type:             watch.Added,
+				Object:           elem.Object,
+				ObjLabels:        objLabels,
+				ObjFields:        objFields,
+				ObjUninitialized: objUninitialized,
+				Key:              elem.Key,
+				ResourceVersion:  w.resourceVersion,
+			}
+		}
+		return result, nil
+	}
+	if resourceVersion < oldest-1 {
+		return nil, errors.NewGone(fmt.Sprintf("too old resource version: %d (%d)", resourceVersion, oldest-1))
+	}
+
+	// Binary search the smallest index at which resourceVersion is greater than the given one.
+	f := func(i int) bool {
+		return w.cache[(w.startIndex+i)%w.capacity].resourceVersion > resourceVersion
+	}
+	first := sort.Search(size, f)
+	result := make([]*watchCacheEvent, size-first)
+	for i := 0; i < size-first; i++ {
+		result[i] = w.cache[(w.startIndex+first+i)%w.capacity].watchCacheEvent
+	}
+	return result, nil
+}
+
+func (w *watchCache) GetAllEventsSince(resourceVersion uint64) ([]*watchCacheEvent, error) {
+	w.RLock()
+	defer w.RUnlock()
+	return w.GetAllEventsSinceThreadUnsafe(resourceVersion)
+}
+
+func (w *watchCache) Resync() error {
+	// Nothing to do
+	return nil
+}

vendor/k8s.io/apiserver/pkg/storage/watch_cache_test.go

@@ -0,0 +1,367 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+)
+
+func makeTestPod(name string, resourceVersion uint64) *v1.Pod {
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:       "ns",
+			Name:            name,
+			ResourceVersion: strconv.FormatUint(resourceVersion, 10),
+		},
+	}
+}
+
+// newTestWatchCache just adds a fake clock.
+func newTestWatchCache(capacity int) *watchCache {
+	keyFunc := func(obj runtime.Object) (string, error) {
+		return NamespaceKeyFunc("prefix", obj)
+	}
+	getAttrsFunc := func(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+		return nil, nil, false, nil
+	}
+	wc := newWatchCache(capacity, keyFunc, getAttrsFunc)
+	wc.clock = clock.NewFakeClock(time.Now())
+	return wc
+}
+
+func TestWatchCacheBasic(t *testing.T) {
+	store := newTestWatchCache(2)
+
+	// Test Add/Update/Delete.
+	pod1 := makeTestPod("pod", 1)
+	if err := store.Add(pod1); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if item, ok, _ := store.Get(pod1); !ok {
+		t.Errorf("didn't find pod")
+	} else {
+		if !apiequality.Semantic.DeepEqual(&storeElement{Key: "prefix/ns/pod", Object: pod1}, item) {
+			t.Errorf("expected %v, got %v", pod1, item)
+		}
+	}
+	pod2 := makeTestPod("pod", 2)
+	if err := store.Update(pod2); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if item, ok, _ := store.Get(pod2); !ok {
+		t.Errorf("didn't find pod")
+	} else {
+		if !apiequality.Semantic.DeepEqual(&storeElement{Key: "prefix/ns/pod", Object: pod2}, item) {
+			t.Errorf("expected %v, got %v", pod1, item)
+		}
+	}
+	pod3 := makeTestPod("pod", 3)
+	if err := store.Delete(pod3); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if _, ok, _ := store.Get(pod3); ok {
+		t.Errorf("found pod")
+	}
+
+	// Test List.
+	store.Add(makeTestPod("pod1", 4))
+	store.Add(makeTestPod("pod2", 5))
+	store.Add(makeTestPod("pod3", 6))
+	{
+		podNames := sets.String{}
+		for _, item := range store.List() {
+			podNames.Insert(item.(*storeElement).Object.(*v1.Pod).ObjectMeta.Name)
+		}
+		if !podNames.HasAll("pod1", "pod2", "pod3") {
+			t.Errorf("missing pods, found %v", podNames)
+		}
+		if len(podNames) != 3 {
+			t.Errorf("found missing/extra items")
+		}
+	}
+
+	// Test Replace.
+	store.Replace([]interface{}{
+		makeTestPod("pod4", 7),
+		makeTestPod("pod5", 8),
+	}, "8")
+	{
+		podNames := sets.String{}
+		for _, item := range store.List() {
+			podNames.Insert(item.(*storeElement).Object.(*v1.Pod).ObjectMeta.Name)
+		}
+		if !podNames.HasAll("pod4", "pod5") {
+			t.Errorf("missing pods, found %v", podNames)
+		}
+		if len(podNames) != 2 {
+			t.Errorf("found missing/extra items")
+		}
+	}
+}
+
+func TestEvents(t *testing.T) {
+	store := newTestWatchCache(5)
+
+	store.Add(makeTestPod("pod", 3))
+
+	// Test for Added event.
+	{
+		_, err := store.GetAllEventsSince(1)
+		if err == nil {
+			t.Errorf("expected error too old")
+		}
+		if _, ok := err.(*errors.StatusError); !ok {
+			t.Errorf("expected error to be of type StatusError")
+		}
+	}
+	{
+		result, err := store.GetAllEventsSince(2)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if len(result) != 1 {
+			t.Fatalf("unexpected events: %v", result)
+		}
+		if result[0].Type != watch.Added {
+			t.Errorf("unexpected event type: %v", result[0].Type)
+		}
+		pod := makeTestPod("pod", uint64(3))
+		if !apiequality.Semantic.DeepEqual(pod, result[0].Object) {
+			t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod)
+		}
+		if result[0].PrevObject != nil {
+			t.Errorf("unexpected item: %v", result[0].PrevObject)
+		}
+	}
+
+	store.Update(makeTestPod("pod", 4))
+	store.Update(makeTestPod("pod", 5))
+
+	// Test with not full cache.
+	{
+		_, err := store.GetAllEventsSince(1)
+		if err == nil {
+			t.Errorf("expected error too old")
+		}
+	}
+	{
+		result, err := store.GetAllEventsSince(3)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if len(result) != 2 {
+			t.Fatalf("unexpected events: %v", result)
+		}
+		for i := 0; i < 2; i++ {
+			if result[i].Type != watch.Modified {
+				t.Errorf("unexpected event type: %v", result[i].Type)
+			}
+			pod := makeTestPod("pod", uint64(i+4))
+			if !apiequality.Semantic.DeepEqual(pod, result[i].Object) {
+				t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod)
+			}
+			prevPod := makeTestPod("pod", uint64(i+3))
+			if !apiequality.Semantic.DeepEqual(prevPod, result[i].PrevObject) {
+				t.Errorf("unexpected item: %v, expected: %v", result[i].PrevObject, prevPod)
+			}
+		}
+	}
+
+	for i := 6; i < 10; i++ {
+		store.Update(makeTestPod("pod", uint64(i)))
+	}
+
+	// Test with full cache - there should be elements from 5 to 9.
+	{
+		_, err := store.GetAllEventsSince(3)
+		if err == nil {
+			t.Errorf("expected error too old")
+		}
+	}
+	{
+		result, err := store.GetAllEventsSince(4)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if len(result) != 5 {
+			t.Fatalf("unexpected events: %v", result)
+		}
+		for i := 0; i < 5; i++ {
+			pod := makeTestPod("pod", uint64(i+5))
+			if !apiequality.Semantic.DeepEqual(pod, result[i].Object) {
+				t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod)
+			}
+		}
+	}
+
+	// Test for delete event.
+	store.Delete(makeTestPod("pod", uint64(10)))
+
+	{
+		result, err := store.GetAllEventsSince(9)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if len(result) != 1 {
+			t.Fatalf("unexpected events: %v", result)
+		}
+		if result[0].Type != watch.Deleted {
+			t.Errorf("unexpected event type: %v", result[0].Type)
+		}
+		pod := makeTestPod("pod", uint64(10))
+		if !apiequality.Semantic.DeepEqual(pod, result[0].Object) {
+			t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod)
+		}
+		prevPod := makeTestPod("pod", uint64(9))
+		if !apiequality.Semantic.DeepEqual(prevPod, result[0].PrevObject) {
+			t.Errorf("unexpected item: %v, expected: %v", result[0].PrevObject, prevPod)
+		}
+	}
+}
+
+func TestWaitUntilFreshAndList(t *testing.T) {
+	store := newTestWatchCache(3)
+
+	// In background, update the store.
+	go func() {
+		store.Add(makeTestPod("foo", 2))
+		store.Add(makeTestPod("bar", 5))
+	}()
+
+	list, resourceVersion, err := store.WaitUntilFreshAndList(5, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if resourceVersion != 5 {
+		t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion)
+	}
+	if len(list) != 2 {
+		t.Errorf("unexpected list returned: %#v", list)
+	}
+}
+
+func TestWaitUntilFreshAndGet(t *testing.T) {
+	store := newTestWatchCache(3)
+
+	// In background, update the store.
+	go func() {
+		store.Add(makeTestPod("foo", 2))
+		store.Add(makeTestPod("bar", 5))
+	}()
+
+	obj, exists, resourceVersion, err := store.WaitUntilFreshAndGet(5, "prefix/ns/bar", nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if resourceVersion != 5 {
+		t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion)
+	}
+	if !exists {
+		t.Fatalf("no results returned: %#v", obj)
+	}
+	if !apiequality.Semantic.DeepEqual(&storeElement{Key: "prefix/ns/bar", Object: makeTestPod("bar", 5)}, obj) {
+		t.Errorf("unexpected element returned: %#v", obj)
+	}
+}
+
+func TestWaitUntilFreshAndListTimeout(t *testing.T) {
+	store := newTestWatchCache(3)
+	fc := store.clock.(*clock.FakeClock)
+
+	// In background, step clock after the below call starts the timer.
+	go func() {
+		for !fc.HasWaiters() {
+			time.Sleep(time.Millisecond)
+		}
+		fc.Step(blockTimeout)
+
+		// Add an object to make sure the test would
+		// eventually fail instead of just waiting
+		// forever.
+		time.Sleep(30 * time.Second)
+		store.Add(makeTestPod("bar", 5))
+	}()
+
+	_, _, err := store.WaitUntilFreshAndList(5, nil)
+	if err == nil {
+		t.Fatalf("unexpected lack of timeout error")
+	}
+}
+
+type testLW struct {
+	ListFunc  func(options metav1.ListOptions) (runtime.Object, error)
+	WatchFunc func(options metav1.ListOptions) (watch.Interface, error)
+}
+
+func (t *testLW) List(options metav1.ListOptions) (runtime.Object, error) {
+	return t.ListFunc(options)
+}
+func (t *testLW) Watch(options metav1.ListOptions) (watch.Interface, error) {
+	return t.WatchFunc(options)
+}
+
+func TestReflectorForWatchCache(t *testing.T) {
+	store := newTestWatchCache(5)
+
+	{
+		_, version, err := store.WaitUntilFreshAndList(0, nil)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if version != 0 {
+			t.Errorf("unexpected resource version: %d", version)
+		}
+	}
+
+	lw := &testLW{
+		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+			fw := watch.NewFake()
+			go fw.Stop()
+			return fw, nil
+		},
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}}, nil
+		},
+	}
+	r := cache.NewReflector(lw, &v1.Pod{}, store, 0)
+	r.ListAndWatch(wait.NeverStop)
+
+	{
+		_, version, err := store.WaitUntilFreshAndList(10, nil)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if version != 10 {
+			t.Errorf("unexpected resource version: %d", version)
+		}
+	}
+}