Rate limit requeues on error

controllers/gce/controller/utils.go

@@ -85,9 +85,9 @@ func (e errorNodePortNotFound) Error() string {
 // invokes the given sync function for every work item inserted.
 type taskQueue struct {
 	// queue is the work queue the worker polls
-	queue *workqueue.Type
+	queue workqueue.RateLimitingInterface
 	// sync is called for each item in the queue
-	sync func(string)
+	sync func(string) error
 	// workerDone is closed when the worker exits
 	workerDone chan struct{}
 }
@@ -106,11 +106,6 @@ func (t *taskQueue) enqueue(obj interface{}) {
 	t.queue.Add(key)
 }
 
-func (t *taskQueue) requeue(key string, err error) {
-	glog.Errorf("Requeuing %v, err %v", key, err)
-	t.queue.Add(key)
-}
-
 // worker processes work in the queue through sync.
 func (t *taskQueue) worker() {
 	for {
@@ -120,7 +115,12 @@ func (t *taskQueue) worker() {
 			return
 		}
 		glog.V(3).Infof("Syncing %v", key)
-		t.sync(key.(string))
+		if err := t.sync(key.(string)); err != nil {
+			glog.Errorf("Requeuing %v, err %v", key, err)
+			t.queue.AddRateLimited(key)
+		} else {
+			t.queue.Forget(key)
+		}
 		t.queue.Done(key)
 	}
 }
@@ -133,9 +133,9 @@ func (t *taskQueue) shutdown() {
 
 // NewTaskQueue creates a new task queue with the given sync function.
 // The sync function is called for every element inserted into the queue.
-func NewTaskQueue(syncFn func(string)) *taskQueue {
+func NewTaskQueue(syncFn func(string) error) *taskQueue {
 	return &taskQueue{
-		queue:      workqueue.New(),
+		queue:      workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
 		sync:       syncFn,
 		workerDone: make(chan struct{}),
 	}