Replace godep with dep

vendor/k8s.io/kubernetes/test/e2e/instrumentation/monitoring/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cadvisor.go",
+        "influxdb.go",
+        "metrics_grabber.go",
+        "stackdriver.go",
+    ],
+    deps = [
+        "//test/e2e/common:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/framework/metrics:go_default_library",
+        "//test/e2e/instrumentation/common:go_default_library",
+        "//vendor/github.com/influxdata/influxdb/client/v2:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/golang.org/x/oauth2/google:go_default_library",
+        "//vendor/google.golang.org/api/monitoring/v3:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e/instrumentation/monitoring/cadvisor.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitoring
+
+import (
+	"fmt"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = instrumentation.SIGDescribe("Cadvisor", func() {
+
+	f := framework.NewDefaultFramework("cadvisor")
+
+	It("should be healthy on every node.", func() {
+		CheckCadvisorHealthOnAllNodes(f.ClientSet, 5*time.Minute)
+	})
+})
+
+func CheckCadvisorHealthOnAllNodes(c clientset.Interface, timeout time.Duration) {
+	// It should be OK to list unschedulable Nodes here.
+	By("getting list of nodes")
+	nodeList, err := c.Core().Nodes().List(metav1.ListOptions{})
+	framework.ExpectNoError(err)
+	var errors []error
+
+	// returns maxRetries, sleepDuration
+	readConfig := func() (int, time.Duration) {
+		// Read in configuration settings, reasonable defaults.
+		retry := framework.TestContext.Cadvisor.MaxRetries
+		if framework.TestContext.Cadvisor.MaxRetries == 0 {
+			retry = 6
+			framework.Logf("Overriding default retry value of zero to %d", retry)
+		}
+
+		sleepDurationMS := framework.TestContext.Cadvisor.SleepDurationMS
+		if sleepDurationMS == 0 {
+			sleepDurationMS = 10000
+			framework.Logf("Overriding default milliseconds value of zero to %d", sleepDurationMS)
+		}
+
+		return retry, time.Duration(sleepDurationMS) * time.Millisecond
+	}
+
+	maxRetries, sleepDuration := readConfig()
+	for {
+		errors = []error{}
+		for _, node := range nodeList.Items {
+			// cadvisor is not accessible directly unless its port (4194 by default) is exposed.
+			// Here, we access '/stats/' REST endpoint on the kubelet which polls cadvisor internally.
+			statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
+			By(fmt.Sprintf("Querying stats from node %s using url %s", node.Name, statsResource))
+			_, err = c.Core().RESTClient().Get().AbsPath(statsResource).Timeout(timeout).Do().Raw()
+			if err != nil {
+				errors = append(errors, err)
+			}
+		}
+		if len(errors) == 0 {
+			return
+		}
+		if maxRetries--; maxRetries <= 0 {
+			break
+		}
+		framework.Logf("failed to retrieve kubelet stats -\n %v", errors)
+		time.Sleep(sleepDuration)
+	}
+	framework.Failf("Failed after retrying %d times for cadvisor to be healthy on all nodes. Errors:\n%v", maxRetries, errors)
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/monitoring/influxdb.go

@@ -0,0 +1,343 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitoring
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	influxdb "github.com/influxdata/influxdb/client/v2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = instrumentation.SIGDescribe("Monitoring", func() {
+	f := framework.NewDefaultFramework("monitoring")
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce")
+	})
+
+	It("should verify monitoring pods and all cluster nodes are available on influxdb using heapster.", func() {
+		testMonitoringUsingHeapsterInfluxdb(f.ClientSet)
+	})
+})
+
+const (
+	influxdbService       = "monitoring-influxdb"
+	influxdbDatabaseName  = "k8s"
+	podlistQuery          = "show tag values from \"cpu/usage\" with key = pod_name"
+	nodelistQuery         = "show tag values from \"cpu/usage\" with key = nodename"
+	sleepBetweenAttempts  = 5 * time.Second
+	testTimeout           = 5 * time.Minute
+	initializationTimeout = 5 * time.Minute
+)
+
+var (
+	rcLabels         = []string{"heapster", "influxGrafana"}
+	expectedServices = map[string]bool{
+		influxdbService:      false,
+		"monitoring-grafana": false,
+	}
+)
+
+// Query sends a command to the server and returns the Response
+func Query(c clientset.Interface, query string) (*influxdb.Response, error) {
+	subResourceProxyAvailable, err := framework.ServerVersionGTE(framework.SubResourceServiceAndNodeProxyVersion, c.Discovery())
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout)
+	defer cancel()
+
+	var result []byte
+	if subResourceProxyAvailable {
+		result, err = c.Core().RESTClient().Get().
+			Context(ctx).
+			Namespace("kube-system").
+			Resource("services").
+			Name(influxdbService+":api").
+			SubResource("proxy").
+			Suffix("query").
+			Param("q", query).
+			Param("db", influxdbDatabaseName).
+			Param("epoch", "s").
+			Do().
+			Raw()
+	} else {
+		result, err = c.Core().RESTClient().Get().
+			Context(ctx).
+			Prefix("proxy").
+			Namespace("kube-system").
+			Resource("services").
+			Name(influxdbService+":api").
+			Suffix("query").
+			Param("q", query).
+			Param("db", influxdbDatabaseName).
+			Param("epoch", "s").
+			Do().
+			Raw()
+	}
+
+	if err != nil {
+		if ctx.Err() != nil {
+			framework.Failf("Failed to query influx db: %v", err)
+		}
+		return nil, err
+	}
+
+	var response influxdb.Response
+	dec := json.NewDecoder(bytes.NewReader(result))
+	dec.UseNumber()
+	err = dec.Decode(&response)
+
+	if err != nil {
+		return nil, err
+	}
+	return &response, nil
+}
+
+func verifyExpectedRcsExistAndGetExpectedPods(c clientset.Interface) ([]string, error) {
+	expectedPods := []string{}
+	// Iterate over the labels that identify the replication controllers that we
+	// want to check. The rcLabels contains the value values for the k8s-app key
+	// that identify the replication controllers that we want to check. Using a label
+	// rather than an explicit name is preferred because the names will typically have
+	// a version suffix e.g. heapster-monitoring-v1 and this will change after a rolling
+	// update e.g. to heapster-monitoring-v2. By using a label query we can check for the
+	// situation when a heapster-monitoring-v1 and heapster-monitoring-v2 replication controller
+	// is running (which would be an error except during a rolling update).
+	for _, rcLabel := range rcLabels {
+		selector := labels.Set{"k8s-app": rcLabel}.AsSelector()
+		options := metav1.ListOptions{LabelSelector: selector.String()}
+		deploymentList, err := c.Extensions().Deployments(metav1.NamespaceSystem).List(options)
+		if err != nil {
+			return nil, err
+		}
+		rcList, err := c.Core().ReplicationControllers(metav1.NamespaceSystem).List(options)
+		if err != nil {
+			return nil, err
+		}
+		psList, err := c.AppsV1beta1().StatefulSets(metav1.NamespaceSystem).List(options)
+		if err != nil {
+			return nil, err
+		}
+		if (len(rcList.Items) + len(deploymentList.Items) + len(psList.Items)) != 1 {
+			return nil, fmt.Errorf("expected to find one replica for RC or deployment with label %s but got %d",
+				rcLabel, len(rcList.Items))
+		}
+		// Check all the replication controllers.
+		for _, rc := range rcList.Items {
+			selector := labels.Set(rc.Spec.Selector).AsSelector()
+			options := metav1.ListOptions{LabelSelector: selector.String()}
+			podList, err := c.Core().Pods(metav1.NamespaceSystem).List(options)
+			if err != nil {
+				return nil, err
+			}
+			for _, pod := range podList.Items {
+				if pod.DeletionTimestamp != nil {
+					continue
+				}
+				expectedPods = append(expectedPods, pod.Name)
+			}
+		}
+		// Do the same for all deployments.
+		for _, rc := range deploymentList.Items {
+			selector := labels.Set(rc.Spec.Selector.MatchLabels).AsSelector()
+			options := metav1.ListOptions{LabelSelector: selector.String()}
+			podList, err := c.Core().Pods(metav1.NamespaceSystem).List(options)
+			if err != nil {
+				return nil, err
+			}
+			for _, pod := range podList.Items {
+				if pod.DeletionTimestamp != nil {
+					continue
+				}
+				expectedPods = append(expectedPods, pod.Name)
+			}
+		}
+		// And for pet sets.
+		for _, ps := range psList.Items {
+			selector := labels.Set(ps.Spec.Selector.MatchLabels).AsSelector()
+			options := metav1.ListOptions{LabelSelector: selector.String()}
+			podList, err := c.Core().Pods(metav1.NamespaceSystem).List(options)
+			if err != nil {
+				return nil, err
+			}
+			for _, pod := range podList.Items {
+				if pod.DeletionTimestamp != nil {
+					continue
+				}
+				expectedPods = append(expectedPods, pod.Name)
+			}
+		}
+	}
+	return expectedPods, nil
+}
+
+func expectedServicesExist(c clientset.Interface) error {
+	serviceList, err := c.Core().Services(metav1.NamespaceSystem).List(metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, service := range serviceList.Items {
+		if _, ok := expectedServices[service.Name]; ok {
+			expectedServices[service.Name] = true
+		}
+	}
+	for service, found := range expectedServices {
+		if !found {
+			return fmt.Errorf("Service %q not found", service)
+		}
+	}
+	return nil
+}
+
+func getAllNodesInCluster(c clientset.Interface) ([]string, error) {
+	// It should be OK to list unschedulable Nodes here.
+	nodeList, err := c.Core().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	result := []string{}
+	for _, node := range nodeList.Items {
+		result = append(result, node.Name)
+	}
+	return result, nil
+}
+
+func getInfluxdbData(c clientset.Interface, query string, tag string) (map[string]bool, error) {
+	response, err := Query(c, query)
+	if err != nil {
+		return nil, err
+	}
+	if len(response.Results) != 1 {
+		return nil, fmt.Errorf("expected only one result from Influxdb for query %q. Got %+v", query, response)
+	}
+	if len(response.Results[0].Series) != 1 {
+		return nil, fmt.Errorf("expected exactly one series for query %q.", query)
+	}
+	if len(response.Results[0].Series[0].Columns) != 2 {
+		framework.Failf("Expected two columns for query %q. Found %v", query, response.Results[0].Series[0].Columns)
+	}
+	result := map[string]bool{}
+	for _, value := range response.Results[0].Series[0].Values {
+		name := value[1].(string)
+		result[name] = true
+	}
+	return result, nil
+}
+
+func expectedItemsExist(expectedItems []string, actualItems map[string]bool) bool {
+	if len(actualItems) < len(expectedItems) {
+		return false
+	}
+	for _, item := range expectedItems {
+		if _, found := actualItems[item]; !found {
+			return false
+		}
+	}
+	return true
+}
+
+func validatePodsAndNodes(c clientset.Interface, expectedPods, expectedNodes []string) bool {
+	pods, err := getInfluxdbData(c, podlistQuery, "pod_id")
+	if err != nil {
+		// We don't fail the test here because the influxdb service might still not be running.
+		framework.Logf("failed to query list of pods from influxdb. Query: %q, Err: %v", podlistQuery, err)
+		return false
+	}
+	nodes, err := getInfluxdbData(c, nodelistQuery, "hostname")
+	if err != nil {
+		framework.Logf("failed to query list of nodes from influxdb. Query: %q, Err: %v", nodelistQuery, err)
+		return false
+	}
+	if !expectedItemsExist(expectedPods, pods) {
+		framework.Logf("failed to find all expected Pods.\nExpected: %v\nActual: %v", expectedPods, pods)
+		return false
+	}
+	if !expectedItemsExist(expectedNodes, nodes) {
+		framework.Logf("failed to find all expected Nodes.\nExpected: %v\nActual: %v", expectedNodes, nodes)
+		return false
+	}
+	return true
+}
+
+func testMonitoringUsingHeapsterInfluxdb(c clientset.Interface) {
+	// Check if heapster pods and services are up.
+	var expectedPods []string
+	rcErr := fmt.Errorf("failed to verify expected RCs within timeout")
+	serviceErr := fmt.Errorf("failed to verify expected services within timeout")
+	err := wait.PollImmediate(sleepBetweenAttempts, initializationTimeout, func() (bool, error) {
+		expectedPods, rcErr = verifyExpectedRcsExistAndGetExpectedPods(c)
+		if rcErr != nil {
+			framework.Logf("Waiting for expected RCs (got error: %v)", rcErr)
+			return false, nil
+		}
+		serviceErr = expectedServicesExist(c)
+		if serviceErr != nil {
+			framework.Logf("Waiting for expected services (got error: %v)", serviceErr)
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		framework.ExpectNoError(rcErr)
+		framework.ExpectNoError(serviceErr)
+		framework.Failf("Failed to verify RCs and services within timeout: %v", err)
+	}
+
+	expectedNodes, err := getAllNodesInCluster(c)
+	framework.ExpectNoError(err)
+	startTime := time.Now()
+	for {
+		if validatePodsAndNodes(c, expectedPods, expectedNodes) {
+			return
+		}
+		if time.Since(startTime) >= testTimeout {
+			// temporary workaround to help debug issue #12765
+			printDebugInfo(c)
+			break
+		}
+		time.Sleep(sleepBetweenAttempts)
+	}
+	framework.Failf("monitoring using heapster and influxdb test failed")
+}
+
+func printDebugInfo(c clientset.Interface) {
+	set := labels.Set{"k8s-app": "heapster"}
+	options := metav1.ListOptions{LabelSelector: set.AsSelector().String()}
+	podList, err := c.Core().Pods(metav1.NamespaceSystem).List(options)
+	if err != nil {
+		framework.Logf("Error while listing pods %v", err)
+		return
+	}
+	for _, pod := range podList.Items {
+		framework.Logf("Kubectl output:\n%v",
+			framework.RunKubectlOrDie("log", pod.Name, "--namespace=kube-system", "--container=heapster"))
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/monitoring/metrics_grabber.go

@@ -0,0 +1,102 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitoring
+
+import (
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/metrics"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+
+	gin "github.com/onsi/ginkgo"
+	gom "github.com/onsi/gomega"
+)
+
+var _ = instrumentation.SIGDescribe("MetricsGrabber", func() {
+	f := framework.NewDefaultFramework("metrics-grabber")
+	var c, ec clientset.Interface
+	var grabber *metrics.MetricsGrabber
+	gin.BeforeEach(func() {
+		var err error
+		c = f.ClientSet
+		ec = f.KubemarkExternalClusterClientSet
+		framework.ExpectNoError(err)
+		grabber, err = metrics.NewMetricsGrabber(c, ec, true, true, true, true, true)
+		framework.ExpectNoError(err)
+	})
+
+	gin.It("should grab all metrics from API server.", func() {
+		gin.By("Connecting to /metrics endpoint")
+		response, err := grabber.GrabFromApiServer()
+		framework.ExpectNoError(err)
+		gom.Expect(response).NotTo(gom.BeEmpty())
+	})
+
+	gin.It("should grab all metrics from a Kubelet.", func() {
+		gin.By("Proxying to Node through the API server")
+		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		gom.Expect(nodes.Items).NotTo(gom.BeEmpty())
+		response, err := grabber.GrabFromKubelet(nodes.Items[0].Name)
+		framework.ExpectNoError(err)
+		gom.Expect(response).NotTo(gom.BeEmpty())
+	})
+
+	gin.It("should grab all metrics from a Scheduler.", func() {
+		gin.By("Proxying to Pod through the API server")
+		// Check if master Node is registered
+		nodes, err := c.Core().Nodes().List(metav1.ListOptions{})
+		framework.ExpectNoError(err)
+
+		var masterRegistered = false
+		for _, node := range nodes.Items {
+			if strings.HasSuffix(node.Name, "master") {
+				masterRegistered = true
+			}
+		}
+		if !masterRegistered {
+			framework.Logf("Master is node api.Registry. Skipping testing Scheduler metrics.")
+			return
+		}
+		response, err := grabber.GrabFromScheduler()
+		framework.ExpectNoError(err)
+		gom.Expect(response).NotTo(gom.BeEmpty())
+	})
+
+	gin.It("should grab all metrics from a ControllerManager.", func() {
+		gin.By("Proxying to Pod through the API server")
+		// Check if master Node is registered
+		nodes, err := c.Core().Nodes().List(metav1.ListOptions{})
+		framework.ExpectNoError(err)
+
+		var masterRegistered = false
+		for _, node := range nodes.Items {
+			if strings.HasSuffix(node.Name, "master") {
+				masterRegistered = true
+			}
+		}
+		if !masterRegistered {
+			framework.Logf("Master is node api.Registry. Skipping testing ControllerManager metrics.")
+			return
+		}
+		response, err := grabber.GrabFromControllerManager()
+		framework.ExpectNoError(err)
+		gom.Expect(response).NotTo(gom.BeEmpty())
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/instrumentation/monitoring/stackdriver.go

@@ -0,0 +1,175 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitoring
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"time"
+
+	"golang.org/x/oauth2/google"
+
+	. "github.com/onsi/ginkgo"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/test/e2e/common"
+	"k8s.io/kubernetes/test/e2e/framework"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+
+	gcm "google.golang.org/api/monitoring/v3"
+)
+
+var (
+	// Stackdriver container metrics, as described here:
+	// https://cloud.google.com/monitoring/api/metrics#gcp-container
+	stackdriverMetrics = []string{
+		"uptime",
+		"memory/bytes_total",
+		"memory/bytes_used",
+		"cpu/reserved_cores",
+		"cpu/usage_time",
+		"memory/page_fault_count",
+		"disk/bytes_used",
+		"disk/bytes_total",
+		"cpu/utilization",
+	}
+
+	pollFrequency = time.Second * 5
+	pollTimeout   = time.Minute * 7
+
+	rcName            = "resource-consumer"
+	memoryUsed        = 64
+	memoryLimit int64 = 200
+	tolerance         = 0.25
+)
+
+var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() {
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce", "gke")
+	})
+
+	f := framework.NewDefaultFramework("stackdriver-monitoring")
+
+	It("should have cluster metrics [Feature:StackdriverMonitoring]", func() {
+		testStackdriverMonitoring(f, 1, 100, 200)
+	})
+
+})
+
+func testStackdriverMonitoring(f *framework.Framework, pods, allPodsCPU int, perPodCPU int64) {
+	projectId := framework.TestContext.CloudConfig.ProjectID
+
+	ctx := context.Background()
+	client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
+	gcmService, err := gcm.New(client)
+
+	// set this env var if accessing Stackdriver test endpoint (default is prod):
+	// $ export STACKDRIVER_API_ENDPOINT_OVERRIDE=https://test-monitoring.sandbox.googleapis.com/
+	basePathOverride := os.Getenv("STACKDRIVER_API_ENDPOINT_OVERRIDE")
+	if basePathOverride != "" {
+		gcmService.BasePath = basePathOverride
+	}
+
+	framework.ExpectNoError(err)
+
+	rc := common.NewDynamicResourceConsumer(rcName, f.Namespace.Name, common.KindDeployment, pods, allPodsCPU, memoryUsed, 0, perPodCPU, memoryLimit, f.ClientSet, f.InternalClientset)
+	defer rc.CleanUp()
+
+	rc.WaitForReplicas(pods, 15*time.Minute)
+
+	metricsMap := map[string]bool{}
+	pollingFunction := checkForMetrics(projectId, gcmService, time.Now(), metricsMap, allPodsCPU, perPodCPU)
+	err = wait.Poll(pollFrequency, pollTimeout, pollingFunction)
+	if err != nil {
+		framework.Logf("Missing metrics: %+v\n", metricsMap)
+	}
+	framework.ExpectNoError(err)
+}
+
+func checkForMetrics(projectId string, gcmService *gcm.Service, start time.Time, metricsMap map[string]bool, cpuUsed int, cpuLimit int64) func() (bool, error) {
+	return func() (bool, error) {
+		counter := 0
+		correctUtilization := false
+		for _, metric := range stackdriverMetrics {
+			metricsMap[metric] = false
+		}
+		for _, metric := range stackdriverMetrics {
+			// TODO: check only for metrics from this cluster
+			ts, err := fetchTimeSeries(projectId, gcmService, metric, start, time.Now())
+			framework.ExpectNoError(err)
+			if len(ts) > 0 {
+				counter = counter + 1
+				metricsMap[metric] = true
+				framework.Logf("Received %v timeseries for metric %v\n", len(ts), metric)
+			} else {
+				framework.Logf("No timeseries for metric %v\n", metric)
+			}
+
+			var sum float64 = 0
+			switch metric {
+			case "cpu/utilization":
+				for _, t := range ts {
+					max := t.Points[0]
+					maxEnd, _ := time.Parse(time.RFC3339, max.Interval.EndTime)
+					for _, p := range t.Points {
+						pEnd, _ := time.Parse(time.RFC3339, p.Interval.EndTime)
+						if pEnd.After(maxEnd) {
+							max = p
+							maxEnd, _ = time.Parse(time.RFC3339, max.Interval.EndTime)
+						}
+					}
+					sum = sum + *max.Value.DoubleValue
+					framework.Logf("Received %v points for metric %v\n",
+						len(t.Points), metric)
+				}
+				framework.Logf("Most recent cpu/utilization sum*cpu/limit: %v\n", sum*float64(cpuLimit))
+				if math.Abs(sum*float64(cpuLimit)-float64(cpuUsed)) > tolerance*float64(cpuUsed) {
+					return false, nil
+				} else {
+					correctUtilization = true
+				}
+			}
+		}
+		if counter < 9 || !correctUtilization {
+			return false, nil
+		}
+		return true, nil
+	}
+}
+
+func createMetricFilter(metric string, container_name string) string {
+	return fmt.Sprintf(`metric.type="container.googleapis.com/container/%s" AND
+				resource.label.container_name="%s"`, metric, container_name)
+}
+
+func fetchTimeSeries(projectId string, gcmService *gcm.Service, metric string, start time.Time, end time.Time) ([]*gcm.TimeSeries, error) {
+	response, err := gcmService.Projects.TimeSeries.
+		List(fullProjectName(projectId)).
+		Filter(createMetricFilter(metric, rcName)).
+		IntervalStartTime(start.Format(time.RFC3339)).
+		IntervalEndTime(end.Format(time.RFC3339)).
+		Do()
+	if err != nil {
+		return nil, err
+	}
+	return response.TimeSeries, nil
+}
+
+func fullProjectName(name string) string {
+	return fmt.Sprintf("projects/%s", name)
+}