From 9877592e5211197ae0cbc0716c3d29fbb98f8cf8 Mon Sep 17 00:00:00 2001 From: Reza Karegar Date: Fri, 16 Feb 2024 16:22:40 -0500 Subject: [PATCH 1/6] Add the metric export logic. --- google-cloud-bigtable-deps-bom/pom.xml | 6 + google-cloud-bigtable-stats/pom.xml | 4 + .../BigtableCreateTimeSeriesExporter.java | 23 +- .../stats/BigtableStackdriverExportUtils.java | 102 +++++++- .../BigtableStackdriverStatsExporter.java | 26 +- .../bigtable/stats/BuiltinViewConstants.java | 20 +- .../cloud/bigtable/stats/BuiltinViews.java | 3 +- .../stats/ConsumerEnvironmentUtils.java | 99 ++++++++ .../BigtableCreateTimeSeriesExporterTest.java | 223 ++++++++++++++++-- 9 files changed, 461 insertions(+), 45 deletions(-) create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index 54301e752e..4a1db51837 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -77,6 +77,12 @@ pom import + + + io.opencensus + opencensus-contrib-resource-util + 0.31.1 + diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index 535e93be98..dda77cfea3 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -38,6 +38,10 @@ io.opencensus opencensus-exporter-stats-stackdriver + + io.opencensus + opencensus-contrib-resource-util + io.opencensus opencensus-impl diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index dca52f1c81..a964d36e69 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -33,13 +33,20 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { private static final Logger logger = Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName()); private final MetricServiceClient metricServiceClient; - private final MonitoredResource monitoredResource; + private final MonitoredResource bigtableMonitoredResource; + private final MonitoredResource gceMonitoredResource; + private final MonitoredResource gkeMonitoredResource; private final String clientId; BigtableCreateTimeSeriesExporter( - MetricServiceClient metricServiceClient, MonitoredResource monitoredResource) { + MetricServiceClient metricServiceClient, + MonitoredResource bigtableMonitoredResource, + MonitoredResource gceMonitoredResource, + MonitoredResource gkeMonitoredResource) { this.metricServiceClient = metricServiceClient; - this.monitoredResource = monitoredResource; + this.bigtableMonitoredResource = bigtableMonitoredResource; + this.gceMonitoredResource = gceMonitoredResource; + this.gkeMonitoredResource = gkeMonitoredResource; this.clientId = BigtableStackdriverExportUtils.getDefaultTaskValue(); } @@ -48,11 +55,7 @@ public void export(Collection metrics) { for (Metric metric : metrics) { // only export bigtable metrics - if (!metric.getMetricDescriptor().getName().contains("bigtable")) { - continue; - } - // TODO: temporarily skip exporting per connection metrics. - if (metric.getMetricDescriptor().getName().contains("per_connection_error_count")) { + if (!BigtableStackdriverExportUtils.shouldExportMetric(metric.getMetricDescriptor())) { continue; } @@ -69,7 +72,9 @@ public void export(Collection metrics) { metric.getMetricDescriptor(), timeSeries, clientId, - monitoredResource), + bigtableMonitoredResource, + gceMonitoredResource, + gkeMonitoredResource), Collectors.toList()))); for (Map.Entry> entry : diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index ff3e93158b..f787552d9b 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -15,6 +15,8 @@ */ package com.google.cloud.bigtable.stats; +import static com.google.cloud.bigtable.stats.BuiltinViewConstants.PER_CONNECTION_ERROR_COUNT_VIEW; + import com.google.api.Distribution.BucketOptions; import com.google.api.Distribution.BucketOptions.Explicit; import com.google.api.Metric; @@ -90,8 +92,8 @@ class BigtableStackdriverExportUtils { return builder.build(); }; - // promote the following metric labels to monitored resource labels - private static final Set PROMOTED_RESOURCE_LABELS = + // promote the following metric labels to Bigtable monitored resource labels + private static final Set PROMOTED_BIGTABLE_RESOURCE_LABELS = ImmutableSet.of( BuiltinMeasureConstants.PROJECT_ID.getName(), BuiltinMeasureConstants.INSTANCE_ID.getName(), @@ -102,17 +104,63 @@ class BigtableStackdriverExportUtils { private static final LabelKey CLIENT_UID_LABEL_KEY = LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client uid"); + static boolean isBigtableTableMetric(MetricDescriptor metricDescriptor) { + return !metricDescriptor.getName().equals(PER_CONNECTION_ERROR_COUNT_VIEW.getName().asString()); + } + + static boolean shouldExportMetric(MetricDescriptor metricDescriptor) { + return metricDescriptor.getName().contains("bigtable") + && (isBigtableTableMetric(metricDescriptor) + || ConsumerEnvironmentUtils.isEnvGce() + || ConsumerEnvironmentUtils.isEnvGke()); + } + static com.google.monitoring.v3.TimeSeries convertTimeSeries( MetricDescriptor metricDescriptor, TimeSeries timeSeries, String clientId, - MonitoredResource monitoredResource) { - String metricName = metricDescriptor.getName(); - List labelKeys = metricDescriptor.getLabelKeys(); + MonitoredResource bigtableMonitoredResource, + MonitoredResource gceMonitoredResource, + MonitoredResource gkeMonitoredResource) { Type metricType = metricDescriptor.getType(); - MonitoredResource.Builder monitoredResourceBuilder = monitoredResource.toBuilder(); + com.google.monitoring.v3.TimeSeries.Builder builder; + if (isBigtableTableMetric(metricDescriptor)) { + builder = + setupBuilderForBigtableResource( + metricDescriptor, bigtableMonitoredResource, timeSeries, clientId); + } else if (ConsumerEnvironmentUtils.isEnvGce()) { + builder = + setupBuilderForGceOrGKEResource( + metricDescriptor, gceMonitoredResource, timeSeries, clientId); + } else if (ConsumerEnvironmentUtils.isEnvGke()) { + builder = + setupBuilderForGceOrGKEResource( + metricDescriptor, gkeMonitoredResource, timeSeries, clientId); + } else { + logger.warning( + "Trying to export metric " + + metricDescriptor.getName() + + " in a non-GCE/GKE environment."); + return com.google.monitoring.v3.TimeSeries.newBuilder().build(); + } + builder.setMetricKind(createMetricKind(metricType)); + builder.setValueType(createValueType(metricType)); + Timestamp startTimeStamp = timeSeries.getStartTimestamp(); + for (Point point : timeSeries.getPoints()) { + builder.addPoints(createPoint(point, startTimeStamp)); + } + return builder.build(); + } + private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForBigtableResource( + MetricDescriptor metricDescriptor, + MonitoredResource bigtableMonitoredResource, + TimeSeries timeSeries, + String clientId) { + List labelKeys = metricDescriptor.getLabelKeys(); + String metricName = metricDescriptor.getName(); + MonitoredResource.Builder monitoredResourceBuilder = bigtableMonitoredResource.toBuilder(); List metricTagKeys = new ArrayList<>(); List metricTagValues = new ArrayList<>(); @@ -120,7 +168,7 @@ static com.google.monitoring.v3.TimeSeries convertTimeSeries( for (int i = 0; i < labelValues.size(); i++) { // If the label is defined in the monitored resource, convert it to // a monitored resource label. Otherwise, keep it as a metric label. - if (PROMOTED_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) { + if (PROMOTED_BIGTABLE_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) { monitoredResourceBuilder.putLabels( labelKeys.get(i).getKey(), labelValues.get(i).getValue()); } else { @@ -135,13 +183,41 @@ static com.google.monitoring.v3.TimeSeries convertTimeSeries( com.google.monitoring.v3.TimeSeries.newBuilder(); builder.setResource(monitoredResourceBuilder.build()); builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues)); - builder.setMetricKind(createMetricKind(metricType)); - builder.setValueType(createValueType(metricType)); - Timestamp startTimeStamp = timeSeries.getStartTimestamp(); - for (Point point : timeSeries.getPoints()) { - builder.addPoints(createPoint(point, startTimeStamp)); + + return builder; + } + + private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForGceOrGKEResource( + MetricDescriptor metricDescriptor, + MonitoredResource monitoredResource, + TimeSeries timeSeries, + String clientId) { + List labelKeys = metricDescriptor.getLabelKeys(); + String metricName = metricDescriptor.getName(); + MonitoredResource.Builder monitoredResourceBuilder = monitoredResource.toBuilder(); + List metricTagKeys = new ArrayList<>(); + List metricTagValues = new ArrayList<>(); + + List labelValues = timeSeries.getLabelValues(); + for (int i = 0; i < labelValues.size(); i++) { + metricTagKeys.add(labelKeys.get(i)); + metricTagValues.add(labelValues.get(i)); } - return builder.build(); + metricTagKeys.add(CLIENT_UID_LABEL_KEY); + metricTagValues.add(LabelValue.create(clientId)); + + if (ConsumerEnvironmentUtils.isEnvGce()) { + ConsumerEnvironmentUtils.putGceResourceLabels(monitoredResourceBuilder); + } else { + ConsumerEnvironmentUtils.putGkeResourceLabels(monitoredResourceBuilder); + } + + com.google.monitoring.v3.TimeSeries.Builder builder = + com.google.monitoring.v3.TimeSeries.newBuilder(); + builder.setResource(monitoredResourceBuilder.build()); + builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues)); + + return builder; } static String getProjectId(MetricDescriptor metricDescriptor, TimeSeries timeSeries) { diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 76a36215c6..3a102135dc 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -43,7 +43,9 @@ public class BigtableStackdriverStatsExporter { // Default export interval is 1 minute private static final Duration EXPORT_INTERVAL = Duration.create(60, 0); - private static final String RESOURCE_TYPE = "bigtable_client_raw"; + private static final String BIGTABLE_RESOURCE_TYPE = "bigtable_client_raw"; + private static final String GCE_RESOURCE_TYPE = "gce_instance"; + private static final String GKE_RESOURCE_TYPE = "k8s_container"; private static final String MONITORING_ENDPOINT = MoreObjects.firstNonNull( @@ -55,13 +57,19 @@ public class BigtableStackdriverStatsExporter { private BigtableStackdriverStatsExporter( MetricServiceClient metricServiceClient, Duration exportInterval, - MonitoredResource monitoredResource) { + MonitoredResource bigtableMonitoredResource, + MonitoredResource gceMonitoredResource, + MonitoredResource gkeMonitoredResource) { IntervalMetricReader.Options.Builder intervalMetricReaderOptionsBuilder = IntervalMetricReader.Options.builder(); intervalMetricReaderOptionsBuilder.setExportInterval(exportInterval); this.intervalMetricReader = IntervalMetricReader.create( - new BigtableCreateTimeSeriesExporter(metricServiceClient, monitoredResource), + new BigtableCreateTimeSeriesExporter( + metricServiceClient, + bigtableMonitoredResource, + gceMonitoredResource, + gkeMonitoredResource), MetricReader.create( MetricReader.Options.builder() .setMetricProducerManager( @@ -76,9 +84,15 @@ public static void register(Credentials credentials) throws IOException { instance == null, "Bigtable Stackdriver stats exporter is already created"); // Default timeout for creating a client is 1 minute MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); - MonitoredResource resourceType = - MonitoredResource.newBuilder().setType(RESOURCE_TYPE).build(); - instance = new BigtableStackdriverStatsExporter(client, EXPORT_INTERVAL, resourceType); + MonitoredResource bigtableResourceType = + MonitoredResource.newBuilder().setType(BIGTABLE_RESOURCE_TYPE).build(); + MonitoredResource gceResourceType = + MonitoredResource.newBuilder().setType(GCE_RESOURCE_TYPE).build(); + MonitoredResource gkeResourceType = + MonitoredResource.newBuilder().setType(GKE_RESOURCE_TYPE).build(); + instance = + new BigtableStackdriverStatsExporter( + client, EXPORT_INTERVAL, bigtableResourceType, gceResourceType, gkeResourceType); } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java index 06364a2288..82ce61e2d3 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java @@ -15,7 +15,25 @@ */ package com.google.cloud.bigtable.stats; -import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.*; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.APPLICATION_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.APP_PROFILE; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.ATTEMPT_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CLIENT_NAME; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CLUSTER; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.INSTANCE_ID; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.METHOD; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.OPERATION_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.PER_CONNECTION_ERROR_COUNT; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.PROJECT_ID; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.RETRY_COUNT; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.SERVER_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.STATUS; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.STREAMING; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.TABLE; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.THROTTLING_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.ZONE; import static io.opencensus.stats.Aggregation.Distribution; import static io.opencensus.stats.Aggregation.Sum; diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java index e62117cb7e..cfa5af0b00 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java @@ -36,7 +36,8 @@ public class BuiltinViews { BuiltinViewConstants.SERVER_LATENCIES_VIEW, BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, - BuiltinViewConstants.THROTTLING_LATENCIES_VIEW); + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + BuiltinViewConstants.PER_CONNECTION_ERROR_COUNT_VIEW); @VisibleForTesting void registerPrivateViews(ViewManager viewManager) { diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java new file mode 100644 index 0000000000..ba17dc2168 --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java @@ -0,0 +1,99 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import com.google.api.MonitoredResource; +import com.google.common.annotations.VisibleForTesting; +import io.opencensus.contrib.resource.util.CloudResource; +import io.opencensus.contrib.resource.util.ContainerResource; +import io.opencensus.contrib.resource.util.HostResource; +import io.opencensus.contrib.resource.util.K8sResource; +import io.opencensus.contrib.resource.util.ResourceUtils; +import io.opencensus.resource.Resource; +import java.util.Objects; + +/** A class for extracting details about consumer environments (GCE and GKE) for metrics. */ +class ConsumerEnvironmentUtils { + + private static ResourceUtilsWrapper resourceUtilsWrapper = new ResourceUtilsWrapper(); + static final String GCE_PROJECT_ID_LABEL = "project_id"; + static final String GCE_INSTANCE_ID_LABEL = "instance_id"; + static final String GCE_ZONE_LABEL = "zone"; + + static final String GKE_PROJECT_ID_LABEL = "project_id"; + static final String GKE_LOCATION_LABEL = "location"; + static final String GKE_CLUSTER_NAME_LABEL = "cluster_name"; + static final String GKE_NAMESPACE_NAME_LABEL = "namespace_name"; + static final String GKE_POD_NAME_LABEL = "pod_name"; + static final String GKE_CONTAINER_NAME_LABEL = "container_name"; + + @VisibleForTesting + public static void setResourceUtilsWrapper(ResourceUtilsWrapper newResourceUtilsWrapper) { + resourceUtilsWrapper = newResourceUtilsWrapper; + } + + public static boolean isEnvGce() { + Resource resource = resourceUtilsWrapper.detectResource(); + return Objects.equals(resource.getType(), HostResource.TYPE) + && Objects.equals( + resource.getLabels().get(CloudResource.PROVIDER_KEY), CloudResource.PROVIDER_GCP); + } + + public static boolean isEnvGke() { + Resource resource = resourceUtilsWrapper.detectResource(); + return Objects.equals(resource.getType(), ContainerResource.TYPE) + && Objects.equals( + resource.getLabels().get(CloudResource.PROVIDER_KEY), CloudResource.PROVIDER_GCP); + } + + public static void putGceResourceLabels(MonitoredResource.Builder builder) { + Resource resource = resourceUtilsWrapper.detectResource(); + if (isEnvGce()) { + builder.putLabels( + GCE_PROJECT_ID_LABEL, resource.getLabels().get(CloudResource.ACCOUNT_ID_KEY)); + builder.putLabels(GCE_INSTANCE_ID_LABEL, resource.getLabels().get(HostResource.ID_KEY)); + builder.putLabels(GCE_ZONE_LABEL, resource.getLabels().get(CloudResource.ZONE_KEY)); + } else { + throw new IllegalStateException("Can only put GCE labels inside a GCE environment."); + } + } + + public static void putGkeResourceLabels(MonitoredResource.Builder builder) { + Resource resource = resourceUtilsWrapper.detectResource(); + if (isEnvGke()) { + builder.putLabels( + GKE_PROJECT_ID_LABEL, resource.getLabels().get(CloudResource.ACCOUNT_ID_KEY)); + builder.putLabels(GKE_LOCATION_LABEL, resource.getLabels().get(CloudResource.ZONE_KEY)); + builder.putLabels( + GKE_CLUSTER_NAME_LABEL, resource.getLabels().get(K8sResource.CLUSTER_NAME_KEY)); + builder.putLabels( + GKE_NAMESPACE_NAME_LABEL, resource.getLabels().get(K8sResource.NAMESPACE_NAME_KEY)); + builder.putLabels(GKE_POD_NAME_LABEL, resource.getLabels().get(K8sResource.POD_NAME_KEY)); + builder.putLabels( + GKE_CONTAINER_NAME_LABEL, resource.getLabels().get(ContainerResource.NAME_KEY)); + } else { + throw new IllegalStateException("Can only put GKE labels inside a GKE environment."); + } + } + + // We wrap the static ResourceUtils.detectResource() method in a non-static method for mocking. + @VisibleForTesting + public static class ResourceUtilsWrapper { + public Resource detectResource() { + return ResourceUtils.detectResource(); + } + } +} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index 26654c09af..47bba056cc 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -23,9 +23,14 @@ import com.google.api.gax.rpc.UnaryCallable; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.cloud.monitoring.v3.stub.MetricServiceStub; +import com.google.common.collect.ImmutableMap; import com.google.monitoring.v3.CreateTimeSeriesRequest; import com.google.protobuf.Empty; import io.opencensus.common.Timestamp; +import io.opencensus.contrib.resource.util.CloudResource; +import io.opencensus.contrib.resource.util.ContainerResource; +import io.opencensus.contrib.resource.util.HostResource; +import io.opencensus.contrib.resource.util.K8sResource; import io.opencensus.metrics.LabelKey; import io.opencensus.metrics.LabelValue; import io.opencensus.metrics.export.Metric; @@ -33,6 +38,7 @@ import io.opencensus.metrics.export.Point; import io.opencensus.metrics.export.TimeSeries; import io.opencensus.metrics.export.Value; +import io.opencensus.resource.Resource; import java.util.Arrays; import org.junit.After; import org.junit.Before; @@ -42,18 +48,31 @@ import org.junit.runners.JUnit4; import org.mockito.ArgumentCaptor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; @RunWith(JUnit4.class) public class BigtableCreateTimeSeriesExporterTest { - private static final String projectId = "fake-project"; - private static final String instanceId = "fake-instance"; + private static final String bigtableProjectId = "fake-bigtable-project"; + private static final String bigtableInstanceId = "fake-bigtable-instance"; private static final String appProfileId = "default"; private static final String tableId = "fake-table"; - private static final String zone = "us-east-1"; - private static final String cluster = "cluster-1"; + private static final String bigtableZone = "us-east-1"; + private static final String bigtableCluster = "cluster-1"; + private static final String clientName = "client-name"; + + private static final String gceProjectId = "fake-gce-project"; + private static final String gceInstanceId = "fake-gce-instance"; + private static final String gceZone = "fake-gce-zone"; + + private static final String gkeProjectId = "fake-gke-project"; + private static final String gkeLocation = "fake-gke-location"; + private static final String gkeClusterName = "fake-gke-cluster"; + private static final String gkeNamespaceName = "fake-gke-namespace"; + private static final String gkePodName = "fake-gke-pod"; + private static final String gkeContainerName = "fake-gke-container"; @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); @@ -69,14 +88,16 @@ public void setUp() { exporter = new BigtableCreateTimeSeriesExporter( fakeMetricServiceClient, - MonitoredResource.newBuilder().setType("bigtable-table").build()); + MonitoredResource.newBuilder().setType("bigtable-table").build(), + MonitoredResource.newBuilder().setType("gce-instance").build(), + MonitoredResource.newBuilder().setType("gke-container").build()); } @After public void tearDown() {} @Test - public void testTimeSeries() { + public void testTimeSeriesForMetricWithBigtableResource() { ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); @@ -89,7 +110,7 @@ public void testTimeSeries() { Metric.create( MetricDescriptor.create( "bigtable/test", - "descritpion", + "description", "ms", MetricDescriptor.Type.CUMULATIVE_DOUBLE, Arrays.asList( @@ -102,11 +123,11 @@ public void testTimeSeries() { Arrays.asList( TimeSeries.create( Arrays.asList( - LabelValue.create(projectId), - LabelValue.create(instanceId), + LabelValue.create(bigtableProjectId), + LabelValue.create(bigtableInstanceId), LabelValue.create(tableId), - LabelValue.create(cluster), - LabelValue.create(zone), + LabelValue.create(bigtableCluster), + LabelValue.create(bigtableZone), LabelValue.create(appProfileId)), Arrays.asList( Point.create( @@ -124,11 +145,11 @@ public void testTimeSeries() { assertThat(timeSeries.getResource().getLabelsMap()) .containsExactly( - BuiltinMeasureConstants.PROJECT_ID.getName(), projectId, - BuiltinMeasureConstants.INSTANCE_ID.getName(), instanceId, + BuiltinMeasureConstants.PROJECT_ID.getName(), bigtableProjectId, + BuiltinMeasureConstants.INSTANCE_ID.getName(), bigtableInstanceId, BuiltinMeasureConstants.TABLE.getName(), tableId, - BuiltinMeasureConstants.CLUSTER.getName(), cluster, - BuiltinMeasureConstants.ZONE.getName(), zone); + BuiltinMeasureConstants.CLUSTER.getName(), bigtableCluster, + BuiltinMeasureConstants.ZONE.getName(), bigtableZone); assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(2); assertThat(timeSeries.getMetric().getLabelsMap()) @@ -139,6 +160,178 @@ public void testTimeSeries() { assertThat(timeSeries.getPoints(0).getValue().getDoubleValue()).isEqualTo(fakeValue); } + @Test + public void testTimeSeriesForMetricWithGceResource() { + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); + + UnaryCallable mockCallable = mock(UnaryCallable.class); + when(mockMetricServiceStub.createServiceTimeSeriesCallable()).thenReturn(mockCallable); + when(mockCallable.call(argumentCaptor.capture())).thenReturn(Empty.getDefaultInstance()); + + ConsumerEnvironmentUtils.ResourceUtilsWrapper resourceUtilsWrapperMock = + Mockito.mock(ConsumerEnvironmentUtils.ResourceUtilsWrapper.class); + ConsumerEnvironmentUtils.setResourceUtilsWrapper(resourceUtilsWrapperMock); + Mockito.when(resourceUtilsWrapperMock.detectResource()) + .thenReturn( + Resource.create( + HostResource.TYPE, + ImmutableMap.of( + CloudResource.PROVIDER_KEY, + CloudResource.PROVIDER_GCP, + CloudResource.ACCOUNT_ID_KEY, + gceProjectId, + HostResource.ID_KEY, + gceInstanceId, + CloudResource.ZONE_KEY, + gceZone))); + + double fakeValue = 10.0; + Metric fakeMetric = + Metric.create( + MetricDescriptor.create( + "bigtable.googleapis.com/internal/client/per_connection_error_count", + "description", + "ms", + MetricDescriptor.Type.CUMULATIVE_DOUBLE, + Arrays.asList( + LabelKey.create(BuiltinMeasureConstants.PROJECT_ID.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.INSTANCE_ID.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.APP_PROFILE.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.CLIENT_NAME.getName(), ""))), + Arrays.asList( + TimeSeries.create( + Arrays.asList( + LabelValue.create(bigtableProjectId), + LabelValue.create(bigtableInstanceId), + LabelValue.create(appProfileId), + LabelValue.create(clientName)), + Arrays.asList( + Point.create( + Value.doubleValue(fakeValue), + Timestamp.fromMillis(System.currentTimeMillis()))), + Timestamp.fromMillis(System.currentTimeMillis())))); + + exporter.export(Arrays.asList(fakeMetric)); + + CreateTimeSeriesRequest request = argumentCaptor.getValue(); + + assertThat(request.getTimeSeriesList()).hasSize(1); + + com.google.monitoring.v3.TimeSeries timeSeries = request.getTimeSeriesList().get(0); + + assertThat(timeSeries.getResource().getLabelsMap()) + .containsExactly( + ConsumerEnvironmentUtils.GCE_PROJECT_ID_LABEL, gceProjectId, + ConsumerEnvironmentUtils.GCE_INSTANCE_ID_LABEL, gceInstanceId, + ConsumerEnvironmentUtils.GCE_ZONE_LABEL, gceZone); + + assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(5); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.PROJECT_ID.getName(), bigtableProjectId); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.INSTANCE_ID.getName(), bigtableInstanceId); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.APP_PROFILE.getName(), appProfileId); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.CLIENT_NAME.getName(), clientName); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsKey(BuiltinMeasureConstants.CLIENT_UID.getName()); + + assertThat(timeSeries.getPoints(0).getValue().getDoubleValue()).isEqualTo(fakeValue); + } + + @Test + public void testTimeSeriesForMetricWithGkeResource() { + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); + + UnaryCallable mockCallable = mock(UnaryCallable.class); + when(mockMetricServiceStub.createServiceTimeSeriesCallable()).thenReturn(mockCallable); + when(mockCallable.call(argumentCaptor.capture())).thenReturn(Empty.getDefaultInstance()); + + ConsumerEnvironmentUtils.ResourceUtilsWrapper resourceUtilsWrapperMock = + Mockito.mock(ConsumerEnvironmentUtils.ResourceUtilsWrapper.class); + ConsumerEnvironmentUtils.setResourceUtilsWrapper(resourceUtilsWrapperMock); + + Mockito.when(resourceUtilsWrapperMock.detectResource()) + .thenReturn( + Resource.create( + ContainerResource.TYPE, + ImmutableMap.of( + CloudResource.PROVIDER_KEY, + CloudResource.PROVIDER_GCP, + CloudResource.ACCOUNT_ID_KEY, + gkeProjectId, + CloudResource.ZONE_KEY, + gkeLocation, + K8sResource.CLUSTER_NAME_KEY, + gkeClusterName, + K8sResource.NAMESPACE_NAME_KEY, + gkeNamespaceName, + K8sResource.POD_NAME_KEY, + gkePodName, + ContainerResource.NAME_KEY, + gkeContainerName))); + + double fakeValue = 10.0; + Metric fakeMetric = + Metric.create( + MetricDescriptor.create( + "bigtable.googleapis.com/internal/client/per_connection_error_count", + "description", + "ms", + MetricDescriptor.Type.CUMULATIVE_DOUBLE, + Arrays.asList( + LabelKey.create(BuiltinMeasureConstants.PROJECT_ID.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.INSTANCE_ID.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.APP_PROFILE.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.CLIENT_NAME.getName(), ""))), + Arrays.asList( + TimeSeries.create( + Arrays.asList( + LabelValue.create(bigtableProjectId), + LabelValue.create(bigtableInstanceId), + LabelValue.create(appProfileId), + LabelValue.create(clientName)), + Arrays.asList( + Point.create( + Value.doubleValue(fakeValue), + Timestamp.fromMillis(System.currentTimeMillis()))), + Timestamp.fromMillis(System.currentTimeMillis())))); + + exporter.export(Arrays.asList(fakeMetric)); + + CreateTimeSeriesRequest request = argumentCaptor.getValue(); + + assertThat(request.getTimeSeriesList()).hasSize(1); + + com.google.monitoring.v3.TimeSeries timeSeries = request.getTimeSeriesList().get(0); + + assertThat(timeSeries.getResource().getLabelsMap()) + .containsExactly( + ConsumerEnvironmentUtils.GKE_PROJECT_ID_LABEL, gkeProjectId, + ConsumerEnvironmentUtils.GKE_LOCATION_LABEL, gkeLocation, + ConsumerEnvironmentUtils.GKE_CLUSTER_NAME_LABEL, gkeClusterName, + ConsumerEnvironmentUtils.GKE_NAMESPACE_NAME_LABEL, gkeNamespaceName, + ConsumerEnvironmentUtils.GKE_POD_NAME_LABEL, gkePodName, + ConsumerEnvironmentUtils.GKE_CONTAINER_NAME_LABEL, gkeContainerName); + + assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(5); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.PROJECT_ID.getName(), bigtableProjectId); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.INSTANCE_ID.getName(), bigtableInstanceId); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.APP_PROFILE.getName(), appProfileId); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.CLIENT_NAME.getName(), clientName); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsKey(BuiltinMeasureConstants.CLIENT_UID.getName()); + + assertThat(timeSeries.getPoints(0).getValue().getDoubleValue()).isEqualTo(fakeValue); + } + private class FakeMetricServiceClient extends MetricServiceClient { protected FakeMetricServiceClient(MetricServiceStub stub) { From 4f7ce65eb7c804b09bb8410f4ff20a4d8d14adf4 Mon Sep 17 00:00:00 2001 From: Reza Karegar Date: Fri, 16 Feb 2024 16:40:25 -0500 Subject: [PATCH 2/6] Fix integration test failure by updating set of metrics. --- .../google/cloud/bigtable/stats/BuiltinViews.java | 13 +++++++++++-- .../google/cloud/bigtable/stats/StatsWrapper.java | 2 +- .../bigtable/stats/ITBuiltinViewConstantsTest.java | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java index cfa5af0b00..2b91ee60c3 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java @@ -36,14 +36,20 @@ public class BuiltinViews { BuiltinViewConstants.SERVER_LATENCIES_VIEW, BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, - BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, - BuiltinViewConstants.PER_CONNECTION_ERROR_COUNT_VIEW); + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW); + // We store views that don't use the Bigtable schema and need different tags in a separate set to + // simplify testing. + static final ImmutableSet NON_BIGTABLE_BUILTIN_VIEWS = + ImmutableSet.of(BuiltinViewConstants.PER_CONNECTION_ERROR_COUNT_VIEW); @VisibleForTesting void registerPrivateViews(ViewManager viewManager) { for (View view : BIGTABLE_BUILTIN_VIEWS) { viewManager.registerView(view); } + for (View view : NON_BIGTABLE_BUILTIN_VIEWS) { + viewManager.registerView(view); + } } public static void registerBigtableBuiltinViews() { @@ -51,5 +57,8 @@ public static void registerBigtableBuiltinViews() { for (View view : BIGTABLE_BUILTIN_VIEWS) { viewManager.registerView(view); } + for (View view : NON_BIGTABLE_BUILTIN_VIEWS) { + viewManager.registerView(view); + } } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java index 0a97b21d67..fc6a072d01 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java @@ -61,7 +61,7 @@ public static List getOperationLatencyViewTagValueStrings() { // the packaging step. Opencensus classes will be relocated when they are packaged but the // integration test files will not be. So the integration tests can't reference any transitive // dependencies that have been relocated. - static Map> getViewToTagMap() { + static Map> getBigtableViewToTagMap() { Map> map = new HashMap<>(); for (View view : BuiltinViews.BIGTABLE_BUILTIN_VIEWS) { List tagKeys = view.getColumns(); diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java index 929ee85f48..c2dcc2a602 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java @@ -27,7 +27,7 @@ public class ITBuiltinViewConstantsTest { @Test public void testBasicTagsExistForAllViews() { - Map> viewToTagMap = StatsWrapper.getViewToTagMap(); + Map> viewToTagMap = StatsWrapper.getBigtableViewToTagMap(); for (String view : viewToTagMap.keySet()) { assertWithMessage(view + " should have all basic tags") .that(viewToTagMap.get(view)) From f7fde8f27034a5c8f0ca5acc9182ad007014123d Mon Sep 17 00:00:00 2001 From: Reza Karegar Date: Tue, 20 Feb 2024 09:36:50 -0500 Subject: [PATCH 3/6] Refactor the creation of MonitoredResource. --- .../BigtableCreateTimeSeriesExporter.java | 23 +++-------- .../stats/BigtableStackdriverExportUtils.java | 41 +++++++++++-------- .../BigtableStackdriverStatsExporter.java | 26 ++---------- .../BigtableCreateTimeSeriesExporterTest.java | 8 +--- 4 files changed, 32 insertions(+), 66 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index a964d36e69..d056e20937 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -15,7 +15,6 @@ */ package com.google.cloud.bigtable.stats; -import com.google.api.MonitoredResource; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.monitoring.v3.CreateTimeSeriesRequest; import com.google.monitoring.v3.ProjectName; @@ -33,20 +32,13 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { private static final Logger logger = Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName()); private final MetricServiceClient metricServiceClient; - private final MonitoredResource bigtableMonitoredResource; - private final MonitoredResource gceMonitoredResource; - private final MonitoredResource gkeMonitoredResource; + // private final MonitoredResource bigtableMonitoredResource; + // private final MonitoredResource gceMonitoredResource; + // private final MonitoredResource gkeMonitoredResource; private final String clientId; - BigtableCreateTimeSeriesExporter( - MetricServiceClient metricServiceClient, - MonitoredResource bigtableMonitoredResource, - MonitoredResource gceMonitoredResource, - MonitoredResource gkeMonitoredResource) { + BigtableCreateTimeSeriesExporter(MetricServiceClient metricServiceClient) { this.metricServiceClient = metricServiceClient; - this.bigtableMonitoredResource = bigtableMonitoredResource; - this.gceMonitoredResource = gceMonitoredResource; - this.gkeMonitoredResource = gkeMonitoredResource; this.clientId = BigtableStackdriverExportUtils.getDefaultTaskValue(); } @@ -69,12 +61,7 @@ public void export(Collection metrics) { Collectors.mapping( timeSeries -> BigtableStackdriverExportUtils.convertTimeSeries( - metric.getMetricDescriptor(), - timeSeries, - clientId, - bigtableMonitoredResource, - gceMonitoredResource, - gkeMonitoredResource), + metric.getMetricDescriptor(), timeSeries, clientId), Collectors.toList()))); for (Map.Entry> entry : diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index f787552d9b..58b20bca2e 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -54,6 +54,9 @@ import javax.annotation.Nullable; class BigtableStackdriverExportUtils { + private static final String BIGTABLE_RESOURCE_TYPE = "bigtable_client_raw"; + private static final String GCE_RESOURCE_TYPE = "gce_instance"; + private static final String GKE_RESOURCE_TYPE = "k8s_container"; private static final Logger logger = Logger.getLogger(BigtableStackdriverExportUtils.class.getName()); @@ -105,38 +108,42 @@ class BigtableStackdriverExportUtils { LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client uid"); static boolean isBigtableTableMetric(MetricDescriptor metricDescriptor) { - return !metricDescriptor.getName().equals(PER_CONNECTION_ERROR_COUNT_VIEW.getName().asString()); + return metricDescriptor.getName().contains("bigtable") + && !metricDescriptor.getName().equals(PER_CONNECTION_ERROR_COUNT_VIEW.getName().asString()); } static boolean shouldExportMetric(MetricDescriptor metricDescriptor) { - return metricDescriptor.getName().contains("bigtable") - && (isBigtableTableMetric(metricDescriptor) - || ConsumerEnvironmentUtils.isEnvGce() - || ConsumerEnvironmentUtils.isEnvGke()); + return isBigtableTableMetric(metricDescriptor) + || (metricDescriptor.getName().equals(PER_CONNECTION_ERROR_COUNT_VIEW.getName().asString()) + && (ConsumerEnvironmentUtils.isEnvGce() || ConsumerEnvironmentUtils.isEnvGke())); } static com.google.monitoring.v3.TimeSeries convertTimeSeries( - MetricDescriptor metricDescriptor, - TimeSeries timeSeries, - String clientId, - MonitoredResource bigtableMonitoredResource, - MonitoredResource gceMonitoredResource, - MonitoredResource gkeMonitoredResource) { + MetricDescriptor metricDescriptor, TimeSeries timeSeries, String clientId) { Type metricType = metricDescriptor.getType(); com.google.monitoring.v3.TimeSeries.Builder builder; if (isBigtableTableMetric(metricDescriptor)) { builder = setupBuilderForBigtableResource( - metricDescriptor, bigtableMonitoredResource, timeSeries, clientId); + metricDescriptor, + MonitoredResource.newBuilder().setType(BIGTABLE_RESOURCE_TYPE), + timeSeries, + clientId); } else if (ConsumerEnvironmentUtils.isEnvGce()) { builder = setupBuilderForGceOrGKEResource( - metricDescriptor, gceMonitoredResource, timeSeries, clientId); + metricDescriptor, + MonitoredResource.newBuilder().setType(GCE_RESOURCE_TYPE), + timeSeries, + clientId); } else if (ConsumerEnvironmentUtils.isEnvGke()) { builder = setupBuilderForGceOrGKEResource( - metricDescriptor, gkeMonitoredResource, timeSeries, clientId); + metricDescriptor, + MonitoredResource.newBuilder().setType(GKE_RESOURCE_TYPE), + timeSeries, + clientId); } else { logger.warning( "Trying to export metric " @@ -155,12 +162,11 @@ static com.google.monitoring.v3.TimeSeries convertTimeSeries( private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForBigtableResource( MetricDescriptor metricDescriptor, - MonitoredResource bigtableMonitoredResource, + MonitoredResource.Builder monitoredResourceBuilder, TimeSeries timeSeries, String clientId) { List labelKeys = metricDescriptor.getLabelKeys(); String metricName = metricDescriptor.getName(); - MonitoredResource.Builder monitoredResourceBuilder = bigtableMonitoredResource.toBuilder(); List metricTagKeys = new ArrayList<>(); List metricTagValues = new ArrayList<>(); @@ -189,12 +195,11 @@ private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForBigtab private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForGceOrGKEResource( MetricDescriptor metricDescriptor, - MonitoredResource monitoredResource, + MonitoredResource.Builder monitoredResourceBuilder, TimeSeries timeSeries, String clientId) { List labelKeys = metricDescriptor.getLabelKeys(); String metricName = metricDescriptor.getName(); - MonitoredResource.Builder monitoredResourceBuilder = monitoredResource.toBuilder(); List metricTagKeys = new ArrayList<>(); List metricTagValues = new ArrayList<>(); diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 3a102135dc..c537e57a30 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -15,7 +15,6 @@ */ package com.google.cloud.bigtable.stats; -import com.google.api.MonitoredResource; import com.google.api.core.InternalApi; import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; @@ -43,9 +42,6 @@ public class BigtableStackdriverStatsExporter { // Default export interval is 1 minute private static final Duration EXPORT_INTERVAL = Duration.create(60, 0); - private static final String BIGTABLE_RESOURCE_TYPE = "bigtable_client_raw"; - private static final String GCE_RESOURCE_TYPE = "gce_instance"; - private static final String GKE_RESOURCE_TYPE = "k8s_container"; private static final String MONITORING_ENDPOINT = MoreObjects.firstNonNull( @@ -55,21 +51,13 @@ public class BigtableStackdriverStatsExporter { private final IntervalMetricReader intervalMetricReader; private BigtableStackdriverStatsExporter( - MetricServiceClient metricServiceClient, - Duration exportInterval, - MonitoredResource bigtableMonitoredResource, - MonitoredResource gceMonitoredResource, - MonitoredResource gkeMonitoredResource) { + MetricServiceClient metricServiceClient, Duration exportInterval) { IntervalMetricReader.Options.Builder intervalMetricReaderOptionsBuilder = IntervalMetricReader.Options.builder(); intervalMetricReaderOptionsBuilder.setExportInterval(exportInterval); this.intervalMetricReader = IntervalMetricReader.create( - new BigtableCreateTimeSeriesExporter( - metricServiceClient, - bigtableMonitoredResource, - gceMonitoredResource, - gkeMonitoredResource), + new BigtableCreateTimeSeriesExporter(metricServiceClient), MetricReader.create( MetricReader.Options.builder() .setMetricProducerManager( @@ -84,15 +72,7 @@ public static void register(Credentials credentials) throws IOException { instance == null, "Bigtable Stackdriver stats exporter is already created"); // Default timeout for creating a client is 1 minute MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); - MonitoredResource bigtableResourceType = - MonitoredResource.newBuilder().setType(BIGTABLE_RESOURCE_TYPE).build(); - MonitoredResource gceResourceType = - MonitoredResource.newBuilder().setType(GCE_RESOURCE_TYPE).build(); - MonitoredResource gkeResourceType = - MonitoredResource.newBuilder().setType(GKE_RESOURCE_TYPE).build(); - instance = - new BigtableStackdriverStatsExporter( - client, EXPORT_INTERVAL, bigtableResourceType, gceResourceType, gkeResourceType); + instance = new BigtableStackdriverStatsExporter(client, EXPORT_INTERVAL); } } diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index 47bba056cc..f6c91d9d4c 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -19,7 +19,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.google.api.MonitoredResource; import com.google.api.gax.rpc.UnaryCallable; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.cloud.monitoring.v3.stub.MetricServiceStub; @@ -85,12 +84,7 @@ public void setUp() { fakeMetricServiceClient = new FakeMetricServiceClient(mockMetricServiceStub); - exporter = - new BigtableCreateTimeSeriesExporter( - fakeMetricServiceClient, - MonitoredResource.newBuilder().setType("bigtable-table").build(), - MonitoredResource.newBuilder().setType("gce-instance").build(), - MonitoredResource.newBuilder().setType("gke-container").build()); + exporter = new BigtableCreateTimeSeriesExporter(fakeMetricServiceClient); } @After From f6559ae0868b3fc5b2490094b4d1d7a28d2d9097 Mon Sep 17 00:00:00 2001 From: Reza Karegar Date: Tue, 20 Feb 2024 12:26:44 -0500 Subject: [PATCH 4/6] Use StackdriverStatsConfiguration to get GCE/GKE monitoring resource. --- .../BigtableCreateTimeSeriesExporter.java | 14 ++-- .../stats/BigtableStackdriverExportUtils.java | 42 +++++------ .../BigtableStackdriverStatsExporter.java | 16 ++++- .../stats/ConsumerEnvironmentUtils.java | 42 ----------- .../BigtableCreateTimeSeriesExporterTest.java | 71 ++++++------------- 5 files changed, 62 insertions(+), 123 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index d056e20937..e8b95e2ab0 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.stats; +import com.google.api.MonitoredResource; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.monitoring.v3.CreateTimeSeriesRequest; import com.google.monitoring.v3.ProjectName; @@ -32,14 +33,14 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { private static final Logger logger = Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName()); private final MetricServiceClient metricServiceClient; - // private final MonitoredResource bigtableMonitoredResource; - // private final MonitoredResource gceMonitoredResource; - // private final MonitoredResource gkeMonitoredResource; + private final MonitoredResource gceOrGkeMonitoredResource; private final String clientId; - BigtableCreateTimeSeriesExporter(MetricServiceClient metricServiceClient) { + BigtableCreateTimeSeriesExporter( + MetricServiceClient metricServiceClient, MonitoredResource gceOrGkeMonitoredResource) { this.metricServiceClient = metricServiceClient; this.clientId = BigtableStackdriverExportUtils.getDefaultTaskValue(); + this.gceOrGkeMonitoredResource = gceOrGkeMonitoredResource; } public void export(Collection metrics) { @@ -61,7 +62,10 @@ public void export(Collection metrics) { Collectors.mapping( timeSeries -> BigtableStackdriverExportUtils.convertTimeSeries( - metric.getMetricDescriptor(), timeSeries, clientId), + metric.getMetricDescriptor(), + timeSeries, + clientId, + gceOrGkeMonitoredResource), Collectors.toList()))); for (Map.Entry> entry : diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 58b20bca2e..ffd1be47a1 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -47,6 +47,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.logging.Level; @@ -55,8 +56,8 @@ class BigtableStackdriverExportUtils { private static final String BIGTABLE_RESOURCE_TYPE = "bigtable_client_raw"; - private static final String GCE_RESOURCE_TYPE = "gce_instance"; - private static final String GKE_RESOURCE_TYPE = "k8s_container"; + static final String GCE_RESOURCE_TYPE = "gce_instance"; + static final String GKE_RESOURCE_TYPE = "k8s_container"; private static final Logger logger = Logger.getLogger(BigtableStackdriverExportUtils.class.getName()); @@ -119,7 +120,10 @@ static boolean shouldExportMetric(MetricDescriptor metricDescriptor) { } static com.google.monitoring.v3.TimeSeries convertTimeSeries( - MetricDescriptor metricDescriptor, TimeSeries timeSeries, String clientId) { + MetricDescriptor metricDescriptor, + TimeSeries timeSeries, + String clientId, + MonitoredResource gceOrGkeMonitoredResource) { Type metricType = metricDescriptor.getType(); com.google.monitoring.v3.TimeSeries.Builder builder; @@ -130,20 +134,10 @@ static com.google.monitoring.v3.TimeSeries convertTimeSeries( MonitoredResource.newBuilder().setType(BIGTABLE_RESOURCE_TYPE), timeSeries, clientId); - } else if (ConsumerEnvironmentUtils.isEnvGce()) { + } else if (ConsumerEnvironmentUtils.isEnvGce() || ConsumerEnvironmentUtils.isEnvGke()) { builder = setupBuilderForGceOrGKEResource( - metricDescriptor, - MonitoredResource.newBuilder().setType(GCE_RESOURCE_TYPE), - timeSeries, - clientId); - } else if (ConsumerEnvironmentUtils.isEnvGke()) { - builder = - setupBuilderForGceOrGKEResource( - metricDescriptor, - MonitoredResource.newBuilder().setType(GKE_RESOURCE_TYPE), - timeSeries, - clientId); + metricDescriptor, gceOrGkeMonitoredResource, timeSeries, clientId); } else { logger.warning( "Trying to export metric " @@ -195,7 +189,7 @@ private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForBigtab private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForGceOrGKEResource( MetricDescriptor metricDescriptor, - MonitoredResource.Builder monitoredResourceBuilder, + MonitoredResource gceOrGkeMonitoredResource, TimeSeries timeSeries, String clientId) { List labelKeys = metricDescriptor.getLabelKeys(); @@ -211,15 +205,17 @@ private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForGceOrG metricTagKeys.add(CLIENT_UID_LABEL_KEY); metricTagValues.add(LabelValue.create(clientId)); - if (ConsumerEnvironmentUtils.isEnvGce()) { - ConsumerEnvironmentUtils.putGceResourceLabels(monitoredResourceBuilder); - } else { - ConsumerEnvironmentUtils.putGkeResourceLabels(monitoredResourceBuilder); - } - com.google.monitoring.v3.TimeSeries.Builder builder = com.google.monitoring.v3.TimeSeries.newBuilder(); - builder.setResource(monitoredResourceBuilder.build()); + if (gceOrGkeMonitoredResource == null + || (!Objects.equals(gceOrGkeMonitoredResource.getType(), GCE_RESOURCE_TYPE) + && !Objects.equals(gceOrGkeMonitoredResource.getType(), GKE_RESOURCE_TYPE))) { + logger.warning( + "MonitoredResource is expected to correspond to GCE or GKE, but was " + + gceOrGkeMonitoredResource + + " instead."); + } + builder.setResource(gceOrGkeMonitoredResource); builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues)); return builder; diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index c537e57a30..856353cfd0 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.stats; +import com.google.api.MonitoredResource; import com.google.api.core.InternalApi; import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; @@ -27,6 +28,7 @@ import io.opencensus.common.Duration; import io.opencensus.exporter.metrics.util.IntervalMetricReader; import io.opencensus.exporter.metrics.util.MetricReader; +import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration; import io.opencensus.metrics.Metrics; import java.io.IOException; import javax.annotation.Nullable; @@ -51,13 +53,15 @@ public class BigtableStackdriverStatsExporter { private final IntervalMetricReader intervalMetricReader; private BigtableStackdriverStatsExporter( - MetricServiceClient metricServiceClient, Duration exportInterval) { + MetricServiceClient metricServiceClient, + Duration exportInterval, + MonitoredResource gceOrGkeMonitoredResource) { IntervalMetricReader.Options.Builder intervalMetricReaderOptionsBuilder = IntervalMetricReader.Options.builder(); intervalMetricReaderOptionsBuilder.setExportInterval(exportInterval); this.intervalMetricReader = IntervalMetricReader.create( - new BigtableCreateTimeSeriesExporter(metricServiceClient), + new BigtableCreateTimeSeriesExporter(metricServiceClient, gceOrGkeMonitoredResource), MetricReader.create( MetricReader.Options.builder() .setMetricProducerManager( @@ -72,7 +76,13 @@ public static void register(Credentials credentials) throws IOException { instance == null, "Bigtable Stackdriver stats exporter is already created"); // Default timeout for creating a client is 1 minute MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); - instance = new BigtableStackdriverStatsExporter(client, EXPORT_INTERVAL); + MonitoredResource gceOrGkeMonitoredResource = null; + if (ConsumerEnvironmentUtils.isEnvGce() || ConsumerEnvironmentUtils.isEnvGke()) { + gceOrGkeMonitoredResource = + StackdriverStatsConfiguration.builder().build().getMonitoredResource(); + } + instance = + new BigtableStackdriverStatsExporter(client, EXPORT_INTERVAL, gceOrGkeMonitoredResource); } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java index ba17dc2168..6eeaa73265 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java @@ -15,12 +15,10 @@ */ package com.google.cloud.bigtable.stats; -import com.google.api.MonitoredResource; import com.google.common.annotations.VisibleForTesting; import io.opencensus.contrib.resource.util.CloudResource; import io.opencensus.contrib.resource.util.ContainerResource; import io.opencensus.contrib.resource.util.HostResource; -import io.opencensus.contrib.resource.util.K8sResource; import io.opencensus.contrib.resource.util.ResourceUtils; import io.opencensus.resource.Resource; import java.util.Objects; @@ -29,16 +27,6 @@ class ConsumerEnvironmentUtils { private static ResourceUtilsWrapper resourceUtilsWrapper = new ResourceUtilsWrapper(); - static final String GCE_PROJECT_ID_LABEL = "project_id"; - static final String GCE_INSTANCE_ID_LABEL = "instance_id"; - static final String GCE_ZONE_LABEL = "zone"; - - static final String GKE_PROJECT_ID_LABEL = "project_id"; - static final String GKE_LOCATION_LABEL = "location"; - static final String GKE_CLUSTER_NAME_LABEL = "cluster_name"; - static final String GKE_NAMESPACE_NAME_LABEL = "namespace_name"; - static final String GKE_POD_NAME_LABEL = "pod_name"; - static final String GKE_CONTAINER_NAME_LABEL = "container_name"; @VisibleForTesting public static void setResourceUtilsWrapper(ResourceUtilsWrapper newResourceUtilsWrapper) { @@ -59,36 +47,6 @@ public static boolean isEnvGke() { resource.getLabels().get(CloudResource.PROVIDER_KEY), CloudResource.PROVIDER_GCP); } - public static void putGceResourceLabels(MonitoredResource.Builder builder) { - Resource resource = resourceUtilsWrapper.detectResource(); - if (isEnvGce()) { - builder.putLabels( - GCE_PROJECT_ID_LABEL, resource.getLabels().get(CloudResource.ACCOUNT_ID_KEY)); - builder.putLabels(GCE_INSTANCE_ID_LABEL, resource.getLabels().get(HostResource.ID_KEY)); - builder.putLabels(GCE_ZONE_LABEL, resource.getLabels().get(CloudResource.ZONE_KEY)); - } else { - throw new IllegalStateException("Can only put GCE labels inside a GCE environment."); - } - } - - public static void putGkeResourceLabels(MonitoredResource.Builder builder) { - Resource resource = resourceUtilsWrapper.detectResource(); - if (isEnvGke()) { - builder.putLabels( - GKE_PROJECT_ID_LABEL, resource.getLabels().get(CloudResource.ACCOUNT_ID_KEY)); - builder.putLabels(GKE_LOCATION_LABEL, resource.getLabels().get(CloudResource.ZONE_KEY)); - builder.putLabels( - GKE_CLUSTER_NAME_LABEL, resource.getLabels().get(K8sResource.CLUSTER_NAME_KEY)); - builder.putLabels( - GKE_NAMESPACE_NAME_LABEL, resource.getLabels().get(K8sResource.NAMESPACE_NAME_KEY)); - builder.putLabels(GKE_POD_NAME_LABEL, resource.getLabels().get(K8sResource.POD_NAME_KEY)); - builder.putLabels( - GKE_CONTAINER_NAME_LABEL, resource.getLabels().get(ContainerResource.NAME_KEY)); - } else { - throw new IllegalStateException("Can only put GKE labels inside a GKE environment."); - } - } - // We wrap the static ResourceUtils.detectResource() method in a non-static method for mocking. @VisibleForTesting public static class ResourceUtilsWrapper { diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index f6c91d9d4c..61f5169afb 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -19,6 +19,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.google.api.MonitoredResource; import com.google.api.gax.rpc.UnaryCallable; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.cloud.monitoring.v3.stub.MetricServiceStub; @@ -29,7 +30,6 @@ import io.opencensus.contrib.resource.util.CloudResource; import io.opencensus.contrib.resource.util.ContainerResource; import io.opencensus.contrib.resource.util.HostResource; -import io.opencensus.contrib.resource.util.K8sResource; import io.opencensus.metrics.LabelKey; import io.opencensus.metrics.LabelValue; import io.opencensus.metrics.export.Metric; @@ -62,29 +62,15 @@ public class BigtableCreateTimeSeriesExporterTest { private static final String bigtableCluster = "cluster-1"; private static final String clientName = "client-name"; - private static final String gceProjectId = "fake-gce-project"; - private static final String gceInstanceId = "fake-gce-instance"; - private static final String gceZone = "fake-gce-zone"; - - private static final String gkeProjectId = "fake-gke-project"; - private static final String gkeLocation = "fake-gke-location"; - private static final String gkeClusterName = "fake-gke-cluster"; - private static final String gkeNamespaceName = "fake-gke-namespace"; - private static final String gkePodName = "fake-gke-pod"; - private static final String gkeContainerName = "fake-gke-container"; - @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); @Mock private MetricServiceStub mockMetricServiceStub; private MetricServiceClient fakeMetricServiceClient; - private BigtableCreateTimeSeriesExporter exporter; @Before public void setUp() { fakeMetricServiceClient = new FakeMetricServiceClient(mockMetricServiceStub); - - exporter = new BigtableCreateTimeSeriesExporter(fakeMetricServiceClient); } @After @@ -92,6 +78,8 @@ public void tearDown() {} @Test public void testTimeSeriesForMetricWithBigtableResource() { + BigtableCreateTimeSeriesExporter exporter = + new BigtableCreateTimeSeriesExporter(fakeMetricServiceClient, null); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); @@ -156,6 +144,13 @@ public void testTimeSeriesForMetricWithBigtableResource() { @Test public void testTimeSeriesForMetricWithGceResource() { + BigtableCreateTimeSeriesExporter exporter = + new BigtableCreateTimeSeriesExporter( + fakeMetricServiceClient, + MonitoredResource.newBuilder() + .setType(BigtableStackdriverExportUtils.GCE_RESOURCE_TYPE) + .putLabels("some-gce-key", "some-gce-value") + .build()); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); @@ -170,15 +165,7 @@ public void testTimeSeriesForMetricWithGceResource() { .thenReturn( Resource.create( HostResource.TYPE, - ImmutableMap.of( - CloudResource.PROVIDER_KEY, - CloudResource.PROVIDER_GCP, - CloudResource.ACCOUNT_ID_KEY, - gceProjectId, - HostResource.ID_KEY, - gceInstanceId, - CloudResource.ZONE_KEY, - gceZone))); + ImmutableMap.of(CloudResource.PROVIDER_KEY, CloudResource.PROVIDER_GCP))); double fakeValue = 10.0; Metric fakeMetric = @@ -215,10 +202,7 @@ public void testTimeSeriesForMetricWithGceResource() { com.google.monitoring.v3.TimeSeries timeSeries = request.getTimeSeriesList().get(0); assertThat(timeSeries.getResource().getLabelsMap()) - .containsExactly( - ConsumerEnvironmentUtils.GCE_PROJECT_ID_LABEL, gceProjectId, - ConsumerEnvironmentUtils.GCE_INSTANCE_ID_LABEL, gceInstanceId, - ConsumerEnvironmentUtils.GCE_ZONE_LABEL, gceZone); + .containsExactly("some-gce-key", "some-gce-value"); assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(5); assertThat(timeSeries.getMetric().getLabelsMap()) @@ -237,6 +221,13 @@ public void testTimeSeriesForMetricWithGceResource() { @Test public void testTimeSeriesForMetricWithGkeResource() { + BigtableCreateTimeSeriesExporter exporter = + new BigtableCreateTimeSeriesExporter( + fakeMetricServiceClient, + MonitoredResource.newBuilder() + .setType(BigtableStackdriverExportUtils.GKE_RESOURCE_TYPE) + .putLabels("some-gke-key", "some-gke-value") + .build()); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); @@ -252,21 +243,7 @@ public void testTimeSeriesForMetricWithGkeResource() { .thenReturn( Resource.create( ContainerResource.TYPE, - ImmutableMap.of( - CloudResource.PROVIDER_KEY, - CloudResource.PROVIDER_GCP, - CloudResource.ACCOUNT_ID_KEY, - gkeProjectId, - CloudResource.ZONE_KEY, - gkeLocation, - K8sResource.CLUSTER_NAME_KEY, - gkeClusterName, - K8sResource.NAMESPACE_NAME_KEY, - gkeNamespaceName, - K8sResource.POD_NAME_KEY, - gkePodName, - ContainerResource.NAME_KEY, - gkeContainerName))); + ImmutableMap.of(CloudResource.PROVIDER_KEY, CloudResource.PROVIDER_GCP))); double fakeValue = 10.0; Metric fakeMetric = @@ -303,13 +280,7 @@ public void testTimeSeriesForMetricWithGkeResource() { com.google.monitoring.v3.TimeSeries timeSeries = request.getTimeSeriesList().get(0); assertThat(timeSeries.getResource().getLabelsMap()) - .containsExactly( - ConsumerEnvironmentUtils.GKE_PROJECT_ID_LABEL, gkeProjectId, - ConsumerEnvironmentUtils.GKE_LOCATION_LABEL, gkeLocation, - ConsumerEnvironmentUtils.GKE_CLUSTER_NAME_LABEL, gkeClusterName, - ConsumerEnvironmentUtils.GKE_NAMESPACE_NAME_LABEL, gkeNamespaceName, - ConsumerEnvironmentUtils.GKE_POD_NAME_LABEL, gkePodName, - ConsumerEnvironmentUtils.GKE_CONTAINER_NAME_LABEL, gkeContainerName); + .containsExactly("some-gke-key", "some-gke-value"); assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(5); assertThat(timeSeries.getMetric().getLabelsMap()) From 3392175168d7c49108bf6590772dc9a4a0874c5e Mon Sep 17 00:00:00 2001 From: Reza Karegar Date: Tue, 20 Feb 2024 12:43:26 -0500 Subject: [PATCH 5/6] Use preconditions to check null values. --- .../stats/BigtableStackdriverExportUtils.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index ffd1be47a1..061b808211 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -22,6 +22,7 @@ import com.google.api.Metric; import com.google.api.MetricDescriptor.MetricKind; import com.google.api.MonitoredResource; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.monitoring.v3.TimeInterval; @@ -192,6 +193,14 @@ private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForGceOrG MonitoredResource gceOrGkeMonitoredResource, TimeSeries timeSeries, String clientId) { + Preconditions.checkNotNull(gceOrGkeMonitoredResource); + if (!Objects.equals(gceOrGkeMonitoredResource.getType(), GCE_RESOURCE_TYPE) + && !Objects.equals(gceOrGkeMonitoredResource.getType(), GKE_RESOURCE_TYPE)) { + logger.warning( + "MonitoredResource is expected to correspond to GCE or GKE, but was " + + gceOrGkeMonitoredResource + + " instead."); + } List labelKeys = metricDescriptor.getLabelKeys(); String metricName = metricDescriptor.getName(); List metricTagKeys = new ArrayList<>(); @@ -207,14 +216,6 @@ private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForGceOrG com.google.monitoring.v3.TimeSeries.Builder builder = com.google.monitoring.v3.TimeSeries.newBuilder(); - if (gceOrGkeMonitoredResource == null - || (!Objects.equals(gceOrGkeMonitoredResource.getType(), GCE_RESOURCE_TYPE) - && !Objects.equals(gceOrGkeMonitoredResource.getType(), GKE_RESOURCE_TYPE))) { - logger.warning( - "MonitoredResource is expected to correspond to GCE or GKE, but was " - + gceOrGkeMonitoredResource - + " instead."); - } builder.setResource(gceOrGkeMonitoredResource); builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues)); From 3c6cf84fd4581332fdca48162990ad8a443bc44e Mon Sep 17 00:00:00 2001 From: Reza Karegar Date: Tue, 20 Feb 2024 13:00:12 -0500 Subject: [PATCH 6/6] Remove unnecesary checks. --- .../stats/BigtableStackdriverExportUtils.java | 13 ------------- .../stats/BigtableCreateTimeSeriesExporterTest.java | 4 ++-- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 061b808211..e7591f0062 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -22,7 +22,6 @@ import com.google.api.Metric; import com.google.api.MetricDescriptor.MetricKind; import com.google.api.MonitoredResource; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.monitoring.v3.TimeInterval; @@ -48,7 +47,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.logging.Level; @@ -57,9 +55,6 @@ class BigtableStackdriverExportUtils { private static final String BIGTABLE_RESOURCE_TYPE = "bigtable_client_raw"; - static final String GCE_RESOURCE_TYPE = "gce_instance"; - static final String GKE_RESOURCE_TYPE = "k8s_container"; - private static final Logger logger = Logger.getLogger(BigtableStackdriverExportUtils.class.getName()); @@ -193,14 +188,6 @@ private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForGceOrG MonitoredResource gceOrGkeMonitoredResource, TimeSeries timeSeries, String clientId) { - Preconditions.checkNotNull(gceOrGkeMonitoredResource); - if (!Objects.equals(gceOrGkeMonitoredResource.getType(), GCE_RESOURCE_TYPE) - && !Objects.equals(gceOrGkeMonitoredResource.getType(), GKE_RESOURCE_TYPE)) { - logger.warning( - "MonitoredResource is expected to correspond to GCE or GKE, but was " - + gceOrGkeMonitoredResource - + " instead."); - } List labelKeys = metricDescriptor.getLabelKeys(); String metricName = metricDescriptor.getName(); List metricTagKeys = new ArrayList<>(); diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index 61f5169afb..a0fb2ca0d1 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -148,7 +148,7 @@ public void testTimeSeriesForMetricWithGceResource() { new BigtableCreateTimeSeriesExporter( fakeMetricServiceClient, MonitoredResource.newBuilder() - .setType(BigtableStackdriverExportUtils.GCE_RESOURCE_TYPE) + .setType("gce-instance") .putLabels("some-gce-key", "some-gce-value") .build()); ArgumentCaptor argumentCaptor = @@ -225,7 +225,7 @@ public void testTimeSeriesForMetricWithGkeResource() { new BigtableCreateTimeSeriesExporter( fakeMetricServiceClient, MonitoredResource.newBuilder() - .setType(BigtableStackdriverExportUtils.GKE_RESOURCE_TYPE) + .setType("gke-container") .putLabels("some-gke-key", "some-gke-value") .build()); ArgumentCaptor argumentCaptor =