diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 487cfdb216..98f3b1a7de 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -19,6 +19,8 @@ branchProtectionRules: - 'Kokoro - Test: Java 17 GraalVM Native Image' - javadoc - conformance + - library_generation + - unmanaged_dependency_check - pattern: 1.22.0-sp isAdminEnforced: true requiredApprovingReviewCount: 1 diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 605555ecae..f5f585bcd0 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -33,6 +33,7 @@ retry_with_backoff 3 10 \ -DskipTests=true \ -Dclirr.skip=true \ -Denforcer.skip=true \ + -Dcheckstyle.skip=true \ -Dmaven.javadoc.skip=true \ -Dgcloud.download.skip=true \ -T 1C @@ -66,7 +67,8 @@ integration) -DtrimStackTrace=false \ -Dclirr.skip=true \ -Denforcer.skip=true \ - -fae \ + -Dcheckstyle.skip=true \ + -DskipUnitTests=true \ verify RETURN_CODE=$? ;; diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a450e7cae..974ce8dd2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [2.49.0](https://github.com/googleapis/java-bigtable/compare/v2.48.0...v2.49.0) (2024-12-03) + + +### Features + +* Add support for table deletion protection ([#2430](https://github.com/googleapis/java-bigtable/issues/2430)) ([687b6df](https://github.com/googleapis/java-bigtable/commit/687b6df14b743358e8207cda26022dfc75338d55)) + + +### Bug Fixes + +* Allow factory to export to different projects ([#2374](https://github.com/googleapis/java-bigtable/issues/2374)) ([06b912c](https://github.com/googleapis/java-bigtable/commit/06b912cc5d63436757008e79edfa8286b2ccac18)) +* Send priming requests on the channel directly ([#2435](https://github.com/googleapis/java-bigtable/issues/2435)) ([b76698d](https://github.com/googleapis/java-bigtable/commit/b76698dfb2c8552185f34e01e924ecc80798ba4f)) + ## [2.48.0](https://github.com/googleapis/java-bigtable/compare/v2.47.0...v2.48.0) (2024-11-19) diff --git a/README.md b/README.md index d114930c70..837665a4aa 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-bigtable' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigtable:2.48.0' +implementation 'com.google.cloud:google-cloud-bigtable:2.49.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.48.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.49.0" ``` ## Authentication @@ -543,7 +543,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigtable/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigtable.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigtable/2.48.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigtable/2.49.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/generation_config.yaml b/generation_config.yaml index b88f583921..c885f1fb10 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,5 +1,5 @@ gapic_generator_version: 2.50.0 -googleapis_commitish: c6eb517e76204c0cd71ab298c165eebbf12682eb +googleapis_commitish: 349841abac6c3e580ccce6e3d6fcc182ed2512c2 libraries_bom_version: 26.50.0 template_excludes: - .gitignore diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index ac89a56f9d..e241a11534 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigtable-bom - 2.48.0 + 2.49.0 pom com.google.cloud @@ -63,37 +63,37 @@ com.google.cloud google-cloud-bigtable - 2.48.0 + 2.49.0 com.google.cloud google-cloud-bigtable-emulator - 0.185.0 + 0.186.0 com.google.cloud google-cloud-bigtable-emulator-core - 0.185.0 + 0.186.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.48.0 + 2.49.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.48.0 + 2.49.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.48.0 + 2.49.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.48.0 + 2.49.0 diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index 42f0b0ea7b..8e59114a52 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -13,7 +13,7 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.48.0 + 2.49.0 pom diff --git a/google-cloud-bigtable-emulator-core/pom.xml b/google-cloud-bigtable-emulator-core/pom.xml index 7fe8685e60..e320a64d24 100644 --- a/google-cloud-bigtable-emulator-core/pom.xml +++ b/google-cloud-bigtable-emulator-core/pom.xml @@ -7,11 +7,11 @@ google-cloud-bigtable-parent com.google.cloud - 2.48.0 + 2.49.0 google-cloud-bigtable-emulator-core - 0.185.0 + 0.186.0 A Java wrapper for the Cloud Bigtable emulator. diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml index 326f8d1ce7..ec853bf2a9 100644 --- a/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-bigtable-emulator/pom.xml @@ -5,7 +5,7 @@ 4.0.0 google-cloud-bigtable-emulator - 0.185.0 + 0.186.0 Google Cloud Java - Bigtable Emulator https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-bigtable-parent - 2.48.0 + 2.49.0 scm:git:git@github.com:googleapis/java-bigtable.git @@ -81,14 +81,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.48.0 + 2.49.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.48.0 + 2.49.0 pom import @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigtable-emulator-core - 0.185.0 + 0.186.0 diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 553f06b9ea..b4be0a9ef5 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-bigtable - 2.48.0 + 2.49.0 jar Google Cloud Bigtable https://github.com/googleapis/java-bigtable @@ -12,11 +12,11 @@ com.google.cloud google-cloud-bigtable-parent - 2.48.0 + 2.49.0 - 2.48.0 + 2.49.0 google-cloud-bigtable @@ -52,14 +52,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.48.0 + 2.49.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.48.0 + 2.49.0 pom import @@ -709,7 +709,6 @@ grpc-auth is not directly used transitively, but is pulled to align with other grpc parts opencensus-impl-core is brought in transitively through opencensus-impl --> - io.grpc:grpc-auth io.opencensus:opencensus-impl-core diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java index 3cb5343804..21bdfd5e09 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java @@ -20,6 +20,6 @@ @InternalApi("For internal use only") public final class Version { // {x-version-update-start:google-cloud-bigtable:current} - public static String VERSION = "2.48.0"; + public static String VERSION = "2.49.0"; // {x-version-update-end} } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequest.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequest.java index 0fbffcb190..c7a0580fde 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequest.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequest.java @@ -123,6 +123,12 @@ public CreateTableRequest addChangeStreamRetention(Duration retention) { return this; } + /** Configures if the table is deletion protected. */ + public CreateTableRequest setDeletionProtection(boolean deletionProtection) { + requestBuilder.getTableBuilder().setDeletionProtection(deletionProtection); + return this; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/Table.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/Table.java index 31aa612f18..979e01cb8c 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/Table.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/Table.java @@ -105,6 +105,7 @@ public com.google.bigtable.admin.v2.Table.ClusterState.ReplicationState toProto( private final List columnFamilies; private final Duration changeStreamRetention; + private final boolean deletionProtection; @InternalApi public static Table fromProto(@Nonnull com.google.bigtable.admin.v2.Table proto) { @@ -135,19 +136,22 @@ public static Table fromProto(@Nonnull com.google.bigtable.admin.v2.Table proto) TableName.parse(proto.getName()), replicationStates.build(), columnFamilies.build(), - changeStreamConfig); + changeStreamConfig, + proto.getDeletionProtection()); } private Table( TableName tableName, Map replicationStatesByClusterId, List columnFamilies, - Duration changeStreamRetention) { + Duration changeStreamRetention, + boolean deletionProtection) { this.instanceId = tableName.getInstance(); this.id = tableName.getTable(); this.replicationStatesByClusterId = replicationStatesByClusterId; this.columnFamilies = columnFamilies; this.changeStreamRetention = changeStreamRetention; + this.deletionProtection = deletionProtection; } /** Gets the table's id. */ @@ -172,6 +176,11 @@ public Duration getChangeStreamRetention() { return changeStreamRetention; } + /** Returns whether this table is deletion protected. */ + public boolean isDeletionProtected() { + return deletionProtection; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -185,12 +194,18 @@ public boolean equals(Object o) { && Objects.equal(instanceId, table.instanceId) && Objects.equal(replicationStatesByClusterId, table.replicationStatesByClusterId) && Objects.equal(columnFamilies, table.columnFamilies) - && Objects.equal(changeStreamRetention, table.changeStreamRetention); + && Objects.equal(changeStreamRetention, table.changeStreamRetention) + && Objects.equal(deletionProtection, table.deletionProtection); } @Override public int hashCode() { return Objects.hashCode( - id, instanceId, replicationStatesByClusterId, columnFamilies, changeStreamRetention); + id, + instanceId, + replicationStatesByClusterId, + columnFamilies, + changeStreamRetention, + deletionProtection); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequest.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequest.java index 034736aa56..4e78051864 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequest.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequest.java @@ -74,6 +74,13 @@ public UpdateTableRequest disableChangeStreamRetention() { return addChangeStreamRetention(Duration.ZERO); } + /** Changes the deletion protection of an existing table. */ + public UpdateTableRequest setDeletionProtection(boolean deletionProtection) { + requestBuilder.getTableBuilder().setDeletionProtection(deletionProtection); + requestBuilder.getUpdateMaskBuilder().addPaths("deletion_protection"); + return this; + } + @InternalApi public com.google.bigtable.admin.v2.UpdateTableRequest toProto( String projectId, String instanceId) { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java index 34ec77bdfc..359d0ff8aa 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java @@ -16,13 +16,10 @@ package com.google.cloud.bigtable.data.v2; import com.google.api.core.BetaApi; -import com.google.api.gax.core.BackgroundResource; import com.google.api.gax.rpc.ClientContext; +import com.google.cloud.bigtable.data.v2.stub.BigtableClientContext; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub; -import io.opentelemetry.api.OpenTelemetry; import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.annotation.Nonnull; /** @@ -66,11 +63,8 @@ @BetaApi("This feature is currently experimental and can change in the future") public final class BigtableDataClientFactory implements AutoCloseable { - private static final Logger logger = Logger.getLogger(BigtableDataClientFactory.class.getName()); - private final BigtableDataSettings defaultSettings; - private final ClientContext sharedClientContext; - private final OpenTelemetry openTelemetry; + private final BigtableClientContext sharedClientContext; /** * Create a instance of this factory. @@ -80,31 +74,16 @@ public final class BigtableDataClientFactory implements AutoCloseable { */ public static BigtableDataClientFactory create(BigtableDataSettings defaultSettings) throws IOException { - ClientContext sharedClientContext = - EnhancedBigtableStub.createClientContext(defaultSettings.getStubSettings()); - OpenTelemetry openTelemetry = null; - try { - // We don't want client side metrics to crash the client, so catch any exception when getting - // the OTEL instance and log the exception instead. - openTelemetry = - EnhancedBigtableStub.getOpenTelemetry( - defaultSettings.getProjectId(), - defaultSettings.getMetricsProvider(), - sharedClientContext.getCredentials(), - defaultSettings.getStubSettings().getMetricsEndpoint()); - } catch (Throwable t) { - logger.log(Level.WARNING, "Failed to get OTEL, will skip exporting client side metrics", t); - } - return new BigtableDataClientFactory(sharedClientContext, defaultSettings, openTelemetry); + BigtableClientContext sharedClientContext = + EnhancedBigtableStub.createBigtableClientContext(defaultSettings.getStubSettings()); + + return new BigtableDataClientFactory(sharedClientContext, defaultSettings); } private BigtableDataClientFactory( - ClientContext sharedClientContext, - BigtableDataSettings defaultSettings, - OpenTelemetry openTelemetry) { + BigtableClientContext sharedClientContext, BigtableDataSettings defaultSettings) { this.sharedClientContext = sharedClientContext; this.defaultSettings = defaultSettings; - this.openTelemetry = openTelemetry; } /** @@ -114,9 +93,7 @@ private BigtableDataClientFactory( */ @Override public void close() throws Exception { - for (BackgroundResource resource : sharedClientContext.getBackgroundResources()) { - resource.close(); - } + sharedClientContext.close(); } /** @@ -132,10 +109,11 @@ public BigtableDataClient createDefault() { try { ClientContext clientContext = sharedClientContext + .getClientContext() .toBuilder() .setTracerFactory( EnhancedBigtableStub.createBigtableTracerFactory( - defaultSettings.getStubSettings(), openTelemetry)) + defaultSettings.getStubSettings(), sharedClientContext.getOpenTelemetry())) .build(); return BigtableDataClient.createWithClientContext(defaultSettings, clientContext); @@ -161,10 +139,11 @@ public BigtableDataClient createForAppProfile(@Nonnull String appProfileId) thro ClientContext clientContext = sharedClientContext + .getClientContext() .toBuilder() .setTracerFactory( EnhancedBigtableStub.createBigtableTracerFactory( - settings.getStubSettings(), openTelemetry)) + settings.getStubSettings(), sharedClientContext.getOpenTelemetry())) .build(); return BigtableDataClient.createWithClientContext(settings, clientContext); } @@ -190,10 +169,11 @@ public BigtableDataClient createForInstance(@Nonnull String projectId, @Nonnull ClientContext clientContext = sharedClientContext + .getClientContext() .toBuilder() .setTracerFactory( EnhancedBigtableStub.createBigtableTracerFactory( - settings.getStubSettings(), openTelemetry)) + settings.getStubSettings(), sharedClientContext.getOpenTelemetry())) .build(); return BigtableDataClient.createWithClientContext(settings, clientContext); @@ -220,10 +200,11 @@ public BigtableDataClient createForInstance( .build(); ClientContext clientContext = sharedClientContext + .getClientContext() .toBuilder() .setTracerFactory( EnhancedBigtableStub.createBigtableTracerFactory( - settings.getStubSettings(), openTelemetry)) + settings.getStubSettings(), sharedClientContext.getOpenTelemetry())) .build(); return BigtableDataClient.createWithClientContext(settings, clientContext); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 928159aa6d..25ff2ff30d 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -30,6 +30,7 @@ import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider; +import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider; import com.google.common.base.MoreObjects; import com.google.common.base.Strings; import io.grpc.ManagedChannelBuilder; @@ -127,6 +128,7 @@ public static Builder newBuilderForEmulator(String hostname, int port) { .setEndpoint(hostname + ":" + port) // disable channel refreshing when creating an emulator .setRefreshingChannel(false) + .setMetricsProvider(NoopMetricsProvider.INSTANCE) // disable exporting metrics for emulator .setTransportChannelProvider( InstantiatingGrpcChannelProvider.newBuilder() .setMaxInboundMessageSize(256 * 1024 * 1024) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimer.java index ecbef85be5..7495ca6ceb 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimer.java @@ -16,18 +16,27 @@ package com.google.cloud.bigtable.data.v2.stub; import com.google.api.core.BetaApi; -import com.google.api.gax.core.FixedCredentialsProvider; -import com.google.api.gax.core.InstantiatingExecutorProvider; +import com.google.api.core.SettableApiFuture; import com.google.api.gax.grpc.ChannelPrimer; -import com.google.api.gax.grpc.GrpcTransportChannel; -import com.google.api.gax.rpc.FixedTransportChannelProvider; import com.google.auth.Credentials; +import com.google.bigtable.v2.BigtableGrpc; +import com.google.bigtable.v2.InstanceName; import com.google.bigtable.v2.PingAndWarmRequest; -import com.google.cloud.bigtable.data.v2.internal.NameUtil; -import com.google.common.base.Preconditions; +import com.google.bigtable.v2.PingAndWarmResponse; +import io.grpc.CallCredentials; +import io.grpc.CallOptions; +import io.grpc.ClientCall; +import io.grpc.Deadline; import io.grpc.ManagedChannel; +import io.grpc.Metadata; +import io.grpc.Status; +import io.grpc.auth.MoreCallCredentials; import java.io.IOException; -import java.util.concurrent.ExecutionException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -41,27 +50,40 @@ class BigtableChannelPrimer implements ChannelPrimer { private static Logger LOG = Logger.getLogger(BigtableChannelPrimer.class.toString()); - private final EnhancedBigtableStubSettings settingsTemplate; + static final Metadata.Key REQUEST_PARAMS = + Metadata.Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER); + private final PingAndWarmRequest request; + private final CallCredentials callCredentials; + private final Map headers; static BigtableChannelPrimer create( - Credentials credentials, String projectId, String instanceId, String appProfileId) { - EnhancedBigtableStubSettings.Builder builder = - EnhancedBigtableStubSettings.newBuilder() - .setProjectId(projectId) - .setInstanceId(instanceId) - .setAppProfileId(appProfileId) - .setCredentialsProvider(FixedCredentialsProvider.create(credentials)) - // Disable refreshing channel here to avoid creating settings in a loop - .setRefreshingChannel(false) - .setExecutorProvider( - InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build()); - - return new BigtableChannelPrimer(builder.build()); + String projectId, + String instanceId, + String appProfileId, + Credentials credentials, + Map headers) { + return new BigtableChannelPrimer(projectId, instanceId, appProfileId, credentials, headers); } - private BigtableChannelPrimer(EnhancedBigtableStubSettings settingsTemplate) { - Preconditions.checkNotNull(settingsTemplate, "settingsTemplate can't be null"); - this.settingsTemplate = settingsTemplate; + BigtableChannelPrimer( + String projectId, + String instanceId, + String appProfileId, + Credentials credentials, + Map headers) { + if (credentials != null) { + callCredentials = MoreCallCredentials.from(credentials); + } else { + callCredentials = null; + } + + request = + PingAndWarmRequest.newBuilder() + .setName(InstanceName.format(projectId, instanceId)) + .setAppProfileId(appProfileId) + .build(); + + this.headers = headers; } @Override @@ -69,8 +91,7 @@ public void primeChannel(ManagedChannel managedChannel) { try { primeChannelUnsafe(managedChannel); } catch (IOException | RuntimeException e) { - LOG.warning( - String.format("Unexpected error while trying to prime a channel: %s", e.getMessage())); + LOG.log(Level.WARNING, "Unexpected error while trying to prime a channel", e); } } @@ -78,35 +99,64 @@ private void primeChannelUnsafe(ManagedChannel managedChannel) throws IOExceptio sendPrimeRequests(managedChannel); } - private void sendPrimeRequests(ManagedChannel managedChannel) throws IOException { - // Wrap the channel in a temporary stub - EnhancedBigtableStubSettings primingSettings = - settingsTemplate - .toBuilder() - .setTransportChannelProvider( - FixedTransportChannelProvider.create(GrpcTransportChannel.create(managedChannel))) - .build(); + private void sendPrimeRequests(ManagedChannel managedChannel) { + try { + ClientCall clientCall = + managedChannel.newCall( + BigtableGrpc.getPingAndWarmMethod(), + CallOptions.DEFAULT + .withCallCredentials(callCredentials) + .withDeadline(Deadline.after(1, TimeUnit.MINUTES))); - try (EnhancedBigtableStub stub = EnhancedBigtableStub.create(primingSettings)) { - PingAndWarmRequest request = - PingAndWarmRequest.newBuilder() - .setName( - NameUtil.formatInstanceName( - primingSettings.getProjectId(), primingSettings.getInstanceId())) - .setAppProfileId(primingSettings.getAppProfileId()) - .build(); - - try { - stub.pingAndWarmCallable().call(request); - } catch (Throwable e) { - // TODO: Not sure if we should swallow the error here. We are pre-emptively swapping - // channels if the new - // channel is bad. - if (e instanceof ExecutionException) { - e = e.getCause(); - } - LOG.warning(String.format("Failed to prime channel: %s", e)); - } + SettableApiFuture future = SettableApiFuture.create(); + clientCall.start( + new ClientCall.Listener() { + PingAndWarmResponse response; + + @Override + public void onMessage(PingAndWarmResponse message) { + response = message; + } + + @Override + public void onClose(Status status, Metadata trailers) { + if (status.isOk()) { + future.set(response); + } else { + future.setException(status.asException()); + } + } + }, + createMetadata(headers, request)); + clientCall.sendMessage(request); + clientCall.halfClose(); + clientCall.request(Integer.MAX_VALUE); + + future.get(1, TimeUnit.MINUTES); + } catch (Throwable e) { + // TODO: Not sure if we should swallow the error here. We are pre-emptively swapping + // channels if the new + // channel is bad. + LOG.log(Level.WARNING, "Failed to prime channel", e); } } + + private static Metadata createMetadata(Map headers, PingAndWarmRequest request) { + Metadata metadata = new Metadata(); + + headers.forEach( + (k, v) -> metadata.put(Metadata.Key.of(k, Metadata.ASCII_STRING_MARSHALLER), v)); + try { + metadata.put( + REQUEST_PARAMS, + String.format( + "name=%s&app_profile_id=%s", + URLEncoder.encode(request.getName(), "UTF-8"), + URLEncoder.encode(request.getAppProfileId(), "UTF-8"))); + } catch (UnsupportedEncodingException e) { + LOG.log(Level.WARNING, "Failed to encode request params", e); + } + + return metadata; + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableClientContext.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableClientContext.java new file mode 100644 index 0000000000..a2587b0dd9 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableClientContext.java @@ -0,0 +1,234 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub; + +import com.google.api.core.ApiFunction; +import com.google.api.core.InternalApi; +import com.google.api.gax.core.BackgroundResource; +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.rpc.ClientContext; +import com.google.auth.Credentials; +import com.google.auth.oauth2.ServiceAccountJwtAccessCredentials; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.internal.JwtCredentialsWithAudience; +import com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider; +import com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider; +import com.google.cloud.bigtable.data.v2.stub.metrics.ErrorCountPerConnectionMetricTracker; +import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider; +import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider; +import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.api.OpenTelemetry; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +/** + * This class wraps all state needed during the lifetime of the Bigtable client. This includes gax's + * {@link ClientContext} plus any additional state that Bigtable Client needs. + */ +@InternalApi +public class BigtableClientContext { + + private static final Logger logger = Logger.getLogger(BigtableClientContext.class.getName()); + + @Nullable private final OpenTelemetry openTelemetry; + private final ClientContext clientContext; + + public static BigtableClientContext create(EnhancedBigtableStubSettings settings) + throws IOException { + EnhancedBigtableStubSettings.Builder builder = settings.toBuilder(); + + // Set up credentials + patchCredentials(builder); + + // Fix the credentials so that they can be shared + Credentials credentials = null; + if (builder.getCredentialsProvider() != null) { + credentials = builder.getCredentialsProvider().getCredentials(); + } + builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); + + // Set up OpenTelemetry + OpenTelemetry openTelemetry = null; + try { + // We don't want client side metrics to crash the client, so catch any exception when getting + // the OTEL instance and log the exception instead. + openTelemetry = + getOpenTelemetryFromMetricsProvider( + settings.getMetricsProvider(), credentials, settings.getMetricsEndpoint()); + } catch (Throwable t) { + logger.log(Level.WARNING, "Failed to get OTEL, will skip exporting client side metrics", t); + } + + // Set up channel + InstantiatingGrpcChannelProvider.Builder transportProvider = + builder.getTransportChannelProvider() instanceof InstantiatingGrpcChannelProvider + ? ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder() + : null; + + ErrorCountPerConnectionMetricTracker errorCountPerConnectionMetricTracker = null; + + if (transportProvider != null) { + // Set up cookie holder if routing cookie is enabled + if (builder.getEnableRoutingCookie()) { + setupCookieHolder(transportProvider); + } + // Set up per connection error count tracker if OpenTelemetry is not null + if (openTelemetry != null) { + errorCountPerConnectionMetricTracker = + setupPerConnectionErrorTracer(builder, transportProvider, openTelemetry); + } + // Inject channel priming if enabled + if (builder.isRefreshingChannel()) { + transportProvider.setChannelPrimer( + BigtableChannelPrimer.create( + builder.getProjectId(), + builder.getInstanceId(), + builder.getAppProfileId(), + credentials, + builder.getHeaderProvider().getHeaders())); + } + + builder.setTransportChannelProvider(transportProvider.build()); + } + + ClientContext clientContext = ClientContext.create(builder.build()); + + if (errorCountPerConnectionMetricTracker != null) { + errorCountPerConnectionMetricTracker.startConnectionErrorCountTracker( + clientContext.getExecutor()); + } + + return new BigtableClientContext(clientContext, openTelemetry); + } + + private BigtableClientContext(ClientContext clientContext, OpenTelemetry openTelemetry) { + this.clientContext = clientContext; + this.openTelemetry = openTelemetry; + } + + public OpenTelemetry getOpenTelemetry() { + return this.openTelemetry; + } + + public ClientContext getClientContext() { + return this.clientContext; + } + + public void close() throws Exception { + for (BackgroundResource resource : clientContext.getBackgroundResources()) { + resource.close(); + } + } + + private static OpenTelemetry getOpenTelemetryFromMetricsProvider( + MetricsProvider metricsProvider, + @Nullable Credentials defaultCredentials, + @Nullable String metricsEndpoint) + throws IOException { + if (metricsProvider instanceof CustomOpenTelemetryMetricsProvider) { + CustomOpenTelemetryMetricsProvider customMetricsProvider = + (CustomOpenTelemetryMetricsProvider) metricsProvider; + return customMetricsProvider.getOpenTelemetry(); + } else if (metricsProvider instanceof DefaultMetricsProvider) { + Credentials credentials = + BigtableDataSettings.getMetricsCredentials() != null + ? BigtableDataSettings.getMetricsCredentials() + : defaultCredentials; + DefaultMetricsProvider defaultMetricsProvider = (DefaultMetricsProvider) metricsProvider; + return defaultMetricsProvider.getOpenTelemetry(metricsEndpoint, credentials); + } else if (metricsProvider instanceof NoopMetricsProvider) { + return null; + } + throw new IOException("Invalid MetricsProvider type " + metricsProvider); + } + + private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings) + throws IOException { + int i = settings.getEndpoint().lastIndexOf(":"); + String host = settings.getEndpoint().substring(0, i); + String audience = settings.getJwtAudienceMapping().get(host); + + if (audience == null) { + return; + } + URI audienceUri = null; + try { + audienceUri = new URI(audience); + } catch (URISyntaxException e) { + throw new IllegalStateException("invalid JWT audience override", e); + } + + CredentialsProvider credentialsProvider = settings.getCredentialsProvider(); + if (credentialsProvider == null) { + return; + } + + Credentials credentials = credentialsProvider.getCredentials(); + if (credentials == null) { + return; + } + + if (!(credentials instanceof ServiceAccountJwtAccessCredentials)) { + return; + } + + ServiceAccountJwtAccessCredentials jwtCreds = (ServiceAccountJwtAccessCredentials) credentials; + JwtCredentialsWithAudience patchedCreds = new JwtCredentialsWithAudience(jwtCreds, audienceUri); + settings.setCredentialsProvider(FixedCredentialsProvider.create(patchedCreds)); + } + + private static ErrorCountPerConnectionMetricTracker setupPerConnectionErrorTracer( + EnhancedBigtableStubSettings.Builder builder, + InstantiatingGrpcChannelProvider.Builder transportProvider, + OpenTelemetry openTelemetry) { + ErrorCountPerConnectionMetricTracker errorCountPerConnectionMetricTracker = + new ErrorCountPerConnectionMetricTracker( + openTelemetry, EnhancedBigtableStub.createBuiltinAttributes(builder.build())); + ApiFunction oldChannelConfigurator = + transportProvider.getChannelConfigurator(); + transportProvider.setChannelConfigurator( + managedChannelBuilder -> { + managedChannelBuilder.intercept(errorCountPerConnectionMetricTracker.getInterceptor()); + + if (oldChannelConfigurator != null) { + managedChannelBuilder = oldChannelConfigurator.apply(managedChannelBuilder); + } + return managedChannelBuilder; + }); + return errorCountPerConnectionMetricTracker; + } + + private static void setupCookieHolder( + InstantiatingGrpcChannelProvider.Builder transportProvider) { + ApiFunction oldChannelConfigurator = + transportProvider.getChannelConfigurator(); + transportProvider.setChannelConfigurator( + managedChannelBuilder -> { + managedChannelBuilder.intercept(new CookiesInterceptor()); + + if (oldChannelConfigurator != null) { + managedChannelBuilder = oldChannelConfigurator.apply(managedChannelBuilder); + } + return managedChannelBuilder; + }); + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index da0831304c..46377fbc41 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -20,7 +20,6 @@ import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLIENT_NAME_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID_KEY; -import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.api.core.BetaApi; @@ -29,13 +28,10 @@ import com.google.api.gax.batching.BatcherImpl; import com.google.api.gax.batching.FlowController; import com.google.api.gax.core.BackgroundResource; -import com.google.api.gax.core.CredentialsProvider; -import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.grpc.GaxGrpcProperties; import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.GrpcCallSettings; import com.google.api.gax.grpc.GrpcRawCallableFactory; -import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.retrying.BasicResultRetryAlgorithm; import com.google.api.gax.retrying.ExponentialRetryAlgorithm; import com.google.api.gax.retrying.RetryAlgorithm; @@ -57,8 +53,6 @@ import com.google.api.gax.tracing.SpanName; import com.google.api.gax.tracing.TracedServerStreamingCallable; import com.google.api.gax.tracing.TracedUnaryCallable; -import com.google.auth.Credentials; -import com.google.auth.oauth2.ServiceAccountJwtAccessCredentials; import com.google.bigtable.v2.BigtableGrpc; import com.google.bigtable.v2.CheckAndMutateRowResponse; import com.google.bigtable.v2.ExecuteQueryRequest; @@ -67,8 +61,6 @@ import com.google.bigtable.v2.GenerateInitialChangeStreamPartitionsResponse; import com.google.bigtable.v2.MutateRowsRequest; import com.google.bigtable.v2.MutateRowsResponse; -import com.google.bigtable.v2.PingAndWarmRequest; -import com.google.bigtable.v2.PingAndWarmResponse; import com.google.bigtable.v2.ReadChangeStreamRequest; import com.google.bigtable.v2.ReadChangeStreamResponse; import com.google.bigtable.v2.ReadRowsRequest; @@ -76,8 +68,6 @@ import com.google.bigtable.v2.RowRange; import com.google.bigtable.v2.SampleRowKeysResponse; import com.google.cloud.bigtable.Version; -import com.google.cloud.bigtable.data.v2.BigtableDataSettings; -import com.google.cloud.bigtable.data.v2.internal.JwtCredentialsWithAudience; import com.google.cloud.bigtable.data.v2.internal.NameUtil; import com.google.cloud.bigtable.data.v2.internal.RequestContext; import com.google.cloud.bigtable.data.v2.internal.SqlRow; @@ -109,12 +99,7 @@ import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable; import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTracerFactory; import com.google.cloud.bigtable.data.v2.stub.metrics.CompositeTracerFactory; -import com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider; -import com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider; -import com.google.cloud.bigtable.data.v2.stub.metrics.ErrorCountPerConnectionMetricTracker; -import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider; import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsTracerFactory; -import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider; import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants; import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersServerStreamingCallable; import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersUnaryCallable; @@ -145,7 +130,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; -import io.grpc.ManagedChannelBuilder; import io.grpc.MethodDescriptor; import io.opencensus.stats.Stats; import io.opencensus.stats.StatsRecorder; @@ -156,8 +140,6 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.time.Duration; import java.util.Collections; import java.util.List; @@ -165,8 +147,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -185,8 +165,6 @@ @InternalApi public class EnhancedBigtableStub implements AutoCloseable { - private static final Logger logger = Logger.getLogger(EnhancedBigtableStub.class.getName()); - private static final String CLIENT_NAME = "Bigtable"; private static final long FLOW_CONTROL_ADJUSTING_INTERVAL_MS = TimeUnit.SECONDS.toMillis(20); private final EnhancedBigtableStubSettings settings; @@ -208,7 +186,6 @@ public class EnhancedBigtableStub implements AutoCloseable { private final UnaryCallable externalBulkMutateRowsCallable; private final UnaryCallable checkAndMutateRowCallable; private final UnaryCallable readModifyWriteRowCallable; - private final UnaryCallable pingAndWarmCallable; private final ServerStreamingCallable generateInitialChangeStreamPartitionsCallable; @@ -220,22 +197,11 @@ public class EnhancedBigtableStub implements AutoCloseable { public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings) throws IOException { - ClientContext clientContext = createClientContext(settings); - OpenTelemetry openTelemetry = null; - try { - // We don't want client side metrics to crash the client, so catch any exception when getting - // the OTEL instance and log the exception instead. - openTelemetry = - getOpenTelemetry( - settings.getProjectId(), - settings.getMetricsProvider(), - clientContext.getCredentials(), - settings.getMetricsEndpoint()); - } catch (Throwable t) { - logger.log(Level.WARNING, "Failed to get OTEL, will skip exporting client side metrics", t); - } + BigtableClientContext bigtableClientContext = createBigtableClientContext(settings); + OpenTelemetry openTelemetry = bigtableClientContext.getOpenTelemetry(); ClientContext contextWithTracer = - clientContext + bigtableClientContext + .getClientContext() .toBuilder() .setTracerFactory(createBigtableTracerFactory(settings, openTelemetry)) .build(); @@ -248,89 +214,9 @@ public static EnhancedBigtableStub createWithClientContext( return new EnhancedBigtableStub(settings, clientContext, false); } - public static ClientContext createClientContext(EnhancedBigtableStubSettings settings) - throws IOException { - EnhancedBigtableStubSettings.Builder builder = settings.toBuilder(); - - // TODO: this implementation is on the cusp of unwieldy, if we end up adding more features - // consider splitting it up by feature. - - // workaround JWT audience issues - patchCredentials(builder); - - // Fix the credentials so that they can be shared - Credentials credentials = null; - if (builder.getCredentialsProvider() != null) { - credentials = builder.getCredentialsProvider().getCredentials(); - } - builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); - - InstantiatingGrpcChannelProvider.Builder transportProvider = - builder.getTransportChannelProvider() instanceof InstantiatingGrpcChannelProvider - ? ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder() - : null; - - OpenTelemetry openTelemetry = null; - try { - // We don't want client side metrics to crash the client, so catch any exception when getting - // the OTEL instance and log the exception instead. - openTelemetry = - getOpenTelemetry( - settings.getProjectId(), - settings.getMetricsProvider(), - credentials, - settings.getMetricsEndpoint()); - } catch (Throwable t) { - logger.log(Level.WARNING, "Failed to get OTEL, will skip exporting client side metrics", t); - } - ErrorCountPerConnectionMetricTracker errorCountPerConnectionMetricTracker; - // Skip setting up ErrorCountPerConnectionMetricTracker if openTelemetry is null - if (openTelemetry != null && transportProvider != null) { - errorCountPerConnectionMetricTracker = - new ErrorCountPerConnectionMetricTracker( - openTelemetry, createBuiltinAttributes(settings)); - ApiFunction oldChannelConfigurator = - transportProvider.getChannelConfigurator(); - transportProvider.setChannelConfigurator( - managedChannelBuilder -> { - if (settings.getEnableRoutingCookie()) { - managedChannelBuilder.intercept(new CookiesInterceptor()); - } - - managedChannelBuilder.intercept(errorCountPerConnectionMetricTracker.getInterceptor()); - - if (oldChannelConfigurator != null) { - managedChannelBuilder = oldChannelConfigurator.apply(managedChannelBuilder); - } - return managedChannelBuilder; - }); - } else { - errorCountPerConnectionMetricTracker = null; - } - - // Inject channel priming - if (settings.isRefreshingChannel()) { - - if (transportProvider != null) { - transportProvider.setChannelPrimer( - BigtableChannelPrimer.create( - credentials, - settings.getProjectId(), - settings.getInstanceId(), - settings.getAppProfileId())); - } - } - - if (transportProvider != null) { - builder.setTransportChannelProvider(transportProvider.build()); - } - - ClientContext clientContext = ClientContext.create(builder.build()); - if (errorCountPerConnectionMetricTracker != null) { - errorCountPerConnectionMetricTracker.startConnectionErrorCountTracker( - clientContext.getExecutor()); - } - return clientContext; + public static BigtableClientContext createBigtableClientContext( + EnhancedBigtableStubSettings settings) throws IOException { + return BigtableClientContext.create(settings); } public static ApiTracerFactory createBigtableTracerFactory( @@ -387,31 +273,7 @@ public static ApiTracerFactory createBigtableTracerFactory( return new CompositeTracerFactory(tracerFactories.build()); } - @Nullable - public static OpenTelemetry getOpenTelemetry( - String projectId, - MetricsProvider metricsProvider, - @Nullable Credentials defaultCredentials, - @Nullable String metricsEndpoint) - throws IOException { - if (metricsProvider instanceof CustomOpenTelemetryMetricsProvider) { - CustomOpenTelemetryMetricsProvider customMetricsProvider = - (CustomOpenTelemetryMetricsProvider) metricsProvider; - return customMetricsProvider.getOpenTelemetry(); - } else if (metricsProvider instanceof DefaultMetricsProvider) { - Credentials credentials = - BigtableDataSettings.getMetricsCredentials() != null - ? BigtableDataSettings.getMetricsCredentials() - : defaultCredentials; - DefaultMetricsProvider defaultMetricsProvider = (DefaultMetricsProvider) metricsProvider; - return defaultMetricsProvider.getOpenTelemetry(projectId, metricsEndpoint, credentials); - } else if (metricsProvider instanceof NoopMetricsProvider) { - return null; - } - throw new IOException("Invalid MetricsProvider type " + metricsProvider); - } - - private static Attributes createBuiltinAttributes(EnhancedBigtableStubSettings settings) { + static Attributes createBuiltinAttributes(EnhancedBigtableStubSettings settings) { return Attributes.of( BIGTABLE_PROJECT_ID_KEY, settings.getProjectId(), @@ -423,41 +285,6 @@ private static Attributes createBuiltinAttributes(EnhancedBigtableStubSettings s "bigtable-java/" + Version.VERSION); } - private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings) - throws IOException { - int i = settings.getEndpoint().lastIndexOf(":"); - String host = settings.getEndpoint().substring(0, i); - String audience = settings.getJwtAudienceMapping().get(host); - - if (audience == null) { - return; - } - URI audienceUri = null; - try { - audienceUri = new URI(audience); - } catch (URISyntaxException e) { - throw new IllegalStateException("invalid JWT audience override", e); - } - - CredentialsProvider credentialsProvider = settings.getCredentialsProvider(); - if (credentialsProvider == null) { - return; - } - - Credentials credentials = credentialsProvider.getCredentials(); - if (credentials == null) { - return; - } - - if (!(credentials instanceof ServiceAccountJwtAccessCredentials)) { - return; - } - - ServiceAccountJwtAccessCredentials jwtCreds = (ServiceAccountJwtAccessCredentials) credentials; - JwtCredentialsWithAudience patchedCreds = new JwtCredentialsWithAudience(jwtCreds, audienceUri); - settings.setCredentialsProvider(FixedCredentialsProvider.create(patchedCreds)); - } - public EnhancedBigtableStub(EnhancedBigtableStubSettings settings, ClientContext clientContext) { this(settings, clientContext, true); } @@ -491,7 +318,6 @@ public EnhancedBigtableStub( createGenerateInitialChangeStreamPartitionsCallable(); readChangeStreamCallable = createReadChangeStreamCallable(new DefaultChangeStreamRecordAdapter()); - pingAndWarmCallable = createPingAndWarmCallable(); executeQueryCallable = createExecuteQueryCallable(); } @@ -1422,28 +1248,6 @@ ServerStreamingCallSettings convertUnaryToServerStreamingSettings( .build(); } - private UnaryCallable createPingAndWarmCallable() { - UnaryCallable pingAndWarm = - GrpcRawCallableFactory.createUnaryCallable( - GrpcCallSettings.newBuilder() - .setMethodDescriptor(BigtableGrpc.getPingAndWarmMethod()) - .setParamsExtractor( - new RequestParamsExtractor() { - @Override - public Map extract(PingAndWarmRequest request) { - return ImmutableMap.of( - "name", request.getName(), - "app_profile_id", request.getAppProfileId()); - } - }) - .build(), - Collections.emptySet()); - return pingAndWarm.withDefaultCallContext( - clientContext - .getDefaultCallContext() - .withRetrySettings(settings.pingAndWarmSettings().getRetrySettings())); - } - private UnaryCallable withRetries( UnaryCallable innerCallable, UnaryCallSettings unaryCallSettings) { UnaryCallable retrying; @@ -1551,10 +1355,6 @@ public ExecuteQueryCallable executeQueryCallable() { return executeQueryCallable; } - UnaryCallable pingAndWarmCallable() { - return pingAndWarmCallable; - } - // private SpanName getSpanName(String methodName) { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java index 8aa53fa198..ff5bcd81c1 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java @@ -58,6 +58,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; @@ -94,7 +95,6 @@ public final class BigtableCloudMonitoringExporter implements MetricExporter { private final MetricServiceClient client; - private final String bigtableProjectId; private final String taskId; // The resource the client application is running on @@ -128,8 +128,7 @@ public final class BigtableCloudMonitoringExporter implements MetricExporter { .collect(ImmutableList.toImmutableList()); public static BigtableCloudMonitoringExporter create( - String projectId, @Nullable Credentials credentials, @Nullable String endpoint) - throws IOException { + @Nullable Credentials credentials, @Nullable String endpoint) throws IOException { MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder(); CredentialsProvider credentialsProvider = Optional.ofNullable(credentials) @@ -164,7 +163,6 @@ public static BigtableCloudMonitoringExporter create( } return new BigtableCloudMonitoringExporter( - projectId, MetricServiceClient.create(settingsBuilder.build()), applicationResource, BigtableExporterUtils.getDefaultTaskValue()); @@ -172,14 +170,10 @@ public static BigtableCloudMonitoringExporter create( @VisibleForTesting BigtableCloudMonitoringExporter( - String projectId, - MetricServiceClient client, - @Nullable MonitoredResource applicationResource, - String taskId) { + MetricServiceClient client, @Nullable MonitoredResource applicationResource, String taskId) { this.client = client; this.taskId = taskId; this.applicationResource = applicationResource; - this.bigtableProjectId = projectId; } @Override @@ -211,15 +205,8 @@ private CompletableResultCode exportBigtableResourceMetrics(Collection metricData.getData().getPoints().stream()) - .allMatch(pd -> bigtableProjectId.equals(BigtableExporterUtils.getProjectId(pd)))) { - logger.log(Level.WARNING, "Metric data has different a projectId. Skip exporting."); - return CompletableResultCode.ofFailure(); - } - - List bigtableTimeSeries; + // List of timeseries by project id + Map> bigtableTimeSeries; try { bigtableTimeSeries = BigtableExporterUtils.convertToBigtableTimeSeries(bigtableMetricData, taskId); @@ -231,37 +218,39 @@ private CompletableResultCode exportBigtableResourceMetrics(Collection> future = exportTimeSeries(projectName, bigtableTimeSeries); - CompletableResultCode bigtableExportCode = new CompletableResultCode(); - ApiFutures.addCallback( - future, - new ApiFutureCallback>() { - @Override - public void onFailure(Throwable throwable) { - if (bigtableExportFailureLogged.compareAndSet(false, true)) { - String msg = "createServiceTimeSeries request failed for bigtable metrics."; - if (throwable instanceof PermissionDeniedException) { - msg += - String.format( - " Need monitoring metric writer permission on project=%s. Follow https://cloud.google.com/bigtable/docs/client-side-metrics-setup to set up permissions.", - projectName.getProject()); - } - logger.log(Level.WARNING, msg, throwable); - } - bigtableExportCode.fail(); - } + bigtableTimeSeries.forEach( + (projectId, ts) -> { + ProjectName projectName = ProjectName.of(projectId); + ApiFuture> future = exportTimeSeries(projectName, ts); + ApiFutures.addCallback( + future, + new ApiFutureCallback>() { + @Override + public void onFailure(Throwable throwable) { + if (bigtableExportFailureLogged.compareAndSet(false, true)) { + String msg = "createServiceTimeSeries request failed for bigtable metrics."; + if (throwable instanceof PermissionDeniedException) { + msg += + String.format( + " Need monitoring metric writer permission on project=%s. Follow https://cloud.google.com/bigtable/docs/client-side-metrics-setup to set up permissions.", + projectName.getProject()); + } + logger.log(Level.WARNING, msg, throwable); + } + bigtableExportCode.fail(); + } - @Override - public void onSuccess(List emptyList) { - // When an export succeeded reset the export failure flag to false so if there's a - // transient failure it'll be logged. - bigtableExportFailureLogged.set(false); - bigtableExportCode.succeed(); - } - }, - MoreExecutors.directExecutor()); + @Override + public void onSuccess(List emptyList) { + // When an export succeeded reset the export failure flag to false so if there's a + // transient failure it'll be logged. + bigtableExportFailureLogged.set(false); + bigtableExportCode.succeed(); + } + }, + MoreExecutors.directExecutor()); + }); return bigtableExportCode; } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java index 5bf6688e17..821c2295e0 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java @@ -63,6 +63,7 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -110,17 +111,24 @@ static String getProjectId(PointData pointData) { return pointData.getAttributes().get(BIGTABLE_PROJECT_ID_KEY); } - static List convertToBigtableTimeSeries(List collection, String taskId) { - List allTimeSeries = new ArrayList<>(); + // Returns a list of timeseries by project id + static Map> convertToBigtableTimeSeries( + List collection, String taskId) { + Map> allTimeSeries = new HashMap<>(); for (MetricData metricData : collection) { if (!metricData.getInstrumentationScopeInfo().getName().equals(METER_NAME)) { // Filter out metric data for instruments that are not part of the bigtable builtin metrics continue; } - metricData.getData().getPoints().stream() - .map(pointData -> convertPointToBigtableTimeSeries(metricData, pointData, taskId)) - .forEach(allTimeSeries::add); + + for (PointData pd : metricData.getData().getPoints()) { + String projectId = getProjectId(pd); + List current = + allTimeSeries.computeIfAbsent(projectId, ignored -> new ArrayList<>()); + current.add(convertPointToBigtableTimeSeries(metricData, pd, taskId)); + allTimeSeries.put(projectId, current); + } } return allTimeSeries; diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsView.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsView.java index ca52581a92..07679af8d2 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsView.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsView.java @@ -38,35 +38,65 @@ private BuiltinMetricsView() {} /** * Register built-in metrics on the {@link SdkMeterProviderBuilder} with application default * credentials and default endpoint. + * + * @deprecated projectId is no longer used. Call {@link + * #registerBuiltinMetrics(SdkMeterProviderBuilder)} instead. */ + @Deprecated public static void registerBuiltinMetrics(String projectId, SdkMeterProviderBuilder builder) throws IOException { BuiltinMetricsView.registerBuiltinMetrics( - projectId, GoogleCredentials.getApplicationDefault(), builder); + GoogleCredentials.getApplicationDefault(), builder, null); + } + + /** + * Register built-in metrics on the {@link SdkMeterProviderBuilder} with application default + * credentials and default endpoint. + */ + public static void registerBuiltinMetrics(SdkMeterProviderBuilder builder) throws IOException { + BuiltinMetricsView.registerBuiltinMetrics( + GoogleCredentials.getApplicationDefault(), builder, null); } /** * Register built-in metrics on the {@link SdkMeterProviderBuilder} with custom credentials and * default endpoint. + * + * @deprecated projectId is no longer used. Call {@link #registerBuiltinMetrics(Credentials, + * SdkMeterProviderBuilder, String)} instead. */ + @Deprecated public static void registerBuiltinMetrics( String projectId, @Nullable Credentials credentials, SdkMeterProviderBuilder builder) throws IOException { - BuiltinMetricsView.registerBuiltinMetrics(projectId, credentials, builder, null); + BuiltinMetricsView.registerBuiltinMetrics(credentials, builder, null); } /** * Register built-in metrics on the {@link SdkMeterProviderBuilder} with custom credentials and * endpoint. + * + * @deprecated projectId is no longer used. Call {@link #registerBuiltinMetrics(Credentials, + * SdkMeterProviderBuilder, String)} instead. */ + @Deprecated public static void registerBuiltinMetrics( String projectId, @Nullable Credentials credentials, SdkMeterProviderBuilder builder, @Nullable String endpoint) throws IOException { - MetricExporter metricExporter = - BigtableCloudMonitoringExporter.create(projectId, credentials, endpoint); + registerBuiltinMetrics(credentials, builder, endpoint); + } + + /** + * Register built-in metrics on the {@link SdkMeterProviderBuilder} with custom credentials and + * endpoint. + */ + public static void registerBuiltinMetrics( + @Nullable Credentials credentials, SdkMeterProviderBuilder builder, @Nullable String endpoint) + throws IOException { + MetricExporter metricExporter = BigtableCloudMonitoringExporter.create(credentials, endpoint); for (Map.Entry entry : BuiltinMetricsConstants.getAllViews().entrySet()) { builder.registerView(entry.getKey(), entry.getValue()); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CustomOpenTelemetryMetricsProvider.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CustomOpenTelemetryMetricsProvider.java index 8c1c5c1c90..d728d657ae 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CustomOpenTelemetryMetricsProvider.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CustomOpenTelemetryMetricsProvider.java @@ -27,7 +27,7 @@ * SdkMeterProviderBuilder sdkMeterProvider = SdkMeterProvider.builder(); * * // register Builtin metrics on your meter provider with default credentials - * BuiltinMetricsView.registerBuiltinMetrics("project-id", sdkMeterProvider); + * BuiltinMetricsView.registerBuiltinMetrics(sdkMeterProvider); * * // register other metrics reader and views * sdkMeterProvider.registerMetricReader(..); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/DefaultMetricsProvider.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/DefaultMetricsProvider.java index c6b0a80c76..ae4df85893 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/DefaultMetricsProvider.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/DefaultMetricsProvider.java @@ -17,7 +17,6 @@ import com.google.api.core.InternalApi; import com.google.auth.Credentials; -import com.google.common.base.MoreObjects; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.metrics.SdkMeterProvider; @@ -36,30 +35,18 @@ public final class DefaultMetricsProvider implements MetricsProvider { public static DefaultMetricsProvider INSTANCE = new DefaultMetricsProvider(); - private OpenTelemetry openTelemetry; - private String projectId; - private DefaultMetricsProvider() {} @InternalApi public OpenTelemetry getOpenTelemetry( - String projectId, String metricsEndpoint, @Nullable Credentials credentials) - throws IOException { - this.projectId = projectId; - if (openTelemetry == null) { - SdkMeterProviderBuilder meterProvider = SdkMeterProvider.builder(); - BuiltinMetricsView.registerBuiltinMetrics( - projectId, credentials, meterProvider, metricsEndpoint); - openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build(); - } - return openTelemetry; + @Nullable String metricsEndpoint, @Nullable Credentials credentials) throws IOException { + SdkMeterProviderBuilder meterProvider = SdkMeterProvider.builder(); + BuiltinMetricsView.registerBuiltinMetrics(credentials, meterProvider, metricsEndpoint); + return OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build(); } @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("projectId", projectId) - .add("openTelemetry", openTelemetry) - .toString(); + return "DefaultMetricsProvider"; } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java index 3cf3ded747..0ba472f783 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java @@ -299,6 +299,43 @@ public void testCreateTable() { assertThat(result).isEqualTo(Table.fromProto(expectedResponse)); } + @Test + public void testCreateTableWithDeletionProtectionSet() { + // Setup + Mockito.when(mockStub.createTableCallable()).thenReturn(mockCreateTableCallable); + + com.google.bigtable.admin.v2.CreateTableRequest expectedRequest = + com.google.bigtable.admin.v2.CreateTableRequest.newBuilder() + .setParent(INSTANCE_NAME) + .setTableId(TABLE_ID) + .setTable( + com.google.bigtable.admin.v2.Table.newBuilder() + .setDeletionProtection(true) + .putColumnFamilies( + "cf1", + ColumnFamily.newBuilder() + .setGcRule(GcRule.getDefaultInstance()) + .setValueType(TypeProtos.intSumType()) + .build())) + .build(); + + com.google.bigtable.admin.v2.Table expectedResponse = + com.google.bigtable.admin.v2.Table.newBuilder().setName(TABLE_NAME).build(); + + Mockito.when(mockCreateTableCallable.futureCall(expectedRequest)) + .thenReturn(ApiFutures.immediateFuture(expectedResponse)); + + // Execute + Table result = + adminClient.createTable( + CreateTableRequest.of(TABLE_ID) + .addFamily("cf1", Type.int64Sum()) + .setDeletionProtection(true)); + + // Verify + assertThat(result).isEqualTo(Table.fromProto(expectedResponse)); + } + @Test public void testUpdateTable() { // Setup diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java index 76413165bd..c95afa9eef 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java @@ -19,6 +19,7 @@ import static com.google.common.truth.Truth.assertWithMessage; import static com.google.common.truth.TruthJUnit.assume; +import com.google.api.gax.rpc.FailedPreconditionException; import com.google.cloud.Policy; import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; import com.google.cloud.bigtable.admin.v2.models.AppProfile; @@ -36,7 +37,10 @@ import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv; import com.google.cloud.bigtable.test_helpers.env.PrefixGenerator; import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; +import java.time.Duration; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -49,6 +53,8 @@ public class BigtableInstanceAdminClientIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + private static final Logger logger = + Logger.getLogger(BigtableInstanceAdminClientIT.class.getName()); @Rule public final PrefixGenerator prefixGenerator = new PrefixGenerator(); private String instanceId = testEnvRule.env().getInstanceId(); @@ -410,7 +416,7 @@ public void createClusterWithAutoscalingTest() { } @Test - public void createClusterWithAutoscalingAndPartialUpdateTest() { + public void createClusterWithAutoscalingAndPartialUpdateTest() throws Exception { String newInstanceId = prefixGenerator.newPrefix(); String newClusterId = newInstanceId + "-c1"; @@ -448,8 +454,16 @@ public void createClusterWithAutoscalingAndPartialUpdateTest() { assertThat(retrievedCluster.getAutoscalingCpuPercentageTarget()).isEqualTo(20); assertThat(retrievedCluster.getStorageUtilizationGibPerNode()).isEqualTo(2561); + // The test might trigger cluster autoscaling, which races against the update cluster calls in + // this test and causing the update cluster calls to fail with "FAILED_PRECONDITION: Cannot + // update cluster that is currently being modified" error. + // In order to avoid test flakiness due to this race condition, we wrap all the update cluster + // call with a retry loop. + // TODO: After we have a proper fix for the issue, remove the + // updateClusterAutoScalingConfigWithRetry function and all the calls to it. + Cluster updatedCluster = - client.updateClusterAutoscalingConfig( + updateClusterAutoScalingConfigWithRetry( ClusterAutoscalingConfig.of(newInstanceId, clusterId).setMaxNodes(3)); assertThat(updatedCluster.getAutoscalingMinServeNodes()).isEqualTo(1); assertThat(updatedCluster.getAutoscalingMaxServeNodes()).isEqualTo(3); @@ -463,7 +477,7 @@ public void createClusterWithAutoscalingAndPartialUpdateTest() { assertThat(retrievedUpdatedCluster.getStorageUtilizationGibPerNode()).isEqualTo(2561); updatedCluster = - client.updateClusterAutoscalingConfig( + updateClusterAutoScalingConfigWithRetry( ClusterAutoscalingConfig.of(newInstanceId, clusterId).setMinNodes(2)); assertThat(updatedCluster.getAutoscalingMinServeNodes()).isEqualTo(2); assertThat(updatedCluster.getAutoscalingMaxServeNodes()).isEqualTo(3); @@ -477,7 +491,7 @@ public void createClusterWithAutoscalingAndPartialUpdateTest() { assertThat(retrievedUpdatedCluster.getStorageUtilizationGibPerNode()).isEqualTo(2561); updatedCluster = - client.updateClusterAutoscalingConfig( + updateClusterAutoScalingConfigWithRetry( ClusterAutoscalingConfig.of(newInstanceId, clusterId) .setCpuUtilizationTargetPercent(40)); assertThat(updatedCluster.getAutoscalingMinServeNodes()).isEqualTo(2); @@ -492,7 +506,7 @@ public void createClusterWithAutoscalingAndPartialUpdateTest() { assertThat(retrievedUpdatedCluster.getStorageUtilizationGibPerNode()).isEqualTo(2561); updatedCluster = - client.updateClusterAutoscalingConfig( + updateClusterAutoScalingConfigWithRetry( ClusterAutoscalingConfig.of(newInstanceId, clusterId) .setCpuUtilizationTargetPercent(45) .setMaxNodes(5)); @@ -508,7 +522,7 @@ public void createClusterWithAutoscalingAndPartialUpdateTest() { assertThat(retrievedUpdatedCluster.getStorageUtilizationGibPerNode()).isEqualTo(2561); updatedCluster = - client.updateClusterAutoscalingConfig( + updateClusterAutoScalingConfigWithRetry( ClusterAutoscalingConfig.of(newInstanceId, clusterId) .setStorageUtilizationGibPerNode(2777)); assertThat(updatedCluster.getAutoscalingMinServeNodes()).isEqualTo(2); @@ -523,7 +537,7 @@ public void createClusterWithAutoscalingAndPartialUpdateTest() { assertThat(retrievedUpdatedCluster.getStorageUtilizationGibPerNode()).isEqualTo(2777); updatedCluster = - client.updateClusterAutoscalingConfig( + updateClusterAutoScalingConfigWithRetry( ClusterAutoscalingConfig.of(newInstanceId, clusterId) // testing default case .setStorageUtilizationGibPerNode(0)); @@ -614,4 +628,20 @@ private void basicClusterOperationTestHelper(String targetInstanceId, String tar assertThat(updatedCluster.getAutoscalingCpuPercentageTarget()).isEqualTo(0); assertThat(updatedCluster.getStorageUtilizationGibPerNode()).isEqualTo(0); } + + private Cluster updateClusterAutoScalingConfigWithRetry( + ClusterAutoscalingConfig clusterAutoscalingConfig) throws Exception { + int retryCount = 0; + int maxRetries = 10; + while (true) { + try { + return client.updateClusterAutoscalingConfig(clusterAutoscalingConfig); + } catch (FailedPreconditionException e) { + if (++retryCount == maxRetries) throw e; + logger.log( + Level.INFO, "Retrying updateClusterAutoscalingConfig, retryCount: " + retryCount); + Thread.sleep(Duration.ofMinutes(1).toMillis()); + } + } + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequestTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequestTest.java index 0f7a58c078..35dae7aeeb 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequestTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequestTest.java @@ -48,7 +48,8 @@ public void testToProto() { .addFamily("another-family", GCRULES.maxAge(100, TimeUnit.HOURS)) .addSplit(splitKey) .addSplit(secondSplitKey) - .addChangeStreamRetention(Duration.ofHours(24)); + .addChangeStreamRetention(Duration.ofHours(24)) + .setDeletionProtection(true); com.google.bigtable.admin.v2.CreateTableRequest requestProto = com.google.bigtable.admin.v2.CreateTableRequest.newBuilder() @@ -70,7 +71,8 @@ public void testToProto() { ChangeStreamConfig.newBuilder() .setRetentionPeriod( com.google.protobuf.Duration.newBuilder().setSeconds(86400)) - .build())) + .build()) + .setDeletionProtection(true)) .setParent(NameUtil.formatInstanceName(PROJECT_ID, INSTANCE_ID)) .addInitialSplits( com.google.bigtable.admin.v2.CreateTableRequest.Split.newBuilder().setKey(splitKey)) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/TableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/TableTest.java index b94be17e7f..20f9c8e514 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/TableTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/TableTest.java @@ -67,6 +67,7 @@ public void testFromProto() { .setSeconds(1) .setNanos(99))) .build()) + .setDeletionProtection(true) .build(); Table result = Table.fromProto(proto); @@ -78,6 +79,7 @@ public void testFromProto() { "cluster1", Table.ReplicationState.READY, "cluster2", Table.ReplicationState.INITIALIZING); assertThat(result.getColumnFamilies()).hasSize(3); + assertThat(result.isDeletionProtected()).isTrue(); for (Entry entry : proto.getColumnFamiliesMap().entrySet()) { assertThat(result.getColumnFamilies()) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequestTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequestTest.java index fabebdccbf..24fe80187c 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequestTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequestTest.java @@ -81,4 +81,36 @@ public void testNoChangeChangeStreamToProto() { .build(); assertThat(request.toProto(PROJECT_ID, INSTANCE_ID)).isEqualTo(requestProto); } + + @Test + public void testEnableDeletionProtection() { + UpdateTableRequest request = UpdateTableRequest.of(TABLE_ID).setDeletionProtection(true); + + com.google.bigtable.admin.v2.UpdateTableRequest requestProto = + com.google.bigtable.admin.v2.UpdateTableRequest.newBuilder() + .setTable( + Table.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, TABLE_ID)) + .setDeletionProtection(true)) + .setUpdateMask(FieldMask.newBuilder().addPaths("deletion_protection").build()) + .build(); + + assertThat(request.toProto(PROJECT_ID, INSTANCE_ID)).isEqualTo(requestProto); + } + + @Test + public void testDisableDeletionProtection() { + UpdateTableRequest request = UpdateTableRequest.of(TABLE_ID).setDeletionProtection(false); + + com.google.bigtable.admin.v2.UpdateTableRequest requestProto = + com.google.bigtable.admin.v2.UpdateTableRequest.newBuilder() + .setTable( + Table.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, TABLE_ID)) + .setDeletionProtection(false)) + .setUpdateMask(FieldMask.newBuilder().addPaths("deletion_protection").build()) + .build(); + + assertThat(request.toProto(PROJECT_ID, INSTANCE_ID)).isEqualTo(requestProto); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index 79d27b7fa3..d929627e12 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -24,10 +24,7 @@ import static com.google.common.truth.TruthJUnit.assume; import com.google.api.client.util.Lists; -import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.models.AppProfile; -import com.google.cloud.bigtable.admin.v2.models.CreateAppProfileRequest; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; import com.google.cloud.bigtable.admin.v2.models.Table; import com.google.cloud.bigtable.data.v2.BigtableDataClient; @@ -73,6 +70,7 @@ import org.junit.After; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -81,6 +79,7 @@ import org.threeten.bp.Duration; import org.threeten.bp.Instant; +@Ignore("Temporarily disable flaky test") @RunWith(JUnit4.class) public class BuiltinMetricsIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @@ -94,12 +93,9 @@ public class BuiltinMetricsIT { private BigtableDataClient clientCustomOtel; private BigtableDataClient clientDefault; private BigtableTableAdminClient tableAdminClient; - private BigtableInstanceAdminClient instanceAdminClient; private MetricServiceClient metricClient; private InMemoryMetricReader metricReader; - private String appProfileCustomOtel; - private String appProfileDefault; public static String[] VIEWS = { "operation_latencies", @@ -131,19 +127,6 @@ public void setup() throws IOException { metricClient = MetricServiceClient.create(); tableAdminClient = testEnvRule.env().getTableAdminClient(); - instanceAdminClient = testEnvRule.env().getInstanceAdminClient(); - appProfileCustomOtel = PrefixGenerator.newPrefix("test1"); - appProfileDefault = PrefixGenerator.newPrefix("test2"); - instanceAdminClient.createAppProfile( - CreateAppProfileRequest.of(testEnvRule.env().getInstanceId(), appProfileCustomOtel) - .setRoutingPolicy( - AppProfile.SingleClusterRoutingPolicy.of(testEnvRule.env().getPrimaryClusterId())) - .setIsolationPolicy(AppProfile.StandardIsolationPolicy.of(AppProfile.Priority.LOW))); - instanceAdminClient.createAppProfile( - CreateAppProfileRequest.of(testEnvRule.env().getInstanceId(), appProfileDefault) - .setRoutingPolicy( - AppProfile.SingleClusterRoutingPolicy.of(testEnvRule.env().getPrimaryClusterId())) - .setIsolationPolicy(AppProfile.StandardIsolationPolicy.of(AppProfile.Priority.LOW))); // When using the custom OTEL instance, we can also register a InMemoryMetricReader on the // SdkMeterProvider to verify the data exported on Cloud Monitoring with the in memory metric @@ -162,9 +145,8 @@ public void setup() throws IOException { BigtableDataClient.create( settings .setMetricsProvider(CustomOpenTelemetryMetricsProvider.create(openTelemetry)) - .setAppProfileId(appProfileCustomOtel) .build()); - clientDefault = BigtableDataClient.create(settings.setAppProfileId(appProfileDefault).build()); + clientDefault = BigtableDataClient.create(settings.build()); } @After @@ -178,12 +160,7 @@ public void tearDown() { if (tableDefault != null) { tableAdminClient.deleteTable(tableDefault.getId()); } - if (instanceAdminClient != null) { - instanceAdminClient.deleteAppProfile( - testEnvRule.env().getInstanceId(), appProfileCustomOtel, true); - instanceAdminClient.deleteAppProfile( - testEnvRule.env().getInstanceId(), appProfileDefault, true); - } + if (clientCustomOtel != null) { clientCustomOtel.close(); } @@ -231,8 +208,8 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { String.format( "metric.type=\"bigtable.googleapis.com/client/%s\" " + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.MutateRow\"" - + " AND resource.labels.table=\"%s\" AND metric.labels.app_profile=\"%s\"", - view, testEnvRule.env().getInstanceId(), tableDefault.getId(), appProfileDefault); + + " AND resource.labels.table=\"%s\"", + view, testEnvRule.env().getInstanceId(), tableDefault.getId()); ListTimeSeriesRequest.Builder requestBuilder = ListTimeSeriesRequest.newBuilder() .setName(name.toString()) @@ -246,8 +223,8 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { String.format( "metric.type=\"bigtable.googleapis.com/client/%s\" " + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.ReadRows\"" - + " AND resource.labels.table=\"%s\" AND metric.labels.app_profile=\"%s\"", - view, testEnvRule.env().getInstanceId(), tableDefault.getId(), appProfileDefault); + + " AND resource.labels.table=\"%s\"", + view, testEnvRule.env().getInstanceId(), tableDefault.getId()); requestBuilder.setFilter(metricFilter); verifyMetricsArePublished(requestBuilder.build(), metricsPollingStopwatch, view); @@ -299,11 +276,8 @@ public void testBuiltinMetricsWithCustomOTEL() throws Exception { String.format( "metric.type=\"bigtable.googleapis.com/client/%s\" " + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.MutateRow\"" - + " AND resource.labels.table=\"%s\" AND metric.labels.app_profile=\"%s\"", - view, - testEnvRule.env().getInstanceId(), - tableCustomOtel.getId(), - appProfileCustomOtel); + + " AND resource.labels.table=\"%s\"", + view, testEnvRule.env().getInstanceId(), tableCustomOtel.getId()); ListTimeSeriesRequest.Builder requestBuilder = ListTimeSeriesRequest.newBuilder() .setName(name.toString()) @@ -320,11 +294,8 @@ public void testBuiltinMetricsWithCustomOTEL() throws Exception { String.format( "metric.type=\"bigtable.googleapis.com/client/%s\" " + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.ReadRows\"" - + " AND resource.labels.table=\"%s\" AND metric.labels.app_profile=\"%s\"", - view, - testEnvRule.env().getInstanceId(), - tableCustomOtel.getId(), - appProfileCustomOtel); + + " AND resource.labels.table=\"%s\"", + view, testEnvRule.env().getInstanceId(), tableCustomOtel.getId()); requestBuilder.setFilter(metricFilter); response = verifyMetricsArePublished(requestBuilder.build(), metricsPollingStopwatch, view); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimerTest.java index e1f22bebbd..709b482477 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimerTest.java @@ -24,6 +24,7 @@ import com.google.bigtable.v2.PingAndWarmRequest; import com.google.bigtable.v2.PingAndWarmResponse; import com.google.cloud.bigtable.data.v2.FakeServiceBuilder; +import com.google.common.collect.ImmutableMap; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.Metadata; @@ -69,10 +70,11 @@ public void setup() throws IOException { primer = BigtableChannelPrimer.create( - OAuth2Credentials.create(new AccessToken(TOKEN_VALUE, null)), "fake-project", "fake-instance", - "fake-app-profile"); + "fake-app-profile", + OAuth2Credentials.create(new AccessToken(TOKEN_VALUE, null)), + ImmutableMap.of("bigtable-feature", "fake-feature")); channel = ManagedChannelBuilder.forAddress("localhost", server.getPort()).usePlaintext().build(); @@ -133,7 +135,7 @@ public PingAndWarmResponse apply(PingAndWarmRequest pingAndWarmRequest) { assertThat(logHandler.logs).hasSize(1); for (LogRecord log : logHandler.logs) { - assertThat(log.getMessage()).contains("FAILED_PRECONDITION"); + assertThat(log.getThrown().getMessage()).contains("FAILED_PRECONDITION"); } } @@ -146,7 +148,21 @@ public void testChannelErrorsAreLogged() { assertThat(logHandler.logs).hasSize(1); for (LogRecord log : logHandler.logs) { - assertThat(log.getMessage()).contains("UnsupportedOperationException"); + assertThat(log.getThrown()).isInstanceOf(UnsupportedOperationException.class); + } + } + + @Test + public void testHeadersAreSent() { + primer.primeChannel(channel); + + for (Metadata metadata : metadataInterceptor.metadataList) { + assertThat(metadata.get(BigtableChannelPrimer.REQUEST_PARAMS)) + .isEqualTo( + "name=projects%2Ffake-project%2Finstances%2Ffake-instance&app_profile_id=fake-app-profile"); + assertThat( + metadata.get(Metadata.Key.of("bigtable-feature", Metadata.ASCII_STRING_MARSHALLER))) + .isEqualTo("fake-feature"); } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java index 495250fe13..fcdb4a0624 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java @@ -84,8 +84,10 @@ import com.google.cloud.bigtable.data.v2.models.TableId; import com.google.cloud.bigtable.data.v2.models.sql.ResultSetMetadata; import com.google.cloud.bigtable.data.v2.models.sql.Statement; +import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider; import com.google.cloud.bigtable.data.v2.stub.sql.ExecuteQueryCallable; import com.google.cloud.bigtable.data.v2.stub.sql.SqlServerStream; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Queues; import com.google.common.io.BaseEncoding; @@ -97,7 +99,6 @@ import io.grpc.CallOptions; import io.grpc.Context; import io.grpc.Deadline; -import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.Metadata; import io.grpc.Metadata.Key; @@ -172,6 +173,7 @@ public void setUp() throws IOException, IllegalAccessException, InstantiationExc .setInstanceId(INSTANCE_ID) .setAppProfileId(APP_PROFILE_ID) .setCredentialsProvider(NoCredentialsProvider.create()) + .setMetricsProvider(NoopMetricsProvider.INSTANCE) .build() .getStubSettings(); @@ -187,9 +189,6 @@ public void tearDown() { @Test public void testJwtAudience() throws InterruptedException, IOException, NoSuchAlgorithmException, ExecutionException { - // close default stub - need to create custom one - enhancedBigtableStub.close(); - // Create fake jwt creds KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); KeyPair keyPair = keyGen.genKeyPair(); @@ -210,9 +209,10 @@ public void testJwtAudience() .setJwtAudienceMapping(ImmutableMap.of("localhost", expectedAudience)) .setCredentialsProvider(FixedCredentialsProvider.create(jwtCreds)) .build(); - enhancedBigtableStub = EnhancedBigtableStub.create(settings); + try (EnhancedBigtableStub stub = EnhancedBigtableStub.create(settings)) { + stub.readRowCallable().futureCall(Query.create("fake-table")).get(); + } // Send rpc and grab the credentials sent - enhancedBigtableStub.readRowCallable().futureCall(Query.create("fake-table")).get(); Metadata metadata = metadataInterceptor.headers.take(); String authValue = metadata.get(Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER)); @@ -226,9 +226,6 @@ public void testJwtAudience() @Test public void testBatchJwtAudience() throws InterruptedException, IOException, NoSuchAlgorithmException, ExecutionException { - // close default stub - need to create custom one - enhancedBigtableStub.close(); - // Create fake jwt creds KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); KeyPair keyPair = keyGen.genKeyPair(); @@ -241,31 +238,30 @@ public void testBatchJwtAudience() .setPrivateKeyId("fake-private-key") .build(); - // Create a fixed channel that will ignore the default endpoint and connect to the emulator - ManagedChannel emulatorChannel = - ManagedChannelBuilder.forAddress("localhost", server.getPort()).usePlaintext().build(); + EnhancedBigtableStubSettings settings = + EnhancedBigtableStubSettings.newBuilder() + .setProjectId("fake-project") + .setInstanceId("fake-instance") + .setEndpoint("batch-bigtable.googleapis.com:443") + .setCredentialsProvider(FixedCredentialsProvider.create(jwtCreds)) + .setMetricsProvider(NoopMetricsProvider.INSTANCE) + // Use a fixed channel that will ignore the default endpoint and connect to the emulator + .setTransportChannelProvider( + FixedTransportChannelProvider.create( + GrpcTransportChannel.create( + ManagedChannelBuilder.forAddress("localhost", server.getPort()) + .usePlaintext() + .build()))) + // Channel refreshing doesn't work with FixedTransportChannelProvider. Disable it for + // the test + .setRefreshingChannel(false) + .build(); Metadata metadata; - try { - EnhancedBigtableStubSettings settings = - EnhancedBigtableStubSettings.newBuilder() - .setProjectId("fake-project") - .setInstanceId("fake-instance") - .setEndpoint("batch-bigtable.googleapis.com:443") - .setCredentialsProvider(FixedCredentialsProvider.create(jwtCreds)) - .setTransportChannelProvider( - FixedTransportChannelProvider.create( - GrpcTransportChannel.create(emulatorChannel))) - // Channel refreshing doesn't work with FixedTransportChannelProvider. Disable it for - // the test - .setRefreshingChannel(false) - .build(); - enhancedBigtableStub = EnhancedBigtableStub.create(settings); + try (EnhancedBigtableStub stub = EnhancedBigtableStub.create(settings)) { // Send rpc and grab the credentials sent - enhancedBigtableStub.readRowCallable().futureCall(Query.create("fake-table")).get(); + stub.readRowCallable().futureCall(Query.create("fake-table")).get(); metadata = metadataInterceptor.headers.take(); - } finally { - emulatorChannel.shutdown(); } String authValue = metadata.get(Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER)); @@ -278,7 +274,6 @@ public void testBatchJwtAudience() @Test public void testFeatureFlags() throws InterruptedException, IOException, ExecutionException { - enhancedBigtableStub.readRowCallable().futureCall(Query.create("fake-table")).get(); Metadata metadata = metadataInterceptor.headers.take(); @@ -291,6 +286,28 @@ public void testFeatureFlags() throws InterruptedException, IOException, Executi assertThat(featureFlags.getLastScannedRowResponses()).isTrue(); } + @Test + public void testPingAndWarmFeatureFlags() + throws InterruptedException, IOException, ExecutionException { + EnhancedBigtableStubSettings settings = + defaultSettings.toBuilder().setRefreshingChannel(true).build(); + try (EnhancedBigtableStub ignored = EnhancedBigtableStub.create(settings)) { + Preconditions.checkState( + !fakeDataService.pingRequests.isEmpty(), "Ping request was not sent during setup"); + Metadata metadata = metadataInterceptor.headers.take(); + + String encodedFeatureFlags = + metadata.get(Key.of("bigtable-features", Metadata.ASCII_STRING_MARSHALLER)); + FeatureFlags featureFlags = + FeatureFlags.parseFrom(BaseEncoding.base64Url().decode(encodedFeatureFlags)); + + assertThat(featureFlags.getReverseScans()).isTrue(); + assertThat(featureFlags.getLastScannedRowResponses()).isTrue(); + assertThat(featureFlags.getRoutingCookie()).isTrue(); + assertThat(featureFlags.getRetryInfo()).isTrue(); + } + } + @Test public void testCheckAndMutateRequestResponseConversion() throws ExecutionException, InterruptedException { @@ -825,15 +842,16 @@ public void testExecuteQueryWaitTimeoutWorksWithMetadataFuture() settings.setStreamWatchdogProvider( InstantiatingWatchdogProvider.create().withCheckInterval(WATCHDOG_CHECK_DURATION)); - EnhancedBigtableStub stub = EnhancedBigtableStub.create(settings.build()); - ApiFuture future = - stub.executeQueryCallable().call(Statement.of(WAIT_TIME_QUERY)).metadataFuture(); - - ExecutionException e = assertThrows(ExecutionException.class, future::get); - assertThat(e.getCause()).isInstanceOf(WatchdogTimeoutException.class); - assertThat(e.getCause().getMessage()) - .contains("Canceled due to timeout waiting for next response"); - assertThat(e).hasMessageThat().contains("Canceled due to timeout waiting for next response"); + try (EnhancedBigtableStub stub = EnhancedBigtableStub.create(settings.build())) { + ApiFuture future = + stub.executeQueryCallable().call(Statement.of(WAIT_TIME_QUERY)).metadataFuture(); + + ExecutionException e = assertThrows(ExecutionException.class, future::get); + assertThat(e.getCause()).isInstanceOf(WatchdogTimeoutException.class); + assertThat(e.getCause().getMessage()) + .contains("Canceled due to timeout waiting for next response"); + assertThat(e).hasMessageThat().contains("Canceled due to timeout waiting for next response"); + } } private static class MetadataInterceptor implements ServerInterceptor { diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporterTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporterTest.java index 81629e2d9d..657db7d8ae 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporterTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporterTest.java @@ -24,7 +24,10 @@ import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.TABLE_ID_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.ZONE_ID_KEY; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.api.Distribution; @@ -35,6 +38,7 @@ import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.cloud.monitoring.v3.stub.MetricServiceStub; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.monitoring.v3.CreateTimeSeriesRequest; import com.google.monitoring.v3.TimeSeries; import com.google.protobuf.Empty; @@ -53,6 +57,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.List; +import java.util.Map; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -89,7 +95,7 @@ public void setUp() { exporter = new BigtableCloudMonitoringExporter( - projectId, fakeMetricServiceClient, /* applicationResource= */ null, taskId); + fakeMetricServiceClient, /* applicationResource= */ null, taskId); attributes = Attributes.builder() @@ -301,7 +307,6 @@ public void testTimeSeriesForMetricWithGceOrGkeResource() { String gceProjectId = "fake-gce-project"; BigtableCloudMonitoringExporter exporter = new BigtableCloudMonitoringExporter( - projectId, fakeMetricServiceClient, MonitoredResource.newBuilder() .setType("gce-instance") @@ -377,6 +382,114 @@ public void testTimeSeriesForMetricWithGceOrGkeResource() { taskId); } + @Test + public void testExportingToMultipleProjects() { + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); + + UnaryCallable mockCallable = mock(UnaryCallable.class); + when(mockMetricServiceStub.createServiceTimeSeriesCallable()).thenReturn(mockCallable); + ApiFuture future = ApiFutures.immediateFuture(Empty.getDefaultInstance()); + when(mockCallable.futureCall(any())).thenReturn(future); + + long startEpoch = 10; + long endEpoch = 15; + HistogramPointData histogramPointData1 = + ImmutableHistogramPointData.create( + startEpoch, + endEpoch, + attributes, + 3d, + true, + 1d, // min + true, + 2d, // max + Arrays.asList(1.0), + Arrays.asList(1L, 2L)); + + MetricData histogramData1 = + ImmutableMetricData.createDoubleHistogram( + resource, + scope, + "bigtable.googleapis.com/internal/client/operation_latencies", + "description", + "ms", + ImmutableHistogramData.create( + AggregationTemporality.CUMULATIVE, ImmutableList.of(histogramPointData1))); + + HistogramPointData histogramPointData2 = + ImmutableHistogramPointData.create( + startEpoch, + endEpoch, + attributes.toBuilder().put(BIGTABLE_PROJECT_ID_KEY, "another-project").build(), + 50d, + true, + 5d, // min + true, + 30d, // max + Arrays.asList(1.0), + Arrays.asList(5L, 10L)); + + MetricData histogramData2 = + ImmutableMetricData.createDoubleHistogram( + resource, + scope, + "bigtable.googleapis.com/internal/client/operation_latencies", + "description", + "ms", + ImmutableHistogramData.create( + AggregationTemporality.CUMULATIVE, ImmutableList.of(histogramPointData2))); + + exporter.export(Arrays.asList(histogramData1, histogramData2)); + + verify(mockCallable, times(2)).futureCall(argumentCaptor.capture()); + + List allValues = argumentCaptor.getAllValues(); + + assertThat(allValues).hasSize(2); + + List> labelsMap = new ArrayList<>(); + List counts = new ArrayList<>(); + allValues.forEach( + value -> { + labelsMap.add(value.getTimeSeriesList().get(0).getResource().getLabelsMap()); + counts.add( + value + .getTimeSeriesList() + .get(0) + .getPoints(0) + .getValue() + .getDistributionValue() + .getCount()); + }); + + assertThat(labelsMap) + .containsExactly( + ImmutableMap.of( + BIGTABLE_PROJECT_ID_KEY.getKey(), + projectId, + INSTANCE_ID_KEY.getKey(), + instanceId, + TABLE_ID_KEY.getKey(), + tableId, + CLUSTER_ID_KEY.getKey(), + cluster, + ZONE_ID_KEY.getKey(), + zone), + ImmutableMap.of( + BIGTABLE_PROJECT_ID_KEY.getKey(), + "another-project", + INSTANCE_ID_KEY.getKey(), + instanceId, + TABLE_ID_KEY.getKey(), + tableId, + CLUSTER_ID_KEY.getKey(), + cluster, + ZONE_ID_KEY.getKey(), + zone)); + assertThat(counts).containsExactly(3l, 15l); + } + private static class FakeMetricServiceClient extends MetricServiceClient { protected FakeMetricServiceClient(MetricServiceStub stub) { diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java index a12dd3cfbd..91b650e6a8 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java @@ -44,6 +44,7 @@ import com.google.cloud.bigtable.data.v2.models.RowMutation; import com.google.cloud.bigtable.data.v2.models.SampleRowKeysRequest; import com.google.cloud.bigtable.data.v2.models.TableId; +import com.google.cloud.bigtable.data.v2.stub.BigtableClientContext; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub; import com.google.common.collect.ImmutableMap; import io.grpc.ForwardingServerCall.SimpleForwardingServerCall; @@ -126,10 +127,11 @@ public void sendHeaders(Metadata headers) { .setAppProfileId(APP_PROFILE_ID) .build(); + BigtableClientContext bigtableClientContext = + EnhancedBigtableStub.createBigtableClientContext(settings.getStubSettings()); ClientContext clientContext = - EnhancedBigtableStub.createClientContext(settings.getStubSettings()); - clientContext = - clientContext + bigtableClientContext + .getClientContext() .toBuilder() .setTracerFactory( EnhancedBigtableStub.createBigtableTracerFactory( @@ -152,10 +154,11 @@ public void sendHeaders(Metadata headers) { .setAppProfileId(APP_PROFILE_ID) .build(); + BigtableClientContext noHeaderBigtableClientContext = + EnhancedBigtableStub.createBigtableClientContext(noHeaderSettings.getStubSettings()); ClientContext noHeaderClientContext = - EnhancedBigtableStub.createClientContext(noHeaderSettings.getStubSettings()); - noHeaderClientContext = - noHeaderClientContext + noHeaderBigtableClientContext + .getClientContext() .toBuilder() .setTracerFactory( EnhancedBigtableStub.createBigtableTracerFactory( diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java index d72eac4056..b651f231da 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java @@ -38,6 +38,7 @@ import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; +import com.google.cloud.bigtable.data.v2.stub.BigtableClientContext; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub; import com.google.cloud.bigtable.data.v2.stub.mutaterows.MutateRowsBatchingDescriptor; import com.google.common.base.Stopwatch; @@ -120,10 +121,11 @@ public void setUp() throws Exception { .setAppProfileId(APP_PROFILE_ID) .build(); + BigtableClientContext bigtableClientContext = + EnhancedBigtableStub.createBigtableClientContext(settings.getStubSettings()); ClientContext clientContext = - EnhancedBigtableStub.createClientContext(settings.getStubSettings()); - clientContext = - clientContext + bigtableClientContext + .getClientContext() .toBuilder() .setTracerFactory( EnhancedBigtableStub.createBigtableTracerFactory( diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index 98e495a22b..94231ba7fa 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.48.0 + 2.49.0 grpc-google-cloud-bigtable-admin-v2 GRPC library for grpc-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 2.48.0 + 2.49.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.48.0 + 2.49.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.48.0 + 2.49.0 pom import diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index aabc9c0816..b09f14695d 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.48.0 + 2.49.0 grpc-google-cloud-bigtable-v2 GRPC library for grpc-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 2.48.0 + 2.49.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.48.0 + 2.49.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.48.0 + 2.49.0 pom import diff --git a/pom.xml b/pom.xml index 2d2408f133..0a945a3853 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ google-cloud-bigtable-parent pom - 2.48.0 + 2.49.0 Google Cloud Bigtable Parent https://github.com/googleapis/java-bigtable @@ -153,27 +153,27 @@ com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.48.0 + 2.49.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.48.0 + 2.49.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.48.0 + 2.49.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.48.0 + 2.49.0 com.google.cloud google-cloud-bigtable - 2.48.0 + 2.49.0 diff --git a/proto-google-cloud-bigtable-admin-v2/pom.xml b/proto-google-cloud-bigtable-admin-v2/pom.xml index 829bf5e42a..48f59de7e1 100644 --- a/proto-google-cloud-bigtable-admin-v2/pom.xml +++ b/proto-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.48.0 + 2.49.0 proto-google-cloud-bigtable-admin-v2 PROTO library for proto-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 2.48.0 + 2.49.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.48.0 + 2.49.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.48.0 + 2.49.0 pom import diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index 56dbfefa75..9b61631b52 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.48.0 + 2.49.0 proto-google-cloud-bigtable-v2 PROTO library for proto-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 2.48.0 + 2.49.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.48.0 + 2.49.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.48.0 + 2.49.0 pom import diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 93dc0ee8cc..30a503385d 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigtable - 2.48.0 + 2.49.0 diff --git a/test-proxy/pom.xml b/test-proxy/pom.xml index ff90a1d171..1bd83d098a 100644 --- a/test-proxy/pom.xml +++ b/test-proxy/pom.xml @@ -12,11 +12,11 @@ google-cloud-bigtable-parent com.google.cloud - 2.48.0 + 2.49.0 - 2.48.0 + 2.49.0 diff --git a/versions.txt b/versions.txt index 7362862a52..021d20cb62 100644 --- a/versions.txt +++ b/versions.txt @@ -1,10 +1,10 @@ # Format: # module:released-version:current-version -google-cloud-bigtable:2.48.0:2.48.0 -grpc-google-cloud-bigtable-admin-v2:2.48.0:2.48.0 -grpc-google-cloud-bigtable-v2:2.48.0:2.48.0 -proto-google-cloud-bigtable-admin-v2:2.48.0:2.48.0 -proto-google-cloud-bigtable-v2:2.48.0:2.48.0 -google-cloud-bigtable-emulator:0.185.0:0.185.0 -google-cloud-bigtable-emulator-core:0.185.0:0.185.0 +google-cloud-bigtable:2.49.0:2.49.0 +grpc-google-cloud-bigtable-admin-v2:2.49.0:2.49.0 +grpc-google-cloud-bigtable-v2:2.49.0:2.49.0 +proto-google-cloud-bigtable-admin-v2:2.49.0:2.49.0 +proto-google-cloud-bigtable-v2:2.49.0:2.49.0 +google-cloud-bigtable-emulator:0.186.0:0.186.0 +google-cloud-bigtable-emulator-core:0.186.0:0.186.0