diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 42327db5e2..459487d388 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:f14e3fefe8e361e85752bd9890c8e56f2fe25f1e89cbb9597e4e3c7a429203a3 + digest: sha256:ae72564aa9c368b9ccd96f4af21f87889fd83b9e60635b80844deb5a2ccd08aa diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index d7476cfe97..bd8960246f 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -57,54 +57,3 @@ retry_with_backoff 3 10 \ -Dclirr.skip=true mvn -B dependency:analyze -DfailOnWarning=true - -echo "****************** DEPENDENCY LIST COMPLETENESS CHECK *******************" -## Run dependency list completeness check -function completenessCheck() { - # Output dep list with compile scope generated using the original pom - # Running mvn dependency:list on Java versions that support modules will also include the module of the dependency. - # This is stripped from the output as it is not present in the flattened pom. - # Only dependencies with 'compile' or 'runtime' scope are included from original dependency list. - msg "Generating dependency list using original pom..." - mvn dependency:list -f pom.xml -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' | sed -e 's/ --.*//' >.org-list.txt - - # Output dep list generated using the flattened pom (only 'compile' and 'runtime' scopes) - msg "Generating dependency list using flattened pom..." - mvn dependency:list -f .flattened-pom.xml -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt - - # Compare two dependency lists - msg "Comparing dependency lists..." - diff .org-list.txt .new-list.txt >.diff.txt - if [[ $? == 0 ]] - then - msg "Success. No diff!" - else - msg "Diff found. See below: " - msg "You can also check .diff.txt file located in $1." - cat .diff.txt - return 1 - fi -} - -# Allow failures to continue running the script -set +e - -error_count=0 -for path in **/.flattened-pom.xml -do - # Check flattened pom in each dir that contains it for completeness - dir=$(dirname "$path") - pushd "$dir" - completenessCheck "$dir" - error_count=$(($error_count + $?)) - popd -done - -if [[ $error_count == 0 ]] -then - msg "All checks passed." - exit 0 -else - msg "Errors found. See log statements above." - exit 1 -fi diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in index cfdc2e7ed9..6aa7cf2b55 100644 --- a/.kokoro/requirements.in +++ b/.kokoro/requirements.in @@ -16,10 +16,12 @@ pycparser==2.21 pyperclip==1.8.2 python-dateutil==2.8.2 requests==2.27.1 +certifi==2022.9.24 importlib-metadata==4.8.3 zipp==3.6.0 google_api_core==2.8.2 google-cloud-storage==2.0.0 +google-resumable-media==2.3.3 google-cloud-core==2.3.1 typing-extensions==4.1.1 urllib3==1.26.12 diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 170f1c63ab..02ae42bb40 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -16,10 +16,12 @@ cachetools==4.2.4 \ # via # -r requirements.in # google-auth -certifi==2022.9.14 \ - --hash=sha256:36973885b9542e6bd01dea287b2b4b3b21236307c56324fcc3f1160f2d655ed5 \ - --hash=sha256:e232343de1ab72c2aa521b625c80f699e356830fd0e2c620b465b304b17b0516 - # via requests +certifi==2022.9.24 \ + --hash=sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14 \ + --hash=sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382 + # via + # -r requirements.in + # requests cffi==1.15.1 \ --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ @@ -218,7 +220,9 @@ google-crc32c==1.3.0 \ google-resumable-media==2.3.3 \ --hash=sha256:27c52620bd364d1c8116eaac4ea2afcbfb81ae9139fb3199652fcac1724bfb6c \ --hash=sha256:5b52774ea7a829a8cdaa8bd2d4c3d4bc660c91b30857ab2668d0eb830f4ea8c5 - # via google-cloud-storage + # via + # -r requirements.in + # google-cloud-storage googleapis-common-protos==1.56.3 \ --hash=sha256:6f1369b58ed6cf3a4b7054a44ebe8d03b29c309257583a2bbdc064cd1e4a1442 \ --hash=sha256:87955d7b3a73e6e803f2572a33179de23989ebba725e05ea42f24838b792e461 diff --git a/CHANGELOG.md b/CHANGELOG.md index 38b415aabb..2f28f8bf6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [2.15.0](https://github.com/googleapis/java-bigtable/compare/v2.14.1...v2.15.0) (2022-10-26) + + +### Features + +* Add apis for Mutation and RowMutationEntry ([#1454](https://github.com/googleapis/java-bigtable/issues/1454)) ([a86934f](https://github.com/googleapis/java-bigtable/commit/a86934f4f70e1998e7368688df8695a4bac7006e)) + + +### Bug Fixes + +* Catch all throwables so version mismatch won't hang the client ([#1402](https://github.com/googleapis/java-bigtable/issues/1402)) ([c03b8a4](https://github.com/googleapis/java-bigtable/commit/c03b8a4245beab7258b8ed8ec76153e6a0275211)) +* Fix attempt status tag for metrics ([#1477](https://github.com/googleapis/java-bigtable/issues/1477)) ([e54cf7d](https://github.com/googleapis/java-bigtable/commit/e54cf7d61128049cea2817b409a8524636b4a6cf)) +* Rename metric names to match the external name ([#1479](https://github.com/googleapis/java-bigtable/issues/1479)) ([28ca7c3](https://github.com/googleapis/java-bigtable/commit/28ca7c310d9a157529555ab6865f89802161443b)) +* Set a longer timeout to wait for callbacks to be executed ([#1478](https://github.com/googleapis/java-bigtable/issues/1478)) ([0d9ff6a](https://github.com/googleapis/java-bigtable/commit/0d9ff6ab3bd5bf853f1d371312036b14c924ae5e)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-monitoring-bom to v3.5.0 ([#1464](https://github.com/googleapis/java-bigtable/issues/1464)) ([d8e58a5](https://github.com/googleapis/java-bigtable/commit/d8e58a5d88de7ea249d7bc141f1ac65592088da6)) +* Update dependency com.google.cloud:google-cloud-monitoring-bom to v3.6.0 ([#1476](https://github.com/googleapis/java-bigtable/issues/1476)) ([331dcfb](https://github.com/googleapis/java-bigtable/commit/331dcfbd06ab578cfe9673ca70cab1f6abd5ddcd)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.0.5 ([#1470](https://github.com/googleapis/java-bigtable/issues/1470)) ([557a4fb](https://github.com/googleapis/java-bigtable/commit/557a4fbed77157c3ccfc3e84a0c952b647844f9b)) +* Update dependency org.graalvm.buildtools:junit-platform-native to v0.9.15 ([#1462](https://github.com/googleapis/java-bigtable/issues/1462)) ([69540cb](https://github.com/googleapis/java-bigtable/commit/69540cb2df25d5fb716e81e7513930b01c310b72)) +* Update dependency org.graalvm.buildtools:junit-platform-native to v0.9.16 ([#1467](https://github.com/googleapis/java-bigtable/issues/1467)) ([53599ca](https://github.com/googleapis/java-bigtable/commit/53599caa0f92bad1365adbc4b58a1dcb1e8a393e)) +* Update dependency org.graalvm.buildtools:native-maven-plugin to v0.9.15 ([#1463](https://github.com/googleapis/java-bigtable/issues/1463)) ([a6612f9](https://github.com/googleapis/java-bigtable/commit/a6612f90cd4a0ec9589ca797ff3a42d23478a6e6)) +* Update dependency org.graalvm.buildtools:native-maven-plugin to v0.9.16 ([#1468](https://github.com/googleapis/java-bigtable/issues/1468)) ([fe0ddb1](https://github.com/googleapis/java-bigtable/commit/fe0ddb1b183cfdf368c2387a645a613bbfd52a69)) + ## [2.14.1](https://github.com/googleapis/java-bigtable/compare/v2.14.0...v2.14.1) (2022-10-05) diff --git a/README.md b/README.md index 2a14cbe979..de9341d00a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.1.2 + 26.1.3 pom import @@ -41,7 +41,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigtable - 2.13.0 + 2.14.1 ``` @@ -49,20 +49,20 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.1.2') +implementation platform('com.google.cloud:libraries-bom:26.1.3') implementation 'com.google.cloud:google-cloud-bigtable' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigtable:2.14.0' +implementation 'com.google.cloud:google-cloud-bigtable:2.14.1' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.14.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.14.1" ``` ## Authentication @@ -490,6 +490,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-bigtable/tree | Filters | [source code](https://github.com/googleapis/java-bigtable/blob/main/samples/snippets/src/main/java/com/example/bigtable/Filters.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigtable&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigtable/Filters.java) | | Hello World | [source code](https://github.com/googleapis/java-bigtable/blob/main/samples/snippets/src/main/java/com/example/bigtable/HelloWorld.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigtable&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigtable/HelloWorld.java) | | Instance Admin Example | [source code](https://github.com/googleapis/java-bigtable/blob/main/samples/snippets/src/main/java/com/example/bigtable/InstanceAdminExample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigtable&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigtable/InstanceAdminExample.java) | +| Key Salting | [source code](https://github.com/googleapis/java-bigtable/blob/main/samples/snippets/src/main/java/com/example/bigtable/KeySalting.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigtable&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigtable/KeySalting.java) | | Quickstart | [source code](https://github.com/googleapis/java-bigtable/blob/main/samples/snippets/src/main/java/com/example/bigtable/Quickstart.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigtable&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigtable/Quickstart.java) | | Reads | [source code](https://github.com/googleapis/java-bigtable/blob/main/samples/snippets/src/main/java/com/example/bigtable/Reads.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigtable&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigtable/Reads.java) | | Table Admin Example | [source code](https://github.com/googleapis/java-bigtable/blob/main/samples/snippets/src/main/java/com/example/bigtable/TableAdminExample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigtable&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigtable/TableAdminExample.java) | diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index 17a3fb8595..3d0667837e 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -3,12 +3,12 @@ 4.0.0 com.google.cloud google-cloud-bigtable-bom - 2.14.1 + 2.15.0 pom com.google.cloud google-cloud-shared-config - 1.5.1 + 1.5.4 @@ -63,42 +63,42 @@ com.google.cloud google-cloud-bigtable - 2.14.1 + 2.15.0 com.google.cloud google-cloud-bigtable-emulator - 0.151.1 + 0.152.0 com.google.cloud google-cloud-bigtable-emulator-core - 0.151.1 + 0.152.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.14.1 + 2.15.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.14.1 + 2.15.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.14.1 + 2.15.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.14.1 + 2.15.0 com.google.cloud google-cloud-bigtable-stats - 2.14.1 + 2.15.0 diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index 46462bd8f6..ef43d9f78c 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -7,13 +7,13 @@ com.google.cloud google-cloud-shared-config - 1.5.1 + 1.5.4 com.google.cloud google-cloud-bigtable-deps-bom - 2.14.1 + 2.15.0 pom @@ -66,14 +66,14 @@ com.google.cloud google-cloud-shared-dependencies - 3.0.4 + 3.0.5 pom import com.google.cloud google-cloud-monitoring-bom - 3.4.6 + 3.6.0 pom import diff --git a/google-cloud-bigtable-emulator-core/pom.xml b/google-cloud-bigtable-emulator-core/pom.xml index e1d07d140f..0c63523866 100644 --- a/google-cloud-bigtable-emulator-core/pom.xml +++ b/google-cloud-bigtable-emulator-core/pom.xml @@ -7,11 +7,11 @@ google-cloud-bigtable-parent com.google.cloud - 2.14.1 + 2.15.0 google-cloud-bigtable-emulator-core - 0.151.1 + 0.152.0 A Java wrapper for the Cloud Bigtable emulator. diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml index 502de53013..b608c31a99 100644 --- a/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-bigtable-emulator/pom.xml @@ -5,7 +5,7 @@ 4.0.0 google-cloud-bigtable-emulator - 0.151.1 + 0.152.0 Google Cloud Java - Bigtable Emulator https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-bigtable-parent - 2.14.1 + 2.15.0 scm:git:git@github.com:googleapis/java-bigtable.git @@ -81,14 +81,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.14.1 + 2.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.14.1 + 2.15.0 pom import @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigtable-emulator-core - 0.151.1 + 0.152.0 diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index ac76c40278..a52b749deb 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -5,7 +5,7 @@ com.google.cloud google-cloud-bigtable-parent - 2.14.1 + 2.15.0 4.0.0 @@ -13,7 +13,7 @@ through Stackdriver. Built-in metrics will be implemented with shaded OpenCensus so it won't interfere with customer's application metrics. --> google-cloud-bigtable-stats - 2.14.1 + 2.15.0 Experimental project to shade OpenCensus dependencies. @@ -21,7 +21,7 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.14.1 + 2.15.0 pom import diff --git a/google-cloud-bigtable/clirr-ignored-differences.xml b/google-cloud-bigtable/clirr-ignored-differences.xml index 8a3edd69c0..2f7631c873 100644 --- a/google-cloud-bigtable/clirr-ignored-differences.xml +++ b/google-cloud-bigtable/clirr-ignored-differences.xml @@ -49,7 +49,7 @@ 8001 com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracer$Builder - change method args is ok because HeaderTracerStreamingCallable is InternalApi + 7004 com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerStreamingCallable @@ -76,4 +76,9 @@ 8001 com/google/cloud/bigtable/data/v2/stub/readrows/ReadRowsConvertExceptionCallable + + 5001 + com/google/cloud/bigtable/gaxx/reframing/ReframingResponseObserver + com/google/api/gax/rpc/StateCheckingResponseObserver + diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index c3eb629779..c5baa2ca44 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-bigtable - 2.14.1 + 2.15.0 jar Google Cloud Bigtable https://github.com/googleapis/java-bigtable @@ -12,11 +12,11 @@ com.google.cloud google-cloud-bigtable-parent - 2.14.1 + 2.15.0 - 2.14.1 + 2.15.0 google-cloud-bigtable @@ -43,14 +43,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.14.1 + 2.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.14.1 + 2.15.0 pom import diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java index 1a7f920145..8fdc3f3a97 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java @@ -20,6 +20,6 @@ @InternalApi("For internal use only") public final class Version { // {x-version-update-start:google-cloud-bigtable:current} - public static String VERSION = "2.14.1"; + public static String VERSION = "2.15.0"; // {x-version-update-end} } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/PartialListClustersException.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/PartialListClustersException.java index 7b95983225..f04bdf326e 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/PartialListClustersException.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/PartialListClustersException.java @@ -33,7 +33,7 @@ public class PartialListClustersException extends RuntimeException { */ @InternalApi public PartialListClustersException(List unavailableZones, List clusters) { - super("Failed to list all instances, some zones where unavailable"); + super("Failed to list all clusters, some zones were unavailable: " + unavailableZones); this.unavailableZones = ImmutableList.copyOf(unavailableZones); this.clusters = ImmutableList.copyOf(clusters); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/PartialListInstancesException.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/PartialListInstancesException.java index d4d5d4e992..a24d66fb27 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/PartialListInstancesException.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/PartialListInstancesException.java @@ -31,7 +31,7 @@ public class PartialListInstancesException extends RuntimeException { @InternalApi public PartialListInstancesException( @Nonnull List unavailableZones, @Nonnull List instances) { - super("Failed to list all instances, some zones where unavailable"); + super("Failed to list all instances, some zones were unavailable: " + unavailableZones); this.unavailableZones = ImmutableList.copyOf(unavailableZones); this.instances = ImmutableList.copyOf(instances); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java index 326d78cfe0..6c1402685d 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java @@ -87,6 +87,19 @@ public static Mutation fromProtoUnsafe(List pro return mutation; } + /** + * Wraps the List of protobuf {@link com.google.bigtable.v2.Mutation}. This methods, like {@link + * #createUnsafe()}, allows setCell operation to use server side timestamp. This is dangerous + * because mutations will no longer be idempotent, which might cause multiple duplicate values to + * be stored in Bigtable. This option should only be used for advanced usecases with extreme care. + */ + @BetaApi + public static Mutation fromProtoUnsafe(Iterable protos) { + Mutation mutation = new Mutation(true); + mutation.mutations.addAll(protos); + return mutation; + } + /** * Constructs a row mutation from an existing protobuf object. * diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutationEntry.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutationEntry.java index 9239b30497..63ffe708a5 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutationEntry.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutationEntry.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.MutateRowsRequest; import com.google.common.base.Preconditions; @@ -54,6 +55,13 @@ public static RowMutationEntry create(@Nonnull ByteString key) { return new RowMutationEntry(key, Mutation.create()); } + /** Creates a new instance from existing mutation. */ + @BetaApi + public static RowMutationEntry createFromMutationUnsafe( + @Nonnull ByteString key, @Nonnull Mutation mutation) { + return new RowMutationEntry(key, mutation); + } + /** * Creates new instance of mutation builder which allows server timestamp for setCell operations. * diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallable.java index ed50532fae..d3ff88af7e 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/ConvertExceptionCallable.java @@ -42,31 +42,32 @@ public void call( innerCallable.call(request, observer, context); } - private class ReadRowsConvertExceptionResponseObserver implements ResponseObserver { + private class ReadRowsConvertExceptionResponseObserver extends SafeResponseObserver { private final ResponseObserver outerObserver; ReadRowsConvertExceptionResponseObserver(ResponseObserver outerObserver) { + super(outerObserver); this.outerObserver = outerObserver; } @Override - public void onStart(StreamController controller) { + protected void onStartImpl(StreamController controller) { outerObserver.onStart(controller); } @Override - public void onResponse(RowT response) { + protected void onResponseImpl(RowT response) { outerObserver.onResponse(response); } @Override - public void onError(Throwable t) { + protected void onErrorImpl(Throwable t) { outerObserver.onError(convertException(t)); } @Override - public void onComplete() { + protected void onCompleteImpl() { outerObserver.onComplete(); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/SafeResponseObserver.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/SafeResponseObserver.java new file mode 100644 index 0000000000..7c65bdf95a --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/SafeResponseObserver.java @@ -0,0 +1,123 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub; + +import com.google.api.core.InternalApi; +import com.google.api.gax.rpc.ResponseObserver; +import com.google.api.gax.rpc.StreamController; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Base implementation of {@link ResponseObserver} that checks the state and catches all the + * throwables. + */ +@InternalApi +public abstract class SafeResponseObserver implements ResponseObserver { + + private static final Logger LOGGER = Logger.getLogger(SafeResponseObserver.class.getName()); + private AtomicBoolean isStarted = new AtomicBoolean(false); + private AtomicBoolean isClosed = new AtomicBoolean(false); + private StreamController streamController; + private ResponseObserver outerObserver; + + public SafeResponseObserver(ResponseObserver outerObserver) { + this.outerObserver = outerObserver; + } + + @Override + public final void onStart(StreamController streamController) { + if (!isStarted.compareAndSet(false, true)) { + throw new IllegalStateException("A stream is already started"); + } + + this.streamController = streamController; + try { + onStartImpl(streamController); + } catch (Throwable t) { + if (!isClosed.compareAndSet(false, true)) { + logException("Tried to cancel a closed stream"); + return; + } + streamController.cancel(); + outerObserver.onError(t); + } + } + + @Override + public final void onResponse(ResponseT response) { + if (isClosed.get()) { + logException("Received a response after the stream is closed"); + return; + } + try { + onResponseImpl(response); + } catch (Throwable t1) { + try { + if (!isClosed.compareAndSet(false, true)) { + logException("Tried to cancel a closed stream"); + return; + } + streamController.cancel(); + } catch (Throwable t2) { + t1.addSuppressed(t2); + } + outerObserver.onError(t1); + } + } + + @Override + public final void onError(Throwable throwable) { + if (!isClosed.compareAndSet(false, true)) { + logException("Received error after the stream is closed"); + return; + } + + try { + onErrorImpl(throwable); + } catch (Throwable t) { + throwable.addSuppressed(t); + outerObserver.onError(throwable); + } + } + + @Override + public final void onComplete() { + if (!isClosed.compareAndSet(false, true)) { + logException("Tried to double close the stream"); + return; + } + + try { + onCompleteImpl(); + } catch (Throwable t) { + outerObserver.onError(t); + } + } + + private void logException(String message) { + LOGGER.log(Level.WARNING, message, new IllegalStateException(message)); + } + + protected abstract void onStartImpl(StreamController streamController); + + protected abstract void onResponseImpl(ResponseT response); + + protected abstract void onErrorImpl(Throwable throwable); + + protected abstract void onCompleteImpl(); +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index c7f09c4db1..5ec4c726e4 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -22,6 +22,7 @@ import com.google.api.gax.rpc.ServerStreamingCallable; import com.google.api.gax.rpc.StreamController; import com.google.bigtable.v2.ResponseParams; +import com.google.cloud.bigtable.data.v2.stub.SafeResponseObserver; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.protobuf.InvalidProtocolBufferException; @@ -68,7 +69,7 @@ public void call( } } - private class BigtableTracerResponseObserver implements ResponseObserver { + private class BigtableTracerResponseObserver extends SafeResponseObserver { private final BigtableTracer tracer; private final ResponseObserver outerObserver; @@ -78,26 +79,28 @@ private class BigtableTracerResponseObserver implements ResponseObser ResponseObserver observer, BigtableTracer tracer, GrpcResponseMetadata metadata) { + super(observer); + this.tracer = tracer; this.outerObserver = observer; this.responseMetadata = metadata; } @Override - public void onStart(final StreamController controller) { + protected void onStartImpl(final StreamController controller) { TracedStreamController tracedController = new TracedStreamController(controller, tracer); outerObserver.onStart(tracedController); } @Override - public void onResponse(ResponseT response) { + protected void onResponseImpl(ResponseT response) { Stopwatch stopwatch = Stopwatch.createStarted(); outerObserver.onResponse(response); tracer.afterResponse(stopwatch.elapsed(TimeUnit.MILLISECONDS)); } @Override - public void onError(Throwable t) { + protected void onErrorImpl(Throwable t) { // server-timing metric will be added through GrpcResponseMetadata#onHeaders(Metadata), // so it's not checking trailing metadata here. Metadata metadata = responseMetadata.getMetadata(); @@ -122,13 +125,14 @@ public void onError(Throwable t) { } } } catch (InvalidProtocolBufferException e) { + t.addSuppressed(t); } outerObserver.onError(t); } @Override - public void onComplete() { + protected void onCompleteImpl() { Metadata metadata = responseMetadata.getMetadata(); Long latency = Util.getGfeLatency(metadata); tracer.recordGfeMetadata(latency, null); @@ -151,6 +155,9 @@ public void onComplete() { } } } catch (InvalidProtocolBufferException e) { + // InvalidProtocolBufferException will only throw if something changed on + // the server side. Location info won't be populated as a result. Ignore + // this error and don't bubble it up to user. } outerObserver.onComplete(); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java index 5e29065860..4704f04a4d 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java @@ -17,6 +17,7 @@ import static com.google.api.gax.tracing.ApiTracerFactory.OperationType; +import com.google.api.gax.retrying.ServerStreamingAttemptException; import com.google.api.gax.tracing.SpanName; import com.google.cloud.bigtable.stats.StatsRecorderWrapper; import com.google.common.annotations.VisibleForTesting; @@ -256,6 +257,14 @@ private void recordAttemptCompletion(@Nullable Throwable status) { serverLatencyTimerIsRunning = false; } } + + // Patch the status until it's fixed in gax. When an attempt failed, + // it'll throw a ServerStreamingAttemptException. Unwrap the exception + // so it could get processed by extractStatus + if (status instanceof ServerStreamingAttemptException) { + status = status.getCause(); + } + recorder.putAttemptLatencies(attemptTimer.elapsed(TimeUnit.MILLISECONDS)); recorder.recordAttempt(Util.extractStatus(status), tableId, zone, cluster); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java index 3c63b1b5f7..3b6b1b40ae 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.stub.metrics; +import com.google.api.gax.retrying.ServerStreamingAttemptException; import com.google.api.gax.tracing.ApiTracerFactory.OperationType; import com.google.api.gax.tracing.SpanName; import com.google.common.base.Stopwatch; @@ -166,6 +167,13 @@ private void recordAttemptCompletion(@Nullable Throwable throwable) { RpcMeasureConstants.BIGTABLE_ATTEMPT_LATENCY, attemptTimer.elapsed(TimeUnit.MILLISECONDS)); + // Patch the throwable until it's fixed in gax. When an attempt failed, + // it'll throw a ServerStreamingAttemptException. Unwrap the exception + // so it could get processed by extractStatus + if (throwable instanceof ServerStreamingAttemptException) { + throwable = throwable.getCause(); + } + TagContextBuilder tagCtx = newTagCtxBuilder() .putLocal( diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/FilterMarkerRowsCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/FilterMarkerRowsCallable.java index 57f987fb7c..181006b6c3 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/FilterMarkerRowsCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/FilterMarkerRowsCallable.java @@ -22,6 +22,7 @@ import com.google.api.gax.rpc.StreamController; import com.google.bigtable.v2.ReadRowsRequest; import com.google.cloud.bigtable.data.v2.models.RowAdapter; +import com.google.cloud.bigtable.data.v2.stub.SafeResponseObserver; /** * Remove the special marker rows generated by {@link RowMergingCallable}. @@ -47,17 +48,18 @@ public void call( innerCallable.call(request, innerObserver, context); } - private class FilteringResponseObserver implements ResponseObserver { + private class FilteringResponseObserver extends SafeResponseObserver { private final ResponseObserver outerObserver; private StreamController innerController; private boolean autoFlowControl = true; FilteringResponseObserver(ResponseObserver outerObserver) { + super(outerObserver); this.outerObserver = outerObserver; } @Override - public void onStart(final StreamController controller) { + protected void onStartImpl(final StreamController controller) { innerController = controller; outerObserver.onStart( @@ -81,7 +83,7 @@ public void request(int count) { } @Override - public void onResponse(RowT response) { + protected void onResponseImpl(RowT response) { if (rowAdapter.isScanMarkerRow(response)) { if (!autoFlowControl) { innerController.request(1); @@ -92,12 +94,12 @@ public void onResponse(RowT response) { } @Override - public void onError(Throwable t) { + protected void onErrorImpl(Throwable t) { outerObserver.onError(t); } @Override - public void onComplete() { + protected void onCompleteImpl() { outerObserver.onComplete(); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/reframing/ReframingResponseObserver.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/reframing/ReframingResponseObserver.java index c46eb55ff9..6f2440fff7 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/reframing/ReframingResponseObserver.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/reframing/ReframingResponseObserver.java @@ -17,8 +17,8 @@ import com.google.api.core.InternalApi; import com.google.api.gax.rpc.ResponseObserver; -import com.google.api.gax.rpc.StateCheckingResponseObserver; import com.google.api.gax.rpc.StreamController; +import com.google.cloud.bigtable.data.v2.stub.SafeResponseObserver; import com.google.common.base.Preconditions; import com.google.common.math.IntMath; import java.util.concurrent.CancellationException; @@ -56,8 +56,7 @@ * } */ @InternalApi -public class ReframingResponseObserver - extends StateCheckingResponseObserver { +public class ReframingResponseObserver extends SafeResponseObserver { // Used as a nonblocking mutex for deliver(). // 0 means unlocked // 1 means locked without contention @@ -97,6 +96,7 @@ public class ReframingResponseObserver public ReframingResponseObserver( ResponseObserver observer, Reframer reframer) { + super(observer); this.outerResponseObserver = observer; this.reframer = reframer; } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index 76c2adfb52..34201d961a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -51,7 +51,7 @@ public class BuiltinMetricsIT { "retry_count", "attempt_latencies", "connectivity_error_count", - "application_latencies" + "application_blocking_latencies" }; @BeforeClass @@ -125,9 +125,9 @@ public void testBuiltinMetrics() throws Exception { // Verify that metrics are published for ReadRows request metricFilter = String.format( - "metric.type=\"bigtable.googleapis.com/client/operation_latencies\" " + "metric.type=\"bigtable.googleapis.com/client/%s\" " + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.ReadRows\"", - testEnvRule.env().getInstanceId()); + view, testEnvRule.env().getInstanceId()); requestBuilder.setFilter(metricFilter); response = metricClient.listTimeSeriesCallable().call(requestBuilder.build()); assertThat(response.getTimeSeriesCount()).isGreaterThan(0); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index d64570488a..76bb7384ed 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -313,10 +313,12 @@ public void testRetryCount() { stub.mutateRowCallable() .call(RowMutation.create(TABLE_ID, "random-row").setCell("cf", "q", "value")); - // onOperationComplete() is called in TracerFinisher which will be called after the mutateRow - // call is returned. So there's a race between when the call returns and when the putRetryCount - // is called in onOperationCompletion(). - verify(statsRecorderWrapper, timeout(20)).putRetryCount(retryCount.capture()); + // In TracedUnaryCallable, we create a future and add a TraceFinisher to the callback. Main + // thread is blocked on waiting for the future to be completed. When onComplete is called on + // the grpc thread, the future is completed, however we might not have enough time for + // TraceFinisher to run. Add a 1 second time out to wait for the callback. This shouldn't have + // any impact on production code. + verify(statsRecorderWrapper, timeout(1000)).putRetryCount(retryCount.capture()); assertThat(retryCount.getValue()).isEqualTo(fakeService.getAttemptCounter().get() - 1); } diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index 9aa416a599..308e5b3fb5 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.14.1 + 2.15.0 grpc-google-cloud-bigtable-admin-v2 GRPC library for grpc-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 2.14.1 + 2.15.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.14.1 + 2.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.14.1 + 2.15.0 pom import diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index 0635259b3c..d6b1e51ace 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.14.1 + 2.15.0 grpc-google-cloud-bigtable-v2 GRPC library for grpc-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 2.14.1 + 2.15.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.14.1 + 2.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.14.1 + 2.15.0 pom import diff --git a/pom.xml b/pom.xml index 9d602166d9..a64d00d373 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ google-cloud-bigtable-parent pom - 2.14.1 + 2.15.0 Google Cloud Bigtable Parent https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.5.3 + 1.5.4 @@ -153,27 +153,27 @@ com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.14.1 + 2.15.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.14.1 + 2.15.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.14.1 + 2.15.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.14.1 + 2.15.0 com.google.cloud google-cloud-bigtable - 2.14.1 + 2.15.0 @@ -207,7 +207,7 @@ org.mockito mockito-core - 4.8.0 + 4.8.1 diff --git a/proto-google-cloud-bigtable-admin-v2/pom.xml b/proto-google-cloud-bigtable-admin-v2/pom.xml index 80140de0c7..c71bc4f23b 100644 --- a/proto-google-cloud-bigtable-admin-v2/pom.xml +++ b/proto-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.14.1 + 2.15.0 proto-google-cloud-bigtable-admin-v2 PROTO library for proto-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 2.14.1 + 2.15.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.14.1 + 2.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.14.1 + 2.15.0 pom import diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index 22930bdd02..56c5aa9b2c 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.14.1 + 2.15.0 proto-google-cloud-bigtable-v2 PROTO library for proto-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 2.14.1 + 2.15.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.14.1 + 2.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.14.1 + 2.15.0 pom import diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 27713f1c19..c56163f327 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-bigtable - 2.13.0 + 2.14.1 diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index cdda421cd4..b9bb862a0e 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -29,7 +29,7 @@ com.google.cloud libraries-bom - 26.1.2 + 26.1.3 pom import @@ -89,7 +89,7 @@ org.graalvm.buildtools junit-platform-native - 0.9.14 + 0.9.16 test @@ -99,7 +99,7 @@ org.graalvm.buildtools native-maven-plugin - 0.9.14 + 0.9.16 true com.example.bigtable.NativeImageBigtableSample diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index d902dbbdf2..3a23e24610 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigtable - 2.14.1 + 2.15.0 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 77a1725039..0facce3f6e 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 26.1.2 + 26.1.3 pom import diff --git a/samples/snippets/src/main/java/com/example/bigtable/KeySalting.java b/samples/snippets/src/main/java/com/example/bigtable/KeySalting.java new file mode 100644 index 0000000000..da5e401347 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigtable/KeySalting.java @@ -0,0 +1,95 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +import com.google.api.core.ApiFuture; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class KeySalting { + private static final String COLUMN_FAMILY_NAME = "stats_summary"; + public static final int SALT_RANGE = 4; + + public static void writeSaltedRow( + String projectId, String instanceId, String tableId, String rowKey) throws IOException { + BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId); + String saltedRowKey = getSaltedRowKey(rowKey, SALT_RANGE); + RowMutation rowMutation = + RowMutation.create(tableId, saltedRowKey) + .setCell(COLUMN_FAMILY_NAME, "os_build", "PQ2A.190405.003"); + + dataClient.mutateRow(rowMutation); + System.out.printf("Successfully wrote row %s as %s\n", rowKey, saltedRowKey); + + dataClient.close(); + } + + public static void readSaltedRow( + String projectId, String instanceId, String tableId, String rowKey) throws IOException { + BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId); + Row row = dataClient.readRow(tableId, getSaltedRowKey(rowKey, SALT_RANGE)); + System.out.printf("Successfully read row %s\n", row.getKey().toStringUtf8()); + } + + public static void scanSaltedRows( + String projectId, String instanceId, String tableId, String prefix) + throws IOException, ExecutionException, InterruptedException { + BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId); + + List queries = new ArrayList<>(); + for (int i = 0; i < SALT_RANGE; i++) { + queries.add(Query.create(tableId).prefix(i + "-" + prefix)); + } + + List>> futures = new ArrayList<>(); + for (Query q : queries) { + futures.add(dataClient.readRowsCallable().all().futureCall(q)); + } + + List rows = new ArrayList<>(); + for (ApiFuture> future : futures) { + rows.addAll(future.get()); + } + + System.out.printf("Successfully fetched %s rows\n", rows.size()); + for (Row row : rows) { + System.out.printf("Successfully read row %s\n", row.getKey().toStringUtf8()); + } + } + + /** + * Generates a salted version of the row key. + * + *

Some Bigtable schema designs will always have hot spots, and a salted row key breaks up + * individual rows and groups of rows that are hot. Row keys are stored in sorted order, so + * prepending a numeric prefix allows those hot rows to be stored in different locations. + * + *

The salted row key is created by hashing the existing row key and taking a modulo of how + * large a prefix range you want to create; then prepending the existing row key with that result. + * This produces a deterministic output, so each row key will always produce the same salted key. + */ + public static String getSaltedRowKey(String rowKey, int saltRange) { + int prefix = rowKey.hashCode() % saltRange; + return prefix + "-" + rowKey; + } +} diff --git a/samples/snippets/src/test/java/com/example/bigtable/KeySaltingTest.java b/samples/snippets/src/test/java/com/example/bigtable/KeySaltingTest.java new file mode 100644 index 0000000000..085d992cea --- /dev/null +++ b/samples/snippets/src/test/java/com/example/bigtable/KeySaltingTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class KeySaltingTest extends MobileTimeSeriesBaseTest { + public static final String ROW_KEY = "phone#4c410523#20190501"; + + @BeforeClass + public static void beforeClass() throws IOException { + initializeVariables(); + createTable(); + } + + @AfterClass + public static void afterClass() throws IOException { + cleanupTable(); + } + + @Test + public void testWriteAndRead() throws IOException { + KeySalting.writeSaltedRow(projectId, instanceId, TABLE_ID, ROW_KEY); + KeySalting.readSaltedRow(projectId, instanceId, TABLE_ID, ROW_KEY); + + String output = bout.toString(); + assertEquals( + "Successfully wrote row phone#4c410523#20190501 as 0-phone#4c410523#20190501\n" + + "Successfully read row 0-phone#4c410523#20190501\n", + output); + } + + @Test + public void testScans() throws IOException, ExecutionException, InterruptedException { + String prefix = "abc-"; + for (int i = 0; i < 8; i++) { + KeySalting.writeSaltedRow(projectId, instanceId, TABLE_ID, prefix + i); + } + bout.reset(); + + KeySalting.scanSaltedRows(projectId, instanceId, TABLE_ID, prefix); + + String output = bout.toString(); + assertEquals( + "Successfully fetched 8 rows\n" + + "Successfully read row 0-abc-3\n" + + "Successfully read row 0-abc-7\n" + + "Successfully read row 1-abc-0\n" + + "Successfully read row 1-abc-4\n" + + "Successfully read row 2-abc-1\n" + + "Successfully read row 2-abc-5\n" + + "Successfully read row 3-abc-2\n" + + "Successfully read row 3-abc-6\n", + output); + } + + @Test + public void testKeySalting() { + Map exampleKeys = new HashMap(); + exampleKeys.put("abc-1", "2-abc-1"); + exampleKeys.put("abc-2", "3-abc-2"); + exampleKeys.put("abc-3", "0-abc-3"); + exampleKeys.put("abc-4", "1-abc-4"); + + exampleKeys.forEach((k, v) -> assertEquals(v, KeySalting.getSaltedRowKey(k, 4))); + } +} diff --git a/versions.txt b/versions.txt index e97e03e75e..4cc3a8e52f 100644 --- a/versions.txt +++ b/versions.txt @@ -1,10 +1,10 @@ # Format: # module:released-version:current-version -google-cloud-bigtable:2.14.1:2.14.1 -grpc-google-cloud-bigtable-admin-v2:2.14.1:2.14.1 -grpc-google-cloud-bigtable-v2:2.14.1:2.14.1 -proto-google-cloud-bigtable-admin-v2:2.14.1:2.14.1 -proto-google-cloud-bigtable-v2:2.14.1:2.14.1 -google-cloud-bigtable-emulator:0.151.1:0.151.1 -google-cloud-bigtable-emulator-core:2.14.1:2.14.1 +google-cloud-bigtable:2.15.0:2.15.0 +grpc-google-cloud-bigtable-admin-v2:2.15.0:2.15.0 +grpc-google-cloud-bigtable-v2:2.15.0:2.15.0 +proto-google-cloud-bigtable-admin-v2:2.15.0:2.15.0 +proto-google-cloud-bigtable-v2:2.15.0:2.15.0 +google-cloud-bigtable-emulator:0.152.0:0.152.0 +google-cloud-bigtable-emulator-core:2.15.0:2.15.0