feat: Add firestoreInDatastoreMode for datastore emulator (#1698)

* feat: Add firestoreInDatastoreMode for datastore emulator

* chore: generate libraries at Mon Jan  6 17:28:37 UTC 2025

* update emulator min version

* update version to 2.2.2

* update version to 2.3.0

* update version to 2.2.1

* update version to 2.3.1

* chore: generate libraries at Mon Jan 13 23:32:38 UTC 2025

* Fix ci testing environment with java 11

* Add Java 11 enviroment to run tests

* Fix java env formatting

* set maven env

* set maven env

* testing java 11

* Add sample test env variable

* Add sample test env variable

* Add sample test env variable

* run with java8 but build with 11

* Add java 11 as java_home sample build

* Add java_home for presubmit

* Fix SUREFIRE_JVM_OPT for sample build

* Fix JAVA_HOME for sample build

* Fix JAVA_HOME for sample build

* Modify Java env for presubmit integration build

* Add setjava to builder script

* Add setjava to builder script

* Add setjava to builder script

* Add debugging info to builder script

* Add debugging info to builder script

* Add debugging info to builder script

* Add debugging info to builder script

* formatting

* Add debugging

* Add debugging

* Add debugging

* Add debugging

* Add debugging

* Add debugging

* Resolve conflicts

* Change sample build to use java11 container

* chore: generate libraries at Thu Jan 23 23:20:47 UTC 2025

---------

Co-authored-by: cloud-java-bot <[email protected]>

Author: cindy-peng

.cloudbuild/samples_build.yaml

@@ -1,19 +1,19 @@
 steps:
-- name: gcr.io/cloud-devrel-public-resources/java8
+- name: gcr.io/cloud-devrel-public-resources/java11
   entrypoint: ls
   args: [
     '-alt',
   ]
-- name: gcr.io/cloud-devrel-public-resources/java8
+- name: gcr.io/cloud-devrel-public-resources/java11
   entrypoint: curl
   args: [
     '--header',
     'Metadata-Flavor: Google',
     'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email'
   ]
-- name: gcr.io/cloud-devrel-public-resources/java8
+- name: gcr.io/cloud-devrel-public-resources/java11
   entrypoint: pwd
-- name: gcr.io/cloud-devrel-public-resources/java8
+- name: gcr.io/cloud-devrel-public-resources/java11
   entrypoint: bash
   args: [
     '.kokoro/build.sh'
@@ -22,7 +22,7 @@ steps:
   - 'JOB_TYPE=samples'
   - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-sample'
   - 'KOKORO_GITHUB_PULL_REQUEST_NUMBER=$_PR_NUMBER'
-- name: gcr.io/cloud-devrel-public-resources/java8
+- name: gcr.io/cloud-devrel-public-resources/java11
   entrypoint: echo
   args: [
     'Sample job succeeded',

.github/workflows/ci.yaml

@@ -59,15 +59,23 @@ jobs:
         env:
           JOB_TYPE: test
   windows:
+    # Building using Java 11 and run the tests with Java 8 runtime
     runs-on: windows-latest
     steps:
     - name: Support longpaths
       run: git config --system core.longpaths true
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
       with:
-        distribution: temurin
         java-version: 8
+        distribution: temurin
+    - name: "Set jvm system property environment variable for surefire plugin (unit tests)"
+      run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}\bin\java" >> $GITHUB_ENV
+      shell: bash
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 11
     - run: java -version
     - run: .kokoro/build.bat
       env:

.kokoro/build.sh

@@ -60,6 +60,18 @@ javadoc)
     RETURN_CODE=$?
     ;;
 integration)
+    # Kokoro integration tests use both JDK 11 and JDK 8. Integration
+    # tests require JDK 11 export as JAVA env variable to run cloud datastore
+    # emulator (https://cloud.google.com/sdk/docs/release-notes#39300_2022-07-12).
+    # For Java 8 environment, we will still run the tests using Java 8 with
+    # SUREFIRE_JVM_OPT for Maven surefire plugin:
+    # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm
+    if [[ -n "${JAVA11_HOME}"  &&  -n "${JAVA8_HOME}" ]]
+    then
+      export JAVA=${JAVA11_HOME}/bin/java
+      export SUREFIRE_JVM_OPT=-Djvm=${JAVA8_HOME}/bin/java
+    fi
+
     mvn -B ${INTEGRATION_TEST_ARGS} \
       -ntp \
       -Penable-integration-tests \

.kokoro/common.sh

@@ -52,7 +52,7 @@ function retry_with_backoff {
     return $exit_code
 }
 
-## Helper functionss
+## Helper functions
 function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
 function msg() { println "$*" >&2; }
 function println() { printf '%s\n' "$(now) $*"; }

.kokoro/presubmit/integration.cfg

@@ -35,4 +35,4 @@ env_vars: {
 env_vars: {
   key: "SECRET_MANAGER_KEYS"
   value: "java-it-service-account"
-}
+}

google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/LocalDatastoreHelper.java

@@ -53,12 +53,13 @@ public class LocalDatastoreHelper extends BaseEmulatorHelper<DatastoreOptions> {
   private final double consistency;
   private final Path gcdPath;
   private boolean storeOnDisk;
+  private boolean firestoreInDatastoreMode;
 
   // Gcloud emulator settings
   private static final String GCLOUD_CMD_TEXT = "gcloud beta emulators datastore start";
   private static final String GCLOUD_CMD_PORT_FLAG = "--host-port=";
   private static final String VERSION_PREFIX = "cloud-datastore-emulator ";
-  private static final String MIN_VERSION = "2.0.2";
+  private static final String MIN_VERSION = "2.3.1";
 
   // Downloadable emulator settings
   private static final String BIN_NAME = "cloud-datastore-emulator/cloud_datastore_emulator";
@@ -74,6 +75,8 @@ public class LocalDatastoreHelper extends BaseEmulatorHelper<DatastoreOptions> {
   private static final String PROJECT_FLAG = "--project=";
   private static final double DEFAULT_CONSISTENCY = 0.9;
   private static final String DEFAULT_PROJECT_ID = PROJECT_ID_PREFIX + UUID.randomUUID();
+  private static final String FIRESTORE_IN_DATASTORE_MODE_FLAG =
+      "--use-firestore-in-datastore-mode";
 
   private static final Logger LOGGER = Logger.getLogger(LocalDatastoreHelper.class.getName());
 
@@ -102,6 +105,7 @@ public static class Builder {
     private int port;
     private Path dataDir;
     private boolean storeOnDisk = true;
+    private boolean firestoreInDatastoreMode = false;
     private String projectId;
 
     private Builder() {}
@@ -110,6 +114,7 @@ private Builder(LocalDatastoreHelper helper) {
       this.consistency = helper.consistency;
       this.dataDir = helper.gcdPath;
       this.storeOnDisk = helper.storeOnDisk;
+      this.firestoreInDatastoreMode = helper.firestoreInDatastoreMode;
     }
 
     public Builder setConsistency(double consistency) {
@@ -137,6 +142,11 @@ public Builder setStoreOnDisk(boolean storeOnDisk) {
       return this;
     }
 
+    public Builder setFirestoreInDatastoreMode(boolean firestoreInDatastoreMode) {
+      this.firestoreInDatastoreMode = firestoreInDatastoreMode;
+      return this;
+    }
+
     /** Creates a {@code LocalDatastoreHelper} object. */
     public LocalDatastoreHelper build() {
       return new LocalDatastoreHelper(this);
@@ -152,14 +162,21 @@ private LocalDatastoreHelper(Builder builder) {
     this.consistency = builder.consistency > 0 ? builder.consistency : DEFAULT_CONSISTENCY;
     this.gcdPath = builder.dataDir;
     this.storeOnDisk = builder.storeOnDisk;
+    this.firestoreInDatastoreMode = builder.firestoreInDatastoreMode;
     String binName = BIN_NAME;
     if (isWindows()) {
       binName = BIN_NAME.replace("/", "\\");
     }
     List<String> gcloudCommand = new ArrayList<>(Arrays.asList(GCLOUD_CMD_TEXT.split(" ")));
     gcloudCommand.add(GCLOUD_CMD_PORT_FLAG + "localhost:" + getPort());
-    gcloudCommand.add(CONSISTENCY_FLAG + builder.consistency);
     gcloudCommand.add(PROJECT_FLAG + projectId);
+    if (builder.firestoreInDatastoreMode) {
+      gcloudCommand.add(FIRESTORE_IN_DATASTORE_MODE_FLAG);
+    } else {
+      // At most one of --consistency | --use-firestore-in-datastore-mode can be specified.
+      // --consistency will be ignored with --use-firestore-in-datastore-mode.
+      gcloudCommand.add(CONSISTENCY_FLAG + builder.consistency);
+    }
     if (!builder.storeOnDisk) {
       gcloudCommand.add("--no-store-on-disk");
     }
@@ -170,8 +187,16 @@ private LocalDatastoreHelper(Builder builder) {
         new GcloudEmulatorRunner(gcloudCommand, VERSION_PREFIX, MIN_VERSION);
     List<String> binCommand = new ArrayList<>(Arrays.asList(binName, "start"));
     binCommand.add("--testing");
+    if (builder.firestoreInDatastoreMode) {
+      // Downloadable emulator runner takes the flag in a different
+      // format: --firestore_in_datastore_mode
+      binCommand.add("--firestore_in_datastore_mode");
+    } else {
+      // At most one of --consistency | --firestore_in_datastore_mode can be specified.
+      // --consistency will be ignored with --firestore_in_datastore_mode.
+      binCommand.add(CONSISTENCY_FLAG + getConsistency());
+    }
     binCommand.add(BIN_CMD_PORT_FLAG + getPort());
-    binCommand.add(CONSISTENCY_FLAG + getConsistency());
     DownloadableEmulatorRunner downloadRunner =
         new DownloadableEmulatorRunner(binCommand, EMULATOR_URL, MD5_CHECKSUM, ACCESS_TOKEN);
     this.emulatorRunners = ImmutableList.of(gcloudRunner, downloadRunner);
@@ -235,6 +260,13 @@ public boolean isStoreOnDisk() {
     return storeOnDisk;
   }
 
+  /**
+   * Returns {@code true} use firestore-in-datastore-mode, otherwise {@code false} use native mode.
+   */
+  public boolean isFirestoreInDatastoreMode() {
+    return firestoreInDatastoreMode;
+  }
+
   /**
    * Creates a local Datastore helper with the specified settings for project ID and consistency.
    *

google-cloud-datastore/src/test/java/com/google/cloud/datastore/testing/ITLocalDatastoreHelperTest.java

@@ -76,11 +76,13 @@ public void testCreateWithBuilder() {
             .setConsistency(0.75)
             .setPort(8081)
             .setStoreOnDisk(false)
+            .setFirestoreInDatastoreMode(true)
             .setDataDir(dataDir)
             .build();
     assertTrue(Math.abs(0.75 - helper.getConsistency()) < TOLERANCE);
     assertTrue(helper.getProjectId().startsWith(PROJECT_ID_PREFIX));
     assertFalse(helper.isStoreOnDisk());
+    assertTrue(helper.isFirestoreInDatastoreMode());
     assertEquals(8081, helper.getPort());
     assertEquals(dataDir, helper.getGcdPath());
     LocalDatastoreHelper incompleteHelper = LocalDatastoreHelper.newBuilder().build();
@@ -103,11 +105,13 @@ public void testCreateWithToBuilder() throws IOException {
             .setConsistency(0.75)
             .setPort(8081)
             .setStoreOnDisk(false)
+            .setFirestoreInDatastoreMode(true)
             .setDataDir(dataDir)
             .build();
     assertTrue(Math.abs(0.75 - helper.getConsistency()) < TOLERANCE);
     assertTrue(helper.getProjectId().startsWith(PROJECT_ID_PREFIX));
     assertFalse(helper.isStoreOnDisk());
+    assertTrue(helper.isFirestoreInDatastoreMode());
     assertEquals(8081, helper.getPort());
     assertEquals(dataDir, helper.getGcdPath());
     LocalDatastoreHelper actualHelper = helper.toBuilder().build();
@@ -119,10 +123,12 @@ public void testCreateWithToBuilder() throws IOException {
             .setConsistency(0.85)
             .setPort(9091)
             .setStoreOnDisk(true)
+            .setFirestoreInDatastoreMode(false)
             .setDataDir(dataDir)
             .build();
     assertTrue(Math.abs(0.85 - actualHelper.getConsistency()) < TOLERANCE);
     assertTrue(actualHelper.isStoreOnDisk());
+    assertFalse(actualHelper.isFirestoreInDatastoreMode());
     assertEquals(9091, actualHelper.getPort());
     assertEquals(dataDir, actualHelper.getGcdPath());
     LocalDatastoreHelper.deleteRecursively(dataDir);
@@ -206,10 +212,28 @@ public void testStartStopResetWithBuilder()
     }
   }
 
+  @Test
+  public void testCreateWithFirestoreInDatastoreMode()
+      throws IOException, InterruptedException, TimeoutException {
+    LocalDatastoreHelper helper =
+        LocalDatastoreHelper.newBuilder().setFirestoreInDatastoreMode(true).build();
+    assertTrue(helper.isFirestoreInDatastoreMode());
+    helper.start();
+    Datastore datastore = helper.getOptions().getService();
+    Key key = datastore.newKeyFactory().setKind("kind").newKey("name");
+    Entity expected = Entity.newBuilder(key).build();
+    datastore.put(expected);
+    assertNotNull(datastore.get(key));
+    Entity actual = datastore.get(key);
+    assertEquals(expected, actual);
+    helper.stop();
+  }
+
   public void assertLocalDatastoreHelpersEquivelent(
       LocalDatastoreHelper expected, LocalDatastoreHelper actual) {
     assertEquals(expected.getConsistency(), actual.getConsistency(), 0);
     assertEquals(expected.isStoreOnDisk(), actual.isStoreOnDisk());
+    assertEquals(expected.isFirestoreInDatastoreMode(), actual.isFirestoreInDatastoreMode());
     assertEquals(expected.getGcdPath(), actual.getGcdPath());
   }
 }