Skip to content

Commit

Permalink
feat: add support to customize gcloud command of LocalDatastoreHelper (…
Browse files Browse the repository at this point in the history 
…#137)

Refactor creation of LocalDatastoreHelper to provide a builder (in addition 
to the existing factory methods) which allow finer grained configuration 
including the ability to set a storage directory to be passed when starting
the emulator.
  • Loading branch information
@athakor
athakor committed Jun 4, 2020
1 parent d572efb commit 976d979
Show file tree
Hide file tree
Showing 2 changed files with 176 additions and 22 deletions.
105 changes: 83 additions & 22 deletions ...loud-datastore/src/main/java/com/google/cloud/datastore/testing/LocalDatastoreHelper.java
View file
Expand Up @@ -35,7 +35,6 @@
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.threeten.bp.Duration;

Expand All @@ -50,6 +49,7 @@ public class LocalDatastoreHelper extends BaseEmulatorHelper<DatastoreOptions> {
private final List<EmulatorRunner> emulatorRunners;
private final double consistency;
private final Path gcdPath;
private boolean storeOnDisk;

// Gcloud emulator settings
private static final String GCLOUD_CMD_TEXT = "gcloud beta emulators datastore start";
Expand Down Expand Up @@ -78,39 +78,87 @@ public class LocalDatastoreHelper extends BaseEmulatorHelper<DatastoreOptions> {
}
}

private LocalDatastoreHelper(double consistency, int port) {
/** A builder for {@code LocalDatastoreHelper} objects. */
public static class Builder {
private double consistency;
private int port;
private Path dataDir;
private boolean storeOnDisk = true;

private Builder() {}

private Builder(LocalDatastoreHelper helper) {
this.consistency = helper.consistency;
this.dataDir = helper.gcdPath;
this.storeOnDisk = helper.storeOnDisk;
}

public Builder setConsistency(double consistency) {
this.consistency = consistency;
return this;
}

public Builder setPort(int port) {
this.port = port;
return this;
}

public Builder setDataDir(Path dataDir) {
this.dataDir = dataDir;
return this;
}

public Builder setStoreOnDisk(boolean storeOnDisk) {
this.storeOnDisk = storeOnDisk;
return this;
}

/** Creates a {@code LocalDatastoreHelper} object. */
public LocalDatastoreHelper build() {
return new LocalDatastoreHelper(this);
}
}

private LocalDatastoreHelper(Builder builder) {
super(
"datastore",
port > 0 ? port : BaseEmulatorHelper.findAvailablePort(DEFAULT_PORT),
builder.port > 0 ? builder.port : BaseEmulatorHelper.findAvailablePort(DEFAULT_PORT),
PROJECT_ID_PREFIX + UUID.randomUUID().toString());
Path tmpDirectory = null;
try {
tmpDirectory = Files.createTempDirectory("gcd");
} catch (IOException ex) {
getLogger().log(Level.WARNING, "Failed to create temporary directory");
}
this.gcdPath = tmpDirectory;
this.consistency = consistency;
this.consistency = builder.consistency > 0 ? builder.consistency : DEFAULT_CONSISTENCY;
this.gcdPath = builder.dataDir;
this.storeOnDisk = builder.storeOnDisk;
String binName = BIN_NAME;
if (isWindows()) {
binName = BIN_NAME.replace("/", "\\");
}
List<String> gcloudCommand = new ArrayList<>(Arrays.asList(GCLOUD_CMD_TEXT.split(" ")));
gcloudCommand.add(GCLOUD_CMD_PORT_FLAG + "localhost:" + getPort());
gcloudCommand.add(CONSISTENCY_FLAG + consistency);
gcloudCommand.add("--no-store-on-disk");
gcloudCommand.add(CONSISTENCY_FLAG + builder.consistency);
if (!builder.storeOnDisk) {
gcloudCommand.add("--no-store-on-disk");
}
GcloudEmulatorRunner gcloudRunner =
new GcloudEmulatorRunner(gcloudCommand, VERSION_PREFIX, MIN_VERSION);
List<String> binCommand = new ArrayList<>(Arrays.asList(binName, "start"));
binCommand.add("--testing");
binCommand.add(BIN_CMD_PORT_FLAG + getPort());
binCommand.add(CONSISTENCY_FLAG + consistency);
if (gcdPath != null) {
gcloudCommand.add("--data-dir=" + gcdPath.toString());
binCommand.add(CONSISTENCY_FLAG + getConsistency());
if (builder.dataDir != null) {
gcloudCommand.add("--data-dir=" + getGcdPath());
}
DownloadableEmulatorRunner downloadRunner =
new DownloadableEmulatorRunner(binCommand, EMULATOR_URL, MD5_CHECKSUM);
emulatorRunners = ImmutableList.of(gcloudRunner, downloadRunner);
this.emulatorRunners = ImmutableList.of(gcloudRunner, downloadRunner);
}

/** Returns a builder for {@code LocalDatastoreHelper} object. */
public LocalDatastoreHelper.Builder toBuilder() {
return new Builder(this);
}

/** Returns a builder for {@code LocalDatastoreHelper} object. */
public static LocalDatastoreHelper.Builder newBuilder() {
return new LocalDatastoreHelper.Builder();
}

@Override
Expand Down Expand Up @@ -153,6 +201,16 @@ public double getConsistency() {
return consistency;
}

/** Returns the data directory path of the local Datastore emulator. */
public Path getGcdPath() {
return gcdPath;
}

/** Returns {@code true} data persist on disk, otherwise {@code false} data not store on disk. */
public boolean isStoreOnDisk() {
return storeOnDisk;
}

/**
* Creates a local Datastore helper with the specified settings for project ID and consistency.
*
Expand All @@ -162,7 +220,7 @@ public double getConsistency() {
* consistency of non-ancestor queries; non-ancestor queries are eventually consistent.
*/
public static LocalDatastoreHelper create(double consistency) {
return create(consistency, 0);
return LocalDatastoreHelper.newBuilder().setConsistency(consistency).setPort(0).build();
}

/**
Expand All @@ -176,7 +234,7 @@ public static LocalDatastoreHelper create(double consistency) {
* emulator will search for a free random port.
*/
public static LocalDatastoreHelper create(double consistency, int port) {
return new LocalDatastoreHelper(consistency, port);
return LocalDatastoreHelper.newBuilder().setConsistency(consistency).setPort(port).build();
}

/**
Expand All @@ -187,7 +245,10 @@ public static LocalDatastoreHelper create(double consistency, int port) {
* emulator will search for a free random port.
*/
public static LocalDatastoreHelper create(int port) {
return new LocalDatastoreHelper(DEFAULT_CONSISTENCY, port);
return LocalDatastoreHelper.newBuilder()
.setConsistency(DEFAULT_CONSISTENCY)
.setPort(port)
.build();
}

/**
Expand All @@ -197,7 +258,7 @@ public static LocalDatastoreHelper create(int port) {
* all writes are immediately visible.
*/
public static LocalDatastoreHelper create() {
return create(DEFAULT_CONSISTENCY);
return LocalDatastoreHelper.newBuilder().setConsistency(DEFAULT_CONSISTENCY).build();
}

/**
Expand Down Expand Up @@ -254,7 +315,7 @@ public void stop() throws IOException, InterruptedException, TimeoutException {
stop(Duration.ofSeconds(20));
}

private static void deleteRecursively(Path path) throws IOException {
static void deleteRecursively(Path path) throws IOException {
if (path == null || !Files.exists(path)) {
return;
}
Expand Down
93 changes: 93 additions & 0 deletions ...atastore/src/test/java/com/google/cloud/datastore/testing/ITLocalDatastoreHelperTest.java
View file
Expand Up @@ -30,8 +30,12 @@
import com.google.cloud.datastore.Entity;
import com.google.cloud.datastore.Key;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.TimeoutException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
Expand All @@ -43,6 +47,17 @@ public class ITLocalDatastoreHelperTest {
private static final double TOLERANCE = 0.00001;
private static final String PROJECT_ID_PREFIX = "test-project-";
private static final String NAMESPACE = "namespace";
private Path dataDir;

@Before
public void setUp() throws IOException {
dataDir = Files.createTempDirectory("gcd");
}

@After
public void tearDown() throws IOException {
LocalDatastoreHelper.deleteRecursively(dataDir);
}

@Test
public void testCreate() {
Expand All @@ -54,6 +69,57 @@ public void testCreate() {
assertTrue(helper.getProjectId().startsWith(PROJECT_ID_PREFIX));
}

@Test
public void testCreateWithBuilder() {
LocalDatastoreHelper helper =
LocalDatastoreHelper.newBuilder()
.setConsistency(0.75)
.setPort(8081)
.setStoreOnDisk(false)
.setDataDir(dataDir)
.build();
assertTrue(Math.abs(0.75 - helper.getConsistency()) < TOLERANCE);
assertTrue(helper.getProjectId().startsWith(PROJECT_ID_PREFIX));
assertFalse(helper.isStoreOnDisk());
assertEquals(8081, helper.getPort());
assertEquals(dataDir, helper.getGcdPath());
LocalDatastoreHelper incompleteHelper = LocalDatastoreHelper.newBuilder().build();
assertTrue(Math.abs(0.9 - incompleteHelper.getConsistency()) < TOLERANCE);
assertTrue(incompleteHelper.getProjectId().startsWith(PROJECT_ID_PREFIX));
}

@Test
public void testCreateWithToBuilder() throws IOException {
LocalDatastoreHelper helper =
LocalDatastoreHelper.newBuilder()
.setConsistency(0.75)
.setPort(8081)
.setStoreOnDisk(false)
.setDataDir(dataDir)
.build();
assertTrue(Math.abs(0.75 - helper.getConsistency()) < TOLERANCE);
assertTrue(helper.getProjectId().startsWith(PROJECT_ID_PREFIX));
assertFalse(helper.isStoreOnDisk());
assertEquals(8081, helper.getPort());
assertEquals(dataDir, helper.getGcdPath());
LocalDatastoreHelper actualHelper = helper.toBuilder().build();
assertLocalDatastoreHelpersEquivelent(helper, actualHelper);
Path dataDir = Files.createTempDirectory("gcd_data_dir");
actualHelper =
helper
.toBuilder()
.setConsistency(0.85)
.setPort(9091)
.setStoreOnDisk(true)
.setDataDir(dataDir)
.build();
assertTrue(Math.abs(0.85 - actualHelper.getConsistency()) < TOLERANCE);
assertTrue(actualHelper.isStoreOnDisk());
assertEquals(9091, actualHelper.getPort());
assertEquals(dataDir, actualHelper.getGcdPath());
LocalDatastoreHelper.deleteRecursively(dataDir);
}

@Test
public void testCreatePort() {
LocalDatastoreHelper helper = LocalDatastoreHelper.create(0.75, 8888);
Expand Down Expand Up @@ -103,4 +169,31 @@ public void testStartStopReset() throws IOException, InterruptedException, Timeo
assertNotNull(ex.getMessage());
}
}

@Test
public void testStartStopResetWithBuilder()
throws IOException, InterruptedException, TimeoutException {
try {
LocalDatastoreHelper helper = LocalDatastoreHelper.newBuilder().build();
helper.start();
Datastore datastore = helper.getOptions().getService();
Key key = datastore.newKeyFactory().setKind("kind").newKey("name");
datastore.put(Entity.newBuilder(key).build());
assertNotNull(datastore.get(key));
helper.reset();
assertNull(datastore.get(key));
helper.stop(Duration.ofMinutes(1));
datastore.get(key);
Assert.fail();
} catch (DatastoreException ex) {
assertNotNull(ex.getMessage());
}
}

public void assertLocalDatastoreHelpersEquivelent(
LocalDatastoreHelper expected, LocalDatastoreHelper actual) {
assertEquals(expected.getConsistency(), actual.getConsistency(), 0);
assertEquals(expected.isStoreOnDisk(), actual.isStoreOnDisk());
assertEquals(expected.getGcdPath(), actual.getGcdPath());
}
}

1 comment on commit 976d979

@jacobg
Copy link

@jacobg jacobg commented on 976d979 Nov 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like LocalDatastoreHelper.reset() hangs if storeOnDisk is true.

Please sign in to comment.