Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for javascript UDFs determinism level #1094

Merged
merged 3 commits into from Feb 9, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 3 additions & 8 deletions google-cloud-bigquery/clirr-ignored-differences.xml
@@ -1,15 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
<differences>
<!-- TODO: REMOVE AFTER RELEASE OF CONNECTION_ID FEAT -->
<!-- TODO: REMOVE AFTER RELEASE -->
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition</className>
<method>java.lang.String getConnectionId()</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition$Builder</className>
<method>com.google.cloud.bigquery.ExternalTableDefinition$Builder setConnectionId(java.lang.String)</method>
<className>com/google/cloud/bigquery/RoutineInfo$Builder</className>
<method>com.google.cloud.bigquery.RoutineInfo$Builder setDeterminismLevel(java.lang.String)</method>
</difference>
</differences>
Expand Up @@ -87,6 +87,12 @@ Builder setLastModifiedTime(Long lastModifiedMillis) {
return this;
}

@Override
public Builder setDeterminismLevel(String determinismLevel) {
infoBuilder.setDeterminismLevel(determinismLevel);
return this;
}

@Override
public Builder setLanguage(String language) {
infoBuilder.setLanguage(language);
Expand Down
Expand Up @@ -62,6 +62,7 @@ public Routine apply(RoutineInfo routineInfo) {
private final String routineType;
private final Long creationTime;
private final String description;
private final String determinismLevel;
private final Long lastModifiedTime;
private final String language;
private final List<RoutineArgument> argumentList;
Expand Down Expand Up @@ -89,6 +90,12 @@ public abstract static class Builder {

abstract Builder setLastModifiedTime(Long lastModifiedMillis);

/**
* Sets the JavaScript UDF determinism levels (e.g. DETERMINISM_LEVEL_UNSPECIFIED,
* DETERMINISTIC, NOT_DETERMINISTIC) only applicable to Javascript UDFs.
*/
public abstract Builder setDeterminismLevel(String determinismLevel);

/** Sets the language for the routine (e.g. SQL or JAVASCRIPT) */
public abstract Builder setLanguage(String language);

Expand Down Expand Up @@ -147,6 +154,7 @@ static class BuilderImpl extends Builder {
private String routineType;
private Long creationTime;
private String description;
private String determinismLevel;
private Long lastModifiedTime;
private String language;
private List<RoutineArgument> argumentList;
Expand All @@ -162,6 +170,7 @@ static class BuilderImpl extends Builder {
this.routineType = routineInfo.routineType;
this.creationTime = routineInfo.creationTime;
this.description = routineInfo.description;
this.determinismLevel = routineInfo.determinismLevel;
this.lastModifiedTime = routineInfo.lastModifiedTime;
this.language = routineInfo.language;
this.argumentList = routineInfo.argumentList;
Expand All @@ -176,6 +185,7 @@ static class BuilderImpl extends Builder {
this.routineType = routinePb.getRoutineType();
this.creationTime = routinePb.getCreationTime();
this.description = routinePb.getDescription();
this.determinismLevel = routinePb.getDeterminismLevel();
this.lastModifiedTime = routinePb.getLastModifiedTime();
this.language = routinePb.getLanguage();
if (routinePb.getArguments() != null) {
Expand Down Expand Up @@ -223,6 +233,12 @@ public Builder setDescription(String description) {
return this;
}

@Override
public Builder setDeterminismLevel(String determinismLevel) {
this.determinismLevel = determinismLevel;
return this;
}

@Override
Builder setLastModifiedTime(Long lastModifiedMillis) {
this.lastModifiedTime = lastModifiedMillis;
Expand Down Expand Up @@ -271,6 +287,7 @@ public RoutineInfo build() {
this.routineType = builder.routineType;
this.creationTime = builder.creationTime;
this.description = builder.description;
this.determinismLevel = builder.determinismLevel;
this.lastModifiedTime = builder.lastModifiedTime;
this.language = builder.language;
this.argumentList = builder.argumentList;
Expand Down Expand Up @@ -304,6 +321,11 @@ public String getDescription() {
return description;
}

/** Returns the determinism level of the JavaScript UDF if defined. */
public String getDeterminismLevel() {
return determinismLevel;
}

/**
* Returns the last modification time of the routine, represented as milliseconds since the epoch.
*/
Expand Down Expand Up @@ -354,6 +376,7 @@ public String toString() {
.add("routineType", routineType)
.add("creationTime", creationTime)
.add("description", description)
.add("determinismLevel", determinismLevel)
.add("lastModifiedTime", lastModifiedTime)
.add("language", language)
.add("arguments", argumentList)
Expand All @@ -371,6 +394,7 @@ public int hashCode() {
routineType,
creationTime,
description,
determinismLevel,
lastModifiedTime,
language,
argumentList,
Expand Down Expand Up @@ -412,6 +436,7 @@ Routine toPb() {
.setDefinitionBody(getBody())
.setCreationTime(getCreationTime())
.setDescription(getDescription())
.setDeterminismLevel(getDeterminismLevel())
.setLastModifiedTime(getLastModifiedTime())
.setLanguage(getLanguage());
if (getRoutineId() != null) {
Expand All @@ -420,6 +445,9 @@ Routine toPb() {
if (getArguments() != null) {
routinePb.setArguments(Lists.transform(getArguments(), RoutineArgument.TO_PB_FUNCTION));
}
if (getReturnType() != null) {
routinePb.setReturnType(getReturnType().toPb());
}
return routinePb;
}

Expand Down
Expand Up @@ -29,6 +29,7 @@ public class RoutineInfoTest {
private static final String ROUTINE_TYPE = "SCALAR_FUNCTION";
private static final Long CREATION_TIME = 10L;
private static final String DESCRIPTION = "description";
private static final String DETERMINISM = "DETERMINISTIC";
private static final Long LAST_MODIFIED_TIME = 20L;
private static final String LANGUAGE = "SQL";

Expand All @@ -55,6 +56,7 @@ public class RoutineInfoTest {
.setRoutineType(ROUTINE_TYPE)
.setCreationTime(CREATION_TIME)
.setDescription(DESCRIPTION)
.setDeterminismLevel(DETERMINISM)
.setLastModifiedTime(LAST_MODIFIED_TIME)
.setLanguage(LANGUAGE)
.setArguments(ARGUMENT_LIST)
Expand All @@ -81,6 +83,7 @@ public void testBuilder() {
assertEquals(ROUTINE_TYPE, ROUTINE_INFO.getRoutineType());
assertEquals(CREATION_TIME, ROUTINE_INFO.getCreationTime());
assertEquals(DESCRIPTION, ROUTINE_INFO.getDescription());
assertEquals(DETERMINISM, ROUTINE_INFO.getDeterminismLevel());
assertEquals(LAST_MODIFIED_TIME, ROUTINE_INFO.getLastModifiedTime());
assertEquals(LANGUAGE, ROUTINE_INFO.getLanguage());
assertEquals(ARGUMENT_LIST, ROUTINE_INFO.getArguments());
Expand All @@ -97,6 +100,7 @@ public void testOf() {
assertNull(routineInfo.getRoutineType());
assertNull(routineInfo.getCreationTime());
assertNull(routineInfo.getDescription());
assertNull(routineInfo.getDeterminismLevel());
assertNull(routineInfo.getLastModifiedTime());
assertNull(routineInfo.getLanguage());
assertNull(routineInfo.getArguments());
Expand All @@ -121,6 +125,7 @@ public void compareRoutineInfo(RoutineInfo expected, RoutineInfo value) {
assertEquals(expected.getRoutineType(), value.getRoutineType());
assertEquals(expected.getCreationTime(), value.getCreationTime());
assertEquals(expected.getDescription(), value.getDescription());
assertEquals(expected.getDeterminismLevel(), value.getDeterminismLevel());
assertEquals(expected.getLastModifiedTime(), value.getLastModifiedTime());
assertEquals(expected.getLanguage(), value.getLanguage());
assertEquals(expected.getArguments(), value.getArguments());
Expand Down
Expand Up @@ -38,6 +38,7 @@
public class RoutineTest {

private static final RoutineId ROUTINE_ID = RoutineId.of("dataset", "routine");
private static final String DETERMINISM_LEVEL = "DETERMINISTIC";
private static final String ETAG = "etag";
private static final String ROUTINE_TYPE = "SCALAR_FUNCTION";
private static final Long CREATION_TIME = 10L;
Expand Down Expand Up @@ -65,6 +66,7 @@ public class RoutineTest {
.setEtag(ETAG)
.setRoutineType(ROUTINE_TYPE)
.setCreationTime(CREATION_TIME)
.setDeterminismLevel(DETERMINISM_LEVEL)
.setLastModifiedTime(LAST_MODIFIED_TIME)
.setLanguage(LANGUAGE)
.setArguments(ARGUMENT_LIST)
Expand Down Expand Up @@ -96,6 +98,7 @@ public void testBuilder() {
.setEtag(ETAG)
.setRoutineType(ROUTINE_TYPE)
.setCreationTime(CREATION_TIME)
.setDeterminismLevel(DETERMINISM_LEVEL)
.setLastModifiedTime(LAST_MODIFIED_TIME)
.setLanguage(LANGUAGE)
.setArguments(ARGUMENT_LIST)
Expand All @@ -104,6 +107,7 @@ public void testBuilder() {
.setBody(BODY)
.build();
assertEquals(ETAG, builtRoutine.getEtag());
assertEquals(DETERMINISM_LEVEL, builtRoutine.getDeterminismLevel());
assertSame(bigquery, builtRoutine.getBigQuery());
}

Expand Down Expand Up @@ -191,6 +195,7 @@ public void compareRoutineInfo(RoutineInfo expected, RoutineInfo value) {
assertEquals(expected.getEtag(), value.getEtag());
assertEquals(expected.getRoutineType(), value.getRoutineType());
assertEquals(expected.getCreationTime(), value.getCreationTime());
assertEquals(expected.getDeterminismLevel(), value.getDeterminismLevel());
assertEquals(expected.getLastModifiedTime(), value.getLastModifiedTime());
assertEquals(expected.getLanguage(), value.getLanguage());
assertEquals(expected.getArguments(), value.getArguments());
Expand Down
Expand Up @@ -1522,7 +1522,6 @@ public void testEmptyListRoutines() {

@Test
public void testRoutineLifecycle() throws InterruptedException {

String routineName = RemoteBigQueryHelper.generateRoutineName();
// Create a routine using SQL.
String sql =
Expand Down Expand Up @@ -1587,6 +1586,34 @@ public void testRoutineAPICreation() {
assertEquals(routine.getRoutineType(), "SCALAR_FUNCTION");
}

@Test
public void testRoutineAPICreationJavascriptUDF() {
String routineName = RemoteBigQueryHelper.generateRoutineName();
RoutineId routineId = RoutineId.of(ROUTINE_DATASET, routineName);
RoutineInfo routineInfo =
RoutineInfo.newBuilder(routineId)
.setLanguage("JAVASCRIPT")
.setRoutineType("SCALAR_FUNCTION")
.setDeterminismLevel("DETERMINISTIC")
.setArguments(
ImmutableList.of(
RoutineArgument.newBuilder()
.setName("instr")
.setKind("FIXED_TYPE")
.setDataType(StandardSQLDataType.newBuilder("STRING").build())
.build()))
.setReturnType(StandardSQLDataType.newBuilder("STRING").build())
.setBody("return instr.toUpperCase();")
.build();

Routine routine = bigquery.create(routineInfo);
assertNotNull(routine);
assertEquals(routine.getLanguage(), "JAVASCRIPT");
assertEquals(routine.getDeterminismLevel(), "DETERMINISTIC");
assertEquals(routine.getRoutineType(), "SCALAR_FUNCTION");
assertEquals(routine.getReturnType(), StandardSQLDataType.newBuilder("STRING").build());
}

@Test
public void testAuthorizeRoutine() {
String routineName = RemoteBigQueryHelper.generateRoutineName();
Expand Down