From 60f4a66ad907f4a838536e405326869487468f35 Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Tue, 10 Nov 2020 15:04:23 -0500 Subject: [PATCH] feat(test): add new MultipleAttemptRule, StdOutCaptureRule & StdErrCaptureRule JUnit 4 rules (#327) * feat(test): add new MultipleAttemptRule JUnit 4 rule * feat(test): add new StdOutCaptureRule & StdErrCaptureRule JUnit 4 rule * fix: line separator failures from windows --- .../testing/junit4/MultipleAttemptsRule.java | 107 ++++++++++ .../testing/junit4/StdErrCaptureRule.java | 56 ++++++ .../testing/junit4/StdOutCaptureRule.java | 56 ++++++ .../cloud/testing/junit4/StdXCaptureRule.java | 188 ++++++++++++++++++ .../testing/junit4/StdXCaptureRuleTest.java | 76 +++++++ .../tests/MultipleAttemptsRuleTest.java | 87 ++++++++ .../junit4/tests/StdErrCaptureRuleTest.java | 36 ++++ .../junit4/tests/StdOutCaptureRuleTest.java | 36 ++++ 8 files changed, 642 insertions(+) create mode 100644 google-cloud-core/src/test/java/com/google/cloud/testing/junit4/MultipleAttemptsRule.java create mode 100644 google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdErrCaptureRule.java create mode 100644 google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdOutCaptureRule.java create mode 100644 google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdXCaptureRule.java create mode 100644 google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdXCaptureRuleTest.java create mode 100644 google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/MultipleAttemptsRuleTest.java create mode 100644 google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/StdErrCaptureRuleTest.java create mode 100644 google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/StdOutCaptureRuleTest.java diff --git a/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/MultipleAttemptsRule.java b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/MultipleAttemptsRule.java new file mode 100644 index 0000000000..86fd82c7b9 --- /dev/null +++ b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/MultipleAttemptsRule.java @@ -0,0 +1,107 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.testing.junit4; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.ArrayList; +import java.util.List; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.MultipleFailureException; +import org.junit.runners.model.Statement; + +/** + * A JUnit rule that allows multiple attempts of a test execution before ultimately reporting + * failure for the test. Attempts will be attempted with an exponential backoff which defaults to a + * starting duration of 1 second. + * + *

If after the maximum number of attempts the test has still not succeeded, all failures will be + * propagated as the result of the test allowing all errors to be visible (regardless if they are + * the same failure or different ones). + * + *

To use this rule add the field declaration to your JUnit 4 Test class: + * + *

Note: It is important that the field is public + * + *

{@code
+ * @Rule
+ * public MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3);
+ * }
+ * + * @see org.junit.Rule + */ +public final class MultipleAttemptsRule implements TestRule { + private final long initialBackoffMillis; + private final int maxAttemptCount; + + /** + * Construct a {@link MultipleAttemptsRule} which will attempt a test up to {@code attemptCount} + * times before ultimately reporting failure of the test. + * + *

The initialBackoffMillis will be set to 1000L. + * + * @param maxAttemptCount max number of attempts before reporting failure, must be greater than 0 + * @see #MultipleAttemptsRule(int, long) + */ + public MultipleAttemptsRule(int maxAttemptCount) { + this(maxAttemptCount, 1000L); + } + + /** + * Construct a {@link MultipleAttemptsRule} which will attempt a test up to {@code attemptCount} + * times before ultimately reporting failure of the test. + * + *

The {@code initialBackoffMillis} will be used as the first pause duration before + * reattempting the test. + * + * @param maxAttemptCount max number of attempts before reporting failure, must be greater than 0 + * @param initialBackoffMillis initial duration in millis to wait between attempts, must be + * greater than or equal to 0 + */ + public MultipleAttemptsRule(int maxAttemptCount, long initialBackoffMillis) { + checkArgument(maxAttemptCount > 0, "attemptCount must be > 0"); + checkArgument(initialBackoffMillis >= 0, "initialBackoffMillis must be >= 0"); + this.initialBackoffMillis = initialBackoffMillis; + this.maxAttemptCount = maxAttemptCount; + } + + @Override + public Statement apply(final Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + List failures = new ArrayList<>(); + + long retryIntervalMillis = initialBackoffMillis; + + for (int i = 1; i <= maxAttemptCount; i++) { + try { + base.evaluate(); + return; + } catch (Throwable t) { + failures.add(t); + Thread.sleep(retryIntervalMillis); + retryIntervalMillis *= 1.5f; + } + } + + MultipleFailureException.assertEmpty(failures); + } + }; + } +} diff --git a/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdErrCaptureRule.java b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdErrCaptureRule.java new file mode 100644 index 0000000000..1640d950e3 --- /dev/null +++ b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdErrCaptureRule.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.testing.junit4; + +import java.io.PrintStream; +import org.junit.Rule; + +/** + * A JUnit rule that allows the capturing stderr (i.e. {@link System#err} during the scope of a + * test. + * + *

Note: If some part of the system holds a reference System.err before this rule is loaded + * into the test lifecycle there is no way for this rule to capture the output. Ensure this rule is + * declared as high in your test file as possible, and ordered using {@link Rule#order()} before + * other Rules if necessary. + * + *

To use this rule add the field declaration to your JUnit 4 Test class: + * + *

Note: It is important that the field is public + * + *

{@code
+ * @Rule
+ * public StdErrCaptureRule stdErrCaptureRule = new StdErrCaptureRule();
+ * }
+ * + * @see org.junit.Rule + * @see Rule#order() + */ +public final class StdErrCaptureRule extends StdXCaptureRule { + + public StdErrCaptureRule() {} + + @Override + protected PrintStream getOriginal() { + return System.err; + } + + @Override + protected void set(PrintStream ps) { + System.setErr(ps); + } +} diff --git a/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdOutCaptureRule.java b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdOutCaptureRule.java new file mode 100644 index 0000000000..0a33997cde --- /dev/null +++ b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdOutCaptureRule.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.testing.junit4; + +import java.io.PrintStream; +import org.junit.Rule; + +/** + * A JUnit rule that allows the capturing stdout (i.e. {@link System#out} during the scope of a + * test. + * + *

Note: If some part of the system holds a reference System.out before this rule is loaded + * into the test lifecycle there is no way for this rule to capture the output. Ensure this rule is + * declared as high in your test file as possible, and ordered using {@link Rule#order()} before + * other Rules if necessary. + * + *

To use this rule add the field declaration to your JUnit 4 Test class: + * + *

Note: It is important that the field is public + * + *

{@code
+ * @Rule
+ * public StdOutCaptureRule stdOutCaptureRule = new StdOutCaptureRule();
+ * }
+ * + * @see org.junit.Rule + * @see Rule#order() + */ +public final class StdOutCaptureRule extends StdXCaptureRule { + + public StdOutCaptureRule() {} + + @Override + protected PrintStream getOriginal() { + return System.out; + } + + @Override + protected void set(PrintStream ps) { + System.setOut(ps); + } +} diff --git a/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdXCaptureRule.java b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdXCaptureRule.java new file mode 100644 index 0000000000..ba17e3e324 --- /dev/null +++ b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdXCaptureRule.java @@ -0,0 +1,188 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.testing.junit4; + +import com.google.common.base.Charsets; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +abstract class StdXCaptureRule implements TestRule { + + private final ByteArrayOutputStream byteArrayOutputStream; + + public StdXCaptureRule() { + byteArrayOutputStream = new ByteArrayOutputStream(); + } + + protected abstract PrintStream getOriginal(); + + protected abstract void set(PrintStream ps); + + /** + * Get a handle to the raw bytes written during the running test so far. + * + * @return A read-only {@link ByteArrayOutputStream} representing the raw bytes written so far. + *

Note the following behavior of the return value: + *

    + *
  1. Calling any "write" method call will result in an {@link IllegalStateException} + *
  2. Calls to {@link ByteArrayOutputStream#toByteArray()} will result in a new copy of the + * underlying array up to that point + *
  3. Calls to {@link ByteArrayOutputStream#flush()} will be silently ignored + *
  4. Calls to {@link ByteArrayOutputStream#close()} will be silently ignored + *
+ */ + public ByteArrayOutputStream getCapturedOutput() { + return new ReadOnlyByteArrayOutputStream(byteArrayOutputStream); + } + + /** + * Return a UTF-8 {@link String} of all bytes written during the running test so far. + * + * @return UTF-8 {@link String} of all bytes written + */ + public String getCapturedOutputAsUtf8String() { + return new String(byteArrayOutputStream.toByteArray(), Charsets.UTF_8); + } + + @Override + public Statement apply(final Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + PrintStream originalOut = getOriginal(); + TeeOutputStream tee = new TeeOutputStream(System.out, byteArrayOutputStream); + boolean outReplaced = false; + try { + set(new PrintStream(tee)); + outReplaced = true; + base.evaluate(); + } finally { + if (outReplaced) { + set(originalOut); + } + } + } + }; + } + + private static final class TeeOutputStream extends OutputStream { + private final OutputStream left; + private final OutputStream right; + + public TeeOutputStream(OutputStream left, OutputStream right) { + this.left = left; + this.right = right; + } + + @Override + public void write(int b) throws IOException { + try { + left.write(b); + } finally { + right.write(b); + } + } + + @Override + public void flush() throws IOException { + try { + left.flush(); + } finally { + right.flush(); + } + } + } + + private static final class ReadOnlyByteArrayOutputStream extends ByteArrayOutputStream { + private final ByteArrayOutputStream delegate; + + public ReadOnlyByteArrayOutputStream(ByteArrayOutputStream delegate) { + this.delegate = delegate; + } + + @Override + public synchronized void write(int b) { + throw getIllegalStateException("write(b)"); + } + + @Override + public synchronized void write(byte[] b, int off, int len) { + throw getIllegalStateException("write(byte[], off, len)"); + } + + @Override + public synchronized void writeTo(OutputStream out) { + throw getIllegalStateException("writeOt(out)"); + } + + @Override + public synchronized void reset() { + throw getIllegalStateException("reset()"); + } + + @Override + public synchronized byte[] toByteArray() { + return delegate.toByteArray(); + } + + @Override + public void close() { + // ignore + } + + @Override + public synchronized int size() { + return delegate.size(); + } + + @Override + public synchronized String toString() { + return delegate.toString(); + } + + @Override + public synchronized String toString(String charsetName) throws UnsupportedEncodingException { + return delegate.toString(charsetName); + } + + @SuppressWarnings("deprecation") + @Override + public synchronized String toString(int hibyte) { + return delegate.toString(hibyte); + } + + @Override + public void flush() { + // ignore + } + + @Override + public void write(byte[] b) { + throw getIllegalStateException("write(byte[])"); + } + + private IllegalStateException getIllegalStateException(String desc) { + return new IllegalStateException(desc + " is forbidden"); + } + } +} diff --git a/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdXCaptureRuleTest.java b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdXCaptureRuleTest.java new file mode 100644 index 0000000000..17d167e1af --- /dev/null +++ b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/StdXCaptureRuleTest.java @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.testing.junit4; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.PrintStream; +import org.junit.Test; + +public final class StdXCaptureRuleTest { + + @Test(expected = IllegalStateException.class) + public void returnedByteArrayOutputStreamIsReadOnly_writeByte() { + getStdXCaptureRule().getCapturedOutput().write(0); + } + + @Test(expected = IllegalStateException.class) + public void returnedByteArrayOutputStreamIsReadOnly_writeByteArray() throws IOException { + getStdXCaptureRule().getCapturedOutput().write(new byte[] {0}); + } + + @Test(expected = IllegalStateException.class) + public void returnedByteArrayOutputStreamIsReadOnly_writeByteArrayRange() { + getStdXCaptureRule().getCapturedOutput().write(new byte[] {0}, 0, 1); + } + + @Test(expected = IllegalStateException.class) + public void returnedByteArrayOutputStreamIsReadOnly_writeTo() throws IOException { + getStdXCaptureRule().getCapturedOutput().writeTo(System.out); + } + + @Test(expected = IllegalStateException.class) + public void returnedByteArrayOutputStreamIsReadOnly_reset() { + getStdXCaptureRule().getCapturedOutput().reset(); + } + + @Test + public void returnedByteArrayOutputStreamIsReadOnly_close() throws IOException { + getStdXCaptureRule().getCapturedOutput().close(); + } + + @Test + public void returnedByteArrayOutputStreamIsReadOnly_flush() throws IOException { + getStdXCaptureRule().getCapturedOutput().flush(); + } + + private static StdXCaptureRule getStdXCaptureRule() { + return new StdXCaptureRule() { + @Override + protected PrintStream getOriginal() { + fail("unexpected call"); + return null; + } + + @Override + protected void set(PrintStream ps) { + fail("unexpected call"); + } + }; + } +} diff --git a/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/MultipleAttemptsRuleTest.java b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/MultipleAttemptsRuleTest.java new file mode 100644 index 0000000000..e1c98b1953 --- /dev/null +++ b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/MultipleAttemptsRuleTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.testing.junit4.tests; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runners.model.MultipleFailureException; +import org.junit.runners.model.Statement; + +public final class MultipleAttemptsRuleTest { + + private static final int NUMBER_OF_ATTEMPTS = 5; + + @Rule public MultipleAttemptsRule rr = new MultipleAttemptsRule(NUMBER_OF_ATTEMPTS, 0); + + private int numberAttempted = 0; + + @Test + public void wontPassUntil5() { + numberAttempted += 1; + assertEquals(NUMBER_OF_ATTEMPTS, numberAttempted); + } + + @Test(expected = IllegalArgumentException.class) + public void errorConstructing_attemptLessThan1() { + new MultipleAttemptsRule(0); + } + + @Test + public void errorConstructing_attemptEquals1() { + new MultipleAttemptsRule(1); + } + + @Test(expected = IllegalArgumentException.class) + public void errorConstructing_attemptGreaterThanOrEqualTo1AndBackoffLessThan0() { + new MultipleAttemptsRule(1, -1); + } + + @Test + public void errorConstructing_attemptGreaterThanOrEqualTo1AndBackoffEqualTo0() { + new MultipleAttemptsRule(1, 0); + } + + @Test + public void allErrorPropagated() { + MultipleAttemptsRule rule = new MultipleAttemptsRule(3, 0); + Statement statement = + rule.apply( + new Statement() { + private int counter = 1; + + @Override + public void evaluate() { + fail("attempt " + counter++); + } + }, + null); + + try { + statement.evaluate(); + } catch (MultipleFailureException mfe) { + // pass + assertThat(mfe.getFailures()).hasSize(3); + } catch (Throwable throwable) { + fail("unexpected error: " + throwable.getMessage()); + } + } +} diff --git a/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/StdErrCaptureRuleTest.java b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/StdErrCaptureRuleTest.java new file mode 100644 index 0000000000..565bb08f12 --- /dev/null +++ b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/StdErrCaptureRuleTest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.testing.junit4.tests; + +import static org.junit.Assert.assertEquals; + +import com.google.cloud.testing.junit4.StdErrCaptureRule; +import org.junit.Rule; +import org.junit.Test; + +public class StdErrCaptureRuleTest { + + @Rule public StdErrCaptureRule stdOutCap = new StdErrCaptureRule(); + + @Test + public void captureSuccessful() { + System.err.println("err world"); + String expected = "err world" + System.lineSeparator(); + String actual = stdOutCap.getCapturedOutputAsUtf8String(); + assertEquals(expected, actual); + } +} diff --git a/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/StdOutCaptureRuleTest.java b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/StdOutCaptureRuleTest.java new file mode 100644 index 0000000000..8230debded --- /dev/null +++ b/google-cloud-core/src/test/java/com/google/cloud/testing/junit4/tests/StdOutCaptureRuleTest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.testing.junit4.tests; + +import static org.junit.Assert.assertEquals; + +import com.google.cloud.testing.junit4.StdOutCaptureRule; +import org.junit.Rule; +import org.junit.Test; + +public class StdOutCaptureRuleTest { + + @Rule public StdOutCaptureRule stdOutCap = new StdOutCaptureRule(); + + @Test + public void captureSuccessful() { + System.out.println("hello world"); + String expected = "hello world" + System.lineSeparator(); + String actual = stdOutCap.getCapturedOutputAsUtf8String(); + assertEquals(expected, actual); + } +}