testing: GrpcCleanupRule

This will ease a lot of test scenarios that we want to automatically shut down servers and channels, with much more flexibility than `GrpcServerRule`. Resolves #3624

**ManagedChannel/Server cleanup details:**
- If the test has already failed, call `shutdownNow()` for each of the resources registered. Throw (an exception including) the original failure. End.
- If the test is successful, call `shutdown()` for each of the resources registered.
- Call `awaitTermination()` with `timeout = deadline - current time` and assert termination for each resource.  If any error occurs, break immediately and call `shutdownNow()` for the current resource and all the rest resources.
-  Throw the first exception encountered if any.

3 files changed, 703 insertions, 1 deletions

diff --git a/testing/build.gradle b/testing/build.gradle
index 3902744f1..d272d7ff0 100644
--- a/testing/build.gradle
+++ b/testing/build.gradle
@@ -16,7 +16,8 @@ dependencies {
     // they'd have to resolve it like normal anyway.
     compileOnly libraries.truth
 
-    testCompile project(':grpc-testing-proto')
+    testCompile project(':grpc-testing-proto'),
+                project(':grpc-core').sourceSets.test.output
 }
 
 javadoc {
diff --git a/testing/src/main/java/io/grpc/testing/GrpcCleanupRule.java b/testing/src/main/java/io/grpc/testing/GrpcCleanupRule.java
new file mode 100644
index 000000000..5e223d6cb
--- /dev/null
+++ b/testing/src/main/java/io/grpc/testing/GrpcCleanupRule.java
@@ -0,0 +1,254 @@
+/*
+ * Copyright 2018, gRPC Authors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.grpc.testing;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Ticker;
+import io.grpc.ExperimentalApi;
+import io.grpc.ManagedChannel;
+import io.grpc.Server;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.NotThreadSafe;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.MultipleFailureException;
+import org.junit.runners.model.Statement;
+
+/**
+ * A JUnit {@link TestRule} that can register gRPC resources and manages its automatic release at
+ * the end of the test. If any of the resources registered to the rule can not be successfully
+ * released, the test will fail.
+ *
+ * @since 1.13.0
+ */
+@ExperimentalApi("https://github.com/grpc/grpc-java/issues/2488")
+@NotThreadSafe
+public final class GrpcCleanupRule implements TestRule {
+
+  private final List<Resource> resources = new ArrayList<Resource>();
+  private long timeoutNanos = TimeUnit.SECONDS.toNanos(10L);
+  private Stopwatch stopwatch = Stopwatch.createUnstarted();
+
+  private Throwable firstException;
+
+  /**
+   * Sets a positive total time limit for the automatic resource cleanup. If any of the resources
+   * registered to the rule fails to be released in time, the test will fail.
+   *
+   * <p>Note that the resource cleanup duration may or may not be counted as part of the JUnit
+   * {@link org.junit.rules.Timeout Timeout} rule's test duration, depending on which rule is
+   * applied first.
+   *
+   * @return this
+   */
+  public GrpcCleanupRule setTimeout(long timeout, TimeUnit timeUnit) {
+    checkArgument(timeout > 0, "timeout should be positive");
+    timeoutNanos = timeUnit.toNanos(timeout);
+    return this;
+  }
+
+  /**
+   * Sets a specified time source for monitoring cleanup timeout.
+   *
+   * @return this
+   */
+  @SuppressWarnings("BetaApi") // Stopwatch.createUnstarted(Ticker ticker) is not Beta. Test only.
+  @VisibleForTesting
+  GrpcCleanupRule setTicker(Ticker ticker) {
+    this.stopwatch = Stopwatch.createUnstarted(ticker);
+    return this;
+  }
+
+  /**
+   * Registers the given channel to the rule. Once registered, the channel will be automatically
+   * shutdown at the end of the test.
+   *
+   * <p>This method need be properly synchronized if used in multiple threads. This method must
+   * not be used during the test teardown.
+   *
+   * @return the input channel
+   */
+  public <T extends ManagedChannel> T register(@Nonnull T channel) {
+    checkNotNull(channel, "channel");
+    register(new ManagedChannelResource(channel));
+    return channel;
+  }
+
+  /**
+   * Registers the given server to the rule. Once registered, the server will be automatically
+   * shutdown at the end of the test.
+   *
+   * <p>This method need be properly synchronized if used in multiple threads. This method must
+   * not be used during the test teardown.
+   *
+   * @return the input server
+   */
+  public <T extends Server> T register(@Nonnull T server) {
+    checkNotNull(server, "server");
+    register(new ServerResource(server));
+    return server;
+  }
+
+  @VisibleForTesting
+  void register(Resource resource) {
+    resources.add(resource);
+  }
+
+  @Override
+  public Statement apply(final Statement base, Description description) {
+    return new Statement() {
+      @Override
+      public void evaluate() throws Throwable {
+        try {
+          base.evaluate();
+        } catch (Throwable t) {
+          firstException = t;
+
+          try {
+            teardown();
+          } catch (Throwable t2) {
+            throw new MultipleFailureException(Arrays.asList(t, t2));
+          }
+
+          throw t;
+        }
+
+        teardown();
+        if (firstException != null) {
+          throw firstException;
+        }
+      }
+    };
+  }
+
+  /**
+   * Releases all the registered resources.
+   */
+  private void teardown() {
+    stopwatch.start();
+
+    if (firstException == null) {
+      for (int i = resources.size() - 1; i >= 0; i--) {
+        resources.get(i).cleanUp();
+      }
+    }
+
+    for (int i = resources.size() - 1; i >= 0; i--) {
+      if (firstException != null) {
+        resources.get(i).forceCleanUp();
+        continue;
+      }
+
+      try {
+        boolean released = resources.get(i).awaitReleased(
+            timeoutNanos - stopwatch.elapsed(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
+        if (!released) {
+          firstException = new AssertionError(
+              "Resource " + resources.get(i) + " can not be released in time at the end of test");
+        }
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        firstException = e;
+      }
+
+      if (firstException != null) {
+        resources.get(i).forceCleanUp();
+      }
+    }
+
+    resources.clear();
+  }
+
+  @VisibleForTesting
+  interface Resource {
+    void cleanUp();
+
+    /**
+     * Error already happened, try the best to clean up. Never throws.
+     */
+    void forceCleanUp();
+
+    /**
+     * Returns true if the resource is released in time.
+     */
+    boolean awaitReleased(long duration, TimeUnit timeUnit) throws InterruptedException;
+  }
+
+  private static final class ManagedChannelResource implements Resource {
+    final ManagedChannel channel;
+
+    ManagedChannelResource(ManagedChannel channel) {
+      this.channel = channel;
+    }
+
+    @Override
+    public void cleanUp() {
+      channel.shutdown();
+    }
+
+    @Override
+    public void forceCleanUp() {
+      channel.shutdownNow();
+    }
+
+    @Override
+    public boolean awaitReleased(long duration, TimeUnit timeUnit) throws InterruptedException {
+      return channel.awaitTermination(duration, timeUnit);
+    }
+
+    @Override
+    public String toString() {
+      return channel.toString();
+    }
+  }
+
+  private static final class ServerResource implements Resource {
+    final Server server;
+
+    ServerResource(Server server) {
+      this.server = server;
+    }
+
+    @Override
+    public void cleanUp() {
+      server.shutdown();
+    }
+
+    @Override
+    public void forceCleanUp() {
+      server.shutdownNow();
+    }
+
+    @Override
+    public boolean awaitReleased(long duration, TimeUnit timeUnit) throws InterruptedException {
+      return server.awaitTermination(duration, timeUnit);
+    }
+
+    @Override
+    public String toString() {
+      return server.toString();
+    }
+  }
+}
diff --git a/testing/src/test/java/io/grpc/testing/GrpcCleanupRuleTest.java b/testing/src/test/java/io/grpc/testing/GrpcCleanupRuleTest.java
new file mode 100644
index 000000000..fb7ecdff2
--- /dev/null
+++ b/testing/src/test/java/io/grpc/testing/GrpcCleanupRuleTest.java
@@ -0,0 +1,447 @@
+/*
+ * Copyright 2018, gRPC Authors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.grpc.testing;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.AdditionalAnswers.delegatesTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import io.grpc.ManagedChannel;
+import io.grpc.Server;
+import io.grpc.internal.FakeClock;
+import io.grpc.testing.GrpcCleanupRule.Resource;
+import java.util.concurrent.TimeUnit;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.junit.runners.model.MultipleFailureException;
+import org.junit.runners.model.Statement;
+import org.mockito.InOrder;
+
+/**
+ * Unit tests for {@link GrpcCleanupRule}.
+ */
+@RunWith(JUnit4.class)
+public class GrpcCleanupRuleTest {
+  public static final FakeClock fakeClock = new FakeClock();
+
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  @Test
+  public void registerChannelReturnSameChannel() {
+    ManagedChannel channel = mock(ManagedChannel.class);
+    assertSame(channel, new GrpcCleanupRule().register(channel));
+  }
+
+  @Test
+  public void registerServerReturnSameServer() {
+    Server server = mock(Server.class);
+    assertSame(server, new GrpcCleanupRule().register(server));
+  }
+
+  @Test
+  public void registerNullChannelThrowsNpe() {
+    ManagedChannel channel = null;
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    thrown.expect(NullPointerException.class);
+    thrown.expectMessage("channel");
+
+    grpcCleanup.register(channel);
+  }
+
+  @Test
+  public void registerNullServerThrowsNpe() {
+    Server server = null;
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    thrown.expect(NullPointerException.class);
+    thrown.expectMessage("server");
+
+    grpcCleanup.register(server);
+  }
+
+  @Test
+  public void singleChannelCleanup() throws Throwable {
+    // setup
+    ManagedChannel channel = mock(ManagedChannel.class);
+    Statement statement = mock(Statement.class);
+    InOrder inOrder = inOrder(statement, channel);
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    // run
+    grpcCleanup.register(channel);
+
+    boolean awaitTerminationFailed = false;
+    try {
+      // will throw because channel.awaitTermination(long, TimeUnit) will return false;
+      grpcCleanup.apply(statement, null /* description*/).evaluate();
+    } catch (AssertionError e) {
+      awaitTerminationFailed = true;
+    }
+
+    // verify
+    assertTrue(awaitTerminationFailed);
+    inOrder.verify(statement).evaluate();
+    inOrder.verify(channel).shutdown();
+    inOrder.verify(channel).awaitTermination(anyLong(), any(TimeUnit.class));
+    inOrder.verify(channel).shutdownNow();
+  }
+
+  @Test
+  public void singleServerCleanup() throws Throwable {
+    // setup
+    Server server = mock(Server.class);
+    Statement statement = mock(Statement.class);
+    InOrder inOrder = inOrder(statement, server);
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    // run
+    grpcCleanup.register(server);
+
+    boolean awaitTerminationFailed = false;
+    try {
+      // will throw because channel.awaitTermination(long, TimeUnit) will return false;
+      grpcCleanup.apply(statement, null /* description*/).evaluate();
+    } catch (AssertionError e) {
+      awaitTerminationFailed = true;
+    }
+
+    // verify
+    assertTrue(awaitTerminationFailed);
+    inOrder.verify(statement).evaluate();
+    inOrder.verify(server).shutdown();
+    inOrder.verify(server).awaitTermination(anyLong(), any(TimeUnit.class));
+    inOrder.verify(server).shutdownNow();
+  }
+
+  @Test
+  public void multiResource_cleanupGracefully() throws Throwable {
+    // setup
+    Resource resource1 = mock(Resource.class);
+    Resource resource2 = mock(Resource.class);
+    Resource resource3 = mock(Resource.class);
+    doReturn(true).when(resource1).awaitReleased(anyLong(), any(TimeUnit.class));
+    doReturn(true).when(resource2).awaitReleased(anyLong(), any(TimeUnit.class));
+    doReturn(true).when(resource3).awaitReleased(anyLong(), any(TimeUnit.class));
+
+    Statement statement = mock(Statement.class);
+    InOrder inOrder = inOrder(statement, resource1, resource2, resource3);
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    // run
+    grpcCleanup.register(resource1);
+    grpcCleanup.register(resource2);
+    grpcCleanup.register(resource3);
+    grpcCleanup.apply(statement, null /* description*/).evaluate();
+
+    // Verify.
+    inOrder.verify(statement).evaluate();
+
+    inOrder.verify(resource3).cleanUp();
+    inOrder.verify(resource2).cleanUp();
+    inOrder.verify(resource1).cleanUp();
+
+    inOrder.verify(resource3).awaitReleased(anyLong(), any(TimeUnit.class));
+    inOrder.verify(resource2).awaitReleased(anyLong(), any(TimeUnit.class));
+    inOrder.verify(resource1).awaitReleased(anyLong(), any(TimeUnit.class));
+
+    inOrder.verifyNoMoreInteractions();
+
+    verify(resource1, never()).forceCleanUp();
+    verify(resource2, never()).forceCleanUp();
+    verify(resource3, never()).forceCleanUp();
+  }
+
+  @Test
+  public void baseTestFails() throws Throwable {
+    // setup
+    Resource resource = mock(Resource.class);
+
+    Statement statement = mock(Statement.class);
+    doThrow(new Exception()).when(statement).evaluate();
+
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    // run
+    grpcCleanup.register(resource);
+
+    boolean baseTestFailed = false;
+    try {
+      grpcCleanup.apply(statement, null /* description*/).evaluate();
+    } catch (Exception e) {
+      baseTestFailed = true;
+    }
+
+    // verify
+    assertTrue(baseTestFailed);
+
+    verify(resource).forceCleanUp();
+    verifyNoMoreInteractions(resource);
+
+    verify(resource, never()).cleanUp();
+    verify(resource, never()).awaitReleased(anyLong(), any(TimeUnit.class));
+  }
+
+  @Test
+  public void multiResource_awaitReleasedFails() throws Throwable {
+    // setup
+    Resource resource1 = mock(Resource.class);
+    Resource resource2 = mock(Resource.class);
+    Resource resource3 = mock(Resource.class);
+    doReturn(true).when(resource1).awaitReleased(anyLong(), any(TimeUnit.class));
+    doReturn(false).when(resource2).awaitReleased(anyLong(), any(TimeUnit.class));
+    doReturn(true).when(resource3).awaitReleased(anyLong(), any(TimeUnit.class));
+
+    Statement statement = mock(Statement.class);
+    InOrder inOrder = inOrder(statement, resource1, resource2, resource3);
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    // run
+    grpcCleanup.register(resource1);
+    grpcCleanup.register(resource2);
+    grpcCleanup.register(resource3);
+
+    boolean cleanupFailed = false;
+    try {
+      grpcCleanup.apply(statement, null /* description*/).evaluate();
+    } catch (AssertionError e) {
+      cleanupFailed = true;
+    }
+
+    // verify
+    assertTrue(cleanupFailed);
+
+    inOrder.verify(statement).evaluate();
+
+    inOrder.verify(resource3).cleanUp();
+    inOrder.verify(resource2).cleanUp();
+    inOrder.verify(resource1).cleanUp();
+
+    inOrder.verify(resource3).awaitReleased(anyLong(), any(TimeUnit.class));
+    inOrder.verify(resource2).awaitReleased(anyLong(), any(TimeUnit.class));
+    inOrder.verify(resource2).forceCleanUp();
+    inOrder.verify(resource1).forceCleanUp();
+
+    inOrder.verifyNoMoreInteractions();
+
+    verify(resource3, never()).forceCleanUp();
+    verify(resource1, never()).awaitReleased(anyLong(), any(TimeUnit.class));
+  }
+
+  @Test
+  public void multiResource_awaitReleasedInterrupted() throws Throwable {
+    // setup
+    Resource resource1 = mock(Resource.class);
+    Resource resource2 = mock(Resource.class);
+    Resource resource3 = mock(Resource.class);
+    doReturn(true).when(resource1).awaitReleased(anyLong(), any(TimeUnit.class));
+    doThrow(new InterruptedException())
+        .when(resource2).awaitReleased(anyLong(), any(TimeUnit.class));
+    doReturn(true).when(resource3).awaitReleased(anyLong(), any(TimeUnit.class));
+
+    Statement statement = mock(Statement.class);
+    InOrder inOrder = inOrder(statement, resource1, resource2, resource3);
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    // run
+    grpcCleanup.register(resource1);
+    grpcCleanup.register(resource2);
+    grpcCleanup.register(resource3);
+
+    boolean cleanupFailed = false;
+    try {
+      grpcCleanup.apply(statement, null /* description*/).evaluate();
+    } catch (InterruptedException e) {
+      cleanupFailed = true;
+    }
+
+    // verify
+    assertTrue(cleanupFailed);
+    assertTrue(Thread.interrupted());
+
+    inOrder.verify(statement).evaluate();
+
+    inOrder.verify(resource3).cleanUp();
+    inOrder.verify(resource2).cleanUp();
+    inOrder.verify(resource1).cleanUp();
+
+    inOrder.verify(resource3).awaitReleased(anyLong(), any(TimeUnit.class));
+    inOrder.verify(resource2).awaitReleased(anyLong(), any(TimeUnit.class));
+    inOrder.verify(resource2).forceCleanUp();
+    inOrder.verify(resource1).forceCleanUp();
+
+    inOrder.verifyNoMoreInteractions();
+
+    verify(resource3, never()).forceCleanUp();
+    verify(resource1, never()).awaitReleased(anyLong(), any(TimeUnit.class));
+  }
+
+  @Test
+  public void multiResource_timeoutCalculation() throws Throwable {
+    // setup
+
+    Resource resource1 = mock(FakeResource.class,
+        delegatesTo(new FakeResource(1 /* cleanupNanos */, 10 /* awaitReleaseNanos */)));
+
+    Resource resource2 = mock(FakeResource.class,
+        delegatesTo(new FakeResource(100 /* cleanupNanos */, 1000 /* awaitReleaseNanos */)));
+
+
+    Statement statement = mock(Statement.class);
+    InOrder inOrder = inOrder(statement, resource1, resource2);
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule().setTicker(fakeClock.getTicker());
+
+    // run
+    grpcCleanup.register(resource1);
+    grpcCleanup.register(resource2);
+    grpcCleanup.apply(statement, null /* description*/).evaluate();
+
+    // verify
+    inOrder.verify(statement).evaluate();
+
+    inOrder.verify(resource2).cleanUp();
+    inOrder.verify(resource1).cleanUp();
+
+    inOrder.verify(resource2).awaitReleased(
+        TimeUnit.SECONDS.toNanos(10) - 100 - 1, TimeUnit.NANOSECONDS);
+    inOrder.verify(resource1).awaitReleased(
+        TimeUnit.SECONDS.toNanos(10) - 100 - 1 - 1000, TimeUnit.NANOSECONDS);
+
+    inOrder.verifyNoMoreInteractions();
+
+    verify(resource2, never()).forceCleanUp();
+    verify(resource1, never()).forceCleanUp();
+  }
+
+  @Test
+  public void multiResource_timeoutCalculation_customTimeout() throws Throwable {
+    // setup
+
+    Resource resource1 = mock(FakeResource.class,
+        delegatesTo(new FakeResource(1 /* cleanupNanos */, 10 /* awaitReleaseNanos */)));
+
+    Resource resource2 = mock(FakeResource.class,
+        delegatesTo(new FakeResource(100 /* cleanupNanos */, 1000 /* awaitReleaseNanos */)));
+
+
+    Statement statement = mock(Statement.class);
+    InOrder inOrder = inOrder(statement, resource1, resource2);
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule()
+        .setTicker(fakeClock.getTicker()).setTimeout(3000, TimeUnit.NANOSECONDS);
+
+    // run
+    grpcCleanup.register(resource1);
+    grpcCleanup.register(resource2);
+    grpcCleanup.apply(statement, null /* description*/).evaluate();
+
+    // verify
+    inOrder.verify(statement).evaluate();
+
+    inOrder.verify(resource2).cleanUp();
+    inOrder.verify(resource1).cleanUp();
+
+    inOrder.verify(resource2).awaitReleased(3000 - 100 - 1, TimeUnit.NANOSECONDS);
+    inOrder.verify(resource1).awaitReleased(3000 - 100 - 1 - 1000, TimeUnit.NANOSECONDS);
+
+    inOrder.verifyNoMoreInteractions();
+
+    verify(resource2, never()).forceCleanUp();
+    verify(resource1, never()).forceCleanUp();
+  }
+
+  @Test
+  public void baseTestFailsThenCleanupFails() throws Throwable {
+    // setup
+    Exception baseTestFailure = new Exception();
+
+    Statement statement = mock(Statement.class);
+    doThrow(baseTestFailure).when(statement).evaluate();
+
+    Resource resource1 = mock(Resource.class);
+    Resource resource2 = mock(Resource.class);
+    Resource resource3 = mock(Resource.class);
+    doThrow(new RuntimeException()).when(resource2).forceCleanUp();
+
+    InOrder inOrder = inOrder(statement, resource1, resource2, resource3);
+    GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+    // run
+    grpcCleanup.register(resource1);
+    grpcCleanup.register(resource2);
+    grpcCleanup.register(resource3);
+
+    Throwable failure = null;
+    try {
+      grpcCleanup.apply(statement, null /* description*/).evaluate();
+    } catch (Throwable e) {
+      failure = e;
+    }
+
+    // verify
+    assertThat(failure).isInstanceOf(MultipleFailureException.class);
+    assertSame(baseTestFailure, ((MultipleFailureException) failure).getFailures().get(0));
+
+    inOrder.verify(statement).evaluate();
+    inOrder.verify(resource3).forceCleanUp();
+    inOrder.verify(resource2).forceCleanUp();
+    inOrder.verifyNoMoreInteractions();
+
+    verify(resource1, never()).cleanUp();
+    verify(resource2, never()).cleanUp();
+    verify(resource3, never()).cleanUp();
+    verify(resource1, never()).forceCleanUp();
+  }
+
+  public static class FakeResource implements Resource {
+    private final long cleanupNanos;
+    private final long awaitReleaseNanos;
+
+    private FakeResource(long cleanupNanos, long awaitReleaseNanos) {
+      this.cleanupNanos = cleanupNanos;
+      this.awaitReleaseNanos = awaitReleaseNanos;
+    }
+
+    @Override
+    public void cleanUp() {
+      fakeClock.forwardTime(cleanupNanos, TimeUnit.NANOSECONDS);
+    }
+
+    @Override
+    public void forceCleanUp() {
+    }
+
+    @Override
+    public boolean awaitReleased(long duration, TimeUnit timeUnit) {
+      fakeClock.forwardTime(awaitReleaseNanos, TimeUnit.NANOSECONDS);
+      return true;
+    }
+  }
+}