Merge remote-tracking branch 'aosp/upstream-master' into master am: 0b3c15f840 am: cc9e6f50aeandroid-vts-8.0_r2android-vts-8.0_r1oreo-dev

am: b1838b10de

Change-Id: I65df3ab235d9746b787758e77937f17bd7691ccf

20 files changed, 1378 insertions, 721 deletions

diff --git a/java/com/google/devtools/build/android/desugar/ClassReaderFactory.java b/java/com/google/devtools/build/android/desugar/ClassReaderFactory.java
index 56f99a3..2b44d76 100644
--- a/java/com/google/devtools/build/android/desugar/ClassReaderFactory.java
+++ b/java/com/google/devtools/build/android/desugar/ClassReaderFactory.java
@@ -15,19 +15,16 @@ package com.google.devtools.build.android.desugar;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.jar.JarFile;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
 import javax.annotation.Nullable;
 import org.objectweb.asm.ClassReader;
 
 class ClassReaderFactory {
-  private final IndexedJars indexedJars;
+  private final IndexedInputs indexedInputs;
   private final CoreLibraryRewriter rewriter;
 
-  public ClassReaderFactory(IndexedJars indexedJars, CoreLibraryRewriter rewriter) {
+  public ClassReaderFactory(IndexedInputs indexedInputs, CoreLibraryRewriter rewriter) {
     this.rewriter = rewriter;
-    this.indexedJars = indexedJars;
+    this.indexedInputs = indexedInputs;
   }
 
   /**
@@ -38,23 +35,19 @@ class ClassReaderFactory {
   @Nullable
   public ClassReader readIfKnown(String internalClassName) {
     String filename = rewriter.unprefix(internalClassName) + ".class";
-    JarFile jarFile = indexedJars.getJarFile(filename);
+    InputFileProvider inputFileProvider = indexedInputs.getInputFileProvider(filename);
 
-    if (jarFile != null) {
-      return getClassReader(internalClassName, jarFile, jarFile.getEntry(filename));
+    if (inputFileProvider != null) {
+      try (InputStream bytecode = inputFileProvider.getInputStream(filename)) {
+        // ClassReader doesn't take ownership and instead eagerly reads the stream's contents
+        return rewriter.reader(bytecode);
+      } catch (IOException e) {
+        // We should've already read through all files in the Jar once at this point, so we don't
+        // expect failures reading some files a second time.
+        throw new IllegalStateException("Couldn't load " + internalClassName, e);
+      }
     }
 
     return null;
   }
-
-  private ClassReader getClassReader(String internalClassName, ZipFile jar, ZipEntry entry) {
-    try (InputStream bytecode = jar.getInputStream(entry)) {
-      // ClassReader doesn't take ownership and instead eagerly reads the stream's contents
-      return rewriter.reader(bytecode);
-    } catch (IOException e) {
-      // We should've already read through all files in the Jar once at this point, so we don't
-      // expect failures reading some files a second time.
-      throw new IllegalStateException("Couldn't load " + internalClassName, e);
-    }
-  }
 }
diff --git a/java/com/google/devtools/build/android/desugar/Desugar.java b/java/com/google/devtools/build/android/desugar/Desugar.java
index 6f9e0f0..22780ea 100644
--- a/java/com/google/devtools/build/android/desugar/Desugar.java
+++ b/java/com/google/devtools/build/android/desugar/Desugar.java
@@ -18,14 +18,14 @@ import static java.nio.charset.StandardCharsets.ISO_8859_1;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.io.ByteStreams;
+import com.google.common.io.Closer;
 import com.google.devtools.build.android.Converters.ExistingPathConverter;
 import com.google.devtools.build.android.Converters.PathConverter;
 import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsParser;
-import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.FileVisitResult;
@@ -34,14 +34,9 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
-import java.util.Enumeration;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.zip.CRC32;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
-import java.util.zip.ZipOutputStream;
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
@@ -62,8 +57,8 @@ class Desugar {
       converter = ExistingPathConverter.class,
       abbrev = 'i',
       help =
-          "Input Jar with classes to desugar (required, the n-th input is paired with the n-th "
-              + "output)."
+        "Input Jar or directory with classes to desugar (required, the n-th input is paired with"
+        + "the n-th output)."
     )
     public List<Path> inputJars;
 
@@ -96,6 +91,15 @@ class Desugar {
     public boolean allowEmptyBootclasspath;
 
     @Option(
+      name = "only_desugar_javac9_for_lint",
+      defaultValue = "false",
+      help =
+          "A temporary flag specifically for android lint, subject to removal anytime (DO NOT USE)",
+      category = "undocumented"
+    )
+    public boolean onlyDesugarJavac9ForLint;
+
+    @Option(
       name = "output",
       allowMultiple = true,
       defaultValue = "",
@@ -103,8 +107,8 @@ class Desugar {
       converter = PathConverter.class,
       abbrev = 'o',
       help =
-          "Output Jar to write desugared classes into (required, the n-th output is paired with "
-              + "the n-th input)."
+          "Output Jar or directory to write desugared classes into (required, the n-th output is "
+              + "paired with the n-th input, output must be a Jar if input is a Jar)."
     )
     public List<Path> outputJars;
 
@@ -171,7 +175,12 @@ class Desugar {
         "Desugar requires the same number of inputs and outputs to pair them");
     checkState(!options.bootclasspath.isEmpty() || options.allowEmptyBootclasspath,
         "At least one --bootclasspath_entry is required");
-
+    for (Path path : options.classpath) {
+      checkState(!Files.isDirectory(path), "Classpath entry must be a jar file: %s", path);
+    }
+    for (Path path : options.bootclasspath) {
+      checkState(!Files.isDirectory(path), "Bootclasspath entry must be a jar file: %s", path);
+    }
     if (options.verbose) {
       System.out.printf("Lambda classes will be written under %s%n", dumpDirectory);
     }
@@ -186,72 +195,86 @@ class Desugar {
 
     // Process each input separately
     for (InputOutputPair inputOutputPair : toInputOutputPairs(options)) {
-      Path inputJar = inputOutputPair.getInput();
-      IndexedJars appIndexedJar = new IndexedJars(ImmutableList.of(inputJar));
-      IndexedJars appAndClasspathIndexedJars = new IndexedJars(options.classpath, appIndexedJar);
-      ClassLoader loader =
-          createClassLoader(rewriter, options.bootclasspath, appAndClasspathIndexedJars);
-
-      try (ZipFile in = new ZipFile(inputJar.toFile());
-          ZipOutputStream out =
-              new ZipOutputStream(
-                  new BufferedOutputStream(Files.newOutputStream(inputOutputPair.getOutput())))) {
+      Path inputPath = inputOutputPair.getInput();
+      Path outputPath = inputOutputPair.getOutput();
+      checkState(
+          Files.isDirectory(inputPath) || !Files.isDirectory(outputPath),
+          "Input jar file requires an output jar file");
+      
+      try (Closer closer = Closer.create();
+          OutputFileProvider outputFileProvider = toOutputFileProvider(outputPath)) {
+        InputFileProvider appInputFiles = toInputFileProvider(closer, inputPath);
+        List<InputFileProvider> classpathInputFiles =
+            toInputFileProvider(closer, options.classpath);
+        IndexedInputs appIndexedInputs = new IndexedInputs(ImmutableList.of(appInputFiles));
+        IndexedInputs appAndClasspathIndexedInputs =
+            new IndexedInputs(classpathInputFiles, appIndexedInputs);
+        ClassLoader loader =
+            createClassLoader(
+                rewriter,
+                toInputFileProvider(closer, options.bootclasspath),
+                appAndClasspathIndexedInputs);
+
         ClassReaderFactory readerFactory =
             new ClassReaderFactory(
                 (options.copyBridgesFromClasspath && !allowDefaultMethods)
-                    ? appAndClasspathIndexedJars
-                    : appIndexedJar,
+                    ? appAndClasspathIndexedInputs
+                    : appIndexedInputs,
                 rewriter);
 
         ImmutableSet.Builder<String> interfaceLambdaMethodCollector = ImmutableSet.builder();
 
-        // Process input Jar, desugaring as we go
-        for (Enumeration<? extends ZipEntry> entries = in.entries(); entries.hasMoreElements(); ) {
-          ZipEntry entry = entries.nextElement();
-          try (InputStream content = in.getInputStream(entry)) {
-            // We can write classes uncompressed since they need to be converted to .dex format for
-            // Android anyways. Resources are written as they were in the input jar to avoid any
-            // danger of accidentally uncompressed resources ending up in an .apk.
-            if (entry.getName().endsWith(".class")) {
+        // Process inputs, desugaring as we go
+        for (String filename : appInputFiles) {
+          try (InputStream content = appInputFiles.getInputStream(filename)) {
+            // We can write classes uncompressed since they need to be converted to .dex format
+            // for Android anyways. Resources are written as they were in the input jar to avoid
+            // any danger of accidentally uncompressed resources ending up in an .apk.
+            if (filename.endsWith(".class")) {
               ClassReader reader = rewriter.reader(content);
               CoreLibraryRewriter.UnprefixingClassWriter writer =
                   rewriter.writer(ClassWriter.COMPUTE_MAXS /*for bridge methods*/);
               ClassVisitor visitor = writer;
-              if (!allowDefaultMethods) {
-                visitor = new Java7Compatibility(visitor, readerFactory);
-              }
 
-              visitor =
-                  new LambdaDesugaring(
-                      visitor, loader, lambdas, interfaceLambdaMethodCollector,
-                      allowDefaultMethods);
+              if (!options.onlyDesugarJavac9ForLint) {
+                if (!allowDefaultMethods) {
+                  visitor = new Java7Compatibility(visitor, readerFactory);
+                }
+
+                visitor =
+                    new LambdaDesugaring(
+                        visitor,
+                        loader,
+                        lambdas,
+                        interfaceLambdaMethodCollector,
+                        allowDefaultMethods);
+              }
 
               if (!allowCallsToObjectsNonNull) {
                 visitor = new ObjectsRequireNonNullMethodInliner(visitor);
               }
               reader.accept(visitor, 0);
 
-              writeStoredEntry(out, entry.getName(), writer.toByteArray());
+              outputFileProvider.write(filename, writer.toByteArray());
             } else {
-              // TODO(bazel-team): Avoid de- and re-compressing resource files
-              ZipEntry destEntry = new ZipEntry(entry);
-              destEntry.setCompressedSize(-1);
-              out.putNextEntry(destEntry);
-              ByteStreams.copy(content, out);
-              out.closeEntry();
+              outputFileProvider.copyFrom(filename, appInputFiles);
             }
           }
         }
 
         ImmutableSet<String> interfaceLambdaMethods = interfaceLambdaMethodCollector.build();
-        if (allowDefaultMethods) {
-          checkState(
-              interfaceLambdaMethods.isEmpty(),
-              "Desugaring with default methods enabled moved interface lambdas");
-        }
+        checkState(
+            !allowDefaultMethods || interfaceLambdaMethods.isEmpty(),
+            "Desugaring with default methods enabled moved interface lambdas");
 
         // Write out the lambda classes we generated along the way
-        for (Map.Entry<Path, LambdaInfo> lambdaClass : lambdas.drain().entrySet()) {
+        ImmutableMap<Path, LambdaInfo> lambdaClasses = lambdas.drain();
+        checkState(
+            !options.onlyDesugarJavac9ForLint || lambdaClasses.isEmpty(),
+            "There should be no lambda classes generated: %s",
+            lambdaClasses.keySet());
+
+        for (Map.Entry<Path, LambdaInfo> lambdaClass : lambdaClasses.entrySet()) {
           try (InputStream bytecode =
               Files.newInputStream(dumpDirectory.resolve(lambdaClass.getKey()))) {
             ClassReader reader = rewriter.reader(bytecode);
@@ -282,7 +305,7 @@ class Desugar {
             reader.accept(visitor, 0);
             String filename =
                 rewriter.unprefix(lambdaClass.getValue().desiredInternalName()) + ".class";
-            writeStoredEntry(out, filename, writer.toByteArray());
+            outputFileProvider.write(filename, writer.toByteArray());
           }
         }
 
@@ -292,7 +315,7 @@ class Desugar {
     }
   }
 
-   private static List<InputOutputPair> toInputOutputPairs(Options options) {
+  private static List<InputOutputPair> toInputOutputPairs(Options options) {
     final ImmutableList.Builder<InputOutputPair> ioPairListbuilder = ImmutableList.builder();
     for (Iterator<Path> inputIt = options.inputJars.iterator(),
                 outputIt = options.outputJars.iterator();
@@ -302,38 +325,22 @@ class Desugar {
     return ioPairListbuilder.build();
   }
 
-  private static void writeStoredEntry(ZipOutputStream out, String filename, byte[] content)
+  private static ClassLoader createClassLoader(
+      CoreLibraryRewriter rewriter,
+      List<InputFileProvider> bootclasspath,
+      IndexedInputs appAndClasspathIndexedInputs)
       throws IOException {
-    // Need to pre-compute checksum for STORED (uncompressed) entries)
-    CRC32 checksum = new CRC32();
-    checksum.update(content);
-
-    ZipEntry result = new ZipEntry(filename);
-    result.setTime(0L); // Use stable timestamp Jan 1 1980
-    result.setCrc(checksum.getValue());
-    result.setSize(content.length);
-    result.setCompressedSize(content.length);
-    // Write uncompressed, since this is just an intermediary artifact that we will convert to .dex
-    result.setMethod(ZipEntry.STORED);
-
-    out.putNextEntry(result);
-    out.write(content);
-    out.closeEntry();
-  }
-
-  private static ClassLoader createClassLoader(CoreLibraryRewriter rewriter,
-      List<Path> bootclasspath, IndexedJars appAndClasspathIndexedJars) throws IOException {
     // Use a classloader that as much as possible uses the provided bootclasspath instead of
     // the tool's system classloader.  Unfortunately we can't do that for java. classes.
     ClassLoader parent = new ThrowingClassLoader();
     if (!bootclasspath.isEmpty()) {
-      parent = new HeaderClassLoader(new IndexedJars(bootclasspath), rewriter, parent);
+      parent = new HeaderClassLoader(new IndexedInputs(bootclasspath), rewriter, parent);
     }
     // Prepend classpath with input jar itself so LambdaDesugaring can load classes with lambdas.
     // Note that inputJar and classpath need to be in the same classloader because we typically get
     // the header Jar for inputJar on the classpath and having the header Jar in a parent loader
     // means the header version is preferred over the real thing.
-    return new HeaderClassLoader(appAndClasspathIndexedJars, rewriter, parent);
+    return new HeaderClassLoader(appAndClasspathIndexedInputs, rewriter, parent);
   }
 
   private static class ThrowingClassLoader extends ClassLoader {
@@ -386,6 +393,36 @@ class Desugar {
     }
   }
 
+  /** Transform a Path to an {@link OutputFileProvider} */
+  private static OutputFileProvider toOutputFileProvider(Path path)
+      throws IOException {
+    if (Files.isDirectory(path)) {
+      return new DirectoryOutputFileProvider(path);
+    } else {
+      return new ZipOutputFileProvider(path);
+    }
+  }
+
+  /** Transform a Path to an InputFileProvider and register it to close it at the end of desugar */
+  private static InputFileProvider toInputFileProvider(Closer closer, Path path)
+      throws IOException {
+    if (Files.isDirectory(path)) {
+      return closer.register(new DirectoryInputFileProvider(path));
+    } else {
+      return closer.register(new ZipInputFileProvider(path));
+    }
+  }
+
+  private static ImmutableList<InputFileProvider> toInputFileProvider(
+      Closer closer, List<Path> paths) throws IOException {
+    ImmutableList.Builder<InputFileProvider> builder = new ImmutableList.Builder<>();
+    for (Path path : paths) {
+      checkState(!Files.isDirectory(path), "Directory is not supported: %s", path);
+      builder.add(closer.register(new ZipInputFileProvider(path)));
+    }
+    return builder.build();
+  }
+  
   /**
    * Pair input and output.
    */
diff --git a/java/com/google/devtools/build/android/desugar/DirectoryInputFileProvider.java b/java/com/google/devtools/build/android/desugar/DirectoryInputFileProvider.java
new file mode 100644
index 0000000..1c5abc9
--- /dev/null
+++ b/java/com/google/devtools/build/android/desugar/DirectoryInputFileProvider.java
@@ -0,0 +1,81 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.desugar;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOError;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+
+/** Input provider is a directory. */
+class DirectoryInputFileProvider implements InputFileProvider {
+
+  private final Path root;
+
+  public DirectoryInputFileProvider(Path root) {
+    this.root = root;
+  }
+
+  @Override
+  public String toString() {
+    return root.getFileName().toString();
+  }
+
+  @Override
+  public InputStream getInputStream(String filename) throws IOException {
+    return new FileInputStream(root.resolve(filename).toFile());
+  }
+
+  @Override
+  public ZipEntry getZipEntry(String filename) {
+    ZipEntry destEntry = new ZipEntry(filename);
+    destEntry.setTime(0L); // Use stable timestamp Jan 1 1980
+    return destEntry;
+  }
+
+  @Override
+  public void close() throws IOException {
+    // Nothing to close
+  }
+
+  @Override
+  public Iterator<String> iterator() {
+    final List<String> entries = new ArrayList<>();
+    try (Stream<Path> paths = Files.walk(root)) {
+      paths.forEach(
+          new Consumer<Path>() {
+            @Override
+            public void accept(Path t) {
+              if (Files.isRegularFile(t)) {
+                // Internally, we use '/' as a common package separator in filename to abstract
+                // that filename can comes from a zip or a directory.
+                entries.add(root.relativize(t).toString().replace(File.separatorChar, '/'));
+              }
+            }
+          });
+    } catch (IOException e) {
+      throw new IOError(e);
+    }
+    return entries.iterator();
+  }
+}
diff --git a/java/com/google/devtools/build/android/desugar/DirectoryOutputFileProvider.java b/java/com/google/devtools/build/android/desugar/DirectoryOutputFileProvider.java
new file mode 100644
index 0000000..782a81e
--- /dev/null
+++ b/java/com/google/devtools/build/android/desugar/DirectoryOutputFileProvider.java
@@ -0,0 +1,59 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.desugar;
+
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/** Output provider is a directory. */
+public class DirectoryOutputFileProvider implements OutputFileProvider {
+
+  private final Path root;
+
+  public DirectoryOutputFileProvider(Path root) {
+    this.root = root;
+  }
+
+  @Override
+  public void copyFrom(String filename, InputFileProvider inputFileProvider) throws IOException {
+    Path path = root.resolve(filename);
+    createParentFolder(path);
+    try (InputStream is = inputFileProvider.getInputStream(filename);
+        OutputStream os = Files.newOutputStream(path)) {
+      ByteStreams.copy(is, os);
+    }
+  }
+
+  @Override
+  public void write(String filename, byte[] content) throws IOException {
+    Path path = root.resolve(filename);
+    createParentFolder(path);
+    Files.write(path, content);
+  }
+
+  @Override
+  public void close() {
+    // Nothing to close
+  }
+
+  private void createParentFolder(Path path) throws IOException {
+    if (!Files.exists(path.getParent())) {
+      Files.createDirectories(path.getParent());
+    }
+  }
+}
diff --git a/java/com/google/devtools/build/android/desugar/HeaderClassLoader.java b/java/com/google/devtools/build/android/desugar/HeaderClassLoader.java
index 44c3932..154b793 100644
--- a/java/com/google/devtools/build/android/desugar/HeaderClassLoader.java
+++ b/java/com/google/devtools/build/android/desugar/HeaderClassLoader.java
@@ -16,8 +16,6 @@ package com.google.devtools.build.android.desugar;
 import java.io.IOError;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.jar.JarFile;
-import java.util.zip.ZipEntry;
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
@@ -35,26 +33,25 @@ import org.objectweb.asm.Opcodes;
  */
 class HeaderClassLoader extends ClassLoader {
 
-  private final IndexedJars indexedJars;
+  private final IndexedInputs indexedInputs;
   private final CoreLibraryRewriter rewriter;
 
   public HeaderClassLoader(
-      IndexedJars indexedJars, CoreLibraryRewriter rewriter, ClassLoader parent) {
+      IndexedInputs indexedInputs, CoreLibraryRewriter rewriter, ClassLoader parent) {
     super(parent);
     this.rewriter = rewriter;
-    this.indexedJars = indexedJars;
+    this.indexedInputs = indexedInputs;
   }
 
   @Override
   protected Class<?> findClass(String name) throws ClassNotFoundException {
     String filename = rewriter.unprefix(name.replace('.', '/') + ".class");
-    JarFile jarfile = indexedJars.getJarFile(filename);
-    if (jarfile == null) {
-      throw new ClassNotFoundException();
+    InputFileProvider inputFileProvider = indexedInputs.getInputFileProvider(filename);
+    if (inputFileProvider == null) {
+      throw new ClassNotFoundException("Class " + name + " not found");
     }
-    ZipEntry entry = jarfile.getEntry(filename);
     byte[] bytecode;
-    try (InputStream content = jarfile.getInputStream(entry)) {
+    try (InputStream content = inputFileProvider.getInputStream(filename)) {
       ClassReader reader = rewriter.reader(content);
       // Have ASM compute maxs so we don't need to figure out how many formal parameters there are
       ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS);
diff --git a/java/com/google/devtools/build/android/desugar/IndexedInputs.java b/java/com/google/devtools/build/android/desugar/IndexedInputs.java
new file mode 100644
index 0000000..58459cc
--- /dev/null
+++ b/java/com/google/devtools/build/android/desugar/IndexedInputs.java
@@ -0,0 +1,75 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.desugar;
+
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Opens the given list of input files and compute an index of all classes in them, to avoid
+ * scanning all inputs over and over for each class to load. An indexed inputs can have a parent
+ * that is firstly used when a file name is searched.
+ */
+class IndexedInputs {
+
+  private final Map<String, InputFileProvider> inputFiles = new HashMap<>();
+
+  /** Parent indexed inputs to use before to search a file name into this indexed inputs. */
+  @Nullable
+  private final IndexedInputs parentIndexedInputs;
+
+  /** Index a list of input files without a parent indexed inputs. */
+  public IndexedInputs(List<InputFileProvider> inputProviders) throws IOException {
+    this(inputProviders, null);
+  }
+
+  /**
+   * Index a list of input files and set a parent indexed inputs that is firstly used during the
+   * search of a file name.
+   */
+  public IndexedInputs(
+      List<InputFileProvider> inputProviders, @Nullable IndexedInputs parentIndexedInputs)
+      throws IOException {
+    this.parentIndexedInputs = parentIndexedInputs;
+    for (InputFileProvider inputProvider : inputProviders) {
+      indexInput(inputProvider);
+    }
+  }
+
+  @Nullable
+  public InputFileProvider getInputFileProvider(String filename) {
+    Preconditions.checkArgument(filename.endsWith(".class"));
+
+    if (parentIndexedInputs != null) {
+      InputFileProvider inputFileProvider = parentIndexedInputs.getInputFileProvider(filename);
+      if (inputFileProvider != null) {
+        return inputFileProvider;
+      }
+    }
+
+    return inputFiles.get(filename);
+  }
+
+  private void indexInput(final InputFileProvider inputFileProvider) throws IOException {
+    for (String relativePath : inputFileProvider) {
+      if (relativePath.endsWith(".class") && !inputFiles.containsKey(relativePath)) {
+        inputFiles.put(relativePath, inputFileProvider);
+      }
+    }
+  }
+}
diff --git a/java/com/google/devtools/build/android/desugar/IndexedJars.java b/java/com/google/devtools/build/android/desugar/IndexedJars.java
deleted file mode 100644
index bdb5b47..0000000
--- a/java/com/google/devtools/build/android/desugar/IndexedJars.java
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright 2017 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package com.google.devtools.build.android.desugar;
-
-import com.google.common.base.Preconditions;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-import javax.annotation.Nullable;
-
-/**
- * Opens the given list of Jar files and compute an index of all classes in them, to avoid
- * scanning all Jars over and over for each class to load. An indexed jars can have a parent
- * that is firstly used when a file name is searched.
- */
-class IndexedJars {
-
-  private final Map<String, JarFile> jarfiles = new HashMap<>();
-
-  /**
-   * Parent indexed jars to use before to search a file name into this indexed jars.
-   */
-  @Nullable
-  private final IndexedJars parentIndexedJar;
-
-  /**
-   * Index a list of Jar files without a parent indexed jars.
-   */
-  public IndexedJars(List<Path> jarFiles) throws IOException {
-    this(jarFiles, null);
-  }
-
-  /**
-   * Index a list of Jar files and set a parent indexed jars that is firstly used during the search
-   * of a file name.
-   */
-  public IndexedJars(List<Path> jarFiles, @Nullable IndexedJars parentIndexedJar)
-      throws IOException {
-    this.parentIndexedJar = parentIndexedJar;
-    for (Path jarfile : jarFiles) {
-      indexJar(jarfile);
-    }
-  }
-
-  @Nullable
-  public JarFile getJarFile(String filename) {
-    Preconditions.checkArgument(filename.endsWith(".class"));
-
-    if (parentIndexedJar != null) {
-      JarFile jarFile = parentIndexedJar.getJarFile(filename);
-      if (jarFile != null) {
-        return jarFile;
-      }
-    }
-
-    return jarfiles.get(filename);
-  }
-
-  private void indexJar(Path jarfile) throws IOException {
-    JarFile jar = new JarFile(jarfile.toFile());
-    for (Enumeration<JarEntry> cur = jar.entries(); cur.hasMoreElements(); ) {
-      JarEntry entry = cur.nextElement();
-      if (entry.getName().endsWith(".class") && !jarfiles.containsKey(entry.getName())) {
-        jarfiles.put(entry.getName(), jar);
-      }
-    }
-  }
-}
diff --git a/java/com/google/devtools/build/android/desugar/InputFileProvider.java b/java/com/google/devtools/build/android/desugar/InputFileProvider.java
new file mode 100644
index 0000000..c2b6353
--- /dev/null
+++ b/java/com/google/devtools/build/android/desugar/InputFileProvider.java
@@ -0,0 +1,36 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.desugar;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.ZipEntry;
+
+/** Input file provider allows to iterate on relative path filename of a directory or a jar file. */
+interface InputFileProvider extends Closeable, Iterable<String> {
+
+  /**
+   * Return a ZipEntry for {@code filename}. If the provider is a {@link ZipInputFileProvider}, the
+   * method returns the existing ZipEntry in order to keep its metadata, otherwise a new one is
+   * created.
+   */
+  ZipEntry getZipEntry(String filename);
+
+  /**
+   * This method returns an input stream allowing to read the file {@code filename}, it is the
+   * responsibility of the caller to close this stream.
+   */
+  InputStream getInputStream(String filename) throws IOException;
+}
diff --git a/java/com/google/devtools/build/android/desugar/LambdaClassMaker.java b/java/com/google/devtools/build/android/desugar/LambdaClassMaker.java
index 2a9f9be..155e323 100644
--- a/java/com/google/devtools/build/android/desugar/LambdaClassMaker.java
+++ b/java/com/google/devtools/build/android/desugar/LambdaClassMaker.java
@@ -57,7 +57,7 @@ class LambdaClassMaker {
    * Returns absolute paths to .class files generated since the last call to this method together
    * with a string descriptor of the factory method.
    */
-  public Map<Path, LambdaInfo> drain() {
+  public ImmutableMap<Path, LambdaInfo> drain() {
     ImmutableMap<Path, LambdaInfo> result = ImmutableMap.copyOf(generatedClasses);
     generatedClasses.clear();
     return result;
diff --git a/java/com/google/devtools/build/android/desugar/OutputFileProvider.java b/java/com/google/devtools/build/android/desugar/OutputFileProvider.java
new file mode 100644
index 0000000..bf3b710
--- /dev/null
+++ b/java/com/google/devtools/build/android/desugar/OutputFileProvider.java
@@ -0,0 +1,29 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.desugar;
+
+import java.io.IOException;
+
+/** Output file provider allows to write files in directory or jar files. */
+interface OutputFileProvider extends AutoCloseable {
+
+  /**
+   * Copy {@code filename} from {@code inputFileProvider} to this output. If input file provider is
+   * a {@link ZipInputFileProvider} then the metadata of the zip entry are kept.
+   */
+  void copyFrom(String filename, InputFileProvider inputFileProvider) throws IOException;
+
+  /** Write {@code content} in {@code filename} to this output */
+  void write(String filename, byte[] content) throws IOException;
+}
diff --git a/java/com/google/devtools/build/android/desugar/ZipInputFileProvider.java b/java/com/google/devtools/build/android/desugar/ZipInputFileProvider.java
new file mode 100644
index 0000000..307c8b8
--- /dev/null
+++ b/java/com/google/devtools/build/android/desugar/ZipInputFileProvider.java
@@ -0,0 +1,64 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.desugar;
+
+import com.google.common.base.Functions;
+import com.google.common.collect.Iterators;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+/** Input provider is a zip file. */
+class ZipInputFileProvider implements InputFileProvider {
+
+  private final Path root;
+
+  private final ZipFile zipFile;
+
+  public ZipInputFileProvider(Path root) throws IOException {
+    this.root = root;
+    this.zipFile = new ZipFile(root.toFile());
+  }
+
+  @Override
+  public void close() throws IOException {
+    zipFile.close();
+  }
+
+  @Override
+  public String toString() {
+    return root.getFileName().toString();
+  }
+
+  @Override
+  public ZipEntry getZipEntry(String filename) {
+    ZipEntry zipEntry = zipFile.getEntry(filename);
+    zipEntry.setCompressedSize(-1);
+    return zipEntry;
+  }
+
+  @Override
+  public InputStream getInputStream(String filename) throws IOException {
+    return zipFile.getInputStream(zipFile.getEntry(filename));
+  }
+
+  @Override
+  public Iterator<String> iterator() {
+    return Iterators.transform(
+        Iterators.forEnumeration(zipFile.entries()), Functions.toStringFunction());
+  }
+}
diff --git a/java/com/google/devtools/build/android/desugar/ZipOutputFileProvider.java b/java/com/google/devtools/build/android/desugar/ZipOutputFileProvider.java
new file mode 100644
index 0000000..3f4f344
--- /dev/null
+++ b/java/com/google/devtools/build/android/desugar/ZipOutputFileProvider.java
@@ -0,0 +1,76 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.desugar;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.ByteStreams;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.zip.CRC32;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/** Output provider is a zip file. */
+public class ZipOutputFileProvider implements OutputFileProvider {
+
+  private final ZipOutputStream out;
+
+  public ZipOutputFileProvider(Path root) throws IOException {
+    out = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(root)));
+  }
+
+  @Override
+  public void copyFrom(String filename, InputFileProvider inputFileProvider) throws IOException {
+    // TODO(bazel-team): Avoid de- and re-compressing resource files
+    out.putNextEntry(inputFileProvider.getZipEntry(filename));
+    try (InputStream is = inputFileProvider.getInputStream(filename)) {
+      ByteStreams.copy(is, out);
+    }
+    out.closeEntry();
+  }
+
+  @Override
+  public void write(String filename, byte[] content) throws IOException {
+    Preconditions.checkArgument(filename.endsWith(".class"));
+    writeStoredEntry(out, filename, content);
+  }
+
+  @Override
+  public void close() throws IOException {
+    out.close();
+  }
+
+  private static void writeStoredEntry(ZipOutputStream out, String filename, byte[] content)
+      throws IOException {
+    // Need to pre-compute checksum for STORED (uncompressed) entries)
+    CRC32 checksum = new CRC32();
+    checksum.update(content);
+
+    ZipEntry result = new ZipEntry(filename);
+    result.setTime(0L); // Use stable timestamp Jan 1 1980
+    result.setCrc(checksum.getValue());
+    result.setSize(content.length);
+    result.setCompressedSize(content.length);
+    // Write uncompressed, since this is just an intermediary artifact that
+    // we will convert to .dex
+    result.setMethod(ZipEntry.STORED);
+
+    out.putNextEntry(result);
+    out.write(content);
+    out.closeEntry();
+  }
+}
diff --git a/java/com/google/devtools/common/options/Converters.java b/java/com/google/devtools/common/options/Converters.java
index c8b4d47..0d19029 100644
--- a/java/com/google/devtools/common/options/Converters.java
+++ b/java/com/google/devtools/common/options/Converters.java
@@ -16,7 +16,6 @@ package com.google.devtools.common.options;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
-
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -30,6 +29,171 @@ import java.util.regex.PatternSyntaxException;
  */
 public final class Converters {
 
+  /** Standard converter for booleans. Accepts common shorthands/synonyms. */
+  public static class BooleanConverter implements Converter<Boolean> {
+    @Override
+    public Boolean convert(String input) throws OptionsParsingException {
+      if (input == null) {
+        return false;
+      }
+      input = input.toLowerCase();
+      if (input.equals("true")
+          || input.equals("1")
+          || input.equals("yes")
+          || input.equals("t")
+          || input.equals("y")) {
+        return true;
+      }
+      if (input.equals("false")
+          || input.equals("0")
+          || input.equals("no")
+          || input.equals("f")
+          || input.equals("n")) {
+        return false;
+      }
+      throw new OptionsParsingException("'" + input + "' is not a boolean");
+    }
+
+    @Override
+    public String getTypeDescription() {
+      return "a boolean";
+    }
+  }
+
+  /** Standard converter for Strings. */
+  public static class StringConverter implements Converter<String> {
+    @Override
+    public String convert(String input) {
+      return input;
+    }
+
+    @Override
+    public String getTypeDescription() {
+      return "a string";
+    }
+  }
+
+  /** Standard converter for integers. */
+  public static class IntegerConverter implements Converter<Integer> {
+    @Override
+    public Integer convert(String input) throws OptionsParsingException {
+      try {
+        return Integer.decode(input);
+      } catch (NumberFormatException e) {
+        throw new OptionsParsingException("'" + input + "' is not an int");
+      }
+    }
+
+    @Override
+    public String getTypeDescription() {
+      return "an integer";
+    }
+  }
+
+  /** Standard converter for longs. */
+  public static class LongConverter implements Converter<Long> {
+    @Override
+    public Long convert(String input) throws OptionsParsingException {
+      try {
+        return Long.decode(input);
+      } catch (NumberFormatException e) {
+        throw new OptionsParsingException("'" + input + "' is not a long");
+      }
+    }
+
+    @Override
+    public String getTypeDescription() {
+      return "a long integer";
+    }
+  }
+
+  /** Standard converter for doubles. */
+  public static class DoubleConverter implements Converter<Double> {
+    @Override
+    public Double convert(String input) throws OptionsParsingException {
+      try {
+        return Double.parseDouble(input);
+      } catch (NumberFormatException e) {
+        throw new OptionsParsingException("'" + input + "' is not a double");
+      }
+    }
+
+    @Override
+    public String getTypeDescription() {
+      return "a double";
+    }
+  }
+
+  /** Standard converter for TriState values. */
+  public static class TriStateConverter implements Converter<TriState> {
+    @Override
+    public TriState convert(String input) throws OptionsParsingException {
+      if (input == null) {
+        return TriState.AUTO;
+      }
+      input = input.toLowerCase();
+      if (input.equals("auto")) {
+        return TriState.AUTO;
+      }
+      if (input.equals("true")
+          || input.equals("1")
+          || input.equals("yes")
+          || input.equals("t")
+          || input.equals("y")) {
+        return TriState.YES;
+      }
+      if (input.equals("false")
+          || input.equals("0")
+          || input.equals("no")
+          || input.equals("f")
+          || input.equals("n")) {
+        return TriState.NO;
+      }
+      throw new OptionsParsingException("'" + input + "' is not a boolean");
+    }
+
+    @Override
+    public String getTypeDescription() {
+      return "a tri-state (auto, yes, no)";
+    }
+  }
+
+  /**
+   * Standard "converter" for Void. Should not actually be invoked. For instance, expansion flags
+   * are usually Void-typed and do not invoke the converter.
+   */
+  public static class VoidConverter implements Converter<Void> {
+    @Override
+    public Void convert(String input) throws OptionsParsingException {
+      if (input == null) {
+        return null; // expected input, return is unused so null is fine.
+      }
+      throw new OptionsParsingException("'" + input + "' unexpected");
+    }
+
+    @Override
+    public String getTypeDescription() {
+      return "";
+    }
+  }
+
+  /**
+   * The converters that are available to the options parser by default. These are used if the
+   * {@code @Option} annotation does not specify its own {@code converter}, and its type is one of
+   * the following.
+   */
+  static final Map<Class<?>, Converter<?>> DEFAULT_CONVERTERS = Maps.newHashMap();
+
+  static {
+    DEFAULT_CONVERTERS.put(String.class, new Converters.StringConverter());
+    DEFAULT_CONVERTERS.put(int.class, new Converters.IntegerConverter());
+    DEFAULT_CONVERTERS.put(long.class, new Converters.LongConverter());
+    DEFAULT_CONVERTERS.put(double.class, new Converters.DoubleConverter());
+    DEFAULT_CONVERTERS.put(boolean.class, new Converters.BooleanConverter());
+    DEFAULT_CONVERTERS.put(TriState.class, new Converters.TriStateConverter());
+    DEFAULT_CONVERTERS.put(Void.class, new Converters.VoidConverter());
+  }
+
   /**
    * Join a list of words as in English.  Examples:
    * "nothing"
@@ -92,7 +256,7 @@ public final class Converters {
 
   public static class LogLevelConverter implements Converter<Level> {
 
-    public static Level[] LEVELS = new Level[] {
+    public static final Level[] LEVELS = new Level[] {
       Level.OFF, Level.SEVERE, Level.WARNING, Level.INFO, Level.FINE,
       Level.FINER, Level.FINEST
     };
@@ -295,32 +459,4 @@ public final class Converters {
     }
   }
 
-  /**
-   * A converter for boolean values. This is already one of the defaults, so clients
-   * should not typically need to add this.
-   */
-  public static class BooleanConverter implements Converter<Boolean> {
-    @Override
-    public Boolean convert(String input) throws OptionsParsingException {
-      if (input == null) {
-        return false;
-      }
-      input = input.toLowerCase();
-      if (input.equals("true") || input.equals("1") || input.equals("yes") ||
-          input.equals("t") || input.equals("y")) {
-        return true;
-      }
-      if (input.equals("false") || input.equals("0") || input.equals("no") ||
-          input.equals("f") || input.equals("n")) {
-        return false;
-      }
-      throw new OptionsParsingException("'" + input + "' is not a boolean");
-    }
-
-    @Override
-    public String getTypeDescription() {
-      return "a boolean";
-    }
-  }
-
 }
diff --git a/java/com/google/devtools/common/options/ExpansionFunction.java b/java/com/google/devtools/common/options/ExpansionFunction.java
new file mode 100644
index 0000000..ffab6e7
--- /dev/null
+++ b/java/com/google/devtools/common/options/ExpansionFunction.java
@@ -0,0 +1,31 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+/**
+ * A function from an option parser's static setup (what flags it knows about) to an expansion
+ * String[] to use for one of its options.
+ */
+public interface ExpansionFunction {
+
+  /**
+   * Compute the expansion for an option. May be called at any time during or after the {@link
+   * OptionsParser}'s construction, or not at all.
+   *
+   * @param optionsData the parser's indexed information about its own options, before expansion
+   *     information is computed
+   * @return An expansion to use for all occurrences of this option in this parser
+   */
+  public String[] getExpansion(IsolatedOptionsData optionsData);
+}
diff --git a/java/com/google/devtools/common/options/IsolatedOptionsData.java b/java/com/google/devtools/common/options/IsolatedOptionsData.java
new file mode 100644
index 0000000..27f42f4
--- /dev/null
+++ b/java/com/google/devtools/common/options/IsolatedOptionsData.java
@@ -0,0 +1,382 @@
+// Copyright 2014 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * An immutable selection of options data corresponding to a set of options classes. The data is
+ * collected using reflection, which can be expensive. Therefore this class can be used internally
+ * to cache the results.
+ *
+ * <p>The data is isolated in the sense that it has not yet been processed to add inter-option-
+ * dependent information -- namely, the results of evaluating expansion functions. The {@link
+ * OptionsData} subclass stores this added information. The reason for the split is so that we can
+ * avoid exposing to expansion functions the effects of evaluating other expansion functions, to
+ * ensure that the order in which they run is not significant.
+ */
+// TODO(brandjon): This class is technically not necessarily immutable due to optionsDefault
+// accepting Object values, and the List in allOptionsField should be ImmutableList. Either fix
+// this or remove @Immutable.
+@Immutable
+class IsolatedOptionsData extends OpaqueOptionsData {
+
+  /**
+   * These are the options-declaring classes which are annotated with {@link Option} annotations.
+   */
+  private final ImmutableMap<Class<? extends OptionsBase>, Constructor<?>> optionsClasses;
+
+  /** Maps option name to Option-annotated Field. */
+  private final ImmutableMap<String, Field> nameToField;
+
+  /** Maps option abbreviation to Option-annotated Field. */
+  private final ImmutableMap<Character, Field> abbrevToField;
+
+  /** For each options class, contains a list of all Option-annotated fields in that class. */
+  private final ImmutableMap<Class<? extends OptionsBase>, List<Field>> allOptionsFields;
+
+  /** Mapping from each Option-annotated field to the default value for that field. */
+  // Immutable like the others, but uses Collections.unmodifiableMap because of null values.
+  private final Map<Field, Object> optionDefaults;
+
+  /**
+   * Mapping from each Option-annotated field to the proper converter.
+   *
+   * @see #findConverter
+   */
+  private final ImmutableMap<Field, Converter<?>> converters;
+
+  /**
+   * Mapping from each Option-annotated field to a boolean for whether that field allows multiple
+   * values.
+   */
+  private final ImmutableMap<Field, Boolean> allowMultiple;
+
+  private IsolatedOptionsData(
+      Map<Class<? extends OptionsBase>, Constructor<?>> optionsClasses,
+      Map<String, Field> nameToField,
+      Map<Character, Field> abbrevToField,
+      Map<Class<? extends OptionsBase>, List<Field>> allOptionsFields,
+      Map<Field, Object> optionDefaults,
+      Map<Field, Converter<?>> converters,
+      Map<Field, Boolean> allowMultiple) {
+    this.optionsClasses = ImmutableMap.copyOf(optionsClasses);
+    this.nameToField = ImmutableMap.copyOf(nameToField);
+    this.abbrevToField = ImmutableMap.copyOf(abbrevToField);
+    this.allOptionsFields = ImmutableMap.copyOf(allOptionsFields);
+    // Can't use an ImmutableMap here because of null values.
+    this.optionDefaults = Collections.unmodifiableMap(optionDefaults);
+    this.converters = ImmutableMap.copyOf(converters);
+    this.allowMultiple = ImmutableMap.copyOf(allowMultiple);
+  }
+
+  protected IsolatedOptionsData(IsolatedOptionsData other) {
+    this(
+        other.optionsClasses,
+        other.nameToField,
+        other.abbrevToField,
+        other.allOptionsFields,
+        other.optionDefaults,
+        other.converters,
+        other.allowMultiple);
+  }
+
+  public Collection<Class<? extends OptionsBase>> getOptionsClasses() {
+    return optionsClasses.keySet();
+  }
+
+  @SuppressWarnings("unchecked") // The construction ensures that the case is always valid.
+  public <T extends OptionsBase> Constructor<T> getConstructor(Class<T> clazz) {
+    return (Constructor<T>) optionsClasses.get(clazz);
+  }
+
+  public Field getFieldFromName(String name) {
+    return nameToField.get(name);
+  }
+
+  public Iterable<Map.Entry<String, Field>> getAllNamedFields() {
+    return nameToField.entrySet();
+  }
+
+  public Field getFieldForAbbrev(char abbrev) {
+    return abbrevToField.get(abbrev);
+  }
+
+  public List<Field> getFieldsForClass(Class<? extends OptionsBase> optionsClass) {
+    return allOptionsFields.get(optionsClass);
+  }
+
+  public Object getDefaultValue(Field field) {
+    return optionDefaults.get(field);
+  }
+
+  public Converter<?> getConverter(Field field) {
+    return converters.get(field);
+  }
+
+  public boolean getAllowMultiple(Field field) {
+    return allowMultiple.get(field);
+  }
+
+  /**
+   * For an option that does not use {@link Option#allowMultiple}, returns its type. For an option
+   * that does use it, asserts that the type is a {@code List<T>} and returns its element type
+   * {@code T}.
+   */
+  private static Type getFieldSingularType(Field field, Option annotation) {
+    Type fieldType = field.getGenericType();
+    if (annotation.allowMultiple()) {
+      // If the type isn't a List<T>, this is an error in the option's declaration.
+      if (!(fieldType instanceof ParameterizedType)) {
+        throw new AssertionError("Type of multiple occurrence option must be a List<...>");
+      }
+      ParameterizedType pfieldType = (ParameterizedType) fieldType;
+      if (pfieldType.getRawType() != List.class) {
+        throw new AssertionError("Type of multiple occurrence option must be a List<...>");
+      }
+      fieldType = pfieldType.getActualTypeArguments()[0];
+    }
+    return fieldType;
+  }
+
+  /**
+   * Returns whether a field should be considered as boolean.
+   *
+   * <p>Can be used for usage help and controlling whether the "no" prefix is allowed.
+   */
+  static boolean isBooleanField(Field field) {
+    return field.getType().equals(boolean.class)
+        || field.getType().equals(TriState.class)
+        || findConverter(field) instanceof BoolOrEnumConverter;
+  }
+
+  /** Returns whether a field has Void type. */
+  static boolean isVoidField(Field field) {
+    return field.getType().equals(Void.class);
+  }
+
+  /**
+   * Returns whether the arg is an expansion option defined by an expansion function (and not a
+   * constant expansion value).
+   */
+  static boolean usesExpansionFunction(Option annotation) {
+    return annotation.expansionFunction() != ExpansionFunction.class;
+  }
+
+  /**
+   * Given an {@code @Option}-annotated field, retrieves the {@link Converter} that will be used,
+   * taking into account the default converters if an explicit one is not specified.
+   */
+  static Converter<?> findConverter(Field optionField) {
+    Option annotation = optionField.getAnnotation(Option.class);
+    if (annotation.converter() == Converter.class) {
+      // No converter provided, use the default one.
+      Type type = getFieldSingularType(optionField, annotation);
+      Converter<?> converter = Converters.DEFAULT_CONVERTERS.get(type);
+      if (converter == null) {
+        throw new AssertionError(
+            "No converter found for "
+                + type
+                + "; possible fix: add "
+                + "converter=... to @Option annotation for "
+                + optionField.getName());
+      }
+      return converter;
+    }
+    try {
+      // Instantiate the given Converter class.
+      Class<?> converter = annotation.converter();
+      Constructor<?> constructor = converter.getConstructor();
+      return (Converter<?>) constructor.newInstance();
+    } catch (Exception e) {
+      // This indicates an error in the Converter, and should be discovered the first time it is
+      // used.
+      throw new AssertionError(e);
+    }
+  }
+
+  private static List<Field> getAllAnnotatedFields(Class<? extends OptionsBase> optionsClass) {
+    List<Field> allFields = Lists.newArrayList();
+    for (Field field : optionsClass.getFields()) {
+      if (field.isAnnotationPresent(Option.class)) {
+        allFields.add(field);
+      }
+    }
+    if (allFields.isEmpty()) {
+      throw new IllegalStateException(optionsClass + " has no public @Option-annotated fields");
+    }
+    return ImmutableList.copyOf(allFields);
+  }
+
+  private static Object retrieveDefaultFromAnnotation(Field optionField) {
+    Converter<?> converter = findConverter(optionField);
+    String defaultValueAsString = OptionsParserImpl.getDefaultOptionString(optionField);
+    // Special case for "null"
+    if (OptionsParserImpl.isSpecialNullDefault(defaultValueAsString, optionField)) {
+      return null;
+    }
+    boolean allowsMultiple = optionField.getAnnotation(Option.class).allowMultiple();
+    // If the option allows multiple values then we intentionally return the empty list as
+    // the default value of this option since it is not always the case that an option
+    // that allows multiple values will have a converter that returns a list value.
+    if (allowsMultiple) {
+      return Collections.emptyList();
+    }
+    // Otherwise try to convert the default value using the converter
+    Object convertedValue;
+    try {
+      convertedValue = converter.convert(defaultValueAsString);
+    } catch (OptionsParsingException e) {
+      throw new IllegalStateException("OptionsParsingException while "
+          + "retrieving default for " + optionField.getName() + ": "
+          + e.getMessage());
+    }
+    return convertedValue;
+  }
+
+  /**
+   * Constructs an {@link IsolatedOptionsData} object for a parser that knows about the given
+   * {@link OptionsBase} classes. No inter-option analysis is done. Performs basic sanity checking
+   * on each option in isolation.
+   */
+  static IsolatedOptionsData from(Collection<Class<? extends OptionsBase>> classes) {
+    Map<Class<? extends OptionsBase>, Constructor<?>> constructorBuilder = Maps.newHashMap();
+    Map<Class<? extends OptionsBase>, List<Field>> allOptionsFieldsBuilder = Maps.newHashMap();
+    Map<String, Field> nameToFieldBuilder = Maps.newHashMap();
+    Map<Character, Field> abbrevToFieldBuilder = Maps.newHashMap();
+    Map<Field, Object> optionDefaultsBuilder = Maps.newHashMap();
+    Map<Field, Converter<?>> convertersBuilder = Maps.newHashMap();
+    Map<Field, Boolean> allowMultipleBuilder = Maps.newHashMap();
+
+    // Read all Option annotations:
+    for (Class<? extends OptionsBase> parsedOptionsClass : classes) {
+      try {
+        Constructor<? extends OptionsBase> constructor =
+            parsedOptionsClass.getConstructor();
+        constructorBuilder.put(parsedOptionsClass, constructor);
+      } catch (NoSuchMethodException e) {
+        throw new IllegalArgumentException(parsedOptionsClass
+            + " lacks an accessible default constructor");
+      }
+      List<Field> fields = getAllAnnotatedFields(parsedOptionsClass);
+      allOptionsFieldsBuilder.put(parsedOptionsClass, fields);
+
+      for (Field field : fields) {
+        Option annotation = field.getAnnotation(Option.class);
+
+        if (annotation.name() == null) {
+          throw new AssertionError("Option cannot have a null name");
+        }
+
+        Type fieldType = getFieldSingularType(field, annotation);
+
+        // Get the converter return type.
+        @SuppressWarnings("rawtypes")
+        Class<? extends Converter> converter = annotation.converter();
+        if (converter == Converter.class) {
+          Converter<?> actualConverter = Converters.DEFAULT_CONVERTERS.get(fieldType);
+          if (actualConverter == null) {
+            throw new AssertionError("Cannot find converter for field of type "
+                + field.getType() + " named " + field.getName()
+                + " in class " + field.getDeclaringClass().getName());
+          }
+          converter = actualConverter.getClass();
+        }
+        if (Modifier.isAbstract(converter.getModifiers())) {
+          throw new AssertionError("The converter type " + converter
+              + " must be a concrete type");
+        }
+        Type converterResultType;
+        try {
+          Method convertMethod = converter.getMethod("convert", String.class);
+          converterResultType = GenericTypeHelper.getActualReturnType(converter, convertMethod);
+        } catch (NoSuchMethodException e) {
+          throw new AssertionError("A known converter object doesn't implement the convert"
+              + " method");
+        }
+
+        if (annotation.allowMultiple()) {
+          if (GenericTypeHelper.getRawType(converterResultType) == List.class) {
+            Type elementType =
+                ((ParameterizedType) converterResultType).getActualTypeArguments()[0];
+            if (!GenericTypeHelper.isAssignableFrom(fieldType, elementType)) {
+              throw new AssertionError("If the converter return type of a multiple occurance " +
+                  "option is a list, then the type of list elements (" + fieldType + ") must be " +
+                  "assignable from the converter list element type (" + elementType + ")");
+            }
+          } else {
+            if (!GenericTypeHelper.isAssignableFrom(fieldType, converterResultType)) {
+              throw new AssertionError("Type of list elements (" + fieldType +
+                  ") for multiple occurrence option must be assignable from the converter " +
+                  "return type (" + converterResultType + ")");
+            }
+          }
+        } else {
+          if (!GenericTypeHelper.isAssignableFrom(fieldType, converterResultType)) {
+            throw new AssertionError("Type of field (" + fieldType +
+                ") must be assignable from the converter " +
+                "return type (" + converterResultType + ")");
+          }
+        }
+
+        if (nameToFieldBuilder.put(annotation.name(), field) != null) {
+          throw new DuplicateOptionDeclarationException(
+              "Duplicate option name: --" + annotation.name());
+        }
+        if (!annotation.oldName().isEmpty()) {
+          if (nameToFieldBuilder.put(annotation.oldName(), field) != null) {
+            throw new DuplicateOptionDeclarationException(
+                "Old option name duplicates option name: --" + annotation.oldName());
+          }
+        }
+        if (annotation.abbrev() != '\0') {
+          if (abbrevToFieldBuilder.put(annotation.abbrev(), field) != null) {
+            throw new DuplicateOptionDeclarationException(
+                  "Duplicate option abbrev: -" + annotation.abbrev());
+          }
+        }
+
+        optionDefaultsBuilder.put(field, retrieveDefaultFromAnnotation(field));
+
+        convertersBuilder.put(field, findConverter(field));
+
+        allowMultipleBuilder.put(field, annotation.allowMultiple());
+      }
+    }
+
+    return new IsolatedOptionsData(
+        constructorBuilder,
+        nameToFieldBuilder,
+        abbrevToFieldBuilder,
+        allOptionsFieldsBuilder,
+        optionDefaultsBuilder,
+        convertersBuilder,
+        allowMultipleBuilder);
+  }
+}
diff --git a/java/com/google/devtools/common/options/Option.java b/java/com/google/devtools/common/options/Option.java
index 3b2ca30..249ee70 100644
--- a/java/com/google/devtools/common/options/Option.java
+++ b/java/com/google/devtools/common/options/Option.java
@@ -47,116 +47,147 @@ public @interface Option {
   String valueHelp() default "";
 
   /**
-   * The default value for the option. This method should only be invoked
-   * directly by the parser implementation. Any access to default values
-   * should go via the parser to allow for application specific defaults.
+   * The default value for the option. This method should only be invoked directly by the parser
+   * implementation. Any access to default values should go via the parser to allow for application
+   * specific defaults.
    *
-   * <p>There are two reasons this is a string.  Firstly, it ensures that
-   * explicitly specifying this option at its default value (as printed in the
-   * usage message) has the same behavior as not specifying the option at all;
-   * this would be very hard to achieve if the default value was an instance of
-   * type T, since we'd need to ensure that {@link #toString()} and {@link
-   * #converter} were dual to each other.  The second reason is more mundane
-   * but also more restrictive: annotation values must be compile-time
-   * constants.
+   * <p>There are two reasons this is a string. Firstly, it ensures that explicitly specifying this
+   * option at its default value (as printed in the usage message) has the same behavior as not
+   * specifying the option at all; this would be very hard to achieve if the default value was an
+   * instance of type T, since we'd need to ensure that {@link #toString()} and {@link #converter}
+   * were dual to each other. The second reason is more mundane but also more restrictive:
+   * annotation values must be compile-time constants.
    *
-   * <p>If an option's defaultValue() is the string "null", the option's
-   * converter will not be invoked to interpret it; a null reference will be
-   * used instead.  (It would be nice if defaultValue could simply return null,
-   * but bizarrely, the Java Language Specification does not consider null to
-   * be a compile-time constant.)  This special interpretation of the string
-   * "null" is only applicable when computing the default value; if specified
-   * on the command-line, this string will have its usual literal meaning.
+   * <p>If an option's defaultValue() is the string "null", the option's converter will not be
+   * invoked to interpret it; a null reference will be used instead. (It would be nice if
+   * defaultValue could simply return null, but bizarrely, the Java Language Specification does not
+   * consider null to be a compile-time constant.) This special interpretation of the string "null"
+   * is only applicable when computing the default value; if specified on the command-line, this
+   * string will have its usual literal meaning.
    *
-   * <p>The default value for flags that set allowMultiple is always the empty
-   * list and its default value is ignored.
+   * <p>The default value for flags that set allowMultiple is always the empty list and its default
+   * value is ignored.
    */
   String defaultValue();
 
   /**
    * A string describing the category of options that this belongs to. {@link
-   * OptionsParser#describeOptions} prints options of the same category grouped
-   * together.
+   * OptionsParser#describeOptions} prints options of the same category grouped together.
+   *
+   * <p>There are three special category values:
+   *
+   * <ul>
+   *   <li>{@code "undocumented"}: options which are useful for (some subset of) users, but not
+   *       meant to be publicly advertised. For example, experimental options which are only meant
+   *       to be used by specific testers or team members, but which should otherwise be treated
+   *       normally. These options will not be listed in the usage info displayed for the {@code
+   *       --help} option. They are otherwise normal - {@link
+   *       OptionsParser.UnparsedOptionValueDescription#isHidden()} returns {@code false} for them,
+   *       and they can be parsed normally from the command line or RC files.
+   *   <li>{@code "hidden"}: options which users should not pass or know about, but which are used
+   *       by the program (e.g., communication between a command-line client and a backend server).
+   *       Like {@code "undocumented"} options, these options will not be listed in the usage info
+   *       displayed for the {@code --help} option. However, in addition to this, calling {@link
+   *       OptionsParser.UnparsedOptionValueDescription#isHidden()} on these options will return
+   *       {@code true} - for example, this can be checked to strip out such secret options when
+   *       logging or otherwise reporting the command line to the user. This category does not
+   *       affect the option in any other way; it can still be parsed normally from the command line
+   *       or an RC file.
+   *   <li>{@code "internal"}: options which are purely for internal use within the JVM, and should
+   *       never be shown to the user, nor ever need to be parsed by the options parser. Like {@code
+   *       "hidden"} options, these options will not be listed in the usage info displayed for the
+   *       --help option, and are considered hidden by {@link
+   *       OptionsParser.UnparsedOptionValueDescription#isHidden()}. Unlike those, this type of
+   *       option cannot be parsed by any call to {@link OptionsParser#parse} - it will be treated
+   *       as if it was not defined.
+   * </ul>
    */
   String category() default "misc";
 
   /**
-   * The converter that we'll use to convert this option into an object or
-   * a simple type. The default is to use the builtin converters.
-   * Custom converters must implement the {@link Converter} interface.
+   * The converter that we'll use to convert the string representation of this option's value into
+   * an object or a simple type. The default is to use the builtin converters ({@link
+   * Converters#DEFAULT_CONVERTERS}). Custom converters must implement the {@link Converter}
+   * interface.
    */
   @SuppressWarnings({"unchecked", "rawtypes"})
   // Can't figure out how to coerce Converter.class into Class<? extends Converter<?>>
   Class<? extends Converter> converter() default Converter.class;
 
   /**
-   * A flag indicating whether the option type should be allowed to occur
-   * multiple times in a single option list.
+   * A flag indicating whether the option type should be allowed to occur multiple times in a single
+   * option list.
    *
-   * <p>If the command can occur multiple times, then the attribute value
-   * <em>must</em> be a list type {@code List<T>}, and the result type of the
-   * converter for this option must either match the parameter {@code T} or
-   * {@code List<T>}. In the latter case the individual lists are concatenated
-   * to form the full options value.
+   * <p>If the command can occur multiple times, then the attribute value <em>must</em> be a list
+   * type {@code List<T>}, and the result type of the converter for this option must either match
+   * the parameter {@code T} or {@code List<T>}. In the latter case the individual lists are
+   * concatenated to form the full options value.
    *
-   * <p>The {@link #defaultValue()} field of the annotation is ignored for repeatable
-   * flags and the default value will be the empty list.
+   * <p>The {@link #defaultValue()} field of the annotation is ignored for repeatable flags and the
+   * default value will be the empty list.
    */
   boolean allowMultiple() default false;
 
   /**
-   * If the option is actually an abbreviation for other options, this field will
-   * contain the strings to expand this option into. The original option is dropped
-   * and the replacement used in its stead. It is recommended that such an option be
-   * of type {@link Void}.
+   * If the option is actually an abbreviation for other options, this field will contain the
+   * strings to expand this option into. The original option is dropped and the replacement used in
+   * its stead. It is recommended that such an option be of type {@link Void}.
    *
-   * An expanded option overrides previously specified options of the same name,
-   * even if it is explicitly specified. This is the original behavior and can
-   * be surprising if the user is not aware of it, which has led to several
-   * requests to change this behavior. This was discussed in the blaze team and
-   * it was decided that it is not a strong enough case to change the behavior.
+   * <p>An expanded option overrides previously specified options of the same name, even if it is
+   * explicitly specified. This is the original behavior and can be surprising if the user is not
+   * aware of it, which has led to several requests to change this behavior. This was discussed in
+   * the blaze team and it was decided that it is not a strong enough case to change the behavior.
    */
   String[] expansion() default {};
 
   /**
-   * If the option requires that additional options be implicitly appended, this field
-   * will contain the additional options. Implicit dependencies are parsed at the end
-   * of each {@link OptionsParser#parse} invocation, and override options specified in
-   * the same call. However, they can be overridden by options specified in a later
-   * call or by options with a higher priority.
+   * A mechanism for specifying an expansion that is a function of the parser's {@link
+   * IsolatedOptionsData}. This can be used to create an option that expands to different strings
+   * depending on what other options the parser knows about.
+   *
+   * <p>If provided (i.e. not {@link ExpansionFunction}{@code .class}), the {@code expansion} field
+   * must not be set. The mechanism of expansion is as if the {@code expansion} field were set to
+   * whatever the return value of this function is.
+   */
+  Class<? extends ExpansionFunction> expansionFunction() default ExpansionFunction.class;
+
+  /**
+   * If the option requires that additional options be implicitly appended, this field will contain
+   * the additional options. Implicit dependencies are parsed at the end of each {@link
+   * OptionsParser#parse} invocation, and override options specified in the same call. However, they
+   * can be overridden by options specified in a later call or by options with a higher priority.
    *
    * @see OptionPriority
    */
   String[] implicitRequirements() default {};
 
   /**
-   * If this field is a non-empty string, the option is deprecated, and a
-   * deprecation warning is added to the list of warnings when such an option
-   * is used.
+   * If this field is a non-empty string, the option is deprecated, and a deprecation warning is
+   * added to the list of warnings when such an option is used.
    */
   String deprecationWarning() default "";
 
   /**
-   * The old name for this option. If an option has a name "foo" and an old name "bar",
-   * --foo=baz and --bar=baz will be equivalent. If the old name is used, a warning will be printed
-   * indicating that the old name is deprecated and the new name should be used.
+   * The old name for this option. If an option has a name "foo" and an old name "bar", --foo=baz
+   * and --bar=baz will be equivalent. If the old name is used, a warning will be printed indicating
+   * that the old name is deprecated and the new name should be used.
    */
   String oldName() default "";
 
   /**
-   * Indicates that this option is a wrapper for other options, and will be unwrapped
-   * when parsed. For example, if foo is a wrapper option, then "--foo=--bar=baz"
-   * will be parsed as the flag "--bar=baz" (rather than --foo taking the value
-   * "--bar=baz"). A wrapper option should have the type {@link Void} (if it is something other
-   * than Void, the parser will not assign a value to it). The
-   * {@link Option#implicitRequirements()}, {@link Option#expansion()}, {@link Option#converter()}
-   * attributes will not be processed. Wrapper options are implicitly repeatable (i.e., as though
-   * {@link Option#allowMultiple()} is true regardless of its value in the annotation).
+   * Indicates that this option is a wrapper for other options, and will be unwrapped when parsed.
+   * For example, if foo is a wrapper option, then "--foo=--bar=baz" will be parsed as the flag
+   * "--bar=baz" (rather than --foo taking the value "--bar=baz"). A wrapper option should have the
+   * type {@link Void} (if it is something other than Void, the parser will not assign a value to
+   * it). The {@link Option#implicitRequirements()}, {@link Option#expansion()}, {@link
+   * Option#converter()} attributes will not be processed. Wrapper options are implicitly repeatable
+   * (i.e., as though {@link Option#allowMultiple()} is true regardless of its value in the
+   * annotation).
    *
    * <p>Wrapper options are provided only for transitioning flags which appear as values to other
    * flags, to top-level flags. Wrapper options should not be used in Invocation Policy, as
-   * expansion flags to other flags, or as implicit requirements to other flags. Use the inner
-   * flags instead.  
+   * expansion flags to other flags, or as implicit requirements to other flags. Use the inner flags
+   * instead.
    */
   boolean wrapperOption() default false;
 }
diff --git a/java/com/google/devtools/common/options/OptionsData.java b/java/com/google/devtools/common/options/OptionsData.java
index ac23d63..e71321c 100644
--- a/java/com/google/devtools/common/options/OptionsData.java
+++ b/java/com/google/devtools/common/options/OptionsData.java
@@ -1,4 +1,4 @@
-// Copyright 2014 The Bazel Authors. All rights reserved.
+// Copyright 2017 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,280 +14,89 @@
 
 package com.google.devtools.common.options;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
-import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
-
 import javax.annotation.concurrent.Immutable;
 
 /**
- * An immutable selection of options data corresponding to a set of options
- * classes. The data is collected using reflection, which can be expensive.
- * Therefore this class can be used internally to cache the results.
+ * This extends IsolatedOptionsData with information that can only be determined once all the {@link
+ * OptionsBase} subclasses for a parser are known. In particular, this includes expansion
+ * information.
  */
 @Immutable
-final class OptionsData extends OpaqueOptionsData {
-
-  /**
-   * These are the options-declaring classes which are annotated with
-   * {@link Option} annotations.
-   */
-  private final Map<Class<? extends OptionsBase>, Constructor<?>> optionsClasses;
-
-  /** Maps option name to Option-annotated Field. */
-  private final Map<String, Field> nameToField;
-
-  /** Maps option abbreviation to Option-annotated Field. */
-  private final Map<Character, Field> abbrevToField;
+final class OptionsData extends IsolatedOptionsData {
 
   /**
-   * For each options class, contains a list of all Option-annotated fields in
-   * that class.
+   * Mapping from each Option-annotated field with a {@code String[]} expansion to that expansion.
    */
-  private final Map<Class<? extends OptionsBase>, List<Field>> allOptionsFields;
+  // TODO(brandjon): This is technically not necessarily immutable due to String[], and should use
+  // ImmutableList. Either fix this or remove @Immutable.
+  private final ImmutableMap<Field, String[]> evaluatedExpansions;
+
+  /** Construct {@link OptionsData} by extending an {@link IsolatedOptionsData} with new info. */
+  private OptionsData(IsolatedOptionsData base, Map<Field, String[]> evaluatedExpansions) {
+    super(base);
+    this.evaluatedExpansions = ImmutableMap.copyOf(evaluatedExpansions);
+  }
 
-  /**
-   * Mapping from each Option-annotated field to the default value for that
-   * field.
-   */
-  private final Map<Field, Object> optionDefaults;
+  private static final String[] EMPTY_EXPANSION = new String[] {};
 
   /**
-   * Mapping from each Option-annotated field to the proper converter.
-   *
-   * @see OptionsParserImpl#findConverter
+   * Returns the expansion of an options field, regardless of whether it was defined using {@link
+   * Option#expansion} or {@link Option#expansionFunction}. If the field is not an expansion option,
+   * returns an empty array.
    */
-  private final Map<Field, Converter<?>> converters;
+  public String[] getEvaluatedExpansion(Field field) {
+    String[] result = evaluatedExpansions.get(field);
+    return result != null ? result : EMPTY_EXPANSION;
+  }
 
   /**
-   * Mapping from each Option-annotated field to a boolean for whether that field allows multiple
-   * values.
+   * Constructs an {@link OptionsData} object for a parser that knows about the given {@link
+   * OptionsBase} classes. In addition to the work done to construct the {@link
+   * IsolatedOptionsData}, this also computes expansion information.
    */
-  private final Map<Field, Boolean> allowMultiple;
-
-  private OptionsData(Map<Class<? extends OptionsBase>, Constructor<?>> optionsClasses,
-                      Map<String, Field> nameToField,
-                      Map<Character, Field> abbrevToField,
-                      Map<Class<? extends OptionsBase>, List<Field>> allOptionsFields,
-                      Map<Field, Object> optionDefaults,
-                      Map<Field, Converter<?>> converters,
-                      Map<Field, Boolean> allowMultiple) {
-    this.optionsClasses = ImmutableMap.copyOf(optionsClasses);
-    this.allOptionsFields = ImmutableMap.copyOf(allOptionsFields);
-    this.nameToField = ImmutableMap.copyOf(nameToField);
-    this.abbrevToField = ImmutableMap.copyOf(abbrevToField);
-    // Can't use an ImmutableMap here because of null values.
-    this.optionDefaults = Collections.unmodifiableMap(optionDefaults);
-    this.converters = ImmutableMap.copyOf(converters);
-    this.allowMultiple = ImmutableMap.copyOf(allowMultiple);
-  }
-
-  public Collection<Class<? extends OptionsBase>> getOptionsClasses() {
-    return optionsClasses.keySet();
-  }
-
-  @SuppressWarnings("unchecked") // The construction ensures that the case is always valid.
-  public <T extends OptionsBase> Constructor<T> getConstructor(Class<T> clazz) {
-    return (Constructor<T>) optionsClasses.get(clazz);
-  }
-
-  public Field getFieldFromName(String name) {
-    return nameToField.get(name);
-  }
-
-  public Iterable<Map.Entry<String, Field>> getAllNamedFields() {
-    return nameToField.entrySet();
-  }
-
-  public Field getFieldForAbbrev(char abbrev) {
-    return abbrevToField.get(abbrev);
-  }
-
-  public List<Field> getFieldsForClass(Class<? extends OptionsBase> optionsClass) {
-    return allOptionsFields.get(optionsClass);
-  }
-
-  public Object getDefaultValue(Field field) {
-    return optionDefaults.get(field);
-  }
-
-  public Converter<?> getConverter(Field field) {
-    return converters.get(field);
-  }
-
-  public boolean getAllowMultiple(Field field) {
-    return allowMultiple.get(field);
-  }
-
-  private static List<Field> getAllAnnotatedFields(Class<? extends OptionsBase> optionsClass) {
-    List<Field> allFields = Lists.newArrayList();
-    for (Field field : optionsClass.getFields()) {
-      if (field.isAnnotationPresent(Option.class)) {
-        allFields.add(field);
-      }
-    }
-    if (allFields.isEmpty()) {
-      throw new IllegalStateException(optionsClass + " has no public @Option-annotated fields");
-    }
-    return ImmutableList.copyOf(allFields);
-  }
-
-  private static Object retrieveDefaultFromAnnotation(Field optionField) {
-    Converter<?> converter = OptionsParserImpl.findConverter(optionField);
-    String defaultValueAsString = OptionsParserImpl.getDefaultOptionString(optionField);
-    // Special case for "null"
-    if (OptionsParserImpl.isSpecialNullDefault(defaultValueAsString, optionField)) {
-      return null;
-    }
-    boolean allowsMultiple = optionField.getAnnotation(Option.class).allowMultiple();
-    // If the option allows multiple values then we intentionally return the empty list as
-    // the default value of this option since it is not always the case that an option
-    // that allows multiple values will have a converter that returns a list value.
-    if (allowsMultiple) {
-      return Collections.emptyList();
-    }
-    // Otherwise try to convert the default value using the converter
-    Object convertedValue;
-    try {
-      convertedValue = converter.convert(defaultValueAsString);
-    } catch (OptionsParsingException e) {
-      throw new IllegalStateException("OptionsParsingException while "
-          + "retrieving default for " + optionField.getName() + ": "
-          + e.getMessage());
-    }
-    return convertedValue;
-  }
-
-  static OptionsData of(Collection<Class<? extends OptionsBase>> classes) {
-    Map<Class<? extends OptionsBase>, Constructor<?>> constructorBuilder = Maps.newHashMap();
-    Map<Class<? extends OptionsBase>, List<Field>> allOptionsFieldsBuilder = Maps.newHashMap();
-    Map<String, Field> nameToFieldBuilder = Maps.newHashMap();
-    Map<Character, Field> abbrevToFieldBuilder = Maps.newHashMap();
-    Map<Field, Object> optionDefaultsBuilder = Maps.newHashMap();
-    Map<Field, Converter<?>> convertersBuilder = Maps.newHashMap();
-    Map<Field, Boolean> allowMultipleBuilder = Maps.newHashMap();
-
-    // Read all Option annotations:
-    for (Class<? extends OptionsBase> parsedOptionsClass : classes) {
-      try {
-        Constructor<? extends OptionsBase> constructor =
-            parsedOptionsClass.getConstructor(new Class[0]);
-        constructorBuilder.put(parsedOptionsClass, constructor);
-      } catch (NoSuchMethodException e) {
-        throw new IllegalArgumentException(parsedOptionsClass
-            + " lacks an accessible default constructor");
-      }
-      List<Field> fields = getAllAnnotatedFields(parsedOptionsClass);
-      allOptionsFieldsBuilder.put(parsedOptionsClass, fields);
-
-      for (Field field : fields) {
-        Option annotation = field.getAnnotation(Option.class);
-
-        // Check that the field type is a List, and that the converter
-        // type matches the element type of the list.
-        Type fieldType = field.getGenericType();
-        if (annotation.allowMultiple()) {
-          if (!(fieldType instanceof ParameterizedType)) {
-            throw new AssertionError("Type of multiple occurrence option must be a List<...>");
-          }
-          ParameterizedType pfieldType = (ParameterizedType) fieldType;
-          if (pfieldType.getRawType() != List.class) {
-            // Throw an assertion, because this indicates an undetected type
-            // error in the code.
-            throw new AssertionError("Type of multiple occurrence option must be a List<...>");
-          }
-          fieldType = pfieldType.getActualTypeArguments()[0];
-        }
-
-        // Get the converter return type.
-        @SuppressWarnings("rawtypes")
-        Class<? extends Converter> converter = annotation.converter();
-        if (converter == Converter.class) {
-          Converter<?> actualConverter = OptionsParserImpl.DEFAULT_CONVERTERS.get(fieldType);
-          if (actualConverter == null) {
-            throw new AssertionError("Cannot find converter for field of type "
-                + field.getType() + " named " + field.getName()
-                + " in class " + field.getDeclaringClass().getName());
-          }
-          converter = actualConverter.getClass();
-        }
-        if (Modifier.isAbstract(converter.getModifiers())) {
-          throw new AssertionError("The converter type (" + converter
-              + ") must be a concrete type");
-        }
-        Type converterResultType;
-        try {
-          Method convertMethod = converter.getMethod("convert", String.class);
-          converterResultType = GenericTypeHelper.getActualReturnType(converter, convertMethod);
-        } catch (NoSuchMethodException e) {
-          throw new AssertionError("A known converter object doesn't implement the convert"
-              + " method");
-        }
-
-        if (annotation.allowMultiple()) {
-          if (GenericTypeHelper.getRawType(converterResultType) == List.class) {
-            Type elementType =
-                ((ParameterizedType) converterResultType).getActualTypeArguments()[0];
-            if (!GenericTypeHelper.isAssignableFrom(fieldType, elementType)) {
-              throw new AssertionError("If the converter return type of a multiple occurance " +
-                  "option is a list, then the type of list elements (" + fieldType + ") must be " +
-                  "assignable from the converter list element type (" + elementType + ")");
-            }
-          } else {
-            if (!GenericTypeHelper.isAssignableFrom(fieldType, converterResultType)) {
-              throw new AssertionError("Type of list elements (" + fieldType +
-                  ") for multiple occurrence option must be assignable from the converter " +
-                  "return type (" + converterResultType + ")");
-            }
-          }
-        } else {
-          if (!GenericTypeHelper.isAssignableFrom(fieldType, converterResultType)) {
-            throw new AssertionError("Type of field (" + fieldType +
-                ") must be assignable from the converter " +
-                "return type (" + converterResultType + ")");
-          }
-        }
-
-        if (annotation.name() == null) {
+  public static OptionsData from(Collection<Class<? extends OptionsBase>> classes) {
+    IsolatedOptionsData isolatedData = IsolatedOptionsData.from(classes);
+
+    // All that's left is to compute expansions.
+    Map<Field, String[]> evaluatedExpansionsBuilder = Maps.newHashMap();
+    for (Map.Entry<String, Field> entry : isolatedData.getAllNamedFields()) {
+      Field field = entry.getValue();
+      Option annotation = field.getAnnotation(Option.class);
+      // Determine either the hard-coded expansion, or the ExpansionFunction class.
+      String[] constExpansion = annotation.expansion();
+      Class<? extends ExpansionFunction> expansionFunctionClass = annotation.expansionFunction();
+      if (constExpansion.length > 0 && usesExpansionFunction(annotation)) {
+        throw new AssertionError(
+            "Cannot set both expansion and expansionFunction for option --" + annotation.name());
+      } else if (constExpansion.length > 0) {
+        evaluatedExpansionsBuilder.put(field, constExpansion);
+      } else if (usesExpansionFunction(annotation)) {
+        if (Modifier.isAbstract(expansionFunctionClass.getModifiers())) {
           throw new AssertionError(
-              "Option cannot have a null name");
+              "The expansionFunction type " + expansionFunctionClass + " must be a concrete type");
         }
-        if (nameToFieldBuilder.put(annotation.name(), field) != null) {
-          throw new DuplicateOptionDeclarationException(
-              "Duplicate option name: --" + annotation.name());
-        }
-        if (!annotation.oldName().isEmpty()) {
-          if (nameToFieldBuilder.put(annotation.oldName(), field) != null) {
-            throw new DuplicateOptionDeclarationException(
-                "Old option name duplicates option name: --" + annotation.oldName());
-          }
-        }
-        if (annotation.abbrev() != '\0') {
-          if (abbrevToFieldBuilder.put(annotation.abbrev(), field) != null) {
-            throw new DuplicateOptionDeclarationException(
-                  "Duplicate option abbrev: -" + annotation.abbrev());
-          }
+        // Evaluate the ExpansionFunction.
+        ExpansionFunction instance;
+        try {
+          Constructor<?> constructor = expansionFunctionClass.getConstructor();
+          instance = (ExpansionFunction) constructor.newInstance();
+        } catch (Exception e) {
+          // This indicates an error in the ExpansionFunction, and should be discovered the first
+          // time it is used.
+          throw new AssertionError(e);
         }
-
-        optionDefaultsBuilder.put(field, retrieveDefaultFromAnnotation(field));
-
-        convertersBuilder.put(field, OptionsParserImpl.findConverter(field));
-
-        allowMultipleBuilder.put(field, annotation.allowMultiple());
+        String[] expansion = instance.getExpansion(isolatedData);
+        evaluatedExpansionsBuilder.put(field, expansion);
       }
     }
-    return new OptionsData(constructorBuilder, nameToFieldBuilder, abbrevToFieldBuilder,
-        allOptionsFieldsBuilder, optionDefaultsBuilder, convertersBuilder, allowMultipleBuilder);
+
+    return new OptionsData(isolatedData, evaluatedExpansionsBuilder);
   }
 }
diff --git a/java/com/google/devtools/common/options/OptionsParser.java b/java/com/google/devtools/common/options/OptionsParser.java
index 946d73b..1c4b278 100644
--- a/java/com/google/devtools/common/options/OptionsParser.java
+++ b/java/com/google/devtools/common/options/OptionsParser.java
@@ -88,7 +88,7 @@ public class OptionsParser implements OptionsProvider {
       ImmutableList<Class<? extends OptionsBase>> optionsClasses) {
     OptionsData result = optionsData.get(optionsClasses);
     if (result == null) {
-      result = OptionsData.of(optionsClasses);
+      result = OptionsData.from(optionsClasses);
       optionsData.put(optionsClasses, result);
     }
     return result;
@@ -140,10 +140,6 @@ public class OptionsParser implements OptionsProvider {
   private final List<String> residue = new ArrayList<String>();
   private boolean allowResidue = true;
 
-  OptionsParser(Collection<Class<? extends OptionsBase>> optionsClasses) {
-    this(OptionsData.of(optionsClasses));
-  }
-
   OptionsParser(OptionsData optionsData) {
     impl = new OptionsParserImpl(optionsData);
   }
@@ -395,12 +391,14 @@ public class OptionsParser implements OptionsProvider {
     }
 
     public boolean isHidden() {
-      return documentationLevel() == DocumentationLevel.HIDDEN;
+      return documentationLevel() == DocumentationLevel.HIDDEN
+          || documentationLevel() == DocumentationLevel.INTERNAL;
     }
 
     boolean isExpansion() {
       Option option = field.getAnnotation(Option.class);
-      return option.expansion().length > 0;
+      return (option.expansion().length > 0
+          || OptionsData.usesExpansionFunction(option));
     }
 
     boolean isImplicitRequirement() {
@@ -455,28 +453,27 @@ public class OptionsParser implements OptionsProvider {
    *
    * <p>We use 'hidden' so that options that form the protocol between the
    * client and the server are not logged.
+   *
+   * <p>Options which are 'internal' are not recognized by the parser at all.
    */
   enum DocumentationLevel {
-    DOCUMENTED, UNDOCUMENTED, HIDDEN
+    DOCUMENTED, UNDOCUMENTED, HIDDEN, INTERNAL
   }
 
   /**
-   * Returns a description of all the options this parser can digest.
-   * In addition to {@link Option} annotations, this method also
-   * interprets {@link OptionsUsage} annotations which give an intuitive short
-   * description for the options.
+   * Returns a description of all the options this parser can digest. In addition to {@link Option}
+   * annotations, this method also interprets {@link OptionsUsage} annotations which give an
+   * intuitive short description for the options. Options of the same category (see {@link
+   * Option#category}) will be grouped together.
    *
-   * @param categoryDescriptions a mapping from category names to category
-   *   descriptions.  Options of the same category (see {@link
-   *   Option#category}) will be grouped together, preceded by the description
-   *   of the category.
-   * @param helpVerbosity if {@code long}, the options will be described
-   *   verbosely, including their types, defaults and descriptions.  If {@code
-   *   medium}, the descriptions are omitted, and if {@code short}, the options
-   *   are just enumerated.
+   * @param categoryDescriptions a mapping from category names to category descriptions.
+   *     Descriptions are optional; if omitted, a string based on the category name will be used.
+   * @param helpVerbosity if {@code long}, the options will be described verbosely, including their
+   *     types, defaults and descriptions. If {@code medium}, the descriptions are omitted, and if
+   *     {@code short}, the options are just enumerated.
    */
-  public String describeOptions(Map<String, String> categoryDescriptions,
-                                HelpVerbosity helpVerbosity) {
+  public String describeOptions(
+      Map<String, String> categoryDescriptions, HelpVerbosity helpVerbosity) {
     StringBuilder desc = new StringBuilder();
     if (!impl.getOptionsClasses().isEmpty()) {
       List<Field> allFields = Lists.newArrayList();
@@ -500,7 +497,7 @@ public class OptionsParser implements OptionsProvider {
         }
 
         if (documentationLevel(prevCategory) == DocumentationLevel.DOCUMENTED) {
-          OptionsUsage.getUsage(optionField, desc, helpVerbosity);
+          OptionsUsage.getUsage(optionField, desc, helpVerbosity, impl.getOptionsData());
         }
       }
     }
@@ -545,7 +542,7 @@ public class OptionsParser implements OptionsProvider {
         }
 
         if (level == DocumentationLevel.DOCUMENTED) {
-          OptionsUsage.getUsageHtml(optionField, desc, escaper);
+          OptionsUsage.getUsageHtml(optionField, desc, escaper, impl.getOptionsData());
         }
       }
       desc.append("</dl>\n");
@@ -614,6 +611,8 @@ public class OptionsParser implements OptionsProvider {
       return DocumentationLevel.UNDOCUMENTED;
     } else if ("hidden".equals(category)) {
       return DocumentationLevel.HIDDEN;
+    } else if ("internal".equals(category)) {
+      return DocumentationLevel.INTERNAL;
     } else {
       return DocumentationLevel.DOCUMENTED;
     }
diff --git a/java/com/google/devtools/common/options/OptionsParserImpl.java b/java/com/google/devtools/common/options/OptionsParserImpl.java
index c15f927..5c6498a 100644
--- a/java/com/google/devtools/common/options/OptionsParserImpl.java
+++ b/java/com/google/devtools/common/options/OptionsParserImpl.java
@@ -32,8 +32,6 @@ import com.google.devtools.common.options.OptionsParser.OptionValueDescription;
 import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -50,100 +48,6 @@ import java.util.Map;
  */
 class OptionsParserImpl {
 
-  /**
-   * A bunch of default converters in case the user doesn't specify a
-   * different one in the field annotation.
-   */
-  static final Map<Class<?>, Converter<?>> DEFAULT_CONVERTERS = Maps.newHashMap();
-
-  static {
-    DEFAULT_CONVERTERS.put(String.class, new Converter<String>() {
-      @Override
-      public String convert(String input) {
-        return input;
-      }
-      @Override
-      public String getTypeDescription() {
-        return "a string";
-      }});
-    DEFAULT_CONVERTERS.put(int.class, new Converter<Integer>() {
-      @Override
-      public Integer convert(String input) throws OptionsParsingException {
-        try {
-          return Integer.decode(input);
-        } catch (NumberFormatException e) {
-          throw new OptionsParsingException("'" + input + "' is not an int");
-        }
-      }
-      @Override
-      public String getTypeDescription() {
-        return "an integer";
-      }});
-    DEFAULT_CONVERTERS.put(double.class, new Converter<Double>() {
-      @Override
-      public Double convert(String input) throws OptionsParsingException {
-        try {
-          return Double.parseDouble(input);
-        } catch (NumberFormatException e) {
-          throw new OptionsParsingException("'" + input + "' is not a double");
-        }
-      }
-      @Override
-      public String getTypeDescription() {
-        return "a double";
-      }});
-    DEFAULT_CONVERTERS.put(boolean.class, new Converters.BooleanConverter());
-    DEFAULT_CONVERTERS.put(TriState.class, new Converter<TriState>() {
-      @Override
-      public TriState convert(String input) throws OptionsParsingException {
-        if (input == null) {
-          return TriState.AUTO;
-        }
-        input = input.toLowerCase();
-        if (input.equals("auto")) {
-          return TriState.AUTO;
-        }
-        if (input.equals("true") || input.equals("1") || input.equals("yes") ||
-            input.equals("t") || input.equals("y")) {
-          return TriState.YES;
-        }
-        if (input.equals("false") || input.equals("0") || input.equals("no") ||
-            input.equals("f") || input.equals("n")) {
-          return TriState.NO;
-        }
-        throw new OptionsParsingException("'" + input + "' is not a boolean");
-      }
-      @Override
-      public String getTypeDescription() {
-        return "a tri-state (auto, yes, no)";
-      }});
-    DEFAULT_CONVERTERS.put(Void.class, new Converter<Void>() {
-      @Override
-      public Void convert(String input) throws OptionsParsingException {
-        if (input == null) {
-          return null;  // expected input, return is unused so null is fine.
-        }
-        throw new OptionsParsingException("'" + input + "' unexpected");
-      }
-      @Override
-      public String getTypeDescription() {
-        return "";
-      }});
-    DEFAULT_CONVERTERS.put(long.class, new Converter<Long>() {
-      @Override
-      public Long convert(String input) throws OptionsParsingException {
-        try {
-          return Long.decode(input);
-        } catch (NumberFormatException e) {
-          throw new OptionsParsingException("'" + input + "' is not a long");
-        }
-      }
-      @Override
-      public String getTypeDescription() {
-        return "a long integer";
-      }});
-  }
-
   private final OptionsData optionsData;
 
   /**
@@ -196,6 +100,10 @@ class OptionsParserImpl {
     this.optionsData = optionsData;
   }
 
+  OptionsData getOptionsData() {
+    return optionsData;
+  }
+
   /**
    * Indicates whether or not the parser will allow long options with a
    * single-dash, instead of the usual double-dash, too, eg. -example instead of just --example.
@@ -440,7 +348,8 @@ class OptionsParserImpl {
 
     // Recurse to remove any implicit or expansion flags that this flag may have added when
     // originally parsed.
-    for (String[] args : new String[][] {option.implicitRequirements(), option.expansion()}) {
+    String[] expansion = optionsData.getEvaluatedExpansion(field);
+    for (String[] args : new String[][] {option.implicitRequirements(), expansion}) {
       Iterator<String> argsIterator = Iterators.forArray(args);
       while (argsIterator.hasNext()) {
         String arg = argsIterator.next();
@@ -578,7 +487,8 @@ class OptionsParserImpl {
       }
 
       // Handle expansion options.
-      if (option.expansion().length > 0) {
+      String[] expansion = optionsData.getEvaluatedExpansion(field);
+      if (expansion.length > 0) {
         Function<Object, String> expansionSourceFunction =
             Functions.constant(
                 "expanded from option --"
@@ -587,7 +497,7 @@ class OptionsParserImpl {
                     + sourceFunction.apply(originalName));
         maybeAddDeprecationWarning(field);
         List<String> unparsed = parse(priority, expansionSourceFunction, null, originalName,
-            ImmutableList.copyOf(option.expansion()));
+            ImmutableList.copyOf(expansion));
         if (!unparsed.isEmpty()) {
           // Throw an assertion, because this indicates an error in the code that specified the
           // expansion for the current option.
@@ -701,7 +611,7 @@ class OptionsParserImpl {
         booleanValue = false;
         if (field != null) {
           // TODO(bazel-team): Add tests for these cases.
-          if (!OptionsParserImpl.isBooleanField(field)) {
+          if (!OptionsData.isBooleanField(field)) {
             throw new OptionsParsingException(
                 "Illegal use of 'no' prefix on non-boolean option: " + arg, arg);
           }
@@ -717,15 +627,18 @@ class OptionsParserImpl {
       throw new OptionsParsingException("Invalid options syntax: " + arg, arg);
     }
 
-    if (field == null) {
+    Option option = field == null ? null : field.getAnnotation(Option.class);
+
+    if (option == null
+        || OptionsParser.documentationLevel(option.category())
+            == OptionsParser.DocumentationLevel.INTERNAL) {
+      // This also covers internal options, which are treated as if they did not exist.
       throw new OptionsParsingException("Unrecognized option: " + arg, arg);
     }
 
-    Option option = field.getAnnotation(Option.class);
-
     if (value == null) {
       // Special-case boolean to supply value based on presence of "no" prefix.
-      if (OptionsParserImpl.isBooleanField(field)) {
+      if (OptionsData.isBooleanField(field)) {
         value = booleanValue ? "1" : "0";
       } else if (field.getType().equals(Void.class) && !option.wrapperOption()) {
         // This is expected, Void type options have no args (unless they're wrapper options).
@@ -782,46 +695,7 @@ class OptionsParserImpl {
     return annotation.defaultValue();
   }
 
-  static boolean isBooleanField(Field field) {
-    return field.getType().equals(boolean.class)
-        || field.getType().equals(TriState.class)
-        || findConverter(field) instanceof BoolOrEnumConverter;
-  }
-
-  static boolean isVoidField(Field field) {
-    return field.getType().equals(Void.class);
-  }
-
   static boolean isSpecialNullDefault(String defaultValueString, Field optionField) {
     return defaultValueString.equals("null") && !optionField.getType().isPrimitive();
   }
-
-  static Converter<?> findConverter(Field optionField) {
-    Option annotation = optionField.getAnnotation(Option.class);
-    if (annotation.converter() == Converter.class) {
-      Type type;
-      if (annotation.allowMultiple()) {
-        // The OptionParserImpl already checked that the type is List<T> for some T;
-        // here we extract the type T.
-        type = ((ParameterizedType) optionField.getGenericType()).getActualTypeArguments()[0];
-      } else {
-        type = optionField.getType();
-      }
-      Converter<?> converter = DEFAULT_CONVERTERS.get(type);
-      if (converter == null) {
-        throw new AssertionError("No converter found for "
-            + type + "; possible fix: add "
-            + "converter=... to @Option annotation for "
-            + optionField.getName());
-      }
-      return converter;
-    }
-    try {
-      Class<?> converter = annotation.converter();
-      Constructor<?> constructor = converter.getConstructor(new Class<?>[0]);
-      return (Converter<?>) constructor.newInstance(new Object[0]);
-    } catch (Exception e) {
-      throw new AssertionError(e);
-    }
-  }
 }
diff --git a/java/com/google/devtools/common/options/OptionsUsage.java b/java/com/google/devtools/common/options/OptionsUsage.java
index b8c19df..aa48cb7 100644
--- a/java/com/google/devtools/common/options/OptionsUsage.java
+++ b/java/com/google/devtools/common/options/OptionsUsage.java
@@ -13,19 +13,17 @@
 // limitations under the License.
 package com.google.devtools.common.options;
 
-import static com.google.devtools.common.options.OptionsParserImpl.findConverter;
-
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.escape.Escaper;
-
 import java.lang.reflect.Field;
 import java.text.BreakIterator;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import javax.annotation.Nullable;
 
 /**
  * A renderer for usage messages. For now this is very simple.
@@ -36,15 +34,17 @@ class OptionsUsage {
   private static final Joiner COMMA_JOINER = Joiner.on(",");
 
   /**
-   * Given an options class, render the usage string into the usage,
-   * which is passed in as an argument.
+   * Given an options class, render the usage string into the usage, which is passed in as an
+   * argument. This will not include information about expansions for options using expansion
+   * functions (it would be unsafe to report this as we cannot know what options from other {@link
+   * OptionsBase} subclasses they depend on until a complete parser is constructed).
    */
   static void getUsage(Class<? extends OptionsBase> optionsClass, StringBuilder usage) {
     List<Field> optionFields =
         Lists.newArrayList(OptionsParser.getAllAnnotatedFields(optionsClass));
     Collections.sort(optionFields, BY_NAME);
     for (Field optionField : optionFields) {
-      getUsage(optionField, usage, OptionsParser.HelpVerbosity.LONG);
+      getUsage(optionField, usage, OptionsParser.HelpVerbosity.LONG, null);
     }
   }
 
@@ -79,10 +79,35 @@ class OptionsUsage {
   }
 
   /**
-   * Append the usage message for a single option-field message to 'usage'.
+   * Returns the expansion for an option, to the extent known. Precisely, if an {@link OptionsData}
+   * object is supplied, the expansion is read from that. Otherwise, the annotation is inspected: If
+   * the annotation uses {@link Option#expansion} it is returned, and if it uses {@link
+   * Option#expansionFunction} null is returned, indicating a lack of definite information. In all
+   * cases, when the option is not an expansion option, an empty array is returned.
+   */
+  private static @Nullable String[] getExpansionIfKnown(
+      Field optionField, Option annotation, @Nullable OptionsData optionsData) {
+    if (optionsData != null) {
+      return optionsData.getEvaluatedExpansion(optionField);
+    } else {
+      if (OptionsData.usesExpansionFunction(annotation)) {
+        return null;
+      } else {
+        // Empty array if it's not an expansion option.
+        return annotation.expansion();
+      }
+    }
+  }
+
+  /**
+   * Appends the usage message for a single option-field message to 'usage'. If {@code optionsData}
+   * is not supplied, options that use expansion functions won't be fully described.
    */
-  static void getUsage(Field optionField, StringBuilder usage,
-                       OptionsParser.HelpVerbosity helpVerbosity) {
+  static void getUsage(
+      Field optionField,
+      StringBuilder usage,
+      OptionsParser.HelpVerbosity helpVerbosity,
+      @Nullable OptionsData optionsData) {
     String flagName = getFlagName(optionField);
     String typeDescription = getTypeDescription(optionField);
     Option annotation = optionField.getAnnotation(Option.class);
@@ -117,9 +142,12 @@ class OptionsUsage {
       usage.append(paragraphFill(annotation.help(), 4, 80)); // (indent, width)
       usage.append('\n');
     }
-    if (annotation.expansion().length > 0) {
+    String[] expansion = getExpansionIfKnown(optionField, annotation, optionsData);
+    if (expansion == null) {
+      usage.append("    Expands to unknown options.\n");
+    } else if (expansion.length > 0) {
       StringBuilder expandsMsg = new StringBuilder("Expands to: ");
-      for (String exp : annotation.expansion()) {
+      for (String exp : expansion) {
         expandsMsg.append(exp).append(" ");
       }
       usage.append(paragraphFill(expandsMsg.toString(), 4, 80)); // (indent, width)
@@ -128,9 +156,11 @@ class OptionsUsage {
   }
 
   /**
-   * Append the usage message for a single option-field message to 'usage'.
+   * Append the usage message for a single option-field message to 'usage'. If {@code optionsData}
+   * is not supplied, options that use expansion functions won't be fully described.
    */
-  static void getUsageHtml(Field optionField, StringBuilder usage, Escaper escaper) {
+  static void getUsageHtml(
+      Field optionField, StringBuilder usage, Escaper escaper, @Nullable OptionsData optionsData) {
     String plainFlagName = optionField.getAnnotation(Option.class).name();
     String flagName = getFlagName(optionField);
     String valueDescription = optionField.getAnnotation(Option.class).valueHelp();
@@ -138,8 +168,7 @@ class OptionsUsage {
     Option annotation = optionField.getAnnotation(Option.class);
     usage.append("<dt><code><a name=\"flag--").append(plainFlagName).append("\"></a>--");
     usage.append(flagName);
-    if (OptionsParserImpl.isBooleanField(optionField)
-        || OptionsParserImpl.isVoidField(optionField)) {
+    if (OptionsData.isBooleanField(optionField) || OptionsData.isVoidField(optionField)) {
       // Nothing for boolean, tristate, boolean_or_enum, or void options.
     } else if (!valueDescription.isEmpty()) {
       usage.append("=").append(escaper.escape(valueDescription));
@@ -157,7 +186,7 @@ class OptionsUsage {
     } else {
       // Don't call the annotation directly (we must allow overrides to certain defaults).
       String defaultValueString = OptionsParserImpl.getDefaultOptionString(optionField);
-      if (OptionsParserImpl.isVoidField(optionField)) {
+      if (OptionsData.isVoidField(optionField)) {
         // Void options don't have a default.
       } else if (OptionsParserImpl.isSpecialNullDefault(defaultValueString, optionField)) {
         usage.append(" default: see description");
@@ -171,10 +200,13 @@ class OptionsUsage {
       usage.append(paragraphFill(escaper.escape(annotation.help()), 0, 80)); // (indent, width)
       usage.append('\n');
     }
-    if (annotation.expansion().length > 0) {
+    String[] expansion = getExpansionIfKnown(optionField, annotation, optionsData);
+    if (expansion == null) {
+      usage.append("    Expands to unknown options.<br>\n");
+    } else if (expansion.length > 0) {
       usage.append("<br/>\n");
       StringBuilder expandsMsg = new StringBuilder("Expands to:<br/>\n");
-      for (String exp : annotation.expansion()) {
+      for (String exp : expansion) {
         // TODO(ulfjack): Can we link to the expanded flags here?
         expandsMsg
             .append("&nbsp;&nbsp;<code>")
@@ -259,12 +291,12 @@ class OptionsUsage {
   };
 
   private static String getTypeDescription(Field optionsField) {
-    return findConverter(optionsField).getTypeDescription();
+    return OptionsData.findConverter(optionsField).getTypeDescription();
   }
 
   static String getFlagName(Field field) {
     String name = field.getAnnotation(Option.class).name();
-    return OptionsParserImpl.isBooleanField(field) ? "[no]" + name : name;
+    return OptionsData.isBooleanField(field) ? "[no]" + name : name;
   }
 
 }