Merge changes I9dc35e9a,I382f4805,I85b454bb,I60935e7d

am: 42e51e0ec4

Change-Id: I929b1d4b64b304b1988f196cc9d59c269f80c76e

10 files changed, 261 insertions, 16 deletions

diff --git a/common/libs/fs/Android.bp b/common/libs/fs/Android.bp
index 01e5f325..59075daa 100644
--- a/common/libs/fs/Android.bp
+++ b/common/libs/fs/Android.bp
@@ -17,6 +17,7 @@ cc_library {
     name: "libcuttlefish_fs",
     srcs: [
         "gce_fs.cpp",
+        "shared_buf.cc",
         "shared_fd.cpp",
     ],
     shared: {
@@ -50,6 +51,7 @@ cc_library_static {
     ],
     srcs: [
         "gce_fs.cpp",
+        "shared_buf.cc",
         "shared_fd.cpp",
     ],
     shared_libs: [
diff --git a/common/libs/fs/shared_buf.cc b/common/libs/fs/shared_buf.cc
new file mode 100644
index 00000000..e929f3b6
--- /dev/null
+++ b/common/libs/fs/shared_buf.cc
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sstream>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "common/libs/fs/shared_buf.h"
+#include "common/libs/fs/shared_fd.h"
+
+namespace cvd {
+
+namespace {
+
+const size_t BUFF_SIZE = 1 << 14;
+
+} // namespace
+
+ssize_t ReadAll(SharedFD fd, std::string* buf) {
+  char buff[BUFF_SIZE];
+  std::stringstream ss;
+  ssize_t read;
+  while ((read = fd->Read(buff, BUFF_SIZE - 1)) > 0) {
+    // this is necessary to avoid problems with having a '\0' in the middle of the buffer
+    ss << std::string(buff, read);
+  }
+  if (read < 0) {
+    return read;
+  }
+  *buf = ss.str();
+  return buf->size();
+}
+
+} // namespace cvd
diff --git a/common/libs/fs/shared_buf.h b/common/libs/fs/shared_buf.h
new file mode 100644
index 00000000..c1ecf375
--- /dev/null
+++ b/common/libs/fs/shared_buf.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "common/libs/fs/shared_fd.h"
+
+namespace cvd {
+
+ssize_t ReadAll(SharedFD fd, std::string* buf);
+
+} // namespace cvd
diff --git a/common/libs/strings/Android.bp b/common/libs/strings/Android.bp
index 273c79ee..fcc67844 100644
--- a/common/libs/strings/Android.bp
+++ b/common/libs/strings/Android.bp
@@ -13,13 +13,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-cc_library_shared {
+cc_library {
     name: "libcuttlefish_strings",
     srcs: [
         "str_split.cpp",
     ],
-    shared_libs: [
-        "libbase",
-    ],
+    shared: {
+        shared_libs: [
+            "libbase",
+        ],
+    },
+    static: {
+        static_libs: [
+            "libbase",
+        ],
+    },
     defaults: ["cuttlefish_host_and_guest"],
 }
diff --git a/common/libs/utils/subprocess.cpp b/common/libs/utils/subprocess.cpp
index 18dbbeb6..cca5e287 100644
--- a/common/libs/utils/subprocess.cpp
+++ b/common/libs/utils/subprocess.cpp
@@ -26,6 +26,9 @@
 #include <set>
 
 #include <glog/logging.h>
+
+#include "common/libs/fs/shared_buf.h"
+
 namespace {
 
 // If a redirected-to file descriptor was already closed, it's possible that
@@ -279,4 +282,34 @@ int execute(const std::vector<std::string>& command) {
   }
   return subprocess.Wait();
 }
+
+int execute_capture_output(const std::vector<std::string>& command,
+                           std::string* output) {
+  Command cmd(command[0]);
+  for (size_t i = 1; i < command.size(); ++i) {
+    cmd.AddParameter(command[i]);
+  }
+  cvd::SharedFD pipe_read, pipe_write;
+  cvd::SharedFD::Pipe(&pipe_read, &pipe_write);
+  cmd.RedirectStdIO(cvd::Subprocess::StdIOChannel::kStdOut, pipe_write);
+  cmd.RedirectStdIO(cvd::Subprocess::StdIOChannel::kStdErr, pipe_write);
+
+  auto subprocess = cmd.Start();
+  if (!subprocess.Started()) {
+    return -1;
+  }
+  {
+    pipe_write->Close();
+    // Force the destructor to run by moving it into a smaller scope.
+    // This is necessary to close the write end of the pipe.
+    cvd::Command forceDelete = std::move(cmd);
+  }
+
+  int read = cvd::ReadAll(pipe_read, output);
+  if (read < 0) {
+    LOG(FATAL) << "Could not read from pipe in execute_capture_output";
+  }
+  return subprocess.Wait();
+}
+
 }  // namespace cvd
diff --git a/common/libs/utils/subprocess.h b/common/libs/utils/subprocess.h
index a2997987..dd435aba 100644
--- a/common/libs/utils/subprocess.h
+++ b/common/libs/utils/subprocess.h
@@ -169,4 +169,8 @@ int execute(const std::vector<std::string>& command,
             const std::vector<std::string>& env);
 int execute(const std::vector<std::string>& command);
 
+// Like execute, but captures stdout and stderr and returns it in "output".
+int execute_capture_output(const std::vector<std::string>& command,
+                           std::string* output);
+
 }  // namespace cvd
diff --git a/host/commands/fetcher/Android.bp b/host/commands/fetcher/Android.bp
index 0c881711..4ab61789 100644
--- a/host/commands/fetcher/Android.bp
+++ b/host/commands/fetcher/Android.bp
@@ -19,6 +19,7 @@ cc_binary_host {
         "build_api.cc",
         "credential_source.cc",
         "curl_wrapper.cc",
+        "install_zip.cc",
         "main.cc",
     ],
     header_libs: [
@@ -29,6 +30,7 @@ cc_binary_host {
         "cuttlefish_auto_resources",
         "libbase",
         "libcuttlefish_fs",
+        "libcuttlefish_strings",
         "libcuttlefish_utils",
         "libcurl",
         "libcrypto",
diff --git a/host/commands/fetcher/install_zip.cc b/host/commands/fetcher/install_zip.cc
new file mode 100644
index 00000000..ad7f08c3
--- /dev/null
+++ b/host/commands/fetcher/install_zip.cc
@@ -0,0 +1,96 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "install_zip.h"
+
+#include <stdlib.h>
+
+#include <string>
+#include <vector>
+
+#include <glog/logging.h>
+
+#include "common/libs/strings/str_split.h"
+#include "common/libs/utils/subprocess.h"
+
+namespace {
+
+std::vector<std::string> ArchiveContents(const std::string& archive) {
+  std::string bsdtar_output;
+  auto bsdtar_ret =
+      cvd::execute_capture_output({"/usr/bin/bsdtar", "-tf", archive},
+                                  &bsdtar_output);
+  return bsdtar_ret == 0
+      ? cvd::StrSplit(bsdtar_output, '\n')
+      : std::vector<std::string>();
+}
+
+} // namespace
+
+bool ExtractImages(const std::string& archive,
+                   const std::string& target_directory,
+                   const std::vector<std::string>& images) {
+  std::vector<std::string> bsdtar_cmd = {
+      "/usr/bin/bsdtar",
+      "-x",
+      "-v",
+      "-C", target_directory,
+      "-f", archive,
+      "-S",
+  };
+  for (const auto& img : images) {
+    bsdtar_cmd.push_back(img);
+  }
+  auto bsdtar_ret = cvd::execute(bsdtar_cmd);
+  if (bsdtar_ret != 0) {
+    LOG(ERROR) << "Unable to extract images. bsdtar returned " << bsdtar_ret;
+    return false;
+  }
+
+  bool extraction_success = true;
+  std::vector<std::string> files =
+      images.size() > 0 ? images : ArchiveContents(archive);
+  for (const auto& file : files) {
+    if (file.find(".img") == std::string::npos) {
+      continue;
+    }
+    std::string extracted_file = target_directory + "/" + file;
+
+    std::string file_output;
+    auto file_ret = cvd::execute_capture_output(
+      {"/usr/bin/file", extracted_file}, &file_output);
+    if (file_ret != 0) {
+      LOG(ERROR) << "Unable to run file on " << file << ", returned" << file_ret;
+      extraction_success = false;
+      continue;
+    }
+    if (file_output.find("Android sparse image,") == std::string::npos) {
+      continue;
+    }
+    std::string inflated_file = extracted_file + ".inflated";
+    auto simg_ret = cvd::execute({"/usr/bin/simg2img", extracted_file, inflated_file});
+    if (simg_ret != 0) {
+      LOG(ERROR) << "Unable to run simg2img on " << file;
+      extraction_success = false;
+      continue;
+    }
+    auto rename_ret = rename(inflated_file.c_str(), extracted_file.c_str());
+    if (rename_ret != 0) {
+      LOG(ERROR) << "Unable to rename deflated version of " << file;
+      extraction_success = false;
+    }
+  }
+  return extraction_success;
+}
diff --git a/host/commands/fetcher/install_zip.h b/host/commands/fetcher/install_zip.h
new file mode 100644
index 00000000..0e365287
--- /dev/null
+++ b/host/commands/fetcher/install_zip.h
@@ -0,0 +1,23 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+bool ExtractImages(const std::string& archive,
+                   const std::string& target_directory,
+                   const std::vector<std::string>& images);
diff --git a/host/commands/fetcher/main.cc b/host/commands/fetcher/main.cc
index 3e555e56..d5c1cac4 100644
--- a/host/commands/fetcher/main.cc
+++ b/host/commands/fetcher/main.cc
@@ -26,6 +26,7 @@
 
 #include "build_api.h"
 #include "credential_source.h"
+#include "install_zip.h"
 
 // TODO(schuffelen): Mixed builds.
 DEFINE_string(build_id, "latest", "Build ID for all artifacts");
@@ -72,18 +73,10 @@ bool download_images(BuildApi* build_api, const std::string& target,
   std::string local_path = target_directory + "/" + img_zip_name;
   build_api->ArtifactToFile(build_id, target, "latest",
                             img_zip_name, local_path);
-  // -o for "overwrite"
-  std::vector<std::string> command = {"/usr/bin/unzip", "-o", local_path,
-                                      "-d", target_directory};
-  for (const auto& image_file : images) {
-    command.push_back(image_file);
-  }
-  int result = cvd::execute(command);
-  if (result != 0) {
-    LOG(ERROR) << "Could not extract " << local_path << "; ran command";
-    for (const auto& argument : command) {
-      LOG(ERROR) << argument;
-    }
+
+  auto could_extract = ExtractImages(local_path, target_directory, images);
+  if (!could_extract) {
+    LOG(ERROR) << "Could not extract " << local_path;
     return false;
   }
   if (unlink(local_path.c_str()) != 0) {
@@ -122,6 +115,15 @@ bool download_host_package(BuildApi* build_api, const std::string& target,
   return true;
 }
 
+bool desparse(const std::string& file) {
+  LOG(INFO) << "Unsparsing " << file;
+  if (cvd::execute({"/bin/dd", "if=" + file, "of=" + file, "conv=notrunc"}) != 0) {
+    LOG(ERROR) << "Could not unsparse " << file;
+    return false;
+  }
+  return true;
+}
+
 } // namespace
 
 int main(int argc, char** argv) {
@@ -156,6 +158,7 @@ int main(int argc, char** argv) {
       LOG(FATAL) << "Could not download images with target "
           << FLAGS_target << " and build id " << build_id;
     }
+    desparse(target_dir + "/userdata.img");
     if (FLAGS_system_image_build_id != "") {
       std::string system_target = FLAGS_system_image_build_target == ""
           ? FLAGS_target