diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-08-09 23:01:06 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-08-09 23:01:06 +0000 |
commit | 84317c337e2561dfe449046165b507cf929e8cf8 (patch) | |
tree | 69a5d9a18b25f31a8d29d74311d2930374deee5f | |
parent | 64abfa8ef3a4e337ee8dc931497c210d16c38fa4 (diff) | |
parent | fd9d4c915779907b71ab5c7d0f95d8e08af6959b (diff) | |
download | soong-android14-qpr1-release.tar.gz |
Snap for 10635039 from fd9d4c915779907b71ab5c7d0f95d8e08af6959b to udc-qpr1-releaseandroid-14.0.0_r27android-14.0.0_r26android-14.0.0_r25android-14.0.0_r24android-14.0.0_r23android-14.0.0_r22android-14.0.0_r21android-14.0.0_r20android-14.0.0_r19android-14.0.0_r18android-14.0.0_r17android-14.0.0_r16android14-qpr1-s2-releaseandroid14-qpr1-release
Change-Id: I9a531797b4405445587292669c47c95ad5e958fc
-rw-r--r-- | cmd/merge_zips/merge_zips.go | 22 | ||||
-rw-r--r-- | cmd/merge_zips/merge_zips_test.go | 68 | ||||
-rw-r--r-- | jar/Android.bp | 1 | ||||
-rw-r--r-- | jar/services.go | 128 | ||||
-rw-r--r-- | java/androidmk.go | 3 | ||||
-rw-r--r-- | java/base.go | 8 | ||||
-rw-r--r-- | java/builder.go | 15 | ||||
-rw-r--r-- | java/device_host_converter_test.go | 3 | ||||
-rw-r--r-- | java/fuzz_test.go | 4 | ||||
-rw-r--r-- | sdk/java_sdk_test.go | 8 | ||||
-rw-r--r-- | third_party/zip/android.go | 2 |
11 files changed, 231 insertions, 31 deletions
diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go index e3d1179b4..a70a9d158 100644 --- a/cmd/merge_zips/merge_zips.go +++ b/cmd/merge_zips/merge_zips.go @@ -122,7 +122,7 @@ func (be ZipEntryFromBuffer) Size() uint64 { } func (be ZipEntryFromBuffer) WriteToZip(dest string, zw *zip.Writer) error { - w, err := zw.CreateHeader(be.fh) + w, err := zw.CreateHeaderAndroid(be.fh) if err != nil { return err } @@ -562,6 +562,8 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } } + var jarServices jar.Services + // Finally, add entries from all the input zips. for _, inputZip := range inputZips { _, copyFully := zipsToNotStrip[inputZip.Name()] @@ -570,6 +572,14 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } for i, entry := range inputZip.Entries() { + if emulateJar && jarServices.IsServiceFile(entry) { + // If this is a jar, collect service files to combine instead of adding them to the zip. + err := jarServices.AddServiceFile(entry) + if err != nil { + return err + } + continue + } if copyFully || !out.isEntryExcluded(entry.Name) { if err := out.copyEntry(inputZip, i); err != nil { return err @@ -585,6 +595,16 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } if emulateJar { + // Combine all the service files into a single list of combined service files and add them to the zip. + for _, serviceFile := range jarServices.ServiceFiles() { + _, err := out.addZipEntry(serviceFile.Name, ZipEntryFromBuffer{ + fh: serviceFile.FileHeader, + content: serviceFile.Contents, + }) + if err != nil { + return err + } + } return out.writeEntries(out.jarSorted()) } else if sortEntries { return out.writeEntries(out.alphanumericSorted()) diff --git a/cmd/merge_zips/merge_zips_test.go b/cmd/merge_zips/merge_zips_test.go index cb5843607..767d4e61f 100644 --- a/cmd/merge_zips/merge_zips_test.go +++ b/cmd/merge_zips/merge_zips_test.go @@ -17,6 +17,7 @@ package main import ( "bytes" "fmt" + "hash/crc32" "os" "strconv" "strings" @@ -27,28 +28,34 @@ import ( ) type testZipEntry struct { - name string - mode os.FileMode - data []byte + name string + mode os.FileMode + data []byte + method uint16 } var ( - A = testZipEntry{"A", 0755, []byte("foo")} - a = testZipEntry{"a", 0755, []byte("foo")} - a2 = testZipEntry{"a", 0755, []byte("FOO2")} - a3 = testZipEntry{"a", 0755, []byte("Foo3")} - bDir = testZipEntry{"b/", os.ModeDir | 0755, nil} - bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil} - bbb = testZipEntry{"b/b/b", 0755, nil} - ba = testZipEntry{"b/a", 0755, []byte("foob")} - bc = testZipEntry{"b/c", 0755, []byte("bar")} - bd = testZipEntry{"b/d", 0700, []byte("baz")} - be = testZipEntry{"b/e", 0700, []byte("")} - - metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil} - manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest")} - manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2")} - moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")} + A = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate} + a = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate} + a2 = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate} + a3 = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate} + bDir = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate} + bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate} + bbb = testZipEntry{"b/b/b", 0755, nil, zip.Deflate} + ba = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate} + bc = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate} + bd = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate} + be = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate} + + service1a = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store} + service1b = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate} + service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store} + service2 = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate} + + metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate} + manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate} + manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate} + moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate} ) type testInputZip struct { @@ -236,6 +243,15 @@ func TestMergeZips(t *testing.T) { "in1": true, }, }, + { + name: "services", + in: [][]testZipEntry{ + {service1a, service2}, + {service1b}, + }, + jar: true, + out: []testZipEntry{service1combined, service2}, + }, } for _, test := range testCases { @@ -256,7 +272,7 @@ func TestMergeZips(t *testing.T) { closeErr := writer.Close() if closeErr != nil { - t.Fatal(err) + t.Fatal(closeErr) } if test.err != "" { @@ -266,12 +282,16 @@ func TestMergeZips(t *testing.T) { t.Fatal("incorrect err, want:", test.err, "got:", err) } return + } else if err != nil { + t.Fatal("unexpected err: ", err) } if !bytes.Equal(want, out.Bytes()) { t.Error("incorrect zip output") t.Errorf("want:\n%s", dumpZip(want)) t.Errorf("got:\n%s", dumpZip(out.Bytes())) + os.WriteFile("/tmp/got.zip", out.Bytes(), 0755) + os.WriteFile("/tmp/want.zip", want, 0755) } }) } @@ -286,8 +306,14 @@ func testZipEntriesToBuf(entries []testZipEntry) []byte { Name: e.name, } fh.SetMode(e.mode) + fh.Method = e.method + fh.UncompressedSize64 = uint64(len(e.data)) + fh.CRC32 = crc32.ChecksumIEEE(e.data) + if fh.Method == zip.Store { + fh.CompressedSize64 = fh.UncompressedSize64 + } - w, err := zw.CreateHeader(&fh) + w, err := zw.CreateHeaderAndroid(&fh) if err != nil { panic(err) } diff --git a/jar/Android.bp b/jar/Android.bp index 46113d877..c03e49174 100644 --- a/jar/Android.bp +++ b/jar/Android.bp @@ -21,6 +21,7 @@ bootstrap_go_package { pkgPath: "android/soong/jar", srcs: [ "jar.go", + "services.go", ], testSrcs: [ "jar_test.go", diff --git a/jar/services.go b/jar/services.go new file mode 100644 index 000000000..d06a6dc99 --- /dev/null +++ b/jar/services.go @@ -0,0 +1,128 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jar + +import ( + "android/soong/third_party/zip" + "bufio" + "hash/crc32" + "sort" + "strings" +) + +const servicesPrefix = "META-INF/services/" + +// Services is used to collect service files from multiple zip files and produce a list of ServiceFiles containing +// the unique lines from all the input zip entries with the same name. +type Services struct { + services map[string]*ServiceFile +} + +// ServiceFile contains the combined contents of all input zip entries with a single name. +type ServiceFile struct { + Name string + FileHeader *zip.FileHeader + Contents []byte + Lines []string +} + +// IsServiceFile returns true if the zip entry is in the META-INF/services/ directory. +func (Services) IsServiceFile(entry *zip.File) bool { + return strings.HasPrefix(entry.Name, servicesPrefix) +} + +// AddServiceFile adds a zip entry in the META-INF/services/ directory to the list of service files that need +// to be combined. +func (j *Services) AddServiceFile(entry *zip.File) error { + if j.services == nil { + j.services = map[string]*ServiceFile{} + } + + service := entry.Name + serviceFile := j.services[service] + fh := entry.FileHeader + if serviceFile == nil { + serviceFile = &ServiceFile{ + Name: service, + FileHeader: &fh, + } + j.services[service] = serviceFile + } + + f, err := entry.Open() + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + serviceFile.Lines = append(serviceFile.Lines, line) + } + } + + if err := scanner.Err(); err != nil { + return err + } + + return nil +} + +// ServiceFiles returns the list of combined service files, each containing all the unique lines from the +// corresponding service files in the input zip entries. +func (j *Services) ServiceFiles() []ServiceFile { + services := make([]ServiceFile, 0, len(j.services)) + + for _, serviceFile := range j.services { + serviceFile.Lines = dedupServicesLines(serviceFile.Lines) + serviceFile.Lines = append(serviceFile.Lines, "") + serviceFile.Contents = []byte(strings.Join(serviceFile.Lines, "\n")) + + serviceFile.FileHeader.UncompressedSize64 = uint64(len(serviceFile.Contents)) + serviceFile.FileHeader.CRC32 = crc32.ChecksumIEEE(serviceFile.Contents) + if serviceFile.FileHeader.Method == zip.Store { + serviceFile.FileHeader.CompressedSize64 = serviceFile.FileHeader.UncompressedSize64 + } + + services = append(services, *serviceFile) + } + + sort.Slice(services, func(i, j int) bool { + return services[i].Name < services[j].Name + }) + + return services +} + +func dedupServicesLines(in []string) []string { + writeIndex := 0 +outer: + for readIndex := 0; readIndex < len(in); readIndex++ { + for compareIndex := 0; compareIndex < writeIndex; compareIndex++ { + if interface{}(in[readIndex]) == interface{}(in[compareIndex]) { + // The value at readIndex already exists somewhere in the output region + // of the slice before writeIndex, skip it. + continue outer + } + } + if readIndex != writeIndex { + in[writeIndex] = in[readIndex] + } + writeIndex++ + } + return in[0:writeIndex] +} diff --git a/java/androidmk.go b/java/androidmk.go index 9c21633fb..291ed90ee 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -340,6 +340,9 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries { // App module names can be overridden. entries.SetString("LOCAL_MODULE", app.installApkName) entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", app.appProperties.PreventInstall) + if app.headerJarFile != nil { + entries.SetPath("LOCAL_SOONG_HEADER_JAR", app.headerJarFile) + } entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", app.exportPackage) if app.dexJarFile.IsSet() { entries.SetPath("LOCAL_SOONG_DEX_JAR", app.dexJarFile.Path()) diff --git a/java/base.go b/java/base.go index 991132321..e29e94ae3 100644 --- a/java/base.go +++ b/java/base.go @@ -1446,7 +1446,13 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { j.implementationJarFile = outputFile if j.headerJarFile == nil { - j.headerJarFile = j.implementationJarFile + // If this module couldn't generate a header jar (for example due to api generating annotation processors) + // then use the implementation jar. Run it through zip2zip first to remove any files in META-INF/services + // so that javac on modules that depend on this module don't pick up annotation processors (which may be + // missing their implementations) from META-INF/services/javax.annotation.processing.Processor. + headerJarFile := android.PathForModuleOut(ctx, "javac-header", jarName) + convertImplementationJarToHeaderJar(ctx, j.implementationJarFile, headerJarFile) + j.headerJarFile = headerJarFile } // enforce syntax check to jacoco filters for any build (http://b/183622051) diff --git a/java/builder.go b/java/builder.go index 462626712..8cc0c2587 100644 --- a/java/builder.go +++ b/java/builder.go @@ -246,6 +246,12 @@ var ( CommandDeps: []string{"${config.ZipAlign}"}, }, ) + + convertImplementationJarToHeaderJarRule = pctx.AndroidStaticRule("convertImplementationJarToHeaderJar", + blueprint.RuleParams{ + Command: `${config.Zip2ZipCmd} -i ${in} -o ${out} -x 'META-INF/services/**/*'`, + CommandDeps: []string{"${config.Zip2ZipCmd}"}, + }) ) func init() { @@ -600,6 +606,15 @@ func TransformJarsToJar(ctx android.ModuleContext, outputFile android.WritablePa }) } +func convertImplementationJarToHeaderJar(ctx android.ModuleContext, implementationJarFile android.Path, + headerJarFile android.WritablePath) { + ctx.Build(pctx, android.BuildParams{ + Rule: convertImplementationJarToHeaderJarRule, + Input: implementationJarFile, + Output: headerJarFile, + }) +} + func TransformJarJar(ctx android.ModuleContext, outputFile android.WritablePath, classesJar android.Path, rulesFile android.Path) { ctx.Build(pctx, android.BuildParams{ diff --git a/java/device_host_converter_test.go b/java/device_host_converter_test.go index 3c9a0f3f1..3413da03d 100644 --- a/java/device_host_converter_test.go +++ b/java/device_host_converter_test.go @@ -135,6 +135,7 @@ func TestHostForDevice(t *testing.T) { hostModule := ctx.ModuleForTests("host_module", config.BuildOSCommonTarget.String()) hostJavac := hostModule.Output("javac/host_module.jar") + hostJavacHeader := hostModule.Output("javac-header/host_module.jar") hostRes := hostModule.Output("res/host_module.jar") hostImportModule := ctx.ModuleForTests("host_import_module", config.BuildOSCommonTarget.String()) @@ -148,7 +149,7 @@ func TestHostForDevice(t *testing.T) { // check classpath of device module with dependency on host_for_device_module expectedClasspath := "-classpath " + strings.Join(android.Paths{ - hostJavac.Output, + hostJavacHeader.Output, hostImportCombined.Output, }.Strings(), ":") diff --git a/java/fuzz_test.go b/java/fuzz_test.go index dd1e96b3e..f29c91327 100644 --- a/java/fuzz_test.go +++ b/java/fuzz_test.go @@ -71,8 +71,8 @@ func TestJavaFuzz(t *testing.T) { } baz := result.ModuleForTests("baz", osCommonTarget).Rule("javac").Output.String() - barOut := filepath.Join("out", "soong", ".intermediates", "bar", osCommonTarget, "javac", "bar.jar") - bazOut := filepath.Join("out", "soong", ".intermediates", "baz", osCommonTarget, "javac", "baz.jar") + barOut := filepath.Join("out", "soong", ".intermediates", "bar", osCommonTarget, "javac-header", "bar.jar") + bazOut := filepath.Join("out", "soong", ".intermediates", "baz", osCommonTarget, "javac-header", "baz.jar") android.AssertStringDoesContain(t, "foo classpath", javac.Args["classpath"], barOut) android.AssertStringDoesContain(t, "foo classpath", javac.Args["classpath"], bazOut) diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 6159ea9c2..c8cc83476 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -160,7 +160,7 @@ java_import { } `), checkAllCopyRules(` -.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/myjavalib.jar +.intermediates/myjavalib/linux_glibc_common/javac-header/myjavalib.jar -> java/myjavalib.jar aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl `), ) @@ -206,7 +206,7 @@ java_import { `), checkAllCopyRules(` .intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/android/myjavalib.jar -.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar +.intermediates/myjavalib/linux_glibc_common/javac-header/myjavalib.jar -> java/linux_glibc/myjavalib.jar `), ) } @@ -799,7 +799,7 @@ java_system_modules_import { libs: ["mysdk_system-module"], } `), - checkAllCopyRules(".intermediates/system-module/linux_glibc_common/javac/system-module.jar -> java/system-module.jar"), + checkAllCopyRules(".intermediates/system-module/linux_glibc_common/javac-header/system-module.jar -> java/system-module.jar"), ) } @@ -879,7 +879,7 @@ java_import { } `), checkAllCopyRules(` -.intermediates/hostjavalib/linux_glibc_common/javac/hostjavalib.jar -> java/hostjavalib.jar +.intermediates/hostjavalib/linux_glibc_common/javac-header/hostjavalib.jar -> java/hostjavalib.jar .intermediates/androidjavalib/android_common/turbine-combined/androidjavalib.jar -> java/androidjavalib.jar .intermediates/myjavalib/android_common/javac/myjavalib.jar -> java/android/myjavalib.jar .intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar diff --git a/third_party/zip/android.go b/third_party/zip/android.go index f8e45c56d..0f41f6200 100644 --- a/third_party/zip/android.go +++ b/third_party/zip/android.go @@ -170,7 +170,7 @@ func (w *Writer) CreateCompressedHeader(fh *FileHeader) (io.WriteCloser, error) func (w *Writer) CreateHeaderAndroid(fh *FileHeader) (io.Writer, error) { writeDataDescriptor := fh.Method != Store if writeDataDescriptor { - fh.Flags &= DataDescriptorFlag + fh.Flags |= DataDescriptorFlag } else { fh.Flags &= ^uint16(DataDescriptorFlag) } |