diff options
30 files changed, 600 insertions, 323 deletions
diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 0000000..6fca532 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,8 @@ +[codespell] + +# Add extra dictionaries to help improvement comments, docs, etc. +builtin = clear,rare,informal,usage,names + +# Always default to highest interactive level to avoid accidentally +# changing a false positive or picking the wrong replacement. +interactive = 3 diff --git a/.gitattributes b/.gitattributes index de2f316..7e8b706 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ .gitignore export-ignore .gitattributes export-ignore +.codespellrc export-ignore @@ -0,0 +1,11 @@ + +Authors of XZ Embedded +====================== + + XZ Embedded is developed and maintained by + Lasse Collin <lasse.collin@tukaani.org>. + + Major parts of XZ Embedded are based on code written by Igor Pavlov, + specifically the LZMA SDK <https://7-zip.org/sdk.html>. Without this + code, XZ Embedded wouldn't exist. + @@ -1,10 +1,13 @@ - -Licensing of XZ Embedded -======================== - - All the files in this package have been written by Lasse Collin - and/or Igor Pavlov. All these files have been put into the - public domain. You can do whatever you want with these files. - - As usual, this software is provided "as is", without any warranty. - +Copyright (C) The XZ Embedded authors and contributors + +Permission to use, copy, modify, and/or distribute this +software for any purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. @@ -8,12 +8,12 @@ third_party { license_type: UNENCUMBERED last_upgrade_date { year: 2024 - month: 2 - day: 26 + month: 5 + day: 30 } identifier { type: "Git" value: "https://git.tukaani.org/xz-embedded.git" - version: "1b6defd544914bfb4065e343296e5db64ef400e6" + version: "d4a9bc83c72d8087fe36ff388e89599626da7873" } } @@ -9,56 +9,45 @@ XZ Embedded be easily used in other environments too, including regular userspace applications. See userspace/xzminidec.c for an example program. + NOTE: The version of XZ Embedded in the Linux kernel lacks a few + build-time-selectable optional features that are present in the + upstream XZ Embedded project: support for concatated .xz files, + CRC64, and ignoring unsupported check. These aren't in Linux + because they don't seem useful there but they would add to the + code size. + This README contains information that is useful only when the copy of XZ Embedded isn't part of the Linux kernel tree. You should also - read linux/Documentation/xz.txt even if you aren't using XZ Embedded - as part of Linux; information in that file is not repeated in this - README. - -Compiling the Linux kernel module + read linux/Documentation/staging/xz.rst even if you aren't using + XZ Embedded as part of Linux; information in that file is not + repeated in this README. - The xz_dec module depends on crc32 module, so make sure that you have - it enabled (CONFIG_CRC32). +Conformance to the .xz file format specification - Building the xz_dec and xz_dec_test modules without support for BCJ - filters: + As of the .xz file format specification version 1.2.0, this + decompressor implementation has the following limitations: - cd linux/lib/xz - make -C /path/to/kernel/source \ - KCPPFLAGS=-I"$(pwd)/../../include" M="$(pwd)" \ - CONFIG_XZ_DEC=m CONFIG_XZ_DEC_TEST=m + - SHA-256 isn't supported. It can be ignored as an unsupported + checked type if that feature is enabled at build time. - Building the xz_dec and xz_dec_test modules with support for BCJ - filters: + - Delta filter is not included. - cd linux/lib/xz - make -C /path/to/kernel/source \ - KCPPFLAGS=-I"$(pwd)/../../include" M="$(pwd)" \ - CONFIG_XZ_DEC=m CONFIG_XZ_DEC_TEST=m CONFIG_XZ_DEC_BCJ=y \ - CONFIG_XZ_DEC_X86=y CONFIG_XZ_DEC_ARM=y \ - CONFIG_XZ_DEC_ARMTHUMB=y CONFIG_XZ_DEC_ARM64=y \ - CONFIG_XZ_DEC_POWERPC=y CONFIG_XZ_DEC_IA64=y \ - CONFIG_XZ_DEC_SPARC=y + - BCJ filters don't support non-default start offset. - If you want only one or a few of the BCJ filters, omit the appropriate - variables. CONFIG_XZ_DEC_BCJ=y is always required to build the support - code shared between all BCJ filters. + - LZMA2 supports at most 3 GiB dictionary. - Most people don't need the xz_dec_test module. You can skip building - it by omitting CONFIG_XZ_DEC_TEST=m from the make command line. + There are a couple of corner cases where things have been simplified + at expense of detecting errors as early as possible. These should not + matter in practice at all since they don't cause security issues. But + it is good to know this if testing the code with the test files from + XZ Utils. Compiler requirements - XZ Embedded should compile as either GNU-C89 (used in the Linux - kernel) or with any C99 compiler. Getting the code to compile with - non-GNU C89 compiler or a C++ compiler should be quite easy as - long as there is a data type for unsigned 64-bit integer (or the - code is modified not to support large files, which needs some more - care than just using 32-bit integer instead of 64-bit). - - If you use GCC, try to use a recent version. For example, on x86-32, - xz_dec_lzma2.c compiled with GCC 3.3.6 is 15-25 % slower than when - compiled with GCC 4.3.3. + XZ Embedded should compile with any C99 or C11 compiler. The code + should also be GNU-C89 compatible still. GNU-C89 was used in the + Linux kernel until 2022. GNU-C89 support likely will be dropped + at some point. Embedding into userspace applications @@ -152,6 +141,7 @@ BCJ filter support XZ_DEC_ARMTHUMB ARM-Thumb Big or little endian XZ_DEC_ARM64 ARM64 Big or little endian XZ_DEC_SPARC SPARC Big or little endian + XZ_DEC_RISCV RISC-V Big or little endian While some architectures are (partially) bi-endian, the endianness setting doesn't change the endianness of the instructions on all @@ -169,7 +159,13 @@ Notes about shared libraries Please don't create a shared library of XZ Embedded itself unless it is fine to rebuild everything depending on that shared library - everytime you upgrade to a newer version of XZ Embedded. There are + every time you upgrade to a newer version of XZ Embedded. There are no API or ABI stability guarantees between different versions of XZ Embedded. +Contact information + + Email: Lasse Collin <lasse.collin@tukaani.org> + IRC: Larhzu on #tukaani on Libera Chat + GitHub: https://github.com/tukaani-project/xz-embedded + diff --git a/linux/Documentation/staging/xz.rst b/linux/Documentation/staging/xz.rst new file mode 100644 index 0000000..5d34d3d --- /dev/null +++ b/linux/Documentation/staging/xz.rst @@ -0,0 +1,89 @@ +.. SPDX-License-Identifier: 0BSD + +============================ +XZ data compression in Linux +============================ + +Introduction +============ + +XZ is a general purpose data compression format with high compression +ratio. The XZ decompressor in Linux is called XZ Embedded. It supports +the LZMA2 filter and optionally also Branch/Call/Jump (BCJ) filters +for executable code. CRC32 is supported for integrity checking. The +home page of XZ Embedded is at <https://tukaani.org/xz/embedded.html>. +There you can find the latest version which includes a few optional +extra features that aren't required in the Linux kernel and information +about using the code outside the Linux kernel. + +For userspace, XZ Utils provide a zlib-like compression library +and a gzip-like command line tool. XZ Utils can be downloaded from +<https://tukaani.org/xz/>. + +XZ related components in the kernel +=================================== + +The xz_dec module provides XZ decompressor with single-call (buffer +to buffer) and multi-call (stateful) APIs. The usage of the xz_dec +module is documented in include/linux/xz.h. + +For decompressing the kernel image, initramfs, and initrd, there +is a wrapper function in lib/decompress_unxz.c. Its API is the +same as in other decompress_*.c files, which is defined in +include/linux/decompress/generic.h. + +For kernel makefiles, three commands are provided for use with +$(call if_changed). They require the xz tool from XZ Utils. + + - $(call if_changed,xzkern) is for compressing the kernel image. + It runs the script scripts/xz_wrap.sh which uses arch-optimized + options and a big LZMA2 dictionary. + + - $(call if_changed,xzkern_with_size) is like xzkern above but this + also appends a four-byte trailer containing the uncompressed size + of the file, which is needed by the boot code on some archs. + + - Other things can be compressed with $(call if_needed,xzmisc) + which will use no BCJ filter and 1 MiB LZMA2 dictionary. + +Notes on compression options +============================ + +Since the XZ Embedded supports only streams with no integrity check or +CRC32, make sure that you don't use some other integrity check type +when encoding files that are supposed to be decoded by the kernel. With +liblzma, you need to use either LZMA_CHECK_NONE or LZMA_CHECK_CRC32 +when encoding. With the xz command line tool, use --check=none or +--check=crc32 to override the default --check=crc64. + +Using CRC32 is strongly recommended unless there is some other layer +which will verify the integrity of the uncompressed data anyway. +Double checking the integrity would probably be waste of CPU cycles. +Note that the headers will always have a CRC32 which will be validated +by the decoder; you can only change the integrity check type (or +disable it) for the actual uncompressed data. + +In userspace, LZMA2 is typically used with dictionary sizes of several +megabytes. The decoder needs to have the dictionary in RAM: + + - In multi-call mode the dictionary is allocated as part of the + decoder state. The reasonable maximum dictionary size for in-kernel + use will depend on the target hardware: a few megabytes should be + fine for desktop systems while 64 KiB to 1 MiB might be more + appropriate on some embedded systems. + + - In single-call mode the output buffer is used as the dictionary + buffer. That is, the size of the dictionary doesn't affect the + decompressor memory usage at all. Only the base data structures + are allocated which take a little less than 30 KiB of memory. + For the best compression, the dictionary should be at least + as big as the uncompressed data. A notable example of single-call + mode is decompressing the kernel itself (except on PowerPC). + +The compression presets in XZ Utils may not be optimal when creating +files for the kernel, so don't hesitate to use custom settings to, +for example, set the dictionary size. Also, xz may produce a smaller +file in single-threaded mode so setting that explicitly is recommended. +Example:: + + xz --threads=1 --check=crc32 --lzma2=dict=512KiB inputfile diff --git a/linux/Documentation/xz.txt b/linux/Documentation/xz.txt deleted file mode 100644 index b1a743e..0000000 --- a/linux/Documentation/xz.txt +++ /dev/null @@ -1,122 +0,0 @@ - -XZ data compression in Linux -============================ - -Introduction - - XZ is a general purpose data compression format with high compression - ratio and relatively fast decompression. The primary compression - algorithm (filter) is LZMA2. Additional filters can be used to improve - compression ratio even further. E.g. Branch/Call/Jump (BCJ) filters - improve compression ratio of executable data. - - The XZ decompressor in Linux is called XZ Embedded. It supports - the LZMA2 filter and optionally also BCJ filters. CRC32 is supported - for integrity checking. The home page of XZ Embedded is at - <https://xz.tukaani.org/xz-embedded/>, where you can find the - latest version and also information about using the code outside - the Linux kernel. - - For userspace, XZ Utils provide a zlib-like compression library - and a gzip-like command line tool. XZ Utils can be downloaded from - <https://xz.tukaani.org/xz-utils/>. - -XZ related components in the kernel - - The xz_dec module provides XZ decompressor with single-call (buffer - to buffer) and multi-call (stateful) APIs. The usage of the xz_dec - module is documented in include/linux/xz.h. - - The xz_dec_test module is for testing xz_dec. xz_dec_test is not - useful unless you are hacking the XZ decompressor. xz_dec_test - allocates a char device major dynamically to which one can write - .xz files from userspace. The decompressed output is thrown away. - Keep an eye on dmesg to see diagnostics printed by xz_dec_test. - See the xz_dec_test source code for the details. - - For decompressing the kernel image, initramfs, and initrd, there - is a wrapper function in lib/decompress_unxz.c. Its API is the - same as in other decompress_*.c files, which is defined in - include/linux/decompress/generic.h. - - scripts/xz_wrap.sh is a wrapper for the xz command line tool found - from XZ Utils. The wrapper sets compression options to values suitable - for compressing the kernel image. - - For kernel makefiles, two commands are provided for use with - $(call if_needed). The kernel image should be compressed with - $(call if_needed,xzkern) which will use a BCJ filter and a big LZMA2 - dictionary. It will also append a four-byte trailer containing the - uncompressed size of the file, which is needed by the boot code. - Other things should be compressed with $(call if_needed,xzmisc) - which will use no BCJ filter and 1 MiB LZMA2 dictionary. - -Notes on compression options - - Since the XZ Embedded supports only streams with no integrity check or - CRC32, make sure that you don't use some other integrity check type - when encoding files that are supposed to be decoded by the kernel. With - liblzma, you need to use either LZMA_CHECK_NONE or LZMA_CHECK_CRC32 - when encoding. With the xz command line tool, use --check=none or - --check=crc32. - - Using CRC32 is strongly recommended unless there is some other layer - which will verify the integrity of the uncompressed data anyway. - Double checking the integrity would probably be waste of CPU cycles. - Note that the headers will always have a CRC32 which will be validated - by the decoder; you can only change the integrity check type (or - disable it) for the actual uncompressed data. - - In userspace, LZMA2 is typically used with dictionary sizes of several - megabytes. The decoder needs to have the dictionary in RAM, thus big - dictionaries cannot be used for files that are intended to be decoded - by the kernel. 1 MiB is probably the maximum reasonable dictionary - size for in-kernel use (maybe more is OK for initramfs). The presets - in XZ Utils may not be optimal when creating files for the kernel, - so don't hesitate to use custom settings. Example: - - xz --check=crc32 --lzma2=dict=512KiB inputfile - - An exception to above dictionary size limitation is when the decoder - is used in single-call mode. Decompressing the kernel itself is an - example of this situation. In single-call mode, the memory usage - doesn't depend on the dictionary size, and it is perfectly fine to - use a big dictionary: for maximum compression, the dictionary should - be at least as big as the uncompressed data itself. - -Future plans - - Creating a limited XZ encoder may be considered if people think it is - useful. LZMA2 is slower to compress than e.g. Deflate or LZO even at - the fastest settings, so it isn't clear if LZMA2 encoder is wanted - into the kernel. - - Support for limited random-access reading is planned for the - decompression code. I don't know if it could have any use in the - kernel, but I know that it would be useful in some embedded projects - outside the Linux kernel. - -Conformance to the .xz file format specification - - There are a couple of corner cases where things have been simplified - at expense of detecting errors as early as possible. These should not - matter in practice all, since they don't cause security issues. But - it is good to know this if testing the code e.g. with the test files - from XZ Utils. - -Reporting bugs - - Before reporting a bug, please check that it's not fixed already - at upstream. See <https://xz.tukaani.org/xz-embedded/> to get the - latest code. - - Report bugs to <xz@tukaani.org> or visit #tukaani on - Libera Chat and talk to the maintainers. We don't actively read LKML or - other kernel-related mailing lists, so if there's something we should - know, please use email or IRC. - - Don't bother Igor Pavlov with questions about the XZ implementation - in the kernel or about XZ Utils. While these two implementations - include essential code that is directly based on Igor Pavlov's code, - these implementations aren't maintained nor supported by him. - diff --git a/linux/MAINTAINERS b/linux/MAINTAINERS new file mode 100644 index 0000000..4d84f38 --- /dev/null +++ b/linux/MAINTAINERS @@ -0,0 +1,12 @@ +XZ EMBEDDED +M: Lasse Collin <lasse.collin@tukaani.org> +S: Maintained +W: https://tukaani.org/xz/embedded.html +B: https://github.com/tukaani-project/xz-embedded/issues +C: irc://irc.libera.chat/tukaani +F: Documentation/staging/xz.rst +F: include/linux/decompress/unxz.h +F: include/linux/xz.h +F: lib/decompress_unxz.c +F: lib/xz/ +F: scripts/xz_wrap.sh diff --git a/linux/include/linux/decompress/unxz.h b/linux/include/linux/decompress/unxz.h index 41728fc..3dd2658 100644 --- a/linux/include/linux/decompress/unxz.h +++ b/linux/include/linux/decompress/unxz.h @@ -1,19 +1,18 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef DECOMPRESS_UNXZ_H #define DECOMPRESS_UNXZ_H -int unxz(unsigned char *in, int in_size, - int (*fill)(void *dest, unsigned int size), - int (*flush)(void *src, unsigned int size), - unsigned char *out, int *in_used, +int unxz(unsigned char *in, long in_size, + long (*fill)(void *dest, unsigned long size), + long (*flush)(void *src, unsigned long size), + unsigned char *out, long *in_used, void (*error)(char *x)); #endif diff --git a/linux/include/linux/xz.h b/linux/include/linux/xz.h index f3801eb..0bf4282 100644 --- a/linux/include/linux/xz.h +++ b/linux/include/linux/xz.h @@ -1,11 +1,10 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * XZ decompressor * * Authors: Lasse Collin <lasse.collin@tukaani.org> * Igor Pavlov <https://7-zip.org/> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_H @@ -216,7 +215,7 @@ XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max); * the multi-call decoder if you don't want to uncompress the whole stream. * * Use xz_dec_run() when XZ data is stored inside some other file format. - * The decoding will stop after one XZ stream has been decompresed. To + * The decoding will stop after one XZ stream has been decompressed. To * decompress regular .xz files which might have multiple concatenated * streams, use xz_dec_catrun() instead. */ diff --git a/linux/lib/decompress_unxz.c b/linux/lib/decompress_unxz.c index 7dfdde7..cae0039 100644 --- a/linux/lib/decompress_unxz.c +++ b/linux/lib/decompress_unxz.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ /* @@ -102,6 +101,8 @@ */ #ifdef STATIC # define XZ_PREBOOT +#else +# include <linux/decompress/unxz.h> #endif #ifdef __KERNEL__ # include <linux/decompress/mm.h> @@ -125,14 +126,21 @@ #ifdef CONFIG_X86 # define XZ_DEC_X86 #endif -#ifdef CONFIG_PPC +#if defined(CONFIG_PPC) && defined(CONFIG_CPU_BIG_ENDIAN) # define XZ_DEC_POWERPC #endif #ifdef CONFIG_ARM -# define XZ_DEC_ARM +# ifdef CONFIG_THUMB2_KERNEL +# define XZ_DEC_ARMTHUMB +# else +# define XZ_DEC_ARM +# endif +#endif +#ifdef CONFIG_ARM64 +# define XZ_DEC_ARM64 #endif -#ifdef CONFIG_IA64 -# define XZ_DEC_IA64 +#ifdef CONFIG_RISCV +# define XZ_DEC_RISCV #endif #ifdef CONFIG_SPARC # define XZ_DEC_SPARC @@ -221,7 +229,7 @@ void *memmove(void *dest, const void *src, size_t size) #endif /* - * Since we need memmove anyway, would use it as memcpy too. + * Since we need memmove anyway, we could use it as memcpy too. * Commented out for now to avoid breaking things. */ /* @@ -248,10 +256,10 @@ void *memmove(void *dest, const void *src, size_t size) * both input and output buffers are available as a single chunk, i.e. when * fill() and flush() won't be used. */ -STATIC int INIT unxz(unsigned char *in, int in_size, - int (*fill)(void *dest, unsigned int size), - int (*flush)(void *src, unsigned int size), - unsigned char *out, int *in_used, +STATIC int INIT unxz(unsigned char *in, long in_size, + long (*fill)(void *dest, unsigned long size), + long (*flush)(void *src, unsigned long size), + unsigned char *out, long *in_used, void (*error)(char *x)) { struct xz_buf b; @@ -329,7 +337,7 @@ STATIC int INIT unxz(unsigned char *in, int in_size, * returned by xz_dec_run(), but probably * it's not too bad. */ - if (flush(b.out, b.out_pos) != (int)b.out_pos) + if (flush(b.out, b.out_pos) != (long)b.out_pos) ret = XZ_BUF_ERROR; b.out_pos = 0; @@ -391,7 +399,17 @@ error_alloc_state: } /* - * This macro is used by architecture-specific files to decompress + * This function is used by architecture-specific files to decompress * the kernel image. */ -#define decompress unxz +#ifdef XZ_PREBOOT +STATIC int INIT __decompress(unsigned char *in, long in_size, + long (*fill)(void *dest, unsigned long size), + long (*flush)(void *src, unsigned long size), + unsigned char *out, long out_size, + long *in_used, + void (*error)(char *x)) +{ + return unxz(in, in_size, fill, flush, out, in_used, error); +} +#endif diff --git a/linux/lib/xz/Kconfig b/linux/lib/xz/Kconfig index c993dda..f5141f4 100644 --- a/linux/lib/xz/Kconfig +++ b/linux/lib/xz/Kconfig @@ -4,7 +4,8 @@ config XZ_DEC help LZMA2 compression algorithm and BCJ filters are supported using the .xz file format as the container. For integrity checking, - CRC32 is supported. See Documentation/xz.txt for more information. + CRC32 is supported. See Documentation/staging/xz.rst for more + information. if XZ_DEC @@ -43,6 +44,11 @@ config XZ_DEC_SPARC default y select XZ_DEC_BCJ +config XZ_DEC_RISCV + bool "RISC-V BCJ filter decoder" if EXPERT + default y + select XZ_DEC_BCJ + config XZ_DEC_MICROLZMA bool "MicroLZMA decoder" default n diff --git a/linux/lib/xz/xz_crc32.c b/linux/lib/xz/xz_crc32.c index 5627b00..effdf34 100644 --- a/linux/lib/xz/xz_crc32.c +++ b/linux/lib/xz/xz_crc32.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: 0BSD + /* * CRC32 using the polynomial from IEEE-802.3 * * Authors: Lasse Collin <lasse.collin@tukaani.org> * Igor Pavlov <https://7-zip.org/> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ /* diff --git a/linux/lib/xz/xz_crc64.c b/linux/lib/xz/xz_crc64.c index 60c40f6..20049ea 100644 --- a/linux/lib/xz/xz_crc64.c +++ b/linux/lib/xz/xz_crc64.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /* * CRC64 using the polynomial from ECMA-182 * @@ -5,9 +7,6 @@ * * Authors: Lasse Collin <lasse.collin@tukaani.org> * Igor Pavlov <https://7-zip.org/> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include "xz_private.h" diff --git a/linux/lib/xz/xz_dec_bcj.c b/linux/lib/xz/xz_dec_bcj.c index 56c02e9..42d7f26 100644 --- a/linux/lib/xz/xz_dec_bcj.c +++ b/linux/lib/xz/xz_dec_bcj.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: 0BSD + /* * Branch/Call/Jump (BCJ) filter decoders * * Authors: Lasse Collin <lasse.collin@tukaani.org> * Igor Pavlov <https://7-zip.org/> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include "xz_private.h" @@ -25,7 +24,8 @@ struct xz_dec_bcj { BCJ_ARM = 7, /* Little endian only */ BCJ_ARMTHUMB = 8, /* Little endian only */ BCJ_SPARC = 9, /* Big or little endian */ - BCJ_ARM64 = 10 /* AArch64 */ + BCJ_ARM64 = 10, /* AArch64 */ + BCJ_RISCV = 11 /* RV32GQC_Zfh, RV64GQC_Zfh */ } type; /* @@ -163,7 +163,9 @@ static size_t bcj_powerpc(struct xz_dec_bcj *s, uint8_t *buf, size_t size) size_t i; uint32_t instr; - for (i = 0; i + 4 <= size; i += 4) { + size &= ~(size_t)3; + + for (i = 0; i < size; i += 4) { instr = get_unaligned_be32(buf + i); if ((instr & 0xFC000003) == 0x48000001) { instr &= 0x03FFFFFC; @@ -220,7 +222,9 @@ static size_t bcj_ia64(struct xz_dec_bcj *s, uint8_t *buf, size_t size) /* Instruction normalized with bit_res for easier manipulation */ uint64_t norm; - for (i = 0; i + 16 <= size; i += 16) { + size &= ~(size_t)15; + + for (i = 0; i < size; i += 16) { mask = branch_table[buf[i] & 0x1F]; for (slot = 0, bit_pos = 5; slot < 3; ++slot, bit_pos += 41) { if (((mask >> slot) & 1) == 0) @@ -268,7 +272,9 @@ static size_t bcj_arm(struct xz_dec_bcj *s, uint8_t *buf, size_t size) size_t i; uint32_t addr; - for (i = 0; i + 4 <= size; i += 4) { + size &= ~(size_t)3; + + for (i = 0; i < size; i += 4) { if (buf[i + 3] == 0xEB) { addr = (uint32_t)buf[i] | ((uint32_t)buf[i + 1] << 8) | ((uint32_t)buf[i + 2] << 16); @@ -291,7 +297,12 @@ static size_t bcj_armthumb(struct xz_dec_bcj *s, uint8_t *buf, size_t size) size_t i; uint32_t addr; - for (i = 0; i + 4 <= size; i += 2) { + if (size < 4) + return 0; + + size -= 4; + + for (i = 0; i <= size; i += 2) { if ((buf[i + 1] & 0xF8) == 0xF0 && (buf[i + 3] & 0xF8) == 0xF8) { addr = (((uint32_t)buf[i + 1] & 0x07) << 19) @@ -319,7 +330,9 @@ static size_t bcj_sparc(struct xz_dec_bcj *s, uint8_t *buf, size_t size) size_t i; uint32_t instr; - for (i = 0; i + 4 <= size; i += 4) { + size &= ~(size_t)3; + + for (i = 0; i < size; i += 4) { instr = get_unaligned_be32(buf + i); if ((instr >> 22) == 0x100 || (instr >> 22) == 0x1FF) { instr <<= 2; @@ -342,7 +355,9 @@ static size_t bcj_arm64(struct xz_dec_bcj *s, uint8_t *buf, size_t size) uint32_t instr; uint32_t addr; - for (i = 0; i + 4 <= size; i += 4) { + size &= ~(size_t)3; + + for (i = 0; i < size; i += 4) { instr = get_unaligned_le32(buf + i); if ((instr >> 26) == 0x25) { @@ -374,6 +389,99 @@ static size_t bcj_arm64(struct xz_dec_bcj *s, uint8_t *buf, size_t size) } #endif +#ifdef XZ_DEC_RISCV +static size_t bcj_riscv(struct xz_dec_bcj *s, uint8_t *buf, size_t size) +{ + size_t i; + uint32_t b1; + uint32_t b2; + uint32_t b3; + uint32_t instr; + uint32_t instr2; + uint32_t instr2_rs1; + uint32_t addr; + + if (size < 8) + return 0; + + size -= 8; + + for (i = 0; i <= size; i += 2) { + instr = buf[i]; + + if (instr == 0xEF) { + /* JAL */ + b1 = buf[i + 1]; + if ((b1 & 0x0D) != 0) + continue; + + b2 = buf[i + 2]; + b3 = buf[i + 3]; + + addr = ((b1 & 0xF0) << 13) | (b2 << 9) | (b3 << 1); + addr -= s->pos + (uint32_t)i; + + buf[i + 1] = (uint8_t)((b1 & 0x0F) + | ((addr >> 8) & 0xF0)); + + buf[i + 2] = (uint8_t)(((addr >> 16) & 0x0F) + | ((addr >> 7) & 0x10) + | ((addr << 4) & 0xE0)); + + buf[i + 3] = (uint8_t)(((addr >> 4) & 0x7F) + | ((addr >> 13) & 0x80)); + + i += 4 - 2; + + } else if ((instr & 0x7F) == 0x17) { + /* AUIPC */ + instr |= (uint32_t)buf[i + 1] << 8; + instr |= (uint32_t)buf[i + 2] << 16; + instr |= (uint32_t)buf[i + 3] << 24; + + if (instr & 0xE80) { + /* AUIPC's rd doesn't equal x0 or x2. */ + instr2 = get_unaligned_le32(buf + i + 4); + + if (((instr << 8) ^ (instr2 - 3)) & 0xF8003) { + i += 6 - 2; + continue; + } + + addr = (instr & 0xFFFFF000) + (instr2 >> 20); + + instr = 0x17 | (2 << 7) | (instr2 << 12); + instr2 = addr; + } else { + /* AUIPC's rd equals x0 or x2. */ + instr2_rs1 = instr >> 27; + + if ((uint32_t)((instr - 0x3117) << 18) + >= (instr2_rs1 & 0x1D)) { + i += 4 - 2; + continue; + } + + addr = get_unaligned_be32(buf + i + 4); + addr -= s->pos + (uint32_t)i; + + instr2 = (instr >> 12) | (addr << 20); + + instr = 0x17 | (instr2_rs1 << 7) + | ((addr + 0x800) & 0xFFFFF000); + } + + put_unaligned_le32(instr, buf + i); + put_unaligned_le32(instr2, buf + i + 4); + + i += 8 - 2; + } + } + + return i; +} +#endif + /* * Apply the selected BCJ filter. Update *pos and s->pos to match the amount * of data that got filtered. @@ -426,6 +534,11 @@ static void bcj_apply(struct xz_dec_bcj *s, filtered = bcj_arm64(s, buf, size); break; #endif +#ifdef XZ_DEC_RISCV + case BCJ_RISCV: + filtered = bcj_riscv(s, buf, size); + break; +#endif default: /* Never reached but silence compiler warnings. */ filtered = 0; @@ -602,6 +715,9 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id) #ifdef XZ_DEC_ARM64 case BCJ_ARM64: #endif +#ifdef XZ_DEC_RISCV + case BCJ_RISCV: +#endif break; default: diff --git a/linux/lib/xz/xz_dec_lzma2.c b/linux/lib/xz/xz_dec_lzma2.c index c929f1c..613939f 100644 --- a/linux/lib/xz/xz_dec_lzma2.c +++ b/linux/lib/xz/xz_dec_lzma2.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: 0BSD + /* * LZMA2 decoder * * Authors: Lasse Collin <lasse.collin@tukaani.org> * Igor Pavlov <https://7-zip.org/> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include "xz_private.h" @@ -1076,7 +1075,7 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, s->lzma2.sequence = SEQ_LZMA_PREPARE; - /* Fall through */ + fallthrough; case SEQ_LZMA_PREPARE: if (s->lzma2.compressed < RC_INIT_BYTES) @@ -1088,7 +1087,7 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, s->lzma2.compressed -= RC_INIT_BYTES; s->lzma2.sequence = SEQ_LZMA_RUN; - /* Fall through */ + fallthrough; case SEQ_LZMA_RUN: /* diff --git a/linux/lib/xz/xz_dec_stream.c b/linux/lib/xz/xz_dec_stream.c index 2c41f5f..2d5fbe1 100644 --- a/linux/lib/xz/xz_dec_stream.c +++ b/linux/lib/xz/xz_dec_stream.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * .xz Stream decoder * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include "xz_private.h" @@ -605,7 +604,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) if (ret != XZ_OK) return ret; - /* Fall through */ + fallthrough; case SEQ_BLOCK_START: /* We need one byte of input to continue. */ @@ -630,7 +629,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->temp.pos = 0; s->sequence = SEQ_BLOCK_HEADER; - /* Fall through */ + fallthrough; case SEQ_BLOCK_HEADER: if (!fill_temp(s, b)) @@ -642,7 +641,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_BLOCK_UNCOMPRESS; - /* Fall through */ + fallthrough; case SEQ_BLOCK_UNCOMPRESS: ret = dec_block(s, b); @@ -651,7 +650,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_BLOCK_PADDING; - /* Fall through */ + fallthrough; case SEQ_BLOCK_PADDING: /* @@ -673,7 +672,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_BLOCK_CHECK; - /* Fall through */ + fallthrough; case SEQ_BLOCK_CHECK: if (s->check_type == XZ_CHECK_CRC32) { @@ -702,7 +701,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_INDEX_PADDING; - /* Fall through */ + fallthrough; case SEQ_INDEX_PADDING: while ((s->index.size + (b->in_pos - s->in_start)) @@ -726,7 +725,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->sequence = SEQ_INDEX_CRC32; - /* Fall through */ + fallthrough; case SEQ_INDEX_CRC32: ret = crc_validate(s, b, 32); @@ -736,7 +735,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) s->temp.size = STREAM_HEADER_SIZE; s->sequence = SEQ_STREAM_FOOTER; - /* Fall through */ + fallthrough; case SEQ_STREAM_FOOTER: if (!fill_temp(s, b)) diff --git a/linux/lib/xz/xz_dec_syms.c b/linux/lib/xz/xz_dec_syms.c index 61098c6..f40817d 100644 --- a/linux/lib/xz/xz_dec_syms.c +++ b/linux/lib/xz/xz_dec_syms.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * XZ decoder module information * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include <linux/module.h> @@ -23,11 +22,6 @@ EXPORT_SYMBOL(xz_dec_microlzma_end); #endif MODULE_DESCRIPTION("XZ decompressor"); -MODULE_VERSION("1.1"); +MODULE_VERSION("1.2"); MODULE_AUTHOR("Lasse Collin <lasse.collin@tukaani.org> and Igor Pavlov"); - -/* - * This code is in the public domain, but in Linux it's simplest to just - * say it's GPL and consider the authors as the copyright holders. - */ -MODULE_LICENSE("GPL"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/linux/lib/xz/xz_dec_test.c b/linux/lib/xz/xz_dec_test.c index da28a19..53d3600 100644 --- a/linux/lib/xz/xz_dec_test.c +++ b/linux/lib/xz/xz_dec_test.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * XZ decoder tester * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include <linux/kernel.h> @@ -212,9 +211,4 @@ module_exit(xz_dec_test_exit); MODULE_DESCRIPTION("XZ decompressor tester"); MODULE_VERSION("1.0"); MODULE_AUTHOR("Lasse Collin <lasse.collin@tukaani.org>"); - -/* - * This code is in the public domain, but in Linux it's simplest to just - * say it's GPL and consider the authors as the copyright holders. - */ -MODULE_LICENSE("GPL"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/linux/lib/xz/xz_lzma2.h b/linux/lib/xz/xz_lzma2.h index 92d852d..d2632b7 100644 --- a/linux/lib/xz/xz_lzma2.h +++ b/linux/lib/xz/xz_lzma2.h @@ -1,11 +1,10 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * LZMA2 definitions * * Authors: Lasse Collin <lasse.collin@tukaani.org> * Igor Pavlov <https://7-zip.org/> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_LZMA2_H diff --git a/linux/lib/xz/xz_private.h b/linux/lib/xz/xz_private.h index 3acc208..a8b1cbe 100644 --- a/linux/lib/xz/xz_private.h +++ b/linux/lib/xz/xz_private.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Private includes and definitions * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_PRIVATE_H @@ -40,6 +39,9 @@ # ifdef CONFIG_XZ_DEC_ARM64 # define XZ_DEC_ARM64 # endif +# ifdef CONFIG_XZ_DEC_RISCV +# define XZ_DEC_RISCV +# endif # ifdef CONFIG_XZ_DEC_MICROLZMA # define XZ_DEC_MICROLZMA # endif @@ -103,7 +105,8 @@ # if defined(XZ_DEC_X86) || defined(XZ_DEC_POWERPC) \ || defined(XZ_DEC_IA64) \ || defined(XZ_DEC_ARM) || defined(XZ_DEC_ARMTHUMB) \ - || defined(XZ_DEC_SPARC) || defined(XZ_DEC_ARM64) + || defined(XZ_DEC_SPARC) || defined(XZ_DEC_ARM64) \ + || defined(XZ_DEC_RISCV) # define XZ_DEC_BCJ # endif #endif diff --git a/linux/lib/xz/xz_stream.h b/linux/lib/xz/xz_stream.h index e5d3188..55f9f6f 100644 --- a/linux/lib/xz/xz_stream.h +++ b/linux/lib/xz/xz_stream.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Definitions for handling the .xz file format * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_STREAM_H @@ -19,7 +18,7 @@ /* * See the .xz file format specification at - * https://xz.tukaani.org/format/xz-file-format.txt + * https://tukaani.org/xz/xz-file-format.txt * to understand the container format. */ diff --git a/linux/scripts/xz_wrap.sh b/linux/scripts/xz_wrap.sh index 7a2d372..f193696 100755 --- a/linux/scripts/xz_wrap.sh +++ b/linux/scripts/xz_wrap.sh @@ -1,23 +1,162 @@ #!/bin/sh +# SPDX-License-Identifier: 0BSD # # This is a wrapper for xz to compress the kernel image using appropriate # compression options depending on the architecture. # # Author: Lasse Collin <lasse.collin@tukaani.org> + +# This has specialized settings for the following archs. However, +# XZ-compressed kernel isn't currently supported on every listed arch. # -# This file has been put into the public domain. -# You can do whatever you want with this file. -# +# Arch Align Notes +# arm 2/4 ARM and ARM-Thumb2 +# arm64 4 +# csky 2 +# loongarch 4 +# mips 2/4 MicroMIPS is 2-byte aligned +# parisc 4 +# powerpc 4 Uses its own wrapper for compressors instead of this. +# riscv 2/4 +# s390 2 +# sh 2 +# sparc 4 +# x86 1 + +# A few archs use 2-byte or 4-byte aligned instructions depending on +# the kernel config. This function is used to check if the relevant +# config option is set to "y". +is_enabled() +{ + grep -q "^$1=y$" include/config/auto.conf +} + +# XZ_VERSION is needed to disable features that aren't available in +# old XZ Utils versions. +XZ_VERSION=$($XZ --robot --version) || exit +XZ_VERSION=$(printf '%s\n' "$XZ_VERSION" | sed -n 's/^XZ_VERSION=//p') +# Assume that no BCJ filter is available. BCJ= -LZMA2OPTS= +# Set the instruction alignment to 1, 2, or 4 bytes. +# +# Set the BCJ filter if one is available. +# It must match the #ifdef usage in lib/decompress_unxz.c. case $SRCARCH in - x86) BCJ=--x86 ;; - powerpc) BCJ=--powerpc ;; - ia64) BCJ=--ia64; LZMA2OPTS=pb=4 ;; - arm) BCJ=--arm ;; - sparc) BCJ=--sparc ;; + arm) + if is_enabled CONFIG_THUMB2_KERNEL; then + ALIGN=2 + BCJ=--armthumb + else + ALIGN=4 + BCJ=--arm + fi + ;; + + arm64) + ALIGN=4 + + # ARM64 filter was added in XZ Utils 5.4.0. + if [ "$XZ_VERSION" -ge 50040002 ]; then + BCJ=--arm64 + else + echo "$0: Upgrading to xz >= 5.4.0" \ + "would enable the ARM64 filter" \ + "for better compression" >&2 + fi + ;; + + csky) + ALIGN=2 + ;; + + loongarch) + ALIGN=4 + ;; + + mips) + if is_enabled CONFIG_CPU_MICROMIPS; then + ALIGN=2 + else + ALIGN=4 + fi + ;; + + parisc) + ALIGN=4 + ;; + + powerpc) + ALIGN=4 + + # The filter is only for big endian instruction encoding. + if is_enabled CONFIG_CPU_BIG_ENDIAN; then + BCJ=--powerpc + fi + ;; + + riscv) + if is_enabled CONFIG_RISCV_ISA_C; then + ALIGN=2 + else + ALIGN=4 + fi + + # RISC-V filter was added in XZ Utils 5.6.0. + if [ "$XZ_VERSION" -ge 50060002 ]; then + BCJ=--riscv + else + echo "$0: Upgrading to xz >= 5.6.0" \ + "would enable the RISC-V filter" \ + "for better compression" >&2 + fi + ;; + + s390) + ALIGN=2 + ;; + + sh) + ALIGN=2 + ;; + + sparc) + ALIGN=4 + BCJ=--sparc + ;; + + x86) + ALIGN=1 + BCJ=--x86 + ;; + + *) + echo "$0: Arch-specific tuning is missing for '$SRCARCH'" >&2 + + # Guess 2-byte-aligned instructions. Guessing too low + # should hurt less than guessing too high. + ALIGN=2 + ;; +esac + +# Select the LZMA2 options matching the instruction alignment. +case $ALIGN in + 1) LZMA2OPTS= ;; + 2) LZMA2OPTS=lp=1 ;; + 4) LZMA2OPTS=lp=2,lc=2 ;; + *) echo "$0: ALIGN wrong or missing" >&2; exit 1 ;; esac -exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB +# Use single-threaded mode because it compresses a little better +# (and uses less RAM) than multithreaded mode. +# +# For the best compression, the dictionary size shouldn't be +# smaller than the uncompressed kernel. 128 MiB dictionary +# needs less than 1400 MiB of RAM in single-threaded mode. +# +# On the archs that use this script to compress the kernel, +# decompression in the preboot code is done in single-call mode. +# Thus the dictionary size doesn't affect the memory requirements +# of the preboot decompressor at all. +exec $XZ --check=crc32 --threads=1 $BCJ --lzma2=$LZMA2OPTS,dict=128MiB diff --git a/userspace/Makefile b/userspace/Makefile index b48d30c..37a62ba 100644 --- a/userspace/Makefile +++ b/userspace/Makefile @@ -1,20 +1,18 @@ -# -# Makefile +# SPDX-License-Identifier: 0BSD # # Author: Lasse Collin <lasse.collin@tukaani.org> -# -# This file has been put into the public domain. -# You can do whatever you want with this file. -# -# gcc -std=gnu89 is used because Linux uses it. It is fine to omit it as -# the code is also C99/C11 compatible. With clang you may wish to omit -# either -std=gnu89 or -pedantic as having both gives quite a few warnings. -CC = gcc -std=gnu89 +# In 2022, Linux finally moved forward from -std=gnu89 and switched +# to -std=gnu11. In practice quite a few C99/C11 features were used +# for decades that were extensions in GNU-C89. +# +# Use the option -Wdeclaration-after-statement for historical reasons +# for now. +CC = gcc -std=gnu11 BCJ_CPPFLAGS = -DXZ_DEC_X86 -DXZ_DEC_ARM -DXZ_DEC_ARMTHUMB -DXZ_DEC_ARM64 \ - -DXZ_DEC_POWERPC -DXZ_DEC_IA64 -DXZ_DEC_SPARC + -DXZ_DEC_RISCV -DXZ_DEC_POWERPC -DXZ_DEC_IA64 -DXZ_DEC_SPARC CPPFLAGS = -DXZ_USE_CRC64 -DXZ_DEC_ANY_CHECK -DXZ_DEC_CONCATENATED -CFLAGS = -ggdb3 -O2 -pedantic -Wall -Wextra -Wno-long-long +CFLAGS = -ggdb3 -O2 -pedantic -Wall -Wextra -Wdeclaration-after-statement RM = rm -f VPATH = ../linux/include/linux ../linux/lib/xz COMMON_SRCS = xz_crc32.c xz_crc64.c xz_dec_stream.c xz_dec_lzma2.c xz_dec_bcj.c diff --git a/userspace/boottest.c b/userspace/boottest.c index 1aef5ed..068f81c 100644 --- a/userspace/boottest.c +++ b/userspace/boottest.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * Test application for xz_boot.c * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include <stdlib.h> @@ -19,6 +18,12 @@ static void error(/*const*/ char *msg) fprintf(stderr, "%s\n", msg); } +/* + * Disable XZ_UNSUPPORTED_CHECK as it's not used in Linux and thus + * decompress_unxz.c doesn't handle it either (it thinks it's a bug). + */ +#undef XZ_DEC_ANY_CHECK + /* Disable the CRC64 support even if it was enabled in the Makefile. */ #undef XZ_USE_CRC64 @@ -27,12 +32,12 @@ static void error(/*const*/ char *msg) static uint8_t in[1024 * 1024]; static uint8_t out[1024 * 1024]; -static int fill(void *buf, unsigned int size) +static long fill(void *buf, unsigned long size) { return fread(buf, 1, size, stdin); } -static int flush(/*const*/ void *buf, unsigned int size) +static long flush(/*const*/ void *buf, unsigned long size) { return fwrite(buf, 1, size, stdout); } @@ -42,7 +47,8 @@ static void test_buf_to_buf(void) size_t in_size; int ret; in_size = fread(in, 1, sizeof(in), stdin); - ret = decompress(in, in_size, NULL, NULL, out, NULL, &error); + ret = __decompress(in, in_size, NULL, NULL, out, sizeof(out), + NULL, &error); /* fwrite(out, 1, FIXME, stdout); */ fprintf(stderr, "ret = %d\n", ret); } @@ -50,17 +56,19 @@ static void test_buf_to_buf(void) static void test_buf_to_cb(void) { size_t in_size; - int in_used; + long in_used; int ret; in_size = fread(in, 1, sizeof(in), stdin); - ret = decompress(in, in_size, NULL, &flush, NULL, &in_used, &error); - fprintf(stderr, "ret = %d; in_used = %d\n", ret, in_used); + ret = __decompress(in, in_size, NULL, &flush, NULL, sizeof(out), + &in_used, &error); + fprintf(stderr, "ret = %d; in_used = %ld\n", ret, in_used); } static void test_cb_to_cb(void) { int ret; - ret = decompress(NULL, 0, &fill, &flush, NULL, NULL, &error); + ret = __decompress(NULL, 0, &fill, &flush, NULL, sizeof(out), + NULL, &error); fprintf(stderr, "ret = %d\n", ret); } @@ -70,11 +78,12 @@ static void test_cb_to_cb(void) */ static void test_cb_to_buf(void) { - int in_used; + long in_used; int ret; - ret = decompress(in, 0, &fill, NULL, out, &in_used, &error); + ret = __decompress(in, 0, &fill, NULL, out, sizeof(out), + &in_used, &error); /* fwrite(out, 1, FIXME, stdout); */ - fprintf(stderr, "ret = %d; in_used = %d\n", ret, in_used); + fprintf(stderr, "ret = %d; in_used = %ld\n", ret, in_used); } int main(int argc, char **argv) diff --git a/userspace/buftest.c b/userspace/buftest.c index 8714a38..1c8a11f 100644 --- a/userspace/buftest.c +++ b/userspace/buftest.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * Test application to test buffer-to-buffer decoding * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include <stdbool.h> diff --git a/userspace/bytetest.c b/userspace/bytetest.c index 0200d2e..add0208 100644 --- a/userspace/bytetest.c +++ b/userspace/bytetest.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * Lazy test for the case when the output size is known * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include <stdbool.h> diff --git a/userspace/xz_config.h b/userspace/xz_config.h index ac1b6c3..3245be0 100644 --- a/userspace/xz_config.h +++ b/userspace/xz_config.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Private includes and definitions for userspace use of XZ Embedded * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_CONFIG_H @@ -21,23 +20,20 @@ /* #define XZ_DEC_ARM */ /* #define XZ_DEC_ARMTHUMB */ /* #define XZ_DEC_ARM64 */ +/* #define XZ_DEC_RISCV */ /* #define XZ_DEC_POWERPC */ /* #define XZ_DEC_IA64 */ /* #define XZ_DEC_SPARC */ /* - * MSVC doesn't support modern C but XZ Embedded is mostly C89 - * so these are enough. + * Visual Studio 2013 update 2 supports only __inline, not inline. + * MSVC v19.0 / VS 2015 and newer support both. */ -#ifdef _MSC_VER -typedef unsigned char bool; -# define true 1 -# define false 0 +#if defined(_MSC_VER) && _MSC_VER < 1900 && !defined(inline) # define inline __inline -#else -# include <stdbool.h> #endif +#include <stdbool.h> #include <stdlib.h> #include <string.h> @@ -56,6 +52,16 @@ typedef unsigned char bool; #endif #define min_t(type, x, y) min(x, y) +#ifndef fallthrough +# if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000 +# define fallthrough [[fallthrough]] +# elif defined(__GNUC__) && __GNUC__ >= 7 +# define fallthrough __attribute__((__fallthrough__)) +# else +# define fallthrough do {} while (0) +# endif +#endif + /* * Some functions have been marked with __always_inline to keep the * performance reasonable even when the compiler is optimizing for diff --git a/userspace/xzminidec.c b/userspace/xzminidec.c index b542109..8b9b2ef 100644 --- a/userspace/xzminidec.c +++ b/userspace/xzminidec.c @@ -1,17 +1,19 @@ +// SPDX-License-Identifier: 0BSD + /* * Simple XZ decoder command line tool * * Author: Lasse Collin <lasse.collin@tukaani.org> - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ /* - * This is really limited: Not all filters from .xz format are supported, - * only CRC32 is supported as the integrity check, and decoding of - * concatenated .xz streams is not supported. Thus, you may want to look - * at xzdec from XZ Utils if a few KiB bigger tool is not a problem. + * This is a very limited .xz decoder. Only LZMA2 and the BCJ filters + * are supported, and the BCJ filters cannot use Filter Properties. + * SHA256 is not supported as an integrity check. The LZMA2 dictionary + * sizes can be at most 64 MiB, but this can be modified by changing + * DICT_SIZE_MAX. + * + * See xzdec from XZ Utils if a few KiB bigger tool is not a problem. */ #include <stdbool.h> @@ -19,6 +21,10 @@ #include <string.h> #include "xz.h" +#ifndef DICT_SIZE_MAX +# define DICT_SIZE_MAX (64U << 20) +#endif + static uint8_t in[BUFSIZ]; static uint8_t out[BUFSIZ]; @@ -45,7 +51,7 @@ int main(int argc, char **argv) * Support up to 64 MiB dictionary. The actually needed memory * is allocated once the headers have been parsed. */ - s = xz_dec_init(XZ_DYNALLOC, 1 << 26); + s = xz_dec_init(XZ_DYNALLOC, DICT_SIZE_MAX); if (s == NULL) { msg = "Memory allocation failed\n"; goto error; |