diff --git a/AUTHORS b/AUTHORS index ed2988d66e4a7..19a3acd0943a1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -529,4 +529,5 @@ a license to everyone to use it as detailed in LICENSE.) * Max Brunsfeld * Basil Fierz * Rod Hyde -* Aleksey Kliger (copyright owned by Microsoft, Inc.) \ No newline at end of file +* Aleksey Kliger (copyright owned by Microsoft, Inc.) +* Miłosz Rachwał diff --git a/ChangeLog.md b/ChangeLog.md index 207fa5cb6c07c..b154c0578788f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -20,6 +20,7 @@ See docs/process.md for more on how version tagging works. Current Trunk ------------- +- Add port for liblzma library (`liblzma.a`). (#12990) 2.0.10: 12/04/2020 ------------------ diff --git a/embuilder.py b/embuilder.py index 818262190ed9e..ab7d5adb7b2d7 100755 --- a/embuilder.py +++ b/embuilder.py @@ -60,6 +60,7 @@ 'harfbuzz', 'icu', 'libjpeg', + 'liblzma', 'libpng', 'ogg', 'regal', @@ -197,6 +198,8 @@ def main(): build_port('ogg', libname('libogg')) elif what == 'libjpeg': build_port('libjpeg', libname('libjpeg')) + elif what == 'liblzma': + build_port('liblzma', libname('liblzma')) elif what == 'libpng': build_port('libpng', libname('libpng')) elif what == 'sdl2': diff --git a/src/settings.js b/src/settings.js index 61cd98939baf1..be6fcb678c6a2 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1197,6 +1197,9 @@ var USE_BZIP2 = 0; // 1 = use libjpeg from emscripten-ports var USE_LIBJPEG = 0; +// 1 = use liblzma from emscripten-ports +var USE_LIBLZMA = 0; + // 1 = use libpng from emscripten-ports var USE_LIBPNG = 0; diff --git a/tests/test_other.py b/tests/test_other.py index 3a4567bb3dc74..dd6a76548de92 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1481,6 +1481,11 @@ def test_libjpeg(self): building.emcc(path_from_root('tests', 'jpeg_test.c'), ['--embed-file', 'screenshot.jpg', '-s', 'USE_LIBJPEG=1'], output_filename='a.out.js') self.assertContained('Image is 600 by 450 with 3 components', self.run_js('a.out.js', args=['screenshot.jpg'])) + def test_liblzma(self): + shutil.copyfile(path_from_root('tests', 'third_party', 'liblzma', 'liblzma_test.c.xz'), 'liblzma_test.c.xz') + building.emcc(path_from_root('tests', 'third_party', 'liblzma', 'liblzma_test.c'), ['--embed-file', 'liblzma_test.c.xz', '-s', 'USE_LIBLZMA=1'], output_filename='a.out.js') + self.assertContained('c3d55d80', self.run_js('a.out.js', args=['liblzma_test.c.xz'])) + def test_bullet(self): building.emcc(path_from_root('tests', 'bullet_hello_world.cpp'), ['-s', 'USE_BULLET=1'], output_filename='a.out.js') self.assertContained('BULLET RUNNING', self.run_process(config.JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) diff --git a/tests/third_party/liblzma/liblzma_test.c b/tests/third_party/liblzma/liblzma_test.c new file mode 100644 index 0000000000000..5b336a12513db --- /dev/null +++ b/tests/third_party/liblzma/liblzma_test.c @@ -0,0 +1,283 @@ +// simple liblzma test program adapted from xz/doc/examples/02_decompress.c +// original author: Lasse Collin + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file liblzma_test.c +/// \brief Decompress .xz files and output crc32 to stdout +/// +/// Usage: ./lzma_test INPUT_FILES... +/// +/// Example: ./lzma_test foo.xz +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include + + +static bool +init_decoder(lzma_stream *strm) +{ + // Initialize a .xz decoder. The decoder supports a memory usage limit + // and a set of flags. + // + // The memory usage of the decompressor depends on the settings used + // to compress a .xz file. It can vary from less than a megabyte to + // a few gigabytes, but in practice (at least for now) it rarely + // exceeds 65 MiB because that's how much memory is required to + // decompress files created with "xz -9". Settings requiring more + // memory take extra effort to use and don't (at least for now) + // provide significantly better compression in most cases. + // + // Memory usage limit is useful if it is important that the + // decompressor won't consume gigabytes of memory. The need + // for limiting depends on the application. In this example, + // no memory usage limiting is used. This is done by setting + // the limit to UINT64_MAX. + // + // The .xz format allows concatenating compressed files as is: + // + // echo foo | xz > foobar.xz + // echo bar | xz >> foobar.xz + // + // When decompressing normal standalone .xz files, LZMA_CONCATENATED + // should always be used to support decompression of concatenated + // .xz files. If LZMA_CONCATENATED isn't used, the decoder will stop + // after the first .xz stream. This can be useful when .xz data has + // been embedded inside another file format. + // + // Flags other than LZMA_CONCATENATED are supported too, and can + // be combined with bitwise-or. See lzma/container.h + // (src/liblzma/api/lzma/container.h in the source package or e.g. + // /usr/include/lzma/container.h depending on the install prefix) + // for details. + lzma_ret ret = lzma_stream_decoder( + strm, UINT64_MAX, LZMA_CONCATENATED); + + // Return successfully if the initialization went fine. + if (ret == LZMA_OK) + return true; + + // Something went wrong. The possible errors are documented in + // lzma/container.h (src/liblzma/api/lzma/container.h in the source + // package or e.g. /usr/include/lzma/container.h depending on the + // install prefix). + // + // Note that LZMA_MEMLIMIT_ERROR is never possible here. If you + // specify a very tiny limit, the error will be delayed until + // the first headers have been parsed by a call to lzma_code(). + const char *msg; + switch (ret) { + case LZMA_MEM_ERROR: + msg = "Memory allocation failed"; + break; + + case LZMA_OPTIONS_ERROR: + msg = "Unsupported decompressor flags"; + break; + + default: + // This is most likely LZMA_PROG_ERROR indicating a bug in + // this program or in liblzma. It is inconvenient to have a + // separate error message for errors that should be impossible + // to occur, but knowing the error code is important for + // debugging. That's why it is good to print the error code + // at least when there is no good error message to show. + msg = "Unknown error, possibly a bug"; + break; + } + + fprintf(stderr, "Error initializing the decoder: %s (error code %u)\n", + msg, ret); + return false; +} + + +static bool +decompress(lzma_stream *strm, const char *inname, FILE *infile, uint32_t *crc) +{ + // When LZMA_CONCATENATED flag was used when initializing the decoder, + // we need to tell lzma_code() when there will be no more input. + // This is done by setting action to LZMA_FINISH instead of LZMA_RUN + // in the same way as it is done when encoding. + // + // When LZMA_CONCATENATED isn't used, there is no need to use + // LZMA_FINISH to tell when all the input has been read, but it + // is still OK to use it if you want. When LZMA_CONCATENATED isn't + // used, the decoder will stop after the first .xz stream. In that + // case some unused data may be left in strm->next_in. + lzma_action action = LZMA_RUN; + + uint8_t inbuf[BUFSIZ]; + uint8_t outbuf[BUFSIZ]; + + strm->next_in = NULL; + strm->avail_in = 0; + strm->next_out = outbuf; + strm->avail_out = sizeof(outbuf); + + while (true) { + if (strm->avail_in == 0 && !feof(infile)) { + strm->next_in = inbuf; + strm->avail_in = fread(inbuf, 1, sizeof(inbuf), + infile); + + if (ferror(infile)) { + fprintf(stderr, "%s: Read error: %s\n", + inname, strerror(errno)); + return false; + } + + // Once the end of the input file has been reached, + // we need to tell lzma_code() that no more input + // will be coming. As said before, this isn't required + // if the LZMA_CONCATENATED flag isn't used when + // initializing the decoder. + if (feof(infile)) + action = LZMA_FINISH; + } + + lzma_ret ret = lzma_code(strm, action); + + if (strm->avail_out == 0 || ret == LZMA_STREAM_END) { + size_t write_size = sizeof(outbuf) - strm->avail_out; + + *crc = lzma_crc32(outbuf, write_size, *crc); + + strm->next_out = outbuf; + strm->avail_out = sizeof(outbuf); + } + + if (ret != LZMA_OK) { + // Once everything has been decoded successfully, the + // return value of lzma_code() will be LZMA_STREAM_END. + // + // It is important to check for LZMA_STREAM_END. Do not + // assume that getting ret != LZMA_OK would mean that + // everything has gone well or that when you aren't + // getting more output it must have successfully + // decoded everything. + if (ret == LZMA_STREAM_END) + return true; + + // It's not LZMA_OK nor LZMA_STREAM_END, + // so it must be an error code. See lzma/base.h + // (src/liblzma/api/lzma/base.h in the source package + // or e.g. /usr/include/lzma/base.h depending on the + // install prefix) for the list and documentation of + // possible values. Many values listen in lzma_ret + // enumeration aren't possible in this example, but + // can be made possible by enabling memory usage limit + // or adding flags to the decoder initialization. + const char *msg; + switch (ret) { + case LZMA_MEM_ERROR: + msg = "Memory allocation failed"; + break; + + case LZMA_FORMAT_ERROR: + // .xz magic bytes weren't found. + msg = "The input is not in the .xz format"; + break; + + case LZMA_OPTIONS_ERROR: + // For example, the headers specify a filter + // that isn't supported by this liblzma + // version (or it hasn't been enabled when + // building liblzma, but no-one sane does + // that unless building liblzma for an + // embedded system). Upgrading to a newer + // liblzma might help. + // + // Note that it is unlikely that the file has + // accidentally became corrupt if you get this + // error. The integrity of the .xz headers is + // always verified with a CRC32, so + // unintentionally corrupt files can be + // distinguished from unsupported files. + msg = "Unsupported compression options"; + break; + + case LZMA_DATA_ERROR: + msg = "Compressed file is corrupt"; + break; + + case LZMA_BUF_ERROR: + // Typically this error means that a valid + // file has got truncated, but it might also + // be a damaged part in the file that makes + // the decoder think the file is truncated. + // If you prefer, you can use the same error + // message for this as for LZMA_DATA_ERROR. + msg = "Compressed file is truncated or " + "otherwise corrupt"; + break; + + default: + // This is most likely LZMA_PROG_ERROR. + msg = "Unknown error, possibly a bug"; + break; + } + + fprintf(stderr, "%s: Decoder error: " + "%s (error code %u)\n", + inname, msg, ret); + return false; + } + } +} + + +extern int +main(int argc, char **argv) +{ + if (argc <= 1) { + fprintf(stderr, "Usage: %s FILES...\n", argv[0]); + return EXIT_FAILURE; + } + + lzma_stream strm = LZMA_STREAM_INIT; + + bool success = true; + + uint32_t crc = 0; + + // Try to decompress all files. + for (int i = 1; i < argc; ++i) { + if (!init_decoder(&strm)) { + // Decoder initialization failed. There's no point + // to retry it so we need to exit. + success = false; + break; + } + + FILE *infile = fopen(argv[i], "rb"); + + if (infile == NULL) { + fprintf(stderr, "%s: Error opening the " + "input file: %s\n", + argv[i], strerror(errno)); + success = false; + } else { + success &= decompress(&strm, argv[i], infile, &crc); + fclose(infile); + } + } + + // Free the memory allocated for the decoder. This only needs to be + // done after the last file. + lzma_end(&strm); + + printf("%08x\n", crc); + + return success ? EXIT_SUCCESS : EXIT_FAILURE; +} + diff --git a/tests/third_party/liblzma/liblzma_test.c.xz b/tests/third_party/liblzma/liblzma_test.c.xz new file mode 100644 index 0000000000000..961fba32770a2 Binary files /dev/null and b/tests/third_party/liblzma/liblzma_test.c.xz differ diff --git a/tools/ports/liblzma.py b/tools/ports/liblzma.py new file mode 100644 index 0000000000000..e9567ef36ffa2 --- /dev/null +++ b/tools/ports/liblzma.py @@ -0,0 +1,160 @@ +# Copyright 2020 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import os +import shutil + +VERSION = '5.2.5' +HASH = '7443674247deda2935220fbc4dfc7665e5bb5a260be8ad858c8bd7d7b9f0f868f04ea45e62eb17c0a5e6a2de7c7500ad2d201e2d668c48ca29bd9eea5a73a3ce' + + +def needed(settings): + return settings.USE_LIBLZMA + + +def get(ports, settings, shared): + libname = ports.get_lib_name('liblzma') + ports.fetch_project('liblzma', 'https://tukaani.org/xz/xz-' + VERSION + '.tar.gz', 'xz-' + VERSION, sha512hash=HASH) + + def create(): + ports.clear_project_build('liblzma') + + source_path = os.path.join(ports.get_dir(), 'liblzma', 'xz-' + VERSION) + dest_path = os.path.join(ports.get_build_dir(), 'liblzma') + + shared.try_delete(dest_path) + os.makedirs(dest_path) + shutil.rmtree(dest_path, ignore_errors=True) + shutil.copytree(source_path, dest_path) + + build_flags = ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_', '-fvisibility=hidden'] + exclude_dirs = ['xzdec', 'xz', 'lzmainfo'] + exclude_files = ['crc32_small.c', 'crc64_small.c', 'crc32_tablegen.c', 'crc64_tablegen.c', 'price_tablegen.c', 'fastpos_tablegen.c' + 'tuklib_exit.c', 'tuklib_mbstr_fw.c', 'tuklib_mbstr_width.c', 'tuklib_open_stdxxx.c', 'tuklib_progname.c'] + include_dirs_rel = ['../common', 'api', 'common', 'check', 'lz', 'rangecoder', 'lzma', 'delta', 'simple'] + + open(os.path.join(dest_path, 'src', 'config.h'), 'w').write(config_h) + + final = os.path.join(dest_path, libname) + include_dirs = [os.path.join(dest_path, 'src', 'liblzma', p) for p in include_dirs_rel] + ports.build_port(os.path.join(dest_path, 'src'), final, flags=build_flags, exclude_dirs=exclude_dirs, exclude_files=exclude_files, includes=include_dirs) + + ports.install_headers(os.path.join(dest_path, 'src', 'liblzma', 'api'), 'lzma.h') + ports.install_headers(os.path.join(dest_path, 'src', 'liblzma', 'api', 'lzma'), '*.h', 'lzma') + + return final + + return [shared.Cache.get(libname, create, what='port')] + + +def clear(ports, settings, shared): + shared.Cache.erase_file(ports.get_lib_name('liblzma')) + + +def process_args(ports): + return [] + + +def show(): + return 'liblzma (USE_LIBLZMA=1; public domain)' + + +config_h = r''' +#define ASSUME_RAM 128 +#define ENABLE_NLS 1 +#define HAVE_CHECK_CRC32 1 +#define HAVE_CHECK_CRC64 1 +#define HAVE_CHECK_SHA256 1 +#define HAVE_CLOCK_GETTIME 1 +#define HAVE_DCGETTEXT 1 +#define HAVE_DECL_CLOCK_MONOTONIC 1 +#define HAVE_DECL_PROGRAM_INVOCATION_NAME 1 +#define HAVE_DECODERS 1 +#define HAVE_DECODER_ARM 1 +#define HAVE_DECODER_ARMTHUMB 1 +#define HAVE_DECODER_DELTA 1 +#define HAVE_DECODER_IA64 1 +#define HAVE_DECODER_LZMA1 1 +#define HAVE_DECODER_LZMA2 1 +#define HAVE_DECODER_POWERPC 1 +#define HAVE_DECODER_SPARC 1 +#define HAVE_DECODER_X86 1 +#define HAVE_DLFCN_H 1 +#define HAVE_ENCODERS 1 +#define HAVE_ENCODER_ARM 1 +#define HAVE_ENCODER_ARMTHUMB 1 +#define HAVE_ENCODER_DELTA 1 +#define HAVE_ENCODER_IA64 1 +#define HAVE_ENCODER_LZMA1 1 +#define HAVE_ENCODER_LZMA2 1 +#define HAVE_ENCODER_POWERPC 1 +#define HAVE_ENCODER_SPARC 1 +#define HAVE_ENCODER_X86 1 +#define HAVE_FCNTL_H 1 +#define HAVE_FUTIMENS 1 +#define HAVE_GETOPT_H 1 +#define HAVE_GETOPT_LONG 1 +#define HAVE_GETTEXT 1 +#define HAVE_IMMINTRIN_H 1 +#define HAVE_INTTYPES_H 1 +#define HAVE_LIMITS_H 1 +#define HAVE_MBRTOWC 1 +#define HAVE_MEMORY_H 1 +#define HAVE_MF_BT2 1 +#define HAVE_MF_BT3 1 +#define HAVE_MF_BT4 1 +#define HAVE_MF_HC3 1 +#define HAVE_MF_HC4 1 +#define HAVE_OPTRESET 1 +#define HAVE_POSIX_FADVISE 1 +#define HAVE_PTHREAD_CONDATTR_SETCLOCK 1 +#define HAVE_PTHREAD_PRIO_INHERIT 1 +#define HAVE_STDBOOL_H 1 +#define HAVE_STDINT_H 1 +#define HAVE_STDLIB_H 1 +#define HAVE_STRINGS_H 1 +#define HAVE_STRING_H 1 +#define HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC 1 +#define HAVE_SYS_PARAM_H 1 +#define HAVE_SYS_STAT_H 1 +#define HAVE_SYS_TIME_H 1 +#define HAVE_SYS_TYPES_H 1 +#define HAVE_UINTPTR_T 1 +#define HAVE_UNISTD_H 1 +#define HAVE_VISIBILITY 1 +#define HAVE_WCWIDTH 1 +#define HAVE__BOOL 1 +#define HAVE___BUILTIN_ASSUME_ALIGNED 1 +#define HAVE___BUILTIN_BSWAPXX 1 +#define MYTHREAD_POSIX 1 +#define NDEBUG 1 +#define PACKAGE "xz" +#define PACKAGE_BUGREPORT "lasse.collin@tukaani.org" +#define PACKAGE_NAME "XZ Utils" +#define PACKAGE_STRING "XZ Utils 5.2.5" +#define PACKAGE_TARNAME "xz" +#define PACKAGE_VERSION "5.2.5" +#define SIZEOF_SIZE_T 4 +#define STDC_HEADERS 1 +#define TUKLIB_CPUCORES_SYSCONF 1 +#define TUKLIB_FAST_UNALIGNED_ACCESS 1 +#define TUKLIB_PHYSMEM_SYSCONF 1 +#ifndef _ALL_SOURCE +# define _ALL_SOURCE 1 +#endif +#ifndef _GNU_SOURCE +# define _GNU_SOURCE 1 +#endif +#ifndef _POSIX_PTHREAD_SEMANTICS +# define _POSIX_PTHREAD_SEMANTICS 1 +#endif +#ifndef _TANDEM_SOURCE +# define _TANDEM_SOURCE 1 +#endif +#ifndef __EXTENSIONS__ +# define __EXTENSIONS__ 1 +#endif +#define VERSION "5.2.5" +'''