From 6f4f4d9733ddfdeef76ecd96e555b065ec493d93 Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Fri, 16 Apr 2021 17:55:30 +0800 Subject: [PATCH] Move string compression to Python ... so that the script can know its uncompressed size and calculate the appropriate buffer size. --- Translations/lzfx.py | 91 +++++++++++++++++++++++++++ Translations/make_translation.py | 25 +++++--- source/Core/lzfx/lzfx-host-compress.c | 48 -------------- source/Makefile | 15 ++--- 4 files changed, 111 insertions(+), 68 deletions(-) create mode 100644 Translations/lzfx.py delete mode 100644 source/Core/lzfx/lzfx-host-compress.c diff --git a/Translations/lzfx.py b/Translations/lzfx.py new file mode 100644 index 00000000..9f389673 --- /dev/null +++ b/Translations/lzfx.py @@ -0,0 +1,91 @@ +import ctypes +import functools +import os +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +@functools.lru_cache(maxsize=None) +def _liblzfx(): + so_path = os.path.join(HERE, "../source/Objects/host/lzfx/liblzfx.so") + liblzfx = ctypes.cdll.LoadLibrary(so_path) + return liblzfx + + +@functools.lru_cache(maxsize=None) +def _fn_lzfx_compress(): + """Returns the lzfx_compress C function. + :: + + /* Buffer-to buffer compression. + + Supply pre-allocated input and output buffers via ibuf and obuf, and + their size in bytes via ilen and olen. Buffers may not overlap. + + On success, the function returns a non-negative value and the argument + olen contains the compressed size in bytes. On failure, a negative + value is returned and olen is not modified. + */ + int lzfx_compress(const void* ibuf, unsigned int ilen, + void* obuf, unsigned int *olen); + """ + + fn = _liblzfx().lzfx_compress + fn.argtype = [ + ctypes.c_char_p, + ctypes.c_uint, + ctypes.c_char_p, + ctypes.POINTER(ctypes.c_uint), + ] + fn.restype = ctypes.c_int + return fn + + +def compress(data: bytes) -> bytes: + """Returns a bytes object of the lzfx-compressed data.""" + + fn_compress = _fn_lzfx_compress() + + output_buffer_len = len(data) + 8 + + ibuf = data + ilen = len(ibuf) + obuf = ctypes.create_string_buffer(output_buffer_len) + olen = ctypes.c_uint(output_buffer_len) + + res = fn_compress(ibuf, ilen, obuf, ctypes.byref(olen)) + + if res < 0: + raise LzfxError(res) + else: + return bytes(obuf[: olen.value]) # type: ignore + + +class LzfxError(Exception): + """Exception raised for lzfx compression or decompression error. + + Attributes: + error_code -- The source error code, which is a negative integer + error_name -- The constant name of the error + message -- explanation of the error + """ + + # define LZFX_ESIZE -1 /* Output buffer too small */ + # define LZFX_ECORRUPT -2 /* Invalid data for decompression */ + # define LZFX_EARGS -3 /* Arguments invalid (NULL) */ + + def __init__(self, error_code): + self.error_code = error_code + if error_code == -1: + self.error_name = "LZFX_ESIZE" + self.message = "Output buffer too small" + elif error_code == -2: + self.error_name = "LZFX_ECORRUPT" + self.message = "Invalid data for decompression" + elif error_code == -3: + self.error_name = "LZFX_EARGS" + self.message = "Arguments invalid (NULL)" + else: + self.error_name = "UNKNOWN" + self.message = "Unknown error" diff --git a/Translations/make_translation.py b/Translations/make_translation.py index 4e25ee71..c4a2c60d 100755 --- a/Translations/make_translation.py +++ b/Translations/make_translation.py @@ -19,6 +19,7 @@ from bdflib import reader as bdfreader from bdflib.model import Font, Glyph import font_tables +import lzfx logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) @@ -505,7 +506,7 @@ def prepare_language(lang: dict, defs: dict, build_version: str) -> LanguageData def write_language( - data: LanguageData, f: TextIO, lzfx_strings: Optional[bytes] = None + data: LanguageData, f: TextIO, strings_bin: Optional[bytes] = None ) -> None: lang = data.lang defs = data.defs @@ -523,7 +524,7 @@ def write_language( except KeyError: lang_name = language_code - if lzfx_strings: + if strings_bin: f.write('#include "lzfx.h"\n') f.write(f"\n// ---- {lang_name} ----\n\n") @@ -540,7 +541,7 @@ def write_language( "extern const uint8_t *const Font_6x8 = USER_FONT_6x8;\n\n" ) - if not lzfx_strings: + if not strings_bin: translation_strings_and_indices_text = get_translation_strings_and_indices_text( lang, defs, symbol_conversion_table ) @@ -551,9 +552,13 @@ def write_language( "void prepareTranslations() {}\n\n" ) else: - write_bytes_as_c_array(f, "translation_data_lzfx", lzfx_strings) + compressed = lzfx.compress(strings_bin) + logging.info( + f"Strings compressed from {len(strings_bin)} to {len(compressed)} bytes (ratio {len(compressed) / len(strings_bin):.3})" + ) + write_bytes_as_c_array(f, "translation_data_lzfx", compressed) f.write( - "static uint8_t translation_data_out_buffer[4096] __attribute__((__aligned__(2)));\n\n" + f"static uint8_t translation_data_out_buffer[{len(strings_bin)}] __attribute__((__aligned__(2)));\n\n" "const TranslationIndexTable *const Tr = reinterpret_cast(translation_data_out_buffer);\n" "const char *const TranslationStrings = reinterpret_cast(translation_data_out_buffer) + sizeof(TranslationIndexTable);\n\n" "void prepareTranslations() {\n" @@ -847,11 +852,11 @@ def parse_args() -> argparse.Namespace: dest="input_pickled", ) parser.add_argument( - "--lzfx-strings", - help="Use compressed TranslationIndices + TranslationStrings data", + "--strings-bin", + help="Use generated TranslationIndices + TranslationStrings data and compress them", type=argparse.FileType("rb"), required=False, - dest="lzfx_strings", + dest="strings_bin", ) parser.add_argument( "--output", "-o", help="Target file", type=argparse.FileType("w"), required=True @@ -895,8 +900,8 @@ def main() -> None: out_ = args.output write_start(out_) - if args.lzfx_strings: - write_language(language_data, out_, args.lzfx_strings.read()) + if args.strings_bin: + write_language(language_data, out_, args.strings_bin.read()) else: write_language(language_data, out_) diff --git a/source/Core/lzfx/lzfx-host-compress.c b/source/Core/lzfx/lzfx-host-compress.c deleted file mode 100644 index b135b975..00000000 --- a/source/Core/lzfx/lzfx-host-compress.c +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include - -#include "lzfx.h" - -/* Program to demonstrate file compression. Don't use it in the real world! */ - -int main(int argc, char **argv) { - if (argc == 3) { - /* open input */ - FILE *f = fopen(argv[1], "rb"); - if (!f) { - printf("Error: %s\n", argv[1]); - return 1; - } - - /* get size */ - fseek(f, 0, SEEK_END); - int inputsize = ftell(f); - fseek(f, 0, SEEK_SET); - - /* read */ - char *input = malloc(inputsize); - fread(input, inputsize, 1, f); - fclose(f); - - /* compress */ - int outputsize = inputsize + 1; /* buffer overflow */ - char *output = malloc(outputsize); - lzfx_compress(input, inputsize, output, &outputsize); - - /* open output */ - f = fopen(argv[2], "wb"); - if (!f) { - printf("Error: %s\n", argv[1]); - return 1; - } - - /* write */ - fwrite(output, outputsize, 1, f); - fclose(f); - - return 0; - } else { - printf("Compresses a file.\n\nUsage: lzfx-raw input output\n"); - return 1; - } -} diff --git a/source/Makefile b/source/Makefile index dcd94972..0bca7726 100644 --- a/source/Makefile +++ b/source/Makefile @@ -364,10 +364,10 @@ $(OUTPUT_DIR)/Core/Gen/translation.files/%.o: Core/Gen/Translation.%.cpp @echo Generating $@ @$(CPP) -c $(filter-out -flto -g3,$(CXXFLAGS)) $< -o $@ -$(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress: Core/lzfx/lzfx-host-compress.c Core/lzfx/lzfx.c +$(HOST_OUTPUT_DIR)/lzfx/liblzfx.so: Core/lzfx/lzfx.c @test -d $(@D) || mkdir -p $(@D) - @echo Building host lzfx tool $@ - @$(HOST_CC) -Wno-unused-result -O $^ -o $@ + @echo Building host lzfx shared library $@ + @$(HOST_CC) -Wno-unused-result -fPIC -shared -O $^ -o $@ $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin: $(OUTPUT_DIR)/Core/Gen/translation.files/%.o @echo Dumping translation strings data from $< @@ -378,18 +378,13 @@ $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin: $(OUTPUT_DIR)/Core/Gen/t @test -s $(@D)/$*.data.TranslationStrings.bin || (rm $(@D)/$*.data.TranslationStrings.bin; echo 'ERROR: Output for .rodata._ZL22TranslationStringsData is empty!' >&2; false) @cat $(@D)/$*.data.TranslationIndices.bin $(@D)/$*.data.TranslationStrings.bin > $@ -$(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.lzfx: $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin $(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress - @echo Compressing translation strings data for $* - @$(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress $< $@ - @echo Compressed from $$(stat --printf="%s" $<) to $$(stat --printf="%s" $@) bytes - -Core/Gen/Translation_lzfx.%.cpp: $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.lzfx $(OUTPUT_DIR)/Core/Gen/translation.files/%.pickle +Core/Gen/Translation_lzfx.%.cpp: $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin $(OUTPUT_DIR)/Core/Gen/translation.files/%.pickle $(HOST_OUTPUT_DIR)/lzfx/liblzfx.so @test -d $(@D) || mkdir -p $(@D) @echo Generating lzfx compressed translation for $* @python3 ../Translations/make_translation.py \ -o $(PWD)/Core/Gen/Translation_lzfx.$*.cpp \ --input-pickled $(OUTPUT_DIR)/Core/Gen/translation.files/$*.pickle \ - --lzfx-strings $(OUTPUT_DIR)/Core/Gen/translation.files/$*.strings.lzfx \ + --strings-bin $(OUTPUT_DIR)/Core/Gen/translation.files/$*.strings.bin \ $*