From d53245f8a3efb6194e94c1aa2c08d251d2a3b4fb Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Sat, 10 Apr 2021 19:40:33 +0800 Subject: [PATCH 1/7] Simplify some Python code --- Translations/make_translation.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Translations/make_translation.py b/Translations/make_translation.py index 296fc9a3..fc3ebbd0 100755 --- a/Translations/make_translation.py +++ b/Translations/make_translation.py @@ -608,14 +608,11 @@ def write_language(lang: dict, defs: dict, f: TextIO) -> None: f" // {offset + remapped.str_start_offset: >4}: {escape(str_table[j])}\n" ) str_offsets[j] = offset + remapped.str_start_offset - converted_str = convert_string(symbol_conversion_table, source_str) - f.write(f' "{converted_str}"') + converted_bytes = convert_string_bytes(symbol_conversion_table, source_str) + f.write(f' "{bytes_to_escaped(converted_bytes)}"') str_offsets[i] = offset - # Sanity check: Each "char" in `converted_str` should be in format - # `\xFF`, so the length should be divisible by 4. - assert len(converted_str) % 4 == 0 # Add the length and the null terminator - offset += len(converted_str) // 4 + 1 + offset += len(converted_bytes) + 1 f.write("\n};\n\n") def get_offset(idx: int) -> int: From 570fa16d442587b4eb1d9926fa2da64a766b28d6 Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Sun, 11 Apr 2021 00:05:06 +0800 Subject: [PATCH 2/7] Some refactoring of make_translation.py --- Translations/make_translation.py | 206 ++++++++++++++++++------------- 1 file changed, 119 insertions(+), 87 deletions(-) diff --git a/Translations/make_translation.py b/Translations/make_translation.py index fc3ebbd0..a05643a6 100755 --- a/Translations/make_translation.py +++ b/Translations/make_translation.py @@ -79,7 +79,7 @@ def write_start(f: TextIO): f.write('#include "Translation.h"\n') -def get_constants() -> List[Tuple[str, str]]: +def get_constants(build_version: str) -> List[Tuple[str, str]]: # Extra constants that are used in the firmware that are shared across all languages return [ ("SymbolPlus", "+"), @@ -94,7 +94,7 @@ def get_constants() -> List[Tuple[str, str]]: ("SymbolVolts", "V"), ("SymbolDC", "DC"), ("SymbolCellCount", "S"), - ("SymbolVersionNumber", buildVersion), + ("SymbolVersionNumber", build_version), ] @@ -116,7 +116,7 @@ def get_debug_menu() -> List[str]: ] -def get_letter_counts(defs: dict, lang: dict) -> List[str]: +def get_letter_counts(defs: dict, lang: dict, build_version: str) -> List[str]: text_list = [] # iterate over all strings obj = lang["menuOptions"] @@ -169,7 +169,7 @@ def get_letter_counts(defs: dict, lang: dict) -> List[str]: for mod in defs["menuGroups"]: eid = mod["id"] text_list.append(obj[eid]["desc"]) - constants = get_constants() + constants = get_constants(build_version) for x in constants: text_list.append(x[1]) text_list.extend(get_debug_menu()) @@ -290,7 +290,15 @@ def bytes_to_escaped(b: bytes) -> str: return "".join((f"\\x{i:02X}" for i in b)) -def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, bytes]]: +@dataclass +class FontMap: + font12: Dict[str, str] + font06: Dict[str, str] + + +def get_font_map_and_table( + text_list: List[str], +) -> Tuple[List[str], FontMap, Dict[str, bytes]]: # the text list is sorted # allocate out these in their order as number codes symbol_map: Dict[str, bytes] = {"\n": bytes([1])} @@ -324,56 +332,56 @@ def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, bytes]] logging.info(f"Generating fonts for {total_symbol_count} symbols") - for sym in chain(ordered_normal_sym_list, ordered_cjk_sym_list): + sym_list = ordered_normal_sym_list + ordered_cjk_sym_list + for sym in sym_list: if sym in symbol_map: raise ValueError("Symbol not found in symbol map") symbol_map[sym] = get_bytes_from_font_index(index) index += 1 - font_table_strings = [] - font_small_table_strings = [] + font12_map: Dict[str, str] = {} + font06_map: Dict[str, str] = {} for sym in ordered_normal_sym_list: if sym not in font_table: logging.error(f"Missing Large font element for {sym}") sys.exit(1) - font_line: str = font_table[sym] - font_table_strings.append( - f"{font_line}//{bytes_to_escaped(symbol_map[sym])} -> {sym}" - ) + font12_map[sym] = font_table[sym] if sym not in font_small_table: logging.error(f"Missing Small font element for {sym}") sys.exit(1) - font_line = font_small_table[sym] - font_small_table_strings.append( - f"{font_line}//{bytes_to_escaped(symbol_map[sym])} -> {sym}" - ) + font06_map[sym] = font_small_table[sym] for sym in ordered_cjk_sym_list: if sym in font_table: raise ValueError("Symbol already exists in font_table") - font_line = get_cjk_glyph(sym) + font_line: str = get_cjk_glyph(sym) if font_line is None: logging.error(f"Missing Large font element for {sym}") sys.exit(1) - font_table_strings.append( - f"{font_line}//{bytes_to_escaped(symbol_map[sym])} -> {sym}" - ) + font12_map[sym] = font_line # No data to add to the small font table - font_small_table_strings.append( - f"// {bytes_to_escaped(symbol_map[sym])} -> {sym}" - ) + font06_map[sym] = "// " # placeholder + return sym_list, FontMap(font12_map, font06_map), symbol_map + + +def make_font_table_cpp( + sym_list: List[str], font_map: FontMap, symbol_map: Dict[str, bytes] +) -> str: output_table = "const uint8_t USER_FONT_12[] = {\n" - for line in font_table_strings: - # join font table int one large string - output_table += line + "\n" + for sym in sym_list: + output_table += ( + f"{font_map.font12[sym]}//{bytes_to_escaped(symbol_map[sym])} -> {sym}\n" + ) output_table += "};\n" + output_table += "const uint8_t USER_FONT_6x8[] = {\n" - for line in font_small_table_strings: - # join font table int one large string - output_table += line + "\n" + for sym in sym_list: + output_table += ( + f"{font_map.font06[sym]}//{bytes_to_escaped(symbol_map[sym])} -> {sym}\n" + ) output_table += "};\n" - return output_table, symbol_map + return output_table def convert_string_bytes(symbol_conversion_table: Dict[str, bytes], text: str) -> bytes: @@ -403,13 +411,14 @@ class TranslationItem: str_index: int -def write_language(lang: dict, defs: dict, f: TextIO) -> None: +def write_language(lang: dict, defs: dict, build_version: str, f: TextIO) -> None: language_code: str = lang["languageCode"] logging.info(f"Generating block for {language_code}") # Iterate over all of the text to build up the symbols & counts - text_list = get_letter_counts(defs, lang) + text_list = get_letter_counts(defs, lang, build_version) # From the letter counts, need to make a symbol translator & write out the font - font_table_text, symbol_conversion_table = get_font_map_and_table(text_list) + sym_list, font_map, symbol_conversion_table = get_font_map_and_table(text_list) + font_table_text = make_font_table_cpp(sym_list, font_map, symbol_conversion_table) try: lang_name = lang["languageLocalName"] @@ -420,6 +429,50 @@ def write_language(lang: dict, defs: dict, f: TextIO) -> None: f.write(font_table_text) f.write(f"\n// ---- {lang_name} ----\n\n") + translation_common_text = get_translation_common_text( + defs, symbol_conversion_table, build_version + ) + f.write(translation_common_text) + f.write( + f"const bool HasFahrenheit = {('true' if lang.get('tempUnitFahrenheit', True) else 'false')};\n\n" + ) + + translation_strings_and_indices_text = get_translation_strings_and_indices_text( + lang, defs, symbol_conversion_table + ) + f.write(translation_strings_and_indices_text) + f.write("const TranslationIndexTable *const Tr = &TranslationIndices;\n") + f.write("const char *const TranslationStrings = TranslationStringsData;\n\n") + + sanity_checks_text = get_translation_sanity_checks_text(defs) + f.write(sanity_checks_text) + + +def get_translation_common_text( + defs: dict, symbol_conversion_table: Dict[str, bytes], build_version +) -> str: + translation_common_text = "" + + # Write out firmware constant options + constants = get_constants(build_version) + for x in constants: + translation_common_text += f'const char* {x[0]} = "{convert_string(symbol_conversion_table, x[1])}";//{x[1]} \n' + translation_common_text += "\n" + + # Debug Menu + translation_common_text += "const char* DebugMenu[] = {\n" + + for c in get_debug_menu(): + translation_common_text += ( + f'\t "{convert_string(symbol_conversion_table, c)}",//{c} \n' + ) + translation_common_text += "};\n\n" + return translation_common_text + + +def get_translation_strings_and_indices_text( + lang: dict, defs: dict, symbol_conversion_table: Dict[str, bytes] +) -> str: str_table: List[str] = [] str_group_messages: List[TranslationItem] = [] str_group_messageswarn: List[TranslationItem] = [] @@ -478,21 +531,6 @@ def write_language(lang: dict, defs: dict, f: TextIO) -> None: str_group_characters.append(TranslationItem(eid, len(str_table))) str_table.append(obj[eid]) - # Write out firmware constant options - constants = get_constants() - for x in constants: - f.write( - f'const char* {x[0]} = "{convert_string(symbol_conversion_table, x[1])}";//{x[1]} \n' - ) - f.write("\n") - - # Debug Menu - f.write("const char* DebugMenu[] = {\n") - - for c in get_debug_menu(): - f.write(f'\t "{convert_string(symbol_conversion_table, c)}",//{c} \n') - f.write("};\n\n") - # ----- Reading SettingsDescriptions obj = lang["menuOptions"] @@ -537,8 +575,6 @@ def write_language(lang: dict, defs: dict, f: TextIO) -> None: ) str_table.append(obj[eid]["desc"]) - f.write("\n") - @dataclass class RemappedTranslationItem: str_index: int @@ -573,14 +609,13 @@ def write_language(lang: dict, defs: dict, f: TextIO) -> None: str_offsets = [-1] * len(str_table) offset = 0 write_null = False - f.write("const char TranslationStringsData[] = {\n") + translation_strings_text = "const char TranslationStringsData[] = {\n" for i, source_str in enumerate(str_table): - if write_null: - f.write(' "\\0"\n') - write_null = True if str_remapping[i] is not None: - write_null = False continue + if write_null: + translation_strings_text += ' "\\0"\n' + write_null = True # Find what items use this string str_used_by = [i] + [ j for j, r in enumerate(str_remapping) if r and r.str_index == i @@ -597,37 +632,35 @@ def write_language(lang: dict, defs: dict, f: TextIO) -> None: ]: for item in group: if item.str_index == j: - f.write(f" // - {pre_info} {item.info}\n") + translation_strings_text += ( + f" // - {pre_info} {item.info}\n" + ) if j == i: - f.write(f" // {offset: >4}: {escape(source_str)}\n") + translation_strings_text += f" // {offset: >4}: {escape(source_str)}\n" str_offsets[j] = offset else: remapped = str_remapping[j] assert remapped is not None - f.write( - f" // {offset + remapped.str_start_offset: >4}: {escape(str_table[j])}\n" - ) + translation_strings_text += f" // {offset + remapped.str_start_offset: >4}: {escape(str_table[j])}\n" str_offsets[j] = offset + remapped.str_start_offset converted_bytes = convert_string_bytes(symbol_conversion_table, source_str) - f.write(f' "{bytes_to_escaped(converted_bytes)}"') + translation_strings_text += f' "{bytes_to_escaped(converted_bytes)}"' str_offsets[i] = offset # Add the length and the null terminator offset += len(converted_bytes) + 1 - f.write("\n};\n\n") + translation_strings_text += "\n}; // TranslationStringsData\n\n" def get_offset(idx: int) -> int: assert str_offsets[idx] >= 0 return str_offsets[idx] - f.write("const TranslationIndexTable TranslationIndices = {\n") + translation_indices_text = "const TranslationIndexTable TranslationIndices = {\n" # ----- Write the messages string indices: for group in [str_group_messages, str_group_messageswarn, str_group_characters]: for item in group: - f.write( - f" .{item.info} = {get_offset(item.str_index)}, // {escape(str_table[item.str_index])}\n" - ) - f.write("\n") + translation_indices_text += f" .{item.info} = {get_offset(item.str_index)}, // {escape(str_table[item.str_index])}\n" + translation_indices_text += "\n" # ----- Write the settings index tables: for group, name in [ @@ -637,30 +670,25 @@ def write_language(lang: dict, defs: dict, f: TextIO) -> None: (str_group_settingmenuentriesdesc, "SettingsMenuEntriesDescriptions"), ]: max_len = 30 - f.write(f" .{name} = {{\n") + translation_indices_text += f" .{name} = {{\n" for item in group: - f.write( - f" /* {item.info.ljust(max_len)[:max_len]} */ {get_offset(item.str_index)}, // {escape(str_table[item.str_index])}\n" - ) - f.write(f" }}, // {name}\n\n") + translation_indices_text += f" /* {item.info.ljust(max_len)[:max_len]} */ {get_offset(item.str_index)}, // {escape(str_table[item.str_index])}\n" + translation_indices_text += f" }}, // {name}\n\n" - f.write("}; // TranslationIndices\n\n") - f.write("const TranslationIndexTable *const Tr = &TranslationIndices;\n") - f.write("const char *const TranslationStrings = TranslationStringsData;\n\n") + translation_indices_text += "}; // TranslationIndices\n\n" - f.write( - f"const bool HasFahrenheit = {('true' if lang.get('tempUnitFahrenheit', True) else 'false')};\n" - ) + return translation_strings_text + translation_indices_text - f.write("\n// Verify SettingsItemIndex values:\n") + +def get_translation_sanity_checks_text(defs: dict) -> str: + sanity_checks_text = "\n// Verify SettingsItemIndex values:\n" for i, mod in enumerate(defs["menuOptions"]): eid = mod["id"] - f.write( + sanity_checks_text += ( f"static_assert(static_cast(SettingsItemIndex::{eid}) == {i});\n" ) - f.write( - f"static_assert(static_cast(SettingsItemIndex::NUM_ITEMS) == {len(defs['menuOptions'])});\n" - ) + sanity_checks_text += f"static_assert(static_cast(SettingsItemIndex::NUM_ITEMS) == {len(defs['menuOptions'])});\n" + return sanity_checks_text def read_version() -> str: @@ -687,23 +715,27 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -if __name__ == "__main__": +def main() -> None: json_dir = HERE args = parse_args() try: - buildVersion = read_version() + build_version = read_version() except FileNotFoundError: logging.error("error: Could not find version info ") sys.exit(1) - logging.info(f"Build version: {buildVersion}") + logging.info(f"Build version: {build_version}") logging.info(f"Making {args.languageCode} from {json_dir}") lang_ = read_translation(json_dir, args.languageCode) defs_ = load_json(os.path.join(json_dir, "translations_def.js"), True) out_ = args.output write_start(out_) - write_language(lang_, defs_, out_) + write_language(lang_, defs_, build_version, out_) logging.info("Done") + + +if __name__ == "__main__": + main() From 2f2575c0b7760d36838644eb2a171f94088375e6 Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Mon, 12 Apr 2021 17:20:19 +0800 Subject: [PATCH 3/7] Refactor make_translation.py to split parsing and writing --- Translations/make_translation.py | 38 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/Translations/make_translation.py b/Translations/make_translation.py index a05643a6..c7787c94 100755 --- a/Translations/make_translation.py +++ b/Translations/make_translation.py @@ -406,18 +406,37 @@ def escape(string: str) -> str: @dataclass -class TranslationItem: - info: str - str_index: int +class LanguageData: + lang: dict + defs: dict + build_version: str + sym_list: List[str] + font_map: FontMap + symbol_conversion_table: Dict[str, bytes] -def write_language(lang: dict, defs: dict, build_version: str, f: TextIO) -> None: +def prepare_language(lang: dict, defs: dict, build_version: str) -> LanguageData: language_code: str = lang["languageCode"] - logging.info(f"Generating block for {language_code}") + logging.info(f"Preparing language data for {language_code}") # Iterate over all of the text to build up the symbols & counts text_list = get_letter_counts(defs, lang, build_version) # From the letter counts, need to make a symbol translator & write out the font sym_list, font_map, symbol_conversion_table = get_font_map_and_table(text_list) + return LanguageData( + lang, defs, build_version, sym_list, font_map, symbol_conversion_table + ) + + +def write_language(data: LanguageData, f: TextIO) -> None: + lang = data.lang + defs = data.defs + build_version = data.build_version + sym_list = data.sym_list + font_map = data.font_map + symbol_conversion_table = data.symbol_conversion_table + + language_code: str = lang["languageCode"] + logging.info(f"Generating block for {language_code}") font_table_text = make_font_table_cpp(sym_list, font_map, symbol_conversion_table) try: @@ -470,6 +489,12 @@ def get_translation_common_text( return translation_common_text +@dataclass +class TranslationItem: + info: str + str_index: int + + def get_translation_strings_and_indices_text( lang: dict, defs: dict, symbol_conversion_table: Dict[str, bytes] ) -> str: @@ -730,9 +755,10 @@ def main() -> None: lang_ = read_translation(json_dir, args.languageCode) defs_ = load_json(os.path.join(json_dir, "translations_def.js"), True) + language_data = prepare_language(lang_, defs_, build_version) out_ = args.output write_start(out_) - write_language(lang_, defs_, build_version, out_) + write_language(language_data, out_) logging.info("Done") From 07fb8adb7f7bbbd48ac57b58e76bd6fac6cd7499 Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Mon, 12 Apr 2021 19:57:29 +0800 Subject: [PATCH 4/7] Impl. initial translation strings compression --- Translations/make_translation.py | 117 +++++++-- source/Core/Drivers/OLED.cpp | 4 +- source/Core/Inc/Translation.h | 8 +- source/Core/Threads/GUIThread.cpp | 2 + source/Core/lzfx/README.md | 79 ++++++ source/Core/lzfx/lzfx-host-compress.c | 52 ++++ source/Core/lzfx/lzfx.c | 352 ++++++++++++++++++++++++++ source/Core/lzfx/lzfx.h | 98 +++++++ source/Makefile | 87 ++++++- 9 files changed, 771 insertions(+), 28 deletions(-) create mode 100644 source/Core/lzfx/README.md create mode 100644 source/Core/lzfx/lzfx-host-compress.c create mode 100644 source/Core/lzfx/lzfx.c create mode 100644 source/Core/lzfx/lzfx.h diff --git a/Translations/make_translation.py b/Translations/make_translation.py index c7787c94..8c858387 100755 --- a/Translations/make_translation.py +++ b/Translations/make_translation.py @@ -5,13 +5,14 @@ import functools import json import logging import os +import pickle import re import subprocess import sys from datetime import datetime from itertools import chain from pathlib import Path -from typing import Dict, List, Optional, TextIO, Tuple, Union +from typing import BinaryIO, Dict, List, Optional, TextIO, Tuple, Union from dataclasses import dataclass from bdflib import reader as bdfreader @@ -405,6 +406,17 @@ def escape(string: str) -> str: return json.dumps(string, ensure_ascii=False) +def write_bytes_as_c_array( + f: TextIO, name: str, data: bytes, indent: int = 2, bytes_per_line: int = 16 +) -> None: + f.write(f"const uint8_t {name}[] = {{\n") + for i in range(0, len(data), bytes_per_line): + f.write(" " * indent) + f.write(", ".join((f"0x{b:02X}" for b in data[i : i + bytes_per_line]))) + f.write(",\n") + f.write(f"}}; // {name}\n\n") + + @dataclass class LanguageData: lang: dict @@ -427,7 +439,9 @@ def prepare_language(lang: dict, defs: dict, build_version: str) -> LanguageData ) -def write_language(data: LanguageData, f: TextIO) -> None: +def write_language( + data: LanguageData, f: TextIO, lzfx_strings: Optional[bytes] = None +) -> None: lang = data.lang defs = data.defs build_version = data.build_version @@ -444,6 +458,9 @@ def write_language(data: LanguageData, f: TextIO) -> None: except KeyError: lang_name = language_code + if lzfx_strings: + f.write('#include "lzfx.h"\n') + f.write(f"\n// ---- {lang_name} ----\n\n") f.write(font_table_text) f.write(f"\n// ---- {lang_name} ----\n\n") @@ -456,12 +473,31 @@ def write_language(data: LanguageData, f: TextIO) -> None: f"const bool HasFahrenheit = {('true' if lang.get('tempUnitFahrenheit', True) else 'false')};\n\n" ) - translation_strings_and_indices_text = get_translation_strings_and_indices_text( - lang, defs, symbol_conversion_table - ) - f.write(translation_strings_and_indices_text) - f.write("const TranslationIndexTable *const Tr = &TranslationIndices;\n") - f.write("const char *const TranslationStrings = TranslationStringsData;\n\n") + if not lzfx_strings: + translation_strings_and_indices_text = get_translation_strings_and_indices_text( + lang, defs, symbol_conversion_table + ) + f.write(translation_strings_and_indices_text) + f.write( + "const TranslationIndexTable *const Tr = &TranslationIndices;\n" + "const char *const TranslationStrings = TranslationStringsData;\n\n" + "extern const uint8_t *const Font_12x16 = USER_FONT_12;\n" + "extern const uint8_t *const Font_6x8 = USER_FONT_6x8;\n\n" + "void prepareTranslations() {}\n\n" + ) + else: + write_bytes_as_c_array(f, "translation_data_lzfx", lzfx_strings) + f.write( + "static uint8_t translation_data_out_buffer[4096] __attribute__((__aligned__(2)));\n\n" + "const TranslationIndexTable *const Tr = reinterpret_cast(translation_data_out_buffer);\n" + "const char *const TranslationStrings = reinterpret_cast(translation_data_out_buffer) + sizeof(TranslationIndexTable);\n\n" + "extern const uint8_t *const Font_12x16 = USER_FONT_12;\n" + "extern const uint8_t *const Font_6x8 = USER_FONT_6x8;\n\n" + "void prepareTranslations() {\n" + " unsigned int outsize = sizeof(translation_data_out_buffer);\n" + " lzfx_decompress(translation_data_lzfx, sizeof(translation_data_lzfx), translation_data_out_buffer, &outsize);\n" + "}\n\n" + ) sanity_checks_text = get_translation_sanity_checks_text(defs) f.write(sanity_checks_text) @@ -733,6 +769,27 @@ def read_version() -> str: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() + parser.add_argument( + "--output-pickled", + help="Write pickled language data for later reuse", + type=argparse.FileType("wb"), + required=False, + dest="output_pickled", + ) + parser.add_argument( + "--input-pickled", + help="Use previously generated pickled language data", + type=argparse.FileType("rb"), + required=False, + dest="input_pickled", + ) + parser.add_argument( + "--lzfx-strings", + help="Use compressed TranslationIndices + TranslationStrings data", + type=argparse.FileType("rb"), + required=False, + dest="lzfx_strings", + ) parser.add_argument( "--output", "-o", help="Target file", type=argparse.FileType("w"), required=True ) @@ -744,21 +801,45 @@ def main() -> None: json_dir = HERE args = parse_args() - try: - build_version = read_version() - except FileNotFoundError: - logging.error("error: Could not find version info ") + if args.input_pickled and args.output_pickled: + logging.error("error: Both --output-pickled and --input-pickled are specified") sys.exit(1) - logging.info(f"Build version: {build_version}") - logging.info(f"Making {args.languageCode} from {json_dir}") + language_data: LanguageData + if args.input_pickled: + logging.info(f"Reading pickled language data from {args.input_pickled.name}...") + language_data = pickle.load(args.input_pickled) + if language_data.lang["languageCode"] != args.languageCode: + logging.error( + f"error: languageCode {args.languageCode} does not match language data {language_data.lang['languageCode']}" + ) + sys.exit(1) + logging.info(f"Read language data for {language_data.lang['languageCode']}") + logging.info(f"Build version: {language_data.build_version}") + else: + try: + build_version = read_version() + except FileNotFoundError: + logging.error("error: Could not find version info ") + sys.exit(1) + + logging.info(f"Build version: {build_version}") + logging.info(f"Making {args.languageCode} from {json_dir}") + + lang_ = read_translation(json_dir, args.languageCode) + defs_ = load_json(os.path.join(json_dir, "translations_def.js"), True) + language_data = prepare_language(lang_, defs_, build_version) - lang_ = read_translation(json_dir, args.languageCode) - defs_ = load_json(os.path.join(json_dir, "translations_def.js"), True) - language_data = prepare_language(lang_, defs_, build_version) out_ = args.output write_start(out_) - write_language(language_data, out_) + if args.lzfx_strings: + write_language(language_data, out_, args.lzfx_strings.read()) + else: + write_language(language_data, out_) + + if args.output_pickled: + logging.info(f"Writing pickled data to {args.output_pickled.name}") + pickle.dump(language_data, args.output_pickled) logging.info("Done") diff --git a/source/Core/Drivers/OLED.cpp b/source/Core/Drivers/OLED.cpp index 309fe87c..ac6ca524 100644 --- a/source/Core/Drivers/OLED.cpp +++ b/source/Core/Drivers/OLED.cpp @@ -117,7 +117,7 @@ void OLED::drawChar(const uint16_t charCode, const FontStyle fontStyle) { static uint8_t fontWidth, fontHeight; switch (fontStyle) { case FontStyle::SMALL: - currentFont = USER_FONT_6x8; + currentFont = Font_6x8; fontHeight = 8; fontWidth = 6; break; @@ -128,7 +128,7 @@ void OLED::drawChar(const uint16_t charCode, const FontStyle fontStyle) { break; case FontStyle::LARGE: default: - currentFont = USER_FONT_12; + currentFont = Font_12x16; fontHeight = 16; fontWidth = 12; break; diff --git a/source/Core/Inc/Translation.h b/source/Core/Inc/Translation.h index 67277906..9c5bcb5d 100644 --- a/source/Core/Inc/Translation.h +++ b/source/Core/Inc/Translation.h @@ -8,8 +8,7 @@ #ifndef TRANSLATION_H_ #define TRANSLATION_H_ #include "stdint.h" -extern const uint8_t USER_FONT_12[]; -extern const uint8_t USER_FONT_6x8[]; + extern const bool HasFahrenheit; extern const char *SymbolPlus; @@ -114,10 +113,15 @@ struct TranslationIndexTable { extern const TranslationIndexTable *const Tr; extern const char *const TranslationStrings; +extern const uint8_t *const Font_12x16; +extern const uint8_t *const Font_6x8; + constexpr uint8_t settings_item_index(const SettingsItemIndex i) { return static_cast(i); } // Use a constexpr function for type-checking. #define SETTINGS_DESC(i) (settings_item_index(i) + 1) const char *translatedString(uint16_t index); +void prepareTranslations(); + #endif /* TRANSLATION_H_ */ diff --git a/source/Core/Threads/GUIThread.cpp b/source/Core/Threads/GUIThread.cpp index 3c72b758..54fd006c 100644 --- a/source/Core/Threads/GUIThread.cpp +++ b/source/Core/Threads/GUIThread.cpp @@ -746,6 +746,8 @@ void showWarnings() { uint8_t idleScreenBGF[sizeof(idleScreenBG)]; /* StartGUITask function */ void startGUITask(void const *argument __unused) { + prepareTranslations(); + OLED::initialize(); // start up the LCD uint8_t tempWarningState = 0; diff --git a/source/Core/lzfx/README.md b/source/Core/lzfx/README.md new file mode 100644 index 00000000..82420cf0 --- /dev/null +++ b/source/Core/lzfx/README.md @@ -0,0 +1,79 @@ +This directory contains file originally by other people. + + +## Simplified LZFX-based compression library + +- `lzfx.c` +- `lzfx.h` + +The above files are obtained from https://github.com/janding/lzfx (commit +448017f). It is a fork by Jan Ding (GitHub user "janding") based on the LZFX +compression library by Andrew Collette. + +### License: + +``` +LZFX is copyright (c) 2009 Andrew Collette and subject to the BSD license +(below). Original LZF copyright statement follows. + +Copyright (c) 2000-2007 Marc Alexander Lehmann + +Redistribution and use in source and binary forms, with or without modifica- +tion, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- +CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- +CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- +ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. +``` + + +## lzfx-boot + +- `lzfx-host-compress.c` (original: `lzfx-raw.c`) + +The above file is obtained from https://github.com/janding/lzfx-boot (commit +88b1596). + +### License: + +``` +BSD 2-Clause License + +Copyright (c) 2017, Jan Ding +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` diff --git a/source/Core/lzfx/lzfx-host-compress.c b/source/Core/lzfx/lzfx-host-compress.c new file mode 100644 index 00000000..1a3e6f79 --- /dev/null +++ b/source/Core/lzfx/lzfx-host-compress.c @@ -0,0 +1,52 @@ +#include +#include + +#include "lzfx.h" + +/* Program to demonstrate file compression. Don't use it in the real world! */ + +int main(int argc, char **argv) +{ + if (argc == 3) + { + /* open input */ + FILE *f = fopen(argv[1], "rb"); + if (!f) { + printf("Error: %s\n", argv[1]); + return 1; + } + + /* get size */ + fseek(f, 0, SEEK_END); + int inputsize = ftell(f); + fseek(f, 0, SEEK_SET); + + /* read */ + char *input = malloc(inputsize); + fread(input, inputsize, 1, f); + fclose(f); + + /* compress */ + int outputsize = inputsize + 1; /* buffer overflow */ + char *output = malloc(outputsize); + lzfx_compress(input, inputsize, output, &outputsize); + + /* open output */ + f = fopen(argv[2], "wb"); + if (!f) { + printf("Error: %s\n", argv[1]); + return 1; + } + + /* write */ + fwrite(output, outputsize, 1, f); + fclose(f); + + return 0; + } + else + { + printf("Compresses a file.\n\nUsage: lzfx-raw input output\n"); + return 1; + } +} \ No newline at end of file diff --git a/source/Core/lzfx/lzfx.c b/source/Core/lzfx/lzfx.c new file mode 100644 index 00000000..47b387a2 --- /dev/null +++ b/source/Core/lzfx/lzfx.c @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2009 Andrew Collette + * http://lzfx.googlecode.com + * + * Implements an LZF-compatible compressor/decompressor based on the liblzf + * codebase written by Marc Lehmann. This code is released under the BSD + * license. License and original copyright statement follow. + * + * + * Copyright (c) 2000-2008 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "lzfx.h" + +#define LZFX_HSIZE (1 << (LZFX_HLOG)) + +/* We need this for memset */ +#ifdef __cplusplus +# include +#else +# include +#endif + +#if __GNUC__ >= 3 +# define fx_expect_false(expr) __builtin_expect((expr) != 0, 0) +# define fx_expect_true(expr) __builtin_expect((expr) != 0, 1) +#else +# define fx_expect_false(expr) (expr) +# define fx_expect_true(expr) (expr) +#endif + +typedef unsigned char u8; +typedef const u8 *LZSTATE[LZFX_HSIZE]; + +/* Define the hash function */ +#define LZFX_FRST(p) (((p[0]) << 8) | p[1]) +#define LZFX_NEXT(v,p) (((v) << 8) | p[2]) +#define LZFX_IDX(h) ((( h >> (3*8 - LZFX_HLOG)) - h ) & (LZFX_HSIZE - 1)) + +/* These cannot be changed, as they are related to the compressed format. */ +#define LZFX_MAX_LIT (1 << 5) - 1 +#define LZFX_MAX_OFF (1 << 13) +#define LZFX_MAX_REF ((1 << 8) + (1 << 3) - 2) + +static +int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen); + +/* Compressed format + + There are two kinds of structures in LZF/LZFX: literal runs and back + references. Literals are encoded as follows: + + LLLLL000 + + Back references are encoded as follows. The smallest possible encoded + length value is 1, as otherwise the control byte would be recognized as + a literal run. At least three bytes must match for a back reference + to be inserted. The offset (distance to the desired data in the output + buffer) is encoded as o - 1, as all offsets are at least 1. The binary + format is: + + oooooLLL oooooooo for backrefs of real length < 7 (1 <= L < 7) + ooooo111 LLLLLLLL oooooooo for backrefs of real length >= 7 (L >= 7) +*/ +int lzfx_compress(const void *const ibuf, const unsigned int ilen, + void *obuf, unsigned int *const olen){ + + /* Hash table; an array of u8*'s which point + to various locations in the input buffer */ + const u8 *htab[LZFX_HSIZE]; + + const u8 **hslot; /* Pointer to entry in hash table */ + unsigned int hval; /* Hash value generated by macros above */ + const u8 *ref; /* Pointer to candidate match location in input */ + + const u8 *ip = (const u8 *)ibuf; + const u8 *const in_end = ip + ilen; + + u8 *op = (u8 *)obuf; + const u8 *const out_end = (olen == NULL ? NULL : op + *olen); + + int lit; /* # of bytes in current literal run */ + +#if defined (WIN32) && defined (_M_X64) + unsigned _int64 off; /* workaround for missing POSIX compliance */ +#else + unsigned long off; +#endif + + if(olen == NULL) return LZFX_EARGS; + if(ibuf == NULL){ + if(ilen != 0) return LZFX_EARGS; + *olen = 0; + return 0; + } + if(obuf == NULL) return LZFX_EARGS; + + memset(htab, 0, sizeof(htab)); + + /* Start a literal run. Whenever we do this the output pointer is + advanced because the current byte will hold the encoded length. */ + lit = 0; op++; + + hval = LZFX_FRST(ip); + + while(ip + 2 < in_end){ /* The NEXT macro reads 2 bytes ahead */ + + hval = LZFX_NEXT(hval, ip); + hslot = htab + LZFX_IDX(hval); + + ref = *hslot; *hslot = ip; + + if( ref < ip + && (off = ip - ref - 1) < LZFX_MAX_OFF + && ip + 4 < in_end /* Backref takes up to 3 bytes, so don't bother */ + && ref > (u8 *)ibuf + && ref[0] == ip[0] + && ref[1] == ip[1] + && ref[2] == ip[2] ) { + + unsigned int len = 3; /* We already know 3 bytes match */ + const unsigned int maxlen = in_end - ip - 2 > LZFX_MAX_REF ? + LZFX_MAX_REF : in_end - ip - 2; + + /* lit == 0: op + 3 must be < out_end (because we undo the run) + lit != 0: op + 3 + 1 must be < out_end */ + if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) + return LZFX_ESIZE; + + op [- lit - 1] = lit << 3;/* Terminate literal run */ + op -= !lit; /* Undo run if length is zero */ + + /* Start checking at the fourth byte */ + while (len < maxlen && ref[len] == ip[len]) + len++; + + /* Format 1: [oooooLLL oooooooo] */ + if (len < 7) { + *op++ = ((off >> 8) << 3) + len; + *op++ = off; + + /* Format 2: [ooooo111 LLLLLLLL oooooooo] */ + } else { + *op++ = ((off >> 8) << 3) + 7; + *op++ = len - 7; + *op++ = off; + } + + lit = 0; op++; + + ip += len - 1; /* ip = initial ip + #octets - 1 */ + + if (fx_expect_false (ip + 3 >= in_end)){ + ip++; /* Code following expects exit at bottom of loop */ + break; + } + + hval = LZFX_FRST (ip); + hval = LZFX_NEXT (hval, ip); + htab[LZFX_IDX (hval)] = ip; + + ip++; /* ip = initial ip + #octets */ + + } else { + /* Keep copying literal bytes */ + + if (fx_expect_false (op >= out_end)) return LZFX_ESIZE; + + lit++; *op++ = *ip++; + + if (fx_expect_false (lit == LZFX_MAX_LIT)) { + op [- lit - 1] = lit << 3; /* stop run */ + lit = 0; op++; /* start run */ + } + + } /* if() found match in htab */ + + } /* while(ip < ilen -2) */ + + /* At most 3 bytes remain in input. We therefore need 4 bytes available + in the output buffer to store them (3 data + ctrl byte).*/ + if (op + 3 > out_end) return LZFX_ESIZE; + + while (ip < in_end) { + + lit++; *op++ = *ip++; + + if (fx_expect_false (lit == LZFX_MAX_LIT)){ + op [- lit - 1] = lit << 3; + lit = 0; op++; + } + } + + op [- lit - 1] = lit << 3; + op -= !lit; + + *olen = op - (u8 *)obuf; + return 0; +} + +/* Decompressor */ +int lzfx_decompress(const void* ibuf, unsigned int ilen, + void* obuf, unsigned int *olen){ + + u8 const *ip = (const u8 *)ibuf; + u8 const *const in_end = ip + ilen; + u8 *op = (u8 *)obuf; + u8 const *const out_end = (olen == NULL ? NULL : op + *olen); + + unsigned int remain_len = 0; + int rc; + + if(olen == NULL) return LZFX_EARGS; + if(ibuf == NULL){ + if(ilen != 0) return LZFX_EARGS; + *olen = 0; + return 0; + } + if(obuf == NULL){ + if(olen != 0) return LZFX_EARGS; + return lzfx_getsize(ibuf, ilen, olen); + } + + do { + unsigned int ctrl = *ip++; + + /* Format LLLLL000: a literal byte string follows, of length L */ + if((ctrl & 0x7) == 0) { + unsigned int len = ctrl >> 3; + + if(fx_expect_false(op + len > out_end)){ + --ip; /* Rewind to control byte */ + goto guess; + } + if(fx_expect_false(ip + len > in_end)) return LZFX_ECORRUPT; + + do + *op++ = *ip++; + while(--len); + + /* Format #1 [oooooLLL oooooooo]: backref of length L+1 + ^^^^^ ^^^^^^^^ + A B + #2 [ooooo111 LLLLLLLL oooooooo] backref of length L+7 + ^^^^^ ^^^^^^^^ + A B + In both cases the location of the backref is computed from the + remaining part of the data as follows: + + location = op - A*256 - B - 1 + */ + } else { + + unsigned int len = ctrl & 0x7; + u8 *ref = op - ((ctrl >> 3) << 8) - 1; + + if(len==7) len += *ip++; /* i.e. format #2 */ + + if(fx_expect_false(op + len > out_end)){ + ip -= (len >= 7) ? 2 : 1; /* Rewind to control byte */ + goto guess; + } + if(fx_expect_false(ip >= in_end)) return LZFX_ECORRUPT; + + ref -= *ip++; + + if(fx_expect_false(ref < (u8*)obuf)) return LZFX_ECORRUPT; + + do + *op++ = *ref++; + while (--len); + } + + } while (ip < in_end); + + *olen = op - (u8 *)obuf; + + return 0; + +guess: + rc = lzfx_getsize(ip, ilen - (ip-(u8*)ibuf), &remain_len); + if(rc>=0) *olen = remain_len + (op - (u8*)obuf); + return rc; +} + +/* Guess len. No parameters may be NULL; this is not checked. */ +static +int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen){ + + u8 const *ip = (const u8 *)ibuf; + u8 const *const in_end = ip + ilen; + int tot_len = 0; + + while (ip < in_end) { + + unsigned int ctrl = *ip++; + + if((ctrl & 0x7) == 0) { + + if(ip + (ctrl >> 3) > in_end) + return LZFX_ECORRUPT; + + tot_len += (ctrl >> 3); + ip += (ctrl >> 3); + + } else { + + unsigned int len = ctrl & 0x7; + + if(len==7){ /* i.e. format #2 */ + len += *ip++; + } + + if(ip >= in_end) return LZFX_ECORRUPT; + + ip++; /* skip the ref byte */ + + tot_len += len; + + } + + } + + *olen = tot_len; + + return 0; +} + + + + diff --git a/source/Core/lzfx/lzfx.h b/source/Core/lzfx/lzfx.h new file mode 100644 index 00000000..4d7c560b --- /dev/null +++ b/source/Core/lzfx/lzfx.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2009 Andrew Collette + * http://lzfx.googlecode.com + * + * Implements an LZF-compatible compressor/decompressor based on the liblzf + * codebase written by Marc Lehmann. This code is released under the BSD + * license. License and original copyright statement follow. + * + * + * Copyright (c) 2000-2008 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZFX_H +#define LZFX_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Documented behavior, including function signatures and error codes, + is guaranteed to remain unchanged for releases with the same major + version number. Releases of the same major version are also able + to read each other's output, although the output itself is not + guaranteed to be byte-for-byte identical. +*/ +#define LZFX_VERSION_MAJOR 0 +#define LZFX_VERSION_MINOR 1 +#define LZFX_VERSION_STRING "0.1" + +/* Hashtable size (2**LZFX_HLOG entries) */ +#ifndef LZFX_HLOG +# define LZFX_HLOG 16 +#endif + +/* Predefined errors. */ +#define LZFX_ESIZE -1 /* Output buffer too small */ +#define LZFX_ECORRUPT -2 /* Invalid data for decompression */ +#define LZFX_EARGS -3 /* Arguments invalid (NULL) */ + +/* Buffer-to buffer compression. + + Supply pre-allocated input and output buffers via ibuf and obuf, and + their size in bytes via ilen and olen. Buffers may not overlap. + + On success, the function returns a non-negative value and the argument + olen contains the compressed size in bytes. On failure, a negative + value is returned and olen is not modified. +*/ +int lzfx_compress(const void* ibuf, unsigned int ilen, + void* obuf, unsigned int *olen); + +/* Buffer-to-buffer decompression. + + Supply pre-allocated input and output buffers via ibuf and obuf, and + their size in bytes via ilen and olen. Buffers may not overlap. + + On success, the function returns a non-negative value and the argument + olen contains the uncompressed size in bytes. On failure, a negative + value is returned. + + If the failure code is LZFX_ESIZE, olen contains the minimum buffer size + required to hold the decompressed data. Otherwise, olen is not modified. + + Supplying a zero *olen is a valid and supported strategy to determine the + required buffer size. This does not require decompression of the entire + stream and is consequently very fast. Argument obuf may be NULL in + this case only. +*/ +int lzfx_decompress(const void* ibuf, unsigned int ilen, + void* obuf, unsigned int *olen); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/source/Makefile b/source/Makefile index eb988648..73568409 100644 --- a/source/Makefile +++ b/source/Makefile @@ -13,8 +13,16 @@ endif ALL_LANGUAGES=BG CS DA DE EN ES FI FR HR HU IT LT NL NL_BE NO PL PT RU SK SL SR_CYRL SR_LATN SV TR UK YUE_HK ZH_CN ZH_TW +# Defines for host tools +ifeq ($(HOST_CC),) +HOST_CC := gcc +endif +HOST_OUTPUT_DIR=Objects/host + + # Enumerate all of the include directories APP_INC_DIR = ./Core/Inc +LZFX_INC_DIR = ./Core/lzfx MINIWARE_INC_CMSIS_DEVICE = ./Core/BSP/Miniware/Vendor/CMSIS/Device/ST/STM32F1xx/Include MINIWARE_CMSIS_CORE_INC_DIR = ./Core/BSP/Miniware/Vendor/CMSIS/Include MINIWARE_HAL_INC_DIR = ./Core/BSP/Miniware/Vendor/STM32F1xx_HAL_Driver/Inc @@ -34,6 +42,7 @@ PINE_NMSIS_INC_DIR = ./Core/BSP/Pine64/Vendor/NMSIS/Core/Include PINE_FREERTOS_PORT_INC_DIR = ./Core/BSP/Pine64/Vendor/OS/FreeRTOS/Source/portable/GCC SOURCE_THREADS_DIR = ./Core/Threads SOURCE_CORE_DIR = ./Core/Src +SOURCE_LZFX_DIR = ./Core/lzfx SOURCE_DRIVERS_DIR = ./Core/Drivers INC_PD_DRIVERS_DIR = ./Core/Drivers/FUSB302 SOURCE_MIDDLEWARES_DIR = ./Middlewares @@ -101,6 +110,7 @@ DEV_CXXFLAGS= -MMD -MP -MF "$(@:%.o=%.d)" -MT "$@" endif INCLUDES = -I$(APP_INC_DIR) \ + -I$(LZFX_INC_DIR) \ -I$(FRTOS_CMIS_INC_DIR) \ -I$(FRTOS_INC_DIR) \ -I$(DRIVER_INC_DIR) \ @@ -114,7 +124,8 @@ SOURCE := $(shell find $(SOURCE_THREADS_DIR) -type f -name '*.c') \ $(shell find $(SOURCE_CORE_DIR) -type f -name '*.c') \ $(shell find $(SOURCE_DRIVERS_DIR) -type f -name '*.c') \ $(shell find $(DEVICE_BSP_DIR) -type f -name '*.c') \ -$(shell find $(SOURCE_MIDDLEWARES_DIR) -type f -name '*.c') +$(shell find $(SOURCE_MIDDLEWARES_DIR) -type f -name '*.c') \ +$(SOURCE_LZFX_DIR)/lzfx.c SOURCE_CPP := $(shell find $(SOURCE_THREADS_DIR) -type f -name '*.cpp') \ $(shell find $(SOURCE_CORE_DIR) -type f -name '*.cpp') \ $(shell find $(SOURCE_DRIVERS_DIR) -type f -name '*.cpp') \ @@ -296,10 +307,25 @@ all: $(ALL_FIRMWARE_TARGETS) $(SIZE) --format=berkeley $< $(OBJCOPY) $< -O binary $@ -$(HEXFILE_DIR)/$(model)_%.elf : $(OUT_OBJS_S) $(OUT_OBJS) $(OUT_OBJS_CPP) $(OUTPUT_DIR)/Core/Gen/Translation.%.o Makefile $(LDSCRIPT) +$(HEXFILE_DIR)/$(model)_%.elf : \ + $(OUT_OBJS_S) $(OUT_OBJS) $(OUT_OBJS_CPP) \ + $(OUTPUT_DIR)/Core/Gen/Translation.%.o \ + Makefile $(LDSCRIPT) @test -d $(@D) || mkdir -p $(@D) @echo Linking $@ - @$(CPP) $(CXXFLAGS) $(OUT_OBJS_S) $(OUT_OBJS) $(OUT_OBJS_CPP) $(OUTPUT_DIR)/Core/Gen/Translation.$*.o $(LIBS) $(LINKER_FLAGS) -o$@ -Wl,-Map=$@.map + @$(CPP) $(CXXFLAGS) $(OUT_OBJS_S) $(OUT_OBJS) $(OUT_OBJS_CPP) \ + $(OUTPUT_DIR)/Core/Gen/Translation.$*.o \ + $(LIBS) $(LINKER_FLAGS) -o$@ -Wl,-Map=$@.map + +$(HEXFILE_DIR)/$(model)_string_compressed_%.elf : \ + $(OUT_OBJS_S) $(OUT_OBJS) $(OUT_OBJS_CPP) \ + $(OUTPUT_DIR)/Core/Gen/Translation_lzfx.%.o \ + Makefile $(LDSCRIPT) + @test -d $(@D) || mkdir -p $(@D) + @echo Linking $@ + @$(CPP) $(CXXFLAGS) $(OUT_OBJS_S) $(OUT_OBJS) $(OUT_OBJS_CPP) \ + $(OUTPUT_DIR)/Core/Gen/Translation_lzfx.$*.o \ + $(LIBS) $(LINKER_FLAGS) -o$@ -Wl,-Map=$@.map $(OUT_OBJS): $(OUTPUT_DIR)/%.o : %.c Makefile @test -d $(@D) || mkdir -p $(@D) @@ -316,10 +342,57 @@ $(OUT_OBJS_S): $(OUTPUT_DIR)/%.o: %.S Makefile @echo 'Building file: $<' @$(AS) -c $(AFLAGS) $< -o $@ -Core/Gen/Translation.%.cpp: ../Translations/translation_%.json Makefile ../Translations/make_translation.py ../Translations/translations_def.js ../Translations/font_tables.py ../Translations/wqy-bitmapsong/wenquanyi_9pt.bdf - @test -d $(@D) || mkdir -p $(@D) +Core/Gen/Translation.%.cpp $(OUTPUT_DIR)/Core/Gen/translation.files/%.pickle: ../Translations/translation_%.json \ + ../Translations/make_translation.py \ + ../Translations/translations_def.js \ + ../Translations/font_tables.py \ + Makefile ../Translations/wqy-bitmapsong/wenquanyi_9pt.bdf + @test -d Core/Gen || mkdir -p Core/Gen + @test -d $(OUTPUT_DIR)/Core/Gen/translation.files || mkdir -p $(OUTPUT_DIR)/Core/Gen/translation.files @echo 'Generating translations for language $*' - @python3 ../Translations/make_translation.py -o $(PWD)/$@ $* + @python3 ../Translations/make_translation.py \ + -o $(PWD)/Core/Gen/Translation.$*.cpp \ + --output-pickled $(OUTPUT_DIR)/Core/Gen/translation.files/$*.pickle \ + $* + +# +# The recipes to produce compressed translation data: +# + +$(OUTPUT_DIR)/Core/Gen/translation.files/%.o: Core/Gen/Translation.%.cpp + @test -d $(@D) || mkdir -p $(@D) + @echo Generating $@ + @$(CPP) -c $(filter-out -flto -g3,$(CXXFLAGS)) $< -o $@ + +$(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress: Core/lzfx/lzfx-host-compress.c Core/lzfx/lzfx.c + @test -d $(@D) || mkdir -p $(@D) + @echo Building host lzfx tool $@ + @$(HOST_CC) -Wno-unused-result -O $^ -o $@ + +$(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin: $(OUTPUT_DIR)/Core/Gen/translation.files/%.o + @echo Dumping translation strings data from $< + @# Extract the raw strings data from the object file + @$(OBJCOPY) -O binary -j .rodata._ZL18TranslationIndices $< $(@D)/$*.data.TranslationIndices.bin + @test -s $(@D)/$*.data.TranslationIndices.bin || (rm $(@D)/$*.data.TranslationIndices.bin; echo 'ERROR: Output for .rodata._ZL18TranslationIndices is empty!' >&2; false) + @$(OBJCOPY) -O binary -j .rodata._ZL22TranslationStringsData $< $(@D)/$*.data.TranslationStrings.bin + @test -s $(@D)/$*.data.TranslationStrings.bin || (rm $(@D)/$*.data.TranslationStrings.bin; echo 'ERROR: Output for .rodata._ZL22TranslationStringsData is empty!' >&2; false) + @cat $(@D)/$*.data.TranslationIndices.bin $(@D)/$*.data.TranslationStrings.bin > $@ + +$(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.lzfx: $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin $(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress + @echo Compressing translation strings data for $* + @$(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress $< $@ + @echo Compressed from $$(stat --printf="%s" $<) to $$(stat --printf="%s" $@) bytes + +Core/Gen/Translation_lzfx.%.cpp: $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.lzfx $(OUTPUT_DIR)/Core/Gen/translation.files/%.pickle + @test -d $(@D) || mkdir -p $(@D) + @echo Generating lzfx compressed translation for $* + @python3 ../Translations/make_translation.py \ + -o $(PWD)/Core/Gen/Translation_lzfx.$*.cpp \ + --input-pickled $(OUTPUT_DIR)/Core/Gen/translation.files/$*.pickle \ + --lzfx-strings $(OUTPUT_DIR)/Core/Gen/translation.files/$*.strings.lzfx \ + $* + + clean : rm -Rf Core/Gen @@ -350,3 +423,5 @@ check-style: -include $(OUT_OBJS:.o=.d) -include $(OUT_OBJS_CPP:.o=.d) -include $(OUTPUT_DIR)/Core/Gen/Translation.*.d +-include $(OUTPUT_DIR)/Core/Gen/Translation_*.d +-include $(OUTPUT_DIR)/Core/Gen/translation.files/*.d From 11f40cfc1b5b8f935d0ff13f4e72a060f6d2dc3a Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Mon, 12 Apr 2021 20:07:06 +0800 Subject: [PATCH 5/7] Reformat lzfx sources --- source/Core/lzfx/lzfx-host-compress.c | 74 ++-- source/Core/lzfx/lzfx.c | 505 +++++++++++++------------- source/Core/lzfx/lzfx.h | 33 +- 3 files changed, 305 insertions(+), 307 deletions(-) diff --git a/source/Core/lzfx/lzfx-host-compress.c b/source/Core/lzfx/lzfx-host-compress.c index 1a3e6f79..b135b975 100644 --- a/source/Core/lzfx/lzfx-host-compress.c +++ b/source/Core/lzfx/lzfx-host-compress.c @@ -5,48 +5,44 @@ /* Program to demonstrate file compression. Don't use it in the real world! */ -int main(int argc, char **argv) -{ - if (argc == 3) - { - /* open input */ - FILE *f = fopen(argv[1], "rb"); - if (!f) { - printf("Error: %s\n", argv[1]); - return 1; - } +int main(int argc, char **argv) { + if (argc == 3) { + /* open input */ + FILE *f = fopen(argv[1], "rb"); + if (!f) { + printf("Error: %s\n", argv[1]); + return 1; + } - /* get size */ - fseek(f, 0, SEEK_END); - int inputsize = ftell(f); - fseek(f, 0, SEEK_SET); + /* get size */ + fseek(f, 0, SEEK_END); + int inputsize = ftell(f); + fseek(f, 0, SEEK_SET); - /* read */ - char *input = malloc(inputsize); - fread(input, inputsize, 1, f); - fclose(f); + /* read */ + char *input = malloc(inputsize); + fread(input, inputsize, 1, f); + fclose(f); - /* compress */ - int outputsize = inputsize + 1; /* buffer overflow */ - char *output = malloc(outputsize); - lzfx_compress(input, inputsize, output, &outputsize); + /* compress */ + int outputsize = inputsize + 1; /* buffer overflow */ + char *output = malloc(outputsize); + lzfx_compress(input, inputsize, output, &outputsize); - /* open output */ - f = fopen(argv[2], "wb"); - if (!f) { - printf("Error: %s\n", argv[1]); - return 1; - } + /* open output */ + f = fopen(argv[2], "wb"); + if (!f) { + printf("Error: %s\n", argv[1]); + return 1; + } - /* write */ - fwrite(output, outputsize, 1, f); - fclose(f); + /* write */ + fwrite(output, outputsize, 1, f); + fclose(f); - return 0; - } - else - { - printf("Compresses a file.\n\nUsage: lzfx-raw input output\n"); - return 1; - } -} \ No newline at end of file + return 0; + } else { + printf("Compresses a file.\n\nUsage: lzfx-raw input output\n"); + return 1; + } +} diff --git a/source/Core/lzfx/lzfx.c b/source/Core/lzfx/lzfx.c index 47b387a2..81d1560a 100644 --- a/source/Core/lzfx/lzfx.c +++ b/source/Core/lzfx/lzfx.c @@ -6,19 +6,19 @@ * codebase written by Marc Lehmann. This code is released under the BSD * license. License and original copyright statement follow. * - * + * * Copyright (c) 2000-2008 Marc Alexander Lehmann - * + * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. - * + * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO @@ -29,7 +29,7 @@ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. -*/ + */ #include "lzfx.h" @@ -37,34 +37,33 @@ /* We need this for memset */ #ifdef __cplusplus -# include +#include #else -# include +#include #endif #if __GNUC__ >= 3 -# define fx_expect_false(expr) __builtin_expect((expr) != 0, 0) -# define fx_expect_true(expr) __builtin_expect((expr) != 0, 1) +#define fx_expect_false(expr) __builtin_expect((expr) != 0, 0) +#define fx_expect_true(expr) __builtin_expect((expr) != 0, 1) #else -# define fx_expect_false(expr) (expr) -# define fx_expect_true(expr) (expr) +#define fx_expect_false(expr) (expr) +#define fx_expect_true(expr) (expr) #endif typedef unsigned char u8; -typedef const u8 *LZSTATE[LZFX_HSIZE]; +typedef const u8 * LZSTATE[LZFX_HSIZE]; /* Define the hash function */ -#define LZFX_FRST(p) (((p[0]) << 8) | p[1]) -#define LZFX_NEXT(v,p) (((v) << 8) | p[2]) -#define LZFX_IDX(h) ((( h >> (3*8 - LZFX_HLOG)) - h ) & (LZFX_HSIZE - 1)) +#define LZFX_FRST(p) (((p[0]) << 8) | p[1]) +#define LZFX_NEXT(v, p) (((v) << 8) | p[2]) +#define LZFX_IDX(h) (((h >> (3 * 8 - LZFX_HLOG)) - h) & (LZFX_HSIZE - 1)) /* These cannot be changed, as they are related to the compressed format. */ -#define LZFX_MAX_LIT (1 << 5) - 1 -#define LZFX_MAX_OFF (1 << 13) -#define LZFX_MAX_REF ((1 << 8) + (1 << 3) - 2) +#define LZFX_MAX_LIT (1 << 5) - 1 +#define LZFX_MAX_OFF (1 << 13) +#define LZFX_MAX_REF ((1 << 8) + (1 << 3) - 2) -static -int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen); +static int lzfx_getsize(const void *ibuf, unsigned int ilen, unsigned int *olen); /* Compressed format @@ -83,270 +82,276 @@ int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen); oooooLLL oooooooo for backrefs of real length < 7 (1 <= L < 7) ooooo111 LLLLLLLL oooooooo for backrefs of real length >= 7 (L >= 7) */ -int lzfx_compress(const void *const ibuf, const unsigned int ilen, - void *obuf, unsigned int *const olen){ +int lzfx_compress(const void *const ibuf, const unsigned int ilen, void *obuf, unsigned int *const olen) { - /* Hash table; an array of u8*'s which point - to various locations in the input buffer */ - const u8 *htab[LZFX_HSIZE]; + /* Hash table; an array of u8*'s which point + to various locations in the input buffer */ + const u8 *htab[LZFX_HSIZE]; - const u8 **hslot; /* Pointer to entry in hash table */ - unsigned int hval; /* Hash value generated by macros above */ - const u8 *ref; /* Pointer to candidate match location in input */ + const u8 ** hslot; /* Pointer to entry in hash table */ + unsigned int hval; /* Hash value generated by macros above */ + const u8 * ref; /* Pointer to candidate match location in input */ - const u8 *ip = (const u8 *)ibuf; - const u8 *const in_end = ip + ilen; + const u8 * ip = (const u8 *)ibuf; + const u8 *const in_end = ip + ilen; - u8 *op = (u8 *)obuf; - const u8 *const out_end = (olen == NULL ? NULL : op + *olen); + u8 * op = (u8 *)obuf; + const u8 *const out_end = (olen == NULL ? NULL : op + *olen); - int lit; /* # of bytes in current literal run */ + int lit; /* # of bytes in current literal run */ -#if defined (WIN32) && defined (_M_X64) - unsigned _int64 off; /* workaround for missing POSIX compliance */ +#if defined(WIN32) && defined(_M_X64) + unsigned _int64 off; /* workaround for missing POSIX compliance */ #else - unsigned long off; + unsigned long off; #endif - if(olen == NULL) return LZFX_EARGS; - if(ibuf == NULL){ - if(ilen != 0) return LZFX_EARGS; - *olen = 0; - return 0; - } - if(obuf == NULL) return LZFX_EARGS; - - memset(htab, 0, sizeof(htab)); - - /* Start a literal run. Whenever we do this the output pointer is - advanced because the current byte will hold the encoded length. */ - lit = 0; op++; - - hval = LZFX_FRST(ip); - - while(ip + 2 < in_end){ /* The NEXT macro reads 2 bytes ahead */ - - hval = LZFX_NEXT(hval, ip); - hslot = htab + LZFX_IDX(hval); - - ref = *hslot; *hslot = ip; - - if( ref < ip - && (off = ip - ref - 1) < LZFX_MAX_OFF - && ip + 4 < in_end /* Backref takes up to 3 bytes, so don't bother */ - && ref > (u8 *)ibuf - && ref[0] == ip[0] - && ref[1] == ip[1] - && ref[2] == ip[2] ) { - - unsigned int len = 3; /* We already know 3 bytes match */ - const unsigned int maxlen = in_end - ip - 2 > LZFX_MAX_REF ? - LZFX_MAX_REF : in_end - ip - 2; - - /* lit == 0: op + 3 must be < out_end (because we undo the run) - lit != 0: op + 3 + 1 must be < out_end */ - if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) - return LZFX_ESIZE; - - op [- lit - 1] = lit << 3;/* Terminate literal run */ - op -= !lit; /* Undo run if length is zero */ - - /* Start checking at the fourth byte */ - while (len < maxlen && ref[len] == ip[len]) - len++; - - /* Format 1: [oooooLLL oooooooo] */ - if (len < 7) { - *op++ = ((off >> 8) << 3) + len; - *op++ = off; - - /* Format 2: [ooooo111 LLLLLLLL oooooooo] */ - } else { - *op++ = ((off >> 8) << 3) + 7; - *op++ = len - 7; - *op++ = off; - } - - lit = 0; op++; - - ip += len - 1; /* ip = initial ip + #octets - 1 */ - - if (fx_expect_false (ip + 3 >= in_end)){ - ip++; /* Code following expects exit at bottom of loop */ - break; - } - - hval = LZFX_FRST (ip); - hval = LZFX_NEXT (hval, ip); - htab[LZFX_IDX (hval)] = ip; - - ip++; /* ip = initial ip + #octets */ - - } else { - /* Keep copying literal bytes */ - - if (fx_expect_false (op >= out_end)) return LZFX_ESIZE; - - lit++; *op++ = *ip++; - - if (fx_expect_false (lit == LZFX_MAX_LIT)) { - op [- lit - 1] = lit << 3; /* stop run */ - lit = 0; op++; /* start run */ - } - - } /* if() found match in htab */ - - } /* while(ip < ilen -2) */ - - /* At most 3 bytes remain in input. We therefore need 4 bytes available - in the output buffer to store them (3 data + ctrl byte).*/ - if (op + 3 > out_end) return LZFX_ESIZE; - - while (ip < in_end) { - - lit++; *op++ = *ip++; - - if (fx_expect_false (lit == LZFX_MAX_LIT)){ - op [- lit - 1] = lit << 3; - lit = 0; op++; - } - } - - op [- lit - 1] = lit << 3; - op -= !lit; - - *olen = op - (u8 *)obuf; + if (olen == NULL) + return LZFX_EARGS; + if (ibuf == NULL) { + if (ilen != 0) + return LZFX_EARGS; + *olen = 0; return 0; + } + if (obuf == NULL) + return LZFX_EARGS; + + memset(htab, 0, sizeof(htab)); + + /* Start a literal run. Whenever we do this the output pointer is + advanced because the current byte will hold the encoded length. */ + lit = 0; + op++; + + hval = LZFX_FRST(ip); + + while (ip + 2 < in_end) { /* The NEXT macro reads 2 bytes ahead */ + + hval = LZFX_NEXT(hval, ip); + hslot = htab + LZFX_IDX(hval); + + ref = *hslot; + *hslot = ip; + + if (ref < ip && (off = ip - ref - 1) < LZFX_MAX_OFF && ip + 4 < in_end /* Backref takes up to 3 bytes, so don't bother */ + && ref > (u8 *)ibuf && ref[0] == ip[0] && ref[1] == ip[1] && ref[2] == ip[2]) { + + unsigned int len = 3; /* We already know 3 bytes match */ + const unsigned int maxlen = in_end - ip - 2 > LZFX_MAX_REF ? LZFX_MAX_REF : in_end - ip - 2; + + /* lit == 0: op + 3 must be < out_end (because we undo the run) + lit != 0: op + 3 + 1 must be < out_end */ + if (fx_expect_false(op - !lit + 3 + 1 >= out_end)) + return LZFX_ESIZE; + + op[-lit - 1] = lit << 3; /* Terminate literal run */ + op -= !lit; /* Undo run if length is zero */ + + /* Start checking at the fourth byte */ + while (len < maxlen && ref[len] == ip[len]) + len++; + + /* Format 1: [oooooLLL oooooooo] */ + if (len < 7) { + *op++ = ((off >> 8) << 3) + len; + *op++ = off; + + /* Format 2: [ooooo111 LLLLLLLL oooooooo] */ + } else { + *op++ = ((off >> 8) << 3) + 7; + *op++ = len - 7; + *op++ = off; + } + + lit = 0; + op++; + + ip += len - 1; /* ip = initial ip + #octets - 1 */ + + if (fx_expect_false(ip + 3 >= in_end)) { + ip++; /* Code following expects exit at bottom of loop */ + break; + } + + hval = LZFX_FRST(ip); + hval = LZFX_NEXT(hval, ip); + htab[LZFX_IDX(hval)] = ip; + + ip++; /* ip = initial ip + #octets */ + + } else { + /* Keep copying literal bytes */ + + if (fx_expect_false(op >= out_end)) + return LZFX_ESIZE; + + lit++; + *op++ = *ip++; + + if (fx_expect_false(lit == LZFX_MAX_LIT)) { + op[-lit - 1] = lit << 3; /* stop run */ + lit = 0; + op++; /* start run */ + } + + } /* if() found match in htab */ + + } /* while(ip < ilen -2) */ + + /* At most 3 bytes remain in input. We therefore need 4 bytes available + in the output buffer to store them (3 data + ctrl byte).*/ + if (op + 3 > out_end) + return LZFX_ESIZE; + + while (ip < in_end) { + + lit++; + *op++ = *ip++; + + if (fx_expect_false(lit == LZFX_MAX_LIT)) { + op[-lit - 1] = lit << 3; + lit = 0; + op++; + } + } + + op[-lit - 1] = lit << 3; + op -= !lit; + + *olen = op - (u8 *)obuf; + return 0; } /* Decompressor */ -int lzfx_decompress(const void* ibuf, unsigned int ilen, - void* obuf, unsigned int *olen){ +int lzfx_decompress(const void *ibuf, unsigned int ilen, void *obuf, unsigned int *olen) { - u8 const *ip = (const u8 *)ibuf; - u8 const *const in_end = ip + ilen; - u8 *op = (u8 *)obuf; - u8 const *const out_end = (olen == NULL ? NULL : op + *olen); - - unsigned int remain_len = 0; - int rc; + u8 const * ip = (const u8 *)ibuf; + u8 const *const in_end = ip + ilen; + u8 * op = (u8 *)obuf; + u8 const *const out_end = (olen == NULL ? NULL : op + *olen); - if(olen == NULL) return LZFX_EARGS; - if(ibuf == NULL){ - if(ilen != 0) return LZFX_EARGS; - *olen = 0; - return 0; - } - if(obuf == NULL){ - if(olen != 0) return LZFX_EARGS; - return lzfx_getsize(ibuf, ilen, olen); - } - - do { - unsigned int ctrl = *ip++; - - /* Format LLLLL000: a literal byte string follows, of length L */ - if((ctrl & 0x7) == 0) { - unsigned int len = ctrl >> 3; - - if(fx_expect_false(op + len > out_end)){ - --ip; /* Rewind to control byte */ - goto guess; - } - if(fx_expect_false(ip + len > in_end)) return LZFX_ECORRUPT; - - do - *op++ = *ip++; - while(--len); - - /* Format #1 [oooooLLL oooooooo]: backref of length L+1 - ^^^^^ ^^^^^^^^ - A B - #2 [ooooo111 LLLLLLLL oooooooo] backref of length L+7 - ^^^^^ ^^^^^^^^ - A B - In both cases the location of the backref is computed from the - remaining part of the data as follows: - - location = op - A*256 - B - 1 - */ - } else { - - unsigned int len = ctrl & 0x7; - u8 *ref = op - ((ctrl >> 3) << 8) - 1; - - if(len==7) len += *ip++; /* i.e. format #2 */ - - if(fx_expect_false(op + len > out_end)){ - ip -= (len >= 7) ? 2 : 1; /* Rewind to control byte */ - goto guess; - } - if(fx_expect_false(ip >= in_end)) return LZFX_ECORRUPT; - - ref -= *ip++; - - if(fx_expect_false(ref < (u8*)obuf)) return LZFX_ECORRUPT; - - do - *op++ = *ref++; - while (--len); - } - - } while (ip < in_end); - - *olen = op - (u8 *)obuf; + unsigned int remain_len = 0; + int rc; + if (olen == NULL) + return LZFX_EARGS; + if (ibuf == NULL) { + if (ilen != 0) + return LZFX_EARGS; + *olen = 0; return 0; + } + if (obuf == NULL) { + if (olen != 0) + return LZFX_EARGS; + return lzfx_getsize(ibuf, ilen, olen); + } + + do { + unsigned int ctrl = *ip++; + + /* Format LLLLL000: a literal byte string follows, of length L */ + if ((ctrl & 0x7) == 0) { + unsigned int len = ctrl >> 3; + + if (fx_expect_false(op + len > out_end)) { + --ip; /* Rewind to control byte */ + goto guess; + } + if (fx_expect_false(ip + len > in_end)) + return LZFX_ECORRUPT; + + do + *op++ = *ip++; + while (--len); + + /* Format #1 [oooooLLL oooooooo]: backref of length L+1 + ^^^^^ ^^^^^^^^ + A B + #2 [ooooo111 LLLLLLLL oooooooo] backref of length L+7 + ^^^^^ ^^^^^^^^ + A B + In both cases the location of the backref is computed from the + remaining part of the data as follows: + + location = op - A*256 - B - 1 + */ + } else { + + unsigned int len = ctrl & 0x7; + u8 * ref = op - ((ctrl >> 3) << 8) - 1; + + if (len == 7) + len += *ip++; /* i.e. format #2 */ + + if (fx_expect_false(op + len > out_end)) { + ip -= (len >= 7) ? 2 : 1; /* Rewind to control byte */ + goto guess; + } + if (fx_expect_false(ip >= in_end)) + return LZFX_ECORRUPT; + + ref -= *ip++; + + if (fx_expect_false(ref < (u8 *)obuf)) + return LZFX_ECORRUPT; + + do + *op++ = *ref++; + while (--len); + } + + } while (ip < in_end); + + *olen = op - (u8 *)obuf; + + return 0; guess: - rc = lzfx_getsize(ip, ilen - (ip-(u8*)ibuf), &remain_len); - if(rc>=0) *olen = remain_len + (op - (u8*)obuf); - return rc; + rc = lzfx_getsize(ip, ilen - (ip - (u8 *)ibuf), &remain_len); + if (rc >= 0) + *olen = remain_len + (op - (u8 *)obuf); + return rc; } /* Guess len. No parameters may be NULL; this is not checked. */ -static -int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen){ +static int lzfx_getsize(const void *ibuf, unsigned int ilen, unsigned int *olen) { - u8 const *ip = (const u8 *)ibuf; - u8 const *const in_end = ip + ilen; - int tot_len = 0; - - while (ip < in_end) { + u8 const * ip = (const u8 *)ibuf; + u8 const *const in_end = ip + ilen; + int tot_len = 0; - unsigned int ctrl = *ip++; + while (ip < in_end) { - if((ctrl & 0x7) == 0) { + unsigned int ctrl = *ip++; - if(ip + (ctrl >> 3) > in_end) - return LZFX_ECORRUPT; + if ((ctrl & 0x7) == 0) { - tot_len += (ctrl >> 3); - ip += (ctrl >> 3); + if (ip + (ctrl >> 3) > in_end) + return LZFX_ECORRUPT; - } else { + tot_len += (ctrl >> 3); + ip += (ctrl >> 3); - unsigned int len = ctrl & 0x7; + } else { - if(len==7){ /* i.e. format #2 */ - len += *ip++; - } + unsigned int len = ctrl & 0x7; - if(ip >= in_end) return LZFX_ECORRUPT; + if (len == 7) { /* i.e. format #2 */ + len += *ip++; + } - ip++; /* skip the ref byte */ + if (ip >= in_end) + return LZFX_ECORRUPT; - tot_len += len; - - } + ip++; /* skip the ref byte */ + tot_len += len; } + } - *olen = tot_len; + *olen = tot_len; - return 0; + return 0; } - - - - diff --git a/source/Core/lzfx/lzfx.h b/source/Core/lzfx/lzfx.h index 4d7c560b..7b6e472e 100644 --- a/source/Core/lzfx/lzfx.h +++ b/source/Core/lzfx/lzfx.h @@ -6,19 +6,19 @@ * codebase written by Marc Lehmann. This code is released under the BSD * license. License and original copyright statement follow. * - * + * * Copyright (c) 2000-2008 Marc Alexander Lehmann - * + * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. - * + * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO @@ -29,7 +29,7 @@ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. -*/ + */ #ifndef LZFX_H #define LZFX_H @@ -44,19 +44,19 @@ extern "C" { to read each other's output, although the output itself is not guaranteed to be byte-for-byte identical. */ -#define LZFX_VERSION_MAJOR 0 -#define LZFX_VERSION_MINOR 1 -#define LZFX_VERSION_STRING "0.1" +#define LZFX_VERSION_MAJOR 0 +#define LZFX_VERSION_MINOR 1 +#define LZFX_VERSION_STRING "0.1" /* Hashtable size (2**LZFX_HLOG entries) */ #ifndef LZFX_HLOG -# define LZFX_HLOG 16 +#define LZFX_HLOG 16 #endif /* Predefined errors. */ -#define LZFX_ESIZE -1 /* Output buffer too small */ -#define LZFX_ECORRUPT -2 /* Invalid data for decompression */ -#define LZFX_EARGS -3 /* Arguments invalid (NULL) */ +#define LZFX_ESIZE -1 /* Output buffer too small */ +#define LZFX_ECORRUPT -2 /* Invalid data for decompression */ +#define LZFX_EARGS -3 /* Arguments invalid (NULL) */ /* Buffer-to buffer compression. @@ -67,8 +67,7 @@ extern "C" { olen contains the compressed size in bytes. On failure, a negative value is returned and olen is not modified. */ -int lzfx_compress(const void* ibuf, unsigned int ilen, - void* obuf, unsigned int *olen); +int lzfx_compress(const void *ibuf, unsigned int ilen, void *obuf, unsigned int *olen); /* Buffer-to-buffer decompression. @@ -87,9 +86,7 @@ int lzfx_compress(const void* ibuf, unsigned int ilen, stream and is consequently very fast. Argument obuf may be NULL in this case only. */ -int lzfx_decompress(const void* ibuf, unsigned int ilen, - void* obuf, unsigned int *olen); - +int lzfx_decompress(const void *ibuf, unsigned int ilen, void *obuf, unsigned int *olen); #ifdef __cplusplus } /* extern "C" */ From 001d91988c890386c8a4f8b0de40dd22ec1039c8 Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Mon, 12 Apr 2021 20:45:26 +0800 Subject: [PATCH 6/7] Fix formatting --- source/Core/Inc/Translation.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Core/Inc/Translation.h b/source/Core/Inc/Translation.h index 9c5bcb5d..62327d7f 100644 --- a/source/Core/Inc/Translation.h +++ b/source/Core/Inc/Translation.h @@ -9,7 +9,7 @@ #define TRANSLATION_H_ #include "stdint.h" -extern const bool HasFahrenheit; +extern const bool HasFahrenheit; extern const char *SymbolPlus; extern const char *SymbolMinus; From 460d48fafbff36d677d9bba08ac9c8b5d6204fe4 Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Tue, 13 Apr 2021 18:30:27 +0800 Subject: [PATCH 7/7] Reduce code duplication --- Translations/make_translation.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Translations/make_translation.py b/Translations/make_translation.py index 8c858387..4aad6fc5 100755 --- a/Translations/make_translation.py +++ b/Translations/make_translation.py @@ -471,6 +471,8 @@ def write_language( f.write(translation_common_text) f.write( f"const bool HasFahrenheit = {('true' if lang.get('tempUnitFahrenheit', True) else 'false')};\n\n" + "extern const uint8_t *const Font_12x16 = USER_FONT_12;\n" + "extern const uint8_t *const Font_6x8 = USER_FONT_6x8;\n\n" ) if not lzfx_strings: @@ -481,8 +483,6 @@ def write_language( f.write( "const TranslationIndexTable *const Tr = &TranslationIndices;\n" "const char *const TranslationStrings = TranslationStringsData;\n\n" - "extern const uint8_t *const Font_12x16 = USER_FONT_12;\n" - "extern const uint8_t *const Font_6x8 = USER_FONT_6x8;\n\n" "void prepareTranslations() {}\n\n" ) else: @@ -491,8 +491,6 @@ def write_language( "static uint8_t translation_data_out_buffer[4096] __attribute__((__aligned__(2)));\n\n" "const TranslationIndexTable *const Tr = reinterpret_cast(translation_data_out_buffer);\n" "const char *const TranslationStrings = reinterpret_cast(translation_data_out_buffer) + sizeof(TranslationIndexTable);\n\n" - "extern const uint8_t *const Font_12x16 = USER_FONT_12;\n" - "extern const uint8_t *const Font_6x8 = USER_FONT_6x8;\n\n" "void prepareTranslations() {\n" " unsigned int outsize = sizeof(translation_data_out_buffer);\n" " lzfx_decompress(translation_data_lzfx, sizeof(translation_data_lzfx), translation_data_out_buffer, &outsize);\n"