Merge branch 'master' into patch-2

Translations/BitmapEditor.html

@@ -141,10 +141,36 @@
 				}
 			}
 		}
+		stringFromMatrix(true, false);
+	}
+	
+	function escapedToMatrix(str) {
+		app.encodedEscapeSequence = str;
+		clearMatrix();
+		var strs = str.split("\\x");
+		var c = 0;
+		var rs = 7;
+		for (var i = 0; i<strs.length; i++) {
+			var d = strs[i];
+			if (d.length > 0) {
+				v = parseInt(d, 16);
+				sv = padLeft(v.toString(2), "0", 8);
+				for (r = 0; r < 8; r++) {
+					paint(getCell(rs - r, c), sv.charAt(r) == '1');
+				}
+				c++;
+				if (c >= app.matrix.cols) {
+					c = 0;
+					rs += 8;
+				}
+			}
+		}
+		stringFromMatrix(false, true);
 	}
 
-	function stringFromMatrix() {
+	function stringFromMatrix(skipEncodedData, skipEncodedEscapeSequence) {
 		var str = "";
+		var strEscaped = "";
 		var delim = "";
 		var blocks = app.matrix.rows / 8;
 		var rs = 7;
@@ -158,11 +184,17 @@
 					}
 				}
 				str += delim + "0x" + padLeft(b.toString(16).toUpperCase(), "0", 2);
+				strEscaped += "\\x" + padLeft(b.toString(16).toUpperCase(), "0", 2);
 				delim = ",";
 			}
 			rs += 8;
 		}
-		app.encodedData = str;
+		if (!skipEncodedData) {
+			app.encodedData = str;
+		}
+		if (!skipEncodedEscapeSequence) {
+			app.encodedEscapeSequence = strEscaped;
+		}
 		return str;
 	}
 	
@@ -175,12 +207,16 @@
 					rows: 16
 				},
 				type: "big",
-				encodedData: ""
+				encodedData: "",
+				encodedEscapeSequence: "",
 			},
 			methods : {
 				VtoMatrix : function(val) {
 					toMatrix(val);
 				},
+				escapedToMatrix : function(val) {
+					escapedToMatrix(val);
+				},
 				
 				VchangeSize : function() {
 					if (app.type == "big") {
@@ -236,10 +272,11 @@
 			<input type="button" value="Clear" onclick="clearMatrix();stringFromMatrix()">
 		</div>
 		<div class="data">
-			<textarea v-model="encodedData" style="width:100%" v-on:change="VtoMatrix(encodedData)" rows=5>
+			<textarea v-model="encodedData" style="width:100%" v-on:change="VtoMatrix(encodedData)" rows=5></textarea>
+			<textarea v-model="encodedEscapeSequence" style="width:100%" v-on:change="escapedToMatrix(encodedEscapeSequence)" rows=5></textarea>
 		</div>
 	</div>
 		
 	
 </body>
-</html>
+</html>

Translations/TranslationEditor.html

@@ -48,8 +48,8 @@
 			} else if (id == "current-lang-file") {
 				if (checkTranslationFile(file.name)) {
 					app.current = json;
-					if (!app.current.cyrillicGlyphs){
+					if (!app.current.fonts){
-						app.current.cyrillicGlyphs = false;
+						app.current.fonts = ["ascii_basic"];
 					}
 					app.meta.currentLoaded = true;
 				}
@@ -137,6 +137,7 @@
 					loaded: false,
 				},
 				obsolete : {},
+				fontToAdd: "latin_extended",
 			},
 			methods : {
 				validateInput: function(valMap, id, mode) {
@@ -246,7 +247,15 @@
 					} else {
 						valMap[id] = message;
 					}
-				}
+				},
+
+				removeFont: function(i) {
+					this.current.fonts.splice(i, 1);
+				},
+
+				addFont: function() {
+					this.current.fonts.push(this.fontToAdd);
+				},
 			}
 		});
 		app.def = def;
@@ -289,12 +298,20 @@
 					<td class="value"><input type="text" v-model="current.languageLocalName" class="short"></td>
 				</tr>
 				<tr v-if="meta.currentLoaded">
-					<td class="label">Font table to use</td>
+					<td class="label">Font tables to use<br>("ascii_basic" must be first)</td>
 					<td class="value">
-						<select v-model="current.cyrillicGlyphs" v-on:change="current.cyrillicGlyphs = current.cyrillicGlyphs=='true'">
+						<ul>
-							<option value="false">Latin Extended</option>
+							<li v-for="(font, i) in current.fonts">
-							<option value="true">Cyrillic Glyphs</option>
+								<button type="button" @click="removeFont(i)" :disabled="i == 0 && font == 'ascii_basic'">-</button> {{ font }}
+							</li>
+						</ul>
+						<select v-model="fontToAdd">
+							<!-- <option value="ascii_basic">ascii_basic: ASCII Basic</option> -->
+							<option value="latin_extended">latin_extended: Latin Extended</option>
+							<option value="cyrillic">cyrillic: Cyrillic Glyphs</option>
+							<option value="cjk">cjk: Chinese/Japanese/Korean</option>
 						</select>
+						<button type="button" @click="addFont()">Add</button>
 					</td>
 				</tr>
 			</table>

Translations/lzfx.py

@@ -0,0 +1,91 @@
+import ctypes
+import functools
+import os
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+
+
+@functools.lru_cache(maxsize=None)
+def _liblzfx():
+    so_path = os.path.join(HERE, "../source/Objects/host/lzfx/liblzfx.so")
+    liblzfx = ctypes.cdll.LoadLibrary(so_path)
+    return liblzfx
+
+
+@functools.lru_cache(maxsize=None)
+def _fn_lzfx_compress():
+    """Returns the lzfx_compress C function.
+    ::
+
+        /*  Buffer-to buffer compression.
+
+            Supply pre-allocated input and output buffers via ibuf and obuf, and
+            their size in bytes via ilen and olen.  Buffers may not overlap.
+
+            On success, the function returns a non-negative value and the argument
+            olen contains the compressed size in bytes.  On failure, a negative
+            value is returned and olen is not modified.
+        */
+        int lzfx_compress(const void* ibuf, unsigned int ilen,
+                                void* obuf, unsigned int *olen);
+    """
+
+    fn = _liblzfx().lzfx_compress
+    fn.argtype = [
+        ctypes.c_char_p,
+        ctypes.c_uint,
+        ctypes.c_char_p,
+        ctypes.POINTER(ctypes.c_uint),
+    ]
+    fn.restype = ctypes.c_int
+    return fn
+
+
+def compress(data: bytes) -> bytes:
+    """Returns a bytes object of the lzfx-compressed data."""
+
+    fn_compress = _fn_lzfx_compress()
+
+    output_buffer_len = len(data) + 8
+
+    ibuf = data
+    ilen = len(ibuf)
+    obuf = ctypes.create_string_buffer(output_buffer_len)
+    olen = ctypes.c_uint(output_buffer_len)
+
+    res = fn_compress(ibuf, ilen, obuf, ctypes.byref(olen))
+
+    if res < 0:
+        raise LzfxError(res)
+    else:
+        return bytes(obuf[: olen.value])  # type: ignore
+
+
+class LzfxError(Exception):
+    """Exception raised for lzfx compression or decompression error.
+
+    Attributes:
+        error_code -- The source error code, which is a negative integer
+        error_name -- The constant name of the error
+        message -- explanation of the error
+    """
+
+    # define LZFX_ESIZE      -1      /* Output buffer too small */
+    # define LZFX_ECORRUPT   -2      /* Invalid data for decompression */
+    # define LZFX_EARGS      -3      /* Arguments invalid (NULL) */
+
+    def __init__(self, error_code):
+        self.error_code = error_code
+        if error_code == -1:
+            self.error_name = "LZFX_ESIZE"
+            self.message = "Output buffer too small"
+        elif error_code == -2:
+            self.error_name = "LZFX_ECORRUPT"
+            self.message = "Invalid data for decompression"
+        elif error_code == -3:
+            self.error_name = "LZFX_EARGS"
+            self.message = "Arguments invalid (NULL)"
+        else:
+            self.error_name = "UNKNOWN"
+            self.message = "Unknown error"

Translations/make_translation.py

@@ -19,6 +19,7 @@ from bdflib import reader as bdfreader
 from bdflib.model import Font, Glyph
 
 import font_tables
+import lzfx
 
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 
@@ -192,7 +193,7 @@ def get_letter_counts(defs: dict, lang: dict, build_version: str) -> List[str]:
     return symbols_by_occurrence
 
 
-def get_cjk_glyph(sym: str) -> str:
+def get_cjk_glyph(sym: str) -> bytes:
     glyph: Glyph = cjk_font()[ord(sym)]
 
     data = glyph.data
@@ -225,15 +226,15 @@ def get_cjk_glyph(sym: str) -> str:
     # top-most pixel. The data goes from the left-most to the right-most column
     # of the top half, then from the left-most to the right-most column of the
     # bottom half.
-    s = ""
+    bs = bytearray()
     for block in range(2):
         for c in range(dst_w):
             b = 0
             for r in range(8):
                 if get_cell(c, r + 8 * block):
                     b |= 0x01 << r
-            s += f"0x{b:02X},"
+            bs.append(b)
-    return s
+    return bytes(bs)
 
 
 def get_bytes_from_font_index(index: int) -> bytes:
@@ -291,14 +292,99 @@ def bytes_to_escaped(b: bytes) -> str:
     return "".join((f"\\x{i:02X}" for i in b))
 
 
+def bytes_to_c_hex(b: bytes) -> str:
+    return ", ".join((f"0x{i:02X}" for i in b)) + ","
+
+
 @dataclass
 class FontMap:
-    font12: Dict[str, str]
+    font12: Dict[str, bytes]
-    font06: Dict[str, str]
+    font06: Dict[str, Optional[bytes]]
+
+
+@dataclass
+class FontMapsPerFont:
+    font12_maps: Dict[str, Dict[str, bytes]]
+    font06_maps: Dict[str, Dict[str, Optional[bytes]]]
+    sym_lists: Dict[str, List[str]]
+
+
+def get_font_map_per_font(text_list: List[str], fonts: List[str]) -> FontMapsPerFont:
+    pending_sym_set = set(text_list)
+    if len(pending_sym_set) != len(text_list):
+        raise ValueError("`text_list` contains duplicated symbols")
+
+    if fonts[0] != font_tables.NAME_ASCII_BASIC:
+        raise ValueError(
+            f'First item in `fonts` must be "{font_tables.NAME_ASCII_BASIC}"'
+        )
+
+    total_symbol_count = len(text_list)
+    # \x00 is for NULL termination and \x01 is for newline, so the maximum
+    # number of symbols allowed is as follow (see also the comments in
+    # `get_bytes_from_font_index`):
+    if total_symbol_count > (0x10 * 0xFF - 15) - 2:  # 4063
+        raise ValueError(
+            f"Error, too many used symbols for this version (total {total_symbol_count})"
+        )
+
+    logging.info(f"Generating fonts for {total_symbol_count} symbols")
+
+    # Collect font bitmaps by the defined font order:
+    font12_maps: Dict[str, Dict[str, bytes]] = {}
+    font06_maps: Dict[str, Dict[str, Optional[bytes]]] = {}
+    sym_lists: Dict[str, List[str]] = {}
+    for font in fonts:
+        font12_maps[font] = {}
+        font12_map = font12_maps[font]
+        font06_maps[font] = {}
+        font06_map = font06_maps[font]
+        sym_lists[font] = []
+        sym_list = sym_lists[font]
+
+        if len(pending_sym_set) == 0:
+            logging.warning(
+                f"Font {font} not used because all symbols already have font bitmaps"
+            )
+            continue
+
+        if font == font_tables.NAME_CJK:
+            is_cjk = True
+        else:
+            is_cjk = False
+            font12: Dict[str, bytes]
+            font06: Dict[str, bytes]
+            font12, font06 = font_tables.get_font_maps_for_name(font)
+
+        for sym in text_list:
+            if sym not in pending_sym_set:
+                continue
+            if is_cjk:
+                font12_line = get_cjk_glyph(sym)
+                if font12_line is None:
+                    continue
+                font06_line = None
+            else:
+                try:
+                    font12_line = font12[sym]
+                    font06_line = font06[sym]
+                except KeyError:
+                    continue
+            font12_map[sym] = font12_line
+            font06_map[sym] = font06_line
+            sym_list.append(sym)
+            pending_sym_set.remove(sym)
+
+        if len(sym_list) == 0:
+            logging.warning(f"Font {font} not used by any symbols on the list")
+    if len(pending_sym_set) > 0:
+        raise KeyError(f"Symbols not found in specified fonts: {pending_sym_set}")
+
+    return FontMapsPerFont(font12_maps, font06_maps, sym_lists)
 
 
 def get_font_map_and_table(
-    text_list: List[str],
+    text_list: List[str], fonts: List[str]
 ) -> Tuple[List[str], FontMap, Dict[str, bytes]]:
     # the text list is sorted
     # allocate out these in their order as number codes
@@ -306,81 +392,71 @@ def get_font_map_and_table(
     index = 2  # start at 2, as 0= null terminator,1 = new line
     forced_first_symbols = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
 
-    # Get the font table, which does not include CJK chars
+    # We enforce that numbers come first.
-    font_table = font_tables.get_font_map()
+    text_list = forced_first_symbols + [
-    font_small_table = font_tables.get_small_font_map()
+        x for x in text_list if x not in forced_first_symbols
-
-    # We want to put all CJK chars after non-CJK ones so that the CJK chars
-    # do not need to be in the small font table to save space.
-    # We assume all symbols not in the font table to be a CJK char.
-    # We also enforce that numbers are first.
-    ordered_normal_sym_list: List[str] = forced_first_symbols + [
-        x for x in text_list if x not in forced_first_symbols and x in font_table
-    ]
-    ordered_cjk_sym_list: List[str] = [
-        x for x in text_list if x not in forced_first_symbols and x not in font_table
     ]
 
-    total_symbol_count = len(ordered_normal_sym_list) + len(ordered_cjk_sym_list)
+    font_maps = get_font_map_per_font(text_list, fonts)
-    # \x00 is for NULL termination and \x01 is for newline, so the maximum
+    font12_maps = font_maps.font12_maps
-    # number of symbols allowed is as follow (see also the comments in
+    font06_maps = font_maps.font06_maps
-    # `get_bytes_from_font_index`):
-    if total_symbol_count > (0x10 * 0xFF - 15) - 2:  # 4063
-        logging.error(
-            f"Error, too many used symbols for this version (total {total_symbol_count})"
-        )
-        sys.exit(1)
 
-    logging.info(f"Generating fonts for {total_symbol_count} symbols")
+    # Build the full font maps
+    font12_map = {}
+    font06_map = {}
+    for font in fonts:
+        font12_map.update(font12_maps[font])
+        font06_map.update(font06_maps[font])
 
-    sym_list = ordered_normal_sym_list + ordered_cjk_sym_list
+    # Collect all symbols by the original symbol order, but also making sure
-    for sym in sym_list:
+    # all symbols with only large font must be placed after all symbols with
-        if sym in symbol_map:
+    # both small and large fonts
-            raise ValueError("Symbol not found in symbol map")
+    sym_list_both_fonts = []
+    sym_list_large_only = []
+    for sym in text_list:
+        if font06_map[sym] is None:
+            sym_list_large_only.append(sym)
+        else:
+            sym_list_both_fonts.append(sym)
+    sym_list = sym_list_both_fonts + sym_list_large_only
+
+    # Assign symbol bytes by font index
+    for index, sym in enumerate(sym_list, index):
+        assert sym not in symbol_map
         symbol_map[sym] = get_bytes_from_font_index(index)
-        index += 1
-
-    font12_map: Dict[str, str] = {}
-    font06_map: Dict[str, str] = {}
-    for sym in ordered_normal_sym_list:
-        if sym not in font_table:
-            logging.error(f"Missing Large font element for {sym}")
-            sys.exit(1)
-        font12_map[sym] = font_table[sym]
-        if sym not in font_small_table:
-            logging.error(f"Missing Small font element for {sym}")
-            sys.exit(1)
-        font06_map[sym] = font_small_table[sym]
-
-    for sym in ordered_cjk_sym_list:
-        if sym in font_table:
-            raise ValueError("Symbol already exists in font_table")
-        font_line: str = get_cjk_glyph(sym)
-        if font_line is None:
-            logging.error(f"Missing Large font element for {sym}")
-            sys.exit(1)
-        font12_map[sym] = font_line
-        # No data to add to the small font table
-        font06_map[sym] = "//                                 "  # placeholder
 
     return sym_list, FontMap(font12_map, font06_map), symbol_map
 
 
 def make_font_table_cpp(
     sym_list: List[str], font_map: FontMap, symbol_map: Dict[str, bytes]
+) -> str:
+    output_table = make_font_table_12_cpp(sym_list, font_map, symbol_map)
+    output_table += make_font_table_06_cpp(sym_list, font_map, symbol_map)
+    return output_table
+
+
+def make_font_table_12_cpp(
+    sym_list: List[str], font_map: FontMap, symbol_map: Dict[str, bytes]
 ) -> str:
     output_table = "const uint8_t USER_FONT_12[] = {\n"
     for sym in sym_list:
-        output_table += (
+        output_table += f"{bytes_to_c_hex(font_map.font12[sym])}//{bytes_to_escaped(symbol_map[sym])} -> {sym}\n"
-            f"{font_map.font12[sym]}//{bytes_to_escaped(symbol_map[sym])} -> {sym}\n"
-        )
     output_table += "};\n"
+    return output_table
 
-    output_table += "const uint8_t USER_FONT_6x8[] = {\n"
+
+def make_font_table_06_cpp(
+    sym_list: List[str], font_map: FontMap, symbol_map: Dict[str, bytes]
+) -> str:
+    output_table = "const uint8_t USER_FONT_6x8[] = {\n"
     for sym in sym_list:
-        output_table += (
+        font_bytes = font_map.font06[sym]
-            f"{font_map.font06[sym]}//{bytes_to_escaped(symbol_map[sym])} -> {sym}\n"
+        if font_bytes:
-        )
+            font_line = bytes_to_c_hex(font_bytes)
+        else:
+            font_line = "//                                 "  # placeholder
+        output_table += f"{font_line}//{bytes_to_escaped(symbol_map[sym])} -> {sym}\n"
     output_table += "};\n"
     return output_table
 
@@ -433,14 +509,20 @@ def prepare_language(lang: dict, defs: dict, build_version: str) -> LanguageData
     # Iterate over all of the text to build up the symbols & counts
     text_list = get_letter_counts(defs, lang, build_version)
     # From the letter counts, need to make a symbol translator & write out the font
-    sym_list, font_map, symbol_conversion_table = get_font_map_and_table(text_list)
+    fonts = lang["fonts"]
+    sym_list, font_map, symbol_conversion_table = get_font_map_and_table(
+        text_list, fonts
+    )
     return LanguageData(
         lang, defs, build_version, sym_list, font_map, symbol_conversion_table
     )
 
 
 def write_language(
-    data: LanguageData, f: TextIO, lzfx_strings: Optional[bytes] = None
+    data: LanguageData,
+    f: TextIO,
+    strings_bin: Optional[bytes] = None,
+    compress_font: bool = False,
 ) -> None:
     lang = data.lang
     defs = data.defs
@@ -451,18 +533,36 @@ def write_language(
 
     language_code: str = lang["languageCode"]
     logging.info(f"Generating block for {language_code}")
-    font_table_text = make_font_table_cpp(sym_list, font_map, symbol_conversion_table)
 
     try:
         lang_name = lang["languageLocalName"]
     except KeyError:
         lang_name = language_code
 
-    if lzfx_strings:
+    if strings_bin or compress_font:
         f.write('#include "lzfx.h"\n')
 
     f.write(f"\n// ---- {lang_name} ----\n\n")
-    f.write(font_table_text)
+
+    if not compress_font:
+        font_table_text = make_font_table_cpp(
+            sym_list, font_map, symbol_conversion_table
+        )
+        f.write(font_table_text)
+    else:
+        font12_uncompressed = bytearray()
+        for sym in sym_list:
+            font12_uncompressed.extend(font_map.font12[sym])
+        font12_compressed = lzfx.compress(bytes(font12_uncompressed))
+        logging.info(
+            f"Font table 12x16 compressed from {len(font12_uncompressed)} to {len(font12_compressed)} bytes (ratio {len(font12_compressed) / len(font12_uncompressed):.3})"
+        )
+        write_bytes_as_c_array(f, "font_12x16_lzfx", font12_compressed)
+        font_table_text = make_font_table_06_cpp(
+            sym_list, font_map, symbol_conversion_table
+        )
+        f.write(font_table_text)
+
     f.write(f"\n// ---- {lang_name} ----\n\n")
 
     translation_common_text = get_translation_common_text(
@@ -471,11 +571,18 @@ def write_language(
     f.write(translation_common_text)
     f.write(
         f"const bool HasFahrenheit = {('true' if lang.get('tempUnitFahrenheit', True) else 'false')};\n\n"
-        "extern const uint8_t *const Font_12x16 = USER_FONT_12;\n"
-        "extern const uint8_t *const Font_6x8 = USER_FONT_6x8;\n\n"
     )
 
-    if not lzfx_strings:
+    if not compress_font:
+        f.write("extern const uint8_t *const Font_12x16 = USER_FONT_12;\n")
+    else:
+        f.write(
+            f"static uint8_t font_out_buffer[{len(font12_uncompressed)}];\n\n"
+            "extern const uint8_t *const Font_12x16 = font_out_buffer;\n"
+        )
+    f.write("extern const uint8_t *const Font_6x8 = USER_FONT_6x8;\n\n")
+
+    if not strings_bin:
         translation_strings_and_indices_text = get_translation_strings_and_indices_text(
             lang, defs, symbol_conversion_table
         )
@@ -483,20 +590,35 @@ def write_language(
         f.write(
             "const TranslationIndexTable *const Tr = &TranslationIndices;\n"
             "const char *const TranslationStrings = TranslationStringsData;\n\n"
-            "void prepareTranslations() {}\n\n"
         )
     else:
-        write_bytes_as_c_array(f, "translation_data_lzfx", lzfx_strings)
+        compressed = lzfx.compress(strings_bin)
+        logging.info(
+            f"Strings compressed from {len(strings_bin)} to {len(compressed)} bytes (ratio {len(compressed) / len(strings_bin):.3})"
+        )
+        write_bytes_as_c_array(f, "translation_data_lzfx", compressed)
         f.write(
-            "static uint8_t translation_data_out_buffer[4096] __attribute__((__aligned__(2)));\n\n"
+            f"static uint8_t translation_data_out_buffer[{len(strings_bin)}] __attribute__((__aligned__(2)));\n\n"
             "const TranslationIndexTable *const Tr = reinterpret_cast<const TranslationIndexTable *>(translation_data_out_buffer);\n"
             "const char *const TranslationStrings = reinterpret_cast<const char *>(translation_data_out_buffer) + sizeof(TranslationIndexTable);\n\n"
-            "void prepareTranslations() {\n"
-            "  unsigned int outsize = sizeof(translation_data_out_buffer);\n"
-            "  lzfx_decompress(translation_data_lzfx, sizeof(translation_data_lzfx), translation_data_out_buffer, &outsize);\n"
-            "}\n\n"
         )
 
+    if not strings_bin and not compress_font:
+        f.write("void prepareTranslations() {}\n\n")
+    else:
+        f.write("void prepareTranslations() {\n" "  unsigned int outsize;\n")
+        if compress_font:
+            f.write(
+                "  outsize = sizeof(font_out_buffer);\n"
+                "  lzfx_decompress(font_12x16_lzfx, sizeof(font_12x16_lzfx), font_out_buffer, &outsize);\n"
+            )
+        if strings_bin:
+            f.write(
+                "  outsize = sizeof(translation_data_out_buffer);\n"
+                "  lzfx_decompress(translation_data_lzfx, sizeof(translation_data_lzfx), translation_data_out_buffer, &outsize);\n"
+            )
+        f.write("}\n\n")
+
     sanity_checks_text = get_translation_sanity_checks_text(defs)
     f.write(sanity_checks_text)
 
@@ -782,11 +904,18 @@ def parse_args() -> argparse.Namespace:
         dest="input_pickled",
     )
     parser.add_argument(
-        "--lzfx-strings",
+        "--strings-bin",
-        help="Use compressed TranslationIndices + TranslationStrings data",
+        help="Use generated TranslationIndices + TranslationStrings data and compress them",
         type=argparse.FileType("rb"),
         required=False,
-        dest="lzfx_strings",
+        dest="strings_bin",
+    )
+    parser.add_argument(
+        "--compress-font",
+        help="Compress the font table",
+        action="store_true",
+        required=False,
+        dest="compress_font",
     )
     parser.add_argument(
         "--output", "-o", help="Target file", type=argparse.FileType("w"), required=True
@@ -830,10 +959,15 @@ def main() -> None:
 
     out_ = args.output
     write_start(out_)
-    if args.lzfx_strings:
+    if args.strings_bin:
-        write_language(language_data, out_, args.lzfx_strings.read())
+        write_language(
+            language_data,
+            out_,
+            args.strings_bin.read(),
+            compress_font=args.compress_font,
+        )
     else:
-        write_language(language_data, out_)
+        write_language(language_data, out_, compress_font=args.compress_font)
 
     if args.output_pickled:
         logging.info(f"Writing pickled data to {args.output_pickled.name}")

Translations/make_translation_test.py

@@ -24,6 +24,12 @@ class TestMakeTranslation(unittest.TestCase):
         self.assertEqual(bytes_to_escaped(b"\x00"), "\\x00")
         self.assertEqual(bytes_to_escaped(b"\xF1\xAB"), "\\xF1\\xAB")
 
+    def test_bytes_to_c_hex(self):
+        from make_translation import bytes_to_c_hex
+
+        self.assertEqual(bytes_to_c_hex(b"\x00"), "0x00,")
+        self.assertEqual(bytes_to_c_hex(b"\xF1\xAB"), "0xF1, 0xAB,")
+
 
 if __name__ == "__main__":
     unittest.main()

Translations/translation_BG.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "BG",
 	"languageLocalName": "Български",
-	"cyrillicGlyphs": true,
+	"fonts": [
+		"ascii_basic",
+		"cyrillic"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Калибрацията завършена!",
 		"SettingsCalibrationWarning": "Уверете се, че върха на поялника е със стайна температура преди да продължите!",

Translations/translation_CS.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "CS",
 	"languageLocalName": "Český",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Kalibrace dokončena!",
 		"SettingsCalibrationWarning": "Ujistěte se, že hrot má pokojovou teplotu!",

Translations/translation_DA.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "DA",
 	"languageLocalName": "Dansk",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Calibration done!",
 		"SettingsCalibrationWarning": "Sørg for at loddespidsen er ved stuetemperatur, inden du fortsætter!",

Translations/translation_DE.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "DE",
 	"languageLocalName": "Deutsch",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"tempUnitFahrenheit": false,
 	"messages": {
 		"SettingsCalibrationDone": "Kalibrierung abgeschlossen!",

Translations/translation_EN.json

@@ -1,7 +1,9 @@
 {
 	"languageCode": "EN",
 	"languageLocalName": "English",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic"
+	],
 	"tempUnitFahrenheit": true,
 	"messages": {
 		"SettingsCalibrationDone": "Calibration done!",

Translations/translation_ES.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "ES",
 	"languageLocalName": "Castellano",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "¡Calibrada!",
 		"SettingsCalibrationWarning": "¡Asegúrate que la punta esté a temperatura ambiente antes de empezar!",

Translations/translation_FI.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "FI",
 	"languageLocalName": "Suomi",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationWarning": "Varmista että kärki on huoneenlämpöinen ennen jatkamista!",
 		"SettingsResetWarning": "Haluatko varmasti palauttaa oletusarvot?",
@@ -301,4 +304,4 @@
 			"desc": "Herätyspulssin kesto (x 250ms)"
 		}
 	}
-}
+}

Translations/translation_FR.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "FR",
 	"languageLocalName": "Français",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Calibration effectuée !",
 		"SettingsCalibrationWarning": "Assurez-vous que la panne soit à température ambiante avant de continuer !",

Translations/translation_HR.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "HR",
 	"languageLocalName": "Hrvatski",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Kalibracija gotova!",
 		"SettingsCalibrationWarning": "Provjerite da je vršak ohlađen na sobnu temperaturu prije nego što nastavite!",

Translations/translation_HU.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "HU",
 	"languageLocalName": "Magyar",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Kalibráció befejezve!",
 		"SettingsCalibrationWarning": "Folytatás előtt győződjön meg róla, hogy a páka szobahőmérsékletű!",

Translations/translation_IT.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "IT",
 	"languageLocalName": "Italiano",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Calibrazione effettuata",
 		"SettingsCalibrationWarning": "Assicurati che la punta si trovi a temperatura ambiente prima di continuare!",

Translations/translation_JA_JP.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "JA_JP",
 	"languageLocalName": "日本語",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"cjk"
+	],
 	"tempUnitFahrenheit": true,
 	"messages": {
 		"SettingsCalibrationDone": "校正完了",
@@ -200,4 +203,4 @@
 			"desc": "電源供給元をオンに保つために使用される、電力パルスの時間長 <x250ms（ミリ秒）>"
 		}
 	}
-}
+}

Translations/translation_LT.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "LT",
 	"languageLocalName": "Lietuvių",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Kalibravimas atliktas!",
 		"SettingsCalibrationWarning": "Prieš tęsdami įsitikinkite, kad antgalis yra kambario temperatūros!",

Translations/translation_NL.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "NL",
 	"languageLocalName": "Nederlands",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Calibratie klaar!",
 		"SettingsCalibrationWarning": "Zorg ervoor dat te punt op kamertemperatuur is voor je verder gaat!",

Translations/translation_NL_BE.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "NL_BE",
 	"languageLocalName": "Vlaams",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Gecalibreerd!",
 		"SettingsCalibrationWarning": "Zorg vooraf dat de punt op kamertemperatuur is!",

Translations/translation_NO.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "NO",
 	"languageLocalName": "Norsk",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Calibration done!",
 		"SettingsCalibrationWarning": "Sørg for at loddespissen har romtemperatur før du fortsetter!",

Translations/translation_PL.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "PL",
 	"languageLocalName": "Polski",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"tempUnitFahrenheit": false,
 	"messages": {
 		"SettingsCalibrationDone": "Kalibracja udana!",

Translations/translation_PT.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "PT",
 	"languageLocalName": "Português",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Calibração terminada!",
 		"SettingsCalibrationWarning": "A ponta deve estar à temperatura ambiente antes de continuar!",

Translations/translation_RU.json

@@ -1,7 +1,11 @@
 {
 	"languageCode": "RU",
 	"languageLocalName": "Русский",
-	"cyrillicGlyphs": true,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended",
+		"cyrillic"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Калибровка завершена!",
 		"SettingsCalibrationWarning": "Прежде чем продолжить, пожалуйста, убедитесь, что жало имеет комнатную температуру!",

Translations/translation_SK.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "SK",
 	"languageLocalName": "Slovenčina",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Kalibrácia hotová!",
 		"SettingsCalibrationWarning": "Najprv sa prosím uistite, že hrot má izbovú teplotu!",

Translations/translation_SL.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "SL",
 	"languageLocalName": "Slovenščina",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Kalibracija opravljena!",
 		"SettingsCalibrationWarning": "Pred nadaljevanjem mora biti konica segreta na sobno temperaturo!",

Translations/translation_SR_CYRL.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "SR_CYRL",
 	"languageLocalName": "Српски",
-	"cyrillicGlyphs": true,
+	"fonts": [
+		"ascii_basic",
+		"cyrillic"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Калибрација готова",
 		"SettingsCalibrationWarning": "Проверите да ли је врх охлађен на собну температуру пре него што наставите",

Translations/translation_SR_LATN.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "SR_LATN",
 	"languageLocalName": "Srpski",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Kalibracija gotova",
 		"SettingsCalibrationWarning": "Proverite da li je vrh ohlađen na sobnu temperaturu pre nego što nastavite",

Translations/translation_SV.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "SV",
 	"languageLocalName": "Svenska",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Calibration done!",
 		"SettingsCalibrationWarning": "Please ensure the tip is at room temperature before continuing!",

Translations/translation_TR.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "TR",
 	"languageLocalName": "Türkçe",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Kalibrasyon tamamlandı!",
 		"SettingsCalibrationWarning": "Lütfen devam etmeden önce ucun oda sıcaklığında olduğunu garantiye alın!",

Translations/translation_UK.json

@@ -1,7 +1,11 @@
 {
 	"languageCode": "UK",
 	"languageLocalName": "Українська",
-	"cyrillicGlyphs": true,
+	"fonts": [
+		"ascii_basic",
+		"latin_extended",
+		"cyrillic"
+	],
 	"messages": {
 		"SettingsCalibrationDone": "Калібрування виконане!",
 		"SettingsCalibrationWarning": "Переконайтеся, що жало охололо до кімнатної температури, перш ніж продовжувати!",

Translations/translation_YUE_HK.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "YUE_HK",
 	"languageLocalName": "廣東話 (香港)",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"cjk"
+	],
 	"tempUnitFahrenheit": true,
 	"messages": {
 		"SettingsCalibrationDone": "校正完成！",

Translations/translation_ZH_CN.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "ZH_CN",
 	"languageLocalName": "简体中文",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"cjk"
+	],
 	"tempUnitFahrenheit": true,
 	"messages": {
 		"SettingsCalibrationDone": "校正完成！",

Translations/translation_ZH_TW.json

@@ -1,7 +1,10 @@
 {
 	"languageCode": "ZH_TW",
 	"languageLocalName": "正體中文",
-	"cyrillicGlyphs": false,
+	"fonts": [
+		"ascii_basic",
+		"cjk"
+	],
 	"tempUnitFahrenheit": true,
 	"messages": {
 		"SettingsCalibrationDone": "校正完成！",

source/Core/lzfx/lzfx-host-compress.c

@@ -1,48 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "lzfx.h"
-
-/* Program to demonstrate file compression. Don't use it in the real world! */
-
-int main(int argc, char **argv) {
-  if (argc == 3) {
-    /* open input */
-    FILE *f = fopen(argv[1], "rb");
-    if (!f) {
-      printf("Error: %s\n", argv[1]);
-      return 1;
-    }
-
-    /* get size */
-    fseek(f, 0, SEEK_END);
-    int inputsize = ftell(f);
-    fseek(f, 0, SEEK_SET);
-
-    /* read */
-    char *input = malloc(inputsize);
-    fread(input, inputsize, 1, f);
-    fclose(f);
-
-    /* compress */
-    int   outputsize = inputsize + 1; /* buffer overflow */
-    char *output     = malloc(outputsize);
-    lzfx_compress(input, inputsize, output, &outputsize);
-
-    /* open output */
-    f = fopen(argv[2], "wb");
-    if (!f) {
-      printf("Error: %s\n", argv[1]);
-      return 1;
-    }
-
-    /* write */
-    fwrite(output, outputsize, 1, f);
-    fclose(f);
-
-    return 0;
-  } else {
-    printf("Compresses a file.\n\nUsage: lzfx-raw input output\n");
-    return 1;
-  }
-}

source/Makefile

@@ -327,6 +327,16 @@ $(HEXFILE_DIR)/$(model)_string_compressed_%.elf : \
 		$(OUTPUT_DIR)/Core/Gen/Translation_lzfx.$*.o \
 		$(LIBS) $(LINKER_FLAGS) -o$@ -Wl,-Map=$@.map
 
+$(HEXFILE_DIR)/$(model)_font_compressed_%.elf : \
+		$(OUT_OBJS_S) $(OUT_OBJS) $(OUT_OBJS_CPP) \
+		$(OUTPUT_DIR)/Core/Gen/Translation_lzfx_font.%.o \
+		Makefile $(LDSCRIPT)
+	@test -d $(@D) || mkdir -p $(@D)
+	@echo Linking $@
+	@$(CPP) $(CXXFLAGS) $(OUT_OBJS_S) $(OUT_OBJS) $(OUT_OBJS_CPP) \
+		$(OUTPUT_DIR)/Core/Gen/Translation_lzfx_font.$*.o \
+		$(LIBS) $(LINKER_FLAGS) -o$@ -Wl,-Map=$@.map
+
 $(OUT_OBJS): $(OUTPUT_DIR)/%.o : %.c Makefile
 	@test -d $(@D) || mkdir -p $(@D)
 	@echo Compiling ${<}
@@ -364,10 +374,10 @@ $(OUTPUT_DIR)/Core/Gen/translation.files/%.o: Core/Gen/Translation.%.cpp
 	@echo Generating $@
 	@$(CPP) -c $(filter-out -flto -g3,$(CXXFLAGS)) $< -o $@
 
-$(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress: Core/lzfx/lzfx-host-compress.c Core/lzfx/lzfx.c
+$(HOST_OUTPUT_DIR)/lzfx/liblzfx.so: Core/lzfx/lzfx.c
 	@test -d $(@D) || mkdir -p $(@D)
-	@echo Building host lzfx tool $@
+	@echo Building host lzfx shared library $@
-	@$(HOST_CC) -Wno-unused-result -O $^ -o $@
+	@$(HOST_CC) -Wno-unused-result -fPIC -shared -O $^ -o $@
 
 $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin: $(OUTPUT_DIR)/Core/Gen/translation.files/%.o
 	@echo Dumping translation strings data from $<
@@ -378,18 +388,22 @@ $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin: $(OUTPUT_DIR)/Core/Gen/t
 	@test -s $(@D)/$*.data.TranslationStrings.bin || (rm $(@D)/$*.data.TranslationStrings.bin; echo 'ERROR: Output for .rodata._ZL22TranslationStringsData is empty!' >&2; false)
 	@cat $(@D)/$*.data.TranslationIndices.bin $(@D)/$*.data.TranslationStrings.bin > $@
 
-$(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.lzfx: $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin $(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress
+Core/Gen/Translation_lzfx.%.cpp: $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.bin $(OUTPUT_DIR)/Core/Gen/translation.files/%.pickle $(HOST_OUTPUT_DIR)/lzfx/liblzfx.so
-	@echo Compressing translation strings data for $*
-	@$(HOST_OUTPUT_DIR)/lzfx/lzfx-host-compress $< $@
-	@echo Compressed from $$(stat --printf="%s" $<) to $$(stat --printf="%s" $@) bytes
-
-Core/Gen/Translation_lzfx.%.cpp: $(OUTPUT_DIR)/Core/Gen/translation.files/%.strings.lzfx $(OUTPUT_DIR)/Core/Gen/translation.files/%.pickle
 	@test -d $(@D) || mkdir -p $(@D)
 	@echo Generating lzfx compressed translation for $*
 	@python3 ../Translations/make_translation.py \
 		-o $(PWD)/Core/Gen/Translation_lzfx.$*.cpp \
 		--input-pickled $(OUTPUT_DIR)/Core/Gen/translation.files/$*.pickle \
-		--lzfx-strings $(OUTPUT_DIR)/Core/Gen/translation.files/$*.strings.lzfx \
+		--strings-bin $(OUTPUT_DIR)/Core/Gen/translation.files/$*.strings.bin \
+		$*
+
+Core/Gen/Translation_lzfx_font.%.cpp: $(OUTPUT_DIR)/Core/Gen/translation.files/%.pickle $(HOST_OUTPUT_DIR)/lzfx/liblzfx.so
+	@test -d $(@D) || mkdir -p $(@D)
+	@echo Generating lzfx compressed translation for $*
+	@python3 ../Translations/make_translation.py \
+		-o $(PWD)/Core/Gen/Translation_lzfx_font.$*.cpp \
+		--input-pickled $(OUTPUT_DIR)/Core/Gen/translation.files/$*.pickle \
+		--compress-font \
 		$*