From 4061a35b6f783ba28972aea7167591be44738fbb Mon Sep 17 00:00:00 2001 From: Alvin Wong Date: Fri, 9 Apr 2021 23:04:38 +0800 Subject: [PATCH] Refactor make_translation bytes escaping - Change font table to store bytes instead of escape sequences - Add function to convert bytes to escape sequences - Update tests --- Translations/make_translation.py | 46 +++++++++++++++++---------- Translations/make_translation_test.py | 30 ++++++++++------- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/Translations/make_translation.py b/Translations/make_translation.py index 6e0b9366..fefc2fe1 100755 --- a/Translations/make_translation.py +++ b/Translations/make_translation.py @@ -234,10 +234,9 @@ def get_cjk_glyph(sym: str) -> str: return s -def get_chars_from_font_index(index: int) -> str: +def get_bytes_from_font_index(index: int) -> bytes: """ - Converts the font table index into its corresponding string escape - sequence(s). + Converts the font table index into its corresponding bytes """ # We want to be able to use more than 254 symbols (excluding \x00 null @@ -273,7 +272,7 @@ def get_chars_from_font_index(index: int) -> str: if page > 0x0F: raise ValueError("page value out of range") if page == 0: - return f"\\x{index:02X}" + return bytes([index]) else: # Into extended range # Leader is 0xFz where z is the page number @@ -283,13 +282,17 @@ def get_chars_from_font_index(index: int) -> str: if leader > 0xFF or value > 0xFF: raise ValueError("value is out of range") - return f"\\x{leader:02X}\\x{value:02X}" + return bytes([leader, value]) -def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, str]]: +def bytes_to_escaped(b: bytes) -> str: + return "".join((f"\\x{i:02X}" for i in b)) + + +def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, bytes]]: # the text list is sorted # allocate out these in their order as number codes - symbol_map = {"\n": "\\x01"} + symbol_map: Dict[str, bytes] = {"\n": bytes([1])} index = 2 # start at 2, as 0= null terminator,1 = new line forced_first_symbols = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] @@ -311,7 +314,7 @@ def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, str]]: total_symbol_count = len(ordered_normal_sym_list) + len(ordered_cjk_sym_list) # \x00 is for NULL termination and \x01 is for newline, so the maximum # number of symbols allowed is as follow (see also the comments in - # `get_chars_from_font_index`): + # `get_bytes_from_font_index`): if total_symbol_count > (0x10 * 0xFF - 15) - 2: # 4063 logging.error( f"Error, too many used symbols for this version (total {total_symbol_count})" @@ -323,7 +326,7 @@ def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, str]]: for sym in chain(ordered_normal_sym_list, ordered_cjk_sym_list): if sym in symbol_map: raise ValueError("Symbol not found in symbol map") - symbol_map[sym] = get_chars_from_font_index(index) + symbol_map[sym] = get_bytes_from_font_index(index) index += 1 font_table_strings = [] @@ -333,12 +336,16 @@ def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, str]]: logging.error(f"Missing Large font element for {sym}") sys.exit(1) font_line: str = font_table[sym] - font_table_strings.append(f"{font_line}//{symbol_map[sym]} -> {sym}") + font_table_strings.append( + f"{font_line}//{bytes_to_escaped(symbol_map[sym])} -> {sym}" + ) if sym not in font_small_table: logging.error(f"Missing Small font element for {sym}") sys.exit(1) font_line: str = font_small_table[sym] - font_small_table_strings.append(f"{font_line}//{symbol_map[sym]} -> {sym}") + font_small_table_strings.append( + f"{font_line}//{bytes_to_escaped(symbol_map[sym])} -> {sym}" + ) for sym in ordered_cjk_sym_list: if sym in font_table: @@ -347,10 +354,12 @@ def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, str]]: if font_line is None: logging.error(f"Missing Large font element for {sym}") sys.exit(1) - font_table_strings.append(f"{font_line}//{symbol_map[sym]} -> {sym}") + font_table_strings.append( + f"{font_line}//{bytes_to_escaped(symbol_map[sym])} -> {sym}" + ) # No data to add to the small font table font_small_table_strings.append( - f"// {symbol_map[sym]} -> {sym}" + f"// {bytes_to_escaped(symbol_map[sym])} -> {sym}" ) output_table = "const uint8_t USER_FONT_12[] = {\n" @@ -366,9 +375,9 @@ def get_font_map_and_table(text_list: List[str]) -> Tuple[str, Dict[str, str]]: return output_table, symbol_map -def convert_string(symbol_conversion_table: Dict[str, str], text: str) -> str: - # convert all of the symbols from the string into escapes for their content - output_string = "" +def convert_string_bytes(symbol_conversion_table: Dict[str, bytes], text: str) -> bytes: + # convert all of the symbols from the string into bytes for their content + output_string = b"" for c in text.replace("\\r", "").replace("\\n", "\n"): if c not in symbol_conversion_table: logging.error(f"Missing font definition for {c}") @@ -378,6 +387,11 @@ def convert_string(symbol_conversion_table: Dict[str, str], text: str) -> str: return output_string +def convert_string(symbol_conversion_table: Dict[str, bytes], text: str) -> str: + # convert all of the symbols from the string into escapes for their content + return bytes_to_escaped(convert_string_bytes(symbol_conversion_table, text)) + + def escape(string: str) -> str: return json.dumps(string, ensure_ascii=False) diff --git a/Translations/make_translation_test.py b/Translations/make_translation_test.py index ea091fd7..39b6f83b 100644 --- a/Translations/make_translation_test.py +++ b/Translations/make_translation_test.py @@ -3,20 +3,26 @@ import unittest class TestMakeTranslation(unittest.TestCase): - def test_get_chars_from_font_index(self): - from make_translation import get_chars_from_font_index + def test_get_bytes_from_font_index(self): + from make_translation import get_bytes_from_font_index - self.assertEqual(get_chars_from_font_index(2), "\\x02") - self.assertEqual(get_chars_from_font_index(239), "\\xEF") - self.assertEqual(get_chars_from_font_index(240), "\\xF0") - self.assertEqual(get_chars_from_font_index(241), "\\xF1\\x01") - self.assertEqual(get_chars_from_font_index(495), "\\xF1\\xFF") - self.assertEqual(get_chars_from_font_index(496), "\\xF2\\x01") - self.assertEqual(get_chars_from_font_index(750), "\\xF2\\xFF") - self.assertEqual(get_chars_from_font_index(751), "\\xF3\\x01") - self.assertEqual(get_chars_from_font_index(0x10 * 0xFF - 15), "\\xFF\\xFF") + self.assertEqual(get_bytes_from_font_index(2), b"\x02") + self.assertEqual(get_bytes_from_font_index(239), b"\xEF") + self.assertEqual(get_bytes_from_font_index(240), b"\xF0") + self.assertEqual(get_bytes_from_font_index(241), b"\xF1\x01") + self.assertEqual(get_bytes_from_font_index(495), b"\xF1\xFF") + self.assertEqual(get_bytes_from_font_index(496), b"\xF2\x01") + self.assertEqual(get_bytes_from_font_index(750), b"\xF2\xFF") + self.assertEqual(get_bytes_from_font_index(751), b"\xF3\x01") + self.assertEqual(get_bytes_from_font_index(0x10 * 0xFF - 15), b"\xFF\xFF") with self.assertRaises(ValueError): - get_chars_from_font_index(0x10 * 0xFF - 14) + get_bytes_from_font_index(0x10 * 0xFF - 14) + + def test_bytes_to_escaped(self): + from make_translation import bytes_to_escaped + + self.assertEqual(bytes_to_escaped(b"\x00"), "\\x00") + self.assertEqual(bytes_to_escaped(b"\xF1\xAB"), "\\xF1\\xAB") if __name__ == "__main__":