Reorganize font table of CJK languages

Move all CJK symbols to the end of the font table so we can skip the
filler for the CJK symbols in the small font table to save space.

Translations/make_translation.py

@@ -289,29 +289,37 @@ def getFontMapAndTable(textList):
     symbolMap["\n"] = "\\x01"  # Force insert the newline char
     index = 2  # start at 2, as 0= null terminator,1 = new line
     forcedFirstSymbols = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
-    # enforce numbers are first
+
-    for sym in forcedFirstSymbols:
+    # Get the font table, which does not include CJK chars
-        symbolMap[sym] = getCharsFromFontIndex(index)
+    fontTable = fontTables.getFontMap()
-        index = index + 1
+    fontSmallTable = fontTables.getSmallFontMap()
-    totalSymbolCount = len(set(textList) | set(forcedFirstSymbols))
+
+    # We want to put all CJK chars after non-CJK ones so that the CJK chars
+    # do not need to be in the small font table to save space.
+    # We assume all symbols not in the font table to be a CJK char.
+    # We also enforce that numbers are first.
+    orderedNormalSymList = forcedFirstSymbols + [x for x in textList if x not in forcedFirstSymbols and x in fontTable]
+    orderedCJKSymList = [x for x in textList if x not in forcedFirstSymbols and x not in fontTable]
+
+    totalSymbolCount = len(orderedNormalSymList) + len(orderedCJKSymList)
     # \x00 is for NULL termination and \x01 is for newline, so the maximum
     # number of symbols allowed is as follow (see also the comments in
     # `getCharsFromFontIndex`):
     if totalSymbolCount > (0x10 * 0xFF - 15) - 2:
         log(f"Error, too many used symbols for this version (total {totalSymbolCount})")
         sys.exit(1)
+
     log("Generating fonts for {} symbols".format(totalSymbolCount))
 
-    for sym in textList:
+    for l in (orderedNormalSymList, orderedCJKSymList):
-        if sym not in symbolMap:
+        for sym in l:
+            assert(sym not in symbolMap)
             symbolMap[sym] = getCharsFromFontIndex(index)
             index = index + 1
-    # Get the font table
+
     fontTableStrings = []
     fontSmallTableStrings = []
-    fontTable = fontTables.getFontMap()
+    for sym in orderedNormalSymList:
-    fontSmallTable = fontTables.getSmallFontMap()
-    for sym in forcedFirstSymbols:
         if sym not in fontTable:
             log("Missing Large font element for {}".format(sym))
             sys.exit(1)
@@ -325,28 +333,18 @@ def getFontMapAndTable(textList):
             fontLine + "//{} -> {}".format(symbolMap[sym], sym)
         )
 
-    for sym in textList:
+    for sym in orderedCJKSymList:
-        if sym not in fontTable:
+        assert(sym not in fontTable)
-            # Assume this is a CJK character.
+        fontLine = getCJKGlyph(sym)
-            fromFont = getCJKGlyph(sym)
+        if fontLine is None:
-            if fromFont is None:
             log("Missing Large font element for {}".format(sym))
             sys.exit(1)
-            # We store the glyph back to the fontTable.
-            fontTable[sym] = fromFont
-            # We also put a "replacement character" in the small font table
-            # for sanity. (It is a question mark with inverted colour.)
-            fontSmallTable[sym] = "0xFD, 0xFE, 0xAE, 0xF6, 0xF9, 0xFF,"
-        if sym not in forcedFirstSymbols:
-            fontLine = fontTable[sym]
         fontTableStrings.append(fontLine + "//{} -> {}".format(symbolMap[sym], sym))
-            if sym not in fontSmallTable:
+        # No data to add to the small font table
-                log("Missing Small font element for {}".format(sym))
-                sys.exit(1)
-            fontLine = fontSmallTable[sym]
         fontSmallTableStrings.append(
-                fontLine + "//{} -> {}".format(symbolMap[sym], sym)
+            "//                                   {} -> {}".format(symbolMap[sym], sym)
         )
+
     outputTable = "const uint8_t USER_FONT_12[] = {" + to_unicode("\n")
     for line in fontTableStrings:
         # join font table int one large string
@@ -380,13 +378,14 @@ def writeLanguage(lang, defs, f):
     # From the letter counts, need to make a symbol translator & write out the font
     (fontTableText, symbolConversionTable) = getFontMapAndTable(textList)
 
-    f.write(fontTableText)
     try:
         langName = lang["languageLocalName"]
     except KeyError:
         langName = languageCode
 
-    f.write(to_unicode("// ---- " + langName + " ----\n\n"))
+    f.write(to_unicode("\n// ---- " + langName + " ----\n\n"))
+    f.write(fontTableText)
+    f.write(to_unicode("\n// ---- " + langName + " ----\n\n"))
 
     # ----- Writing SettingsDescriptions
     obj = lang["menuOptions"]