sv/IronOS
1
0
Fork 0
forked from me/IronOS
Code Pull Requests Activity

Generate per-language translation sources (#806)

Browse Source 
This generates dedicates Translation.cpp files for translation language
and derives all language-specific data from them.

The Makefile is extended to also take care of generating these source
files.
This allows reuse of nearly all object files between builds of different
languages for the same model and regenerating the translation sources if
necessary.
This speeds up the release builds and the normal write-compile-cycle
considerably.
It also eliminates miscompilations when manually building different
languages.
This commit is contained in:
Thomas Weißschuh
2021-02-02 09:44:34 +01:00
committed by GitHub
parent 5e372310cd
commit 15e51f9faa
36 changed files with 108 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
Translations/make_translation.py
View File

@@ -1,6 +1,7 @@
#!/usr/bin/env python3
# coding=utf-8
from __future__ import print_function
import argparse
import json
import os
import io
@@ -10,8 +11,7 @@ import fontTables
import re
import subprocess
TRANSLATION_CPP = "Translation.cpp"
UNIT_H = "unit.h"
HERE = os.path.dirname(__file__)
try:
to_unicode = unicode
@@ -19,6 +19,10 @@ except NameError:
to_unicode = str
def log(message):
print(message, file=sys.stdout)
# Loading a single JSON file
def loadJson(fileName, skipFirstLine):
with io.open(fileName, mode="r", encoding="utf-8") as f:
@@ -30,48 +34,33 @@ def loadJson(fileName, skipFirstLine):
return obj
# Reading all language translations into a dictionary by langCode
def readTranslations(jsonDir):
langDict = {}
UnitDict = {}
def readTranslation(jsonDir, langCode):
fileName = 'translation_{}.json'.format(langCode)
# Read all translation files from the input dir
for fileName in os.listdir(jsonDir):
fileWithPath = os.path.join(jsonDir, fileName)
fileWithPath = os.path.join(jsonDir, fileName)
lf = fileName.lower()
try:
lang = loadJson(fileWithPath, False)
except json.decoder.JSONDecodeError as e:
log("Failed to decode " + fileName)
log(str(e))
sys.exit(2)
# Read only translation_XX.json
if lf.startswith("translation_") and lf.endswith(".json"):
try:
lang = loadJson(fileWithPath, False)
except json.decoder.JSONDecodeError as e:
print("Failed to decode " + lf)
print(str(e))
sys.exit(2)
# Extract lang code from file name
langCode = fileName[12:-5].upper()
# ...and the one specified in the JSON file...
try:
langCodeFromJson = lang["languageCode"]
except KeyError:
langCodeFromJson = "(missing)"
# Extract lang code from file name
langCode = fileName[12:-5].upper()
# ...and the one specified in the JSON file...
try:
langCodeFromJson = lang["languageCode"]
except KeyError:
langCodeFromJson = "(missing)"
# ...cause they should be the same!
if langCode != langCodeFromJson:
raise ValueError(
"Invalid languageCode " + langCodeFromJson + " in file " + fileName
)
try:
TempUnitF_FromJson = lang["tempUnitFahrenheit"]
except KeyError:
TempUnitF_FromJson = True # Default to true.
# ...cause they should be the same!
if langCode != langCodeFromJson:
raise ValueError(
"Invalid languageCode " + langCodeFromJson + " in file " + fileName
)
langDict[langCode] = lang
UnitDict[langCode] = TempUnitF_FromJson
return langDict, UnitDict
return lang
def writeStart(f):
@@ -85,26 +74,6 @@ def writeStart(f):
)
def writeStartUnit(f):
f.write(
to_unicode(
"""// WARNING: THIS FILE WAS AUTO GENERATED BY make_translation.py. PLEASE DO NOT EDIT.
/**
* °F Fahrenheit Support
* You will find the default Fahrenheit configuration in the translation_xx.json
* If tempUnitFahrenheit is set to:
* true - you can switch in menu settings to Fahrenheit or Celsius.
* false - you see only Celsius. All settings are then is in Celsius only.
*/
#ifndef _UNIT_H
#define _UNIT_H\n
"""
)
)
def escapeC(s):
return s.replace('"', '\\"')
@@ -220,9 +189,9 @@ def getFontMapAndTable(textList):
symbolMap[sym] = "\\x%0.2X" % index
index = index + 1
if len(textList) > (253 - len(forcedFirstSymbols)):
print("Error, too many used symbols for this version")
log("Error, too many used symbols for this version")
exit(1)
print("Generating fonts for {} symbols".format(len(textList)))
log("Generating fonts for {} symbols".format(len(textList)))
for sym in textList:
if sym not in symbolMap:
@@ -235,12 +204,12 @@ def getFontMapAndTable(textList):
fontSmallTable = fontTables.getSmallFontMap()
for sym in forcedFirstSymbols:
if sym not in fontTable:
print("Missing Large font element for {}".format(sym))
log("Missing Large font element for {}".format(sym))
exit(1)
fontLine = fontTable[sym]
fontTableStrings.append(fontLine + "//{} -> {}".format(symbolMap[sym], sym))
if sym not in fontSmallTable:
print("Missing Small font element for {}".format(sym))
log("Missing Small font element for {}".format(sym))
exit(1)
fontLine = fontSmallTable[sym]
fontSmallTableStrings.append(
@@ -249,13 +218,13 @@ def getFontMapAndTable(textList):
for sym in textList:
if sym not in fontTable:
print("Missing Large font element for {}".format(sym))
log("Missing Large font element for {}".format(sym))
exit(1)
if sym not in forcedFirstSymbols:
fontLine = fontTable[sym]
fontTableStrings.append(fontLine + "//{} -> {}".format(symbolMap[sym], sym))
if sym not in fontSmallTable:
print("Missing Small font element for {}".format(sym))
log("Missing Small font element for {}".format(sym))
exit(1)
fontLine = fontSmallTable[sym]
fontSmallTableStrings.append(
@@ -279,21 +248,20 @@ def convStr(symbolConversionTable, text):
outputString = ""
for c in text.replace("\\r", "").replace("\\n", "\n"):
if c not in symbolConversionTable:
print("Missing font definition for {}".format(c))
log("Missing font definition for {}".format(c))
else:
outputString = outputString + symbolConversionTable[c]
return outputString
def writeLanguage(languageCode, defs, f):
print("Generating block for " + languageCode)
lang = langDict[languageCode]
def writeLanguage(lang, defs, f):
languageCode = lang['languageCode']
log("Generating block for " + languageCode)
# Iterate over all of the text to build up the symbols & counts
textList = getLetterCounts(defs, lang)
# From the letter counts, need to make a symbol translator & write out the font
(fontTableText, symbolConversionTable) = getFontMapAndTable(textList)
f.write(to_unicode("\n#ifdef LANG_" + languageCode + "\n"))
f.write(fontTableText)
try:
langName = lang["languageLocalName"]
@@ -486,29 +454,13 @@ def writeLanguage(languageCode, defs, f):
)
f.write(to_unicode("};\n\n"))
# ----- Block end
f.write(to_unicode("#endif\n"))
f.write("const bool HasFahrenheit = " + (
"true" if lang.get('tempUnitFahrenheit', True) else "false") +
";\n")
def writeUnit(languageCode, defs, f, UnitCodes):
print("Generating unit block for " + languageCode)
lang = langDict[languageCode]
unit = UnitDict[UnitCodes]
try:
langName = lang["languageLocalName"]
except KeyError:
langName = languageCode
f.write(to_unicode(" #ifdef LANG_" + languageCode + "\n"))
if unit:
f.write(to_unicode(" #define ENABLED_FAHRENHEIT_SUPPORT" + "\n"))
else:
f.write(to_unicode(" //#define ENABLED_FAHRENHEIT_SUPPORT" + "\n"))
# ----- Block end
f.write(to_unicode(" #endif /* ---- " + langName + " ---- */\n"))
def readVersion():
def readVersion(jsonDir):
with open(os.path.relpath(jsonDir + "/../source/version.h"), "r") as version_file:
try:
for line in version_file:
@@ -535,37 +487,6 @@ def readVersion():
return version
def read_opts():
"""Reading input parameters
First parameter = json directory
Second parameter = translation directory
Third paramter = unit directory
"""
if len(sys.argv) > 1:
jsonDir = sys.argv[1]
else:
jsonDir = "."
if len(sys.argv) > 2:
outFileTranslationCPP = sys.argv[2]
else:
outDir = os.path.relpath(jsonDir + "/../source/Core/Gen")
if not os.path.exists(outDir):
os.makedirs(outDir)
outFileTranslationCPP = os.path.join(outDir, TRANSLATION_CPP)
if len(sys.argv) > 3:
outFileUnitH = sys.argv[3]
else:
outDir = os.path.relpath(jsonDir + "/../source/Core/Inc")
outFileUnitH = os.path.join(outDir, UNIT_H)
if len(sys.argv) > 4:
raise Exception("Too many parameters!")
return jsonDir, outFileTranslationCPP, outFileUnitH
def orderOutput(langDict):
# These languages go first
mandatoryOrder = ["EN"]
@@ -581,41 +502,33 @@ def orderOutput(langDict):
return mandatoryOrder
def writeTarget(outFileTranslationCPP, outFileUnitH, defs, langCodes, UnitCodes):
# Start writing the file
with io.open(outFileTranslationCPP, "w", encoding="utf-8", newline="\n") as f:
writeStart(f)
for langCode in langCodes:
writeLanguage(langCode, defs, f)
with io.open(outFileUnitH, "w", encoding="utf-8", newline="\n") as f:
writeStartUnit(f)
for langCode, UnitCode in zip(langCodes, UnitCodes):
writeUnit(langCode, defs, f, UnitCode)
f.write(to_unicode("\n#endif /* _UNIT_H */\n"))
def parseArgs():
parser = argparse.ArgumentParser()
parser.add_argument(
'--output', '-o',
help='Target file', type=argparse.FileType('w'), required=True)
parser.add_argument('languageCode', help='Language to generate')
return parser.parse_args()
if __name__ == "__main__":
try:
jsonDir, outFileTranslationCPP, outFileUnitH = read_opts()
except:
print("usage: make_translation.py {json dir} {cpp dir}")
sys.exit(1)
jsonDir = HERE
args = parseArgs()
try:
buildVersion = readVersion()
buildVersion = readVersion(jsonDir)
except:
print("error: could not get/extract build version")
log("error: could not get/extract build version")
sys.exit(1)
print("Build version: " + buildVersion)
print("Making " + outFileTranslationCPP + " from " + jsonDir)
print("Making " + outFileUnitH + " from " + jsonDir)
log("Build version: " + buildVersion)
log("Making " + args.languageCode + " from " + jsonDir)
langDict, UnitDict = readTranslations(jsonDir)
lang = readTranslation(jsonDir, args.languageCode)
defs = loadJson(os.path.join(jsonDir, "translations_def.js"), True)
langCodes = orderOutput(langDict)
UnitCodes = orderOutput(UnitDict)
writeTarget(outFileTranslationCPP, outFileUnitH, defs, langCodes, UnitCodes)
out = args.output
writeStart(out)
writeLanguage(lang, defs, out)
print("Done")
log("Done")