From 91c9c73d4ac1f2a9b683cd8ee3f84a311f3ea980 Mon Sep 17 00:00:00 2001 From: "Ben V. Brown" Date: Mon, 7 Feb 2022 21:22:13 +1100 Subject: [PATCH 1/5] First pass of metadata generator --- source/metadata.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 source/metadata.py diff --git a/source/metadata.py b/source/metadata.py new file mode 100755 index 00000000..94d3bfc9 --- /dev/null +++ b/source/metadata.py @@ -0,0 +1,80 @@ +#! python3 + +import json +from pathlib import Path +import os +import re +import subprocess + +# Creates an index metadata json file of the hexfiles folder +# This is used by automation like the Pinecil updater + +HERE = Path(__file__).resolve().parent + +HexFileFolder = os.path.join(HERE, "Hexfile") +OutputJSONPath = os.path.join(HexFileFolder, "index.json") +TranslationsFilesPath = os.path.join(HERE.parent, "Translations") + + +def load_json(filename: str, skip_first_line: bool): + with open(filename) as f: + if skip_first_line: + f.readline() + return json.loads(f.read()) + + +def read_git_tag(): + return f"{subprocess.check_output(['git', 'rev-parse', '--short=7', 'HEAD']).strip().decode('ascii').upper()}" + + +def read_version(): + with open(HERE / "version.h") as version_file: + for line in version_file: + if re.findall(r"^.*(?<=(#define)).*(?<=(BUILD_VERSION))", line): + matches = re.findall(r"\"(.+?)\"", line) + if matches: + return matches[0] + raise Exception("Could not parse version") + + +# Fetch our file listings +translation_files = [os.path.join(TranslationsFilesPath, f) for f in os.listdir(TranslationsFilesPath) if os.path.isfile(os.path.join(TranslationsFilesPath, f)) and f.endswith(".json")] +output_files = [os.path.join(HexFileFolder, f) for f in os.listdir(HexFileFolder) if os.path.isfile(os.path.join(HexFileFolder, f))] + +parsed_languages = {} +for path in translation_files: + lang: dict = load_json(path, skip_first_line=False) + code = lang.get("languageCode", None) + if code is not None: + parsed_languages[code] = lang + +# Now that we have the languages, we can generate our index of info on each file + +output_json = {"git_tag": read_git_tag(), "release": read_version(), "contents": {}} + +device_model_name = None +for file_path in output_files: + if file_path.endswith(".hex") or file_path.endswith(".dfu"): + # Find out what language this file is + name: str = os.path.basename(file_path) + matches = re.findall(r"^([a-zA-Z]+)_(.+)\.(.+)$", name) + if matches: + matches = matches[0] + if len(matches) == 3: + if device_model_name is None: + device_model_name = matches[0] + lang_code = matches[1] + lang_file = parsed_languages.get(lang_code, None) + if lang_file is None: + raise Exception(f"Could not match language code {lang_code}") + file_record = {"language_code": lang_code, "language_name": lang_file.get("languageLocalName", None)} + output_json["contents"][name] = file_record + else: + print(f"failed to parse {matches}") + +if device_model_name is None: + raise Exception("No files parsed") + +output_json["model"] = device_model_name +with open(OutputJSONPath, "w", encoding="utf8") as json_file: + json.dump(output_json, json_file, ensure_ascii=False) From 5e777a97d26638b78c905c8ac896981fa67d1b07 Mon Sep 17 00:00:00 2001 From: "Ben V. Brown" Date: Mon, 7 Feb 2022 21:31:33 +1100 Subject: [PATCH 2/5] Draft ci --- .github/workflows/push.yml | 9 +++++++++ source/metadata.py | 8 +++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6c1e45b1..4f0b3b9a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -42,6 +42,15 @@ jobs: source/Hexfile/LICENSE_RELEASE.md if-no-files-found: error + - name: Generate json index file + run: cd source && python3 metadata.py ${{ matrix.model }}.json + + - name: Archive ${{ matrix.model }} index file + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.model }}.json + path: source/Hexfile/${{ matrix.model }}.json + build_multi-lang: runs-on: ubuntu-20.04 container: diff --git a/source/metadata.py b/source/metadata.py index 94d3bfc9..cd672bfc 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -5,14 +5,20 @@ from pathlib import Path import os import re import subprocess +import sys # Creates an index metadata json file of the hexfiles folder # This is used by automation like the Pinecil updater + +if len(sys.argv) != 2: + print("Requires the output json name as an arg") + exit(1) + HERE = Path(__file__).resolve().parent HexFileFolder = os.path.join(HERE, "Hexfile") -OutputJSONPath = os.path.join(HexFileFolder, "index.json") +OutputJSONPath = os.path.join(HexFileFolder, sys.argv[1]) TranslationsFilesPath = os.path.join(HERE.parent, "Translations") From 8c5c166109ff52644fe2ce67fc7a841d9b96088d Mon Sep 17 00:00:00 2001 From: "Ben V. Brown" Date: Mon, 7 Feb 2022 21:34:01 +1100 Subject: [PATCH 3/5] Update metadata.py --- source/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/metadata.py b/source/metadata.py index cd672bfc..09458a18 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -63,7 +63,7 @@ for file_path in output_files: if file_path.endswith(".hex") or file_path.endswith(".dfu"): # Find out what language this file is name: str = os.path.basename(file_path) - matches = re.findall(r"^([a-zA-Z]+)_(.+)\.(.+)$", name) + matches = re.findall(r"^([a-zA-Z0-9]+)_(.+)\.(.+)$", name) if matches: matches = matches[0] if len(matches) == 3: From ee0eeec08f3168396be51a6ccc07404b9f13f60f Mon Sep 17 00:00:00 2001 From: "Ben V. Brown" Date: Mon, 7 Feb 2022 21:37:59 +1100 Subject: [PATCH 4/5] Update push.yml --- .github/workflows/push.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4f0b3b9a..bcfabc33 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -89,6 +89,15 @@ jobs: source/Hexfile/LICENSE_RELEASE.md if-no-files-found: error + - name: Generate json index file + run: cd source && python3 metadata.py ${{ matrix.model }}.json + + - name: Archive ${{ matrix.model }} index file + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.model }}_multi-lang.json + path: source/Hexfile/${{ matrix.model }}.json + tests: runs-on: ubuntu-20.04 container: From d8e0d860846ac7c97c61c8022280a6489059bc05 Mon Sep 17 00:00:00 2001 From: "Ben V. Brown" Date: Mon, 7 Feb 2022 21:48:57 +1100 Subject: [PATCH 5/5] Update metadata.py --- source/metadata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/metadata.py b/source/metadata.py index 09458a18..9349bef8 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -69,8 +69,11 @@ for file_path in output_files: if len(matches) == 3: if device_model_name is None: device_model_name = matches[0] - lang_code = matches[1] + lang_code: str = matches[1] lang_file = parsed_languages.get(lang_code, None) + if lang_file is None and lang_code.startswith("multi_"): + # Multi files wont match, but we fake this by just taking the filename to it + lang_file = {"languageLocalName": lang_code.replace("multi_", "").replace("compressed_", "")} if lang_file is None: raise Exception(f"Could not match language code {lang_code}") file_record = {"language_code": lang_code, "language_name": lang_file.get("languageLocalName", None)}