Implement ci target for Makefile to reproduce github CI actions & artifacts (#1769)
* Implement CI target in Makefile to emulate github CI actions & artifacts * Improve filter for metadata * metadata.py: update usage output for wrong number of input arguments / code review * metadata.py: remove excessive checks for the second input argument / code review * metadata.py: remove hard-coded model for multi-lang builds in ModelName argument processing / code review * metadata.py: remove hard-coded models for multi-lang builds in file name pattern processing / code review * metadata.py: update usage output to remove ambiguity about json extension for output file * metadata.py: unify new lines style formatting * metadata.py: sort the list of processing files in alphanumeric order before looping through them to get the same lang order on every generation in every json output file
This commit is contained in:
Ivan Zorin
GitHub
53
Makefile
53
Makefile
@@ -51,6 +51,14 @@ DOCKER_CMD=$(DOCKER_BIN) -f $(DOCKER_YML) run --rm builder
|
||||
# MkDocs config
|
||||
MKDOCS_YML=$(CURDIR)/scripts/IronOS-mkdocs.yml
|
||||
|
||||
# supported models
|
||||
MODELS=TS100 TS80 TS80P Pinecil MHP30 Pinecilv2 S60 TS101 # target names & dir names
|
||||
MODELS_ML=Pinecil Pinecilv2 # target names
|
||||
MODELS_MULTILANG=Pinecil_multi-lang Pinecilv2_multi-lang # dir names
|
||||
|
||||
# zip command (to pack artifacts)
|
||||
ZIP=zip -q -j -r
|
||||
|
||||
|
||||
### targets
|
||||
|
||||
@@ -91,7 +99,7 @@ list:
|
||||
@echo
|
||||
@echo "Full list of current supported IDs:"
|
||||
@echo " * LANG_ID: $(shell echo "`ls Translations/ | grep -e "^translation_.*.json$$" | sed -e 's,^translation_,,g; s,\.json$$,,g; ' | tr '\n' ' '`")"
|
||||
@echo " * MODEL_ID: TS100 TS101 TS80 TS80P MHP30 Pinecil Pinecilv2 S60"
|
||||
@echo " * MODEL_ID: $(MODELS)"
|
||||
@echo
|
||||
@echo "For example, to make a local build of IronOS firmware for TS100 with English language, just type:"
|
||||
@echo
|
||||
@@ -171,11 +179,41 @@ build-all:
|
||||
@chmod 0777 $(OUT_DIR)
|
||||
cd source && bash ./build.sh
|
||||
@echo "All Firmware built"
|
||||
@cp -r $(OUT_HEX)/*.bin $(OUT_DIR)
|
||||
@cp -r $(OUT_HEX)/*.hex $(OUT_DIR)
|
||||
@cp -r $(OUT_HEX)/*.dfu $(OUT_DIR)
|
||||
@for model in $(MODELS); do \
|
||||
mkdir -p $(OUT_DIR)/$${model} ; \
|
||||
cp -r $(OUT_HEX)/$${model}_*.bin $(OUT_DIR)/$${model}/ ; \
|
||||
cp -r $(OUT_HEX)/$${model}_*.hex $(OUT_DIR)/$${model}/ ; \
|
||||
cp -r $(OUT_HEX)/$${model}_*.dfu $(OUT_DIR)/$${model}/ ; \
|
||||
done;
|
||||
@echo "Resulting output directory: $(OUT_DIR)"
|
||||
|
||||
# target to build multilang supported builds for Pinecil & PinecilV2
|
||||
build-multilang:
|
||||
@for modelml in $(MODELS_ML); do \
|
||||
$(MAKE) -C source/ -j2 model=$${modelml} firmware-multi_compressed_European firmware-multi_compressed_Bulgarian+Russian+Serbian+Ukrainian firmware-multi_Chinese+Japanese ; \
|
||||
mkdir -p $(OUT_DIR)/$${modelml}_multi-lang ; \
|
||||
cp $(OUT_HEX)/$${modelml}_multi_*.bin $(OUT_DIR)/$${modelml}_multi-lang ; \
|
||||
cp $(OUT_HEX)/$${modelml}_multi_*.hex $(OUT_DIR)/$${modelml}_multi-lang ; \
|
||||
cp $(OUT_HEX)/$${modelml}_multi_*.dfu $(OUT_DIR)/$${modelml}_multi-lang ; \
|
||||
done;
|
||||
@echo "Resulting output directory: $(OUT_DIR)"
|
||||
|
||||
# target to reproduce zips according to github CI settings; artifacts will be in $(OUT_DIR)/CI/*.zip
|
||||
ci: tests build-all build-multilang
|
||||
@mkdir -p $(OUT_DIR)/metadata;
|
||||
@for m in $(MODELS) $(MODELS_MULTILANG); do \
|
||||
cp LICENSE scripts/LICENSE_RELEASE.md $(OUT_DIR)/$${m}/ ; \
|
||||
$(ZIP) $(OUT_DIR)/$${m}.zip $(OUT_DIR)/$${m} ; \
|
||||
./source/metadata.py $${m}.json $${m}; \
|
||||
cp $(OUT_HEX)/$${m}.json $(OUT_DIR)/metadata; \
|
||||
done;
|
||||
@$(ZIP) $(OUT_DIR)/metadata.zip $(OUT_DIR)/metadata
|
||||
@mkdir -p $(OUT_DIR)/CI
|
||||
@mv $(OUT_DIR)/*.zip $(OUT_DIR)/CI
|
||||
@chmod 0777 $(OUT_DIR)/CI
|
||||
@chmod 0666 $(OUT_DIR)/CI/*.zip
|
||||
@echo "Resulting artifacts directory: $(OUT_DIR)/CI"
|
||||
|
||||
# pass-through target for Makefile inside source/ dir
|
||||
%:
|
||||
$(MAKE) -C source/ $@
|
||||
@@ -191,4 +229,9 @@ clean-build:
|
||||
clean-full: clean-build docker-clean
|
||||
|
||||
# phony targets
|
||||
.PHONY: help list docker-check docker-shell docker-build docker-clean docs docs-deploy test-md test-sh test-py test-ccpp tests clean-build clean-full
|
||||
.PHONY: help list
|
||||
.PHONY: docker-check docker-shell docker-build docker-clean
|
||||
.PHONY: docs docs-deploy
|
||||
.PHONY: test-md test-sh test-py test-ccpp tests
|
||||
.PHONY: build-all build-multilang ci
|
||||
.PHONY: clean-build clean-full
|
||||
|
||||
@@ -9,7 +9,7 @@ WORKDIR /build/ironos
|
||||
# Installing the two compilers (ARM & RISCV), python3 & pip, clang tools, etc.:
|
||||
## - compilers: gcc-*, newlib-*
|
||||
## - python3: py*, black (required to check Python code formatting)
|
||||
## - misc: findutils, make, git, diffutils
|
||||
## - misc: findutils, make, git, diffutils, zip
|
||||
## - musl-dev (required for the multi lang firmwares)
|
||||
## - clang (required for clang-format to check C++ code formatting)
|
||||
## - shellcheck (to check sh scripts)
|
||||
@@ -17,7 +17,7 @@ WORKDIR /build/ironos
|
||||
ARG APK_COMPS="gcc-riscv-none-elf gcc-arm-none-eabi newlib-riscv-none-elf \
|
||||
newlib-arm-none-eabi"
|
||||
ARG APK_PYTHON="python3 py3-pip black"
|
||||
ARG APK_MISC="findutils make git diffutils"
|
||||
ARG APK_MISC="findutils make git diffutils zip"
|
||||
ARG APK_DEV="musl-dev clang bash clang-extra-tools shellcheck"
|
||||
|
||||
# PIP packages to check & test Python code
|
||||
|
||||
@@ -10,27 +10,33 @@ import sys
|
||||
# Creates an index metadata json file of the hexfiles folder
|
||||
# This is used by automation like the Pinecil updater
|
||||
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print("Requires the output json name as an arg")
|
||||
if len(sys.argv) < 2 or len(sys.argv) > 3:
|
||||
print("Usage: metadata.py OUTPUT_FILE [model]")
|
||||
print(" OUTPUT_FILE - the name of output file in json format with meta info about binary files")
|
||||
print(" model [optional] - name of the model (as for `make model=NAME`) to scan files for explicitly (all files in source/Hexfile by default otherwise)")
|
||||
exit(1)
|
||||
|
||||
# If model is provided explicitly to scan related files only for json output, then process the argument
|
||||
ModelName = None
|
||||
if len(sys.argv) == 3:
|
||||
ModelName = sys.argv[2]
|
||||
if ModelName.endswith("_multi-lang"):
|
||||
# rename on-the-fly for direct compatibility with make targets like PINECILMODEL_multi-lang
|
||||
ModelName = ModelName.rstrip("-lang")
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
HexFileFolder = os.path.join(HERE, "Hexfile")
|
||||
OutputJSONPath = os.path.join(HexFileFolder, sys.argv[1])
|
||||
TranslationsFilesPath = os.path.join(HERE.parent, "Translations")
|
||||
|
||||
|
||||
def load_json(filename: str):
|
||||
with open(filename) as f:
|
||||
return json.loads(f.read())
|
||||
|
||||
|
||||
def read_git_tag():
|
||||
return f"{subprocess.check_output(['git', 'rev-parse', '--short=7', 'HEAD']).strip().decode('ascii').upper()}"
|
||||
|
||||
|
||||
def read_version():
|
||||
with open(HERE / "version.h") as version_file:
|
||||
for line in version_file:
|
||||
@@ -40,10 +46,9 @@ def read_version():
|
||||
return matches[0]
|
||||
raise Exception("Could not parse version")
|
||||
|
||||
|
||||
# Fetch our file listings
|
||||
translation_files = [os.path.join(TranslationsFilesPath, f) for f in os.listdir(TranslationsFilesPath) if os.path.isfile(os.path.join(TranslationsFilesPath, f)) and f.endswith(".json")]
|
||||
output_files = [os.path.join(HexFileFolder, f) for f in os.listdir(HexFileFolder) if os.path.isfile(os.path.join(HexFileFolder, f))]
|
||||
output_files = [os.path.join(HexFileFolder, f) for f in sorted(os.listdir(HexFileFolder)) if os.path.isfile(os.path.join(HexFileFolder, f))]
|
||||
|
||||
parsed_languages = {}
|
||||
for path in translation_files:
|
||||
@@ -61,6 +66,13 @@ for file_path in output_files:
|
||||
if file_path.endswith(".hex") or file_path.endswith(".dfu"):
|
||||
# Find out what language this file is
|
||||
name: str = os.path.basename(file_path)
|
||||
if ModelName is not None:
|
||||
# If ModelName is provided as the second argument (compatible with make model=NAME fully) but current file name doesn't match the model name, then skip it
|
||||
if not name.startswith(ModelName + "_"):
|
||||
continue
|
||||
# If build of interest is not multi-lang one but scanning one is not MODEL_LANG-ID here, then skip it to avoid mess in json between MODEL_LANG-ID & MODEL_multi'
|
||||
if not ModelName.endswith("_multi") and not re.match(r"^" + ModelName + "_" + "([A-Z]+).*$", name):
|
||||
continue
|
||||
matches = re.findall(r"^([a-zA-Z0-9]+)_(.+)\.(.+)$", name)
|
||||
if matches:
|
||||
matches = matches[0]
|
||||
|
||||
Reference in New Issue
Block a user