From 8264cc6d003093af11df06ff421615666970750a Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 7 Oct 2023 16:20:40 +0200 Subject: [PATCH 1/8] reduce continer size ans security attack surface --- CHANGELOG.md | 3 +++ app/Dockerfile | 34 ++++++++++++++-------------------- app/entrypoint.sh | 8 +++----- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 717d75b..88ef3ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- move from slim-bookworm to an alpine base image +- install python requirements with pip wheel + ## [0.1.0] - 2023-10-06 - refactoring of the connection classes diff --git a/app/Dockerfile b/app/Dockerfile index f90177f..c56433f 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -3,31 +3,26 @@ ARG UID=1000 ARG GID=1000 # set base image (host OS) -FROM python:3.11-slim-bookworm AS builder - +FROM python:3.11-alpine AS base USER root -# install gosu for a better su+exec command -RUN set -eux; \ - apt-get update; \ - apt-get install -y gosu; \ - rm -rf /var/lib/apt/lists/*; \ -# verify that the binary works - gosu nobody true +RUN apk update && \ + apk upgrade +RUN apk add --no-cache su-exec -RUN pip install --upgrade pip +FROM base as builder +RUN apk add --no-cache build-base && \ + python -m pip install --no-cache-dir -U pip wheel # copy the dependencies file to the working directory -COPY ./requirements.txt . - -# install dependencies -RUN pip install --user -r requirements.txt +COPY ./requirements.txt /root/ +RUN python -OO -m pip wheel --no-cache-dir --wheel-dir=/root/wheels -r /root/requirements.txt # # second unnamed stage -FROM python:3.11-slim-bookworm +FROM base ARG SERVICE_NAME ARG VERSION ARG UID @@ -43,16 +38,15 @@ WORKDIR /home/$SERVICE_NAME # update PATH environment variable ENV HOME=/home/$SERVICE_NAME -ENV PATH=/home/$SERVICE_NAME/.local:$PATH VOLUME ["/home/$SERVICE_NAME/log", "/home/$SERVICE_NAME/config"] # copy only the dependencies installation from the 1st stage image -COPY --from=builder --chown=$SERVICE_NAME:$SERVICE_NAME /root/.local /home/$SERVICE_NAME/.local -COPY --from=builder /usr/sbin/gosu /usr/sbin/gosu +COPY --from=builder /root/wheels /root/wheels +RUN python -m pip install --no-cache --no-index /root/wheels/* +RUN rm -rf /root/wheels -COPY entrypoint.sh /root/entrypoint.sh -RUN chmod +x /root/entrypoint.sh +COPY --chmod=0700 entrypoint.sh /root/entrypoint.sh # copy the content of the local src and config directory to the working directory COPY config . diff --git a/app/entrypoint.sh b/app/entrypoint.sh index 7935f3b..87148ce 100644 --- a/app/entrypoint.sh +++ b/app/entrypoint.sh @@ -10,17 +10,15 @@ echo "#" if [ "$user" = '0' ]; then mkdir -p /home/$SERVICE_NAME/log /home/$SERVICE_NAME/config - if id $SERVICE_NAME ; then - echo "user still exists" - else + if ! id $SERVICE_NAME &> /dev/null; then addgroup --gid $GID $SERVICE_NAME 2> /dev/null - adduser --ingroup $SERVICE_NAME --shell /bin/false --disabled-password --no-create-home --comment "" --uid $UID $SERVICE_NAME + adduser -G $SERVICE_NAME -s /bin/false -D -H -g "" -u $UID $SERVICE_NAME fi chown -R $SERVICE_NAME:$SERVICE_NAME /home/$SERVICE_NAME || true echo "######################################################" echo "#" - exec gosu $SERVICE_NAME "$@" + exec su-exec $SERVICE_NAME "$@" else exec "$@" fi From ddba3f62855c0302333ef3bb4d27ccfa8e75e478 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 7 Oct 2023 16:39:39 +0200 Subject: [PATCH 2/8] optimize and update some comments --- app/Dockerfile | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/app/Dockerfile b/app/Dockerfile index c56433f..c732b10 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -2,7 +2,8 @@ ARG SERVICE_NAME="tsun-proxy" ARG UID=1000 ARG GID=1000 -# set base image (host OS) +# +# first stage for our base image FROM python:3.11-alpine AS base USER root @@ -10,19 +11,21 @@ RUN apk update && \ apk upgrade RUN apk add --no-cache su-exec +# +# second stage for building wheels packages FROM base as builder RUN apk add --no-cache build-base && \ python -m pip install --no-cache-dir -U pip wheel -# copy the dependencies file to the working directory +# copy the dependencies file to the root dir and install requirements COPY ./requirements.txt /root/ RUN python -OO -m pip wheel --no-cache-dir --wheel-dir=/root/wheels -r /root/requirements.txt # -# second unnamed stage -FROM base +# third stage for our runtime image +FROM base as runtime ARG SERVICE_NAME ARG VERSION ARG UID @@ -41,14 +44,13 @@ ENV HOME=/home/$SERVICE_NAME VOLUME ["/home/$SERVICE_NAME/log", "/home/$SERVICE_NAME/config"] -# copy only the dependencies installation from the 1st stage image +# install the requirements from the wheels packages from the builder stage COPY --from=builder /root/wheels /root/wheels -RUN python -m pip install --no-cache --no-index /root/wheels/* -RUN rm -rf /root/wheels - -COPY --chmod=0700 entrypoint.sh /root/entrypoint.sh +RUN python -m pip install --no-cache --no-index /root/wheels/* && \ + rm -rf /root/wheels # copy the content of the local src and config directory to the working directory +COPY --chmod=0700 entrypoint.sh /root/entrypoint.sh COPY config . COPY src . From ed14ed484b8d712caf0380a1e5de18c2ae6b8e25 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 7 Oct 2023 20:55:26 +0200 Subject: [PATCH 3/8] add build support for release candidates (rc) --- app/build.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/build.sh b/app/build.sh index 9ebc27e..687b32b 100755 --- a/app/build.sh +++ b/app/build.sh @@ -9,20 +9,22 @@ arr=(${VERSION//./ }) MAJOR=${arr[0]} IMAGE=tsun-gen3-proxy -if [[ $1 == dev ]];then +if [[ $1 == dev ]] || [[ $1 == rc ]] ;then IMAGE=docker.io/sallius/${IMAGE} -VERSION=${VERSION}-dev +VERSION=${VERSION}-$1 elif [[ $1 == rel ]];then IMAGE=ghcr.io/s-allius/${IMAGE} else echo argument missing! -echo try: $0 '[dev|rel]' +echo try: $0 '[dev|rc|rel]' exit 1 fi echo version: $VERSION build-date: $BUILD_DATE image: $IMAGE if [[ $1 == dev ]];then -docker build --build-arg "VERSION=${VERSION}" --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" -t ${IMAGE}:latest app +docker build --build-arg "VERSION=${VERSION}" --build-arg "LOG_LVL=DEBUG" --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" -t ${IMAGE}:latest app +elif [[ $1 == rc ]];then +docker build --no-cache --build-arg "VERSION=${VERSION}" --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" -t ${IMAGE}:latest app elif [[ $1 == rel ]];then docker build --no-cache --build-arg "VERSION=${VERSION}" --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" -t ${IMAGE}:latest -t ${IMAGE}:${MAJOR} -t ${IMAGE}:${VERSION} app docker push ghcr.io/s-allius/tsun-gen3-proxy:latest From b06d8325045207b26ae2eff6cc526b7fba772725 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 7 Oct 2023 20:58:18 +0200 Subject: [PATCH 4/8] set log level to DEBUG for dev versions --- app/Dockerfile | 3 +++ app/src/server.py | 24 +++++++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/app/Dockerfile b/app/Dockerfile index c732b10..e88ea0f 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -30,10 +30,13 @@ ARG SERVICE_NAME ARG VERSION ARG UID ARG GID +ARG LOG_LVL + ENV VERSION=$VERSION ENV SERVICE_NAME=$SERVICE_NAME ENV UID=$UID ENV GID=$GID +ENV LOG_LVL=$LOG_LVL # set the working directory in the container diff --git a/app/src/server.py b/app/src/server.py index 727004f..459f4f4 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -32,8 +32,16 @@ def handle_SIGTERM(loop): logging.info('Shutdown complete') - - +def get_log_level() -> int: + '''checks if LOG_LVL is set in the environment and returns the corresponding logging.LOG_LEVEL''' + log_level = os.getenv('LOG_LVL', 'INFO') + if log_level== 'DEBUG': + log_level = logging.DEBUG + elif log_level== 'WARN': + log_level = logging.WARNING + else: + log_level = logging.INFO + return log_level if __name__ == "__main__": @@ -41,11 +49,17 @@ if __name__ == "__main__": # Setup our daily, rotating logger # serv_name = os.getenv('SERVICE_NAME', 'proxy') - version = os.getenv('VERSION', 'unknown') - + version = os.getenv('VERSION', 'unknown') + logging.config.fileConfig('logging.ini') logging.info(f'Server "{serv_name} - {version}" will be started') - logging.getLogger().setLevel(logging.DEBUG if __debug__ else logging.INFO) + + # set lowest-severity for 'root', 'msg', 'conn' and 'data' logger + log_level = get_log_level() + logging.getLogger().setLevel(log_level) + logging.getLogger('msg').setLevel(log_level) + logging.getLogger('conn').setLevel(log_level) + logging.getLogger('data').setLevel(log_level) # read config file Config.read() From e1536cb697b74239b3cd0cf4248433b9a5ed53f2 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 7 Oct 2023 21:03:49 +0200 Subject: [PATCH 5/8] adapt log levels, optimize expensive hex dump logs --- app/src/async_stream.py | 6 +++--- app/src/messages.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index d216550..2e9a4d9 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -21,7 +21,7 @@ class AsyncStream(Message): Our puplic methods ''' def set_serial_no(self, serial_no : str): - logger.info(f'SerialNo: {serial_no}') + logger.debug(f'SerialNo: {serial_no}') if self.unique_id != serial_no: @@ -40,7 +40,7 @@ class AsyncStream(Message): if not inverters['allow_all']: self.unique_id = None - logger.error('ignore message from unknow inverter!') + logger.warning('ignore message from unknow inverter! (SerialNo: {serial_no})') return self.unique_id = serial_no @@ -67,7 +67,7 @@ class AsyncStream(Message): except (ConnectionResetError, ConnectionAbortedError, RuntimeError) as error: - logger.error(f'In loop for {self.addr}: {error}') + logger.warning(f'In loop for {self.addr}: {error}') self.close() return except Exception: diff --git a/app/src/messages.py b/app/src/messages.py index e9a5838..aaa668c 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -18,7 +18,8 @@ def hex_dump_memory(level, info, data, num): lines = [] lines.append(info) tracer = logging.getLogger('tracer') - + if not tracer.isEnabledFor(level): return + #data = list((num * ctypes.c_byte).from_address(ptr)) @@ -294,7 +295,7 @@ class Message(metaclass=IterRegistry): def msg_unknown(self): - logger.error (f"Unknow Msg: ID:{self.msg_id}") + logger.warning (f"Unknow Msg: ID:{self.msg_id}") self.forward(self._recv_buffer, self.header_len+self.data_len) From fa567f68c0301ac222d1507298366778e7f9d705 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 7 Oct 2023 21:14:57 +0200 Subject: [PATCH 6/8] - disable DEBUG log for releases - support building of release candidates --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88ef3ef..054b510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - move from slim-bookworm to an alpine base image - install python requirements with pip wheel +- disable DEBUG log for releases +- support building of release candidates ## [0.1.0] - 2023-10-06 From 3225566b9b76bba70a8ab76f864819849637a8d6 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 7 Oct 2023 21:24:49 +0200 Subject: [PATCH 7/8] fix formating of a log message --- app/src/async_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 2e9a4d9..921758a 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -40,7 +40,7 @@ class AsyncStream(Message): if not inverters['allow_all']: self.unique_id = None - logger.warning('ignore message from unknow inverter! (SerialNo: {serial_no})') + logger.warning(f'ignore message from unknow inverter! (SerialNo: {serial_no})') return self.unique_id = serial_no From 2cf7a2db364858664c6a513879f576b4ae01bc48 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 7 Oct 2023 23:08:39 +0200 Subject: [PATCH 8/8] Version 0.2.0 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 054b510..e25e475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0] - 2023-10-07 + +This version halves the size of the Docker image and reduces the attack surface for security vulnerabilities, by omitting unneeded code. The feature set is exactly the same as the previous release version 0.1.0. + +### Changes + - move from slim-bookworm to an alpine base image - install python requirements with pip wheel - disable DEBUG log for releases