From f870096a664f407bfa01c9c4261a0ad674b771bc Mon Sep 17 00:00:00 2001 From: LordGrey <48840279+Lord-Grey@users.noreply.github.com> Date: Fri, 19 Mar 2021 22:53:14 +0100 Subject: [PATCH] Docker Compile for local code (#1211) * Docker Compile for local code * Fine Tuning Co-authored-by: Paulchen Panther <16664240+Paulchen-Panther@users.noreply.github.com> --- CompileHowto.md | 105 +++++++++++++++-------------- CrossCompileHowto.md | 93 -------------------------- bin/scripts/docker-compile.sh | 122 ++++++++++++++++++++++++---------- 3 files changed, 144 insertions(+), 176 deletions(-) delete mode 100644 CrossCompileHowto.md diff --git a/CompileHowto.md b/CompileHowto.md index 5b662e83..978a65a0 100644 --- a/CompileHowto.md +++ b/CompileHowto.md @@ -1,3 +1,4 @@ + # With Docker If you are using [Docker](https://www.docker.com/), you can compile Hyperion inside a docker container. This keeps your system clean and with a simple script it's easy to use. Supported is also cross compiling for Raspberry Pi (Debian Stretch or higher). To compile Hyperion just execute one of the following commands. @@ -7,58 +8,67 @@ Note: call the script with `./docker-compile.sh -h` for more options. ## Native compilation on Raspberry Pi for: **Raspbian Stretch** -``` +```console wget -qN https://raw.github.com/hyperion-project/hyperion.ng/master/bin/scripts/docker-compile.sh && chmod +x *.sh && ./docker-compile.sh -i rpi-raspbian ``` **Raspbian Buster** -``` +```console wget -qN https://raw.github.com/hyperion-project/hyperion.ng/master/bin/scripts/docker-compile.sh && chmod +x *.sh && ./docker-compile.sh -i rpi-raspbian -t buster ``` ## Cross compilation on x86_64 for: **x86_64 (Debian Stretch):** -``` +```console wget -qN https://raw.github.com/hyperion-project/hyperion.ng/master/bin/scripts/docker-compile.sh && chmod +x *.sh && ./docker-compile.sh -i x86_64 ``` **x86_64 (Debian Buster):** -``` +```console wget -qN https://raw.github.com/hyperion-project/hyperion.ng/master/bin/scripts/docker-compile.sh && chmod +x *.sh && ./docker-compile.sh -i x86_64 -t buster ``` **Raspberry Pi v1 & ZERO (Debian Stretch)** -``` +```console wget -qN https://raw.github.com/hyperion-project/hyperion.ng/master/bin/scripts/docker-compile.sh && chmod +x *.sh && ./docker-compile.sh -i armv6l ``` **Raspberry Pi v1 & ZERO (Debian Buster)** -``` +```console wget -qN https://raw.github.com/hyperion-project/hyperion.ng/master/bin/scripts/docker-compile.sh && chmod +x *.sh && ./docker-compile.sh -i armv6l -t buster ``` **Raspberry Pi 2/3/4 (Debian Stretch)** -``` +```console wget -qN https://raw.github.com/hyperion-project/hyperion.ng/master/bin/scripts/docker-compile.sh && chmod +x *.sh && ./docker-compile.sh -i armv7l ``` **Raspberry Pi 2/3/4 (Debian Buster)** -``` +```console wget -qN https://raw.github.com/hyperion-project/hyperion.ng/master/bin/scripts/docker-compile.sh && chmod +x *.sh && ./docker-compile.sh -i armv7l -t buster ``` +## Cross compilation on x86_64 for developers +Using additional options you can cross compile locally +-l: use a local hyperion source code directory rather than cloning from GitHub +-c: do incremental compiles, Note: you need to keep the image and tag stable +**Compile code in $HYPERION_HOME incrementally for Raspberry Pi 2/3/4 (Debian Buster)** +```console +cd $HYPERION_HOME +./bin/scripts/docker-compile.sh -l -c -i armv7l -t buster +``` # The usual way ## Debian/Ubuntu/Win10LinuxSubsystem -``` +```console sudo apt-get update sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libqt5sql5-sqlite libqt5svg5-dev libqt5x11extras5-dev libusb-1.0-0-dev python3-dev libcec-dev libxcb-image0-dev libxcb-util0-dev libxcb-shm0-dev libxcb-render0-dev libxcb-randr0-dev libxrandr-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev libjpeg-dev libturbojpeg0-dev libssl-dev zlib1g-dev ``` **on RPI you need the videocore IV headers** -``` +```console sudo apt-get install libraspberrypi-dev ``` **OSMC on Raspberry Pi** -``` +```console sudo apt-get install rbp-userland-dev-osmc ``` @@ -71,7 +81,7 @@ See [AUR](https://aur.archlinux.org/packages/?O=0&SeB=nd&K=hyperion&outdated=&SB ## Fedora The following dependencies are needed to build hyperion.ng on fedora. -``` +```console sudo dnf -y groupinstall "Development Tools" sudo dnf install python3-devel qt-devel qt5-qtbase-devel qt5-qtserialport-devel libjpeg-devel xrandr xcb-util-image-devel qt5-qtx11extras-devel turbojpeg-devel libusb-devel avahi-libs avahi-compat-libdns_sd-devel xcb-util-devel dbus-devel openssl-devel fedora-packager rpmdevtools gcc libcec-devel ``` @@ -81,13 +91,8 @@ After installing the dependencies, you can continue with the compile instruction To install on OS X you either need Homebrew or Macport but Homebrew is the recommended way to install the packages. To use Homebrew XCode is required as well, use `brew doctor` to check your install. First you need to install the dependencies: -``` -brew install qt5 -brew install python3 -brew install cmake -brew install libusb -brew install doxygen -brew install zlib +```console +brew install qt5 python3 cmake libusb doxygen zlib ``` ## Windows (WIP) @@ -110,17 +115,17 @@ We assume a 64bit Windows 10. Install the following; # Compiling and installing Hyperion -### The general quick way (without big comments) +## The general quick way (without big comments) -complete automated process for Mac/Linux: -```bash +**complete automated process for Mac/Linux:** +```console wget -qO- https://raw.githubusercontent.com/hyperion-project/hyperion.ng/master/bin/compile.sh | sh ``` -some more detailed way: (or more or less the content of the script above) -be sure you fulfill the prerequisites above. +**some more detailed way: (or more or less the content of the script above)** -```bash +```console +# be sure you fulfill the prerequisites above git clone --recursive https://github.com/hyperion-project/hyperion.ng.git hyperion cd hyperion mkdir build @@ -138,92 +143,94 @@ bin/hyperiond # webui is located on localhost:8090 or 8091 ``` +## The detailed way (with many comments) -### Download +**Download:** Creates hyperion directory and checkout the code from github - -``` +```console export HYPERION_DIR="hyperion" git clone --recursive --depth 1 https://github.com/hyperion-project/hyperion.ng.git "$HYPERION_DIR" ``` -### Preparations +**Preparations:** Change into hyperion folder and create a build folder -``` +```console cd "$HYPERION_DIR" mkdir build cd build ``` -### Generate the make files: - +**Generate the make files:** To generate make files with automatic platform detection and default settings: - -This should fit to *RPI, x86, amlogic/wetek* -``` +This should fit to *RPI, x86, amlogic/wetek: +```console cmake -DCMAKE_BUILD_TYPE=Release .. ``` *Developers on x86* linux should use: -``` +```console cmake -DPLATFORM=x11-dev -DCMAKE_BUILD_TYPE=Release .. ``` To use framebuffer instead of dispmanx (for example on the *cubox-i*): -``` +```console cmake -DENABLE_FB=ON -DCMAKE_BUILD_TYPE=Release .. ``` To generate make files on OS X: Platform should be auto detected and refer to osx, you can also force osx: -``` +```console cmake -DPLATFORM=osx -DCMAKE_BUILD_TYPE=Release .. ``` +In case you would like to build with a dedicated Qt version, provide the version's location via the CMAKE_PREFIX_PATH: +```console +cmake -DCMAKE_PREFIX_PATH=/opt/Qt/5.15.2/gcc_64 -DCMAKE_BUILD_TYPE=Release .. +``` + To generate files on Windows (Release+Debug capable): Platform should be auto detected and refer to windows, you can also force windows: -```sh +```posh # You might need to setup MSVC env first call "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat" cmake -DPLATFORM=windows -G "Visual Studio 16 2019" .. ``` -### Run make to build Hyperion +**Run make to build Hyperion:** The `-j $(nproc)` specifies the amount of CPU cores to use. -```bash +```console make -j $(nproc) ``` On a mac you can use ``sysctl -n hw.ncpu`` to get the number of available CPU cores to use. -```bash +```console make -j $(sysctl -n hw.ncpu) ``` -On Windows run -```bash +On Windows run: +```posh cmake --build . --config Release -- -maxcpucount ``` Maintainer: To build installer, install [NSIS](https://nsis.sourceforge.io/Main_Page) and set env `VCINSTALLDIR="C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC"` -### Install hyperion into your system - +**Install hyperion into your system:** Copy all necessary files to ``/usr/local/share/hyperion`` -```bash +```console sudo make install/strip ``` If you want to install into another location call this before installing -```bash +```console cmake -DCMAKE_INSTALL_PREFIX=/home/pi/apps .. ``` This will install to ``/home/pi/apps/share/hyperion`` -### Integrating hyperion into your system +**Integrating hyperion into your system:** ... ToDo diff --git a/CrossCompileHowto.md b/CrossCompileHowto.md deleted file mode 100644 index 4617370a..00000000 --- a/CrossCompileHowto.md +++ /dev/null @@ -1,93 +0,0 @@ -# Cross-Compile Hyperion-NG -Leverage the power of a host environment (here Ubuntu) compiling for a target platform (here Raspberry Pi). -Use a clean Raspbian Stretch Lite (on target) and Ubuntu 18/19 (on host) to execute the steps outlined below. -## On the Target system (here Raspberry Pi) -Install required additional packages. -``` -sudo apt-get install qtbase5-dev libqt5serialport5-dev libqt5svg5-dev libusb-1.0-0-dev python3-dev libcec-dev libxcb-util0-dev libxcb-randr0-dev libxrandr-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev libjpeg-dev libturbojpeg0-dev libqt5sql5-sqlite aptitude qt5-default rsync libssl-dev zlib1g-dev -``` -## On the Host system (here Ubuntu) -Update the Ubuntu environment to the latest stage and install required additional packages. -``` -sudo apt-get update -sudo apt-get upgrade -sudo apt-get -qq -y install git rsync cmake build-essential qtbase5-dev libqt5serialport5-dev libqt5svg5-dev libqt5sql5-sqlite libqt5x11extras5-dev libusb-1.0-0-dev python3-dev libcec-dev libxcb-image0-dev libxcb-util0-dev libxcb-shm0-dev libxcb-render0-dev libxcb-randr0-dev libxrandr-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev libjpeg-dev libturbojpeg0-dev libssl-dev zlib1g-dev -``` - -Refine the target IP or hostname, plus userID as required and set-up cross-compilation environment: -``` -export TARGET_IP=x.x.x.x -export TARGET_USER=pi -``` -``` -export CROSSROOT="$HOME/crosscompile" -export RASCROSS_DIR="$CROSSROOT/raspberrypi" -export ROOTFS_DIR="$RASCROSS_DIR/rootfs" -export TOOLCHAIN_DIR="$RASCROSS_DIR/tools" -export QT5_DIR="$CROSSROOT/Qt5" -export HYPERION_DIR="$HOME/hyperion.ng" -``` -Get native files from target platform into the host-environment: -``` -mkdir -p "$ROOTFS_DIR/lib" -mkdir -p "$ROOTFS_DIR/usr" -rsync -rl --delete-after --copy-unsafe-links $TARGET_USER@$TARGET_IP:/lib "$ROOTFS_DIR" -rsync -rl --delete-after --copy-unsafe-links $TARGET_USER@$TARGET_IP:/usr/include "$ROOTFS_DIR/usr" -rsync -rl --delete-after --copy-unsafe-links $TARGET_USER@$TARGET_IP:/usr/lib "$ROOTFS_DIR/usr" -``` - -### Raspberry Pi specific steps -Get Raspberry Pi firmware: -``` -mkdir -p "$RASCROSS_DIR/firmware" -git clone --depth 1 https://github.com/raspberrypi/firmware.git "$RASCROSS_DIR/firmware" -ln -s "$RASCROSS_DIR/firmware/hardfp/opt" "$ROOTFS_DIR/opt" -``` -Get toolchain files which allows to build ARM executables on x86 platforms: -``` -mkdir -p "$TOOLCHAIN_DIR" -cd $TOOLCHAIN_DIR -wget -c https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz --no-check-certificate -tar -xvf gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz - -``` -### Install the Qt5 framework -``` -mkdir -p "$QT5_DIR" -cd "$QT5_DIR" -wget -c https://download.qt.io/new_archive/qt/5.7/5.7.1/qt-opensource-linux-x64-5.7.1.run -chmod +x $QT5_DIR/*.run -``` -Display absolute installation directory to be used in Qt5 installer: -``` -echo $HOME/crosscompile/Qt5 -``` -Start the Qt5 installation. -Follow the dialogs and install in absolute directory of ```$HOME/crosscompile/Qt5``` (copy from above) -``` -./qt-opensource-linux-x64-5.7.1.run -``` -### Get the Hyperion-NG source files -``` -git clone --recursive https://github.com/hyperion-project/hyperion.ng.git "$HYPERION_DIR" -``` -### Get required submodules for Hyperion -``` -cd "$HYPERION_DIR" -git fetch --recurse-submodules -j2 -``` -### Compile Hyperion-NG -``` -cd "$HYPERION_DIR" -chmod +x "$HYPERION_DIR/bin/"*.sh -./bin/create_all_releases.sh -``` -### Transfer output packages to target platform and install Hyperion-NG -Output packages for target platform (.deb, .tar.gz, .sh) can be found here: -``` -$HYPERION_DIR/deploy/rpi -``` -# Install Hyperion-NG on target platform -t.b.d. - - diff --git a/bin/scripts/docker-compile.sh b/bin/scripts/docker-compile.sh index 3ff06ad7..eb9a24bc 100644 --- a/bin/scripts/docker-compile.sh +++ b/bin/scripts/docker-compile.sh @@ -10,32 +10,44 @@ BUILD_TYPE="Release" # the docker image at GitHub Container Registry BUILD_IMAGE="x86_64" # the docker tag at GitHub Container Registry -BUILD_TAG="stretch" +BUILD_TAG="buster" # build packages (.deb .zip ...) BUILD_PACKAGES=true # packages string inserted to cmake cmd PACKAGES="" +#Run build using GitHub code files +BUILD_LOCAL=0 +#Build from scratch +BUILD_INCREMENTAL=0 +#Verbose output +_VERBOSE=0 # get current path to this script, independent of calling pushd . > /dev/null -SCRIPT_PATH="${BASH_SOURCE[0]}" -if ([ -h "${SCRIPT_PATH}" ]); then - while([ -h "${SCRIPT_PATH}" ]); do cd `dirname "$SCRIPT_PATH"`; - SCRIPT_PATH=`readlink "${SCRIPT_PATH}"`; done +BASE_PATH="${BASH_SOURCE[0]}" +if ([ -h "${BASE_PATH}" ]); then + while([ -h "${BASE_PATH}" ]); do cd `dirname "${BASE_PATH}"`; + BASE_PATH=`readlink "${BASE_PATH}"`; done fi -cd `dirname ${SCRIPT_PATH}` > /dev/null -SCRIPT_PATH=`pwd`; +cd `dirname ${BASE_PATH}` > /dev/null +BASE_PATH=`pwd`; popd > /dev/null +BASE_PATH=`pwd`;function log () { + if [[ $_V -eq 1 ]]; then + echo "$@" + fi +} + set +e -$DOCKER ps >/dev/null 2>&1 +${DOCKER} ps >/dev/null 2>&1 if [ $? != 0 ]; then DOCKER="sudo docker" fi # check if docker is available -if ! $DOCKER ps >/dev/null; then +if ! ${DOCKER} ps >/dev/null; then echo "Error connecting to docker:" - $DOCKER ps + ${DOCKER} ps printHelp exit 1 fi @@ -46,7 +58,7 @@ function printHelp { echo "######################################################## ## A script to compile Hyperion inside a docker container ## Requires installed Docker: https://www.docker.com/ -## Without arguments it will compile Hyperion for Debian Stretch (x86_64). +## Without arguments it will compile Hyperion for Debian Buster (x86_64) and uses Hyperion code from GitHub repository. ## Supports Raspberry Pi (armv6l, armv7l) cross compilation (Debian Stretch/Buster) and native compilation (Raspbian Stretch/Buster) ## ## Homepage: https://www.hyperion-project.org @@ -54,15 +66,25 @@ echo "######################################################## ######################################################## # These are possible arguments to modify the script behaviour with their default values # -# docker-compile.sh -h # Show this help message -# docker-compile.sh -i x86_64 # The docker image, one of x86_64 | armv6l | armv7l | rpi-raspbian -# docker-compile.sh -t stretch # The docker tag, stretch or buster -# docker-compile.sh -b Release # cmake Release or Debug build -# docker-compile.sh -p true # If true build packages with CPack +# docker-compile.sh -h # Show this help message +# docker-compile.sh -i x86_64 # The docker image, one of x86_64 | armv6l | armv7l | rpi-raspbian +# docker-compile.sh -t stretch # The docker tag, stretch or buster +# docker-compile.sh -b Release # cmake Release or Debug build +# docker-compile.sh -p true # If true, build packages with CPack +# docker-compile.sh -l # Run build using local code files +# docker-compile.sh -c # Run incremental build, i.e. do not delete files created during previous build # More informations to docker tags at: https://github.com/Hyperion-Project/hyperion.docker-ci" } -while getopts i:t:b:p:h option +function log () { + if [[ $_VERBOSE -eq 1 ]]; then + echo "$@" + fi +} + +echo "Compile Hyperion using a Docker container" + +while getopts i:t:b:p:lcvh option do case "${option}" in @@ -70,51 +92,83 @@ do t) BUILD_TAG=${OPTARG};; b) BUILD_TYPE=${OPTARG};; p) BUILD_PACKAGES=${OPTARG};; + l) BUILD_LOCAL=1;; + c) BUILD_INCREMENTAL=1;; + v) _VERBOSE=1;; h) printHelp; exit 0;; esac done # determine package creation -if [ $BUILD_PACKAGES == "true" ]; then +if [ ${BUILD_PACKAGES} == "true" ]; then PACKAGES="package" fi -echo "---> Initialize with IMAGE:TAG=${BUILD_IMAGE}:${BUILD_TAG}, BUILD_TYPE=${BUILD_TYPE}, BUILD_PACKAGES=${BUILD_PACKAGES}" +echo "---> Initialize with IMAGE:TAG=${BUILD_IMAGE}:${BUILD_TAG}, BUILD_TYPE=${BUILD_TYPE}, BUILD_PACKAGES=${BUILD_PACKAGES}, BUILD_LOCAL=${BUILD_LOCAL}, BUILD_INCREMENTAL=${BUILD_INCREMENTAL}" + +CODE_PATH=${BASE_PATH}; +BUILD_DIR="build-${BUILD_IMAGE}-${BUILD_TAG}" +BUILD_PATH="${BASE_PATH}/${BUILD_DIR}" +DEPLOY_DIR="deploy/${BUILD_IMAGE}/${BUILD_TAG}" +DEPLOY_PATH="${BASE_PATH}/${DEPLOY_DIR}" + +log "---> BASE_PATH = ${BASE_PATH}" +log "---> BUILD_DIR = ${BUILD_DIR}" +log "---> BUILD_PATH = ${BUILD_PATH}" +log "---> DEPLOY_DIR = ${DEPLOY_DIR}" +log "---> DEPLOY_PATH = ${DEPLOY_PATH}" # cleanup deploy folder, create folder for ownership -sudo rm -fr $SCRIPT_PATH/deploy >/dev/null 2>&1 -mkdir $SCRIPT_PATH/deploy >/dev/null 2>&1 +sudo rm -fr ${DEPLOY_PATH} >/dev/null 2>&1 +mkdir -p ${DEPLOY_PATH} >/dev/null 2>&1 # get Hyperion source, cleanup previous folder +if [ ${BUILD_LOCAL} == 0 ]; then +CODE_PATH="${CODE_PATH}/hyperion/" echo "---> Downloading Hyperion source code from ${GIT_REPO_URL}" -sudo rm -fr $SCRIPT_PATH/hyperion >/dev/null 2>&1 -git clone --recursive --depth 1 -q $GIT_REPO_URL $SCRIPT_PATH/hyperion || { echo "---> Failed to download Hyperion source code! Abort"; exit 1; } +sudo rm -fr ${CODE_PATH} >/dev/null 2>&1 +git clone --recursive --depth 1 -q ${GIT_REPO_URL} ${CODE_PATH} || { echo "---> Failed to download Hyperion source code! Abort"; exit 1; } +fi + +#Remove previous build area, if no incremental build +if [ ${BUILD_INCREMENTAL} != 1 ]; then +sudo rm -fr ${BASE_PATH}/${BUILD_PATH} >/dev/null 2>&1 +fi +mkdir -p ${BASE_PATH}/${BUILD_PATH} >/dev/null 2>&1 + +echo "---> Compiling Hyperion from source code at ${CODE_PATH}" # Steps: # Update lokal docker image # Remove container after stop -# Mount /deploy to /deploy +# Mount deploment path to /deploy # Mount source dir to /source # Use target docker image # execute inside container all commands on bash echo "---> Startup docker..." $DOCKER run --rm \ - -v "${SCRIPT_PATH}/deploy:/deploy" \ - -v "${SCRIPT_PATH}/hyperion:/source:ro" \ - $REGISTRY_URL/$BUILD_IMAGE:$BUILD_TAG \ - /bin/bash -c "mkdir hyperion && cp -r /source/. /hyperion && - cd /hyperion && mkdir build && cd build && + -v "${DEPLOY_PATH}:/deploy" \ + -v "${CODE_PATH}/:/source:rw" \ + ${REGISTRY_URL}/${BUILD_IMAGE}:${BUILD_TAG} \ + /bin/bash -c "mkdir -p /source/${BUILD_DIR} && cd /source/${BUILD_DIR} && cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} .. || exit 2 && make -j $(nproc) ${PACKAGES} || exit 3 && - echo '---> Copy binaries and packages to host folder: ${SCRIPT_PATH}/deploy' && - cp -v /hyperion/build/bin/h* /deploy/ 2>/dev/null || : && - cp -v /hyperion/build/Hyperion-* /deploy/ 2>/dev/null || : && + echo '---> Copying packages to host folder: ${DEPLOY_PATH}' && + cp -v /source/${BUILD_DIR}/Hyperion-* /deploy 2>/dev/null || : && exit 0; exit 1 " || { echo "---> Hyperion compilation failed! Abort"; exit 4; } # overwrite file owner to current user -sudo chown -fR $(stat -c "%U:%G" $SCRIPT_PATH/deploy) $SCRIPT_PATH/deploy +sudo chown -fR $(stat -c "%U:%G" ${BASE_PATH}) ${BUILD_PATH} +sudo chown -fR $(stat -c "%U:%G" ${BASE_PATH}) ${DEPLOY_PATH} -echo "---> Script finished, view folder ${SCRIPT_PATH}/deploy for compiled packages and binaries" +if [ ${BUILD_LOCAL} == 1 ]; then + echo "---> Find compiled binaries in: ${BUILD_PATH}/bin" +fi + +if [ ${BUILD_PACKAGES} == "true" ]; then + echo "---> Find deployment packages in: ${DEPLOY_PATH}" +fi + echo "---> Script finished" exit 0