From 9d029c4ae45f59a590f427578c6d023389c93eb0 Mon Sep 17 00:00:00 2001 From: Simran Date: Tue, 27 Oct 2020 18:21:54 +0100 Subject: [PATCH 1/2] Docker: Add more & update libraries for FFmpeg (#1279) * Fix error: x265 not found using pkg-config Need to checkout tag or fetch tags, see https://bitbucket.org/multicoreware/x265_git/issues/572/pc-file-is-not-generated * Add libs ogg, vorbis, theora, webp, xvid, openjpeg, png to ffmpeg * ffmpeg x265 uses libnuma (but optional) ci-common image not updated yet, pkg-config error resolved by checking out a tag. Installing libnuma was a suggested fix here: https://stackoverflow.com/questions/51918409/compiling-ffmpeg-x265-not-found-using-pkg-config * Add Docker Images readme --- docker/README.md | 62 +++++++ docker/ci-ffmpeg/Dockerfile | 33 +++- docker/scripts/build_ffmpeg.sh | 180 +++++++++++++++---- docker/scripts/common/install_yumpackages.sh | 1 + 4 files changed, 242 insertions(+), 34 deletions(-) create mode 100644 docker/README.md diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..750c17e64 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,62 @@ +# Docker Images + +Olive uses Docker containers for continuous integration on Linux. +No Docker images are involved for the Windows and macOS CI. + +## Overview + +`ci-common` is the shared build image with GCC, Clang and packages that are +needed by most dependent images. It is used to compile Olive's dependencies +in a controlled environment. The final CI image `ci-olive` is assembled from +images maintained by the Olive team as well as from +[aswf-docker](https://github.com/AcademySoftwareFoundation/aswf-docker/). + +Dependency hierarchy: + +1. `ci-common` +2. `ci-otio`, `ci-crashpad`, `ci-ffmpeg` +3. `ci-olive` + +## Usage + +Pull images from [Docker Hub](https://hub.docker.com/u/olivevideoeditor): + +``` +docker pull olivevideoeditor/ci-common:2 +docker pull olivevideoeditor/ci-package-otio +docker pull olivevideoeditor/ci-package-crashpad +docker pull olivevideoeditor/ci-package-ffmpeg:4.2.4 +docker pull olivevideoeditor/ci-olive:2021.2 +``` + +Use `ci-olive` image as local build container, by mounting working copy at +`~/olive` into guest system at `/opt/olive/olive`: + +```bash +docker run --rm -it -v ~/olive:/opt/olive/olive olivevideoeditor/ci-olive:2021.2 +mkdir build +cd build +cmake .. -G Ninja +cmake --build . +``` + +Rebuild all images locally: + +``` +cd docker +docker build -t olivevideoeditor/ci-common:2 -f ci-common/Dockerfile . +docker build -t olivevideoeditor/ci-package-otio -f ci-otio/Dockerfile . +docker build -t olivevideoeditor/ci-package-crashpad -f ci-crashpad/Dockerfile . +docker build -t olivevideoeditor/ci-package-ffmpeg:4.2.4 -f ci-ffmpeg/Dockerfile . +docker build -t olivevideoeditor/ci-olive:2021.2 -f ci-olive/Dockerfile . +``` + +Publish images: + +``` +docker push olivevideoeditor/ci-common:2 +docker push olivevideoeditor/ci-package-otio +docker push olivevideoeditor/ci-package-crashpad +docker push olivevideoeditor/ci-package-ffmpeg:4.2.4 +docker push olivevideoeditor/ci-olive:2021.2 +``` diff --git a/docker/ci-ffmpeg/Dockerfile b/docker/ci-ffmpeg/Dockerfile index 8a664ca02..0c3d05273 100644 --- a/docker/ci-ffmpeg/Dockerfile +++ b/docker/ci-ffmpeg/Dockerfile @@ -9,8 +9,17 @@ ARG CI_COMMON_VERSION=2 ARG FFMPEG_VERSION=4.2.4 ARG NASM_VERSION=2.15.03 ARG YASM_VERSION=1.3.0 -ARG LAME_VERSION=3.100 +ARG X265_VERSION=3.4 +ARG OGG_VERSION=1.3.4 ARG OPUS_VERSION=1.3.1 +ARG VORBIS_VERSION=1.3.7 +ARG THEORA_VERSION=1.1.1 +ARG VPX_VERSION=1.9.0 +ARG WEBP_VERSION=1.1.0 +ARG LAME_VERSION=3.100 +ARG XVID_VERSION=1.3.5 +ARG OPENJPEG_VERSION=2.3.1 +ARG LIBPNG_VERSION=1.6.31 # TODO: Make version of x264, x265 and libvpx selectable? ARG NUM_JOBS=2 @@ -21,8 +30,17 @@ ARG CI_COMMON_VERSION ARG FFMPEG_VERSION ARG NASM_VERSION ARG YASM_VERSION -ARG LAME_VERSION +ARG X265_VERSION +ARG OGG_VERSION ARG OPUS_VERSION +ARG VORBIS_VERSION +ARG THEORA_VERSION +ARG VPX_VERSION +ARG WEBP_VERSION +ARG LAME_VERSION +ARG XVID_VERSION +ARG OPENJPEG_VERSION +ARG LIBPNG_VERSION # TODO: Make version of x264, x265 and libvpx selectable? ARG NUM_JOBS=2 @@ -43,8 +61,17 @@ ENV OLIVE_ORG=${OLIVE_ORG} \ FFMPEG_VERSION=${FFMPEG_VERSION} \ NASM_VERSION=${NASM_VERSION} \ YASM_VERSION=${YASM_VERSION} \ - LAME_VERSION=${LAME_VERSION} \ + X265_VERSION=${X265_VERSION} \ + OGG_VERSION=${OGG_VERSION} \ OPUS_VERSION=${OPUS_VERSION} \ + VORBIS_VERSION=${VORBIS_VERSION} \ + THEORA_VERSION=${THEORA_VERSION} \ + VPX_VERSION=${VPX_VERSION} \ + WEBP_VERSION=${WEBP_VERSION} \ + LAME_VERSION=${LAME_VERSION} \ + XVID_VERSION=${XVID_VERSION} \ + OPENJPEG_VERSION=${OPENJPEG_VERSION} \ + LIBPNG_VERSION=${LIBPNG_VERSION} \ NUM_JOBS=${NUM_JOBS} \ OLIVE_INSTALL_PREFIX=/usr/local diff --git a/docker/scripts/build_ffmpeg.sh b/docker/scripts/build_ffmpeg.sh index 1f2c36a91..7853903b3 100644 --- a/docker/scripts/build_ffmpeg.sh +++ b/docker/scripts/build_ffmpeg.sh @@ -46,6 +46,7 @@ wait # Set up libx264 { + # TODO: Checkout stable branch instead of master? git clone --depth 1 "https://code.videolan.org/videolan/x264.git" cd x264 ./configure \ @@ -53,7 +54,7 @@ wait --enable-shared \ --enable-pic \ --disable-cli - make -j${NUM_JOBS} + make make install cd .. rm -rf x264 @@ -63,18 +64,121 @@ wait { # BitBucket dropped support for Mercurial repos #hg clone https://bitbucket.org/multicoreware/x265 - git clone --depth 1 "https://bitbucket.org/multicoreware/x265_git.git" x265 + # Need to fetch tags (off by default for shallow clones) or checkout a tag to avoid pkg-config error + git clone --depth 1 --branch "${X265_VERSION}" "https://bitbucket.org/multicoreware/x265_git.git" x265 cd x265/build/linux cmake \ -G "Unix Makefiles" \ -DCMAKE_INSTALL_PREFIX="${OLIVE_INSTALL_PREFIX}" \ ../../source - make -j${NUM_JOBS} + make make install cd ../../.. rm -rf x265 } & +# Set up libogg +{ + curl -fLsS -o libogg.tar.gz "http://downloads.xiph.org/releases/ogg/libogg-${OGG_VERSION}.tar.gz" + tar xf libogg.tar.gz + rm -f libogg.tar.gz + cd libogg* + ./configure \ + --prefix="${OLIVE_INSTALL_PREFIX}" \ + --enable-shared + make + make install + cd .. + rm -rf libogg* +} & + +# Set up libopus +{ + curl -fLsS -o opus.tar.gz "https://archive.mozilla.org/pub/opus/opus-${OPUS_VERSION}.tar.gz" + tar xf opus.tar.gz + rm -f opus.tar.gz + cd opus* + ./configure \ + --prefix="${OLIVE_INSTALL_PREFIX}" \ + --enable-shared + make + make install + cd .. + rm -rf opus* +} & + +# join jobs, libvorbis and libtheora depend on libogg +wait + +# Set up libvorbis +{ + curl -fLsS -o libvorbis.tar.gz "http://downloads.xiph.org/releases/vorbis/libvorbis-${VORBIS_VERSION}.tar.gz" + tar xf libvorbis.tar.gz + rm -f libvorbis.tar.gz + cd libvorbis* + ./configure \ + --prefix="${OLIVE_INSTALL_PREFIX}" \ + --with-ogg="${OLIVE_INSTALL_PREFIX}" \ + --enable-shared + make + make install + cd .. + rm -rf libvorbis* +} & + +# Set up libtheora +{ + curl -fLsS -o libtheora.tar.gz "http://downloads.xiph.org/releases/theora/libtheora-${THEORA_VERSION}.tar.gz" + tar xf libtheora.tar.gz + rm -f libtheora.tar.gz + cd libtheora* + ./configure \ + --prefix="${OLIVE_INSTALL_PREFIX}" \ + --with-ogg="${OLIVE_INSTALL_PREFIX}" \ + --enable-shared + make + make install + cd .. + rm -rf libtheora* +} & + +# Set up libvpx +{ + git clone --depth 1 --branch "v${VPX_VERSION}" "https://chromium.googlesource.com/webm/libvpx.git" + cd libvpx + ./configure \ + --prefix="${OLIVE_INSTALL_PREFIX}" \ + --enable-shared \ + --enable-pic \ + --enable-vp8 \ + --enable-vp9 \ + --enable-vp9-highbitdepth \ + --as=yasm \ + --disable-examples \ + --disable-unit-tests \ + --disable-docs \ + --disable-install-bins + make + make install + cd .. + rm -rf libvpx +} & + +### Set up libwebp +{ + curl -fLsS -o libwebp.tar.gz "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz" + tar xf libwebp.tar.gz + rm -f libwebp.tar.gz + cd libwebp* + ./configure \ + --prefix="${OLIVE_INSTALL_PREFIX}" \ + --enable-shared + make + make install + cd .. + rm -rf libwebp* +} & + # Set up libmp3lame { curl -fLsS -o lame.tar.gz "https://downloads.sourceforge.net/project/lame/lame/${LAME_VERSION}/lame-${LAME_VERSION}.tar.gz" @@ -86,45 +190,54 @@ wait --enable-shared \ --enable-nasm \ --disable-frontend - make -j${NUM_JOBS} + make make install cd .. rm -rf lame* } & -# Set up libopus +### Set up xvid { - curl -fLsS -o opus.tar.gz "https://archive.mozilla.org/pub/opus/opus-${OPUS_VERSION}.tar.gz" - tar xf opus.tar.gz - rm -f opus.tar.gz - cd opus* + curl -fLsS -o xvidcore.tar.gz "http://downloads.xvid.org/downloads/xvidcore-${XVID_VERSION}.tar.gz" + tar xf xvidcore.tar.gz + rm -f xvidcore.tar.gz + cd xvidcore/build/generic ./configure \ --prefix="${OLIVE_INSTALL_PREFIX}" \ - --enable-shared - make -j${NUM_JOBS} + --bindir="${OLIVE_INSTALL_PREFIX}/bin" + make make install - cd .. - rm -rf opus* + cd ../../.. + rm -rf xvidcore } & -# Set up libvpx +# Set up openjpeg { - git clone --depth 1 "https://chromium.googlesource.com/webm/libvpx.git" - cd libvpx - ./configure \ - --prefix="${OLIVE_INSTALL_PREFIX}" \ - --enable-shared \ - --enable-pic \ - --enable-vp9-highbitdepth \ - --as=yasm \ - --disable-examples \ - --disable-unit-tests \ - --disable-docs \ - --disable-install-bins - make -j${NUM_JOBS} + curl -fLsS -o openjpeg.tar.gz "https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz" + tar xf openjpeg.tar.gz + rm -f openjpeg.tar.gz + cd openjpeg* + cmake \ + -DBUILD_THIRDPARTY:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX="${OLIVE_INSTALL_PREFIX}" \ + . + make make install cd .. - rm -rf libvpx + rm -rf openjpeg* +} & + +# Set up libpng +{ + git clone --depth 1 "https://git.code.sf.net/p/libpng/code" libpng --branch "v${LIBPNG_VERSION}" + cd libpng + ./autogen.sh + ./configure \ + --prefix="${OLIVE_INSTALL_PREFIX}" + make check + make install + cd .. + rm -rf libpng } & # join all jobs @@ -142,11 +255,16 @@ PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" ./configure \ --enable-version3 \ --enable-shared \ --enable-libfreetype \ - --enable-libmp3lame \ - --enable-libopus \ - --enable-libvpx \ --enable-libx264 \ --enable-libx265 \ + --enable-libopus \ + --enable-libvorbis \ + --enable-libtheora \ + --enable-libvpx \ + --enable-libwebp \ + --enable-libmp3lame \ + --enable-libxvid \ + --enable-libopenjpeg \ --prefix="${OLIVE_INSTALL_PREFIX}" \ --extra-libs=-lpthread \ --extra-libs=-lm \ diff --git a/docker/scripts/common/install_yumpackages.sh b/docker/scripts/common/install_yumpackages.sh index f5003675c..064c5e1f1 100644 --- a/docker/scripts/common/install_yumpackages.sh +++ b/docker/scripts/common/install_yumpackages.sh @@ -26,6 +26,7 @@ yum install --setopt=tsflags=nodocs -y \ libxkbcommon-x11-devel \ libXScrnSaver libXScrnSaver-devel \ mesa-libGL-devel \ + numactl-devel \ openjpeg2-devel \ pciutils-devel \ pulseaudio-libs pulseaudio-libs-devel \ From 2959f1eb11ff63262783e027ccfb8694225e5bfd Mon Sep 17 00:00:00 2001 From: Simran Date: Wed, 28 Oct 2020 09:10:16 +0100 Subject: [PATCH 2/2] Update 50-build_issue.md There is a separate label for continuous integration now, but we don't have a separate issue template for that. It's unlikely that someone wants to report CI issue anyway. --- .github/ISSUE_TEMPLATE/50-build_issue.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/50-build_issue.md b/.github/ISSUE_TEMPLATE/50-build_issue.md index 556e4cabe..8ac8af650 100644 --- a/.github/ISSUE_TEMPLATE/50-build_issue.md +++ b/.github/ISSUE_TEMPLATE/50-build_issue.md @@ -1,8 +1,8 @@ --- name: Build/Packaging about: >- - Report an issue related to compiling or packaging (including continuous - integration). Note that we do not officially support custom build + Report an issue related to compiling or packaging. + Note that we do not officially support custom build configurations and may not address issues involving them. title: '[BUILD] ' labels: 'Building/Packaging, Triage'