From 14c683a63a26846d9a5ee564210c5a4acb6a11dc Mon Sep 17 00:00:00 2001 From: Carl Kittelberger Date: Mon, 7 Aug 2017 11:38:02 +0200 Subject: [PATCH] Initial commit. --- .env | 1 + docker-compose.yml | 51 ++++++ images/caddy/Caddyfile | 18 +++ images/caddy/Dockerfile | 9 ++ images/caddy/time.txt | 1 + images/ffmpeg/Dockerfile | 154 ++++++++++++++++++ images/ffmpeg/ffmpeg-hls.sh | 314 ++++++++++++++++++++++++++++++++++++ 7 files changed, 548 insertions(+) create mode 100644 .env create mode 100644 docker-compose.yml create mode 100644 images/caddy/Caddyfile create mode 100644 images/caddy/Dockerfile create mode 100644 images/caddy/time.txt create mode 100644 images/ffmpeg/Dockerfile create mode 100755 images/ffmpeg/ffmpeg-hls.sh diff --git a/.env b/.env new file mode 100644 index 0000000..a1d960e --- /dev/null +++ b/.env @@ -0,0 +1 @@ +HLS_SETTINGS=-hls_list_size 5 -hls_time 6 -hls_enc 0 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..957521b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +version: "3.1" + +volumes: + published: + +networks: + web_front: + external: true + +services: + ffmpeg: + build: images/ffmpeg + #build: /srv/nfs4/library/ffmpeg + restart: always + volumes: + - published:/var/out + working_dir: /var/out + environment: + OUTPUT_MP3_BITRATES: 320k 192k 128k 72k 56k + OUTPUT_AAC_BITRATES: 320k 192k 128k 96k + OUTPUT_AAC_HE_BITRATES: 64k 48k + OUTPUT_AAC_HEV2_BITRATES: 48k 24k + OUTPUT_OPUS_BITRATES: 320k 192k 128k 64k 48k 24k + OUTPUT_VORBIS_BITRATES: 320k 192k 128k 64k 48k + CHANNEL_ID: rektnetwork + FFMPEG_ARGS: -async 2 + FFMPEG_LOG_LEVEL: debug + UTC_TIMING_URL: "https://direct.streaminginter.net/adapt/rektnetwork/time.txt" + command: /usr/local/bin/ffmpeg-hls + + caddy: + #image: icedream/caddy + build: images/caddy + volumes: + - published:/data:ro + #ports: + # - "10080:80" + labels: + oss.dreamnetwork.proxy.target.tls: "1" + oss.dreamnetwork.proxy.target.tls.cert: /etc/letsencrypt/live/streaminginter.net/fullchain.pem + oss.dreamnetwork.proxy.target.tls.key: /etc/letsencrypt/live/streaminginter.net/privkey.pem + oss.dreamnetwork.proxy.target.allowhttp: "1" + oss.dreamnetwork.proxy.target.host: "http://direct.streaminginter.net, https://direct.streaminginter.net" + oss.dreamnetwork.proxy.target.path: "/adapt/rektnetwork/" + oss.dreamnetwork.proxy.source.port: "80" + oss.dreamnetwork.proxy.source.proto: "http" + oss.dreamnetwork.proxy.source.trimpath: "/adapt/rektnetwork" + expose: + - "80" + networks: + - web_front diff --git a/images/caddy/Caddyfile b/images/caddy/Caddyfile new file mode 100644 index 0000000..67fb7da --- /dev/null +++ b/images/caddy/Caddyfile @@ -0,0 +1,18 @@ +http://* + +header / Cache-Control "no-store" +header / Access-Control-Allow-Origin "*" +header / Access-Control-Expose-Headers "Content-Length" +header / Access-Control-Allow-Headers "Range" + +gzip { + ext .m3u8 .mpd .xml +} + +templates /time.txt + +push dash.mpd *.hdr +push hls.m3u8 *.m3u8 + +root /data +browse / diff --git a/images/caddy/Dockerfile b/images/caddy/Dockerfile new file mode 100644 index 0000000..b50f185 --- /dev/null +++ b/images/caddy/Dockerfile @@ -0,0 +1,9 @@ +FROM icedream/caddy + +COPY Caddyfile / +COPY time.txt /data/ +WORKDIR / + +VOLUME ["/data/"] + +EXPOSE 80 \ No newline at end of file diff --git a/images/caddy/time.txt b/images/caddy/time.txt new file mode 100644 index 0000000..ae2cbdc --- /dev/null +++ b/images/caddy/time.txt @@ -0,0 +1 @@ +{{.Now "2006-01-02T15:04:05Z07:00"}} \ No newline at end of file diff --git a/images/ffmpeg/Dockerfile b/images/ffmpeg/Dockerfile new file mode 100644 index 0000000..63802e4 --- /dev/null +++ b/images/ffmpeg/Dockerfile @@ -0,0 +1,154 @@ +FROM alpine:3.6 + +ARG FFMPEG_PACKAGES="\ + x264-libs \ + x265 \ + libcrypto1.0 \ + libssl1.0 \ + libass \ + libogg \ + libvorbis \ + libtheora \ + libvpx \ + libgomp \ + lame \ + xvidcore \ + freetype \ + fontconfig \ + fribidi \ +" + +RUN apk add --no-cache ${FFMPEG_PACKAGES} + +ARG FFMPEG_DEV_PACKAGES="\ + x264-dev \ + x265-dev \ + openssl-dev \ + libass-dev \ + libogg-dev \ + libvorbis-dev \ + libtheora-dev \ + libvpx-dev \ + lame-dev \ + xvidcore-dev \ + freetype-dev \ + fontconfig-dev \ + fribidi-dev \ +" + +ARG SRC=/usr/local + +ARG OPENCOREAMR_VERSION=0.1.5 +ARG OPENCOREAMR_SHA256SUM=2c006cb9d5f651bfb5e60156dbff6af3c9d35c7bbcc9015308c0aff1e14cd341 + +ARG OPUS_VERSION=1.2.1 +ARG OPUS_SHA256SUM=cfafd339ccd9c5ef8d6ab15d7e1a412c054bf4cb4ecbbbcc78c12ef2def70732 + +# fdk-aac 0.1.5+git +ARG FDKAAC_VERSION=af5863a78efdfccd003dd6bea68c4a2cd2ad9f37 +ARG FDKAAC_SHA256SUM=474776ab326f8c9bfd8e244e823a41acd328ce20dc163f8ebef3e7fdd2e879f6 + +ARG LIBVIDSTAB_VERSION=v1.1.0 +ARG LIBVIDSTAB_SHA256SUM=14d2a053e56edad4f397be0cb3ef8eb1ec3150404ce99a426c4eb641861dc0bb + +ARG FFMPEG_VERSION=50aeb6e4edf635147a6651859ec63d15a67f6b87 + +RUN \ + apk add --no-cache --virtual .build-deps \ + alpine-sdk \ + coreutils \ + automake \ + autoconf \ + file \ + libtool \ + cmake \ + yasm \ + ${FFMPEG_DEV_PACKAGES} \ +\ + && DIR=$(mktemp -d) && cd ${DIR} && \ + curl -sL https://downloads.sf.net/project/opencore-amr/opencore-amr/opencore-amr-${OPENCOREAMR_VERSION}.tar.gz | \ + tar -zx --strip-components=1 && \ + ./configure --prefix="${SRC}" --bindir="${SRC}/bin" --enable-shared --disable-doc --datadir=${DIR} && \ + make -j$(nproc) && \ + make install && \ + rm -rf ${DIR} && \ +\ + DIR=$(mktemp -d) && cd ${DIR} && \ + curl -sLO https://archive.mozilla.org/pub/opus/opus-${OPUS_VERSION}.tar.gz && \ + echo ${OPUS_SHA256SUM} opus-${OPUS_VERSION}.tar.gz | sha256sum --check && \ + tar -zx --strip-components=1 -f opus-${OPUS_VERSION}.tar.gz && \ + autoreconf -fiv && \ + ./configure --prefix="${SRC}" --disable-static --disable-doc --datadir="${DIR}" && \ + make -j$(nproc) && \ + make install && \ + rm -rf ${DIR} && \ +\ + DIR=$(mktemp -d) && cd ${DIR} && \ + curl -sL https://github.com/mstorsjo/fdk-aac/archive/${FDKAAC_VERSION}.tar.gz | \ + tar -zx --strip-components=1 && \ + autoreconf -fiv && \ + ./configure --prefix="${SRC}" --disable-static --disable-doc --datadir="${DIR}" && \ + make -j$(nproc) && \ + make install && \ + make distclean && \ + rm -rf ${DIR} && \ +\ + DIR=$(mktemp -d) && cd ${DIR} && \ + curl -sLO https://github.com/georgmartius/vid.stab/archive/${LIBVIDSTAB_VERSION}.tar.gz &&\ + echo ${LIBVIDSTAB_SHA256SUM} ${LIBVIDSTAB_VERSION}.tar.gz | sha256sum --check && \ + tar -zx --strip-components=1 -f ${LIBVIDSTAB_VERSION}.tar.gz && \ + cmake -DCMAKE_INSTALL_PREFIX="${SRC}" . && \ + make -j$(nproc) && \ + make install && \ + rm -rf ${DIR} && \ +\ + DIR=$(mktemp -d) && cd ${DIR} && \ + curl -sLO https://git.ffmpeg.org/gitweb/ffmpeg.git/snapshot/${FFMPEG_VERSION}.tar.gz && \ + tar -zx --strip-components=1 -f ${FFMPEG_VERSION}.tar.gz && \ + ./configure \ + --bindir="${SRC}/bin" \ + --disable-debug \ + --disable-doc \ + --disable-ffplay \ + --disable-static \ + --enable-avresample \ + --enable-gpl \ + --enable-libass \ + --enable-libopencore-amrnb \ + --enable-libopencore-amrwb \ + --enable-libfdk_aac \ + --enable-libfreetype \ + --enable-libvidstab \ + --enable-libmp3lame \ + --enable-libopus \ + --enable-libtheora \ + --enable-libvorbis \ + --enable-libvpx \ + --enable-libx264 \ + --enable-libx265 \ + --enable-libxvid \ + --enable-nonfree \ + --enable-openssl \ + --enable-postproc \ + --enable-shared \ + --enable-small \ + --enable-version3 \ + --extra-cflags="-I${SRC}/include" \ + --extra-ldflags="-L${SRC}/lib" \ + --extra-libs=-ldl \ + --prefix="${SRC}" && \ + make -j$(nproc) && \ + make install && \ + make distclean && \ + hash -r && \ + cd tools && \ + make qt-faststart && \ + cp qt-faststart ${SRC}/bin && \ + rm -rf ${DIR} \ + && apk del --no-cache .build-deps + +### + +RUN apk add --no-cache bash bc + +COPY ffmpeg-hls.sh /usr/local/bin/ffmpeg-hls diff --git a/images/ffmpeg/ffmpeg-hls.sh b/images/ffmpeg/ffmpeg-hls.sh new file mode 100755 index 0000000..e1add7c --- /dev/null +++ b/images/ffmpeg/ffmpeg-hls.sh @@ -0,0 +1,314 @@ +#!/bin/bash -e + +# a +# .og +# ...f = OGG/FLAC +# ...o = OGG/Opus +# ...v = OGG/Vorbis +# .mpe = MP3 +# .a +# ..lc = LC-AAC +# ..he = AAC-HE +# ..hp = AAC-HE v2 (PS) +# v +# .avc = H.264 +# .hvc = H.265 +# .vp +# ...g = VP-7 +# ...h = VP-8 +# s +# .wvt = WebVTT subtitles + +ffmpeg_version="$(ffmpeg -version | awk '{print $3}' | head -n1)" + +# function version_gt() { test "$(echo "$@" | tr " " "\n" | sort -V | head -n 1)" != "$1"; } +# function version_le() { test "$(echo "$@" | tr " " "\n" | sort -V | head -n 1)" == "$1"; } +# function version_lt() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" != "$1"; } +# function version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; } + +function contains_audio() { + case "$1" in + a*|*_a*) + true + ;; + *) + false + ;; + esac +} + +function contains_video() { + case "$1" in + v*|*_v*) + true + ;; + *) + false + ;; + esac +} + +function contains_subtitles() { + case "$1" in + s*|*_s*) + true + ;; + *) + false + ;; + esac +} + +function is_hls_compatible() { + result="1" + for i in ${1//_/ }; do + case "$i" in + ampe|aalc|aah[ep]|vavc) + ;; + aogo) + # opus, ffmpeg can code this but no player supports it and it can even confuse bitrade adaption + result="0" + ;; + *) + result="0" + ;; + esac + done + if [ "$result" -eq 1 ]; then true; else false; fi +} + +function is_webm_dash_compatible() { + result="1" + for i in ${1//_/ }; do + case "$i" in + aog[ov]|vvp[gh]|svvt) + ;; + *) + result="0" + ;; + esac + done + if [ "$result" -eq 1 ]; then true; else false; fi +} + +function parse_bitrate() { + echo "$1" |\ + sed \ + -e 's/g/\*1000000000/gi' \ + -e 's/m/\*1000000/gi' \ + -e 's/k/\*1000/gi' |\ + bc +} + +source_url="http://publish.streaminginter.net:61120/${CHANNEL_ID}/master_signal" +if [ -n "${INPUT_URL}" ]; then + source_url="${INPUT_URL}" +fi +echo "* Source URL: ${source_url}" + +ffmpeg_log_level="${LOG_LEVEL:-warning}" + +ffmpeg_args=() +ffmpeg_args+=(-loglevel "${ffmpeg_log_level}") +ffmpeg_args+=(-i "${source_url}") +if [ -n "${FFMPEG_ARGS}" ]; then + ffmpeg_args+=(${FFMPEG_ARGS}) +fi + +hls_args="" +if [ -n "${HLS_ARGS}" ]; then + hls_args+=":${HLS_ARGS}" +fi + +hls_args="${hls_args}:use_localtime=1" +hls_args="${hls_args}:use_localtime_mkdir=1" +# Reference: https://developer.apple.com/library/content/documentation/General/Reference/HLSAuthoringSpec/Requirements.html +# - 8.4. -> program_data_time +# - 8.13. -> discont_start +hls_args="${hls_args}:hls_flags=discont_start+append_list+omit_endlist+temp_file+delete_segments+program_date_time" +hls_args="${hls_args}:hls_init_time=${HLS_INIT_TIME:-0}" +hls_args="${hls_args}:hls_time=${HLS_TIME:-6}" +hls_args="${hls_args}:hls_list_size=${HLS_LIST_SIZE:-10}" +hls_args="${hls_args}:hls_start_number_source=generic" +hls_args="${hls_args}:hls_ts_options=mpegts_service_type=digital_radio\\\\:mpegts_copyts=1" + +# if version_gt ${ffmpeg_version} 3.3.3; then + hls_args="${hls_args}:hls_segment_type=${HLS_SEGMENT_TYPE:-mpegts}" +# else +# echo "* HLS_SEGMENT_TYPE forced to mpegts, currently installed ffmpge $ffmpeg_version does not support hls_segment_type" +# fi + +# if version_gt ${ffmpeg_version} 3.3.3; then + if [ ! -z "${HLS_ENCRYPT}" ] && [ "${HLS_ENCRYPT}" -ne 0 ]; then + echo "* Will generate encrypted HLS chunks" + hls_args="${hls_args}:hls_enc=1" + else + echo "* Will NOT generate encrypted HLS chunks" + fi +# else +# echo "* HLS_ENCRYPT forced to 0, currently installed ffmpeg $ffmpeg_version does not support hls_enc" +# fi + +webm_chunk_args="" +if [ -n "${WEBM_CHUNK_ARGS}" ]; then + webm_chunk_args+=":${WEBM_CHUNK_ARGS}" +fi +webm_chunk_duration="${WEBM_CHUNK_DURATION:-5000}" +webm_chunk_start_index="${WEBM_CHUNK_START_INDEX:-0}" +webm_chunk_args="${webm_chunk_args}:audio_chunk_duration=${webm_chunk_duration}" +webm_chunk_args="${webm_chunk_args}:chunk_start_index=${webm_chunk_start_index}" + +webm_manifest_input_args=() +webm_manifest_output_args=() +webm_manifest_adaptation_sets="" +wemb_manifest_mpds=() +webm_manifest_next_adaptationset_index=0 +webm_manifest_next_stream_index=0 + +function make_stream() { + local codec_shorthand=$1 + local codec_id=$2 + local codec_options_="$3" + local codec_options=(${codec_options_}) + local bitrates_=$4[@] + local bitrates=("${!bitrates_}") + + if [ -z "${bitrates}" ]; then + return + fi + + echo "* Bitrates for ${codec_shorthand}: ${bitrates[*]}" >&2 + + old_webm_manifest_next_stream_index=${webm_manifest_next_stream_index} + + for bitrate in ${bitrates}; do + parsed_bitrate="$(parse_bitrate "$bitrate")" + output="" + + # HLS + if is_hls_compatible "${codec_shorthand}"; then + # if [ -n "$output" ]; then + # # append to tee output name + # output+="|" + # fi + + hls_m3u8="hls_${codec_shorthand}${parsed_bitrate}.m3u8" + hls_chk="hls_${codec_shorthand}${parsed_bitrate}/c%s.ts" + + mkdir -p "$(dirname "${hls_m3u8}")" "$(dirname "${hls_chk}")" + output+="[f=hls${hls_args}:hls_segment_filename=${hls_chk}]${hls_m3u8}" + + # Add format to HLS index + cat <>hls.m3u8 +#EXT-X-STREAM-INF:PROGRAM-ID=1,AVERAGE-BANDWIDTH=$(echo "${parsed_bitrate}*1.1" | bc),BANDWIDTH=$(echo "${parsed_bitrate} * 1.1" | bc),CODECS="$codec_id" +${hls_m3u8} +EOF + fi + + # WebM-DASH + if is_webm_dash_compatible "${codec_shorthand}"; then + if [ -n "$output" ]; then + # append to tee output name + output+="|" + fi + + dash_prefix="dash_${codec_shorthand}${parsed_bitrate}" + dash_hdr="${dash_prefix}.hdr" + dash_chk="${dash_prefix}_%d.chk" + + rm -f "${dash_hdr}" *.chk + + mkdir -p "$(dirname "${dash_hdr}")" "$(dirname "${dash_chk}")" + + webm_manifest_mpds+=("${dash_hdr}") + extra_args="" + + output+="[f=webm_chunk${webm_chunk_args}${extra_args}:header=${dash_hdr}]${dash_chk}" + + # Temporary file to store bandwidth for final manifest + webm_manifest_input_args+=( + -y -f webm_dash_manifest + -live 1 + -bandwidth "${parsed_bitrate}" + -i "${dash_hdr}" + ) + if [ "${old_webm_manifest_next_stream_index}" = "${webm_manifest_next_stream_index}" ]; then + if [ -n "${webm_manifest_adaptation_sets}" ]; then + webm_manifest_adaptation_sets="${webm_manifest_adaptation_sets%,} " + fi + webm_manifest_adaptation_sets+="id=${webm_manifest_next_adaptationset_index},streams=" + webm_manifest_next_adaptationset_index=$((${webm_manifest_next_adaptationset_index}+1)) + fi + webm_manifest_adaptation_sets+="${webm_manifest_next_stream_index}," + webm_manifest_output_args+=(-map ${webm_manifest_next_stream_index}) + webm_manifest_next_stream_index=$((webm_manifest_next_stream_index+1)) + fi + + ffmpeg_args+=( + "${codec_options[@]}" + -b:a "${parsed_bitrate}" + -map_metadata 0 + -metadata "service_provider=streaminginter.net" + -metadata "service_name=${CHANNEL_NAME:-Livestream}" + -y -f tee + ) + if contains_audio "${codec_shorthand}"; then + ffmpeg_args+=(-map 0:a) + fi + if contains_video "${codec_shorthand}"; then + ffmpeg_args+=(-map 0:v) + fi + ffmpeg_args+=("${output}") + done +} + +make_webm_dash_manifest() { + ffmpeg_args_2=( + -loglevel "${ffmpeg_log_level}" + "${webm_manifest_input_args[@]}" + -c copy + "${webm_manifest_output_args[@]}" + -y -f webm_dash_manifest -live 1 + -adaptation_sets "${webm_manifest_adaptation_sets}" + -chunk_start_index "${webm_chunk_start_index}" + -chunk_duration_ms "${webm_chunk_duration}" + -time_shift_buffer_depth 7200 + -minimum_update_period 7200 + ) + if [ -n "${UTC_TIMING_URL}" ]; then + ffmpeg_args_2+=(-utc_timing_url "${UTC_TIMING_URL}") + fi + ffmpeg_args_2+=(dash.mpd) + + set -x + ffmpeg "${ffmpeg_args_2[@]}" +} + +find -name '*.mpd' -or -name '*.chk' -or -name '*.m3u8' -delete + +echo "#EXTM3U" >hls.m3u8 + +make_stream aogo opus "-c:a libopus -compression_level 10" OUTPUT_OPUS_BITRATES +make_stream aogv vorbis "-c:a libvorbis" OUTPUT_VORBIS_BITRATES +make_stream aalc mp4a.40.2 "-c:a libfdk_aac -cutoff 20000" OUTPUT_AAC_BITRATES +make_stream aahe mp4a.40.5 "-c:a libfdk_aac -profile:a aac_he" OUTPUT_AAC_HE_BITRATES +make_stream aahp mp4a.40.29 "-c:a libfdk_aac -profile:a aac_he_v2" OUTPUT_AAC_HEV2_BITRATES +make_stream ampe mp4a.40.34 "-c:a libmp3lame" OUTPUT_MP3_BITRATES + +# WebM DASH master manifest generation +( + echo "Waiting for DASH manifests to be generated..." + for mpd in "${webm_manifest_mpds[@]}"; do + echo "Waiting for $mpd..." + while [ ! -f "$mpd" ]; do sleep 1; done + done + echo "Generating WebM DASH manifest..." + make_webm_dash_manifest + echo "WebM DASH manifest generated." +) & + +# Delete old WebM chunks regularly +watch -t -n 30 'find -name "*.chk" -mmin 120 -delete' & + +echo "Starting ffmpeg..." +ffmpeg "${ffmpeg_args[@]}" \ No newline at end of file