Eli/static ffmpeg (#721)

* downloader: handle non-archived binaries

* manifest: add ffmpeg

* ci: don't try and push docker images on forks

* Add ffprobe to manifest

* Dockerfile: Do not install ffmpeg as runtime dependency.

* Add ffmpeg apt install back in

* Update catalyst (#724)

* [AUTO-COMMIT] Update `manifest.yaml` (#722)

Files changed:
M	manifest.yaml

Co-authored-by: livepeer-docker <[email protected]>

* go.mod: Bump `testcontainers` package version

* Revert "go.mod: Bump `testcontainers` package version"

This reverts commit a7b570d.

* leave app versions unchanged

* Dockerfile: Use a different ppa for installing ffmpeg

---------

Co-authored-by: Eli Mallon <[email protected]>
Co-authored-by: hjpotter92 <[email protected]>
Co-authored-by: livepeer-robot <[email protected]>
Co-authored-by: livepeer-docker <[email protected]>

Author: 5 people

.github/workflows/build.yaml

@@ -112,6 +112,7 @@ jobs:
 
   upload:
     name: Upload artifacts to google bucket
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     permissions:
       contents: "read"
       id-token: "write"

.github/workflows/catalyst.yaml

@@ -74,11 +74,13 @@ jobs:
 
       - name: Login to DockerHub
         uses: docker/login-action@v3
+        if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
         with:
           username: ${{ secrets.CI_DOCKERHUB_USERNAME }}
           password: ${{ secrets.CI_DOCKERHUB_TOKEN }}
 
       - name: Log in to the Container registry
+        if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
@@ -108,7 +110,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           platforms: ${{ matrix.targets.platforms }}
-          push: true
+          push: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
           target: ${{ matrix.docker_images.target }}
           build-args: |
             GIT_VERSION=${{ github.ref_type == 'tag' && github.ref_name || github.event.pull_request.head.sha || github.sha }}

Dockerfile

@@ -44,10 +44,11 @@ FROM	catalyst-${BUILD_TARGET}-build	as	catalyst-build
 FROM	node:18.14.0 as node-build
 ARG	LIVEPEER_W3_VERSION=v0.2.2
 WORKDIR /app
-RUN	git clone --depth 1 --branch ${LIVEPEER_W3_VERSION} https://github.com/livepeer/go-tools.git
-RUN	npm install --prefix /app/go-tools/w3
+
 # chown needed to make everything owned by one user for userspace podman execution
-RUN	chown -R root:root /app/go-tools/w3
+RUN	git clone --depth 1 --branch ${LIVEPEER_W3_VERSION} https://github.com/livepeer/go-tools.git \
+	&& npm install --prefix /app/go-tools/w3 \
+	&& chown -R root:root /app/go-tools/w3
 
 FROM	rust:1.73.0 as rust-build
 RUN	cargo install --version 0.6.2 c2patool
@@ -60,8 +61,10 @@ LABEL	maintainer="Amritanshu Varshney <[email protected]>"
 
 ARG	BUILD_TARGET
 
-RUN	apt update && apt install -yqq wget
-RUN	wget -O - https://deb.nodesource.com/setup_18.x | bash
+RUN	apt update && apt install -yqq wget software-properties-common \
+	&& wget -O - https://deb.nodesource.com/setup_18.x | bash \
+	&& add-apt-repository -y ppa:ubuntuhandbook1/ffmpeg6
+
 RUN	apt update && apt install -yqq \
 	curl \
 	ca-certificates \
@@ -74,12 +77,14 @@ RUN	apt update && apt install -yqq \
 	&& rm -rf /var/lib/apt/lists/*
 
 # Most of ./scripts is for livepeer-in-a-box except livepeer-vmagent, which is used in production in Catalyst proper
-ADD ./scripts/livepeer-vmagent /usr/local/bin
+ADD	./scripts/livepeer-vmagent	/usr/local/bin
+
 COPY --from=catalyst-build	/opt/bin/		/usr/local/bin/
 COPY --from=rust-build 		/usr/local/cargo/bin/c2patool /bin/
 COPY --from=node-build		/app/go-tools/w3	/opt/local/lib/livepeer-w3
-RUN	ln -s /opt/local/lib/livepeer-w3/livepeer-w3.js /usr/local/bin/livepeer-w3 && \
-    	npm install -g ipfs-car
+
+RUN	ln -s /opt/local/lib/livepeer-w3/livepeer-w3.js /usr/local/bin/livepeer-w3 \
+	&& npm install -g ipfs-car
 
 EXPOSE	1935	4242	8080	8889/udp
 
@@ -100,27 +105,27 @@ RUN	apt update && apt install -yqq \
 	coturn \
 	&& rm -rf /var/lib/apt/lists/*
 
-RUN curl -L -O https://binaries.cockroachdb.com/cockroach-v23.1.5.linux-$TARGETARCH.tgz \
+RUN	curl -L -O https://binaries.cockroachdb.com/cockroach-v23.1.5.linux-$TARGETARCH.tgz \
 	&& tar xzvf cockroach-v23.1.5.linux-$TARGETARCH.tgz \
 	&& mv cockroach-v23.1.5.linux-$TARGETARCH/cockroach /usr/bin/cockroach \
 	&& rm -rf cockroach-v23.1.5.linux-$TARGETARCH.tgz cockroach-v23.1.5.linux-$TARGETARCH \
 	&& cockroach --version
 
-RUN curl -o /usr/bin/minio https://dl.min.io/server/minio/release/linux-$TARGETARCH/minio \
+RUN	curl -o /usr/bin/minio https://dl.min.io/server/minio/release/linux-$TARGETARCH/minio \
 	&& curl -o /usr/bin/mc https://dl.min.io/client/mc/release/linux-$TARGETARCH/mc \
 	&& chmod +x /usr/bin/minio /usr/bin/mc \
 	&& minio --version \
 	&& mc --version
 
-ADD ./scripts /usr/local/bin
-ADD ./config/full-stack.json /etc/livepeer/full-stack.json
+COPY --link	./scripts /usr/local/bin
+COPY --link	./config/full-stack.json /etc/livepeer/full-stack.json
 
-ENV CATALYST_DOWNLOADER_PATH=/usr/local/bin \
+ENV	CATALYST_DOWNLOADER_PATH=/usr/local/bin \
 	CATALYST_DOWNLOADER_MANIFEST=/etc/livepeer/manifest.yaml \
 	CATALYST_DOWNLOADER_UPDATE_MANIFEST=true \
 	COCKROACH_DB_SNAPSHOT=https://github.com/iameli-streams/livepeer-in-a-box-database-snapshots/raw/2eb77195f64f22abf3f0de39e6f6930b82a4c098/livepeer-studio-bootstrap.tar.gz
 
-RUN mkdir /data
+RUN	mkdir /data
 
 CMD	["/usr/local/bin/catalyst", "--", "/usr/local/bin/MistController", "-c", "/etc/livepeer/full-stack.json"]
 

cmd/downloader/downloader/downloader.go

@@ -78,18 +78,24 @@ func DownloadService(flags types.CliFlags, manifest *types.BoxManifest, service
 	}
 
 	glog.Infof("downloaded %s. Getting ready for extraction!", projectInfo.ArchiveFileName)
-	if projectInfo.Platform == "windows" {
+	if strings.HasSuffix(projectInfo.ArchiveFileName, ".zip") {
 		glog.V(7).Info("extracting zip archive!")
 		err = ExtractZipArchive(archivePath, downloadPath, service)
 		if err != nil {
 			return err
 		}
-	} else {
+	} else if strings.HasSuffix(projectInfo.ArchiveFileName, ".tar.gz") {
 		glog.V(7).Infof("extracting tarball archive!")
 		err = ExtractTarGzipArchive(archivePath, downloadPath, service)
 		if err != nil {
 			return err
 		}
+	} else {
+		glog.V(7).Infof("moving %s to %s!", archivePath, downloadPath)
+		err = MoveBinaryIntoPlace(archivePath, downloadPath, service)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -130,6 +136,20 @@ func ExtractZipArchive(archiveFile, extractPath string, service *types.Service)
 	return nil
 }
 
+// no gzip, no anything, just put it there!
+func MoveBinaryIntoPlace(archiveFile, extractPath string, service *types.Service) error {
+	var outputPath string
+	if len(service.ArchivePath) > 0 {
+		outputPath = filepath.Join(extractPath, service.ArchivePath)
+	}
+	if len(service.OutputPath) > 0 {
+		outputPath = filepath.Join(extractPath, service.OutputPath)
+	}
+	os.Rename(archiveFile, outputPath)
+	os.Chmod(outputPath, 0755)
+	return nil
+}
+
 // ExtractTarGzipArchive processes a tarball file and extracts a
 // single file from the service definition.
 func ExtractTarGzipArchive(archiveFile, extractPath string, service *types.Service) error {