Change from python webserver to abom-http

Author: plengauer

.github/workflows/test.yaml

@@ -10,11 +10,21 @@ jobs:
   test-clean-install:
     needs: build
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        mode: ['cache', 'proxy']
     steps:
+      - run: sudo apt-get update && sudo apt-get install -y debconf-utils
+      - run: echo gitaptly gitaptly/MODE string ${{ matrix.mode }} | sudo debconf-set-selections
+      - run: curl https://api.github.com/repos/plengauer/abom-http/releases/latest | jq -r '.assets[].browser_download_url' | xargs wget -O ./abomhttp.deb && sudo apt-get install ./abomhttp.deb
       - uses: actions/download-artifact@v4
         with:
           name: package.deb
       - run: sudo apt-get -y install ./package.deb
+      - run: cat /var/lib/gitaptly/dists/stable/main/binary-all/Packages && find /var/lib/gitaptly
+      - run: ps -ef | grep ncat
+      - run: time wget http://127.0.0.1:8000/dists/stable/main/binary-all/Packages
+      - run: time wget http://127.0.0.1:8000/pool/main/plengauer/Utilities/auto_management_1.3.3.deb
       - run: echo "deb [arch=all] http://127.0.0.1:8000/ stable main" | sudo tee /etc/apt/sources.list.d/example.list
       - run: sudo apt-get update --allow-insecure-repositories
       - run: sudo apt-get -y install --allow-unauthenticated auto-management
@@ -23,8 +33,13 @@ jobs:
   test-token:
     needs: build
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        mode: ['cache', 'proxy']
     steps:
-      - run: sudo apt-get install -y debconf-utils
+      - run: sudo apt-get update && sudo apt-get install -y debconf-utils
+      - run: echo gitaptly gitaptly/MODE string ${{ matrix.mode }} | sudo debconf-set-selections
+      - run: curl https://api.github.com/repos/plengauer/abom-http/releases/latest | jq -r '.assets[].browser_download_url' | xargs wget -O ./abomhttp.deb && sudo apt-get install ./abomhttp.deb
       - run: echo gitaptly gitaptly/GITHUB_API_TOKEN string $GITHUB_API_TOKEN | sudo debconf-set-selections
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -40,7 +55,13 @@ jobs:
   test-observed:
     needs: build
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        mode: ['cache', 'proxy']
     steps:
+      - run: sudo apt-get update && sudo apt-get install -y debconf-utils
+      - run: echo gitaptly gitaptly/MODE string ${{ matrix.mode }} | sudo debconf-set-selections
+      - run: curl https://api.github.com/repos/plengauer/abom-http/releases/latest | jq -r '.assets[].browser_download_url' | xargs wget -O ./abomhttp.deb && sudo apt-get install ./abomhttp.deb
       - run: wget -O - https://raw.githubusercontent.com/plengauer/opentelemetry-bash/main/INSTALL.sh | sh -E
       - uses: actions/download-artifact@v4
         with:
@@ -54,14 +75,31 @@ jobs:
   test-upgrade:
     needs: build
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        mode: ['cache', 'proxy']
     steps:
-      - run: wget https://github.com/plengauer/GitAptly/releases/download/v2.0.0/gitaptly_2.0.0.deb
-      - run: sudo apt-get -y install ./gitaptly_2.0.0.deb
+      - run: sudo apt-get update && sudo apt-get install -y debconf-utils
+      - run: echo gitaptly gitaptly/MODE string ${{ matrix.mode }} | sudo debconf-set-selections
+      - run: curl https://api.github.com/repos/plengauer/abom-http/releases/latest | jq -r '.assets[].browser_download_url' | xargs wget -O ./abomhttp.deb && sudo apt-get install ./abomhttp.deb
+      # - run: wget https://github.com/plengauer/GitAptly/releases/download/v2.0.0/gitaptly_2.0.0.deb
+      # - run: sudo apt-get -y install ./gitaptly_2.0.0.deb
+      # - run: cat /var/lib/gitaptly/dists/stable/main/binary-all/Packages && find /var/lib/gitaptly && ls -la /var/lib/gitaptly/*/main/*/* || true
+      - run: wget https://github.com/plengauer/GitAptly/releases/download/v3.14.0/gitaptly_3.14.0.deb
+      - run: sudo apt-get -y install ./gitaptly_3.14.0.deb
+      - run: cat /var/lib/gitaptly/dists/stable/main/binary-all/Packages && find /var/lib/gitaptly && ls -la /var/lib/gitaptly/*/main/*/* || true
       - uses: actions/download-artifact@v4
         with:
           name: package.deb
       - run: sudo apt-get -y install ./package.deb
+      - run: cat /var/lib/gitaptly/dists/stable/main/binary-all/Packages && find /var/lib/gitaptly && ls -la /var/lib/gitaptly/*/main/*/* || true
       - run: echo "deb [arch=all] http://127.0.0.1:8000/ stable main" | sudo tee /etc/apt/sources.list.d/example.list
       - run: sudo apt-get update --allow-insecure-repositories
       - run: sudo apt-get -y install --allow-unauthenticated auto-management
       - run: sudo apt-get -y remove gitaptly
+
+  test-all:
+    needs: [ test-clean-install, test-token, test-observed, test-upgrade ]
+    runs-on: ubuntu-latest
+    steps:
+      - run: exit 0

VERSION

@@ -1 +1 @@
-3.15.0
+4.0.0

package/DEBIAN/control

@@ -1,7 +1,7 @@
 Package: gitaptly
 Version: __VERSION__
 Architecture: all
-Depends: bash, debconf, coreutils, util-linux, findutils, jq, cron, gzip, dpkg-dev, curl, wget, parallel, python3, python3-pip, python3-venv
+Depends: bash, debconf, coreutils, util-linux, findutils, jq, cron, gzip, dpkg-dev, curl, wget, parallel, abom-http
 Suggests: opentelemetry-shell (>= 4.4.2)
 Priority: extra
 Maintainer: Philipp Lengauer <[email protected]>

package/DEBIAN/postinst

@@ -7,6 +7,8 @@ elif [ "$1" != 'configure' ] && [ "$1" != 'reconfigure' ]; then
 fi
 
 rm -rf /opt/gitaptly/* || true
+rm -rf /var/lib/gitaptly/cgi-bin && sed -i "s/cgi-bin/pool/" /var/lib/gitaptly/dists/stable/main/binary-all/Packages || true # legacy
+
 
 . /usr/share/debconf/confmodule
 db_get gitaptly/PORT
@@ -31,21 +33,14 @@ db_stop
 
 source /opt/gitaptly/env
 
-python3 -m venv /opt/gitaptly/
-source /opt/gitaptly/bin/activate
-if [ -n "$OTLP_TRACES_ENDPOINT" ] || [ -n "$OTLP_METRICS_ENDPOINT" ] || [ -n "$OTLP_LOGS_ENDPOINT" ]; then
-  pip3 install opentelemetry-distro opentelemetry-exporter-otlp
-  pip3 install opentelemetry-instrumentation opentelemetry-instrumentation-system-metrics
-  opentelemetry-bootstrap -a install
-fi
-deactivate
-
+touch /var/lib/gitaptly/pool
 if [ "$MODE" = "cache" ]; then
-  find /var/lib/gitaptly/cgi-bin/main/*/*/ -type l -exec rm {} '+' || true
+  if ! [ -d /var/lib/gitaptly/pool ]; then rm -rf /var/lib/gitaptly/pool && mkdir -p /var/lib/gitaptly/pool; fi
 elif [ "$MODE" = "proxy" ]; then
-  find /var/lib/gitaptly/pool/main/*/*/ -type f -exec rm {} '+' || true
+  rm -rf /var/lib/gitaptly/pool || true
+  ln --symbolic /usr/share/gitaptly/serve /var/lib/gitaptly/pool
 fi
-gitaptly_update
+bash /usr/share/gitaptly/update
 
 systemctl daemon-reload
 systemctl enable gitaptly

package/etc/cron.d/gitaptly

@@ -1,2 +1,2 @@
-0 * * * * root gitaptly_update
+0 * * * * root /usr/share/gitaptly/update
 

package/etc/systemd/system/gitaptly.service

@@ -6,8 +6,7 @@ ExitType=main
 Restart=always
 RestartSec=1
 StartLimitBurst=10
-User=root
-WorkingDirectory=/opt/gitaptly
-ExecStart=gitaptly_run_ftp
+User=nobody
+ExecStart=/usr/share/gitaptly/run_ftp
 [Install]
 WantedBy=multi-user.target

package/usr/bin/gitaptly_reset

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 directory=$1
 
 cd $directory

package/usr/bin/gitaptly_serve

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -e
+parallel rm -rf ::: /var/lib/gitaptly/dists/stable/Release /var/lib/gitaptly/dists/stable/main/binary-all/Packages.gz /var/lib/gitaptly/dists/stable/main/binary-all/Packages /var/lib/gitaptly/pool/main/*

package/usr/bin/gitaptly_create_release package/usr/share/gitaptly/create_release

@@ -1,16 +1,17 @@
 #!/bin/bash -e
 source /opt/gitaptly/env
-source /opt/gitaptly/bin/activate
-cmd=(python3 -m http.server --cgi --directory /var/lib/gitaptly $PORT)
 if [ -n "$OTLP_TRACES_ENDPOINT" ] || [ -n "$OTLP_METRICS_ENDPOINT" ]; then
-  export OTEL_SERVICE_NAME="GitAptly"
+  export OTEL_SERVICE_NAME=GitAptly
+  export OTEL_LOGS_EXPORTER=otlp
   export OTEL_TRACES_EXPORTER=otlp
   export OTEL_METRICS_EXPORTER=otlp
   export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
   export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="$OTLP_TRACES_ENDPOINT"
-  export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="$OTLP_METRICS_ENDPOINT"
   export OTEL_EXPORTER_OTLP_TRACES_HEADERS=authorization=$(echo "$OTLP_TRACES_HEADER" | jq -Rr @uri)
+  export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="$OTLP_METRICS_ENDPOINT"
   export OTEL_EXPORTER_OTLP_METRICS_HEADERS=authorization=$(echo "$OTLP_METRICS_HEADER" | jq -Rr @uri)
-  cmd=(opentelemetry-instrument "${cmd[@]}")
+  export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT="$OTLP_LOGS_ENDPOINT"
+  export OTEL_EXPORTER_OTLP_LOGS_HEADERS=authorization=$(echo "$OTLP_LOGS_HEADER" | jq -Rr @uri)
+  export OTEL_SHELL_EXPERIMENTAL_OBSERVE_PIPES=TRUE
 fi
-exec "${cmd[@]}"
+abomhttp "$PORT" /var/lib/gitaptly

package/usr/share/gitaptly/reset

@@ -1,15 +1,17 @@
-#!/bin/bash -e
+#!/bin/bash
+set -e
 source /opt/gitaptly/env
 if [ -n "$GITHUB_API_TOKEN" ]; then
   auth_header="Authorization: Bearer $GITHUB_API_TOKEN"
 else
   auth_header='foo: bar'
 fi
 per_page=100
-url="https://api.github.com/repos/$1/releases?per_page=$per_page"
-curl --no-progress-meter --fail --retry 16 --retry-all-errors --head -H "$auth_header" "$url" \
+host=api.github.com
+path="/repos/$1/releases?per_page=$per_page"
+printf "HEAD $path HTTP/1.1\r\nConnection: close\r\nUser-Agent: gitaptly\r\nHost: $host\r\n$auth_header\r\n\r\n" | ncat --ssl --no-shutdown "$host" 443 \
   | grep '^link: ' | cut -d ' '  -f 2- | tr -d ' <>' | tr ',' '\n' \
   | grep 'rel="last"' | cut -d ';' -f1 | cut -d '?' -f 2- | tr '&' '\n' \
   | grep '^page=' | cut -d = -f 2 \
-  | xargs seq 1 | xargs parallel -q curl --no-progress-meter --fail --retry 16 --retry-all-errors -H "$auth_header" "$url"\&page={} ::: \
+  | xargs seq 1 | xargs parallel -q curl --no-progress-meter --fail --retry 16 --retry-all-errors -H "$auth_header" https://"$host""$path"\&page={} ::: \
   | jq '.[] | .assets[] | .browser_download_url' -r | (grep '.deb$' || true)

package/usr/bin/gitaptly_run_ftp package/usr/share/gitaptly/run_ftp

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+method="$1"
+path="$2"
+
+if [ "$method" != GET ]; then
+  printf 'HTTP/1.1 405 Method Forbidden\r\n\r\n'
+  exit 0
+fi
+
+file=$(echo "$path" | rev | cut -d/ -f1 | rev)
+repo=$(echo "$path" | rev | cut -d/ -f2 | rev)
+owner=$(echo "$path" | rev | cut -d/ -f3 | rev)
+
+# if [ -f /usr/bin/opentelemetry_shell.sh ]; then
+#   span_handle="$(otel_span_current)"
+#   otel_span_attribute $span_handle http.route=/$owner/$repo
+#   otel_span_attribute $span_handle github.repository.owner=$owner
+#   otel_span_attribute $span_handle github.repository.name=$repo
+#   otel_span_attribute $span_handle github.repository.file=$file
+#   metric_handle=$(otel_metric_create "gitaptly.downloads")
+#   otel_metric_attribute $metric_handle github.repository.owner=$owner
+#   otel_metric_attribute $metric_handle github.repository.repo=$repo
+#   otel_metric_attribute $metric_handle github.repository.file=$file
+#   otel_metric_add $metric_handle 1
+# fi
+
+printf 'HTTP/1.1 200 OK\r\n'
+printf 'content-encoding: identity\r\n'
+printf 'content-type: application/vnd.debian.binary-package\r\n'
+printf '\r\n'
+bash /usr/share/gitaptly/scan $owner/$repo | grep "$file"'$' | head --lines=1 | xargs wget -O -
+
+# printf 'HTTP/1.1 308 Permanent Redirect\r\n'
+# printf 'location: ' && bash /usr/share/gitaptly/scan $owner/$repo | grep "$file"'$' | head --lines=1 && printf '\r\n'
+# printf '\r\n'

package/usr/bin/gitaptly_scan package/usr/share/gitaptly/scan

@@ -1,4 +1,5 @@
-#!/bin/bash -e
+#!/bin/bash
+set -e
 source /opt/gitaptly/env
 if [ -f /usr/bin/opentelemetry_shell.sh ]; then
   export OTEL_SERVICE_NAME=GitAptly
@@ -17,36 +18,33 @@ if [ -f /usr/bin/opentelemetry_shell.sh ]; then
 fi
 cd /var/lib/gitaptly
 
-# TODO handle if one mode is switched to the other (packages and evrything needs to be cleared and fully re-initialized)
-# TODO clean if deb packages are remove
-# TODO better recovery in proxy mode, check if Packages is there, and if not, redo a full scan
-
-cat /etc/gitaptly.conf | xargs -r -I {} mkdir -p pool/main/{}
 if [ "$MODE" = 'cache' ]; then
-  cat /etc/gitaptly.conf | sort -u | sort -R | parallel -q bash -c 'gitaptly_scan {} | xargs -r wget -nc -P /pool/main/{}/'
+  cat /etc/gitaptly.conf | xargs -r -I {} mkdir -p pool/main/{}
+  cat /etc/gitaptly.conf | sort -u | sort -R | parallel -q bash -c 'bash /usr/share/gitaptly/scan {} | xargs -r wget -nc -P pool/main/{}/'
   dpkg-scanpackages --multiversion pool/ > dists/stable/main/binary-all/Packages
 elif [ "$MODE" = 'proxy' ]; then
-  gitaptly_proxy_process_url() {
+  indexify() {
     set -e
     url="$*"
     owner="$(echo "$url" | cut -d "/" -f 4)"
     repo="$(echo "$url" | cut -d "/" -f 5)"
     file="$(echo "$url" | rev | cut -d "/" -f 1 | rev)"
-    if [ -f cgi-bin/main/"$owner"/"$repo"/"$file" ]; then return 0; fi
-    wget -nc -O pool/main/"$owner"/"$repo"/"$file" "$url"
-    dpkg-scanpackages --multiversion pool/main/$owner/$repo/$file \
-      | sed "s/Filename: .*/Filename: cgi-bin\/main\/$owner\/$repo\/$file/" \
+    temporary="$(mktemp -d)"
+    wget -nc -O "$temporary"/package.deb "$url"
+    dpkg-scanpackages --multiversion "$temporary"/package.deb \
+      | sed "s/Filename: .*/Filename: pool\/main\/$owner\/$repo\/$file/" \
       | flock dists/stable/main/binary-all/Packages tee -a dists/stable/main/binary-all/Packages > /dev/null
-    rm pool/main/"$owner"/"$repo"/"$file"
-    ln --symbolic /usr/bin/gitaptly_serve cgi-bin/main/"$owner"/"$repo"/"$file"
+    rm -rf "$temporary"
   }
-  export -f gitaptly_proxy_process_url
+  export -f indexify
   touch dists/stable/main/binary-all/Packages
-  cat /etc/gitaptly.conf | xargs -r -I {} mkdir -p cgi-bin/main/{}
-  cat /etc/gitaptly.conf | sort -u | sort -R | parallel gitaptly_scan | parallel gitaptly_proxy_process_url
+  index="$(mktemp)"
+  cat dists/stable/main/binary-all/Packages | grep '^Filename: ' | cut -d ' ' -f 2- | cut -d / -f 3- | awk -F'/' '{print "^https://.*/"$1"/"$2"/.*/"$3"$"}' > "$index"
+  cat /etc/gitaptly.conf | sort -u | sort -R | parallel bash /usr/share/gitaptly/scan | grep -v -f "$index" | parallel indexify
+  rm "$index"
 else
   exit 1
 fi
 
 gzip -9 < dists/stable/main/binary-all/Packages > dists/stable/main/binary-all/Packages.gz
-gitaptly_create_release dists/stable > dists/stable/Release
+bash /usr/share/gitaptly/create_release dists/stable > dists/stable/Release