Skip to content

Commit e6e1dff

Browse files
committed
switch to much simpler/nicer and WAY more efficient "COPY --parents" new hotness
1 parent debded3 commit e6e1dff

2 files changed

Lines changed: 94 additions & 41 deletions

File tree

‎Dockerfile‎

Lines changed: 34 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,18 @@
1-
# copy all our files into a multi-stage build we can COPY from and then throw out:
2-
FROM nginx:alpine AS src
3-
COPY . /src/
1+
# syntax=docker/dockerfile:1.7-labs
2+
# ^ need that first line comment for COPY --parents 2024 goodness
43

5-
# now start the real image setup
64
FROM nginx:alpine
75

86
WORKDIR /usr/share/nginx/html
97

108
RUN rm -f index.html && \
11-
ln -sf /usr/share/nginx/html/default.conf /etc/nginx/conf.d/default.conf &&\
12-
apk add --no-cache rsync
9+
ln -sf /usr/share/nginx/html/default.conf /etc/nginx/conf.d/default.conf
1310

14-
# split COPY . . into many layers so we can pull in parallel -- MUCH faster.
15-
# (and one 5GB layer is brutal — any network hiccup on docker pull restarts from zero)
11+
# split COPY . . into many layers so a single file change only re-pulls one small layer.
12+
# (one 5GB layer is brutal — any network hiccup on docker pull restarts from zero)
1613
#
17-
# uses rsync via --mount=type=bind from throwaway stage 1 (never pushed to registry).
18-
# shell globs are used (not Go filepath.Match) so special chars like - and * just work.
19-
# subdirs are preserved naturally by rsync -a, no special-casing needed.
14+
# COPY --parents preserves directory structure so subdir/file never clobbers top-level file.
15+
# layers are content-hashed so unchanged buckets are skipped on docker pull.
2016
#
2117
# pattern based on 5th char having the most uniform distribution:
2218
# ls |cut -b5 |sort |uniq -c
@@ -25,36 +21,33 @@ RUN rm -f index.html && \
2521
# [aA]=524 [bB]=266 [cC]=346 [dD]=294 [eE]=381 [fF]=167
2622
# [gG]=262 [hH]=207 [iI]=317 [jJ]=164 [kK]=202 [lL]=298
2723
# [mM]=387 [nN]=343 [oO]=378 [pP]=282 [rR]=398 [sS]=529
28-
# [tT]=459 [uU]=188 [qQvVwWxXyYzZ]=402 [0-9]=476 [._-]=233
29-
# short (1-4 char names/dirs) = safety net catchall
24+
# [tT]=459 [uU]=188 [qQvVwWxXyYzZ]=402 [0-9]=476
25+
# short (1-4 char names/dirs) + non-alphanumeric 5th char = safety net
3026

31-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[aA]* ./
32-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[bB]* ./
33-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[cC]* ./
34-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[dD]* ./
35-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[eE]* ./
36-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[fF]* ./
37-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[gG]* ./
38-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[hH]* ./
39-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[iI]* ./
40-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[jJ]* ./
41-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[kK]* ./
42-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[lL]* ./
43-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[mM]* ./
44-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[nN]* ./
45-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[oO]* ./
46-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[pP]* ./
27+
COPY --parents ????[aA]* .
28+
COPY --parents ????[bB]* .
29+
COPY --parents ????[cC]* .
30+
COPY --parents ????[dD]* .
31+
COPY --parents ????[eE]* .
32+
COPY --parents ????[fF]* .
33+
COPY --parents ????[gG]* .
34+
COPY --parents ????[hH]* .
35+
COPY --parents ????[iI]* .
36+
COPY --parents ????[jJ]* .
37+
COPY --parents ????[kK]* .
38+
COPY --parents ????[lL]* .
39+
COPY --parents ????[mM]* .
40+
COPY --parents ????[nN]* .
41+
COPY --parents ????[oO]* .
42+
COPY --parents ????[pP]* .
4743
# skip qQ until later
48-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[rR]* ./
49-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[sS]* ./
50-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[tT]* ./
51-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[uU]* ./
44+
COPY --parents ????[rR]* .
45+
COPY --parents ????[sS]* .
46+
COPY --parents ????[tT]* .
47+
COPY --parents ????[uU]* .
5248
# round up together a bunch of unpopular chars
53-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[qQvVwWxXyYzZ]* ./
54-
# all digits together a decent size layer
55-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[0-9]* ./
56-
# get anything else, eg: - _ * . chars..
57-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[^a-zA-Z0-9]* ./
58-
# grab any rare short 1,2,3,4-char filenames or dirnames
59-
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/???? /mnt/??? /mnt/?? /mnt/? ./ \
60-
2>/dev/null || true
49+
COPY --parents ????[qQvVwWxXyYzZ]* .
50+
# all digits together
51+
COPY --parents ????[0-9]* .
52+
# safety net: short names/dirs and any non-alphanumeric 5th char (eg: - _ . *)
53+
COPY --parents ?[^/] ??[^/] ???[^/] ????[^/] ?/ ??/ ???/ ????/ ????[^a-zA-Z0-9]* .

‎Dockerfile.rsync‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# copy all our files into a multi-stage build we can COPY from and then throw out:
2+
FROM nginx:alpine AS src
3+
COPY . /src/
4+
5+
# now start the real image setup
6+
FROM nginx:alpine
7+
8+
WORKDIR /usr/share/nginx/html
9+
10+
RUN rm -f index.html && \
11+
ln -sf /usr/share/nginx/html/default.conf /etc/nginx/conf.d/default.conf &&\
12+
apk add --no-cache rsync
13+
14+
# split COPY . . into many layers so we can pull in parallel -- MUCH faster.
15+
# (and one 5GB layer is brutal — any network hiccup on docker pull restarts from zero)
16+
#
17+
# uses rsync via --mount=type=bind from throwaway stage 1 (never pushed to registry).
18+
# shell globs are used (not Go filepath.Match) so special chars like - and * just work.
19+
# subdirs are preserved naturally by rsync -a, no special-casing needed.
20+
#
21+
# pattern based on 5th char having the most uniform distribution:
22+
# ls |cut -b5 |sort |uniq -c
23+
#
24+
# counts (approximate, from original analysis):
25+
# [aA]=524 [bB]=266 [cC]=346 [dD]=294 [eE]=381 [fF]=167
26+
# [gG]=262 [hH]=207 [iI]=317 [jJ]=164 [kK]=202 [lL]=298
27+
# [mM]=387 [nN]=343 [oO]=378 [pP]=282 [rR]=398 [sS]=529
28+
# [tT]=459 [uU]=188 [qQvVwWxXyYzZ]=402 [0-9]=476 [._-]=233
29+
# short (1-4 char names/dirs) = safety net catchall
30+
31+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[aA]* ./
32+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[bB]* ./
33+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[cC]* ./
34+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[dD]* ./
35+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[eE]* ./
36+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[fF]* ./
37+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[gG]* ./
38+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[hH]* ./
39+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[iI]* ./
40+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[jJ]* ./
41+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[kK]* ./
42+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[lL]* ./
43+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[mM]* ./
44+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[nN]* ./
45+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[oO]* ./
46+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[pP]* ./
47+
# skip qQ until later
48+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[rR]* ./
49+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[sS]* ./
50+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[tT]* ./
51+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[uU]* ./
52+
# round up together a bunch of unpopular chars
53+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[qQvVwWxXyYzZ]* ./
54+
# all digits together a decent size layer
55+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[0-9]* ./
56+
# get anything else, eg: - _ * . chars..
57+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/????[^a-zA-Z0-9]* ./
58+
# grab any rare short 1,2,3,4-char filenames or dirnames
59+
RUN --mount=type=bind,from=src,source=/src,target=/mnt rsync -a /mnt/???? /mnt/??? /mnt/?? /mnt/? ./ \
60+
2>/dev/null || true

0 commit comments

Comments
 (0)