rpmrepo/src/ctl/push.py at 19104b9fe6c2f4fea9188a0dccbb01b953d81cc2 · osbuild/rpmrepo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
"""rpmrepo - Push RPM Repository

This module implements the functions that push local RPM repository
snapshots to configured remote storage.
"""

# pylint: disable=duplicate-code,invalid-name,too-few-public-methods

import contextlib
import os

import boto3


class Push(contextlib.AbstractContextManager):
    """Push RPM repository"""

    def __init__(self, cache):
        self._cache = cache
        self._path_conf = os.path.join(cache, "conf")
        self._path_data = os.path.join(cache, "index/data")
        self._path_snapshot = os.path.join(cache, "index/snapshot")

    def __exit__(self, exc_type, exc_value, exc_tb):
        pass

    def push_data_s3(self, storage, platform_id):
        """Push data to S3"""

        assert os.access(os.path.join(self._path_conf, "index.ok"), os.R_OK)
        assert storage in ["public", "rhvpn"]

        s3c = boto3.client("s3")

        n_total = 0
        for _, _, entries in os.walk(self._path_data):
            for entry in entries:
                n_total += 1

        i_total = 0
        for level, _, entries in os.walk(self._path_data):
            levelpath = os.path.relpath(level, self._path_data)
            if levelpath == ".":
                path = platform_id
            else:
                path = os.path.join(platform_id, levelpath)

            for entry in entries:
                i_total += 1
                key = f"data/{storage}/{path}/{entry}"

                try:
                    s3c.copy_object(
                        Bucket="rpmrepo-storage",
                        Key=key,
                        CopySource={"Bucket": "rpmrepo-storage", "Key": key},
                        MetadataDirective="COPY",
                    )
                    print(f"[{i_total}/{n_total}] '{key}' (exists, touched)")
                    continue
                except s3c.exceptions.ClientError as e:
                    if e.response["Error"]["Code"] not in ("404", "NoSuchKey"):
                        raise

                print(f"[{i_total}/{n_total}] '{key}'")

                with open(os.path.join(level, entry), "rb") as filp:
                    s3c.upload_fileobj(
                        filp,
                        "rpmrepo-storage",
                        key,
                    )

    def push_snapshot_s3(self, snapshot_id, snapshot_suffix):
        """Push snapshot to S3"""

        assert os.access(os.path.join(self._path_conf, "index.ok"), os.R_OK)

        s3c = boto3.client("s3")

        n_total = 0
        for _, _, entries in os.walk(self._path_snapshot):
            for entry in entries:
                n_total += 1

        i_total = 0
        for level, _subdirs, entries in os.walk(self._path_snapshot):
            levelpath = os.path.relpath(level, self._path_snapshot)
            if levelpath == ".":
                path = os.path.join(snapshot_id + snapshot_suffix)
            else:
                path = os.path.join(snapshot_id + snapshot_suffix, levelpath)

            for entry in entries:
                i_total += 1

                with open(os.path.join(level, entry), "rb") as filp:
                    checksum = filp.read().decode()

                print(f"[{i_total}/{n_total}] '{path}/{entry}' -> {checksum}")

                s3c.put_object(
                    Body=b"",
                    Bucket="rpmrepo-storage",
                    Key=f"data/ref/{path}/{entry}",
                    Metadata={"rpmrepo-checksum": checksum},
                )

        s3c.put_object(
            Body=b"",
            Bucket="rpmrepo-storage",
            Key=f"data/thread/{snapshot_id}/{snapshot_id}{snapshot_suffix}",
        )