master.cfg

# -*- python -*-  vi:ft=python:
# kate: indent-mode python; hl python;
# vim:set ts=8 sw=4 sts=4 et:

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory (although the filename
# can be changed with the --basedir option to 'mktap buildbot master').

# It has one job: define a dictionary named BuildmasterConfig. This
# dictionary has a variety of keys to control different aspects of the
# buildmaster. They are documented in docs/config.xhtml .

import os
import subprocess
import sys

from datetime import datetime, timedelta
from functools import partial
from zoneinfo import ZoneInfo

from buildbot.plugins import reporters, schedulers, util
from buildbot import locks
from twisted.python import log
from twisted.internet import defer

import sentry_sdk
from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration

sys.path.append(os.path.dirname(__file__))

# Make sure locals are up to date on reconfig
for k in list(sys.modules):
    if k.split(".")[0] in ["custom"]:
        sys.modules.pop(k)

from custom import MAIN_BRANCH_NAME  # noqa: E402
from custom.auth import set_up_authorization  # noqa: E402
from custom.email_formatter import MESSAGE_FORMATTER  # noqa: E402
from custom.pr_reporter import GitHubPullRequestReporter  # noqa: E402
from custom.discord_reporter import DiscordReporter  # noqa: E402
from custom.pr_testing import (  # noqa: E402
    CustomGitHubEventHandler,
    should_pr_be_tested,
)
from custom.settings import Settings  # noqa: E402
from custom.steps import Git, GitHub  # noqa: E402
from custom.workers import get_workers  # noqa: E402
from custom.schedulers import GitHubPrScheduler # noqa: E402
from custom.release_dashboard import get_release_status_app    # noqa: E402
from custom.builders import (  # noqa: E402
    get_builders,
    STABLE,
    ONLY_MAIN_BRANCH,
)


def set_up_sentry():
    try:
        release_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True)
    except subprocess.SubprocessError:
        log.err('Could not get current SHA for the release!')
        return
    sentry_dsn = settings.get('sentry_dsn', None)
    if sentry_dsn is None:
        log.err('The sentry DSN could not be found in the settings!')
        return
    sentry_sdk.init(dsn=sentry_dsn, release=release_sha,
                    integrations=[SqlalchemyIntegration()])

    def logToSentry(event):
        if not event.get('isError') or 'failure' not in event:
            return

        f = event['failure']
        sentry_sdk.capture_exception((f.type, f.value, f.getTracebackObject()))

    log.addObserver(logToSentry)


settings_path = os.path.join('/etc', 'buildbot', 'settings.yaml')
settings_path = os.environ.get('PYBUILDBOT_SETTINGS_PATH', settings_path)

try:
    settings = Settings.from_file(settings_path)
    set_up_sentry()
except FileNotFoundError:
    log.err(f"WARNING: settings file could not be found at {settings_path}")
    settings = Settings()


WORKERS = get_workers(settings)
WORKERS_BY_NAME = {w.name: w for w in WORKERS}

BUILDERS = get_builders(settings)

AUTH, AUTHZ = set_up_authorization(settings)

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

c["db_url"] = str(settings.db_url)

# configure a janitor which will delete all logs older than 6 months,
# and will run on sundays at noon
c["configurators"] = [
    util.JanitorConfigurator(
        logHorizon=timedelta(days=180),
        hour=12,
        dayOfWeek=6,
    )
]

# Note: these cache values are not currently tuned in any meaningful way.
# Some are taken straight from the buildbot docs at
# https://docs.buildbot.net/4.2.1/manual/configuration/global.html#caches
# and others are just guesses.  For now, they're mostly meant to see if
# there's any appreciable impact on performance or memory usage.
c["caches"] = {
    "Changes": 100,
    "Builds": 500,
    "chdicts": 100,
    "BuildRequests": 100,
    "SourceStamps": 200,
    "ssdicts": 200,
    "objectids": 10,
    "usdicts": 100,
}

# workers are set up in workers.py
c["workers"] = [w.bb_worker for w in WORKERS]

# repo url, buildbot category name, git branch name
git_url = str(settings.git_url)
git_branches = [
    (git_url, MAIN_BRANCH_NAME, "main"),
    (git_url, "3.14", "3.14"),
    (git_url, "3.13", "3.13"),
    (git_url, "3.12", "3.12"),
    (git_url, "3.11", "3.11"),
    (git_url, "3.10", "3.10"),
]

# common Git() and GitHub() keyword arguments
GIT_KWDS = {
    "timeout": 3600,
    # "git clean -fdx": remove all files which are not tracked by Git,
    # ignoring the .gitignore rules (ex: remove also ".o" files).
    "mode": "full",
    "method": "fresh",
}

c["builders"] = []
c["schedulers"] = []

parallel = {w.name: f"-j{w.parallel_tests}" for w in WORKERS if w.parallel_tests}
extra_factory_args = {
    "cstratak-RHEL8-ppc64le": {
        # Increase the timeout on this slow worker
        "timeout_factor": 2,
    },
    "bcannon-wasi": {
        # Increase the timeout on this slow worker
        "timeout_factor": 2,
    },

}

# Build factory args from worker properties
for w in WORKERS:
    if w.exclude_test_resources:
        if w.name not in extra_factory_args:
            extra_factory_args[w.name] = {}
        extra_factory_args[w.name]["exclude_test_resources"] = w.exclude_test_resources

# The following with the worker owners' agreement
cpulock = locks.WorkerLock(
    "cpu",
    maxCountForWorker={
        w.name: w.parallel_builders for w in WORKERS if w.parallel_builders
    },
)


def is_important_file(filename):
    unimportant_prefixes = (
        ".azure-pipelines/",
        ".devcontainer/"
        ".github/",
        ".well-known/",
        "Doc/",
        "InternalDocs/",
        "Misc/",
        ".coveragerc",
        ".editorconfig",
        ".gitattributes",
        ".gitignore",
        ".mailmap",
        "LICENSE"
    )
    unimportant_suffixes = (
        ".txt",
        ".md",
        ".rst",
        ".yml",
        ".yaml",
        "README",
        "ruff.toml",
    )
    if filename.lstrip("\\/").startswith(unimportant_prefixes):
        return False
    return not filename.endswith(unimportant_suffixes)


def is_important_change(change):
    return any(is_important_file(filename) for filename in change.files)


def is_within_time_range(now, start, end):
    if start <= end:
        return start <= now <= end
    else:
        return now >= start or now <= end


def get_delay(now, end):
    today = datetime.today()
    now = datetime.combine(today, now)
    end = datetime.combine(today, end)

    if now > end:
        end += timedelta(days=1)

    difference = end - now
    return difference.total_seconds()


# Avoid a build to be started between start and end time and delay such build
# at end time
def no_builds_between(start, end, *, day_of_week=None, tz=None):
    start = datetime.strptime(start, "%H:%M").time()
    end = datetime.strptime(end, "%H:%M").time()
    def canStartBuild(builder, wfb, request):
        now_dt = datetime.now(tz=tz)
        if day_of_week is not None and now_dt.weekday() != day_of_week:
            return True
        now = now_dt.time()
        if is_within_time_range(now, start, end):
            delay = get_delay(now, end)
            # Adapted from: https://docs.buildbot.net/current/manual/customization.html#canstartbuild-functions
            wfb.worker.quarantine_timeout = delay
            wfb.worker.putInQuarantine()
            # This does not take the worker out of quarantine, it only resets
            # the timeout value to default (restarting the default
            # exponential backoff)
            wfb.worker.resetQuarantine()
            return False
        # Schedule the build now
        return True
    return canStartBuild


github_status_builders = []
release_status_builders = []
mail_status_builders = []

# Regular builders

for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches):
    buildernames = []
    refleakbuildernames = []
    for name, worker_name, buildfactory, stability, tier in BUILDERS:
        if any(
            pattern in name for pattern in ONLY_MAIN_BRANCH
        ) and branchname != MAIN_BRANCH_NAME:
            # Workers known to be broken on older branches: let's focus on
            # supporting these platforms in the main branch.
            continue

        worker = WORKERS_BY_NAME[worker_name]
        if worker.not_branches and branchname in worker.not_branches:
            continue
        if worker.branches and branchname not in worker.branches:
            continue

        buildername = name + " " + branchname
        source = Git(repourl=git_url, branch=git_branch, **GIT_KWDS)
        f = buildfactory(
            source,
            parallel=parallel.get(worker_name),
            branch=branchname,
            **extra_factory_args.get(worker_name, {}),
        )
        tags = [branchname, stability, *getattr(f, "tags", [])]
        if tier:
            tags.append(tier)

        # Only 3.11+ for WebAssembly builds
        if "wasm" in tags:
            # WASM wasn't a supported platform until 3.11.
            if branchname in {"3.10"}:
                continue
            # Tier 3 support is 3.11 & 3.12.
            elif "nondebug" in tags and branchname not in {"3.11", "3.12"}:
                continue
            # Tier 2 support is 3.13+.
            elif "nondebug" not in tags and branchname in {"3.11", "3.12"}:
                continue

        # Only 3.13+ for NoGIL builds
        if 'nogil' in tags and branchname in {"3.10", "3.11", "3.12"}:
            continue

        if 'refleak' in tags:
            refleakbuildernames.append(buildername)
        else:
            buildernames.append(buildername)
        # disable notifications for unstable builders
        # (all these lists are the same now, but we might need to
        # diverge gain later.)
        if stability == STABLE:
            mail_status_builders.append(buildername)
            github_status_builders.append(buildername)
            release_status_builders.append(buildername)

        builder = util.BuilderConfig(
            name=buildername,
            workernames=[worker_name],
            builddir="%s.%s%s"
            % (branchname, worker_name, getattr(f, "buildersuffix", "")),
            factory=f,
            tags=tags,
            locks=[cpulock.access("counting")],
        )

        # This worker runs pyperformance at 12am UTC. If a build is scheduled between
        # 10pm UTC and 2am UTC, it will be delayed to 2am UTC.
        if worker_name == "diegorusso-aarch64-bigmem":
            builder.canStartBuild = no_builds_between("22:00", "2:00")

        # This worker restarts every day at 9am UTC to work around issues stemming from
        # failing bigmem tests trashing disk space and fragmenting RAM. Builds scheduled
        # between 07:20am - 9:20am UTC will be delayed to 9:20am UTC.
        if worker_name == "ambv-bb-win11":
            builder.canStartBuild = no_builds_between("7:20", "9:20")

        # These workers are reprovisioned every Wednesday at 9am PT. Builds
        # scheduled between 8am - 10am PT on Wednesdays will be delayed to
        # 10am PT.
        if worker_name in ("itamaro-win64-srv-22-aws", "itamaro-centos-aws"):
            builder.canStartBuild = no_builds_between(
                "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles")
            )

        c["builders"].append(builder)

    c["schedulers"].append(
        schedulers.SingleBranchScheduler(
            name=branchname,
            change_filter=util.ChangeFilter(branch=git_branch),
            treeStableTimer=30,  # seconds
            builderNames=buildernames,
            fileIsImportant=is_important_change,
        )
    )
    if refleakbuildernames:
        c["schedulers"].append(
            schedulers.SingleBranchScheduler(
                name=branchname + "-refleak",
                change_filter=util.ChangeFilter(branch=git_branch),
                # Wait this many seconds for no commits before starting a build
                # NB: During extremely busy times, this can cause the builders
                # to never actually fire.  The current expectation is that it
                # won't ever actually be that busy, but we need to keep an eye
                # on that.
                treeStableTimer=1 * 60 * 60,  # h * m * s
                builderNames=refleakbuildernames,
            )
        )


# Set up Pull Request builders

stable_pull_request_builders = []
all_pull_request_builders = []

for name, worker_name, buildfactory, stability, tier in BUILDERS:
    buildername = f"{name} PR"

    source = GitHub(repourl=git_url, **GIT_KWDS)

    f = buildfactory(
        source,
        parallel=parallel.get(worker_name),
        # Use the same downstream branch names as the "custom"
        # builder (check what the factories are doing with this
        # parameter for more info).
        branch="3",
        **extra_factory_args.get(worker_name, {}),
    )

    tags = ["PullRequest", stability, *getattr(f, "tags", [])]
    if tier:
        tags.append(tier)

    # Don't use the WASI buildbot meant for 3.11 and 3.12 on PRs;
    # it's tier 3 only and getting it to work on PRs against `main`
    # is too much work.
    if "wasi" in tags and "nondebug" in tags:
        continue

    all_pull_request_builders.append(buildername)
    if stability == STABLE:
        stable_pull_request_builders.append(buildername)

    builder = util.BuilderConfig(
        name=buildername,
        workernames=[worker_name],
        builddir="%s.%s%s"
        % ("pull_request", worker_name, getattr(f, "buildersuffix", "")),
        factory=f,
        tags=tags,
        locks=[cpulock.access("counting")],
    )

    # This worker runs pyperformance at 12am. If a build is scheduled between
    # 10pm and 2am, it will be delayed at 2am.
    if worker_name == "diegorusso-aarch64-bigmem":
        builder.canStartBuild = no_builds_between("22:00", "2:00")

    # These workers are reprovisioned every Wednesday at 9am PT. Builds
    # scheduled between 8am - 10am PT on Wednesdays will be delayed to
    # 10am PT.
    if worker_name in ("itamaro-win64-srv-22-aws", "itamaro-centos-aws"):
        builder.canStartBuild = no_builds_between(
            "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles")
        )

    c["builders"].append(builder)

c["schedulers"].append(
    GitHubPrScheduler(
        name="pull-request-scheduler",
        change_filter=util.ChangeFilter(filter_fn=should_pr_be_tested),
        treeStableTimer=30,  # seconds
        builderNames=all_pull_request_builders,
        stable_builder_names=set(stable_pull_request_builders),
    )
)


# Set up aditional schedulers

c["schedulers"].append(
    schedulers.ForceScheduler(
        name="force",
        builderNames=[builder.name for builder in c["builders"]],
        reason=util.FixedParameter(name="reason", label="reason", default=""),
        codebases=[
            util.CodebaseParameter(
                "",
                label="CPython repository",
                # will generate nothing in the form, but branch, revision, repository,
                # and project are needed by buildbot scheduling system so we
                # need to pass a value ("")
                branch=util.FixedParameter(name="branch", default=""),
                revision=util.FixedParameter(name="revision", default=""),
                repository=util.FixedParameter(name="repository", default=""),
                project=util.FixedParameter(name="project", default=""),
            ),
        ],
    )
)

# 'workerPortnum' defines the TCP port to listen on. This must match the value
# configured into the buildworkers (with their --master option)

c["protocols"] = {"pb": {"port": "tcp:{}".format(settings.worker_port)}}

# 'www' is the configuration for everything accessible via
# http[s]://buildbot.python.org/all/

c["www"] = dict(
    port=f"tcp:{int(settings.web_port)}",
    auth=AUTH,
    authz=AUTHZ,
    change_hook_dialects={
        "github": {
            "class": partial(
                CustomGitHubEventHandler,
                builder_names=all_pull_request_builders,
            ),
            "secret": str(settings.github_change_hook_secret),
            "strict": True,
            "token": settings.github_status_token,
        },
    },
    plugins=dict(waterfall_view={}, console_view={}, grid_view={}),
    avatar_methods=[util.AvatarGitHub(token=settings.github_status_token)],
    ws_ping_interval=30,
)

# 'services' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/reporters/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c["services"] = []


status_email = str(settings.status_email)
if bool(settings.send_mail):
    c["services"].append(
        reporters.MailNotifier(
            generators=[
                reporters.BuildSetStatusGenerator(
                    mode='problem',
                    builders=mail_status_builders,
                    message_formatter=MESSAGE_FORMATTER,
                ),
                reporters.WorkerMissingGenerator(workers='all'),
            ],
            fromaddr=str(settings.from_email),
            relayhost=str(settings.email_relay_host),
            extraRecipients=[status_email],
            sendToInterestedUsers=False,
            extraHeaders={"Reply-To": status_email},
        )
    )

if bool(settings.irc_notice):
    irc_args = dict(
        host=str(settings.irc_host),
        nick=str(settings.irc_nick),
        channels=[dict(channel=str(settings.irc_channel))],
        notify_events=set(
            settings.get(
                'irc_notify_events',
                # 'cancelled' is not logged to avoid spaming IRC when
                # a "pull-request-scheduler" is cancelled
                ['better', 'worse', 'exception']
                )
            ),
        useColors=True,
        )
    password = settings.get('irc_password', None)
    if password:
        irc_args['useSASL'] = True
        irc_args['password'] = password
    c["services"].append(reporters.IRC(**irc_args))

c["services"].append(
    reporters.GitHubStatusPush(
        str(settings.github_status_token),
        generators=[
            reporters.BuildStartEndStatusGenerator(
                builders=github_status_builders + all_pull_request_builders,
            ),
        ],
        verbose=bool(settings.verbosity),
    )
)

start_formatter = reporters.MessageFormatterRenderable('Build started.')
end_formatter = reporters.MessageFormatterRenderable('Build done.')
pending_formatter = reporters.MessageFormatterRenderable('Build pending.')
c["services"].append(
    GitHubPullRequestReporter(
        str(settings.github_status_token),
        generators=[
            reporters.BuildRequestGenerator(formatter=pending_formatter),
            reporters.BuildStartEndStatusGenerator(
                builders=github_status_builders,
                start_formatter=start_formatter,
                end_formatter=end_formatter,
            ),
        ],
        verbose=bool(settings.verbosity),
    )
)

c["services"].append(
    DiscordReporter(
        str(settings.discord_webhook),
        generators=[
            reporters.BuildRequestGenerator(formatter=pending_formatter),
            reporters.BuildStartEndStatusGenerator(
                builders=github_status_builders,
                start_formatter=start_formatter,
                end_formatter=end_formatter,
            ),
        ],
        verbose=bool(settings.verbosity),
    )
)


# if you set 'manhole', you can telnet into the buildmaster and get an
# interactive python shell, which may be useful for debugging buildbot
# internals. It is probably only useful for buildbot developers.
# from buildbot.master import Manhole
# c['manhole'] = Manhole(9999, "admin", "oneddens")

# the 'title' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'titleURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.

c["title"] = "Python"
c["titleURL"] = "https://www.python.org/"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.Waterfall page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c["buildbotURL"] = str(settings.buildbot_url)

# disable sending of 'buildbotNetUsageData' for now, to improve startup time
c["buildbotNetUsageData"] = None

c['change_source'] = []

c['www']['plugins']['wsgi_dashboards'] = [
    {
        'name': 'release_status',
        'caption': 'Release Status',
        'app': get_release_status_app(
            release_status_builders,
            test_result_dir='/data/www/buildbot/test-results/'),
        'order': 2,
        'icon': 'rocket'
    }
]