mirror of
https://github.com/JulienMalka/snowfield.git
synced 2025-04-01 01:30:53 +02:00
chore: remove unused python source code
This commit is contained in:
parent
ab88939664
commit
9f20d8a79c
GPG key ID:
6FC74C847011FD83
3 changed files with 0 additions and 775 deletions
|
|
@ -1,437 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import json
|
|
||||||
import multiprocessing
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import uuid
|
|
||||||
from collections import defaultdict
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Generator, List
|
|
||||||
|
|
||||||
from buildbot.plugins import steps, util
|
|
||||||
from buildbot.process import buildstep, logobserver
|
|
||||||
from buildbot.process.properties import Properties
|
|
||||||
from buildbot.process.results import ALL_RESULTS, statusToString
|
|
||||||
from buildbot.steps.trigger import Trigger
|
|
||||||
from twisted.internet import defer
|
|
||||||
from buildbot.steps.source.github import GitHub
|
|
||||||
from buildbot.process.results import FAILURE
|
|
||||||
from buildbot.steps.master import SetProperty
|
|
||||||
|
|
||||||
def failure(step):
|
|
||||||
return (step.getProperty("GitFailed")=="failed")
|
|
||||||
|
|
||||||
class BuildTrigger(Trigger):
|
|
||||||
"""
|
|
||||||
Dynamic trigger that creates a build for every attribute.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, scheduler: str, jobs: list[dict[str, str]], **kwargs):
|
|
||||||
if "name" not in kwargs:
|
|
||||||
kwargs["name"] = "trigger"
|
|
||||||
self.jobs = jobs
|
|
||||||
self.config = None
|
|
||||||
Trigger.__init__(
|
|
||||||
self,
|
|
||||||
waitForFinish=True,
|
|
||||||
schedulerNames=[scheduler],
|
|
||||||
#haltOnFailure=True,
|
|
||||||
flunkOnFailure=False,
|
|
||||||
sourceStamps=[],
|
|
||||||
alwaysUseLatest=False,
|
|
||||||
updateSourceStamp=False,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def createTriggerProperties(self, props):
|
|
||||||
return props
|
|
||||||
|
|
||||||
def getSchedulersAndProperties(self):
|
|
||||||
build_props = self.build.getProperties()
|
|
||||||
sch = self.schedulerNames[0]
|
|
||||||
triggered_schedulers = []
|
|
||||||
for job in self.jobs:
|
|
||||||
|
|
||||||
attr = job.get("attr", "eval-error")
|
|
||||||
name = attr
|
|
||||||
drv_path = job.get("drvPath")
|
|
||||||
error = job.get("error")
|
|
||||||
out_path = job.get("outputs", {}).get("out")
|
|
||||||
|
|
||||||
build_props.setProperty(f"{attr}-out_path", out_path, "nix-eval")
|
|
||||||
build_props.setProperty(f"{attr}-drv_path", drv_path, "nix-eval")
|
|
||||||
|
|
||||||
props = Properties()
|
|
||||||
props.setProperty("virtual_builder_name", name, "jobs evaluation")
|
|
||||||
props.setProperty("virtual_builder_tags", "", "nix-eval")
|
|
||||||
props.setProperty("attr", attr, "nix-eval")
|
|
||||||
props.setProperty("drv_path", drv_path, "nix-eval")
|
|
||||||
props.setProperty("out_path", out_path, "nix-eval")
|
|
||||||
# we use this to identify builds when running a retry
|
|
||||||
props.setProperty("build_uuid", str(uuid.uuid4()), "nix-eval")
|
|
||||||
props.setProperty("error", error, "nix-eval")
|
|
||||||
triggered_schedulers.append((sch, props))
|
|
||||||
return triggered_schedulers
|
|
||||||
|
|
||||||
def getCurrentSummary(self):
|
|
||||||
"""
|
|
||||||
The original build trigger will the generic builder name `nix-build` in this case, which is not helpful
|
|
||||||
"""
|
|
||||||
if not self.triggeredNames:
|
|
||||||
return {"step": "running"}
|
|
||||||
summary = []
|
|
||||||
if self._result_list:
|
|
||||||
for status in ALL_RESULTS:
|
|
||||||
count = self._result_list.count(status)
|
|
||||||
if count:
|
|
||||||
summary.append(
|
|
||||||
f"{self._result_list.count(status)} {statusToString(status, count)}"
|
|
||||||
)
|
|
||||||
return {"step": f"({', '.join(summary)})"}
|
|
||||||
|
|
||||||
|
|
||||||
class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
|
||||||
"""
|
|
||||||
Parses the output of `nix-eval-jobs` and triggers a `nix-build` build for
|
|
||||||
every attribute.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
|
||||||
kwargs = self.setupShellMixin(kwargs)
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
self.observer = logobserver.BufferLogObserver()
|
|
||||||
self.addLogObserver("stdio", self.observer)
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
|
||||||
# run nix-instanstiate to generate the dict of stages
|
|
||||||
cmd = yield self.makeRemoteShellCommand()
|
|
||||||
yield self.runCommand(cmd)
|
|
||||||
|
|
||||||
# if the command passes extract the list of stages
|
|
||||||
result = cmd.results()
|
|
||||||
failures = 0
|
|
||||||
if result == util.SUCCESS:
|
|
||||||
# create a ShellCommand for each stage and add them to the build
|
|
||||||
jobs = []
|
|
||||||
for line in self.observer.getStdout().split("\n"):
|
|
||||||
if line != "":
|
|
||||||
job = json.loads(line)
|
|
||||||
if "error" not in job.keys():
|
|
||||||
jobs.append(job)
|
|
||||||
else:
|
|
||||||
failures +=1
|
|
||||||
self.build.addStepsAfterCurrentStep(
|
|
||||||
[BuildTrigger(scheduler="nix-build", name="nix-build", jobs=jobs)]
|
|
||||||
)
|
|
||||||
|
|
||||||
if failures > 0:
|
|
||||||
return util.FAILURE
|
|
||||||
else:
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
class RetryCounter:
|
|
||||||
def __init__(self, retries: int) -> None:
|
|
||||||
self.builds: dict[uuid.UUID, int] = defaultdict(lambda: retries)
|
|
||||||
|
|
||||||
def retry_build(self, id: uuid.UUID) -> int:
|
|
||||||
retries = self.builds[id]
|
|
||||||
if retries > 1:
|
|
||||||
self.builds[id] = retries - 1
|
|
||||||
return retries
|
|
||||||
else:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
RETRY_COUNTER = RetryCounter(retries=2)
|
|
||||||
|
|
||||||
|
|
||||||
class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
|
||||||
"""
|
|
||||||
Builds a nix derivation if evaluation was successful,
|
|
||||||
otherwise this shows the evaluation error.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
|
||||||
kwargs = self.setupShellMixin(kwargs)
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
self.observer = logobserver.BufferLogObserver()
|
|
||||||
self.addLogObserver("stdio", self.observer)
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
|
||||||
error = self.getProperty("error")
|
|
||||||
if error is not None:
|
|
||||||
attr = self.getProperty("attr")
|
|
||||||
# show eval error
|
|
||||||
self.build.results = util.FAILURE
|
|
||||||
log = yield self.addLog("nix_error")
|
|
||||||
log.addStderr(f"{attr} failed to evaluate:\n{error}")
|
|
||||||
return util.FAILURE
|
|
||||||
|
|
||||||
# run `nix build`
|
|
||||||
cmd = yield self.makeRemoteShellCommand()
|
|
||||||
yield self.runCommand(cmd)
|
|
||||||
|
|
||||||
res = cmd.results()
|
|
||||||
if res == util.FAILURE:
|
|
||||||
retries = RETRY_COUNTER.retry_build(self.getProperty("build_uuid"))
|
|
||||||
if retries > 0:
|
|
||||||
return util.RETRY
|
|
||||||
return res
|
|
||||||
|
|
||||||
class CreatePr(steps.ShellCommand):
|
|
||||||
"""
|
|
||||||
Creates a pull request if none exists
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, **kwargs: Any) -> None:
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
self.addLogObserver(
|
|
||||||
"stdio", logobserver.LineConsumerLogObserver(self.check_pr_exists)
|
|
||||||
)
|
|
||||||
|
|
||||||
def check_pr_exists(self):
|
|
||||||
ignores = [
|
|
||||||
re.compile(
|
|
||||||
"""a pull request for branch ".*" into branch ".*" already exists:"""
|
|
||||||
),
|
|
||||||
re.compile("No commits between .* and .*"),
|
|
||||||
]
|
|
||||||
while True:
|
|
||||||
_, line = yield
|
|
||||||
if any(ignore.search(line) is not None for ignore in ignores):
|
|
||||||
self.skipped = True
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def run(self):
|
|
||||||
self.skipped = False
|
|
||||||
cmd = yield self.makeRemoteShellCommand()
|
|
||||||
yield self.runCommand(cmd)
|
|
||||||
if self.skipped:
|
|
||||||
return util.SKIPPED
|
|
||||||
return cmd.results()
|
|
||||||
|
|
||||||
|
|
||||||
def nix_update_flake_config(
|
|
||||||
worker_names: list[str],
|
|
||||||
repo: str,
|
|
||||||
projectname: str,
|
|
||||||
github_token_secret: str,
|
|
||||||
github_bot_user: str,
|
|
||||||
) -> util.BuilderConfig:
|
|
||||||
"""
|
|
||||||
Updates the flake an opens a PR for it.
|
|
||||||
"""
|
|
||||||
factory = util.BuildFactory()
|
|
||||||
url_with_secret = util.Interpolate(
|
|
||||||
f"https://git:%(secret:{github_token_secret})s@github.com/{projectname}"
|
|
||||||
)
|
|
||||||
factory.addStep(
|
|
||||||
steps.Git(
|
|
||||||
repourl=url_with_secret,
|
|
||||||
alwaysUseLatest=True,
|
|
||||||
method="clobber",
|
|
||||||
submodules=True,
|
|
||||||
branch="update_flake_lock",
|
|
||||||
haltOnFailure=False,
|
|
||||||
#flunkOnFailure=False,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
factory.addStep(SetProperty(property="GitFailed", value="failed", doStepIf=(lambda step: step.build.results == FAILURE)))
|
|
||||||
|
|
||||||
factory.addStep(
|
|
||||||
steps.Git(
|
|
||||||
repourl=url_with_secret,
|
|
||||||
alwaysUseLatest=True,
|
|
||||||
method="clobber",
|
|
||||||
submodules=True,
|
|
||||||
haltOnFailure=True,
|
|
||||||
mode="full",
|
|
||||||
branch="main",
|
|
||||||
doStepIf=failure,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
factory.addStep(steps.ShellCommand(
|
|
||||||
name="Creating branch",
|
|
||||||
command=[
|
|
||||||
"git",
|
|
||||||
"checkout",
|
|
||||||
"-b",
|
|
||||||
"update_flake_lock"
|
|
||||||
],
|
|
||||||
haltOnFailure=True,
|
|
||||||
doStepIf=failure,
|
|
||||||
)
|
|
||||||
|
|
||||||
)
|
|
||||||
|
|
||||||
factory.addStep(
|
|
||||||
steps.ShellCommand(
|
|
||||||
name="Update flake",
|
|
||||||
env=dict(
|
|
||||||
GIT_AUTHOR_NAME=github_bot_user,
|
|
||||||
GIT_AUTHOR_EMAIL="julien@malka.sh",
|
|
||||||
GIT_COMMITTER_NAME="Julien Malka",
|
|
||||||
GIT_COMMITTER_EMAIL="julien@malka.sh",
|
|
||||||
),
|
|
||||||
command=[
|
|
||||||
"nix",
|
|
||||||
"flake",
|
|
||||||
"update",
|
|
||||||
"--commit-lock-file",
|
|
||||||
"--commit-lockfile-summary",
|
|
||||||
"flake.lock: Update",
|
|
||||||
],
|
|
||||||
haltOnFailure=True,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
factory.addStep(
|
|
||||||
steps.ShellCommand(
|
|
||||||
name="Push to the update_flake_lock branch",
|
|
||||||
command=[
|
|
||||||
"git",
|
|
||||||
"push",
|
|
||||||
"origin",
|
|
||||||
"HEAD:refs/heads/update_flake_lock",
|
|
||||||
],
|
|
||||||
haltOnFailure=True,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
factory.addStep(
|
|
||||||
CreatePr(
|
|
||||||
name="Create pull-request",
|
|
||||||
env=dict(GITHUB_TOKEN=util.Secret(github_token_secret)),
|
|
||||||
command=[
|
|
||||||
"gh",
|
|
||||||
"pr",
|
|
||||||
"create",
|
|
||||||
"--repo",
|
|
||||||
projectname,
|
|
||||||
"--title",
|
|
||||||
"flake.lock: Update",
|
|
||||||
"--body",
|
|
||||||
"Automatic buildbot update",
|
|
||||||
"--head",
|
|
||||||
"refs/heads/update_flake_lock",
|
|
||||||
"--base",
|
|
||||||
"main",
|
|
||||||
],
|
|
||||||
haltOnFailure = True,
|
|
||||||
flunkOnFailure = False,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return util.BuilderConfig(
|
|
||||||
name=f"nix-update-flake-{repo}",
|
|
||||||
workernames=worker_names,
|
|
||||||
factory=factory,
|
|
||||||
properties=dict(virtual_builder_name=f"nix-update-flake-{repo}"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def nix_eval_config(
|
|
||||||
worker_names: list[str],
|
|
||||||
repo: str,
|
|
||||||
github_token_secret: str,
|
|
||||||
) -> util.BuilderConfig:
|
|
||||||
"""
|
|
||||||
Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
|
|
||||||
For each evaluated attribute a new build pipeline is started.
|
|
||||||
If all builds succeed and the build was for a PR opened by the flake update bot,
|
|
||||||
this PR is merged.
|
|
||||||
"""
|
|
||||||
factory = util.BuildFactory()
|
|
||||||
# check out the source
|
|
||||||
url_with_secret = util.Interpolate(
|
|
||||||
f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s"
|
|
||||||
)
|
|
||||||
factory.addStep(
|
|
||||||
GitHub(
|
|
||||||
logEnviron = False,
|
|
||||||
repourl=url_with_secret,
|
|
||||||
method="clobber",
|
|
||||||
submodules=True,
|
|
||||||
haltOnFailure=True,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
factory.addStep(
|
|
||||||
NixEvalCommand(
|
|
||||||
logEnviron = False,
|
|
||||||
env={},
|
|
||||||
name="Evaluation of hydraJobs",
|
|
||||||
command=[
|
|
||||||
"nix-eval-jobs",
|
|
||||||
"--force-recurse",
|
|
||||||
"--workers",
|
|
||||||
8,
|
|
||||||
"--option",
|
|
||||||
"accept-flake-config",
|
|
||||||
"true",
|
|
||||||
"--gc-roots-dir",
|
|
||||||
# FIXME: don't hardcode this
|
|
||||||
"/var/lib/buildbot-worker/gcroot",
|
|
||||||
"--flake",
|
|
||||||
".#hydraJobs",
|
|
||||||
"--impure",
|
|
||||||
],
|
|
||||||
haltOnFailure=False,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return util.BuilderConfig(
|
|
||||||
name=f"nix-eval-{repo}",
|
|
||||||
workernames=worker_names,
|
|
||||||
factory=factory,
|
|
||||||
properties=dict(virtual_builder_name=f"nix-eval-{repo}"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def nix_build_config(
|
|
||||||
worker_names: list[str],
|
|
||||||
) -> util.BuilderConfig:
|
|
||||||
"""
|
|
||||||
Builds one nix flake attribute.
|
|
||||||
"""
|
|
||||||
factory = util.BuildFactory()
|
|
||||||
factory.addStep(
|
|
||||||
NixBuildCommand(
|
|
||||||
env={},
|
|
||||||
name="Build of flake attribute",
|
|
||||||
command=[
|
|
||||||
"nix-build",
|
|
||||||
"--option",
|
|
||||||
"keep-going",
|
|
||||||
"true",
|
|
||||||
"--accept-flake-config",
|
|
||||||
"--out-link",
|
|
||||||
util.Interpolate("result-%(prop:attr)s"),
|
|
||||||
util.Property("drv_path"),
|
|
||||||
],
|
|
||||||
haltOnFailure=True,
|
|
||||||
warnOnFailure=True,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
factory.addStep(
|
|
||||||
steps.ShellCommand(
|
|
||||||
name="Attic push",
|
|
||||||
env={},
|
|
||||||
command=[
|
|
||||||
"attic",
|
|
||||||
"push",
|
|
||||||
"julien",
|
|
||||||
util.Interpolate("result-%(prop:attr)s"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return util.BuilderConfig(
|
|
||||||
name="nix-build",
|
|
||||||
workernames=worker_names,
|
|
||||||
properties=[],
|
|
||||||
collapseRequests=False,
|
|
||||||
env={},
|
|
||||||
factory=factory,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
@ -1,280 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
from datetime import timedelta
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from buildbot.plugins import reporters, schedulers, secrets, util, worker
|
|
||||||
from buildbot.process.properties import Interpolate
|
|
||||||
|
|
||||||
# allow to import modules
|
|
||||||
sys.path.append(str(Path(__file__).parent))
|
|
||||||
|
|
||||||
from buildbot_nix import (
|
|
||||||
nix_build_config,
|
|
||||||
nix_eval_config,
|
|
||||||
nix_update_flake_config,
|
|
||||||
)
|
|
||||||
|
|
||||||
def read_secret_file(secret_name: str) -> str:
|
|
||||||
directory = os.environ.get("CREDENTIALS_DIRECTORY")
|
|
||||||
if directory is None:
|
|
||||||
print("directory not set", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
return Path(directory).joinpath(secret_name).read_text()
|
|
||||||
|
|
||||||
|
|
||||||
ORG = os.environ["GITHUB_ORG"]
|
|
||||||
REPO = os.environ["GITHUB_REPO"]
|
|
||||||
BUILDBOT_URL = os.environ["BUILDBOT_URL"]
|
|
||||||
BUILDBOT_GITHUB_USER = os.environ["BUILDBOT_GITHUB_USER"]
|
|
||||||
|
|
||||||
|
|
||||||
def build_config() -> dict[str, Any]:
|
|
||||||
c = {}
|
|
||||||
c["buildbotNetUsageData"] = None
|
|
||||||
print(ORG, REPO)
|
|
||||||
|
|
||||||
# configure a janitor which will delete all logs older than one month, and will run on sundays at noon
|
|
||||||
c["configurators"] = [
|
|
||||||
util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
|
|
||||||
]
|
|
||||||
|
|
||||||
c["schedulers"] = [
|
|
||||||
# build all pushes to default branch
|
|
||||||
schedulers.SingleBranchScheduler(
|
|
||||||
name="main-nix-config",
|
|
||||||
change_filter=util.ChangeFilter(
|
|
||||||
repository=f"https://github.com/JulienMalka/nix-config",
|
|
||||||
filter_fn=lambda c: c.branch
|
|
||||||
== c.properties.getProperty("github.repository.default_branch"),
|
|
||||||
),
|
|
||||||
builderNames=["nix-eval-nix-config"],
|
|
||||||
),
|
|
||||||
schedulers.SingleBranchScheduler(
|
|
||||||
name="main-linkal",
|
|
||||||
change_filter=util.ChangeFilter(
|
|
||||||
repository=f"https://github.com/JulienMalka/Linkal",
|
|
||||||
filter_fn=lambda c: c.branch
|
|
||||||
== c.properties.getProperty("github.repository.default_branch"),
|
|
||||||
),
|
|
||||||
builderNames=["nix-eval-linkal"],
|
|
||||||
),
|
|
||||||
|
|
||||||
schedulers.SingleBranchScheduler(
|
|
||||||
name="main-nixos-proxmox",
|
|
||||||
change_filter=util.ChangeFilter(
|
|
||||||
repository=f"https://github.com/JulienMalka/nixos-proxmox",
|
|
||||||
filter_fn=lambda c: c.branch
|
|
||||||
== c.properties.getProperty("github.repository.default_branch"),
|
|
||||||
),
|
|
||||||
builderNames=["nix-eval-nixos-proxmox"],
|
|
||||||
),
|
|
||||||
|
|
||||||
|
|
||||||
# build all pull requests
|
|
||||||
schedulers.SingleBranchScheduler(
|
|
||||||
name="prs-nix-config",
|
|
||||||
change_filter=util.ChangeFilter(
|
|
||||||
repository=f"https://github.com/{ORG}/{REPO}", category="pull"
|
|
||||||
),
|
|
||||||
builderNames=["nix-eval-nix-config"],
|
|
||||||
),
|
|
||||||
|
|
||||||
schedulers.SingleBranchScheduler(
|
|
||||||
name="prs-linkal",
|
|
||||||
change_filter=util.ChangeFilter(
|
|
||||||
repository=f"https://github.com/JulienMalka/Linkal", category="pull"
|
|
||||||
),
|
|
||||||
builderNames=["nix-eval-linkal"],
|
|
||||||
),
|
|
||||||
|
|
||||||
schedulers.SingleBranchScheduler(
|
|
||||||
name="prs-nixos-proxmox",
|
|
||||||
change_filter=util.ChangeFilter(
|
|
||||||
repository=f"https://github.com/JulienMalka/nixos-proxmox", category="pull"
|
|
||||||
),
|
|
||||||
builderNames=["nix-eval-nixos-proxmox"],
|
|
||||||
),
|
|
||||||
|
|
||||||
# this is triggered from `nix-eval`
|
|
||||||
schedulers.Triggerable(
|
|
||||||
name="nix-build",
|
|
||||||
builderNames=["nix-build"],
|
|
||||||
),
|
|
||||||
# allow to manually trigger a nix-build
|
|
||||||
schedulers.ForceScheduler(name="force", builderNames=["nix-eval-nix-config"]),
|
|
||||||
# allow to manually update flakes
|
|
||||||
schedulers.ForceScheduler(
|
|
||||||
name="update-flake-nix-config",
|
|
||||||
builderNames=["nix-update-flake-linkal"],
|
|
||||||
buttonName="Update flakes",
|
|
||||||
),
|
|
||||||
schedulers.ForceScheduler(
|
|
||||||
name="update-flake-linkal",
|
|
||||||
builderNames=["nix-update-flake-nix-config"],
|
|
||||||
buttonName="Update flakes",
|
|
||||||
),
|
|
||||||
schedulers.ForceScheduler(
|
|
||||||
name="update-flake-nixos-proxmox",
|
|
||||||
builderNames=["nix-update-flake-nixos-proxmox"],
|
|
||||||
buttonName="Update flakes",
|
|
||||||
),
|
|
||||||
|
|
||||||
|
|
||||||
# updates flakes once a weeek
|
|
||||||
schedulers.Nightly(
|
|
||||||
name="update-flake-daily-nix-config",
|
|
||||||
builderNames=["nix-update-flake-nix-config"],
|
|
||||||
hour=2,
|
|
||||||
minute=0,
|
|
||||||
),
|
|
||||||
schedulers.Nightly(
|
|
||||||
name="update-flake-daily-linkal",
|
|
||||||
builderNames=["nix-update-flake-linkal"],
|
|
||||||
dayOfWeek=6,
|
|
||||||
hour=1,
|
|
||||||
minute=0,
|
|
||||||
),
|
|
||||||
schedulers.Nightly(
|
|
||||||
name="update-flake-daily-nixos-proxmox",
|
|
||||||
builderNames=["nix-update-flake-nixos-proxmox"],
|
|
||||||
dayOfWeek=5,
|
|
||||||
hour=1,
|
|
||||||
minute=0,
|
|
||||||
),
|
|
||||||
|
|
||||||
|
|
||||||
]
|
|
||||||
|
|
||||||
github_api_token = read_secret_file("github-token")
|
|
||||||
c["services"] = [
|
|
||||||
reporters.GitHubStatusPush(
|
|
||||||
token=github_api_token,
|
|
||||||
# Since we dynamically create build steps,
|
|
||||||
# we use `virtual_builder_name` in the webinterface
|
|
||||||
# so that we distinguish what has beeing build
|
|
||||||
context=Interpolate("%(prop:virtual_builder_name)s"),
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
worker_config = json.loads(read_secret_file("buildbot-nix-workers"))
|
|
||||||
|
|
||||||
credentials = os.environ.get("CREDENTIALS_DIRECTORY", ".")
|
|
||||||
|
|
||||||
systemd_secrets = secrets.SecretInAFile(dirname=credentials)
|
|
||||||
c["secretsProviders"] = [systemd_secrets]
|
|
||||||
c["workers"] = []
|
|
||||||
worker_names = []
|
|
||||||
for item in worker_config:
|
|
||||||
cores = item.get("cores", 0)
|
|
||||||
for i in range(cores):
|
|
||||||
worker_name = f"{item['name']}-{i}"
|
|
||||||
c["workers"].append(worker.Worker(worker_name, item["pass"]))
|
|
||||||
worker_names.append(worker_name)
|
|
||||||
c["builders"] = [
|
|
||||||
# Since all workers run on the same machine, we only assign one of them to do the evaluation.
|
|
||||||
# This should prevent exessive memory usage.
|
|
||||||
nix_eval_config(
|
|
||||||
[worker_names[0]],
|
|
||||||
"nix-config",
|
|
||||||
github_token_secret="github-token",
|
|
||||||
),
|
|
||||||
nix_eval_config(
|
|
||||||
[worker_names[0]],
|
|
||||||
"linkal",
|
|
||||||
github_token_secret="github-token",
|
|
||||||
),
|
|
||||||
nix_eval_config(
|
|
||||||
[worker_names[0]],
|
|
||||||
"nixos-proxmox",
|
|
||||||
github_token_secret="github-token",
|
|
||||||
),
|
|
||||||
|
|
||||||
|
|
||||||
nix_build_config(worker_names),
|
|
||||||
nix_update_flake_config(
|
|
||||||
worker_names,
|
|
||||||
"nix-config",
|
|
||||||
f"{ORG}/{REPO}",
|
|
||||||
github_token_secret="github-token",
|
|
||||||
github_bot_user=BUILDBOT_GITHUB_USER,
|
|
||||||
),
|
|
||||||
nix_update_flake_config(
|
|
||||||
worker_names,
|
|
||||||
"linkal",
|
|
||||||
f"JulienMalka/Linkal",
|
|
||||||
github_token_secret="github-token",
|
|
||||||
github_bot_user=BUILDBOT_GITHUB_USER,
|
|
||||||
),
|
|
||||||
nix_update_flake_config(
|
|
||||||
worker_names,
|
|
||||||
"nixos-proxmox",
|
|
||||||
f"JulienMalka/nixos-proxmox",
|
|
||||||
github_token_secret="github-token",
|
|
||||||
github_bot_user=BUILDBOT_GITHUB_USER,
|
|
||||||
),
|
|
||||||
|
|
||||||
|
|
||||||
]
|
|
||||||
|
|
||||||
github_admins = os.environ.get("GITHUB_ADMINS", "").split(",")
|
|
||||||
|
|
||||||
c["www"] = {
|
|
||||||
"avatar_methods": [util.AvatarGitHub()],
|
|
||||||
'port': "tcp:1810:interface=\\:\\:",
|
|
||||||
"auth": util.GitHubAuth("bba3e144501aa5b8a5dd", str(read_secret_file("github-oauth-secret")).strip()),
|
|
||||||
"authz": util.Authz(
|
|
||||||
roleMatchers=[
|
|
||||||
util.RolesFromUsername(roles=["admin"], usernames=github_admins)
|
|
||||||
],
|
|
||||||
allowRules=[
|
|
||||||
util.AnyEndpointMatcher(role="admin", defaultDeny=False),
|
|
||||||
util.AnyControlEndpointMatcher(role="admins"),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
"plugins": dict(console_view={}, badges = {
|
|
||||||
"left_pad" : 5,
|
|
||||||
"left_text": "Build Status", # text on the left part of the image
|
|
||||||
"left_color": "#555", # color of the left part of the image
|
|
||||||
"right_pad" : 5,
|
|
||||||
"border_radius" : 5, # Border Radius on flat and plastic badges
|
|
||||||
# style of the template availables are "flat", "flat-square", "plastic"
|
|
||||||
"template_name": "flat.svg.j2", # name of the template
|
|
||||||
"font_face": "DejaVu Sans",
|
|
||||||
"font_size": 11,
|
|
||||||
"color_scheme": { # color to be used for right part of the image
|
|
||||||
"exception": "#007ec6",
|
|
||||||
"failure": "#e05d44",
|
|
||||||
"retry": "#007ec6",
|
|
||||||
"running": "#007ec6",
|
|
||||||
"skipped": "a4a61d",
|
|
||||||
"success": "#4c1",
|
|
||||||
"unknown": "#9f9f9f",
|
|
||||||
"warnings": "#dfb317"
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
"change_hook_dialects": dict(
|
|
||||||
github={
|
|
||||||
"secret": str(read_secret_file("github-webhook-secret")).strip(),
|
|
||||||
"strict": True,
|
|
||||||
"token": github_api_token,
|
|
||||||
"github_property_whitelist": "*",
|
|
||||||
}
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
c["db"] = {"db_url": os.environ.get("DB_URL", "sqlite:///state.sqlite")}
|
|
||||||
|
|
||||||
c['protocols'] = {'pb': {'port': "tcp:interface=\\:\\::port=9989"}}
|
|
||||||
c["buildbotURL"] = BUILDBOT_URL
|
|
||||||
c["collapseRequests"] = False
|
|
||||||
|
|
||||||
return c
|
|
||||||
|
|
||||||
|
|
||||||
BuildmasterConfig = build_config()
|
|
||||||
|
|
||||||
|
|
@ -1,58 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import multiprocessing
|
|
||||||
import os
|
|
||||||
import socket
|
|
||||||
from io import open
|
|
||||||
|
|
||||||
from buildbot_worker.bot import Worker
|
|
||||||
from twisted.application import service
|
|
||||||
|
|
||||||
|
|
||||||
def require_env(key: str) -> str:
|
|
||||||
val = os.environ.get(key)
|
|
||||||
assert val is not None, "val is not set"
|
|
||||||
return val
|
|
||||||
|
|
||||||
|
|
||||||
def setup_worker(application: service.Application, id: int) -> None:
|
|
||||||
basedir = f"{require_env('BUILDBOT_DIR')}-{id}"
|
|
||||||
os.makedirs(basedir, mode=0o700, exist_ok=True)
|
|
||||||
|
|
||||||
master_url = require_env("MASTER_URL")
|
|
||||||
hostname = socket.gethostname()
|
|
||||||
workername = f"{hostname}-{id}"
|
|
||||||
|
|
||||||
with open(
|
|
||||||
require_env("WORKER_PASSWORD_FILE"), "r", encoding="utf-8"
|
|
||||||
) as passwd_file:
|
|
||||||
passwd = passwd_file.read().strip("\r\n")
|
|
||||||
keepalive = 600
|
|
||||||
umask = None
|
|
||||||
maxdelay = 300
|
|
||||||
numcpus = None
|
|
||||||
allow_shutdown = None
|
|
||||||
|
|
||||||
s = Worker(
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
workername,
|
|
||||||
passwd,
|
|
||||||
basedir,
|
|
||||||
keepalive,
|
|
||||||
connection_string=master_url,
|
|
||||||
umask=umask,
|
|
||||||
maxdelay=maxdelay,
|
|
||||||
numcpus=numcpus,
|
|
||||||
allow_shutdown=allow_shutdown,
|
|
||||||
)
|
|
||||||
s.setServiceParent(application)
|
|
||||||
|
|
||||||
|
|
||||||
# note: this line is matched against to check that this is a worker
|
|
||||||
# directory; do not edit it.
|
|
||||||
application = service.Application("buildbot-worker")
|
|
||||||
|
|
||||||
for i in range(14):
|
|
||||||
setup_worker(application, i)
|
|
||||||
|
|
||||||
Loading…
Add table
Reference in a new issue