mirror of
https://github.com/JulienMalka/snowfield.git
synced 2025-04-04 03:01:05 +02:00
Simplified config (hope it works)
This commit is contained in:
parent
25fee2fbb9
commit
45b19e73e3
GPG key ID:
6FC74C847011FD83
1 changed files with 4 additions and 234 deletions
|
|
@ -129,7 +129,6 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
# FIXME this leaks memory... but probably not enough that we care
|
|
||||||
class RetryCounter:
|
class RetryCounter:
|
||||||
def __init__(self, retries: int) -> None:
|
def __init__(self, retries: int) -> None:
|
||||||
self.builds: dict[uuid.UUID, int] = defaultdict(lambda: retries)
|
self.builds: dict[uuid.UUID, int] = defaultdict(lambda: retries)
|
||||||
|
|
@ -142,9 +141,6 @@ class RetryCounter:
|
||||||
else:
|
else:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
# For now we limit this to two. Often this allows us to make the error log
|
|
||||||
# shorter because we won't see the logs for all previous succeeded builds
|
|
||||||
RETRY_COUNTER = RetryCounter(retries=2)
|
RETRY_COUNTER = RetryCounter(retries=2)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -159,7 +155,6 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self.observer = logobserver.BufferLogObserver()
|
self.observer = logobserver.BufferLogObserver()
|
||||||
self.addLogObserver("stdio", self.observer)
|
self.addLogObserver("stdio", self.observer)
|
||||||
self.logEnviron = False
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
def run(self) -> Generator[Any, object, Any]:
|
||||||
|
|
@ -183,75 +178,6 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
return util.RETRY
|
return util.RETRY
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
class UpdateBuildOutput(steps.BuildStep):
|
|
||||||
"""
|
|
||||||
Updates store paths in a public www directory.
|
|
||||||
This is useful to prefetch updates without having to evaluate
|
|
||||||
on the target machine.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, branches: list[str], **kwargs):
|
|
||||||
self.branches = branches
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
|
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
|
||||||
props = self.build.getProperties()
|
|
||||||
if props.getProperty("branch") not in self.branches:
|
|
||||||
return util.SKIPPED
|
|
||||||
attr = os.path.basename(props.getProperty("attr"))
|
|
||||||
out_path = props.getProperty("out_path")
|
|
||||||
# XXX don't hardcode this
|
|
||||||
p = Path("/var/www/buildbot/nix-outputs/")
|
|
||||||
os.makedirs(p, exist_ok=True)
|
|
||||||
with open(p / attr, "w") as f:
|
|
||||||
f.write(out_path)
|
|
||||||
return util.SUCCESS
|
|
||||||
|
|
||||||
|
|
||||||
class MergePr(steps.ShellCommand):
|
|
||||||
"""
|
|
||||||
Merge a pull request for specified branches and pull request owners
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
base_branches: list[str],
|
|
||||||
owners: list[str],
|
|
||||||
**kwargs: Any,
|
|
||||||
) -> None:
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
self.base_branches = base_branches
|
|
||||||
self.owners = owners
|
|
||||||
self.observer = logobserver.BufferLogObserver()
|
|
||||||
self.addLogObserver("stdio", self.observer)
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def reconfigService(
|
|
||||||
self,
|
|
||||||
base_branches: list[str],
|
|
||||||
owners: list[str],
|
|
||||||
**kwargs: Any,
|
|
||||||
) -> Generator[Any, object, Any]:
|
|
||||||
self.base_branches = base_branches
|
|
||||||
self.owners = owners
|
|
||||||
super().reconfigService(**kwargs)
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
|
||||||
props = self.build.getProperties()
|
|
||||||
if props.getProperty("basename") not in self.base_branches:
|
|
||||||
return util.SKIPPED
|
|
||||||
if props.getProperty("event") not in ["pull_request"]:
|
|
||||||
return util.SKIPPED
|
|
||||||
if not any(owner in self.owners for owner in props.getProperty("owners")):
|
|
||||||
return util.SKIPPED
|
|
||||||
|
|
||||||
cmd = yield self.makeRemoteShellCommand()
|
|
||||||
yield self.runCommand(cmd)
|
|
||||||
return cmd.results()
|
|
||||||
|
|
||||||
|
|
||||||
class CreatePr(steps.ShellCommand):
|
class CreatePr(steps.ShellCommand):
|
||||||
"""
|
"""
|
||||||
Creates a pull request if none exists
|
Creates a pull request if none exists
|
||||||
|
|
@ -376,78 +302,6 @@ class Machine:
|
||||||
self.attr_name = attr_name
|
self.attr_name = attr_name
|
||||||
|
|
||||||
|
|
||||||
class DeployTrigger(Trigger):
|
|
||||||
"""
|
|
||||||
Dynamic trigger that creates a deploy step for every machine.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, scheduler: str, machines: list[Machine], **kwargs):
|
|
||||||
if "name" not in kwargs:
|
|
||||||
kwargs["name"] = "trigger"
|
|
||||||
self.machines = machines
|
|
||||||
self.config = None
|
|
||||||
Trigger.__init__(
|
|
||||||
self,
|
|
||||||
waitForFinish=True,
|
|
||||||
schedulerNames=[scheduler],
|
|
||||||
haltOnFailure=True,
|
|
||||||
flunkOnFailure=True,
|
|
||||||
sourceStamps=[],
|
|
||||||
alwaysUseLatest=False,
|
|
||||||
updateSourceStamp=False,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def createTriggerProperties(self, props):
|
|
||||||
return props
|
|
||||||
|
|
||||||
def getSchedulersAndProperties(self):
|
|
||||||
build_props = self.build.getProperties()
|
|
||||||
repo_name = build_props.getProperty(
|
|
||||||
"github.base.repo.full_name",
|
|
||||||
build_props.getProperty("github.repository.full_name"),
|
|
||||||
)
|
|
||||||
|
|
||||||
sch = self.schedulerNames[0]
|
|
||||||
|
|
||||||
triggered_schedulers = []
|
|
||||||
for m in self.machines:
|
|
||||||
out_path = build_props.getProperty(f"nixos-{m.attr_name}-out_path")
|
|
||||||
props = Properties()
|
|
||||||
name = m.attr_name
|
|
||||||
if repo_name is not None:
|
|
||||||
name = f"{repo_name}: Deploy {name}"
|
|
||||||
props.setProperty("virtual_builder_name", name, "deploy")
|
|
||||||
props.setProperty("attr", m.attr_name, "deploy")
|
|
||||||
props.setProperty("out_path", out_path, "deploy")
|
|
||||||
triggered_schedulers.append((sch, props))
|
|
||||||
return triggered_schedulers
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def run(self):
|
|
||||||
props = self.build.getProperties()
|
|
||||||
if props.getProperty("branch") not in self.branches:
|
|
||||||
return util.SKIPPED
|
|
||||||
res = yield super().__init__()
|
|
||||||
return res
|
|
||||||
|
|
||||||
def getCurrentSummary(self):
|
|
||||||
"""
|
|
||||||
The original build trigger will the generic builder name `nix-build` in this case, which is not helpful
|
|
||||||
"""
|
|
||||||
if not self.triggeredNames:
|
|
||||||
return {"step": "running"}
|
|
||||||
summary = []
|
|
||||||
if self._result_list:
|
|
||||||
for status in ALL_RESULTS:
|
|
||||||
count = self._result_list.count(status)
|
|
||||||
if count:
|
|
||||||
summary.append(
|
|
||||||
f"{self._result_list.count(status)} {statusToString(status, count)}"
|
|
||||||
)
|
|
||||||
return {"step": f"({', '.join(summary)})"}
|
|
||||||
|
|
||||||
|
|
||||||
def nix_eval_config(
|
def nix_eval_config(
|
||||||
worker_names: list[str],
|
worker_names: list[str],
|
||||||
github_token_secret: str,
|
github_token_secret: str,
|
||||||
|
|
@ -467,6 +321,7 @@ def nix_eval_config(
|
||||||
)
|
)
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
steps.Git(
|
steps.Git(
|
||||||
|
logEnviron = False,
|
||||||
repourl=url_with_secret,
|
repourl=url_with_secret,
|
||||||
method="clean",
|
method="clean",
|
||||||
submodules=True,
|
submodules=True,
|
||||||
|
|
@ -476,6 +331,7 @@ def nix_eval_config(
|
||||||
|
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
NixEvalCommand(
|
NixEvalCommand(
|
||||||
|
logEnviron = False,
|
||||||
env={},
|
env={},
|
||||||
name="Eval flake",
|
name="Eval flake",
|
||||||
command=[
|
command=[
|
||||||
|
|
@ -494,46 +350,6 @@ def nix_eval_config(
|
||||||
haltOnFailure=True,
|
haltOnFailure=True,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
# Merge flake-update pull requests if CI succeeds
|
|
||||||
if len(automerge_users) > 0:
|
|
||||||
factory.addStep(
|
|
||||||
MergePr(
|
|
||||||
name="Merge pull-request",
|
|
||||||
env=dict(GITHUB_TOKEN=util.Secret(github_token_secret)),
|
|
||||||
base_branches=["master"],
|
|
||||||
owners=automerge_users,
|
|
||||||
command=[
|
|
||||||
"gh",
|
|
||||||
"pr",
|
|
||||||
"merge",
|
|
||||||
"--repo",
|
|
||||||
util.Property("project"),
|
|
||||||
"--rebase",
|
|
||||||
util.Property("pullrequesturl"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
# factory.addStep(
|
|
||||||
# DeployTrigger(scheduler="nixos-deploy", name="nixos-deploy", machines=machines)
|
|
||||||
# )
|
|
||||||
# factory.addStep(
|
|
||||||
# DeployNixOS(
|
|
||||||
# name="Deploy NixOS machines",
|
|
||||||
# env=dict(GITHUB_TOKEN=util.Secret(github_token_secret)),
|
|
||||||
# base_branches=["master"],
|
|
||||||
# owners=automerge_users,
|
|
||||||
# command=[
|
|
||||||
# "gh",
|
|
||||||
# "pr",
|
|
||||||
# "merge",
|
|
||||||
# "--repo",
|
|
||||||
# util.Property("project"),
|
|
||||||
# "--rebase",
|
|
||||||
# util.Property("pullrequesturl"),
|
|
||||||
# ],
|
|
||||||
# )
|
|
||||||
# )
|
|
||||||
|
|
||||||
return util.BuilderConfig(
|
return util.BuilderConfig(
|
||||||
name="nix-eval",
|
name="nix-eval",
|
||||||
|
|
@ -554,6 +370,7 @@ def nix_build_config(
|
||||||
factory = util.BuildFactory()
|
factory = util.BuildFactory()
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
NixBuildCommand(
|
NixBuildCommand(
|
||||||
|
logEnviron = False,
|
||||||
env={},
|
env={},
|
||||||
name="Build flake attr",
|
name="Build flake attr",
|
||||||
command=[
|
command=[
|
||||||
|
|
@ -569,24 +386,7 @@ def nix_build_config(
|
||||||
haltOnFailure=True,
|
haltOnFailure=True,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if has_cachix_auth_token or has_cachix_signing_key:
|
|
||||||
if has_cachix_signing_key:
|
|
||||||
env = dict(CACHIX_SIGNING_KEY=util.Secret("cachix-signing-key"))
|
|
||||||
else:
|
|
||||||
env = dict(CACHIX_AUTH_TOKEN=util.Secret("cachix-auth-token"))
|
|
||||||
factory.addStep(
|
|
||||||
steps.ShellCommand(
|
|
||||||
name="Upload cachix",
|
|
||||||
env=env,
|
|
||||||
command=[
|
|
||||||
"cachix",
|
|
||||||
"push",
|
|
||||||
util.Secret("cachix-name"),
|
|
||||||
util.Interpolate("result-%(prop:attr)s"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
)
|
|
||||||
factory.addStep(UpdateBuildOutput(name="Update build output", branches=["master"]))
|
|
||||||
return util.BuilderConfig(
|
return util.BuilderConfig(
|
||||||
name="nix-build",
|
name="nix-build",
|
||||||
workernames=worker_names,
|
workernames=worker_names,
|
||||||
|
|
@ -596,33 +396,3 @@ def nix_build_config(
|
||||||
factory=factory,
|
factory=factory,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# def nixos_deployment_config(worker_names: list[str]) -> util.BuilderConfig:
|
|
||||||
# factory = util.BuildFactory()
|
|
||||||
# factory.addStep(
|
|
||||||
# NixBuildCommand(
|
|
||||||
# env={},
|
|
||||||
# name="Deploy NixOS",
|
|
||||||
# command=[
|
|
||||||
# "nix",
|
|
||||||
# "build",
|
|
||||||
# "--option",
|
|
||||||
# "keep-going",
|
|
||||||
# "true",
|
|
||||||
# "-L",
|
|
||||||
# "--out-link",
|
|
||||||
# util.Interpolate("result-%(prop:attr)s"),
|
|
||||||
# util.Property("drv_path"),
|
|
||||||
# ],
|
|
||||||
# haltOnFailure=True,
|
|
||||||
# )
|
|
||||||
# )
|
|
||||||
# return util.BuilderConfig(
|
|
||||||
# name="nix-build",
|
|
||||||
# workernames=worker_names,
|
|
||||||
# properties=[],
|
|
||||||
# collapseRequests=False,
|
|
||||||
# env={},
|
|
||||||
# factory=factory,
|
|
||||||
# )
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue