Compare commits

..

1 Commits

Author SHA1 Message Date
Pierre Tardy
b8da0b44fe
need to install via pip install 2017-05-03 17:53:14 +02:00
17 changed files with 79 additions and 510 deletions

3
.gitignore vendored
View File

@ -1,3 +0,0 @@
/venv
gitea.env
/data

View File

@ -1,3 +0,0 @@
{
"python.analysis.typeCheckingMode": "basic"
}

View File

@ -1,2 +0,0 @@
# buildbot-docker-example-config
example configuration for running buildbot in docker

View File

@ -3,8 +3,7 @@
import os
from buildbot.plugins import changes, reporters, schedulers, steps, util, worker
from buildbot.www.authz.roles import RolesFromBase
from buildbot.plugins import *
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
@ -13,67 +12,55 @@ from buildbot.www.authz.roles import RolesFromBase
# a shorter alias to save typing.
c = BuildmasterConfig = {}
GITEA_SECRET = os.environ.get("GITEA_SECRET")
GITEA_TOKEN = os.environ.get("GITEA_TOKEN")
CLIENT_ID = os.environ.get("GITEA_CLIENT_ID")
assert CLIENT_ID
CLIENT_SECRET = os.environ.get("GITEA_CLIENT_SECRET")
assert CLIENT_SECRET
####### WORKERS
# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password. The same
# worker name and password must be configured on the worker.
c["workers"] = [
worker.Worker("worker-rust-1-65", "pass", properties={"rust_version": "1.65"}),
worker.Worker("worker-rust-1-72", "pass", properties={"rust_version": "1.72"}),
worker.Worker("worker-rust-mdbook", "pass"),
]
rust_workers_1_65 = ["worker-rust-1-65"]
rust_workers_1_72 = ["worker-rust-1-72"]
rust_workers_mdbook = ["worker-rust-mdbook"]
c['workers'] = [worker.Worker("example-worker", 'pass')]
if "BUILDBOT_MQ_URL" in os.environ:
c["mq"] = {
"type": "wamp",
"router_url": os.environ["BUILDBOT_MQ_URL"],
"realm": os.environ.get("BUILDBOT_MQ_REALM", "buildbot"),
"debug": "BUILDBOT_MQ_DEBUG" in os.environ,
"debug_websockets": "BUILDBOT_MQ_DEBUG" in os.environ,
"debug_lowlevel": "BUILDBOT_MQ_DEBUG" in os.environ,
if 'BUILDBOT_MQ_URL' in os.environ:
c['mq'] = {
'type' : 'wamp',
'router_url': os.environ['BUILDBOT_MQ_URL'],
'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'),
'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ,
'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ,
'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ,
}
# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c["protocols"] = {"pb": {"port": os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
####### CHANGESOURCES
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot clone of radn.
# about source code changes. Here we point to the buildbot clone of pyflakes.
c["change_source"] = []
c["change_source"].append(
changes.GitPoller(
"https://gitea.parrrate.ru/PTV/radn-rs.git",
workdir="gitpoller/radn-rs",
branch="main",
pollinterval=300,
)
)
c["change_source"].append(
changes.GitPoller(
"https://gitea.parrrate.ru/PTV/exercises.git",
workdir="gitpoller/exercises",
branch="latest",
pollinterval=300,
)
)
c['change_source'] = []
c['change_source'].append(changes.GitPoller(
'git://github.com/buildbot/pyflakes.git',
workdir='gitpoller-workdir', branch='master',
pollinterval=300))
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build
c['schedulers'] = []
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="all",
change_filter=util.ChangeFilter(branch='master'),
treeStableTimer=None,
builderNames=["runtests"]))
c['schedulers'].append(schedulers.ForceScheduler(
name="force",
builderNames=["runtests"]))
####### BUILDERS
@ -81,258 +68,49 @@ c["change_source"].append(
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
# all_repositories = {
# r"https://gitea.parrrate.ru/PTV/radn-rs.git": "radn-rs",
# }
factory = util.BuildFactory()
# check out the source
factory.addStep(steps.Git(repourl='http://github.com/buildbot/pyflakes.git', mode='incremental'))
# run the tests (note that this will require that 'trial' is installed)
factory.addStep(steps.ShellCommand(command=["trial", "pyflakes"]))
c['builders'] = []
c['builders'].append(
util.BuilderConfig(name="runtests",
workernames=["example-worker"],
factory=factory))
# def codebaseGenerator(chdict):
# return all_repositories.get(chdict["repository"])
####### STATUS TARGETS
# c["codebaseGenerator"] = codebaseGenerator
c["builders"] = []
radn_builders = []
exercises_builders = []
CARGO_TARGET_DIR = "/buildbot/_rust/radn-rs/target"
RADN_ENV = {"CARGO_TARGET_DIR": CARGO_TARGET_DIR}
DOC_DIR = f"{CARGO_TARGET_DIR}/doc"
def radn_rs_factory():
factory = util.BuildFactory()
factory.addStep(
steps.Git(
repourl="https://gitea.parrrate.ru/PTV/radn-rs.git", mode="incremental"
)
)
return factory
def exercises_factory():
factory = util.BuildFactory()
factory.addStep(
steps.Git(
repourl="https://gitea.parrrate.ru/PTV/exercises.git", mode="incremental"
)
)
return factory
def radn_append_factory(factory, name: str, workernames: list[str]):
c["builders"].append(
util.BuilderConfig(name=name, workernames=workernames, factory=factory)
)
radn_builders.append(name)
def exercises_append_factory(factory, name: str, workernames: list[str]):
c["builders"].append(
util.BuilderConfig(name=name, workernames=workernames, factory=factory)
)
exercises_builders.append(name)
def cargo_test(name: str, workernames: list[str]):
factory = radn_rs_factory()
factory.addStep(
steps.ShellCommand(
command=["cargo", "test", "--workspace"],
env=RADN_ENV,
)
)
radn_append_factory(factory, name, workernames)
def cargo_clippy(name: str, workernames: list[str]):
factory = radn_rs_factory()
factory.addStep(
steps.ShellCommand(
command=[
"cargo",
"clippy",
"--workspace",
"--examples",
"--tests",
"--",
"--deny=warnings",
],
env=RADN_ENV,
)
)
radn_append_factory(factory, name, workernames)
def cargo_fmt(name: str, workernames: list[str]):
factory = radn_rs_factory()
factory.addStep(
steps.ShellCommand(
command=[
"cargo",
"fmt",
"--check",
"--all",
],
env=RADN_ENV,
)
)
radn_append_factory(factory, name, workernames)
def cargo_doc(name: str, workernames: list[str], specific: bool, latest: bool):
factory = radn_rs_factory()
factory.addStep(
steps.ShellCommand(
command=[
"cargo",
"doc",
"--workspace",
"--no-deps",
],
env=RADN_ENV,
)
)
if specific:
factory.addStep(
steps.DirectoryUpload(
workersrc=DOC_DIR,
masterdest=util.Interpolate(
"/buildbot_share/docs/radn-rs/%(prop:rust_version)s/"
),
url=util.Interpolate(
"https://radn.parrrate.ru/latest/docs/%(prop:rust_version)s/radn/"
),
compress="gz",
)
)
if latest:
factory.addStep(
steps.DirectoryUpload(
workersrc=DOC_DIR,
masterdest="/buildbot_share/docs/radn-rs/",
url="https://radn.parrrate.ru/latest/docs/radn/",
compress="gz",
)
)
radn_append_factory(factory, name, workernames)
def mdbook_test(name: str, workernames: list[str]):
factory = exercises_factory()
factory.addStep(
steps.ShellCommand(
command=["mdbook", "test", "."],
)
)
exercises_append_factory(factory, name, workernames)
cargo_test("cargo test (1.65)", rust_workers_1_65)
cargo_clippy("cargo clippy (1.65)", rust_workers_1_65)
cargo_clippy("cargo clippy (1.72)", rust_workers_1_72)
cargo_fmt("cargo fmt (1.72)", rust_workers_1_72)
cargo_doc("cargo doc (1.72)", rust_workers_1_72, False, True)
mdbook_test("mdbook test", rust_workers_mdbook)
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes.
c["schedulers"] = []
c["schedulers"].append(
schedulers.SingleBranchScheduler(
name="all-radn",
change_filter=util.ChangeFilter(branch="main"),
treeStableTimer=None,
builderNames=radn_builders,
)
)
c["schedulers"].append(
schedulers.SingleBranchScheduler(
name="all-exercises",
change_filter=util.ChangeFilter(branch="latest"),
treeStableTimer=None,
builderNames=exercises_builders,
)
)
c["schedulers"].append(
schedulers.ForceScheduler(name="force-radn", builderNames=radn_builders)
)
c["schedulers"].append(
schedulers.ForceScheduler(name="force-exercises", builderNames=exercises_builders)
)
####### REPORTER TARGETS
# 'services' is a list of Reporter Targets. The results of each build will be
# pushed to these targets. buildbot/reporters/*.py has a variety to choose from,
# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# like IRC bots.
c["services"] = []
c['status'] = []
####### PROJECT IDENTITY
# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c["title"] = "RADN"
c["titleURL"] = "https://gitea.parrrate.ru/PTV/radn-rs"
c['title'] = "Pyflakes"
c['titleURL'] = "https://launchpad.net/pyflakes"
# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
c["buildbotURL"] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
class RolesFromCustom(RolesFromBase):
def getRolesFromUser(self, userDetails: dict):
if userDetails.get("is_admin") is True:
return ["admins"]
else:
return []
c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
# minimalistic config to activate new web UI
c["www"] = {
"port": os.environ.get("BUILDBOT_WEB_PORT", 8010),
"plugins": {
"waterfall_view": {},
"console_view": {},
},
"change_hook_dialects": {},
"auth": util.GiteaAuth(
endpoint="https://gitea.parrrate.ru",
client_id=CLIENT_ID,
client_secret=CLIENT_SECRET,
),
"authz": util.Authz(
allowRules=[util.AnyControlEndpointMatcher(role="admins")],
roleMatchers=[RolesFromCustom()],
),
}
if GITEA_SECRET:
c["www"]["change_hook_dialects"]["gitea"] = {
"secret": GITEA_SECRET,
"onlyIncludePushCommit": True,
}
if GITEA_TOKEN:
c["services"].append(
reporters.GiteaStatusPush(
"https://gitea.parrrate.ru", token=GITEA_TOKEN, verbose=True
),
)
c['www'] = dict(port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
plugins=dict(waterfall_view={}, console_view={}))
####### DB URL
c["db"] = {
c['db'] = {
# This specifies what database buildbot uses to store its state. You can leave
# this at its default for all but the largest installations.
"db_url": os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ),
'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ),
}
c["buildbotNetUsageData"] = None

View File

@ -1,2 +0,0 @@
FROM buildbot/buildbot-master:v3.11.5
RUN /buildbot_venv/bin/pip3 install buildbot_gitea

View File

@ -19,7 +19,7 @@ services:
env_file: db.env
environment:
- BUILDBOT_CONFIG_DIR=config
- BUILDBOT_CONFIG_URL=https://gitea.parrrate.ru/PTV/buildbot/archive/master.tar.gz
- BUILDBOT_CONFIG_URL=https://github.com/buildbot/buildbot-docker-example-config/archive/master.tar.gz
- BUILDBOT_WORKER_PORT=9989
- BUILDBOT_WEB_URL=http://localhost:8080/
- BUILDBOT_WEB_PORT=8080

View File

@ -1,2 +1,3 @@
FROM pypy:3-onbuild
from pypy:2-onbuild
RUN pip install /usr/src/app/*.whl
CMD /usr/src/app/start_buildbot.sh

View File

@ -1,2 +1,2 @@
FROM pypy:2-onbuild
from pypy:2-onbuild
CMD ["twistd", "-ny", "buildbot.tac"]

View File

@ -1,2 +0,0 @@
FROM nginx
COPY nginx-default.conf /etc/nginx/conf.d/default.conf

View File

@ -1,122 +1,36 @@
networks:
v6d:
external: true
buildbot-db: {}
buildbot-worker: {}
buildbot-docs: {}
version: '2'
services:
buildbot:
image: "buildbot-master-parrrate"
build: ../master
image: buildbot/buildbot-master:master
env_file:
- db.env
- gitea.env
- db.env
environment:
- BUILDBOT_CONFIG_DIR=config
- BUILDBOT_CONFIG_URL=https://gitea.parrrate.ru/PTV/buildbot/archive/master.tar.gz
- BUILDBOT_WORKER_PORT=9989
- BUILDBOT_WEB_URL=https://buildbot.parrrate.ru/
- BUILDBOT_WEB_PORT=tcp:port=8010
- BUILDBOT_CONFIG_DIR=config
- BUILDBOT_CONFIG_URL=https://github.com/buildbot/buildbot-docker-example-config/archive/master.tar.gz
- BUILDBOT_WORKER_PORT=9989
- BUILDBOT_WEB_URL=http://localhost:8080/
- BUILDBOT_WEB_PORT=8080
links:
- db
depends_on:
- db
ports:
- "8010:8010"
networks:
- v6d
- buildbot-db
- buildbot-worker
volumes:
- ../data/docs:/buildbot_share/docs/:rw
deploy:
restart_policy:
condition: unless-stopped
delay: 30s
window: 300s
- "8080:8080"
db:
env_file:
- db.env
- db.env
image: "postgres:9.4"
expose:
- 5432
networks:
- buildbot-db
volumes:
- ../data/db:/var/lib/postgresql/data
deploy:
restart_policy:
condition: unless-stopped
delay: 30s
window: 300s
- 5432
rust-worker-1-65:
build:
context: ../worker
args:
- RUST_VERSION_ARG=1.65
worker:
image: "buildbot/buildbot-worker:master"
environment:
BUILDMASTER: buildbot
BUILDMASTER_PORT: 9989
WORKERNAME: worker-rust-1-65
WORKERPASS: pass
WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
networks:
- buildbot-worker
deploy:
restart_policy:
condition: unless-stopped
delay: 30s
window: 300s
BUILDMASTER: buildbot
BUILDMASTER_PORT: 9989
WORKERNAME: example-worker
WORKERPASS: pass
WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
rust-worker-1-72:
build:
context: ../worker
args:
- RUST_VERSION_ARG=1.72
environment:
BUILDMASTER: buildbot
BUILDMASTER_PORT: 9989
WORKERNAME: worker-rust-1-72
WORKERPASS: pass
WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
networks:
- buildbot-worker
deploy:
restart_policy:
condition: unless-stopped
delay: 30s
window: 300s
rust-worker-mdbook:
build:
context: ../worker
dockerfile: Dockerfile.mdbook
environment:
BUILDMASTER: buildbot
BUILDMASTER_PORT: 9989
WORKERNAME: worker-rust-mdbook
WORKERPASS: pass
WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
networks:
- buildbot-worker
deploy:
restart_policy:
condition: unless-stopped
delay: 30s
window: 300s
nginx:
container_name: buildbot-nginx
build:
context: .
dockerfile: Dockerfile.Nginx
networks:
v6d: {}
volumes:
- ../data/docs:/buildbot_share/docs/:ro
deploy:
restart_policy:
condition: unless-stopped
delay: 30s
window: 300s
links:
- buildbot

View File

@ -1,22 +0,0 @@
server {
listen 80;
listen [::]:80;
server_name radn.parrrate.ru;
location /docs {
root /buildbot_share;
index index.html index.htm;
autoindex on;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
server {
listen 80;
listen [::]:80;
server_name *.parrrate.ru;
}

View File

@ -1,12 +0,0 @@
ARG RUST_VERSION_ARG=1.65
FROM rust:${RUST_VERSION_ARG}
COPY worker-setup.sh worker-setup.sh
RUN bash worker-setup.sh
USER buildbot
WORKDIR /buildbot
COPY worker-setup-user.sh worker-setup-user.sh
RUN bash worker-setup-user.sh
COPY buildbot.tac /buildbot/buildbot.tac
RUN rustup component add clippy
RUN rustup component add rustfmt
CMD ["/usr/bin/dumb-init", "/buildbot/venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]

View File

@ -1,10 +0,0 @@
FROM rust:1.79
COPY worker-setup.sh worker-setup.sh
RUN bash worker-setup.sh
USER buildbot
WORKDIR /buildbot
COPY worker-setup-user.sh worker-setup-user.sh
RUN bash worker-setup-user.sh
COPY buildbot.tac /buildbot/buildbot.tac
RUN cargo install mdbook
CMD ["/usr/bin/dumb-init", "/buildbot/venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]

View File

@ -1,48 +0,0 @@
import fnmatch
import os
import sys
from buildbot_worker.bot import Worker
from twisted.application import service
from twisted.python.log import FileLogObserver, ILogObserver
# setup worker
basedir = os.environ.get("BUILDBOT_BASEDIR", os.path.abspath(os.path.dirname(__file__)))
application = service.Application("buildbot-worker")
application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
# and worker on the same process!
buildmaster_host = os.environ.get("BUILDMASTER", "localhost")
port = int(os.environ.get("BUILDMASTER_PORT", 9989))
workername = os.environ.get("WORKERNAME", "docker")
passwd = os.environ.get("WORKERPASS")
# delete the password from the environ so that it is not leaked in the log
blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split()
for name in list(os.environ.keys()):
for toremove in blacklist:
if fnmatch.fnmatch(name, toremove):
del os.environ[name]
keepalive = 600
umask = None
maxdelay = 300
allow_shutdown = None
maxretries = 10
delete_leftover_dirs = False
s = Worker(
buildmaster_host,
port,
workername,
passwd,
basedir,
keepalive,
umask=umask,
maxdelay=maxdelay,
allow_shutdown=allow_shutdown,
maxRetries=maxretries,
delete_leftover_dirs=delete_leftover_dirs,
)
s.setServiceParent(application)

View File

@ -1,3 +0,0 @@
python3 -m venv /buildbot/venv/
/buildbot/venv/bin/pip3 --no-cache-dir install 'twisted[tls]'
/buildbot/venv/bin/pip3 --no-cache-dir install buildbot-worker

View File

@ -1,17 +0,0 @@
apt-get update \
&& apt-get -y upgrade \
&& apt-get -y install -q \
build-essential \
git \
libffi-dev \
libssl-dev \
python3-dev \
python3-setuptools \
python3-pip \
python3-venv \
dumb-init \
curl \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir /buildbot \
&& useradd -ms /bin/bash buildbot \
&& chown -R buildbot /buildbot