remove compose version

build and css

.gitignore

@@ -0,0 +1,3 @@
+/venv
+gitea.env
+/data

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.analysis.typeCheckingMode": "basic"
+}

README.md

@@ -0,0 +1,2 @@
+# buildbot-docker-example-config
+example configuration for running buildbot in docker

master.cfg

@@ -3,7 +3,8 @@
 
 import os
 
-from buildbot.plugins import *
+from buildbot.plugins import changes, reporters, schedulers, steps, util, worker
+from buildbot.www.authz.roles import RolesFromBase
 
 # This is a sample buildmaster config file. It must be installed as
 # 'master.cfg' in your buildmaster's base directory.
@@ -12,55 +13,67 @@ from buildbot.plugins import *
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
 
+GITEA_SECRET = os.environ.get("GITEA_SECRET")
+GITEA_TOKEN = os.environ.get("GITEA_TOKEN")
+
+CLIENT_ID = os.environ.get("GITEA_CLIENT_ID")
+assert CLIENT_ID
+CLIENT_SECRET = os.environ.get("GITEA_CLIENT_SECRET")
+assert CLIENT_SECRET
+
 ####### WORKERS
 
 # The 'workers' list defines the set of recognized workers. Each element is
 # a Worker object, specifying a unique worker name and password.  The same
 # worker name and password must be configured on the worker.
 
-c['workers'] = [worker.Worker("example-worker", 'pass')]
+c["workers"] = [
+    worker.Worker("worker-rust-1-65", "pass", properties={"rust_version": "1.65"}),
+    worker.Worker("worker-rust-1-72", "pass", properties={"rust_version": "1.72"}),
+    worker.Worker("worker-rust-mdbook", "pass"),
+]
+rust_workers_1_65 = ["worker-rust-1-65"]
+rust_workers_1_72 = ["worker-rust-1-72"]
+rust_workers_mdbook = ["worker-rust-mdbook"]
 
-if 'BUILDBOT_MQ_URL' in os.environ:
+if "BUILDBOT_MQ_URL" in os.environ:
-    c['mq'] = {
+    c["mq"] = {
-        'type' : 'wamp',
+        "type": "wamp",
-        'router_url': os.environ['BUILDBOT_MQ_URL'],
+        "router_url": os.environ["BUILDBOT_MQ_URL"],
-        'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'),
+        "realm": os.environ.get("BUILDBOT_MQ_REALM", "buildbot"),
-        'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+        "debug": "BUILDBOT_MQ_DEBUG" in os.environ,
-        'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+        "debug_websockets": "BUILDBOT_MQ_DEBUG" in os.environ,
-        'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+        "debug_lowlevel": "BUILDBOT_MQ_DEBUG" in os.environ,
     }
 # 'protocols' contains information about protocols which master will use for
 # communicating with workers. You must define at least 'port' option that workers
 # could connect to your master with this protocol.
 # 'port' must match the value configured into the workers (with their
 # --master option)
-c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
+c["protocols"] = {"pb": {"port": os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
 
 ####### CHANGESOURCES
 
 # the 'change_source' setting tells the buildmaster how it should find out
-# about source code changes.  Here we point to the buildbot clone of pyflakes.
+# about source code changes.  Here we point to the buildbot clone of radn.
 
-c['change_source'] = []
+c["change_source"] = []
-c['change_source'].append(changes.GitPoller(
+c["change_source"].append(
-        'git://github.com/buildbot/pyflakes.git',
+    changes.GitPoller(
-        workdir='gitpoller-workdir', branch='master',
+        "https://gitea.parrrate.ru/PTV/radn-rs.git",
-        pollinterval=300))
+        workdir="gitpoller/radn-rs",
-
+        branch="main",
-####### SCHEDULERS
+        pollinterval=300,
-
+    )
-# Configure the Schedulers, which decide how to react to incoming changes.  In this
+)
-# case, just kick off a 'runtests' build
+c["change_source"].append(
-
+    changes.GitPoller(
-c['schedulers'] = []
+        "https://gitea.parrrate.ru/PTV/exercises.git",
-c['schedulers'].append(schedulers.SingleBranchScheduler(
+        workdir="gitpoller/exercises",
-                            name="all",
+        branch="latest",
-                            change_filter=util.ChangeFilter(branch='master'),
+        pollinterval=300,
-                            treeStableTimer=None,
+    )
-                            builderNames=["runtests"]))
+)
-c['schedulers'].append(schedulers.ForceScheduler(
-                            name="force",
-                            builderNames=["runtests"]))
 
 ####### BUILDERS
 
@@ -68,49 +81,258 @@ c['schedulers'].append(schedulers.ForceScheduler(
 # what steps, and which workers can execute them.  Note that any particular build will
 # only take place on one worker.
 
+# all_repositories = {
+#     r"https://gitea.parrrate.ru/PTV/radn-rs.git": "radn-rs",
+# }
+
+
+# def codebaseGenerator(chdict):
+#     return all_repositories.get(chdict["repository"])
+
+
+# c["codebaseGenerator"] = codebaseGenerator
+
+c["builders"] = []
+
+radn_builders = []
+exercises_builders = []
+
+CARGO_TARGET_DIR = "/buildbot/_rust/radn-rs/target"
+RADN_ENV = {"CARGO_TARGET_DIR": CARGO_TARGET_DIR}
+DOC_DIR = f"{CARGO_TARGET_DIR}/doc"
+
+
+def radn_rs_factory():
     factory = util.BuildFactory()
-# check out the source
+    factory.addStep(
-factory.addStep(steps.Git(repourl='http://github.com/buildbot/pyflakes.git', mode='incremental'))
+        steps.Git(
-# run the tests (note that this will require that 'trial' is installed)
+            repourl="https://gitea.parrrate.ru/PTV/radn-rs.git", mode="incremental"
-factory.addStep(steps.ShellCommand(command=["trial", "pyflakes"]))
+        )
+    )
+    return factory
 
-c['builders'] = []
-c['builders'].append(
-    util.BuilderConfig(name="runtests",
-      workernames=["example-worker"],
-      factory=factory))
 
-####### STATUS TARGETS
+def exercises_factory():
+    factory = util.BuildFactory()
+    factory.addStep(
+        steps.Git(
+            repourl="https://gitea.parrrate.ru/PTV/exercises.git", mode="incremental"
+        )
+    )
+    return factory
 
-# 'status' is a list of Status Targets. The results of each build will be
+
-# pushed to these targets. buildbot/status/*.py has a variety to choose from,
+def radn_append_factory(factory, name: str, workernames: list[str]):
+    c["builders"].append(
+        util.BuilderConfig(name=name, workernames=workernames, factory=factory)
+    )
+    radn_builders.append(name)
+
+
+def exercises_append_factory(factory, name: str, workernames: list[str]):
+    c["builders"].append(
+        util.BuilderConfig(name=name, workernames=workernames, factory=factory)
+    )
+    exercises_builders.append(name)
+
+
+def cargo_test(name: str, workernames: list[str]):
+    factory = radn_rs_factory()
+    factory.addStep(
+        steps.ShellCommand(
+            command=["cargo", "test", "--workspace"],
+            env=RADN_ENV,
+        )
+    )
+    radn_append_factory(factory, name, workernames)
+
+
+def cargo_clippy(name: str, workernames: list[str]):
+    factory = radn_rs_factory()
+    factory.addStep(
+        steps.ShellCommand(
+            command=[
+                "cargo",
+                "clippy",
+                "--workspace",
+                "--examples",
+                "--tests",
+                "--",
+                "--deny=warnings",
+            ],
+            env=RADN_ENV,
+        )
+    )
+    radn_append_factory(factory, name, workernames)
+
+
+def cargo_fmt(name: str, workernames: list[str]):
+    factory = radn_rs_factory()
+    factory.addStep(
+        steps.ShellCommand(
+            command=[
+                "cargo",
+                "fmt",
+                "--check",
+                "--all",
+            ],
+            env=RADN_ENV,
+        )
+    )
+    radn_append_factory(factory, name, workernames)
+
+
+def cargo_doc(name: str, workernames: list[str], specific: bool, latest: bool):
+    factory = radn_rs_factory()
+    factory.addStep(
+        steps.ShellCommand(
+            command=[
+                "cargo",
+                "doc",
+                "--workspace",
+                "--no-deps",
+            ],
+            env=RADN_ENV,
+        )
+    )
+    if specific:
+        factory.addStep(
+            steps.DirectoryUpload(
+                workersrc=DOC_DIR,
+                masterdest=util.Interpolate(
+                    "/buildbot_share/docs/radn-rs/%(prop:rust_version)s/"
+                ),
+                url=util.Interpolate(
+                    "https://radn.parrrate.ru/latest/docs/%(prop:rust_version)s/radn/"
+                ),
+                compress="gz",
+            )
+        )
+    if latest:
+        factory.addStep(
+            steps.DirectoryUpload(
+                workersrc=DOC_DIR,
+                masterdest="/buildbot_share/docs/radn-rs/",
+                url="https://radn.parrrate.ru/latest/docs/radn/",
+                compress="gz",
+            )
+        )
+    radn_append_factory(factory, name, workernames)
+
+
+def mdbook_test(name: str, workernames: list[str]):
+    factory = exercises_factory()
+    factory.addStep(
+        steps.ShellCommand(
+            command=["mdbook", "test", "."],
+        )
+    )
+    exercises_append_factory(factory, name, workernames)
+
+
+cargo_test("cargo test (1.65)", rust_workers_1_65)
+cargo_clippy("cargo clippy (1.65)", rust_workers_1_65)
+cargo_clippy("cargo clippy (1.72)", rust_workers_1_72)
+cargo_fmt("cargo fmt (1.72)", rust_workers_1_72)
+cargo_doc("cargo doc (1.72)", rust_workers_1_72, False, True)
+mdbook_test("mdbook test", rust_workers_mdbook)
+
+####### SCHEDULERS
+
+# Configure the Schedulers, which decide how to react to incoming changes.
+
+c["schedulers"] = []
+c["schedulers"].append(
+    schedulers.SingleBranchScheduler(
+        name="all-radn",
+        change_filter=util.ChangeFilter(branch="main"),
+        treeStableTimer=None,
+        builderNames=radn_builders,
+    )
+)
+c["schedulers"].append(
+    schedulers.SingleBranchScheduler(
+        name="all-exercises",
+        change_filter=util.ChangeFilter(branch="latest"),
+        treeStableTimer=None,
+        builderNames=exercises_builders,
+    )
+)
+c["schedulers"].append(
+    schedulers.ForceScheduler(name="force-radn", builderNames=radn_builders)
+)
+c["schedulers"].append(
+    schedulers.ForceScheduler(name="force-exercises", builderNames=exercises_builders)
+)
+
+####### REPORTER TARGETS
+
+# 'services' is a list of Reporter Targets. The results of each build will be
+# pushed to these targets. buildbot/reporters/*.py has a variety to choose from,
 # like IRC bots.
 
-c['status'] = []
+c["services"] = []
 
 ####### PROJECT IDENTITY
 
 # the 'title' string will appear at the top of this buildbot installation's
 # home pages (linked to the 'titleURL').
 
-c['title'] = "Pyflakes"
+c["title"] = "RADN"
-c['titleURL'] = "https://launchpad.net/pyflakes"
+c["titleURL"] = "https://gitea.parrrate.ru/PTV/radn-rs"
 
 # the 'buildbotURL' string should point to the location where the buildbot's
 # internal web server is visible. This typically uses the port number set in
 # the 'www' entry below, but with an externally-visible host name which the
 # buildbot cannot figure out without some help.
 
-c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
+c["buildbotURL"] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
+
+
+class RolesFromCustom(RolesFromBase):
+    def getRolesFromUser(self, userDetails: dict):
+        if userDetails.get("is_admin") is True:
+            return ["admins"]
+        else:
+            return []
+
 
 # minimalistic config to activate new web UI
-c['www'] = dict(port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
+c["www"] = {
-                plugins=dict(waterfall_view={}, console_view={}))
+    "port": os.environ.get("BUILDBOT_WEB_PORT", 8010),
+    "plugins": {
+        "waterfall_view": {},
+        "console_view": {},
+    },
+    "change_hook_dialects": {},
+    "auth": util.GiteaAuth(
+        endpoint="https://gitea.parrrate.ru",
+        client_id=CLIENT_ID,
+        client_secret=CLIENT_SECRET,
+    ),
+    "authz": util.Authz(
+        allowRules=[util.AnyControlEndpointMatcher(role="admins")],
+        roleMatchers=[RolesFromCustom()],
+    ),
+}
+if GITEA_SECRET:
+    c["www"]["change_hook_dialects"]["gitea"] = {
+        "secret": GITEA_SECRET,
+        "onlyIncludePushCommit": True,
+    }
+if GITEA_TOKEN:
+    c["services"].append(
+        reporters.GiteaStatusPush(
+            "https://gitea.parrrate.ru", token=GITEA_TOKEN, verbose=True
+        ),
+    )
 
 ####### DB URL
 
-c['db'] = {
+c["db"] = {
     # This specifies what database buildbot uses to store its state.  You can leave
     # this at its default for all but the largest installations.
-    'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ),
+    "db_url": os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ),
 }
+
+c["buildbotNetUsageData"] = None

master/Dockerfile

@@ -0,0 +1,2 @@
+FROM buildbot/buildbot-master:v3.11.5
+RUN /buildbot_venv/bin/pip3 install buildbot_gitea

multimaster/docker-compose.yml

@@ -19,7 +19,7 @@ services:
     env_file: db.env
     environment:
         - BUILDBOT_CONFIG_DIR=config
-        - BUILDBOT_CONFIG_URL=https://github.com/buildbot/buildbot-docker-example-config/archive/master.tar.gz
+        - BUILDBOT_CONFIG_URL=https://gitea.parrrate.ru/PTV/buildbot/archive/master.tar.gz
         - BUILDBOT_WORKER_PORT=9989
         - BUILDBOT_WEB_URL=http://localhost:8080/
         - BUILDBOT_WEB_PORT=8080

pypy/master/Dockerfile

@@ -1,2 +1,2 @@
-from pypy:2-onbuild
+FROM pypy:3-onbuild
 CMD /usr/src/app/start_buildbot.sh

pypy/master/requirements.txt

@@ -1,2 +1,2 @@
-buildbot==0.8.14
+buildbot[bundle]
 psycopg2cffi-compat

pypy/worker/Dockerfile

@@ -1,2 +1,2 @@
-from pypy:2-onbuild
+FROM pypy:2-onbuild
 CMD ["twistd", "-ny", "buildbot.tac"]

simple/Dockerfile.Nginx

@@ -0,0 +1,2 @@
+FROM nginx
+COPY nginx-default.conf /etc/nginx/conf.d/default.conf

simple/docker-compose.yml

@@ -1,36 +1,122 @@
-version: '2'
+networks:
+  v6d:
+    external: true
+  buildbot-db: {}
+  buildbot-worker: {}
+  buildbot-docs: {}
+
 services:
   buildbot:
-    image: buildbot/buildbot-master:master
+    image: "buildbot-master-parrrate"
+    build: ../master
     env_file:
       - db.env
+      - gitea.env
     environment:
       - BUILDBOT_CONFIG_DIR=config
-        - BUILDBOT_CONFIG_URL=https://github.com/buildbot/buildbot-docker-example-config/archive/master.tar.gz
+      - BUILDBOT_CONFIG_URL=https://gitea.parrrate.ru/PTV/buildbot/archive/master.tar.gz
       - BUILDBOT_WORKER_PORT=9989
-        - BUILDBOT_WEB_URL=http://localhost:8080/
+      - BUILDBOT_WEB_URL=https://buildbot.parrrate.ru/
-        - BUILDBOT_WEB_PORT=8080
+      - BUILDBOT_WEB_PORT=tcp:port=8010
-    links:
-      - db
     depends_on:
       - db
     ports:
-      - "8080:8080"
+      - "8010:8010"
+    networks:
+      - v6d
+      - buildbot-db
+      - buildbot-worker
+    volumes:
+      - ../data/docs:/buildbot_share/docs/:rw
+    deploy:
+      restart_policy:
+        condition: unless-stopped
+        delay: 30s
+        window: 300s
+
   db:
     env_file:
       - db.env
     image: "postgres:9.4"
     expose:
       - 5432
+    networks:
+      - buildbot-db
+    volumes:
+      - ../data/db:/var/lib/postgresql/data
+    deploy:
+      restart_policy:
+        condition: unless-stopped
+        delay: 30s
+        window: 300s
 
-  worker:
+  rust-worker-1-65:
-    image: "buildbot/buildbot-worker:master"
+    build:
+      context: ../worker
+      args:
+        - RUST_VERSION_ARG=1.65
     environment:
       BUILDMASTER: buildbot
       BUILDMASTER_PORT: 9989
-        WORKERNAME: example-worker
+      WORKERNAME: worker-rust-1-65
       WORKERPASS: pass
       WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
+    networks:
+      - buildbot-worker
+    deploy:
+      restart_policy:
+        condition: unless-stopped
+        delay: 30s
+        window: 300s
 
-    links:
+  rust-worker-1-72:
-        - buildbot
+    build:
+      context: ../worker
+      args:
+        - RUST_VERSION_ARG=1.72
+    environment:
+      BUILDMASTER: buildbot
+      BUILDMASTER_PORT: 9989
+      WORKERNAME: worker-rust-1-72
+      WORKERPASS: pass
+      WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
+    networks:
+      - buildbot-worker
+    deploy:
+      restart_policy:
+        condition: unless-stopped
+        delay: 30s
+        window: 300s
+
+  rust-worker-mdbook:
+    build:
+      context: ../worker
+      dockerfile: Dockerfile.mdbook
+    environment:
+      BUILDMASTER: buildbot
+      BUILDMASTER_PORT: 9989
+      WORKERNAME: worker-rust-mdbook
+      WORKERPASS: pass
+      WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
+    networks:
+      - buildbot-worker
+    deploy:
+      restart_policy:
+        condition: unless-stopped
+        delay: 30s
+        window: 300s
+
+  nginx:
+    container_name: buildbot-nginx
+    build:
+      context: .
+      dockerfile: Dockerfile.Nginx
+    networks:
+      v6d: {}
+    volumes:
+      - ../data/docs:/buildbot_share/docs/:ro
+    deploy:
+      restart_policy:
+        condition: unless-stopped
+        delay: 30s
+        window: 300s

simple/nginx-default.conf

@@ -0,0 +1,22 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  radn.parrrate.ru;
+
+    location /docs {
+        root           /buildbot_share;
+        index     index.html index.htm;
+        autoindex                   on;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}
+
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  *.parrrate.ru;
+}

worker/Dockerfile

@@ -0,0 +1,12 @@
+ARG RUST_VERSION_ARG=1.65
+FROM rust:${RUST_VERSION_ARG}
+COPY worker-setup.sh worker-setup.sh
+RUN bash worker-setup.sh
+USER buildbot
+WORKDIR /buildbot
+COPY worker-setup-user.sh worker-setup-user.sh
+RUN bash worker-setup-user.sh
+COPY buildbot.tac /buildbot/buildbot.tac
+RUN rustup component add clippy
+RUN rustup component add rustfmt
+CMD ["/usr/bin/dumb-init", "/buildbot/venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]

worker/Dockerfile.mdbook

@@ -0,0 +1,10 @@
+FROM rust:1.79
+COPY worker-setup.sh worker-setup.sh
+RUN bash worker-setup.sh
+USER buildbot
+WORKDIR /buildbot
+COPY worker-setup-user.sh worker-setup-user.sh
+RUN bash worker-setup-user.sh
+COPY buildbot.tac /buildbot/buildbot.tac
+RUN cargo install mdbook
+CMD ["/usr/bin/dumb-init", "/buildbot/venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]

worker/buildbot.tac

@@ -0,0 +1,48 @@
+import fnmatch
+import os
+import sys
+
+from buildbot_worker.bot import Worker
+from twisted.application import service
+from twisted.python.log import FileLogObserver, ILogObserver
+
+# setup worker
+basedir = os.environ.get("BUILDBOT_BASEDIR", os.path.abspath(os.path.dirname(__file__)))
+application = service.Application("buildbot-worker")
+
+
+application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
+# and worker on the same process!
+buildmaster_host = os.environ.get("BUILDMASTER", "localhost")
+port = int(os.environ.get("BUILDMASTER_PORT", 9989))
+workername = os.environ.get("WORKERNAME", "docker")
+passwd = os.environ.get("WORKERPASS")
+
+# delete the password from the environ so that it is not leaked in the log
+blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split()
+for name in list(os.environ.keys()):
+    for toremove in blacklist:
+        if fnmatch.fnmatch(name, toremove):
+            del os.environ[name]
+
+keepalive = 600
+umask = None
+maxdelay = 300
+allow_shutdown = None
+maxretries = 10
+delete_leftover_dirs = False
+
+s = Worker(
+    buildmaster_host,
+    port,
+    workername,
+    passwd,
+    basedir,
+    keepalive,
+    umask=umask,
+    maxdelay=maxdelay,
+    allow_shutdown=allow_shutdown,
+    maxRetries=maxretries,
+    delete_leftover_dirs=delete_leftover_dirs,
+)
+s.setServiceParent(application)

worker/worker-setup-user.sh

@@ -0,0 +1,3 @@
+python3 -m venv /buildbot/venv/
+/buildbot/venv/bin/pip3 --no-cache-dir install 'twisted[tls]'
+/buildbot/venv/bin/pip3 --no-cache-dir install buildbot-worker

worker/worker-setup.sh

@@ -0,0 +1,17 @@
+apt-get update \
+&& apt-get -y upgrade \
+&& apt-get -y install -q \
+    build-essential \
+    git \
+    libffi-dev \
+    libssl-dev \
+    python3-dev \
+    python3-setuptools \
+    python3-pip \
+    python3-venv \
+    dumb-init \
+    curl \
+&& rm -rf /var/lib/apt/lists/* \
+&& mkdir /buildbot \
+&& useradd -ms /bin/bash buildbot \
+&& chown -R buildbot /buildbot