multi-stage build for metrics

.vscode/settings.json

@@ -1,5 +1,8 @@
 {
     "python.analysis.typeCheckingMode": "basic",
     "python.formatting.blackArgs": ["--line-length", "120"],
-    "isort.args": ["--profile", "black"]
+    "isort.args": ["--profile", "black"],
+    "python.analysis.extraPaths": [
+        "xmetrics"
+    ]
 }

README.md

@@ -1,6 +1,7 @@
 ```sh
 cd /code/
 git clone --recurse-submodules https://gitea.parrrate.ru/PTV/radn-rs.git
+git clone --recurse-submodules https://gitea.parrrate.ru/PTV/exercises.git
 ```
 ```sh
 clear && docker compose up -d --build metrics && docker cp radn-metrics:/code/metrics.png ./metrics/metrics.png

docker-compose.yml

@@ -7,6 +7,7 @@ volumes:
   radn-rs: {}
   book-monads: {}
   book-radn: {}
+  exercises: {}
 
 services:
   radn-rs-dev:
@@ -57,6 +58,21 @@ services:
       radn: {}
     tty: true
     stop_signal: SIGKILL
+  exercises:
+    container_name: exercises
+    build:
+      context: radn-rs
+      dockerfile: Dockerfile.Book.Exercises
+    environment:
+      MDBOOK_OUTPUT__HTML__GIT_REPOSITORY_URL: "https://gitea.parrrate.ru/PTV/exercises"
+      MDBOOK_OUTPUT__HTML__EDIT_URL_TEMPLATE: "https://gitea.parrrate.ru/PTV/exercises/_edit/latest/{path}"
+    volumes:
+      - radn-rs:/code/:ro
+      - exercises:/data/book/:rw
+    networks:
+      radn: {}
+    tty: true
+    stop_signal: SIGKILL
   nginx:
     container_name: radn-nginx
     build:

metrics/Dockerfile

@@ -17,10 +17,13 @@ RUN python3 /code/metrics.py
 RUN git fetch && git checkout ce65688e47b07f14ef2861971f64296c2197e778
 RUN python3 /code/metrics.py
 
-
 RUN git fetch && git checkout 7f1d72898ddd9ab40bf91e553f70a023a64fe647
 RUN python3 /code/metrics.py
 
+
+RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48
+RUN python3 /code/metrics.py
+
 FROM python:3.11
 RUN python3 -m pip install matplotlib
 WORKDIR /code/

nginx-default.conf

@@ -56,6 +56,22 @@ server {
     }
 }
 
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  exercises.parrrate.ru;
+
+    location / {
+        proxy_pass http://exercises/;
+
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+
+        proxy_buffering off;
+    }
+}
+
 server {
     listen       80;
     listen  [::]:80;

radn-rs/Dockerfile

@@ -5,7 +5,6 @@ RUN rustup component add rustfmt
 RUN rustup component add clippy
 RUN cargo install mdbook
 RUN cargo install rust-code-analysis-cli
-RUN apt-get update
+RUN apt-get update && install -y cloc
-RUN apt-get install -y cloc
 
 WORKDIR /code/

radn-rs/Dockerfile.Book.Exercises

@@ -0,0 +1,7 @@
+FROM rust:1.69
+
+RUN cargo install mdbook
+
+WORKDIR /code/exercises/
+
+CMD [ "mdbook", "serve", "--dest-dir", "/data/book/", "--port", "80", "--hostname", "0.0.0.0" ]

xmetrics/Dockerfile

@@ -11,7 +11,7 @@ ENV SRCPATTERN="*.cs"
 FROM metrics-base as metrics-radn
 RUN git clone https://gitea.parrrate.ru/PTV/radn-rs.git
 WORKDIR /code/radn-rs/
-RUN git fetch && git checkout 7f1d72898ddd9ab40bf91e553f70a023a64fe647
+RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48
 
 FROM metrics-base as metrics-mdbook
 RUN git clone https://github.com/rust-lang/mdBook.git
@@ -47,15 +47,89 @@ RUN git fetch && git checkout f81543a34ee363dcc00e8632fd7cfcd4a3478b23
 ENV SRCDIR="."
 ENV SRCPATTERN="*.[ach]*"
 
-FROM metrics-radn as metrics
+FROM metrics-radn as metrics-repo
+
+FROM metrics-repo as metrics-commits
+COPY common.py /code/common.py
+COPY commits.py /code/commits.py
+RUN python3 /code/commits.py
+
+FROM metrics-repo as metrics
+COPY common.py /code/common.py
 COPY metrics.py /code/metrics.py
+COPY --from=metrics-commits /code/commits.dat commits.dat
 RUN python3 /code/metrics.py
 
-FROM python:3.11
+FROM python:3.11 as metrics-process
-RUN python3 -m pip install matplotlib
+RUN python3 -m pip install numpy
-RUN python3 -m pip install scipy
 WORKDIR /code/
-COPY --from=metrics /code/metrics.json metrics.json
+
+FROM metrics-process as metrics-data
+COPY --from=metrics /code/metrics.dat metrics.dat
+
+FROM metrics-process as metrics-plot
+RUN python3 -m pip install matplotlib
+
+FROM metrics-data as metrics-entries
+COPY entries.py entries.py
+RUN python3 entries.py
+
+FROM metrics-data as metrics-y
+COPY cors.py cors.py
+RUN python3 cors.py
+
+FROM metrics-plot as metrics-c
+COPY --from=metrics-y /code/Y.dat Y.dat
+COPY k2c.py k2c.py
+
+FROM metrics-c as metrics-c-linear
+COPY c_linear.py c_linear.py
+RUN python3 c_linear.py
+
+FROM metrics-c as metrics-c-proportional
+COPY c_proportional.py c_proportional.py
+RUN python3 c_proportional.py
+
+FROM metrics-process as metrics-x-linear
+COPY --from=metrics-y /code/Y.dat Y.dat
+COPY x_linear.py x_linear.py
+RUN python3 x_linear.py
+
+FROM metrics-process as metrics-x-proportional
+COPY --from=metrics-y /code/Y.dat Y.dat
+COPY x_proportional.py x_proportional.py
+RUN python3 x_proportional.py
+
+FROM metrics-plot as metrics-render
 RUN mkdir /code/metrics/
-COPY render.py render.py
+
-RUN python3 render.py
+FROM metrics-render as metrics-render-ploc
+COPY --from=metrics-entries /code/entries.dat entries.dat
+COPY render_ploc.py render_ploc.py
+RUN python3 render_ploc.py
+
+FROM metrics-render as metrics-render-hist
+COPY --from=metrics-y /code/Y.dat Y.dat
+COPY render_hist.py render_hist.py
+
+FROM metrics-render-hist as metrics-render-01x
+COPY --from=metrics-x-linear /code/X.dat X.dat
+COPY --from=metrics-c-linear /code/C.dat C.dat
+RUN python3 render_hist.py
+
+FROM metrics-render-hist as metrics-render-02y
+COPY --from=metrics-x-linear /code/X.dat X.dat
+COPY --from=metrics-c-proportional /code/C.dat C.dat
+RUN python3 render_hist.py
+
+FROM metrics-render-hist as metrics-render-03xy
+COPY --from=metrics-x-proportional /code/X.dat X.dat
+COPY --from=metrics-c-proportional /code/C.dat C.dat
+RUN python3 render_hist.py
+
+FROM metrics-render
+COPY --from=metrics-y /code/Y.dat Y.dat
+COPY --from=metrics-render-ploc /code/metrics/out.png /code/metrics/metrics.png
+COPY --from=metrics-render-01x /code/metrics/out.png /code/metrics/metrics-01x.png
+COPY --from=metrics-render-02y /code/metrics/out.png /code/metrics/metrics-02y.png
+COPY --from=metrics-render-03xy /code/metrics/out.png /code/metrics/metrics-03xy.png

xmetrics/c_linear.py

@@ -0,0 +1,11 @@
+import pickle
+
+import numpy as np
+from k2c import k2c
+
+with open("Y.dat", "rb") as file:
+    Y = pickle.load(file)
+M = Y.shape[0]
+K = np.arange(M) / M
+with open("C.dat", "wb") as file:
+    pickle.dump(k2c(K), file)

xmetrics/c_proportional.py

@@ -0,0 +1,13 @@
+import pickle
+
+import numpy as np
+from k2c import k2c
+
+with open("Y.dat", "rb") as file:
+    Y = pickle.load(file)
+M = Y.shape[0]
+W = Y.max(axis=1)
+I = W.cumsum()
+K = (I - W / 2) / I.max()
+with open("C.dat", "wb") as file:
+    pickle.dump(k2c(K), file)

xmetrics/commits.py

@@ -0,0 +1,41 @@
+import pickle
+from subprocess import check_output
+
+from common import counter
+
+args = []
+# args = ["--topo-order"]
+original_commits = check_output(["git", "log", "--pretty=%H", *args], text=True).splitlines()
+N = len(original_commits)
+filtered_commits = set(original_commits[:: N // min(N, 4096)])
+total_changes: dict[str, int] = {}
+chosen_parents: dict[str, str | None] = {}
+for i, commit in enumerate(reversed(original_commits)):
+    print(f"P={i / N:6f}", flush=True)
+    parents = check_output(["git", "log", "--pretty=%P", "-n", "1", commit], text=True).split()
+    chosen_parent = None
+    if commit in filtered_commits:
+        ctr = counter(commit, True)
+        changes = ctr.total()
+        for parent in parents:
+            pctr = counter(parent, True)
+            maybe_changes = (pctr - ctr).total() + (ctr - pctr).total() + total_changes[parent]
+            if maybe_changes > changes:
+                changes = maybe_changes
+                chosen_parent = parent
+    else:
+        changes = 0
+        for parent in parents:
+            maybe_changes = total_changes[parent]
+            if maybe_changes > changes:
+                changes = maybe_changes
+                chosen_parent = parent
+    total_changes[commit] = changes
+    chosen_parents[commit] = chosen_parent
+commit = original_commits[0]
+commits = []
+while commit is not None:
+    commits.append(commit)
+    commit = chosen_parents[commit]
+with open("/code/commits.dat", "wb") as file:
+    pickle.dump(commits, file)

xmetrics/common.py

@@ -0,0 +1,23 @@
+import os
+from collections import Counter
+from functools import lru_cache
+from pathlib import Path
+from subprocess import check_output
+
+
+@lru_cache()
+def _counter(commit: str, unique: bool) -> Counter:
+    check_output(["git", "checkout", commit], text=True)
+    ctr = Counter()
+    for path in Path(os.getenv("SRCDIR", "src")).rglob(os.getenv("SRCPATTERN", "*.rs")):
+        lines = path.read_bytes().splitlines()
+        lines = (line.strip() for line in lines)
+        lines = (line for line in lines if line)
+        if unique:
+            lines = set(lines)
+        ctr.update(lines)
+    return ctr
+
+
+def counter(commit: str, unique: bool) -> Counter:
+    return _counter(commit, unique)

xmetrics/cors.py

@@ -0,0 +1,10 @@
+import pickle
+
+import numpy as np
+
+with open("metrics.dat", "rb") as file:
+    metrics = pickle.load(file)
+cors = metrics["cors"]
+Y = np.array(cors)
+with open("Y.dat", "wb") as file:
+    pickle.dump(Y, file)

xmetrics/entries.py

@@ -0,0 +1,9 @@
+import pickle
+
+import numpy as np
+
+with open("metrics.dat", "rb") as file:
+    metrics = pickle.load(file)
+entries = metrics["entries"]
+with open("entries.dat", "wb") as file:
+    pickle.dump(np.array(entries).transpose(), file)

xmetrics/k2c.py

@@ -0,0 +1,6 @@
+import numpy as np
+from matplotlib.colors import hsv_to_rgb
+
+
+def k2c(K: np.ndarray):
+    return hsv_to_rgb(np.array([3.0 * K % 1, 0.4 + K * 0, 0.6 + 0.15 * K]).transpose())

xmetrics/metrics.py

@@ -1,28 +1,19 @@
-import json
+import pickle
-import os
 from collections import Counter
-from pathlib import Path
-from subprocess import check_output
 
-args = []
+from common import counter
-# args = ["--topo-order"]
+
-commits = check_output(["git", "log", "--pretty=%H", *args], text=True).splitlines()
+with open("commits.dat", "rb") as file:
-commits = commits[::len(commits) // min(len(commits), 4096)]
+    commits: list[str] = pickle.load(file)
 entries = []
 last_ctr = Counter()
 last_cor = []
 cors = []
 C = min(len(commits), 720)
 for i, commit in enumerate(reversed(commits)):
+    print(f"P={i/len(commits):6f}", flush=True)
     print("running", commit, flush=True)
-    check_output(["git", "checkout", commit], text=True)
+    current_ctr = counter(commit, True)
-    current_ctr = Counter()
-    for path in Path(os.getenv("SRCDIR", "src")).rglob(os.getenv("SRCPATTERN", "*.rs")):
-        lines = path.read_bytes().splitlines()
-        lines = (line.strip() for line in lines)
-        lines = (line for line in lines if line)
-        lines = set(lines)
-        current_ctr.update(lines)
     added = current_ctr - last_ctr
     deleted = last_ctr - current_ctr
     entries.append((i, current_ctr.total(), added.total(), deleted.total()))
@@ -44,5 +35,5 @@ for i, commit in enumerate(reversed(commits)):
         cor.append(cor_ctr[j])
     last_ctr = current_ctr
     last_cor = current_cor
-with open("/code/metrics.json", "w") as file:
+with open("/code/metrics.dat", "wb") as file:
-    json.dump({"entries": entries, "cors": cors}, file)
+    pickle.dump({"entries": entries, "cors": cors}, file)

xmetrics/render.py

@@ -1,126 +0,0 @@
-import json
-from functools import cache
-
-import matplotlib.pyplot as plt
-import numpy as np
-from matplotlib.colors import hsv_to_rgb
-
-plt.rcParams["figure.figsize"] = [18, 9]
-plt.style.use("dark_background")
-plt.subplots_adjust(left=0.05, right=0.99, top=0.95, bottom=0.05)
-plt.margins(x=0.025)
-
-
-with open("metrics.json", "r") as file:
-    metrics = json.load(file)
-    entries = metrics["entries"]
-    cors = metrics["cors"]
-
-
-def render_ploc():
-    X, Y, Ya, Yd = np.array(entries).transpose()
-    plt.clf()
-    plt.plot(X, Y, linewidth=1.0)
-    plt.fill_between(
-        X,
-        Y - Ya,
-        Y,
-        alpha=0.5,
-        color="green",
-        linewidth=0.0,
-    )
-    plt.fill_between(
-        X,
-        Y,
-        Y + Yd,
-        alpha=0.5,
-        color="red",
-        linewidth=0.0,
-    )
-    plt.ylim(bottom=0)
-    plt.savefig("/code/metrics/metrics.png")
-
-
-@cache
-def get_m():
-    return len(cors)
-
-
-@cache
-def get_n():
-    return len(cors[0])
-
-
-@cache
-def get_y():
-    return np.array(cors)
-
-
-@cache
-def _get_x(px: bool):
-    if px:
-        return abs(np.diff(get_y(), axis=1, prepend=0)).sum(axis=0).cumsum()
-    else:
-        return np.arange(get_n())
-
-
-def get_x(*, px: bool):
-    return _get_x(px)
-
-
-@cache
-def get_w():
-    return get_y().max(axis=1)
-
-
-@cache
-def _get_z(pc: bool):
-    M = get_m()
-    if pc:
-        W = get_w()
-        I = W.cumsum()
-        assert I.shape == (M,)
-        return (I - W / 2) / I.max() * M
-    else:
-        return np.arange(M)
-
-
-def get_z(*, pc: bool):
-    return _get_z(pc)
-
-
-@cache
-def _get_c(pc: bool):
-    M = get_m()
-    Z = get_z(pc=pc)
-    W = get_w()
-    return hsv_to_rgb(np.array([3.0 * Z / M % 1, 0.4 + Z * 0, 0.6 + 0.15 * (Z / M)]).transpose())
-
-
-def get_c(*, pc: bool):
-    return _get_c(pc)
-
-
-def render_cors(*, px, pc):
-    M = get_m()
-    N = get_n()
-    Y = get_y()
-    X = get_x(px=px)
-    assert Y.shape == (M, N)
-    plt.clf()
-    C = get_c(pc=pc)
-    plt.stackplot(
-        X,
-        Y,
-        colors=C,
-        aa=False,
-        linewidth=0.0,
-    )
-    plt.savefig(f"/code/metrics/metrics-px{int(px)}-pc{int(pc)}.png")
-
-
-render_ploc()
-render_cors(px=False, pc=False)
-render_cors(px=False, pc=True)
-render_cors(px=True, pc=False)
-render_cors(px=True, pc=True)

xmetrics/render_hist.py

@@ -0,0 +1,23 @@
+import pickle
+
+import matplotlib.pyplot as plt
+
+plt.rcParams["figure.figsize"] = [18, 9]
+plt.style.use("dark_background")
+plt.subplots_adjust(left=0.05, right=0.99, top=0.95, bottom=0.05)
+plt.margins(x=0.025)
+
+with open("Y.dat", "rb") as file:
+    Y = pickle.load(file)
+with open("X.dat", "rb") as file:
+    X = pickle.load(file)
+with open("C.dat", "rb") as file:
+    C = pickle.load(file)
+plt.stackplot(
+    X,
+    Y,
+    colors=C,
+    aa=False,
+    linewidth=0.0,
+)
+plt.savefig(f"/code/metrics/out.png")

xmetrics/render_ploc.py

@@ -0,0 +1,31 @@
+import pickle
+
+import matplotlib.pyplot as plt
+
+plt.rcParams["figure.figsize"] = [18, 9]
+plt.style.use("dark_background")
+plt.subplots_adjust(left=0.05, right=0.99, top=0.95, bottom=0.05)
+plt.margins(x=0.025)
+
+with open("entries.dat", "rb") as file:
+    entries_t = pickle.load(file)
+X, Y, Ya, Yd = entries_t
+plt.plot(X, Y, linewidth=1.0)
+plt.fill_between(
+    X,
+    Y - Ya,
+    Y,
+    alpha=0.5,
+    color="green",
+    linewidth=0.0,
+)
+plt.fill_between(
+    X,
+    Y,
+    Y + Yd,
+    alpha=0.5,
+    color="red",
+    linewidth=0.0,
+)
+plt.ylim(bottom=0)
+plt.savefig("/code/metrics/out.png")

xmetrics/x_linear.py

@@ -0,0 +1,10 @@
+import pickle
+
+import numpy as np
+
+with open("Y.dat", "rb") as file:
+    Y = pickle.load(file)
+N = Y.shape[1]
+X = np.arange(N)
+with open("X.dat", "wb") as file:
+    pickle.dump(X, file)

xmetrics/x_proportional.py

@@ -0,0 +1,10 @@
+import pickle
+
+import numpy as np
+
+with open("Y.dat", "rb") as file:
+    Y = pickle.load(file)
+N = Y.shape[1]
+X = abs(np.diff(Y, axis=1, prepend=0)).sum(axis=0).cumsum()
+with open("X.dat", "wb") as file:
+    pickle.dump(X, file)