diff --git a/.vscode/settings.json b/.vscode/settings.json index 6c48529..2b44416 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,8 @@ { "python.analysis.typeCheckingMode": "basic", "python.formatting.blackArgs": ["--line-length", "120"], - "isort.args": ["--profile", "black"] + "isort.args": ["--profile", "black"], + "python.analysis.extraPaths": [ + "xmetrics" + ] } diff --git a/README.md b/README.md index 11dc075..d1fea50 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ ```sh cd /code/ git clone --recurse-submodules https://gitea.parrrate.ru/PTV/radn-rs.git +git clone --recurse-submodules https://gitea.parrrate.ru/PTV/exercises.git ``` ```sh clear && docker compose up -d --build metrics && docker cp radn-metrics:/code/metrics.png ./metrics/metrics.png diff --git a/docker-compose.yml b/docker-compose.yml index 286ca3c..298ad22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ volumes: radn-rs: {} book-monads: {} book-radn: {} + exercises: {} services: radn-rs-dev: @@ -57,6 +58,21 @@ services: radn: {} tty: true stop_signal: SIGKILL + exercises: + container_name: exercises + build: + context: radn-rs + dockerfile: Dockerfile.Book.Exercises + environment: + MDBOOK_OUTPUT__HTML__GIT_REPOSITORY_URL: "https://gitea.parrrate.ru/PTV/exercises" + MDBOOK_OUTPUT__HTML__EDIT_URL_TEMPLATE: "https://gitea.parrrate.ru/PTV/exercises/_edit/latest/{path}" + volumes: + - radn-rs:/code/:ro + - exercises:/data/book/:rw + networks: + radn: {} + tty: true + stop_signal: SIGKILL nginx: container_name: radn-nginx build: diff --git a/metrics/Dockerfile b/metrics/Dockerfile index 6b286ad..61e6b13 100644 --- a/metrics/Dockerfile +++ b/metrics/Dockerfile @@ -17,10 +17,13 @@ RUN python3 /code/metrics.py RUN git fetch && git checkout ce65688e47b07f14ef2861971f64296c2197e778 RUN python3 /code/metrics.py - RUN git fetch && git checkout 7f1d72898ddd9ab40bf91e553f70a023a64fe647 RUN python3 /code/metrics.py + +RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48 +RUN python3 /code/metrics.py + FROM python:3.11 RUN python3 -m pip install matplotlib WORKDIR /code/ diff --git a/nginx-default.conf b/nginx-default.conf index 5512fd8..0369c75 100644 --- a/nginx-default.conf +++ b/nginx-default.conf @@ -56,6 +56,22 @@ server { } } +server { + listen 80; + listen [::]:80; + server_name exercises.parrrate.ru; + + location / { + proxy_pass http://exercises/; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + proxy_buffering off; + } +} + server { listen 80; listen [::]:80; diff --git a/radn-rs/Dockerfile b/radn-rs/Dockerfile index e8712c1..542c078 100644 --- a/radn-rs/Dockerfile +++ b/radn-rs/Dockerfile @@ -5,7 +5,6 @@ RUN rustup component add rustfmt RUN rustup component add clippy RUN cargo install mdbook RUN cargo install rust-code-analysis-cli -RUN apt-get update -RUN apt-get install -y cloc +RUN apt-get update && install -y cloc WORKDIR /code/ diff --git a/radn-rs/Dockerfile.Book.Exercises b/radn-rs/Dockerfile.Book.Exercises new file mode 100644 index 0000000..b93d6e2 --- /dev/null +++ b/radn-rs/Dockerfile.Book.Exercises @@ -0,0 +1,7 @@ +FROM rust:1.69 + +RUN cargo install mdbook + +WORKDIR /code/exercises/ + +CMD [ "mdbook", "serve", "--dest-dir", "/data/book/", "--port", "80", "--hostname", "0.0.0.0" ] diff --git a/xmetrics/Dockerfile b/xmetrics/Dockerfile index 4a92c85..b8ce2d4 100644 --- a/xmetrics/Dockerfile +++ b/xmetrics/Dockerfile @@ -11,7 +11,7 @@ ENV SRCPATTERN="*.cs" FROM metrics-base as metrics-radn RUN git clone https://gitea.parrrate.ru/PTV/radn-rs.git WORKDIR /code/radn-rs/ -RUN git fetch && git checkout 7f1d72898ddd9ab40bf91e553f70a023a64fe647 +RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48 FROM metrics-base as metrics-mdbook RUN git clone https://github.com/rust-lang/mdBook.git @@ -47,15 +47,89 @@ RUN git fetch && git checkout f81543a34ee363dcc00e8632fd7cfcd4a3478b23 ENV SRCDIR="." ENV SRCPATTERN="*.[ach]*" -FROM metrics-radn as metrics +FROM metrics-radn as metrics-repo + +FROM metrics-repo as metrics-commits +COPY common.py /code/common.py +COPY commits.py /code/commits.py +RUN python3 /code/commits.py + +FROM metrics-repo as metrics +COPY common.py /code/common.py COPY metrics.py /code/metrics.py +COPY --from=metrics-commits /code/commits.dat commits.dat RUN python3 /code/metrics.py -FROM python:3.11 -RUN python3 -m pip install matplotlib -RUN python3 -m pip install scipy +FROM python:3.11 as metrics-process +RUN python3 -m pip install numpy WORKDIR /code/ -COPY --from=metrics /code/metrics.json metrics.json + +FROM metrics-process as metrics-data +COPY --from=metrics /code/metrics.dat metrics.dat + +FROM metrics-process as metrics-plot +RUN python3 -m pip install matplotlib + +FROM metrics-data as metrics-entries +COPY entries.py entries.py +RUN python3 entries.py + +FROM metrics-data as metrics-y +COPY cors.py cors.py +RUN python3 cors.py + +FROM metrics-plot as metrics-c +COPY --from=metrics-y /code/Y.dat Y.dat +COPY k2c.py k2c.py + +FROM metrics-c as metrics-c-linear +COPY c_linear.py c_linear.py +RUN python3 c_linear.py + +FROM metrics-c as metrics-c-proportional +COPY c_proportional.py c_proportional.py +RUN python3 c_proportional.py + +FROM metrics-process as metrics-x-linear +COPY --from=metrics-y /code/Y.dat Y.dat +COPY x_linear.py x_linear.py +RUN python3 x_linear.py + +FROM metrics-process as metrics-x-proportional +COPY --from=metrics-y /code/Y.dat Y.dat +COPY x_proportional.py x_proportional.py +RUN python3 x_proportional.py + +FROM metrics-plot as metrics-render RUN mkdir /code/metrics/ -COPY render.py render.py -RUN python3 render.py + +FROM metrics-render as metrics-render-ploc +COPY --from=metrics-entries /code/entries.dat entries.dat +COPY render_ploc.py render_ploc.py +RUN python3 render_ploc.py + +FROM metrics-render as metrics-render-hist +COPY --from=metrics-y /code/Y.dat Y.dat +COPY render_hist.py render_hist.py + +FROM metrics-render-hist as metrics-render-01x +COPY --from=metrics-x-linear /code/X.dat X.dat +COPY --from=metrics-c-linear /code/C.dat C.dat +RUN python3 render_hist.py + +FROM metrics-render-hist as metrics-render-02y +COPY --from=metrics-x-linear /code/X.dat X.dat +COPY --from=metrics-c-proportional /code/C.dat C.dat +RUN python3 render_hist.py + +FROM metrics-render-hist as metrics-render-03xy +COPY --from=metrics-x-proportional /code/X.dat X.dat +COPY --from=metrics-c-proportional /code/C.dat C.dat +RUN python3 render_hist.py + +FROM metrics-render +COPY --from=metrics-y /code/Y.dat Y.dat +COPY --from=metrics-render-ploc /code/metrics/out.png /code/metrics/metrics.png +COPY --from=metrics-render-01x /code/metrics/out.png /code/metrics/metrics-01x.png +COPY --from=metrics-render-02y /code/metrics/out.png /code/metrics/metrics-02y.png +COPY --from=metrics-render-03xy /code/metrics/out.png /code/metrics/metrics-03xy.png diff --git a/xmetrics/c_linear.py b/xmetrics/c_linear.py new file mode 100644 index 0000000..973d886 --- /dev/null +++ b/xmetrics/c_linear.py @@ -0,0 +1,11 @@ +import pickle + +import numpy as np +from k2c import k2c + +with open("Y.dat", "rb") as file: + Y = pickle.load(file) +M = Y.shape[0] +K = np.arange(M) / M +with open("C.dat", "wb") as file: + pickle.dump(k2c(K), file) diff --git a/xmetrics/c_proportional.py b/xmetrics/c_proportional.py new file mode 100644 index 0000000..91b204e --- /dev/null +++ b/xmetrics/c_proportional.py @@ -0,0 +1,13 @@ +import pickle + +import numpy as np +from k2c import k2c + +with open("Y.dat", "rb") as file: + Y = pickle.load(file) +M = Y.shape[0] +W = Y.max(axis=1) +I = W.cumsum() +K = (I - W / 2) / I.max() +with open("C.dat", "wb") as file: + pickle.dump(k2c(K), file) diff --git a/xmetrics/commits.py b/xmetrics/commits.py new file mode 100644 index 0000000..73be4df --- /dev/null +++ b/xmetrics/commits.py @@ -0,0 +1,41 @@ +import pickle +from subprocess import check_output + +from common import counter + +args = [] +# args = ["--topo-order"] +original_commits = check_output(["git", "log", "--pretty=%H", *args], text=True).splitlines() +N = len(original_commits) +filtered_commits = set(original_commits[:: N // min(N, 4096)]) +total_changes: dict[str, int] = {} +chosen_parents: dict[str, str | None] = {} +for i, commit in enumerate(reversed(original_commits)): + print(f"P={i / N:6f}", flush=True) + parents = check_output(["git", "log", "--pretty=%P", "-n", "1", commit], text=True).split() + chosen_parent = None + if commit in filtered_commits: + ctr = counter(commit, True) + changes = ctr.total() + for parent in parents: + pctr = counter(parent, True) + maybe_changes = (pctr - ctr).total() + (ctr - pctr).total() + total_changes[parent] + if maybe_changes > changes: + changes = maybe_changes + chosen_parent = parent + else: + changes = 0 + for parent in parents: + maybe_changes = total_changes[parent] + if maybe_changes > changes: + changes = maybe_changes + chosen_parent = parent + total_changes[commit] = changes + chosen_parents[commit] = chosen_parent +commit = original_commits[0] +commits = [] +while commit is not None: + commits.append(commit) + commit = chosen_parents[commit] +with open("/code/commits.dat", "wb") as file: + pickle.dump(commits, file) diff --git a/xmetrics/common.py b/xmetrics/common.py new file mode 100644 index 0000000..0ab266c --- /dev/null +++ b/xmetrics/common.py @@ -0,0 +1,23 @@ +import os +from collections import Counter +from functools import lru_cache +from pathlib import Path +from subprocess import check_output + + +@lru_cache() +def _counter(commit: str, unique: bool) -> Counter: + check_output(["git", "checkout", commit], text=True) + ctr = Counter() + for path in Path(os.getenv("SRCDIR", "src")).rglob(os.getenv("SRCPATTERN", "*.rs")): + lines = path.read_bytes().splitlines() + lines = (line.strip() for line in lines) + lines = (line for line in lines if line) + if unique: + lines = set(lines) + ctr.update(lines) + return ctr + + +def counter(commit: str, unique: bool) -> Counter: + return _counter(commit, unique) diff --git a/xmetrics/cors.py b/xmetrics/cors.py new file mode 100644 index 0000000..bab4f06 --- /dev/null +++ b/xmetrics/cors.py @@ -0,0 +1,10 @@ +import pickle + +import numpy as np + +with open("metrics.dat", "rb") as file: + metrics = pickle.load(file) +cors = metrics["cors"] +Y = np.array(cors) +with open("Y.dat", "wb") as file: + pickle.dump(Y, file) diff --git a/xmetrics/entries.py b/xmetrics/entries.py new file mode 100644 index 0000000..f3ec2a9 --- /dev/null +++ b/xmetrics/entries.py @@ -0,0 +1,9 @@ +import pickle + +import numpy as np + +with open("metrics.dat", "rb") as file: + metrics = pickle.load(file) +entries = metrics["entries"] +with open("entries.dat", "wb") as file: + pickle.dump(np.array(entries).transpose(), file) diff --git a/xmetrics/k2c.py b/xmetrics/k2c.py new file mode 100644 index 0000000..bbbe76d --- /dev/null +++ b/xmetrics/k2c.py @@ -0,0 +1,6 @@ +import numpy as np +from matplotlib.colors import hsv_to_rgb + + +def k2c(K: np.ndarray): + return hsv_to_rgb(np.array([3.0 * K % 1, 0.4 + K * 0, 0.6 + 0.15 * K]).transpose()) diff --git a/xmetrics/metrics.py b/xmetrics/metrics.py index 1732275..5c0d097 100644 --- a/xmetrics/metrics.py +++ b/xmetrics/metrics.py @@ -1,28 +1,19 @@ -import json -import os +import pickle from collections import Counter -from pathlib import Path -from subprocess import check_output -args = [] -# args = ["--topo-order"] -commits = check_output(["git", "log", "--pretty=%H", *args], text=True).splitlines() -commits = commits[::len(commits) // min(len(commits), 4096)] +from common import counter + +with open("commits.dat", "rb") as file: + commits: list[str] = pickle.load(file) entries = [] last_ctr = Counter() last_cor = [] cors = [] C = min(len(commits), 720) for i, commit in enumerate(reversed(commits)): + print(f"P={i/len(commits):6f}", flush=True) print("running", commit, flush=True) - check_output(["git", "checkout", commit], text=True) - current_ctr = Counter() - for path in Path(os.getenv("SRCDIR", "src")).rglob(os.getenv("SRCPATTERN", "*.rs")): - lines = path.read_bytes().splitlines() - lines = (line.strip() for line in lines) - lines = (line for line in lines if line) - lines = set(lines) - current_ctr.update(lines) + current_ctr = counter(commit, True) added = current_ctr - last_ctr deleted = last_ctr - current_ctr entries.append((i, current_ctr.total(), added.total(), deleted.total())) @@ -44,5 +35,5 @@ for i, commit in enumerate(reversed(commits)): cor.append(cor_ctr[j]) last_ctr = current_ctr last_cor = current_cor -with open("/code/metrics.json", "w") as file: - json.dump({"entries": entries, "cors": cors}, file) +with open("/code/metrics.dat", "wb") as file: + pickle.dump({"entries": entries, "cors": cors}, file) diff --git a/xmetrics/render.py b/xmetrics/render.py deleted file mode 100644 index 8cf00cd..0000000 --- a/xmetrics/render.py +++ /dev/null @@ -1,126 +0,0 @@ -import json -from functools import cache - -import matplotlib.pyplot as plt -import numpy as np -from matplotlib.colors import hsv_to_rgb - -plt.rcParams["figure.figsize"] = [18, 9] -plt.style.use("dark_background") -plt.subplots_adjust(left=0.05, right=0.99, top=0.95, bottom=0.05) -plt.margins(x=0.025) - - -with open("metrics.json", "r") as file: - metrics = json.load(file) - entries = metrics["entries"] - cors = metrics["cors"] - - -def render_ploc(): - X, Y, Ya, Yd = np.array(entries).transpose() - plt.clf() - plt.plot(X, Y, linewidth=1.0) - plt.fill_between( - X, - Y - Ya, - Y, - alpha=0.5, - color="green", - linewidth=0.0, - ) - plt.fill_between( - X, - Y, - Y + Yd, - alpha=0.5, - color="red", - linewidth=0.0, - ) - plt.ylim(bottom=0) - plt.savefig("/code/metrics/metrics.png") - - -@cache -def get_m(): - return len(cors) - - -@cache -def get_n(): - return len(cors[0]) - - -@cache -def get_y(): - return np.array(cors) - - -@cache -def _get_x(px: bool): - if px: - return abs(np.diff(get_y(), axis=1, prepend=0)).sum(axis=0).cumsum() - else: - return np.arange(get_n()) - - -def get_x(*, px: bool): - return _get_x(px) - - -@cache -def get_w(): - return get_y().max(axis=1) - - -@cache -def _get_z(pc: bool): - M = get_m() - if pc: - W = get_w() - I = W.cumsum() - assert I.shape == (M,) - return (I - W / 2) / I.max() * M - else: - return np.arange(M) - - -def get_z(*, pc: bool): - return _get_z(pc) - - -@cache -def _get_c(pc: bool): - M = get_m() - Z = get_z(pc=pc) - W = get_w() - return hsv_to_rgb(np.array([3.0 * Z / M % 1, 0.4 + Z * 0, 0.6 + 0.15 * (Z / M)]).transpose()) - - -def get_c(*, pc: bool): - return _get_c(pc) - - -def render_cors(*, px, pc): - M = get_m() - N = get_n() - Y = get_y() - X = get_x(px=px) - assert Y.shape == (M, N) - plt.clf() - C = get_c(pc=pc) - plt.stackplot( - X, - Y, - colors=C, - aa=False, - linewidth=0.0, - ) - plt.savefig(f"/code/metrics/metrics-px{int(px)}-pc{int(pc)}.png") - - -render_ploc() -render_cors(px=False, pc=False) -render_cors(px=False, pc=True) -render_cors(px=True, pc=False) -render_cors(px=True, pc=True) diff --git a/xmetrics/render_hist.py b/xmetrics/render_hist.py new file mode 100644 index 0000000..f583c2e --- /dev/null +++ b/xmetrics/render_hist.py @@ -0,0 +1,23 @@ +import pickle + +import matplotlib.pyplot as plt + +plt.rcParams["figure.figsize"] = [18, 9] +plt.style.use("dark_background") +plt.subplots_adjust(left=0.05, right=0.99, top=0.95, bottom=0.05) +plt.margins(x=0.025) + +with open("Y.dat", "rb") as file: + Y = pickle.load(file) +with open("X.dat", "rb") as file: + X = pickle.load(file) +with open("C.dat", "rb") as file: + C = pickle.load(file) +plt.stackplot( + X, + Y, + colors=C, + aa=False, + linewidth=0.0, +) +plt.savefig(f"/code/metrics/out.png") diff --git a/xmetrics/render_ploc.py b/xmetrics/render_ploc.py new file mode 100644 index 0000000..091851e --- /dev/null +++ b/xmetrics/render_ploc.py @@ -0,0 +1,31 @@ +import pickle + +import matplotlib.pyplot as plt + +plt.rcParams["figure.figsize"] = [18, 9] +plt.style.use("dark_background") +plt.subplots_adjust(left=0.05, right=0.99, top=0.95, bottom=0.05) +plt.margins(x=0.025) + +with open("entries.dat", "rb") as file: + entries_t = pickle.load(file) +X, Y, Ya, Yd = entries_t +plt.plot(X, Y, linewidth=1.0) +plt.fill_between( + X, + Y - Ya, + Y, + alpha=0.5, + color="green", + linewidth=0.0, +) +plt.fill_between( + X, + Y, + Y + Yd, + alpha=0.5, + color="red", + linewidth=0.0, +) +plt.ylim(bottom=0) +plt.savefig("/code/metrics/out.png") diff --git a/xmetrics/x_linear.py b/xmetrics/x_linear.py new file mode 100644 index 0000000..36a9078 --- /dev/null +++ b/xmetrics/x_linear.py @@ -0,0 +1,10 @@ +import pickle + +import numpy as np + +with open("Y.dat", "rb") as file: + Y = pickle.load(file) +N = Y.shape[1] +X = np.arange(N) +with open("X.dat", "wb") as file: + pickle.dump(X, file) diff --git a/xmetrics/x_proportional.py b/xmetrics/x_proportional.py new file mode 100644 index 0000000..c4c697b --- /dev/null +++ b/xmetrics/x_proportional.py @@ -0,0 +1,10 @@ +import pickle + +import numpy as np + +with open("Y.dat", "rb") as file: + Y = pickle.load(file) +N = Y.shape[1] +X = abs(np.diff(Y, axis=1, prepend=0)).sum(axis=0).cumsum() +with open("X.dat", "wb") as file: + pickle.dump(X, file)