From 3a5734b98b04fbe4c42e40847ee95c099037ed72 Mon Sep 17 00:00:00 2001
From: timofey <tim@ongoteam.yaconnect.com>
Date: Wed, 2 Aug 2023 02:54:34 +0000
Subject: [PATCH] cache mount
---
metrics/Dockerfile | 18 +-----------------
metrics/metrics.py | 17 +++++++++++------
xmetrics/Dockerfile | 34 +++++++++++++++++-----------------
xmetrics/c_proportional.py | 1 -
xmetrics/commits.py | 1 -
xmetrics/common.py | 14 +++++++++-----
xmetrics/metrics.py | 2 +-
7 files changed, 39 insertions(+), 48 deletions(-)
diff --git a/metrics/Dockerfile b/metrics/Dockerfile
index 61e6b13..41aa910 100644
--- a/metrics/Dockerfile
+++ b/metrics/Dockerfile
@@ -5,24 +5,8 @@ RUN git clone https://gitea.parrrate.ru/PTV/radn-rs.git
WORKDIR /code/radn-rs/
COPY metrics.py /code/metrics.py
-RUN git fetch && git checkout 54a6912baf715816e3dcbfd074c0e2d7b8c74fe3
-RUN python3 /code/metrics.py
-
-RUN git fetch && git checkout 936f41735a4f7913e49fb16c44e89901bf703568
-RUN python3 /code/metrics.py
-
-RUN git fetch && git checkout f13a1382f8781231e01d375a13041884c87ddf0c
-RUN python3 /code/metrics.py
-
-RUN git fetch && git checkout ce65688e47b07f14ef2861971f64296c2197e778
-RUN python3 /code/metrics.py
-
-RUN git fetch && git checkout 7f1d72898ddd9ab40bf91e553f70a023a64fe647
-RUN python3 /code/metrics.py
-
-
RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48
-RUN python3 /code/metrics.py
+RUN --mount=type=cache,target=/code/cache/ python3 /code/metrics.py
FROM python:3.11
RUN python3 -m pip install matplotlib
diff --git a/metrics/metrics.py b/metrics/metrics.py
index 9c1d298..517d584 100644
--- a/metrics/metrics.py
+++ b/metrics/metrics.py
@@ -1,4 +1,5 @@
import json
+from pathlib import Path
from subprocess import check_output
@@ -16,20 +17,24 @@ entry_schema = {
},
}
+SOURCE = Path(__file__).read_text()
+cache_path = Path("/code/cache/cache.json")
+cache_path.parent.mkdir(exist_ok=True)
try:
- with open("/code/cache.json", "r") as file:
- cache = json.load(file)
+ with cache_path.open("r") as file:
+ cache: dict = json.load(file)
+ if cache.get("SOURCE") != SOURCE:
+ cache.clear()
except FileNotFoundError:
cache = {}
+cache["SOURCE"] = SOURCE
commits = check_output(["git", "log", "--pretty=%H"], text=True).splitlines()
entries = []
for i, commit in enumerate(commits):
if commit not in cache:
print("running", commit)
check_output(["git", "checkout", commit], text=True)
- lines = check_output(
- ["rust-code-analysis-cli", "-m", "-p", "src", "-O", "json"], text=True
- ).splitlines()
+ lines = check_output(["rust-code-analysis-cli", "-m", "-p", "src", "-O", "json"], text=True).splitlines()
print(len(lines))
commit_entries = []
for line in lines:
@@ -37,7 +42,7 @@ for i, commit in enumerate(commits):
commit_entries.append(filter_schema(entry, entry_schema))
cache[commit] = commit_entries
entries.extend((i, entry) for entry in cache[commit])
-with open("/code/cache.json", "w") as file:
+with cache_path.open("w") as file:
json.dump(cache, file)
with open("/code/metrics.json", "w") as file:
json.dump(entries, file)
diff --git a/xmetrics/Dockerfile b/xmetrics/Dockerfile
index b8ce2d4..77938de 100644
--- a/xmetrics/Dockerfile
+++ b/xmetrics/Dockerfile
@@ -2,52 +2,52 @@ FROM python:3.11 as metrics-base
WORKDIR /code/
FROM metrics-base as metrics-tmg
-RUN git clone https://github.com/OnGoTeam/TMGmod.git
-WORKDIR /code/TMGmod/
+RUN git clone https://github.com/OnGoTeam/TMGmod.git repo
+WORKDIR /code/repo/
RUN git fetch && git checkout 82db35f2db92572c98d84ef781f64f8d65d2f023
ENV SRCDIR="build/src"
ENV SRCPATTERN="*.cs"
FROM metrics-base as metrics-radn
-RUN git clone https://gitea.parrrate.ru/PTV/radn-rs.git
-WORKDIR /code/radn-rs/
+RUN git clone https://gitea.parrrate.ru/PTV/radn-rs.git repo
+WORKDIR /code/repo/
RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48
FROM metrics-base as metrics-mdbook
-RUN git clone https://github.com/rust-lang/mdBook.git
-WORKDIR /code/mdBook/
+RUN git clone https://github.com/rust-lang/mdBook.git repo
+WORKDIR /code/repo/
RUN git fetch && git checkout 904aa530b5f59387f19feb0dbe0e959de3f247d2
FROM metrics-base as metrics-rustup
-RUN git clone https://github.com/rust-lang/rustup.git
-WORKDIR /code/rustup/
+RUN git clone https://github.com/rust-lang/rustup.git repo
+WORKDIR /code/repo/
RUN git fetch && git checkout 849adb7c1b8c97e4145e350a934c3a665f61798e
FROM metrics-base as metrics-cargo
-RUN git clone https://github.com/rust-lang/cargo.git
-WORKDIR /code/cargo/
+RUN git clone https://github.com/rust-lang/cargo.git repo
+WORKDIR /code/repo/
RUN git fetch && git checkout 5b377cece0e0dd0af686cf53ce4637d5d85c2a10
FROM metrics-base as metrics-rust
-RUN git clone https://github.com/rust-lang/rust.git
-WORKDIR /code/rust/
+RUN git clone https://github.com/rust-lang/rust.git repo
+WORKDIR /code/repo/
RUN git fetch && git checkout 32d81eccd64513bacef9dfa1574543ada6b45d85
FROM metrics-base as metrics-ocen
-RUN git clone https://github.com/ocen-lang/ocen.git
-WORKDIR /code/ocen/
+RUN git clone https://github.com/ocen-lang/ocen.git repo
+WORKDIR /code/repo/
RUN git fetch && git checkout 0b1a63d0e141fc3032f10c70b5cae117429e2dcb
ENV SRCDIR="."
ENV SRCPATTERN="*.[oa][ec]"
FROM metrics-base as metrics-aecor
-RUN git clone https://github.com/mustafaquraish/aecor.git
-WORKDIR /code/aecor/
+RUN git clone https://github.com/mustafaquraish/aecor.git repo
+WORKDIR /code/repo/
RUN git fetch && git checkout f81543a34ee363dcc00e8632fd7cfcd4a3478b23
ENV SRCDIR="."
ENV SRCPATTERN="*.[ach]*"
-FROM metrics-radn as metrics-repo
+FROM metrics-tmg as metrics-repo
FROM metrics-repo as metrics-commits
COPY common.py /code/common.py
diff --git a/xmetrics/c_proportional.py b/xmetrics/c_proportional.py
index 91b204e..872eaf5 100644
--- a/xmetrics/c_proportional.py
+++ b/xmetrics/c_proportional.py
@@ -1,6 +1,5 @@
import pickle
-import numpy as np
from k2c import k2c
with open("Y.dat", "rb") as file:
diff --git a/xmetrics/commits.py b/xmetrics/commits.py
index 73be4df..c259ba0 100644
--- a/xmetrics/commits.py
+++ b/xmetrics/commits.py
@@ -4,7 +4,6 @@ from subprocess import check_output
from common import counter
args = []
-# args = ["--topo-order"]
original_commits = check_output(["git", "log", "--pretty=%H", *args], text=True).splitlines()
N = len(original_commits)
filtered_commits = set(original_commits[:: N // min(N, 4096)])
diff --git a/xmetrics/common.py b/xmetrics/common.py
index 0ab266c..7dd1934 100644
--- a/xmetrics/common.py
+++ b/xmetrics/common.py
@@ -8,15 +8,19 @@ from subprocess import check_output
@lru_cache()
def _counter(commit: str, unique: bool) -> Counter:
check_output(["git", "checkout", commit], text=True)
- ctr = Counter()
+ if unique:
+ all_lines = set()
+ else:
+ all_lines = Counter()
for path in Path(os.getenv("SRCDIR", "src")).rglob(os.getenv("SRCPATTERN", "*.rs")):
lines = path.read_bytes().splitlines()
lines = (line.strip() for line in lines)
lines = (line for line in lines if line)
- if unique:
- lines = set(lines)
- ctr.update(lines)
- return ctr
+ all_lines.update(lines)
+ if isinstance(all_lines, set):
+ return Counter(all_lines)
+ else:
+ return all_lines
def counter(commit: str, unique: bool) -> Counter:
diff --git a/xmetrics/metrics.py b/xmetrics/metrics.py
index 5c0d097..622a5db 100644
--- a/xmetrics/metrics.py
+++ b/xmetrics/metrics.py
@@ -11,7 +11,7 @@ last_cor = []
cors = []
C = min(len(commits), 720)
for i, commit in enumerate(reversed(commits)):
- print(f"P={i/len(commits):6f}", flush=True)
+ print(f"P={i / len(commits):6f}", flush=True)
print("running", commit, flush=True)
current_ctr = counter(commit, True)
added = current_ctr - last_ctr