PTV/radn-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cache mount

Browse Source
This commit is contained in:
AF 2023-08-02 02:54:34 +00:00
parent 82f9f40715
commit 3a5734b98b
7 changed files with 39 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
metrics/Dockerfile
View File

@ -5,24 +5,8 @@ RUN git clone https://gitea.parrrate.ru/PTV/radn-rs.git
WORKDIR /code/radn-rs/ WORKDIR /code/radn-rs/
COPY metrics.py /code/metrics.py COPY metrics.py /code/metrics.py
RUN git fetch && git checkout 54a6912baf715816e3dcbfd074c0e2d7b8c74fe3
RUN python3 /code/metrics.py
RUN git fetch && git checkout 936f41735a4f7913e49fb16c44e89901bf703568
RUN python3 /code/metrics.py
RUN git fetch && git checkout f13a1382f8781231e01d375a13041884c87ddf0c
RUN python3 /code/metrics.py
RUN git fetch && git checkout ce65688e47b07f14ef2861971f64296c2197e778
RUN python3 /code/metrics.py
RUN git fetch && git checkout 7f1d72898ddd9ab40bf91e553f70a023a64fe647
RUN python3 /code/metrics.py
RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48 RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48
RUN python3 /code/metrics.py RUN --mount=type=cache,target=/code/cache/ python3 /code/metrics.py
FROM python:3.11 FROM python:3.11
RUN python3 -m pip install matplotlib RUN python3 -m pip install matplotlib

17
metrics/metrics.py
View File

@ -1,4 +1,5 @@
import json import json
from pathlib import Path
from subprocess import check_output from subprocess import check_output
@ -16,20 +17,24 @@ entry_schema = {
}, },
} }
SOURCE = Path(__file__).read_text()
cache_path = Path("/code/cache/cache.json")
cache_path.parent.mkdir(exist_ok=True)
try: try:
with open("/code/cache.json", "r") as file: with cache_path.open("r") as file:
cache = json.load(file) cache: dict = json.load(file)
if cache.get("SOURCE") != SOURCE:
cache.clear()
except FileNotFoundError: except FileNotFoundError:
cache = {} cache = {}
cache["SOURCE"] = SOURCE
commits = check_output(["git", "log", "--pretty=%H"], text=True).splitlines() commits = check_output(["git", "log", "--pretty=%H"], text=True).splitlines()
entries = [] entries = []
for i, commit in enumerate(commits): for i, commit in enumerate(commits):
if commit not in cache: if commit not in cache:
print("running", commit) print("running", commit)
check_output(["git", "checkout", commit], text=True) check_output(["git", "checkout", commit], text=True)
lines = check_output( lines = check_output(["rust-code-analysis-cli", "-m", "-p", "src", "-O", "json"], text=True).splitlines()
["rust-code-analysis-cli", "-m", "-p", "src", "-O", "json"], text=True
).splitlines()
print(len(lines)) print(len(lines))
commit_entries = [] commit_entries = []
for line in lines: for line in lines:
@ -37,7 +42,7 @@ for i, commit in enumerate(commits):
commit_entries.append(filter_schema(entry, entry_schema)) commit_entries.append(filter_schema(entry, entry_schema))
cache[commit] = commit_entries cache[commit] = commit_entries
entries.extend((i, entry) for entry in cache[commit]) entries.extend((i, entry) for entry in cache[commit])
with open("/code/cache.json", "w") as file: with cache_path.open("w") as file:
json.dump(cache, file) json.dump(cache, file)
with open("/code/metrics.json", "w") as file: with open("/code/metrics.json", "w") as file:
json.dump(entries, file) json.dump(entries, file)

34
xmetrics/Dockerfile
View File

@ -2,52 +2,52 @@ FROM python:3.11 as metrics-base
WORKDIR /code/ WORKDIR /code/
FROM metrics-base as metrics-tmg FROM metrics-base as metrics-tmg
RUN git clone https://github.com/OnGoTeam/TMGmod.git RUN git clone https://github.com/OnGoTeam/TMGmod.git repo
WORKDIR /code/TMGmod/ WORKDIR /code/repo/
RUN git fetch && git checkout 82db35f2db92572c98d84ef781f64f8d65d2f023 RUN git fetch && git checkout 82db35f2db92572c98d84ef781f64f8d65d2f023
ENV SRCDIR="build/src" ENV SRCDIR="build/src"
ENV SRCPATTERN="*.cs" ENV SRCPATTERN="*.cs"
FROM metrics-base as metrics-radn FROM metrics-base as metrics-radn
RUN git clone https://gitea.parrrate.ru/PTV/radn-rs.git RUN git clone https://gitea.parrrate.ru/PTV/radn-rs.git repo
WORKDIR /code/radn-rs/ WORKDIR /code/repo/
RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48 RUN git fetch && git checkout 1a8d70957b7d0f3f08f1c0f11ff04ebc007a6b48
FROM metrics-base as metrics-mdbook FROM metrics-base as metrics-mdbook
RUN git clone https://github.com/rust-lang/mdBook.git RUN git clone https://github.com/rust-lang/mdBook.git repo
WORKDIR /code/mdBook/ WORKDIR /code/repo/
RUN git fetch && git checkout 904aa530b5f59387f19feb0dbe0e959de3f247d2 RUN git fetch && git checkout 904aa530b5f59387f19feb0dbe0e959de3f247d2
FROM metrics-base as metrics-rustup FROM metrics-base as metrics-rustup
RUN git clone https://github.com/rust-lang/rustup.git RUN git clone https://github.com/rust-lang/rustup.git repo
WORKDIR /code/rustup/ WORKDIR /code/repo/
RUN git fetch && git checkout 849adb7c1b8c97e4145e350a934c3a665f61798e RUN git fetch && git checkout 849adb7c1b8c97e4145e350a934c3a665f61798e
FROM metrics-base as metrics-cargo FROM metrics-base as metrics-cargo
RUN git clone https://github.com/rust-lang/cargo.git RUN git clone https://github.com/rust-lang/cargo.git repo
WORKDIR /code/cargo/ WORKDIR /code/repo/
RUN git fetch && git checkout 5b377cece0e0dd0af686cf53ce4637d5d85c2a10 RUN git fetch && git checkout 5b377cece0e0dd0af686cf53ce4637d5d85c2a10
FROM metrics-base as metrics-rust FROM metrics-base as metrics-rust
RUN git clone https://github.com/rust-lang/rust.git RUN git clone https://github.com/rust-lang/rust.git repo
WORKDIR /code/rust/ WORKDIR /code/repo/
RUN git fetch && git checkout 32d81eccd64513bacef9dfa1574543ada6b45d85 RUN git fetch && git checkout 32d81eccd64513bacef9dfa1574543ada6b45d85
FROM metrics-base as metrics-ocen FROM metrics-base as metrics-ocen
RUN git clone https://github.com/ocen-lang/ocen.git RUN git clone https://github.com/ocen-lang/ocen.git repo
WORKDIR /code/ocen/ WORKDIR /code/repo/
RUN git fetch && git checkout 0b1a63d0e141fc3032f10c70b5cae117429e2dcb RUN git fetch && git checkout 0b1a63d0e141fc3032f10c70b5cae117429e2dcb
ENV SRCDIR="." ENV SRCDIR="."
ENV SRCPATTERN="*.[oa][ec]" ENV SRCPATTERN="*.[oa][ec]"
FROM metrics-base as metrics-aecor FROM metrics-base as metrics-aecor
RUN git clone https://github.com/mustafaquraish/aecor.git RUN git clone https://github.com/mustafaquraish/aecor.git repo
WORKDIR /code/aecor/ WORKDIR /code/repo/
RUN git fetch && git checkout f81543a34ee363dcc00e8632fd7cfcd4a3478b23 RUN git fetch && git checkout f81543a34ee363dcc00e8632fd7cfcd4a3478b23
ENV SRCDIR="." ENV SRCDIR="."
ENV SRCPATTERN="*.[ach]*" ENV SRCPATTERN="*.[ach]*"
FROM metrics-radn as metrics-repo FROM metrics-tmg as metrics-repo
FROM metrics-repo as metrics-commits FROM metrics-repo as metrics-commits
COPY common.py /code/common.py COPY common.py /code/common.py

1
xmetrics/c_proportional.py
View File

@ -1,6 +1,5 @@
import pickle import pickle
import numpy as np
from k2c import k2c from k2c import k2c
with open("Y.dat", "rb") as file: with open("Y.dat", "rb") as file:

1
xmetrics/commits.py
View File

@ -4,7 +4,6 @@ from subprocess import check_output
from common import counter from common import counter
args = [] args = []
# args = ["--topo-order"]
original_commits = check_output(["git", "log", "--pretty=%H", *args], text=True).splitlines() original_commits = check_output(["git", "log", "--pretty=%H", *args], text=True).splitlines()
N = len(original_commits) N = len(original_commits)
filtered_commits = set(original_commits[:: N // min(N, 4096)]) filtered_commits = set(original_commits[:: N // min(N, 4096)])

14
xmetrics/common.py
View File

@ -8,15 +8,19 @@ from subprocess import check_output
@lru_cache() @lru_cache()
def _counter(commit: str, unique: bool) -> Counter: def _counter(commit: str, unique: bool) -> Counter:
check_output(["git", "checkout", commit], text=True) check_output(["git", "checkout", commit], text=True)
ctr = Counter() if unique:
all_lines = set()
else:
all_lines = Counter()
for path in Path(os.getenv("SRCDIR", "src")).rglob(os.getenv("SRCPATTERN", "*.rs")): for path in Path(os.getenv("SRCDIR", "src")).rglob(os.getenv("SRCPATTERN", "*.rs")):
lines = path.read_bytes().splitlines() lines = path.read_bytes().splitlines()
lines = (line.strip() for line in lines) lines = (line.strip() for line in lines)
lines = (line for line in lines if line) lines = (line for line in lines if line)
if unique: all_lines.update(lines)
lines = set(lines) if isinstance(all_lines, set):
ctr.update(lines) return Counter(all_lines)
return ctr else:
return all_lines
def counter(commit: str, unique: bool) -> Counter: def counter(commit: str, unique: bool) -> Counter:

2
xmetrics/metrics.py
View File

@ -11,7 +11,7 @@ last_cor = []
cors = [] cors = []
C = min(len(commits), 720) C = min(len(commits), 720)
for i, commit in enumerate(reversed(commits)): for i, commit in enumerate(reversed(commits)):
print(f"P={i/len(commits):6f}", flush=True) print(f"P={i / len(commits):6f}", flush=True)
print("running", commit, flush=True) print("running", commit, flush=True)
current_ctr = counter(commit, True) current_ctr = counter(commit, True)
added = current_ctr - last_ctr added = current_ctr - last_ctr