From 838322ad404981c194853a2675e1e1b1ce090503 Mon Sep 17 00:00:00 2001
From: Ralf Jung <jung@mpi-sws.org>
Date: Thu, 11 Jan 2024 11:51:54 +0100
Subject: [PATCH] remove some long-dead files
---
benchmark/.gitignore | 3 -
benchmark/export.py | 50 ---------------
benchmark/gitlab-extract.py | 118 ------------------------------------
benchmark/parse_log.py | 60 ------------------
benchmark/visualize.py | 40 ------------
5 files changed, 271 deletions(-)
delete mode 100644 benchmark/.gitignore
delete mode 100755 benchmark/export.py
delete mode 100755 benchmark/gitlab-extract.py
delete mode 100644 benchmark/parse_log.py
delete mode 100755 benchmark/visualize.py
diff --git a/benchmark/.gitignore b/benchmark/.gitignore
deleted file mode 100644
index b0922ce45..000000000
--- a/benchmark/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-__pycache__
-build-times*
-gitlab-extract
diff --git a/benchmark/export.py b/benchmark/export.py
deleted file mode 100755
index 54c336ca7..000000000
--- a/benchmark/export.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-import argparse, sys, pprint, itertools, subprocess
-import requests
-import parse_log
-
-# read command-line arguments
-parser = argparse.ArgumentParser(description='Export iris-coq build times to grafana')
-parser.add_argument("-f", "--file",
- dest="file", required=True,
- help="Filename to get the data from.")
-parser.add_argument("-c", "--commits",
- dest="commits",
- help="Restrict the graph to the given commits.")
-parser.add_argument("-p", "--project",
- dest="project", required=True,
- help="Project name sent to the server.")
-parser.add_argument("-b", "--branch",
- dest="branch", required=True,
- help="Branch name sent to the server.")
-parser.add_argument("--config",
- dest="config", required=True,
- help="The config string.")
-parser.add_argument("-s", "--server",
- dest="server", required=True,
- help="The server (URL) to send the data to.")
-parser.add_argument("-u", "--user",
- dest="user", required=True,
- help="Username for HTTP auth.")
-parser.add_argument("--password",
- dest="password", required=True,
- help="Password for HTTP auth.")
-args = parser.parse_args()
-pp = pprint.PrettyPrinter()
-log_file = sys.stdin if args.file == "-" else open(args.file, "r")
-
-results = parse_log.parse(log_file, parse_times = parse_log.PARSE_RAW)
-if args.commits:
- commits = set(parse_log.parse_git_commits(args.commits))
- results = filter(lambda r: r.commit in commits, results)
-results = list(results)
-
-for datapoint in results:
- times = '\n'.join(datapoint.times)
- commit = datapoint.commit
- print("Sending {}...".format(commit), end='')
- date = subprocess.check_output(['git', 'show', commit, '-s', '--pretty=%cI']).strip().decode('UTF-8')
- headers = {'X-Project': args.project, 'X-Branch': args.branch, 'X-Commit': commit, 'X-Config': args.config, 'X-Date': date}
- r = requests.post(args.server+"/build_times", data=times, headers=headers, auth=(args.user, args.password))
- print(" {}".format(r.text.strip()))
- r.raise_for_status()
diff --git a/benchmark/gitlab-extract.py b/benchmark/gitlab-extract.py
deleted file mode 100755
index 926b06ff0..000000000
--- a/benchmark/gitlab-extract.py
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/usr/bin/env python3
-import argparse, pprint, sys, glob, zipfile, subprocess
-import requests
-import parse_log
-
-def last(it):
- r = None
- for i in it:
- r = i
- return r
-
-def first(it):
- for i in it:
- return i
- return None
-
-def req(path):
- url = '%s/api/v3/%s' % (args.server, path)
- r = requests.get(url, headers={'PRIVATE-TOKEN': args.private_token})
- r.raise_for_status()
- return r
-
-# read command-line arguments
-parser = argparse.ArgumentParser(description='Extract iris-coq build logs from GitLab')
-parser.add_argument("-t", "--private-token",
- dest="private_token", required=True,
- help="The private token used to authenticate access.")
-parser.add_argument("-s", "--server",
- dest="server", default="https://gitlab.mpi-sws.org/",
- help="The GitLab server to contact.")
-parser.add_argument("-p", "--project",
- dest="project", default="FP/iris-coq",
- help="The name of the project on GitLab.")
-parser.add_argument("-f", "--file",
- dest="file", required=True,
- help="Filename to store the load in.")
-parser.add_argument("-c", "--commits",
- dest="commits",
- help="The commits to fetch. Default is everything since the most recent entry in the log file.")
-parser.add_argument("-a", "--artifacts",
- dest="artifacts",
- help="Location of the artifacts (following GitLab's folder structure). If not given (which should be the common case), the artifacts will be downloaded from GitLab.")
-parser.add_argument("-b", "--blacklist-branch",
- dest="blacklist_branch",
- help="Skip the commit if it is contained in the given branch.")
-args = parser.parse_args()
-log_file = sys.stdout if args.file == "-" else open(args.file, "a")
-
-# determine commit, if missing
-if args.commits is None:
- if args.file == "-":
- raise Exception("If you do not give explicit commits, you have to give a logfile so that we can determine the missing commits.")
- last_result = last(parse_log.parse(open(args.file, "r"), parse_times = parse_log.PARSE_NOT))
- args.commits = "{}..origin/master".format(last_result.commit)
-
-projects = req("projects?per_page=512")
-project = first(filter(lambda p: p['path_with_namespace'] == args.project, projects.json()))
-if project is None:
- sys.stderr.write("Project not found.\n")
- sys.exit(1)
-
-BREAK = False
-for commit in parse_log.parse_git_commits(args.commits):
- if BREAK:
- break
- # test to skip the commit
- if args.blacklist_branch is not None:
- branches = subprocess.check_output(["git", "branch", "-r", "--contains", commit]).decode("utf-8")
- if args.blacklist_branch in map(lambda x: x.strip(), branches.split('\n')):
- continue
- # Find out more about the commit
- print("Fetching {}...".format(commit), end='')
- commit_data = req("/projects/{}/repository/commits/{}".format(project['id'], commit))
- if commit_data.status_code != 200:
- raise Exception("Commit not found?")
- builds = req("/projects/{}/repository/commits/{}/builds".format(project['id'], commit))
- if builds.status_code != 200:
- raise Exception("Build not found?")
- # iterate over builds by decreasing ID, and look for the artifact
- found_build = False
- for build in builds.json():
- if build['status'] in ('created', 'pending', 'running'):
- # build still not yet done, don't fetch this or any later commit
- BREAK = True
- print(" build still in progress, aborting")
- break
- if build['status'] != 'success':
- # build failed or cancelled, skip to next
- continue
- # now fetch the build times
- if args.artifacts:
- artifact_zip = glob.glob('{}/*/{}/{}/artifacts.zip'.format(args.artifacts, project['id'], build['id']))
- if not artifact_zip:
- # no artifact at this build, try another one
- continue
- assert len(artifact_zip) == 1, "Found too many artifacts"
- artifact_zip = artifact_zip[0]
- with zipfile.ZipFile(artifact_zip) as artifact:
- with artifact.open('build-time.txt') as build_times:
- # Output into log file
- log_file.write("# {}\n".format(commit))
- log_file.write(build_times.read().decode('UTF-8'))
- log_file.flush()
- else:
- build_times = requests.get("{}/builds/{}/artifacts/raw/build-time.txt".format(project['web_url'], build['id']))
- if build_times.status_code != 200:
- # no artifact at this build, try another one
- continue
- # Output in the log file format
- log_file.write("# {}\n".format(commit))
- log_file.write(build_times.text)
- log_file.flush()
- # don't fetch another build
- found_build = True
- print(" success")
- break
- if not found_build and not BREAK:
- print(" found no succeessful build")
diff --git a/benchmark/parse_log.py b/benchmark/parse_log.py
deleted file mode 100644
index 310c7822d..000000000
--- a/benchmark/parse_log.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import re, subprocess
-
-class Result:
- def __init__(self, commit, times):
- self.commit = commit
- self.times = times
-
-PARSE_NOT = 0
-PARSE_RAW = 1
-PARSE_FULL = 2
-
-def parse(file, parse_times = PARSE_FULL):
- '''[file] should be a file-like object, an iterator over the lines.
- yields a list of Result objects.'''
- commit_re = re.compile("^# ([a-z0-9]+)$")
- time_re = re.compile("^([a-zA-Z0-9_/-]+) \((real|user): ([0-9.]+).* mem: ([0-9]+) ko\)$")
- commit = None
- times = None
- for line in file:
- line = line.strip()
- # next commit?
- m = commit_re.match(line)
- if m is not None:
- # previous commit, if any, is done now
- if commit is not None:
- yield Result(commit, times)
- # start recording next commit
- commit = m.group(1)
- if parse_times != PARSE_NOT:
- times = [] if parse_times == PARSE_RAW else {} # reset the recorded times
- continue
- # next file time?
- m = time_re.match(line)
- if m is not None:
- if times is not None:
- if parse_times == PARSE_RAW:
- times.append(line)
- else:
- name = m.group(1)
- time = float(m.group(2))
- times[name] = time
- continue
- # nothing else we know about, ignore
- print("Ignoring line",line,"(in commit {})".format(commit))
- # end of file. previous commit, if any, is done now.
- if commit is not None:
- yield Result(commit, times)
-
-def parse_git_commits(commits):
- '''Returns an iterable of SHA1s'''
- if commits.find('..') >= 0:
- # a range of commits
- commits = subprocess.check_output(["git", "rev-list", commits])
- else:
- # a single commit
- commits = subprocess.check_output(["git", "rev-parse", commits])
- output = commits.decode("utf-8").strip()
- if not output: # empty output
- return []
- return reversed(output.split('\n'))
diff --git a/benchmark/visualize.py b/benchmark/visualize.py
deleted file mode 100755
index 06eba983d..000000000
--- a/benchmark/visualize.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-import argparse, sys, pprint, itertools
-import matplotlib.pyplot as plt
-import parse_log
-
-markers = itertools.cycle([(3, 0), (3, 0, 180), (4, 0), (4, 0, 45), (8, 0)])
-
-# read command-line arguments
-parser = argparse.ArgumentParser(description='Visualize iris-coq build times')
-parser.add_argument("-f", "--file",
- dest="file", required=True,
- help="Filename to get the data from.")
-parser.add_argument("-t", "--timings", nargs='+',
- dest="timings",
- help="The names of the Coq files (with or without the extension) whose timings should be extracted")
-parser.add_argument("-c", "--commits",
- dest="commits",
- help="Restrict the graph to the given commits.")
-args = parser.parse_args()
-pp = pprint.PrettyPrinter()
-log_file = sys.stdin if args.file == "-" else open(args.file, "r")
-
-results = parse_log.parse(log_file, parse_times = parse_log.PARSE_FULL)
-if args.commits:
- commits = set(parse_log.parse_git_commits(args.commits))
- results = filter(lambda r: r.commit in commits, results)
-results = list(results)
-
-timings = list(map(lambda t: t[:-2] if t.endswith(".v") else t, args.timings))
-for timing in timings:
- plt.plot(list(map(lambda r: r.times.get(timing), results)), marker=next(markers), markersize=8)
-plt.legend(timings, loc = 'upper left', bbox_to_anchor=(1.05, 1.0))
-plt.xticks(range(len(results)), list(map(lambda r: r.commit[:7], results)), rotation=70)
-plt.subplots_adjust(bottom=0.2, right=0.7) # more space for the commit labels and legend
-
-plt.xlabel('Commit')
-plt.ylabel('Time (s)')
-plt.title('Time to compile files')
-plt.grid(True)
-plt.show()
--
GitLab