Skip to content
Snippets Groups Projects
Commit 838322ad authored by Ralf Jung's avatar Ralf Jung
Browse files

remove some long-dead files

parent 031cb7cb
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
benchmark/.gitignore deleted 100644 → 0
+ 0
− 3
View file @ 031cb7cb
__pycache__
build-times*
gitlab-extract
benchmark/export.py deleted 100755 → 0
+ 0
− 50
View file @ 031cb7cb
#!/usr/bin/env python3
import argparse, sys, pprint, itertools, subprocess
import requests
import parse_log
# read command-line arguments
parser = argparse.ArgumentParser(description='Export iris-coq build times to grafana')
parser.add_argument("-f", "--file",
dest="file", required=True,
help="Filename to get the data from.")
parser.add_argument("-c", "--commits",
dest="commits",
help="Restrict the graph to the given commits.")
parser.add_argument("-p", "--project",
dest="project", required=True,
help="Project name sent to the server.")
parser.add_argument("-b", "--branch",
dest="branch", required=True,
help="Branch name sent to the server.")
parser.add_argument("--config",
dest="config", required=True,
help="The config string.")
parser.add_argument("-s", "--server",
dest="server", required=True,
help="The server (URL) to send the data to.")
parser.add_argument("-u", "--user",
dest="user", required=True,
help="Username for HTTP auth.")
parser.add_argument("--password",
dest="password", required=True,
help="Password for HTTP auth.")
args = parser.parse_args()
pp = pprint.PrettyPrinter()
log_file = sys.stdin if args.file == "-" else open(args.file, "r")
results = parse_log.parse(log_file, parse_times = parse_log.PARSE_RAW)
if args.commits:
commits = set(parse_log.parse_git_commits(args.commits))
results = filter(lambda r: r.commit in commits, results)
results = list(results)
for datapoint in results:
times = '\n'.join(datapoint.times)
commit = datapoint.commit
print("Sending {}...".format(commit), end='')
date = subprocess.check_output(['git', 'show', commit, '-s', '--pretty=%cI']).strip().decode('UTF-8')
headers = {'X-Project': args.project, 'X-Branch': args.branch, 'X-Commit': commit, 'X-Config': args.config, 'X-Date': date}
r = requests.post(args.server+"/build_times", data=times, headers=headers, auth=(args.user, args.password))
print(" {}".format(r.text.strip()))
r.raise_for_status()
benchmark/gitlab-extract.py deleted 100755 → 0
+ 0
− 118
View file @ 031cb7cb
#!/usr/bin/env python3
import argparse, pprint, sys, glob, zipfile, subprocess
import requests
import parse_log
def last(it):
r = None
for i in it:
r = i
return r
def first(it):
for i in it:
return i
return None
def req(path):
url = '%s/api/v3/%s' % (args.server, path)
r = requests.get(url, headers={'PRIVATE-TOKEN': args.private_token})
r.raise_for_status()
return r
# read command-line arguments
parser = argparse.ArgumentParser(description='Extract iris-coq build logs from GitLab')
parser.add_argument("-t", "--private-token",
dest="private_token", required=True,
help="The private token used to authenticate access.")
parser.add_argument("-s", "--server",
dest="server", default="https://gitlab.mpi-sws.org/",
help="The GitLab server to contact.")
parser.add_argument("-p", "--project",
dest="project", default="FP/iris-coq",
help="The name of the project on GitLab.")
parser.add_argument("-f", "--file",
dest="file", required=True,
help="Filename to store the load in.")
parser.add_argument("-c", "--commits",
dest="commits",
help="The commits to fetch. Default is everything since the most recent entry in the log file.")
parser.add_argument("-a", "--artifacts",
dest="artifacts",
help="Location of the artifacts (following GitLab's folder structure). If not given (which should be the common case), the artifacts will be downloaded from GitLab.")
parser.add_argument("-b", "--blacklist-branch",
dest="blacklist_branch",
help="Skip the commit if it is contained in the given branch.")
args = parser.parse_args()
log_file = sys.stdout if args.file == "-" else open(args.file, "a")
# determine commit, if missing
if args.commits is None:
if args.file == "-":
raise Exception("If you do not give explicit commits, you have to give a logfile so that we can determine the missing commits.")
last_result = last(parse_log.parse(open(args.file, "r"), parse_times = parse_log.PARSE_NOT))
args.commits = "{}..origin/master".format(last_result.commit)
projects = req("projects?per_page=512")
project = first(filter(lambda p: p['path_with_namespace'] == args.project, projects.json()))
if project is None:
sys.stderr.write("Project not found.\n")
sys.exit(1)
BREAK = False
for commit in parse_log.parse_git_commits(args.commits):
if BREAK:
break
# test to skip the commit
if args.blacklist_branch is not None:
branches = subprocess.check_output(["git", "branch", "-r", "--contains", commit]).decode("utf-8")
if args.blacklist_branch in map(lambda x: x.strip(), branches.split('\n')):
continue
# Find out more about the commit
print("Fetching {}...".format(commit), end='')
commit_data = req("/projects/{}/repository/commits/{}".format(project['id'], commit))
if commit_data.status_code != 200:
raise Exception("Commit not found?")
builds = req("/projects/{}/repository/commits/{}/builds".format(project['id'], commit))
if builds.status_code != 200:
raise Exception("Build not found?")
# iterate over builds by decreasing ID, and look for the artifact
found_build = False
for build in builds.json():
if build['status'] in ('created', 'pending', 'running'):
# build still not yet done, don't fetch this or any later commit
BREAK = True
print(" build still in progress, aborting")
break
if build['status'] != 'success':
# build failed or cancelled, skip to next
continue
# now fetch the build times
if args.artifacts:
artifact_zip = glob.glob('{}/*/{}/{}/artifacts.zip'.format(args.artifacts, project['id'], build['id']))
if not artifact_zip:
# no artifact at this build, try another one
continue
assert len(artifact_zip) == 1, "Found too many artifacts"
artifact_zip = artifact_zip[0]
with zipfile.ZipFile(artifact_zip) as artifact:
with artifact.open('build-time.txt') as build_times:
# Output into log file
log_file.write("# {}\n".format(commit))
log_file.write(build_times.read().decode('UTF-8'))
log_file.flush()
else:
build_times = requests.get("{}/builds/{}/artifacts/raw/build-time.txt".format(project['web_url'], build['id']))
if build_times.status_code != 200:
# no artifact at this build, try another one
continue
# Output in the log file format
log_file.write("# {}\n".format(commit))
log_file.write(build_times.text)
log_file.flush()
# don't fetch another build
found_build = True
print(" success")
break
if not found_build and not BREAK:
print(" found no succeessful build")
benchmark/parse_log.py deleted 100644 → 0
+ 0
− 60
View file @ 031cb7cb
import re, subprocess
class Result:
def __init__(self, commit, times):
self.commit = commit
self.times = times
PARSE_NOT = 0
PARSE_RAW = 1
PARSE_FULL = 2
def parse(file, parse_times = PARSE_FULL):
'''[file] should be a file-like object, an iterator over the lines.
yields a list of Result objects.'''
commit_re = re.compile("^# ([a-z0-9]+)$")
time_re = re.compile("^([a-zA-Z0-9_/-]+) \((real|user): ([0-9.]+).* mem: ([0-9]+) ko\)$")
commit = None
times = None
for line in file:
line = line.strip()
# next commit?
m = commit_re.match(line)
if m is not None:
# previous commit, if any, is done now
if commit is not None:
yield Result(commit, times)
# start recording next commit
commit = m.group(1)
if parse_times != PARSE_NOT:
times = [] if parse_times == PARSE_RAW else {} # reset the recorded times
continue
# next file time?
m = time_re.match(line)
if m is not None:
if times is not None:
if parse_times == PARSE_RAW:
times.append(line)
else:
name = m.group(1)
time = float(m.group(2))
times[name] = time
continue
# nothing else we know about, ignore
print("Ignoring line",line,"(in commit {})".format(commit))
# end of file. previous commit, if any, is done now.
if commit is not None:
yield Result(commit, times)
def parse_git_commits(commits):
'''Returns an iterable of SHA1s'''
if commits.find('..') >= 0:
# a range of commits
commits = subprocess.check_output(["git", "rev-list", commits])
else:
# a single commit
commits = subprocess.check_output(["git", "rev-parse", commits])
output = commits.decode("utf-8").strip()
if not output: # empty output
return []
return reversed(output.split('\n'))
benchmark/visualize.py deleted 100755 → 0
+ 0
− 40
View file @ 031cb7cb
#!/usr/bin/env python3
import argparse, sys, pprint, itertools
import matplotlib.pyplot as plt
import parse_log
markers = itertools.cycle([(3, 0), (3, 0, 180), (4, 0), (4, 0, 45), (8, 0)])
# read command-line arguments
parser = argparse.ArgumentParser(description='Visualize iris-coq build times')
parser.add_argument("-f", "--file",
dest="file", required=True,
help="Filename to get the data from.")
parser.add_argument("-t", "--timings", nargs='+',
dest="timings",
help="The names of the Coq files (with or without the extension) whose timings should be extracted")
parser.add_argument("-c", "--commits",
dest="commits",
help="Restrict the graph to the given commits.")
args = parser.parse_args()
pp = pprint.PrettyPrinter()
log_file = sys.stdin if args.file == "-" else open(args.file, "r")
results = parse_log.parse(log_file, parse_times = parse_log.PARSE_FULL)
if args.commits:
commits = set(parse_log.parse_git_commits(args.commits))
results = filter(lambda r: r.commit in commits, results)
results = list(results)
timings = list(map(lambda t: t[:-2] if t.endswith(".v") else t, args.timings))
for timing in timings:
plt.plot(list(map(lambda r: r.times.get(timing), results)), marker=next(markers), markersize=8)
plt.legend(timings, loc = 'upper left', bbox_to_anchor=(1.05, 1.0))
plt.xticks(range(len(results)), list(map(lambda r: r.commit[:7], results)), rotation=70)
plt.subplots_adjust(bottom=0.2, right=0.7) # more space for the commit labels and legend
plt.xlabel('Commit')
plt.ylabel('Time (s)')
plt.title('Time to compile files')
plt.grid(True)
plt.show()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment