219 lines
9.1 KiB
Python
219 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import subprocess
|
|
import os, sys, stat, errno, shutil
|
|
import logging
|
|
import requests
|
|
import simplejson as json
|
|
import datetime
|
|
from tabulate import tabulate
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
logger = logging.getLogger('build')
|
|
logger.setLevel(logging.DEBUG)
|
|
fh = logging.FileHandler('debug.log')
|
|
fh.setLevel(logging.DEBUG)
|
|
eh = logging.FileHandler('error.log')
|
|
eh.setLevel(logging.INFO)
|
|
ch = logging.StreamHandler()
|
|
ch.setLevel(logging.DEBUG)
|
|
# add the handlers to the logger
|
|
logger.addHandler(fh)
|
|
logger.addHandler(eh)
|
|
logger.addHandler(ch)
|
|
|
|
# TODO: add multiple label support. Joining on a comma does an AND not an OR in github api, so we need to make multiple requests
|
|
LABEL_TO_FETCH = 'pr:bleeding-edge-merge'
|
|
|
|
GITHUB_API = 'https://api.github.com/repos'
|
|
PULL_REPOS = ['https://github.com/citra-emu/citra-bleeding-edge', 'https://github.com/citra-emu/citra']
|
|
MAIN_REPO = PULL_REPOS[1]
|
|
PUSH_REPO = 'git@github.com:citra-emu/citra-bleeding-edge'
|
|
# PUSH_REPO = 'git@github.com:jroweboy/lemon'
|
|
|
|
GITHUB_API_HEADERS = {
|
|
'User-Agent': 'citra-emu/lemonbot'
|
|
}
|
|
GITHUB_API_PARAMS = {
|
|
'client_id': 'xxxx',
|
|
'client_secret': 'yyyy'
|
|
}
|
|
|
|
class TagMergeBot:
|
|
''' Fetches all the pull requests on a repository and merges all the pull requests with a specified tag on them. '''
|
|
def __init__(self, main_repo, pull_repos, push_repo):
|
|
repositories = []
|
|
for repo in pull_repos:
|
|
path_split = urlparse(repo).path.split("/")
|
|
repository = {}
|
|
repository['url'] = repo
|
|
repository['owner'] = path_split[1]
|
|
repository['name'] = path_split[2]
|
|
repository['remote_name'] = '{owner}_{name}'.format(owner=repository['owner'], name=repository['name'])
|
|
repositories.append(repository)
|
|
self.repos = repositories
|
|
self.previous_prs = {}
|
|
self.current_prs = {}
|
|
self.base_path = os.path.abspath(os.path.dirname(__file__))
|
|
self.tracking_path = os.path.join(base_dir, 'tracking_repo')
|
|
self.branch_name = 'bleeding_edge'
|
|
self.main_repo = main_repo
|
|
self.push_repo = {"name": "push_remote", "url": push_repo}
|
|
|
|
def load_last_prs(self):
|
|
self.previous_prs = {}
|
|
if os.path.isfile('previous_run.json'):
|
|
with open('previous_run.json') as infile:
|
|
self.previous_prs = json.load(infile)
|
|
|
|
def save_prs_to_file(self):
|
|
with open('previous_run.json', 'w') as outfile:
|
|
json.dump(self.current_prs, outfile)
|
|
|
|
def fetch_latest_prs(self):
|
|
# Fetch all the pull requests with the label (up to 100... hopefully citra doesn't ever get more than that)
|
|
self.current_prs = {}
|
|
for repo in self.repos:
|
|
self.current_prs[repo['remote_name']] = []
|
|
res = requests.get(GITHUB_API+'/{owner}/{repo}/issues?labels={labels}&per_page=100'.format(labels=LABEL_TO_FETCH, owner=repo['owner'], repo=repo['name']), params=GITHUB_API_PARAMS, headers=GITHUB_API_HEADERS)
|
|
if res.status_code != 200:
|
|
logger.error("Could not retrive pull requests from github. Status: {status}".format(status=res.status_code))
|
|
return False
|
|
issues = res.json()
|
|
for issue in issues:
|
|
pr_response = requests.get(issue['pull_request']['url'], params=GITHUB_API_PARAMS, headers=GITHUB_API_HEADERS)
|
|
if res.status_code != 200:
|
|
logger.warn("Couldn't fetch the PRs details for PR {pr}. Status: {status}".format(pr=issue[''], status=res.status_code))
|
|
continue
|
|
pr = pr_response.json()
|
|
self.current_prs[repo['remote_name']].append({"number": pr['number'], "commit": pr['head']['sha'], "ref": pr['head']['ref'], "author": pr['user']['login']})
|
|
return True
|
|
|
|
def reclone(self):
|
|
# TODO: Update the branches that need updating instead of wiping the folder and fresh cloning.
|
|
if os.path.isdir(self.tracking_path):
|
|
logger.debug("Removing tracking_repo")
|
|
shutil.rmtree(self.tracking_path, ignore_errors=False, onerror=handleRemoveReadonly)
|
|
logger.debug("Cloning the repository")
|
|
_git("clone", "--recursive", self.main_repo, self.tracking_path, cwd=self.base_path)
|
|
# setup the pull/push remotes
|
|
for repo in self.repos:
|
|
_git("remote", "add", repo['remote_name'], repo['url'], cwd=self.tracking_path)
|
|
_git("remote", "add", self.push_repo['name'], self.push_repo['url'], cwd=self.tracking_path)
|
|
|
|
def pull_branches(self):
|
|
for repo in self.repos:
|
|
for pr in self.current_prs[repo['remote_name']]:
|
|
_git("fetch", repo['remote_name'], "pull/{0}/head:{1}".format(pr['number'], pr['ref']), cwd=self.tracking_path)
|
|
|
|
def merge(self):
|
|
|
|
readme_content = ""
|
|
|
|
# _git("branch", "-D", branch_name, cwd=self.repo_path)
|
|
# _git("checkout", "master", cwd=self.repo_path)
|
|
# checkout a new branch based off master
|
|
_git("checkout", "-b", self.branch_name, cwd=self.tracking_path)
|
|
total_failed = 0
|
|
merges = []
|
|
for repo in self.repos:
|
|
for pr in self.current_prs[repo['remote_name']]:
|
|
merge = [pr['number'], pr['ref'], "`" + pr['commit'] + "`", pr['author']]
|
|
retval, _ = _git("merge", pr["ref"], cwd=self.tracking_path)
|
|
if retval != 0:
|
|
merge.append("Failed") # TODO: Link to a log to show why this happened
|
|
_git("merge", "--abort", cwd=self.tracking_path)
|
|
logger.warn("Branch ({0}, {1}) failed to merge.".format(pr["ref"], pr["number"]))
|
|
total_failed += 1
|
|
else:
|
|
merge.append("Merged")
|
|
merges.append(merge)
|
|
|
|
readme_content += "# lemonbot merge log\n\nScroll down for the original README.md!\n"
|
|
readme_content += "\n======\n\n"
|
|
readme_content += tabulate(merges, ["PR", "Ref", "Commit", "Author", "Status"], tablefmt="pipe") + "\n"
|
|
|
|
readme_path = os.path.join(self.tracking_path, 'README.md')
|
|
try:
|
|
with open(readme_path, 'r') as original_readme:
|
|
readme_content += "\nEnd of merge log. You can find the original README.md below the break.\n"
|
|
readme_content += "\n======\n\n"
|
|
readme_content += original_readme.read()
|
|
except IOError:
|
|
readme_content += "\nEnd of merge log. No original README.md existed.\n"
|
|
|
|
try:
|
|
with open(readme_path, 'w') as readme:
|
|
readme.write(readme_content)
|
|
readme.close()
|
|
_git("add", "README.md", cwd=self.tracking_path)
|
|
_git("commit", "-m", "lemonbot merge log", cwd=self.tracking_path)
|
|
except IOError:
|
|
logger.warn("Could not write README.md")
|
|
|
|
return total_failed
|
|
|
|
def sync(self):
|
|
_git("push", "-f", self.push_repo["name"], "master", cwd=self.tracking_path)
|
|
|
|
def remove_remote_branch(self):
|
|
_git("push", self.push_repo["name"], "--delete", self.branch_name, cwd=self.tracking_path)
|
|
|
|
def push(self):
|
|
_git("push", "-f", self.push_repo["name"], self.branch_name, cwd=self.tracking_path)
|
|
|
|
# TODO: Use python git instead of calling git from the shell.
|
|
def _git(*args, cwd=None):
|
|
command = ["git"] + list(args)
|
|
logger.debug(" git command: " + " ".join(command))
|
|
if not cwd:
|
|
logger.warn("Cowardly refusing to call the previous git command without a working directory")
|
|
return
|
|
p = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
stdout, stderr = p.communicate()
|
|
if stdout:
|
|
logger.debug(" stdout: " + stdout.decode("utf-8"))
|
|
if stderr:
|
|
logger.debug(" stderr: " + stderr.decode("utf-8"))
|
|
return p.returncode, stdout.decode("utf-8")
|
|
|
|
def handleRemoveReadonly(func, path, exc):
|
|
excvalue = exc[1]
|
|
if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
|
|
os.chmod(path, stat.S_IRWXU| stat.S_IRWXG| stat.S_IRWXO) # 0777
|
|
func(path)
|
|
else:
|
|
raise
|
|
|
|
if __name__ == "__main__":
|
|
logger.info("=== Starting build at {date} ===".format(date=datetime.datetime.now()))
|
|
base_dir = os.path.abspath(os.path.dirname(__file__))
|
|
|
|
t = TagMergeBot(MAIN_REPO, PULL_REPOS, PUSH_REPO)
|
|
ret = t.fetch_latest_prs()
|
|
if not ret:
|
|
logger.error("Error occurred. Aborting early")
|
|
sys.exit(-1)
|
|
logger.debug("Loading last run's information")
|
|
t.load_last_prs()
|
|
if t.previous_prs == t.current_prs:
|
|
logger.info("No changes since last run. Exiting.")
|
|
sys.exit(0)
|
|
if not t.current_prs:
|
|
logger.warn("No PRs to merge. Add the label {label} to some PRs".format(label=LABEL_TO_FETCH))
|
|
sys.exit(0)
|
|
# actually merge the branches and carry on.
|
|
# TODO: Rework this whole workflow to be sane.
|
|
# m = MergeBot(PULL_REPOS, PUSH_REPO)
|
|
t.reclone()
|
|
t.sync()
|
|
t.pull_branches()
|
|
failed = t.merge()
|
|
logger.info("Number of failed merges: {failed}".format(failed=failed))
|
|
t.remove_remote_branch()
|
|
t.push()
|
|
|
|
# save this runs information
|
|
t.save_prs_to_file()
|