ceph-release-notes

gistfile1.txt

#!/usr/bin/env python
# Originally modified from A. Israel's script seen at
# https://gist.github.com/aisrael/b2b78d9dfdd176a232b9

from __future__ import print_function
import argparse
import github
import os
import re
import sys
import requests
import time

from git import Repo

# Regex patterns
fixes_re = re.compile(r"Fixes\:? #(\d+)")
reviewed_by_re = re.compile(r"Rev(.*)By", re.IGNORECASE)
merge_re = re.compile(r"Merge (pull request|PR) #(\d+).*")
signed_off_re = re.compile(r"Signed-off-by: (.+) <")
tracker_re = re.compile(r"http://tracker.ceph.com/issues/(\d+)")
rst_link_re = re.compile(r"([a-zA-Z0-9])_(\W)")

# FIXED: Prevent "eef" bug by being more specific with the prefix removal
release_re = re.compile(r"^(nautilus|octopus|pacific|quincy|reef|squid|tentacle):\s*", re.IGNORECASE)

prefixes = ['bluestore', 'build/ops', 'cephfs', 'cephx', 'cli', 'cmake',
            'common', 'core', 'crush', 'doc', 'fs', 'librados', 'librbd',
            'log', 'mds', 'mgr', 'mon', 'msg', 'objecter', 'osd', 'pybind',
            'rbd', 'rbd-mirror', 'rbd-nbd', 'rgw', 'tests', 'tools']

tracker_uri = "http://tracker.ceph.com/issues/{0}.json"

def get_original_issue(issue, verbose):
    try:
        r = requests.get(tracker_uri.format(issue), params={"include": "relations"}).json()
        if r["issue"]["tracker"]["name"] != "Backport":
            return issue
        if "relations" not in r["issue"]:
            return issue
        copied_to = [str(i['issue_id']) for i in r["issue"]["relations"] if i["relation_type"] == "copied_to"]
        return copied_to[0] if copied_to and len(copied_to) == 1 else issue
    except:
        return issue

def split_component(title, gh, number):
    title_re = '(' + '|'.join(prefixes) + ')(:.*)'
    match = re.match(title_re, title)
    if match:
        return match.group(1)+match.group(2)
    else:
        try:
            issue = gh.repos("ceph")("ceph").issues(number).get()
            issue_labels = {it['name'] for it in issue['labels']}
            if 'documentation' in issue_labels:
                return 'doc: ' + title
            item = set(prefixes).intersection(issue_labels)
            if item:
                return ",".join(sorted(item)) + ': ' + title
        except:
            pass
        return 'UNKNOWN: ' + title

def _title_message(commit, pr, strict):
    title = pr['title']
    message_lines = commit.message.split('\n')
    if strict or len(message_lines) < 1:
        return (title, None)
    lines = [line.strip() for line in message_lines[1:] if not reviewed_by_re.match(line) and line.strip()]
    if not lines or lines[0] == pr['title'].strip():
        return (title, None)
    if len(lines) == 1:
        return (lines[0], None)
    message = "    " + "\n    ".join(lines)
    return (title, message)

def make_release_notes(gh, repo, ref, cherry_picks, plaintext, html, markdown, verbose, strict, use_tags, include_pr_messages):
    issue2prs, pr2issues, pr2info, merges, merge_commits = {}, {}, {}, {}, {}

    for commit in repo.iter_commits(ref, merges=True):
        merge = merge_re.match(commit.summary)
        if merge:
            PR_num = merge.group(2)
            merges[PR_num] = "merge_commit"
            merge_commits[PR_num] = commit
            
    if cherry_picks:
        for PR in cherry_picks.split(','):
            merges[PR] = "cherry_pick"

    # RESTORED: Real-time progress printing
    for number in merges:
        print ("Considering PR#" + number)
        if int(number) < 1311: 
            print ("Ignoring low-numbered PR")
            continue
        
        pr = None
        for attempts in range(30):
            try:
                pr = gh.repos("ceph")("ceph").pulls(number).get()
                break
            except:
                time.sleep(2 * (attempts + 1))
        
        if not pr: continue

        commit = merge_commits.get(number) if merges[number] == "merge_commit" else None
        if merges[number] == "cherry_pick":
            try: commit = repo.commit(pr['merge_commit_sha'])
            except: pass

        title, message = _title_message(commit, pr, strict) if commit else (pr['title'], pr['body'])
        
        # FIXED: Safer prefix removal logic
        title = release_re.sub('', title)
        # Only strip whitespace, NOT specific characters that break names
        title = title.strip()
        
        if use_tags:
            title = split_component(title, gh, number)

        title = title.replace('*', r'\*')
        title = rst_link_re.sub(r'\1\_\2', title)
        
        issues = fixes_re.findall(pr['body'] or "") + tracker_re.findall(pr['body'] or "")
        authors = {pr['user']['login']: 1}
        if commit:
            for c in repo.iter_commits("{sha1}^1..{sha1}^2".format(sha1=commit.hexsha)):
                for author in re.findall(r"Signed-off-by:\s*(.*?)\s*<", c.message):
                    authors[author] = 1
        
        author_str = ", ".join(authors.keys())
        pr2info[number] = (author_str, title, message)
        for issue in set(issues):
            issue2prs.setdefault(issue, set([])).add(number)
            pr2issues.setdefault(number, set([])).add(issue)
        sys.stdout.write('.')

    print (" done collecting merges.")

    for pr, (author, title, message) in sorted(pr2info.items(), key=lambda x: x[1][1].lower()):
        # Formatting back to original RST style
        issue_list = [f"`issue#{i} <http://tracker.ceph.com/issues/{i}>`_" for i in pr2issues.get(pr, [])]
        issues_str = ", ".join(issue_list) + ", " if issue_list else ""
        
        print ("* {title} ({issues}`pr#{pr} <https://github.com/ceph/ceph/pull/{pr}>`_, {author})".format(
            title=title, issues=issues_str, pr=pr, author=author))
        
        if include_pr_messages and message:
            print (message)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Make ceph release notes', formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument("--rev", "-r", help="git revision range")
    parser.add_argument("--cherry_picks", "-c", help="PRs associated with cherry-picks")
    parser.add_argument("--token", default=os.getenv("GITHUB_ACCESS_TOKEN"), help="Github Access Token")
    parser.add_argument("--use-tags", default=False, action='store_true', help="Use github tags to guess the component")
    parser.add_argument("--strict", action='store_true', help="Strict mode")
    parser.add_argument("--include-pr-messages", default=False, action='store_true', help="Include full PR message")
    parser.add_argument("repo", help="path to ceph git repo")
    
    # Placeholders for original flags
    parser.add_argument("--text", "-t", action='store_true')
    parser.add_argument("--html", action='store_true')
    parser.add_argument("--markdown", action='store_true')
    parser.add_argument("--verbose", "-v", action='store_true')

    args = parser.parse_args()
    gh = github.GitHub(args.token)
    
    make_release_notes(gh, Repo(args.repo), args.rev, args.cherry_picks, args.text, args.html, args.markdown, args.verbose, args.strict, args.use_tags, args.include_pr_messages)