First working version

Author: Jeroen Baten

bugzilla2github.py

@@ -18,13 +18,15 @@
 # 2. Create a separate repo for testing purposes. THERE IS NO UNDO!
 # 3. Copy bugzilla2github.conf.sample to bugzilla2github.conf
 #    - Change all settings to fit your setup
-# 4. Run the script. Good luck....
+# 4. Please understand there is not API to delete issues. So test thoroughly before the final run!
+# 5. Run the script. Good luck....
 
-import json, getopt, os, pprint, re, requests, sys, time, xml.etree.ElementTree
+import json, requests, sys
 import psycopg2, psycopg2.extras
 import ConfigParser
-from pprint import pprint,pformat
-from urlparse import urljoin
+from pprint import pprint
+from dateutil.tz import tzlocal
+
 
 reload(sys)
 sys.setdefaultencoding('utf-8')
@@ -68,7 +70,8 @@ def read_bugs(conn):
         # create a new cursor object
         cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
         # execute the SELECT statement
-        cur.execute(""" SELECT * from bugs where bug_id=1295 """,)
+        #cur.execute(""" SELECT * from bugs where bug_id=1284 """,)
+        cur.execute(""" SELECT * from bugs order by bug_id  """, )
         colnames = [desc[0] for desc in cur.description]
         results = cur.fetchall()
     except (Exception, psycopg2.DatabaseError) as error:
@@ -174,8 +177,69 @@ def get_reporter(reporters,reporter_id):
     reporterstr=reporterobj["realname"]+" \<<"+reporterobj["login_name"]+">\>"
     return reporterstr
 
+def get_components(conn):
+    """
+    Retrieve all components from Bugzilla database
+    :param conn:  database connection object
+    :return: object with all results
+    """
+    try:
+        # create a new cursor object
+        cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        # execute the SELECT statement
+        cur.execute(""" SELECT * FROM components """)
+        results = cur.fetchall()
+        colnames = [desc[0] for desc in cur.description]
+    except (Exception, psycopg2.DatabaseError) as error:
+        print(error)
+    # Build indexed dictionary from results
+    components={}
+    for result in results:
+        components[result["id"]]=result
+    return components,colnames
+
+
+def get_component(components,component_id):
+    """
+    routine to find componentname for componentid
+    :param components: dictionary with all components, indexed on componentid
+    :param component_id: id of component to find
+    :return: string componentname
+    """
+    # ['userid', 'login_name', 'cryptpassword', 'realname', 'disabledtext', 'disable_mail', 'mybugslink', 'extern_id']
+    componentobj=components[component_id]
+    componentstr=componentobj["name"]
+    return componentstr
+
+def get_duplicates(conn):
+    """
+        Because we have relative few duplicates (55) compared to the total number of bugs (1709)
+        we retrieve all duplicates from Bugzilla database into dictionary of lists for quick lookup
+        :param conn:  database connection object
+        :return: object with all results (a dictionary of lists)
+        """
+    duplicates=dict()
+    try:
+        # create a new cursor object
+        cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        # execute the SELECT statement
+        cur.execute(""" SELECT * FROM duplicates """ )
+        results = cur.fetchall()
+
+        for result in results:
+            if duplicates.get(result["dupe_of"]) is None:
+                duplicates[result["dupe_of"]]=list()
+                duplicates[result["dupe_of"]].append(result["dupe"])
+            else:
+                duplicates[result["dupe_of"]].append(result["dupe"])
+
+    except (Exception, psycopg2.DatabaseError) as error:
+        print(error)
+
+    return duplicates
+
 
-def create_body(reporters,bug):
+def create_body(reporters,duplicates,bug):
     """
     Routine to create issue body text
     :param bug: object containing bug record
@@ -191,6 +255,37 @@ def create_body(reporters,bug):
     # CC: [email protected]
     #
     # Last updated: 2012-09-10 05:56:17 +0000
+    #
+    # Example record info
+    # bug_id -> 1295
+    # assigned_to -> 6
+    # bug_file_loc ->
+    # bug_severity -> critical
+    # bug_status -> RESOLVED
+    # creation_ts -> 2011-12-14 18:05:08
+    # delta_ts -> 2012-01-13 10:31:16
+    # short_desc -> TaskSource is null for some TaskElements
+    # op_sys -> All
+    # priority -> P5
+    # product_id -> 2
+    # rep_platform -> All
+    # reporter -> 6
+    # version -> libreplan-1.2 (1.2.x)
+    # component_id -> 10
+    # resolution -> FIXED
+    # target_milestone -> ---
+    # qa_contact -> None
+    # status_whiteboard ->
+    # votes -> 0
+    # lastdiffed -> 2012-01-13 10:31:16
+    # everconfirmed -> 1
+    # reporter_accessible -> 1
+    # cclist_accessible -> 1
+    # estimated_time -> 0.00
+    # remaining_time -> 0.00
+    # deadline -> None
+    # alias -> None
+    # cf_browser -> ---
     body=""
 
     # ret["body"].append("# Bugzilla Bug ID: %d" % ret["number"])
@@ -212,19 +307,18 @@ def create_body(reporters,bug):
     # ret["body"].append("Version: " + ret["version"])
     body+="Version: " + str(bug["version"])+"\n"
 
-    # [I'm an inline-style link](https://www.google.com)
-    body+="\n---\n"
-    body+="(Note: this issue was migrated automatically with [bugzilla2github.py tool](https://github.com/LibrePlan/bugzilla2github) )\n"
-
-
     # if "cc" in bug:
     #     ret["body"].append("CC:   " + ", ".join(emails_convert(bug.pop("cc"))))
     # TODO: chose not to support this. Use Link to original issue to find info
+
     # # Extra information
     # ret["body"].append("")
     # if "dup_id" in bug:
     #     ret["body"].append("Duplicates:   " + ids_convert(bug.pop("dup_id")))
-    # TODO: chose not to support this. Use Link to original issue to find info
+    if duplicates.get(bug["bug_id"]) is not None:
+        for duplicate in duplicates[bug["bug_id"]]:
+            body+="Duplicate: [" + str(duplicate) +"]("+BUGZILLA_URL+"show_bug.cgi?id="+str(bug["bug_id"])+"))\n"
+
     # if "dependson" in bug:
     #     ret["body"].append("Depends on:   " + ids_convert(bug.pop("dependson")))
     # TODO: chose not to support this. Use Link to original issue to find info
@@ -233,9 +327,14 @@ def create_body(reporters,bug):
     # TODO: chose not to support this. Use Link to original issue to find info
     # if "see_also" in bug:
     #     ret["body"].append("See also:     " + bug.pop("see_also"))
-    # TODO: chose not to support this. Use Link to original issue to find info
+
     # ret["body"].append("Last updated: " + ret["updated_at"])
+    body+="Last updated: " + str(bug["delta_ts"]) +"\n"
     # ret["body"].append("")
+    # [I'm an inline-style link](https://www.google.com)
+    body+="\n---\n"
+    body+="(Note: this issue was migrated automatically with [bugzilla2github.py tool](https://github.com/LibrePlan/bugzilla2github) )\n"
+
     return body
 
 # Start connection to database
@@ -247,26 +346,31 @@ def create_body(reporters,bug):
 
 # read all bug reporters
 reporters,colnames=get_reporters(conn)
+
+# read all components
+components,colnames=get_components(conn)
+
+# Create a duplicates lookup dictionary of lists
+duplicates=get_duplicates(conn)
+
 # read all bug reports
 bugs,colnames=read_bugs(conn)
+
 #pprint(bugs)
 #pprint(colnames)
 issue={}
 for bug in bugs:
     bug_id=bug['bug_id']
-    for field in colnames:
-       print str(field) + " -> " + str(bug[str(field)])
+    # for field in colnames:
+    #    print str(field) + " -> " + str(bug[str(field)])
     print "bug_id",bug_id
-    #print "bug_severity",bug['bug_severity']
-    #print "bug_status", bug['bug_status']
-    #print "short_desc", bug['short_desc']
-    #print "priority",bug['priority']
-    #print "version",bug['version']
-    #print "resolution",bug['resolution']
 
-    body=create_body(reporters,bug)
+    # Check for duplicates
+    #duplicates = find_duplicates(conn,bug_id)
 
-    # Create issues object to send to GitHub
+    body=create_body(reporters,duplicates,bug)
+
+    # Create issues object to send to GitHub: only title and body are needed.
     # And an issue with a comment only needs the comment body added:
     #
     # {
@@ -281,17 +385,62 @@ def create_body(reporters,bug):
     #   ]
     # }
 
+    # Complete info:
+    # "title": "Imported from some other system",
+    # "body": "...",
+    # "created_at": "2014-01-01T12:34:58Z",
+    # "closed_at": "2014-01-02T12:24:56Z",
+    # "updated_at": "2014-01-03T11:34:53Z",
+    # "assignee": "jonmagic",
+    # "milestone": 1,
+    # "closed": true,
+    # "labels": [
+    #     "bug",
+    #     "low"
+    # ]
+
+    if bug["bug_status"]=="RESOLVED":
+        closed=True
+    else:
+        closed=False
+
+    # Let's create some labels
+    # NOTE to self: An empty label means issue not added to GitHub!
+    labels=[]
+    if bug["bug_severity"]:
+        labels.append(bug["bug_severity"])
+    if bug["priority"]:
+        labels.append(bug["priority"])
+    if bug["resolution"]:
+        labels.append(bug["resolution"].lower())
+    bug_component=get_component(components,bug["component_id"])
+    if bug_component:
+        labels.append(bug_component)
+    #labels.append("mytest")
+
+    # TODO We currently do not know WHEN an issue was closed, so do not store it in the issue.
+    # Well, we could find it through bug_activity table but too much hassle actually.
+    # Good news is, that we do close an issue when it is closed
+    # "closed_at":str(bug["delta_ts"].replace(tzinfo=tzlocal()).isoformat()),
     issue={"issue":   {"title":bug['short_desc'],
-                       "body":body}}
+                       "body":body,
+                       "created_at": str(bug["creation_ts"].replace(tzinfo=tzlocal()).isoformat()),
+                       "closed": closed,
+                       "labels": labels,
+                       }
+           }
+
+    #pprint(issue)
 
     issuecomments=[]
     pprint(issuecomments)
     comments,colnames=read_comments(conn,bug_id)
-    pprint(colnames)
     for comment in comments:
         # ['comment_id', 'bug_id', 'who', 'bug_when', 'work_time', 'thetext', 'isprivate', 'already_wrapped', 'type', 'extra_data']
         comment_id=comment["comment_id"]
         attach_id=comment["extra_data"]
+        comment_type=comment["type"] # Very important: 5=attachment
+
         issuecomment=""
         # pprint(comment)
         print " comment_id", comment["comment_id"]
@@ -302,14 +451,13 @@ def create_body(reporters,bug):
         # ret.append("Date: " + comment.pop("bug_when"))
         issuecomment += "Date: " + str(comment["bug_when"]) + "\n"
         # ret.append("From: " + email_convert(comment.pop("who"),
-        issuecomment += "From: " + get_reporter(reporters, comment["who"]) + "\n"
+        issuecomment += "From: " + get_reporter(reporters, comment["who"]) + "\n\n"
         #                     comment.pop("who.name", None)))
-        # ret.append("")
         # ret.append(comment.pop("thetext", ""))
         issuecomment += comment["thetext"] + "\n\n"
 
 
-        if attach_id is not None:
+        if comment_type==5 and attach_id is not None:
             print " attach_id",attach_id
             # check attachment record for this comment
             attachment,colnames=read_attachment(conn,attach_id)
@@ -318,7 +466,6 @@ def create_body(reporters,bug):
             # 'attach_id', 'bug_id', 'creation_ts', 'modification_time', 'description', 'mimetype', 'ispatch', 'filename',
             #  'submitter_id', 'isobsolete', 'isprivate', 'isurl']
             #pprint(attachment)
-            #attachment=attachment.pop
             filename=attachment["filename"]
             #filesize=attachment[""]
             print " We have an attachment:",filename
@@ -327,41 +474,52 @@ def create_body(reporters,bug):
             #                       attach.pop("type"), attach.pop("size")))
             # ret.append("> Description:   " + attach.pop("desc"))
             # TODO: create link to repo file as: https://github.com/LibrePlan/bugzilla2githubattachments/blob/master/LICENSE
-            issuecomment += "\n---\n\nAttached file: [" + filename + "](" + FILEREPO_URL + str(bug_id) + '_' +  str(comment_id) + "_" + filename + ")\n"
-            issuecomment += "Description: " + attachment["description"] + "\n---\n"
+            issuecomment += "\n---\nAttached file: [" + filename + "](" + FILEREPO_URL + str(bug_id) + '_' +  str(comment_id) + "_" + filename + ")\n"
+            issuecomment += "File description: " + attachment["description"] + "\n"
 
         # add stuff to issuecomments list
         issuedict = dict()
         issuedict["body"] = issuecomment
         issuecomments.append(issuedict)
 
-
-    #issue["issue"].append(issuecomments)
-    #commentsdict={ "comments", issuecomments }
-    #commentsdict=dict()
-    #commentsdict.setdefault("comments", issuecomments)
     issue["comments"]=issuecomments
     # What have we created?
+    print "*" * 40 + " Final issue object " + "*" * 40
     pprint(issue)
 
     # Now let's try to add this issue to a github repo
     # https://api.github.com/repos/${GITHUB_USERNAME}/foo/import/issues
     urlparts=(str(GITHUB_URL),"repos" ,str(GITHUB_OWNER) , str(GITHUB_REPO) , "import/issues")
     url="/".join(urlparts)
-    # if url[0] == "/":
-    #     u = "%s%s" % (GITHUB_URL, url)
-    # else:
-    #     u = "%s/repos/%s/%s/%s" % (GITHUB_URL, GITHUB_OWNER, GITHUB_REPO, url)
+
     pprint(url)
     d=issue
-    #sys.exit(4)
-    #result = requests.post(u, params={"access_token": GITHUB_TOKEN,
     headers={"Authorization": "token "+ GITHUB_TOKEN,
              "Accept": "application/vnd.github.golden-comet-preview+json" }
     result=requests.post(url, headers=headers, data = json.dumps(d))
+    print result.status_code
+    # Check if something went wrong
+    if result.status_code<>202:
+        print result.json()
+        result_dict=result.json()
+        # Let's check if it all worked
+        result_id=result_dict["id"]
+        import_issues_url=result_dict["import_issues_url"]
+        print "Result ID = "+str(result_id)
+        print "import_issues_url= " + str(result_dict["import_issues_url"])
+        # First get ID of issue added
+        # Next request result
+        # curl -H "Authorization: token ${GITHUB_TOKEN}" \
+        #-H "Accept: application/vnd.github.golden-comet-preview+json" \
+        #https://api.github.com/repos/#{GITHUB_USERNAME}/foo/import/issues/7
+        url2=import_issues_url + "/"+ str(result_id)
+        #print(url2)
+        result2=requests.get(import_issues_url, headers=headers, data = json.dumps(d))
+        pprint(result2)
+        print result2.json()
+        # Bail out on error
+        sys.exit(2)
 
-    pprint(result)
-    print result.json()
 # The end of handling all bugs
 
 # close the communication with the PostgresQL database

freeze.txt

@@ -4,5 +4,7 @@ idna==2.6
 pkg-resources==0.0.0
 psycopg2==2.7.4
 psycopg2-binary==2.7.4
+python-dateutil==2.6.1
 requests==2.18.4
+six==1.11.0
 urllib3==1.22