#!/usr/bin/python -tt
# -*- coding: utf-8 -*-
#
# Copyright © 2008 Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use, modify,
# copy, or redistribute it subject to the terms and conditions of the GNU
# General Public License v.2, or (at your option) any later version. This
# program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details. You should have received a copy of the GNU
# General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source
# code or documentation are not subject to the GNU General Public License and
# may only be used or replicated with the express permission of Red Hat, Inc.
#
# Red Hat Author(s): Toshio Kuratomi <tkuratom@redhat.com>
# Author(s): Mike Watters <valholla75@fedoraproject.org>
#
'''
sync information from the packagedb into bugzilla
This short script takes information about package onwership and imports it
into bugzilla.
'''
import sys
import os
import getopt
import xmlrpclib
import codecs
import smtplib
import bugzilla
from configobj import ConfigObj, flatten_errors
from email.Message import Message
from fedora.client.fas2 import AccountSystem
from fedora.client.pkgdb import PackageDB
from validate import Validator
import q
vldtr = Validator()
# configspec to set default values and validate types
configspec = '''
[global]
bugzilla.url = string(default = 'https://bugdev.devel.redhat.com/bugzilla-cvs/xmlrpc.cgi')
bugzilla.username = string(default = '')
bugzilla.password = string(default = '')
bugzilla.component_api = string(default = 'component.get')
fas.url = string(default = '')
fas.username = string(default = '')
fas.password = string(default = '')
notify.email = force_list(default = list(''))
debug = boolean(default = 'False')
'''.splitlines()
cfg = ConfigObj('/etc/pkgdb-sync-bugzilla.cfg', configspec = configspec)
res = cfg.validate(vldtr, preserve_errors=True)
for entry in flatten_errors(cfg, res):
section_list, key, error = entry
if error == False:
restore_default(key)
BZSERVER = cfg['global']['bugzilla.url']
BZUSER = cfg['global']['bugzilla.username']
BZPASS = cfg['global']['bugzilla.password']
BZCOMPAPI = cfg['global']['bugzilla.component_api']
FASURL = cfg['global']['fas.url']
FASUSER = cfg['global']['fas.username']
FASPASS = cfg['global']['fas.password']
NOTIFYEMAIL = cfg['global']['notify.email']
PKGDBSERVER = cfg['global']['pkgdbserver.url']
DRY_RUN = cfg['global']['debug']
class DataChangedError(Exception):
'''Raised when data we are manipulating changes while we're modifying it.'''
pass
class ProductCache(dict):
@q.t
def __init__(self, bz, acls):
self.bz = bz
self.acls = acls
# Ask bugzilla for a section of the pkglist.
# Save the information from the section that we want.
@q.t
def __getitem__(self, key):
try:
return q.q/super(ProductCache, self).__getitem__(key)
except KeyError:
# We can only cache products we have pkgdb information for
if key not in self.acls:
raise
if BZCOMPAPI == 'getcomponentsdetails':
q.q('getcomponentsdetail')
# Old API -- in python-bugzilla. But with current server, this
# gives ProxyError
products = self.server.getcomponentsdetails(key)
elif BZCOMPAPI == 'component.get':
q.q('component.get')
# Way that's undocumented in the partner-bugzilla api but works
# currently
pkglist = acls[key].keys()
query = [dict(product=key, component=p) for p in pkglist]
q.q(query)
raw_data = self.bz._proxy.Component.get(dict(names=query))
q.q(raw_data)
products = {}
for package in raw_data['components']:
# Reformat data to be the same as what's returned from
# getcomponentsdetails
product = dict(initialowner=package['default_assignee'],
description=package['description'],
initialqacontact=package['default_qa_contact'],
initialcclist=package['default_cc'])
products[package['name'].lower()] = product
q.q('past loop')
self[key] = products
return super(ProductCache, self).__getitem__(key)
class Bugzilla(object):
@q.t
def __init__(self, bzServer, username, password, acls):
self.bzXmlRpcServer = bzServer
self.username = username
self.password = password
self.server = bugzilla.Bugzilla(url=self.bzXmlRpcServer, user=self.username,password=self.password)
self.productCache = ProductCache(self.server, acls)
# Connect to the fedora account system
self.fas = AccountSystem(base_url=FASURL, username=FASUSER,
password=FASPASS)
self.userCache = self.fas.people_by_key(key='username',
fields=['bugzilla_email'])
def _get_bugzilla_email(self, username):
'''Return the bugzilla email address for a user.
First looks in a cache for a username => bugzilla email. If not found,
reloads the cache from fas and tries again.
'''
try:
return self.userCache[username]['bugzilla_email'].lower()
except KeyError:
person = self.fas.person_by_username(username)
self.userCache[username] = {'bugzilla_email': person.bugzilla_email}
return self.userCache[username]['bugzilla_email'].lower()
@q.t
def add_edit_component(self, package, collection, owner, description,
qacontact=None, cclist=None):
'''Add or update a component to have the values specified.
'''
# Turn the cclist into something usable by bugzilla
if not cclist or 'people' not in cclist:
initialCCList = list()
else:
initialCCList = [self._get_bugzilla_email(cc) for cc in \
cclist['people']]
# Add owner to the cclist so comaintainers taking over a bug don't
# have to do this manually
owner = self._get_bugzilla_email(owner)
if owner not in initialCCList:
initialCCList.append(owner)
# Lookup product
try:
q.q('here')
product = self.productCache[collection]
q.q('there')
except xmlrpclib.Fault as e:
# Output something useful in args
e.args = (e.faultCode, e.faultString)
raise
except xmlrpclib.ProtocolError as e:
e.args = ('ProtocolError', e.errcode, e.errmsg)
raise
pkgKey = package.lower()
if pkgKey in product:
# edit the package information
data = {}
# Grab bugzilla email for things changable via xmlrpc
if qacontact:
qacontact = self._get_bugzilla_email(qacontact)
else:
qacontact = 'extras-qa@fedoraproject.org'
# Check for changes to the owner, qacontact, or description
if product[pkgKey]['initialowner'] != owner:
data['initialowner'] = owner
if product[pkgKey]['description'] != description:
data['description'] = description
if product[pkgKey]['initialqacontact'] != qacontact and (
qacontact or product[pkgKey]['initialqacontact']):
data['initialqacontact'] = qacontact
if len(product[pkgKey]['initialcclist']) != len(initialCCList):
data['initialcclist'] = initialCCList
else:
for ccMember in product[pkgKey]['initialcclist']:
if ccMember not in initialCCList:
data['initialcclist'] = initialCCList
break
if data:
### FIXME: initialowner has been made mandatory for some
# reason. Asking dkl why.
data['initialowner'] = owner
# Changes occurred. Submit a request to change via xmlrpc
data['product'] = collection
data['component'] = package
if DRY_RUN:
print '[EDITCOMP] Changing via editComponent(%s, %s, "xxxxx")' % (
data, self.username)
print '[EDITCOMP] Former values: %s|%s|%s|%s' % (
product[pkgKey]['initialowner'],
product[pkgKey]['description'],
product[pkgKey]['initialqacontact'],
product[pkgKey]['initialcclist'])
else:
try:
self.server.editcomponent(data)
except xmlrpclib.Fault, e:
# Output something useful in args
e.args = (data, e.faultCode, e.faultString)
raise
except xmlrpclib.ProtocolError, e:
e.args = ('ProtocolError', e.errcode, e.errmsg)
raise
else:
# Add component
if qacontact:
qacontact = self._get_bugzilla_email(qacontact)
else:
qacontact = 'extras-qa@fedoraproject.org'
data = {'product': collection,
'component': package,
'description': description,
'initialowner': owner,
'initialqacontact': qacontact}
if initialCCList:
data['initialcclist'] = initialCCList
if DRY_RUN:
print '[ADDCOMP] Adding new component AddComponent:(%s, %s, "xxxxx")' % (
data, self.username)
else:
try:
self.server.addcomponent(data)
except xmlrpclib.Fault, e:
# Output something useful in args
e.args = (data, e.faultCode, e.faultString)
raise
def send_email(fromAddress, toAddress, subject, message):
'''Send an email if there's an error.
This will be replaced by sending messages to a log later.
'''
msg = Message()
msg.add_header('To', ','.join(toAddress))
msg.add_header('From', fromAddress)
msg.add_header('Subject', subject)
msg.set_payload(message)
smtp = smtplib.SMTP('bastion')
smtp.sendmail(fromAddress, toAddress, msg.as_string())
smtp.quit()
if __name__ == '__main__':
sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
opts, args = getopt.getopt(sys.argv[1:], '', ('usage', 'help'))
if len(args) > 0:
print '''
Usage: bz-make-components.py
Sync package information from the package database to bugzilla.
'''
sys.exit(1)
# Non-fatal errors to alert people about
errors = []
# Get bugzilla information from the package database
pkgdb = PackageDB(base_url=PKGDBSERVER)
acls = pkgdb.get_bugzilla_acls()
# Initialize the connection to bugzilla
bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS, acls)
for product in acls.keys():
if product not in ('Fedora', 'Fedora EPEL'):
continue
for pkg in acls[product]:
pkgInfo = acls[product][pkg]
try:
bugzilla.add_edit_component(pkg, product,
pkgInfo['owner'], pkgInfo['summary'],
pkgInfo['qacontact'], pkgInfo['cclist'])
except ValueError, e:
# A username didn't have a bugzilla address
errors.append(str(e.args))
except DataChangedError, e:
# A Package or Collection was returned via xmlrpc but wasn't
# present when we tried to change it
errors.append(str(e.args))
except xmlrpclib.ProtocolError, e:
# Unrecoverable and likely means that nothing is going to
# succeed.
errors.append(str(e.args))
break
except xmlrpclib.Error, e:
# An error occurred in the xmlrpc call. Shouldn't happen but
# we better see what it is
errors.append(str(e.args))
# Send notification of errors
if errors:
#print '[DEBUG]', '\n'.join(errors)
send_email('accounts@fedoraproject.org',
NOTIFYEMAIL,
'Errors while syncing bugzilla with the PackageDB',
'''
The following errors were encountered while updating bugzilla with information
from the Package Database. Please have the problems taken care of:
%s
''' % ('\n'.join(errors),))
sys.exit(0)