path: root/modules/laymandbtools/gitoliteextractor.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
# Licensed under GPL 2 or later

import sys
from optparse import OptionParser
USAGE = 'Usage: %prog  [--fixes-only]  conf/user.conf conf/proj.conf bar/repositories.xml [baz/extended.xml]'
parser = OptionParser(usage=USAGE)
parser.add_option('--fixes-only',
                  dest = 'fixes_only',
                  default = False,
                  action = 'store_true',
                  help = 'do not add entries that are missing completely')
(opts, args) = parser.parse_args()
if len(args) not in (3, 4):
    parser.print_help()
    sys.exit(1)
gitolite_conf_locations = args[0:2]
repositories_xml_location = args[2]
extended_xml_location = (len(args) == 4) and args[3] or None

import xml.etree.ElementTree as ET
from ConfigParser import ConfigParser
import re

from layman.dbtools.sharedutils import * # local
from laymandbtools.gitoliteparser import RepoDatabase


# From gitolite config
# ..to repositories.xml
repo_name_mapping = {
    'ruby-overlay':'ruby',
    'sci':'science',
    'perl-overlay':'perl-experimental',
    'xfce':'xfce-dev',
    'gnome-perf':'leio-gnome-perf',
    'flameeyes':'flameeyes-overlay',
}


def sort_as_in(elements, tag_order):
    order_map = dict((v, i) for i, v in enumerate(tag_order))
    deco = (t for t in enumerate(elements))
    deco_sorted = sorted(deco, key=lambda (i, v): (order_map[v.tag], i))
    return list(v for _, v in deco_sorted)

class ChangeLog:
    def __init__(self, gitolite_repo_name, is_new):
        self.empty = True
        self.gitolite_repo_name = gitolite_repo_name
        self.is_new = is_new

    def log(self, kind, details):
        if self.empty:
            if self.is_new:
                print 'Repo "%s" missing completely' % self.gitolite_repo_name
            else:
                print 'Analyzing repo "%s":' % self.gitolite_repo_name
            self.empty = False

        if not self.is_new:
            print '- Missing %s "%s"' % (kind, details)


OWNER_REGEX = re.compile('^([^<]+) (?:\([^)]+\) )?<([^ ]+@[^ ]+)>$')
NOT_AN_OVERLAY_MESSAGE = 'Skipping %s (not an overlay)'


gitolite_conf = RepoDatabase()
for filename in gitolite_conf_locations:
    gitolite_conf.feed(filename)


a = ET.parse(open(repositories_xml_location))
repositories = a.getroot()
overlays_gentoo_org_dict = dict([[e.find('name').text, e] for e in repositories])

def oct_string_to_int(os):
    l = len(os)
    res = 0
    for i, v in enumerate(os):
        res = res + (8**(l-1-i)) * int(v)
    return res

assert oct_string_to_int('0713') == 0713
assert oct_string_to_int('103') == 0103


# GLOBAL_DIRMODE = oct_string_to_int(gitosis_conf.get('gitosis', 'dirmode'))
# GLOBAL_GITWEB = gitosis_conf.getboolean('gitosis', 'gitweb')

def is_public(section_name):
    #local_dirmode = GLOBAL_DIRMODE
    #if gitosis_conf.has_option(section_name, 'dirmode'):
    #    local_dirmode = oct_string_to_int(gitosis_conf.get(section_name, 'dirmode'))

    #local_gitweb = GLOBAL_GITWEB
    #if gitosis_conf.has_option(section_name, 'gitweb'):
    #    local_gitweb = gitosis_conf.getboolean(section_name, 'gitweb')

    return True  # ((local_dirmode & 0005) == 0005) and local_gitweb


for section_name in gitolite_conf.names():
    if True:
        _repo_base = section_name

        try:
            owner_part, gitolite_repo_name = _repo_base.split('/')
        except (ValueError) as e:
            # TODO print NOT_AN_OVERLAY_MESSAGE % gitolite_repo_name
            continue

        if owner_part == 'proj':
            owner_type = "project"
        elif owner_part in ('dev', 'user'):
            owner_type = "person"
        else:
            # TODO print NOT_AN_OVERLAY_MESSAGE % gitolite_repo_name
            continue

        terms_status, is_overlay, dont_add_to_layman, owner_contact, _description = gitolite_conf.data(section_name)

        if dont_add_to_layman:
            continue

        overlay_status_clear = False
        if not is_overlay is None:
            if not is_overlay:
                continue
            overlay_status_clear = True

        if not overlay_status_clear \
                and not gitolite_repo_name.endswith('overlay') \
                and not _description.lower().endswith('overlay'):
            continue

        if not is_public(section_name):
            # TODO print 'Skipping %s (not public)' % gitolite_repo_name
            continue

        repositores_xml_repo_name = repo_name_mapping.get(gitolite_repo_name, gitolite_repo_name)
        is_new = repositores_xml_repo_name not in overlays_gentoo_org_dict
        if is_new:
            if opts.fixes_only:
                continue
            repo = ET.Element('repo')
            repositories.append(repo)
            name = ET.Element('name')
            name.text = gitolite_repo_name
            repo.append(name)
        else:
            repo = overlays_gentoo_org_dict[repositores_xml_repo_name]
        log = ChangeLog(gitolite_repo_name, is_new)

        if 'status' not in repo.attrib:
            if owner_part == 'user':
                repo.attrib['status'] = 'unofficial'
            else:
                repo.attrib['status'] = 'official'
            log.log('attribute', 'status')

        if 'quality' not in repo.attrib:
            repo.attrib['quality'] = 'experimental'
            log.log('attribute', 'quality')

        homepage = repo.find('homepage')
        if homepage == None:
            homepage = ET.Element('homepage')
            homepage.text = 'http://git.overlays.gentoo.org/gitweb/?p=%s.git;a=summary' % _repo_base
            repo.append(homepage)
            log.log('homepage', homepage.text)

        description = repo.find('description')
        if description == None:
            description = ET.Element('description', lang='en')
            description.text = _description
            repo.append(description)
            log.log('description', _description)


        _owner = owner_contact
        _owner_match = OWNER_REGEX.match(_owner)

        owner = repo.find('owner')
        if owner == None:
            owner = ET.Element('owner', type=owner_type)
            repo.append(owner)
            log.log('owner', 'TODO')

        owner_name = owner.find('name')
        if owner_name == None:
            owner_name = ET.Element('name')
            owner_name.text = _owner_match.group(1)
            log.log('owner name', owner_name.text)

        owner_email = owner.find('email')
        if owner_email == None:
            owner_email = ET.Element('email')
            owner_email.text = _owner_match.group(2)
            log.log('owner email', owner_email.text)

        owner[:] = [owner_email, owner_name]


        _sources = set((source.attrib['type'], source.text) for source in repo.findall('source'))
        source_uris = (
            'git://git.overlays.gentoo.org/%s.git' % _repo_base,
            'http://git.overlays.gentoo.org/gitroot/%s.git' % _repo_base,
            'git+ssh://git@git.overlays.gentoo.org/%s.git' % _repo_base,
        )
        for uri in source_uris:
            if ('git', uri) not in _sources and \
                    ('git', uri[:-len('.git')]) not in _sources:
                source = ET.Element('source', type='git')
                source.text = uri
                repo.append(source)
                log.log('git source', uri)


        _feeds = set(feed.text for feed in repo.findall('feed'))
        feed_uris = (
            'http://git.overlays.gentoo.org/gitweb/?p=%s.git;a=atom' % _repo_base,
            'http://git.overlays.gentoo.org/gitweb/?p=%s.git;a=rss' % _repo_base,
        )
        for uri in feed_uris:
            if uri not in _feeds:
                feed = ET.Element('feed')
                feed.text = uri
                repo.append(feed)
                log.log('feed', uri)

        repo[:] = sort_as_in(repo[:], (
                'name', 'description', 'longdescription',
                'homepage', 'owner', 'source', 'feed'))

        if is_new or not log.empty:
            if extended_xml_location == None:
                TERM_WIDTH = 67
                print '-'*TERM_WIDTH
                sys.stdout.write('  ')
                indent(repo, 1)
                repo.tail = '\n'
                ET.ElementTree(repo).write(sys.stdout)
                print '-'*TERM_WIDTH
            print


if extended_xml_location != None:
    indent(repositories)
    extended_xml = open(extended_xml_location, 'w')
    extended_xml.write("""\
<?xml version="1.0" encoding="UTF-8"?>
<!-- $Header$ -->
<?xml-stylesheet href="/xsl/repositories.xsl" type="text/xsl"?>
<!DOCTYPE repositories SYSTEM "/dtd/repositories.dtd">
""")
    a.write(extended_xml, encoding='utf-8')
    extended_xml.close()