path: root/buildbot_gentoo_ci/steps/logs.py

# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

import os
import re
import gzip
import io
import hashlib
import json

from portage.versions import catpkgsplit

from twisted.internet import defer
from twisted.python import log

from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot.plugins import util

from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
from buildbot_gentoo_ci.steps import bugs

def PersOutputOfLogParser(rc, stdout, stderr):
    build_summery_output = {}
    build_summery_output['rc'] = rc
    summary_log_dict = {}
    # split the lines
    for line in stdout.split('\n'):
        #FIXME: check if line start with {"[1-9]": {
        if line.startswith('{'):
            for k, v in json.loads(line).items():
                summary_log_dict[int(k)] = {
                                    'text' : v['text'],
                                    'pattern_infos' : [],
                                    }
                for s in v['pattern_info']:
                    summary_log_dict[int(k)]['pattern_infos'].append({
                                                            'type' : s['type'],
                                                            'status' : s['status'],
                                                            'id' : s['id'],
                                                            'search_pattern' : s['search_pattern'],
                                                            })
    build_summery_output['summary_log_dict'] = summary_log_dict
    #FIXME: Handling of stderr output
    return {
        'build_summery_output' : build_summery_output
        }

def PersOutputOfEmergeInfo(rc, stdout, stderr):
    #FIXME: line for package info
    emerge_info_output = {}
    emerge_info_output['rc'] = rc
    emerge_info_list = []
    emerge_package_info = []
    for line in stdout.split('\n'):
        if line.startswith('['):
            emerge_package_info.append(line)
        else:
            emerge_info_list.append(line)
    emerge_info_output['emerge_info'] = emerge_info_list
    emerge_info_output['emerge_package_info'] = emerge_package_info
    return {
        'emerge_info_output' : emerge_info_output
        }

class SetupPropertys(BuildStep):
    
    name = 'SetupPropertys'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        # set this in config
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
        default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
        version_data = yield self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
        self.setProperty("project_data", project_data, 'project_data')
        self.setProperty("default_project_data", default_project_data, 'default_project_data')
        self.setProperty("version_data", version_data, 'version_data')
        self.setProperty("status", 'completed', 'status')
        if self.getProperty('faild_cpv'):
            log_cpv = self.getProperty('faild_cpv')
        else:
            log_cpv = self.getProperty('cpv')
        self.setProperty("log_cpv", log_cpv, 'log_cpv')
        self.setProperty("bgo", dict( match=False), 'bgo')
        self.descriptionDone = 'Runing log checker on ' + log_cpv
        return SUCCESS

class SetupParserBuildLoger(BuildStep):

    name = 'SetupParserBuildLoger'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.aftersteps_list = []
        workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
        log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
        mastersrc_log = yield os.path.join(workdir, log_cpv['full_logname'])
        log_py = 'log_parser.py'
        config_log_py = 'logparser.json'
        mastersrc_py = yield os.path.join(self.master.basedir, log_py)
        mastersrc_config = yield os.path.join(self.master.basedir, config_log_py)
        # Upload logfile to worker
        self.aftersteps_list.append(steps.FileDownload(
                                                    mastersrc=mastersrc_log,
                                                    workerdest=log_cpv['full_logname']
                                                    ))
        # Upload log parser py code
        self.aftersteps_list.append(steps.FileDownload(
                                                    mastersrc=mastersrc_py,
                                                    workerdest=log_py
                                                    ))
        # Upload log parser py config
        self.aftersteps_list.append(steps.FileDownload(
                                                    mastersrc=mastersrc_config,
                                                    workerdest=config_log_py
                                                    ))
        # Run the log parser code
        command = []
        command.append('python3')
        command.append(log_py)
        command.append('-f')
        command.append(log_cpv['full_logname'])
        command.append('-u')
        command.append(self.getProperty('project_data')['uuid'])
        command.append('-d')
        command.append(util.Secret("log_parser_database"))
        self.aftersteps_list.append(steps.SetPropertyFromCommand(
                                                            name = 'RunBuildLogParser',
                                                            haltOnFailure = True,
                                                            flunkOnFailure = True,
                                                            command=command,
                                                            strip=False,
                                                            extract_fn=PersOutputOfLogParser,
                                                            timeout=3600
                                                            ))
        yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
        return SUCCESS

class MakeIssue(BuildStep):

    name = 'MakeIssue'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def logIssue(self):
        separator1 = '\n'
        separator2 = ' '
        log = yield self.addLog('issue')
        self.error_dict['cpv'] = self.getProperty('log_cpv')
        yield log.addStdout('Title:' + '\n')
        yield log.addStdout(separator2.join([self.getProperty('log_cpv'), '-', self.error_dict['title']]) + separator1)
        yield log.addStdout('Summary:' + '\n')
        for line in self.summary_log_list:
            yield log.addStdout(line + '\n')
        yield log.addStdout('Attachments:' + '\n')
        yield log.addStdout('emerge_info.log' + '\n')
        log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
        yield log.addStdout(log_cpv['full_logname'] + '\n')
        yield log.addStdout('world.log' + '\n')

    def getNiceErrorLine(self, line):
        # strip away hex addresses, loong path names, line and time numbers and other stuff
        # https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L467
        # FIXME: Add the needed line when needed
        if re.search(': line', line):
            line = re.sub(r"\d", "<snip>", line)
        # Shorten the path
        if line.startswith('/usr/'):
            line = line.replace(os.path.split(line.split(' ', 1)[0])[0], '/...')
        if re.search(': \d:\d: ', line):
            line = re.sub(r":\d:\d: ", ": ", line)
        return line

    def ClassifyIssue(self):
        # get the title for the issue
        text_issue_list = []
        text_phase_list = []
        for k, v in sorted(self.summary_log_dict.items()):
            # get the issue error
            for s in v['pattern_infos']:
                if s['type'] == self.error_dict['phase'] and s['status'] == 'error':
                    text_issue_list.append(v['text'])
        # add the issue error
        if text_issue_list != []:
            self.error_dict['title_issue'] = text_issue_list[0].replace('*', '').strip()
            self.error_dict['title_issue_nice'] = self.getNiceErrorLine(text_issue_list[0].replace('*', '').strip())
        else:
            self.error_dict['title_issue'] = 'title_issue : None'
            self.error_dict['title_nice'] = 'title_issue : None'
        self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
        #set the error title
        self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' + self.error_dict['title_issue']

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        self.summary_log_dict = self.getProperty('build_summery_output')['summary_log_dict']
        error = False
        warning = False
        self.summary_log_list = []
        self.error_dict = {}
        self.aftersteps_list = []
        #self.error_dict['hash'] = hashlib.sha256()
        for k, v in sorted(self.summary_log_dict.items()):
            self.summary_log_list.append(v['text'])
            #self.error_dict['hash'].update(v['text'].encode('utf-8'))
            for s in v['pattern_infos']:
                if s['status'] == 'warning':
                    warning = True
            # check if the build did fail
            if v['text'].startswith(' * ERROR:') and v['text'].endswith(' phase):'):
                # get phase error
                phase_error = v['text'].split(' (')[1].split(' phase')[0]
                self.error_dict['phase'] = phase_error
                error = True
        # add build log
        # add issue/bug/pr report
        if error:
            yield self.ClassifyIssue()
            print(self.error_dict)
            yield self.logIssue()
            self.setProperty("status", 'failed', 'status')
            self.setProperty("error_dict", self.error_dict, 'error_dict')
            self.aftersteps_list.append(bugs.GetBugs())
        if warning:
            self.setProperty("status", 'warning', 'status')
            self.setProperty("bgo", False, 'bgo')
        self.setProperty("summary_log_list", self.summary_log_list, 'summary_log_list')
        if self.aftersteps_list is not []:
            yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
        return SUCCESS

class Summarylog(BuildStep):

    name = 'Summarylog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('summary')
        # add line for line
        for line in self.getProperty('summary_log_list'):
            yield log.addStdout(line + '\n')
        return SUCCESS

class ReadEmergeInfoLog(BuildStep):

    name = 'ReadEmergeInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        emerge_info_output = {}
        emerge_info_list = []
        emerge_package_info = []
        # Read the file and add it to a property
        workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
        with open(os.path.join(workdir, 'emerge_info.txt'), encoding='utf-8') as source:
            emerge_info = source.read()
        # set emerge_info_output Property
        for line in emerge_info.split('\n'):
            if line.startswith('['):
                emerge_package_info.append(line)
            else:
                emerge_info_list.append(line)
        emerge_info_output['emerge_info'] = emerge_info_list
        emerge_info_output['emerge_package_info'] = emerge_package_info
        self.setProperty("emerge_info_output", emerge_info_output, 'emerge_info_output')
        return SUCCESS

class setEmergeInfoLog(BuildStep):

    name = 'setEmergeInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('emerge_info')
        #FIXME: add emerge info to db
        # add line for line
        for line in self.getProperty('emerge_info_output')['emerge_info']:
            yield log.addStdout(line + '\n')
        return SUCCESS

class setPackageInfoLog(BuildStep):

    name = 'setPackageInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('package_info')
        #FIXME: add package info to db
        # add line for line
        for line in self.getProperty('emerge_info_output')['emerge_package_info']:
            yield log.addStdout(line + '\n')
        return SUCCESS


class Upload(BuildStep):

    name = 'Upload'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
        bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs'
        file_path = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']) ,log_cpv['full_logname'])
        aftersteps_list = []
        aftersteps_list.append(minio.putFileToMinio(file_path, log_cpv['full_logname'], bucket))
        yield self.build.addStepsAfterCurrentStep(aftersteps_list)
        return SUCCESS

class ParserPkgCheckLog(BuildStep):

    name = 'ParserPkgCheckLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnWarnings = False
    flunkOnFailure = False
    warnOnWarnings = False

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        if self.getProperty("pkg_check_log_data") is None:
            return SKIPPED
        returnstatus = SUCCESS
        error = False
        warning = False
        log = yield self.addLog('Pkgcheck')
        print(self.getProperty("pkg_check_log_data"))
        for a in self.getProperty("pkg_check_log_data"):
            status = ''
            print(a)
            if isinstance(a, dict):
                for k, i in a.items():
                    if k.startswith('_'):
                        if k == '_info':
                            status = 'INFO: '
                        if k == '_error':
                            status = 'ERROR: '
                            error = True
                        if k == '_warning':
                            status = 'WARNING: '
                            warning = True
                        if k == '_style':
                            status = 'STYLE: '
                        if isinstance(i, dict):
                            for b, c in i.items():
                                yield log.addStdout(status + b + c + '\n')
                    else:
                        yield log.addStdout(i + '\n')
        if error:
            returnstatus = FAILURE
        if warning and not error:
            returnstatus = WARNINGS
        return returnstatus

class setBuildStatus(BuildStep):

    name = 'setBuildStatus'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        project_build_data = self.getProperty('project_build_data')
        yield self.gentooci.db.builds.setStatusBuilds(
                                                    project_build_data['id'],
                                                    self.getProperty('status')
                                                    )
        if self.getProperty('status') == 'failed':
            return FAILURE
        if self.getProperty('status') == 'warning':
            return WARNINGS
        return SUCCESS