Add support for SetupBugReportSteps and compression with xz

Signed-off-by: Magnus Granberg <zorry@gentoo.org>

3 files changed, 132 insertions, 32 deletions

diff --git a/buildbot_gentoo_ci/config/config.py b/buildbot_gentoo_ci/config/config.py
index e2f33be..696ab0f 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -93,6 +93,7 @@ class GentooCiConfig(util.ComparableMixin):
         "db_url",
         "project",
         "worker_config",
+        "bug_config",
         "repository_basedir"
     ])
 
@@ -152,3 +153,5 @@ class GentooCiConfig(util.ComparableMixin):
             self.project['repository_basedir'] = DEFAULT_REPOSITORY_BASEDIR
         if 'worker_config' in config_dict:
             self.project['worker_config'] = config_dict['worker_config']
+        if 'bug_config' in config_dict:
+            self.project['bug_config'] = config_dict['bug_config']
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index e4fc951..ee11166 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import os
@@ -14,6 +14,7 @@ from twisted.internet import defer
 from twisted.python import log
 
 from buildbot.process.buildstep import BuildStep
+from buildbot.process.properties import Properties
 from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
 from buildbot.process.results import WARNINGS
@@ -21,7 +22,7 @@ from buildbot.process.results import SKIPPED
 from buildbot.plugins import steps
 from buildbot.plugins import util
 
-from buildbot_gentoo_ci.steps import minio
+#from buildbot_gentoo_ci.steps import minio
 from buildbot_gentoo_ci.steps import master as master_steps
 from buildbot_gentoo_ci.steps import bugs
 
@@ -91,13 +92,15 @@ class SetupPropertys(BuildStep):
         self.setProperty("default_project_data", default_project_data, 'default_project_data')
         self.setProperty("version_data", version_data, 'version_data')
         self.setProperty("status", 'completed', 'status')
-        if self.getProperty('faild_cpv'):
+        if isinstance(self.getProperty('faild_cpv'), str):
             log_cpv = self.getProperty('faild_cpv')
         else:
             log_cpv = self.getProperty('cpv')
         self.setProperty("log_cpv", log_cpv, 'log_cpv')
-        self.setProperty("bgo", dict( match=False), 'bgo')
+        self.setProperty("bgo", False, 'bgo')
         self.descriptionDone = 'Runing log checker on ' + log_cpv
+        logsdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
+        self.setProperty("logsdir", logsdir, 'logsdir')
         return SUCCESS
 
 class SetupParserBuildLoger(BuildStep):
@@ -115,9 +118,9 @@ class SetupParserBuildLoger(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         self.aftersteps_list = []
-        workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
         log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
-        mastersrc_log = yield os.path.join(workdir, log_cpv['full_logname'])
+        build_log_file_compressed = log_cpv['full_logname'] + '.xz'
+        mastersrc_log = yield os.path.join(self.getProperty('logsdir'), build_log_file_compressed)
         log_py = 'log_parser.py'
         config_log_py = 'logparser.json'
         mastersrc_py = yield os.path.join(self.master.basedir, log_py)
@@ -125,7 +128,7 @@ class SetupParserBuildLoger(BuildStep):
         # Upload logfile to worker
         self.aftersteps_list.append(steps.FileDownload(
                                                     mastersrc=mastersrc_log,
-                                                    workerdest=log_cpv['full_logname']
+                                                    workerdest=build_log_file_compressed
                                                     ))
         # Upload log parser py code
         self.aftersteps_list.append(steps.FileDownload(
@@ -137,6 +140,16 @@ class SetupParserBuildLoger(BuildStep):
                                                     mastersrc=mastersrc_config,
                                                     workerdest=config_log_py
                                                     ))
+        #Untar the log
+        shell_commad_list = []
+        shell_commad_list.append('xz')
+        shell_commad_list.append('-dv')
+        shell_commad_list.append(build_log_file_compressed)
+        self.aftersteps_list.append(
+            steps.ShellCommand(
+                            name = 'Decompress build log with xz',
+                            command = shell_commad_list,
+        ))
         # Run the log parser code
         command = []
         command.append('python3')
@@ -171,23 +184,6 @@ class MakeIssue(BuildStep):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    @defer.inlineCallbacks
-    def logIssue(self):
-        separator1 = '\n'
-        separator2 = ' '
-        log = yield self.addLog('issue')
-        self.error_dict['cpv'] = self.getProperty('log_cpv')
-        yield log.addStdout('Title:' + '\n')
-        yield log.addStdout(separator2.join([self.getProperty('log_cpv'), '-', self.error_dict['title']]) + separator1)
-        yield log.addStdout('Summary:' + '\n')
-        for line in self.summary_log_list:
-            yield log.addStdout(line + '\n')
-        yield log.addStdout('Attachments:' + '\n')
-        yield log.addStdout('emerge_info.log' + '\n')
-        log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
-        yield log.addStdout(log_cpv['full_logname'] + '\n')
-        yield log.addStdout('world.log' + '\n')
-
     def getNiceErrorLine(self, line):
         # strip away hex addresses, loong path names, line and time numbers and other stuff
         # https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L467
@@ -211,12 +207,16 @@ class MakeIssue(BuildStep):
                 if s['type'] == self.error_dict['phase'] and s['status'] == 'error':
                     text_issue_list.append(v['text'])
         # add the issue error
+        self.error_dict['cpv'] = self.getProperty('log_cpv')
         if text_issue_list != []:
-            self.error_dict['title_issue'] = text_issue_list[0].replace('*', '').strip()
-            self.error_dict['title_issue_nice'] = self.getNiceErrorLine(text_issue_list[0].replace('*', '').strip())
+            title_line = text_issue_list[0].replace('*', '').strip()
+            self.error_dict['title_issue'] = title_line
+            self.error_dict['title_issue_nice'] = self.getNiceErrorLine(title_line)
+            self.error_dict['title_found'] = True
         else:
             self.error_dict['title_issue'] = 'title_issue : None'
             self.error_dict['title_nice'] = 'title_issue : None'
+            self.error_dict['title_found'] = False
         self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
         #set the error title
         self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' + self.error_dict['title_issue']
@@ -243,13 +243,14 @@ class MakeIssue(BuildStep):
                 phase_error = v['text'].split(' (')[1].split(' phase')[0]
                 self.error_dict['phase'] = phase_error
                 error = True
-        # add build log
+        #FIXME: write summary_log_list to a file
         # add issue/bug/pr report
         if error:
             yield self.ClassifyIssue()
             print(self.error_dict)
-            yield self.logIssue()
             self.setProperty("status", 'failed', 'status')
+            #FIXME: nice description for post bug
+            self.error_dict['summary_log_nice'] = ''
             self.setProperty("error_dict", self.error_dict, 'error_dict')
             self.aftersteps_list.append(bugs.GetBugs())
         if warning:
@@ -295,14 +296,13 @@ class ReadEmergeInfoLog(BuildStep):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    @defer.inlineCallbacks
+    #@defer.inlineCallbacks
     def run(self):
         emerge_info_output = {}
         emerge_info_list = []
         emerge_package_info = []
         # Read the file and add it to a property
-        workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
-        with open(os.path.join(workdir, 'emerge_info.txt'), encoding='utf-8') as source:
+        with open(os.path.join(self.getProperty('logsdir'), 'emerge_info.txt'), encoding='utf-8') as source:
             emerge_info = source.read()
         # set emerge_info_output Property
         for line in emerge_info.split('\n'):
@@ -379,7 +379,7 @@ class Upload(BuildStep):
     def run(self):
         log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
         bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs'
-        file_path = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']) ,log_cpv['full_logname'])
+        file_path = yield os.path.join(self.getProperty('logsdir') ,log_cpv['full_logname'])
         aftersteps_list = []
         aftersteps_list.append(minio.putFileToMinio(file_path, log_cpv['full_logname'], bucket))
         yield self.build.addStepsAfterCurrentStep(aftersteps_list)
@@ -435,6 +435,83 @@ class ParserPkgCheckLog(BuildStep):
             returnstatus = WARNINGS
         return returnstatus
 
+class SetupBugReportSteps(BuildStep):
+    name = 'SetupBugReportSteps'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        bug_config = self.gentooci.config.project['bug_config']
+        # add bug enable profile and repo
+        if bug_config['enable'] and self.getProperty('status') == 'failed' and self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
+            aftersteps_list = []
+            #Post
+            p = Properties()
+            p.master = self.master
+            separator = ' '
+            separator2 = '\n'
+            bug_args = {}
+            bug_args['user'] = bug_config['user']
+            bug_args['passwd'] = yield p.render(util.Secret("bugs_password"))
+            bug_params = {}
+            title = separator.join([bug_config['extra_summery'], self.getProperty('error_dict')['cpv'], self.getProperty('error_dict')['title']])
+            # bug title max 170
+            if len(title) >= 170:
+                title = title[:167] + '...'
+            bug_params['summary'] = title
+            # max 16384
+            description_list = self.getProperty('summary_log_list')
+            del description_list[-6:]
+            description_text = separator2.join(description_list)
+            bug_params['description'] = description_text + separator
+            bug_params['assigned_to'] = bug_config['user']
+            #params['cc'] = cc
+            bug_params['url'] = f"https://ci.dev.gentoo.org:8010/#/builders/7/builds/{self.getProperty('buildnumber')}"
+            print(bug_params)
+            # post
+            aftersteps_list.append(bugs.Post(bug_args, bug_params))
+            # add buildbot info
+            bug_params2 = {}
+            bug_params2['comment'] = separator2.join(bug_config['buildbot_comment'])
+            aftersteps_list.append(bugs.Modify(bug_args, bug_params2))
+            # add build info
+            comment_list = []
+            comment_list.append(f"Was trying to build {self.getProperty('cpv')} Commit: {self.getProperty('revision')} Repository: {self.getProperty('repository')}")
+            if self.getProperty('faild_cpv') != self.getProperty('cpv'):
+                comment_list.append(f"Depend: {self.getProperty('faild_cpv')} fail to build")
+            comment_list.append(f"Package info:")
+            for line in self.getProperty('emerge_info_output')['emerge_package_info']:
+                comment_list.append(f"{line}")
+            comment_list.append(f"Emerge info:")
+            for line in self.getProperty('emerge_info_output')['emerge_info']:
+                comment_list.append(f"{line}")
+            bug_params3 = {}
+            bug_params3['comment'] = separator2.join(comment_list)
+            aftersteps_list.append(bugs.Modify(bug_args, bug_params3))
+            # Attach build log
+            file_list = []
+            with os.scandir(self.getProperty('logsdir')) as entries:
+                for entry in entries:
+                    file_list.append(entry.name)
+            print(file_list)
+            for filename in file_list:
+                bug_params4 = {}
+                bug_params4['filename'] = yield os.path.join(self.getProperty('logsdir'), filename)
+                #bug_params4['comment'] = filename
+                aftersteps_list.append(bugs.Attach(bug_args, bug_params4))
+            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+            return SUCCESS
+        return SKIPPED
+
 class setBuildStatus(BuildStep):
 
     name = 'setBuildStatus'
diff --git a/gentooci.cfg b/gentooci.cfg
index cd4bfda..474da76 100644
--- a/gentooci.cfg
+++ b/gentooci.cfg
@@ -26,6 +26,9 @@ makeconf_list.append('PORTAGE_ELOG_CLASSES="*"')
 makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"')
 # use ansifilter on the logs
 makeconf_list.append('PORTAGE_LOG_FILTER_FILE_CMD="bash -c \'ansifilter --ignore-clear; exec cat\'"')
+# use xz for Portage compression of both installed files and binary packages:
+makeconf_list.append('PORTAGE_COMPRESS="xz"')
+makeconf_list.append('BINPKG_COMPRESS="xz"')
 
 # This specifies what project buildbot uses for Gentoo Ci as default
 c['project'] = {
@@ -43,3 +46,20 @@ c['worker_config'] = {
     'repository_basedir' : 'repositorys',
     'portage_repos_path' : '/var/db/repos',
 }
+# This specifies bug configs
+c['bug_config'] = {
+    'enable' : True,
+    'aout_assignee' : False,
+    'user' : 'LarryTheCow@Cow.mu',
+    'extra_summery' : '',
+    'buildbot_comment' : [
+    'This was build on a tinderbox that use Buildbot API[1].',
+    'We use docker images to build on.',
+    'Web: https://ci.dev.gentoo.org:8010/#/',
+    'Code: https://gitweb.gentoo.org/proj/tinderbox-cluster.git',
+    'Wiki: https://wiki.gentoo.org/wiki/Project:Tinderbox-cluster',
+    'How to get more info: https://wiki.gentoo.org/wiki/Project:Tinderbox-cluster/something',
+    'IRC: #gentoo-ci on libera',
+    '[1] https://buildbot.net/',
+    ],
+}