From 16f9c0fc440bb4c15742b7e02827fad64d73db82 Mon Sep 17 00:00:00 2001 From: Magnus Granberg Date: Sat, 23 Mar 2024 15:13:36 +0100 Subject: Redo Trigger for parse_build_log in CheckEmergeLogs Signed-off-by: Magnus Granberg --- buildbot_gentoo_ci/steps/builders.py | 180 +++++++++++++++++------------------ 1 file changed, 87 insertions(+), 93 deletions(-) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index c033196..73e82f5 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -895,8 +895,10 @@ class CheckEmergeLogs(BuildStep): '-v' ] emerge_output = self.getProperty('emerge_output') - self.faild_cpv = emerge_output['failed'] + print(f"emerge_output: {emerge_output}") + self.setProperty('faild_cpv', emerge_output['failed'], 'faild_cpv') self.faild_version_data = False + self.log_data = {} package_dict = emerge_output['packages'] #FIXME: Prosees the logs and do stuff @@ -916,7 +918,7 @@ class CheckEmergeLogs(BuildStep): print(emerge_output) # this should be set in the config retrays = 5 - if self.getProperty('rerun') <= retrays and self.faild_cpv: + if self.getProperty('rerun') <= retrays and emerge_output['failed']: # when we need to change use. we could rerun pre-build with # --autounmask-use=y --autounmask-write=y --autounmask-only=y # but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS @@ -997,26 +999,24 @@ class CheckEmergeLogs(BuildStep): self.aftersteps_list.append(CheckEmergeLogs('extra-build')) else: # trigger parse_build_log with info about pre-build and it fail - if self.faild_cpv: + if emerge_output['failed']: pass self.setProperty('rerun', 0, 'rerun') # Make Logfile dict if self.step == 'extra-build' or self.step == 'build': - print(emerge_output) log_dict = {} # get cpv, logname and log path for log_path in emerge_output['log_paths']: c = log_path.split('/')[6] full_logname = log_path.split('/')[7] - print(full_logname) + print(f"full_logname: {full_logname}") pv = full_logname.split(':')[0] cpv = c + '/' + pv log_dict[cpv] = dict( log_path = log_path, full_logname = full_logname ) - print('log_dict') - print(log_dict) + print(f"log_dict: {log_dict}") if self.step == 'extra-build': #FIXME: Check if extra build did work self.aftersteps_list.append(RunEmerge(step='pre-build')) @@ -1028,94 +1028,88 @@ class CheckEmergeLogs(BuildStep): # local_log_path dir set in config # format /var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz rebuild = False - if self.step == 'build': + if self.step == 'build' and log_dict != {}: retrays = 1 - # Find log for cpv that was requested or did failed - if not log_dict == {}: - print('log_dict not emty') - # requested cpv - cpv = self.getProperty('cpv') - if cpv in log_dict or self.faild_cpv in log_dict: - print(cpv) - print(self.faild_cpv) + if isinstance(emerge_output['failed'], str): + log_cpv = emerge_output['failed'] + if self.getProperty('rerun') < retrays: + # set rebuild + # haskell + if c == 'dev-haskell': + rebuild = 'haskell' + # binary + for cpv, v in package_dict.items(): + if cpv == emerge_output['failed']: + if v['binary']: + rebuild = 'binary' + print(f"rebuild is: {rebuild}") + else: + log_cpv = self.getProperty('cpv') + if not rebuild: + if log_cpv in log_dict: + print(f"Package {log_cpv} in log_dict") + self.log_data[log_cpv] = log_dict[log_cpv] yield self.createDistDir() - if cpv in log_dict: - self.log_data[cpv] = log_dict[cpv] - yield self.getLogFile(cpv, log_dict) - if isinstance(self.faild_cpv, str): - print('faild_cpv is string') - c = yield catpkgsplit(self.faild_cpv)[0] - if c == 'dev-haskell': - rebuild = 'haskell' - for cpv, v in package_dict.items(): - if cpv == self.faild_cpv: - if v['binary']: - rebuild = 'binary' - if not rebuild or self.getProperty('rerun') >= retrays: - # failed and build requested cpv - if cpv == self.faild_cpv: - self.faild_version_data = self.getProperty("version_data") - else: - # failed but not build requested cpv - self.log_data[self.faild_cpv] = log_dict[self.faild_cpv] - yield self.getLogFile(self.faild_cpv, log_dict) - repository = False - for cpv, v in package_dict.items(): - if cpv == self.faild_cpv: - repository = v['repository'] - if repository: - self.faild_version_data = yield self.getVersionData(self.faild_cpv, repository) - self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv') - print(self.faild_version_data) - print('getEmergeFiles') - self.getEmergeFiles(self.faild_cpv) - self.getBuildWorkDirs(self.faild_cpv) - rebuild = False - else: - print('getEmergeFiles') - self.getEmergeFiles(cpv) - if not rebuild: - self.aftersteps_list.append(steps.Trigger( - name = 'Setup properties for log parser and trigger it', - schedulerNames=['parse_build_log'], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - 'cpv' : self.getProperty("cpv"), - 'faild_version_data' : self.faild_version_data, - 'project_build_data' : self.getProperty('project_build_data'), - 'log_build_data' : self.log_data, - 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"), - 'repository_data' : self.getProperty('repository_data'), - 'faild_cpv' : self.faild_cpv, - 'build_workername' : self.getProperty('workername') - } - )) - if rebuild: - #FIXME: Set build timeout in config - build_timeout = 6600 - shell_commad_list = [] - exclude = False - # rebuild broken haskell - if rebuild == 'haskell': - shell_commad_list.append('haskell-updater') - shell_commad_list.append('--') - shell_commad_list.append('--usepkg=n') - if rebuild == 'binary': - c = yield catpkgsplit(self.faild_cpv)[0] - p = yield catpkgsplit(self.faild_cpv)[1] - exclude = c + '/' + p - if shell_commad_list != []: - self.aftersteps_list.append( - steps.ShellCommand( - command=shell_commad_list, - workdir='/', - env=self.getProperty("build_env"), - timeout=build_timeout - )) - self.aftersteps_list.append(RunEmerge(step='build', exclude=exclude)) - self.aftersteps_list.append(CheckEmergeLogs('build')) - self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun') + yield self.getLogFile(log_cpv, log_dict) + yield self.getEmergeFiles(log_cpv) + if isinstance(emerge_output['failed'], str): + # get extra logs + self.getBuildWorkDirs(emerge_output['failed']) + #set faild_version_data + if self.getProperty('cpv') == emerge_output['failed']: + self.faild_version_data = self.getProperty("version_data") + else: + repository = False + for cpv, v in package_dict.items(): + if cpv == emerge_output['failed']: + repository = v['repository'] + if repository: + self.faild_version_data = yield self.getVersionData(emerge_output['failed'], repository) + print(f"Failed package version data: {self.faild_version_data}") + else: + print(f"No log file in log_dict match package: {log_cpv}") + return WARNINGS + self.aftersteps_list.append(steps.Trigger( + name = 'Setup properties for log parser and trigger it', + schedulerNames=['parse_build_log'], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + 'cpv' : self.getProperty("cpv"), + 'faild_version_data' : self.faild_version_data, + 'project_build_data' : self.getProperty('project_build_data'), + 'log_build_data' : self.log_data, + 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"), + 'repository_data' : self.getProperty('repository_data'), + 'faild_cpv' : emerge_output['failed'], + 'build_workername' : self.getProperty('workername') + } + )) + if rebuild: + #FIXME: Set build timeout in config + build_timeout = 6600 + shell_commad_list = [] + exclude = False + # rebuild broken haskell + if rebuild == 'haskell': + shell_commad_list.append('haskell-updater') + shell_commad_list.append('--') + shell_commad_list.append('--usepkg=n') + if rebuild == 'binary': + c = yield catpkgsplit(emerge_output['failed'])[0] + p = yield catpkgsplit(emerge_output['failed'])[1] + exclude = c + '/' + p + if shell_commad_list != []: + self.aftersteps_list.append( + steps.ShellCommand( + command=shell_commad_list, + workdir='/', + env=self.getProperty("build_env"), + timeout=build_timeout + )) + self.aftersteps_list.append(RunEmerge(step='build', exclude=exclude)) + self.aftersteps_list.append(CheckEmergeLogs('build')) + self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun') if not self.step is None and self.aftersteps_list != []: yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS -- cgit v1.2.3-65-gdbad