diff options
author | Magnus Granberg <zorry@gentoo.org> | 2021-04-26 22:26:04 +0200 |
---|---|---|
committer | Magnus Granberg <zorry@gentoo.org> | 2021-04-26 22:26:04 +0200 |
commit | b2f1c115420e05c27fb7a57cf45734cb7f4a7731 (patch) | |
tree | 530824b64c5a1744b43a1e1b3ed71fbd279239bb | |
parent | Support more then one Chaange USE (diff) | |
download | tinderbox-cluster-b2f1c115420e05c27fb7a57cf45734cb7f4a7731.tar.gz tinderbox-cluster-b2f1c115420e05c27fb7a57cf45734cb7f4a7731.tar.bz2 tinderbox-cluster-b2f1c115420e05c27fb7a57cf45734cb7f4a7731.zip |
Add more checks and move * and >>> check to last
Signed-off-by: Magnus Granberg <zorry@gentoo.org>
-rw-r--r-- | buildbot_gentoo_ci/steps/logs.py | 121 |
1 files changed, 70 insertions, 51 deletions
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index 178d71e..0960b9b 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -57,39 +57,46 @@ class ParserBuildLog(BuildStep): self.summery_dict = {} self.index = 1 self.log_search_pattern_list = [] - self.max_text_lines = self.index -1 + self.max_text_lines = 0 super().__init__(**kwargs) + #FIXME: ansifilter + def ansiFilter(self, text): + return text + @defer.inlineCallbacks def get_log_search_pattern(self): # get pattern from the projects # add that to log_search_pattern_list for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])): - self.log_search_pattern_list.append(project_pattern) - # get the default profile pattern + # check if the search pattern is vaild + try: + re.compile(project_pattern['search']) + except re.error: + print("Non valid regex pattern") + print(project_pattern) + else: + self.log_search_pattern_list.append(project_pattern) + # get the default project pattern # add if not pattern is in project ignore + self.project_pattern_ignore = yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('project_data')['uuid']) for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])): - match = True - for project_pattern_ignore in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('default_project_data')['uuid'])): - if project_pattern['search'] == project_pattern_ignore['search']: - match = False - if match: - self.log_search_pattern_list.append(project_pattern) + if not project_pattern['search'] in self.project_pattern_ignore: + # check if the search pattern is vaild + try: + re.compile(project_pattern['search']) + except re.error: + print("Non valid regex pattern") + print(project_pattern) + else: + self.log_search_pattern_list.append(project_pattern) def search_buildlog(self, tmp_index): # get text line to search - text_line = self.logfile_text_dict[tmp_index] + text_line = self.ansiFilter(self.logfile_text_dict[tmp_index]) # loop true the pattern list for match for search_pattern in self.log_search_pattern_list: search_hit = False - # we add all line that start with ' * ' as info - # we add all line that start with '>>>' but not '>>> /' as info - if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')): - self.summery_dict[tmp_index] = {} - self.summery_dict[tmp_index]['text'] = text_line - self.summery_dict[tmp_index]['type'] = 'info' - self.summery_dict[tmp_index]['status'] = 'info' - self.summery_dict[tmp_index]['search_pattern_id'] = 0 if search_pattern['search_type'] == 'in': if search_pattern['search'] in text_line: search_hit = True @@ -100,63 +107,74 @@ class ParserBuildLog(BuildStep): if text_line.endswith(search_pattern['search']): search_hit = True if search_pattern['search_type'] == 'search': - if search_pattern['search'] in text_line: + if re.search(search_pattern['search'], text_line): search_hit = True + # add the line if the pattern match if search_hit: print(text_line) - print(search_pattern['search']) + print(search_pattern) + print(tmp_index) self.summery_dict[tmp_index] = {} self.summery_dict[tmp_index]['text'] = text_line self.summery_dict[tmp_index]['type'] = search_pattern['type'] self.summery_dict[tmp_index]['status'] = search_pattern['status'] self.summery_dict[tmp_index]['search_pattern_id'] = search_pattern['id'] # add upper text lines if requested - # max 10 - if search_pattern['start'] != 0 and search_hit: - i = tmp_index - i_start = i - search_pattern['start'] - match = True - while match: - i = i - 1 - if i < 0 or i < i_start: - match = False - else: - self.summery_dict[i] = {} - self.summery_dict[i]['text'] = self.logfile_text_dict[i] - self.summery_dict[i]['type'] = search_pattern['type'] - self.summery_dict[i]['status'] = 'info' + # max 5 + if search_pattern['start'] != 0: + i = tmp_index - search_pattern['start'] - 1 + match = True + while match: + i = i + 1 + if i < (tmp_index - 9) or i == tmp_index: + match = False + else: + if not i in self.summery_dict: + self.summery_dict[i] = {} + self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i]) + self.summery_dict[i]['type'] = 'info' + self.summery_dict[i]['status'] = 'info' # add lower text lines if requested - # max 10 - if search_pattern['end'] != 0 and search_hit: - i = tmp_index - i_end = i + search_pattern['end'] - match = True - while match: - i = i + 1 - if i > self.max_text_lines or i > i_end: - match = False - else: - self.summery_dict[i] = {} - self.summery_dict[i]['text'] = self.logfile_text_dict[i] - self.summery_dict[i]['type'] = search_pattern['type'] - self.summery_dict[i]['status'] = 'info' + # max 5 + if search_pattern['end'] != 0: + i = tmp_index + end = tmp_index + search_pattern['end'] + match = True + while match: + i = i + 1 + if i > self.max_text_lines or i > end: + match = False + else: + if not i in self.summery_dict: + self.summery_dict[i] = {} + self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i]) + self.summery_dict[i]['type'] = 'info' + self.summery_dict[i]['status'] = 'info' + else: + # we add all line that start with ' * ' as info + # we add all line that start with '>>>' but not '>>> /' as info + if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')): + if not tmp_index in self.summery_dict: + self.summery_dict[tmp_index] = {} + self.summery_dict[tmp_index]['text'] = text_line + self.summery_dict[tmp_index]['type'] = 'info' + self.summery_dict[tmp_index]['status'] = 'info' @defer.inlineCallbacks def run(self): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] - #FIXME: - # get the log parser pattern from db yield self.get_log_search_pattern() # open the log file # read it to a buffer # make a dict of the buffer - # maby use mulitiprocces to speed up the search + # maybe use mulitiprocces to speed up the search print(self.getProperty('log_build_data')) if self.getProperty('faild_cpv'): log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')] else: log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')] file_path = yield os.path.join(self.master.basedir, 'cpv_logs', log_cpv['full_logname']) + #FIXME: decode it to utf-8 with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f: for text_line in f: self.logfile_text_dict[self.index] = text_line.strip('\n') @@ -168,6 +186,7 @@ class ParserBuildLog(BuildStep): if self.index >= 20: del self.logfile_text_dict[self.index - 19] self.index = self.index + 1 + self.max_text_lines = self.index f.close() # check last 10 lines in logfile_text_dict yield self.search_buildlog(self.index - 10) |