1# Copyright 2013-2019 The Distro Tracker Developers
2# See the COPYRIGHT file at the top-level directory of this distribution and
3# at https://deb.li/DTAuthors
4#
5# This file is part of Distro Tracker. It is subject to the license terms
6# in the LICENSE file found in the top-level directory of this
7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker,
8# including this file, may be copied, modified, propagated, or distributed
9# except according to the terms contained in the LICENSE file.
10"""
11The Distro-Tracker-specific tasks for :mod:`distro_tracker.debci_status` app.
12"""
14import collections
15import datetime
16import json
17import logging
18import os.path
20from django.conf import settings
21from django.db import transaction
23from distro_tracker.core.models import (
24 ActionItem,
25 ActionItemType,
26 PackageData,
27 Repository,
28 SourcePackageName
29)
30from distro_tracker.core.tasks import BaseTask
31from distro_tracker.core.tasks.mixins import PackageTagging
32from distro_tracker.core.tasks.schedulers import IntervalScheduler
33from distro_tracker.core.utils.http import get_resource_text
35logger = logging.getLogger(__name__)
38class UpdateDebciStatusTask(BaseTask):
39 """
40 Updates packages' debci status.
41 """
43 class Scheduler(IntervalScheduler):
44 interval = 3600
46 ACTION_ITEM_TYPE_NAME = 'debci-failed-tests'
47 ITEM_DESCRIPTION = (
48 '<a href="{base_url}">Debci</a> reports ' +
49 '<a href="{debci_url}">failed tests</a> '
50 )
51 ITEM_FULL_DESCRIPTION_TEMPLATE = 'debci_status/debci-action-item.html'
53 def initialize(self, *args, **kwargs):
54 super().initialize(*args, **kwargs)
55 self.debci_action_item_type = ActionItemType.objects.create_or_update(
56 type_name=self.ACTION_ITEM_TYPE_NAME,
57 full_description_template=self.ITEM_FULL_DESCRIPTION_TEMPLATE)
59 @property
60 def base_url(self):
61 return getattr(settings, 'DISTRO_TRACKER_DEBCI_URL')
63 def get_debci_status(self, repo):
64 url = self.base_url + '/data/status/' + \
65 repo + '/amd64/packages.json'
67 content = get_resource_text(url, ignore_http_error=404)
68 if content is None:
69 logger.warning("Tried to fetch non-existing debci URL: %s", url)
70 return []
72 debci_status = json.loads(content)
73 return debci_status
75 def __get_debci_dir(self, package_name):
76 if package_name[:3] == 'lib':
77 debci_dir = package_name[:4]
78 else:
79 debci_dir = package_name[:1]
81 return os.path.join(debci_dir, package_name)
83 def __get_debci_url_main(self, package_name):
84 return os.path.join(self.base_url, 'packages',
85 self.__get_debci_dir(package_name))
87 def __get_debci_url_logfile(self, package_name, repo):
88 return os.path.join(self.base_url,
89 'data/packages/' + repo + '/amd64',
90 self.__get_debci_dir(package_name),
91 'latest-autopkgtest/log.gz')
93 def update_action_item(self, package, debci_statuses):
94 """
95 Updates the :class:`ActionItem` for the given package based on the
96 :class:`DebciStatus <distro_tracker.debci_status.DebciStatus`
97 If the package has test failures an :class:`ActionItem` is created.
98 """
99 debci_action_item = package.get_action_item_for_type(
100 self.debci_action_item_type.type_name)
101 if 'fail' not in [s['result']['status'] for s in debci_statuses]:
102 if debci_action_item:
103 debci_action_item.delete()
104 return
106 if debci_action_item is None:
107 debci_action_item = ActionItem(
108 package=package,
109 item_type=self.debci_action_item_type,
110 severity=ActionItem.SEVERITY_HIGH)
112 package_name = package.name
113 url = self.__get_debci_url_main(package_name)
115 debci_action_item.short_description = self.ITEM_DESCRIPTION.format(
116 debci_url=url,
117 base_url=self.base_url)
119 debci_action_item.extra_data = []
120 for debci_status in debci_statuses:
121 repo_codename = debci_status['repository']
123 log = self.__get_debci_url_logfile(package_name, repo_codename)
125 result = debci_status['result']
126 try:
127 duration = str(datetime.timedelta(
128 seconds=int(result['duration_seconds'])))
129 except TypeError:
130 duration = 'an unknown time'
132 debci_action_item.extra_data.append({
133 'duration': duration,
134 'previous_status': result['previous_status'],
135 'status': result['status'],
136 'date': result['date'],
137 'base_url': self.base_url,
138 'url': url,
139 'log': log,
140 'repository': repo_codename
141 })
143 debci_action_item.save()
145 def execute_main(self):
146 all_debci_status = collections.defaultdict(lambda: {})
148 repos = getattr(settings, 'DISTRO_TRACKER_DEBCI_REPOSITORIES', None)
149 if repos is None:
150 repos = Repository.objects.all().values_list('suite', flat=True)
152 for repo_codename in repos:
153 for status in self.get_debci_status(repo_codename):
154 if status and 'package' in status:
155 package = status['package']
156 all_debci_status[package][repo_codename] = status
158 # import pprint
159 # pprint.pprint(all_debci_status)
160 with transaction.atomic():
161 # Delete obsolete data
162 PackageData.objects.filter(key='debci').delete()
163 packages = []
164 infos = []
165 for package_name in all_debci_status:
166 try:
167 package = SourcePackageName.objects.get(
168 name=package_name)
169 packages.append(package)
170 except SourcePackageName.DoesNotExist:
171 continue
173 value = []
174 items = all_debci_status[package_name].items()
175 for repo_codename, result in items:
176 url = self.__get_debci_url_main(package_name)
177 value.append({'result': result,
178 'repository': repo_codename,
179 'url': url})
181 infos.append(
182 PackageData(
183 package=package,
184 key='debci',
185 value=value
186 )
187 )
189 self.update_action_item(package, value)
191 PackageData.objects.bulk_create(infos)
192 ActionItem.objects.delete_obsolete_items(
193 [self.debci_action_item_type], packages)
196class TagPackagesWithDebciFailures(BaseTask, PackageTagging):
197 """
198 Performs an update of 'debci-failures' tag for packages.
199 """
201 class Scheduler(IntervalScheduler):
202 interval = 3600
204 TAG_NAME = 'tag:debci-failures'
205 TAG_DISPLAY_NAME = 'debci failures'
206 TAG_COLOR_TYPE = 'danger'
207 TAG_DESCRIPTION = 'The package has test failures'
208 TAG_TABLE_TITLE = 'Packages with test failures'
210 def packages_to_tag(self):
211 try:
212 action_type = ActionItemType.objects.get(
213 type_name='debci-failed-tests')
214 except ActionItemType.DoesNotExist:
215 return []
217 items = action_type.action_items.all().prefetch_related('package')
218 return [item.package for item in items]