1# Copyright 2013-2015 The Distro Tracker Developers
2# See the COPYRIGHT file at the top-level directory of this distribution and
3# at https://deb.li/DTAuthors
4#
5# This file is part of Distro Tracker. It is subject to the license terms
6# in the LICENSE file found in the top-level directory of this
7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker,
8# including this file, may be copied, modified, propagated, or distributed
9# except according to the terms contained in the LICENSE file.
10"""Debian specific rules for various Distro-Tracker hooks."""
12import os.path
13import re
14from urllib.parse import quote_plus
16from django import forms
17from django.conf import settings
18from django.db.models import Prefetch
19from django.utils.http import urlencode
20from django.utils.safestring import mark_safe
22import requests
24from distro_tracker.core.models import (
25 ActionItem,
26 PackageData,
27 UserEmail
28)
29from distro_tracker.core.package_tables import create_table
30from distro_tracker.core.utils import get_decoded_message_payload, get_or_none
31from distro_tracker.core.utils.email_messages import get_message_body
32from distro_tracker.core.utils.http import HttpCache
33from distro_tracker.debci_status.tracker_package_tables import DebciTableField
34from distro_tracker.mail import mail_news
35from distro_tracker.vendor.common import PluginProcessingError
36from distro_tracker.vendor.debian.tracker_tasks import UpdateNewQueuePackages
39from .models import DebianBugDisplayManager, DebianContributor
40from .tracker_package_tables import UpstreamTableField
43def _simplify_pkglist(pkglist, multi_allowed=True, default=None):
44 """Replace a single-list item by its sole item. A longer list is left
45 as-is (provided multi_allowed is True). An empty list returns the default
46 value."""
47 if len(pkglist) == 1 and pkglist[0]:
48 return pkglist[0]
49 elif len(pkglist) > 1 and multi_allowed:
50 return pkglist
51 return default
54def _classify_bts_message(msg, package, keyword):
55 bts_package = msg.get('X-Debian-PR-Source',
56 msg.get('X-Debian-PR-Package', ''))
57 pkglist = re.split(r'\s+', bts_package.strip())
58 # Don't override default package assignation when we find multiple package
59 # associated to the mail, otherwise we will send multiple copies of a mail
60 # that we already receive multiple times
61 multi_allowed = package is None
62 pkg_result = _simplify_pkglist(pkglist, multi_allowed=multi_allowed,
63 default=package)
65 # We override the package/keyword only...
66 if package is None: # When needed, because we don't have a suggestion
67 override_suggestion = True
68 else: # Or when package suggestion matches the one found in the header
69 override_suggestion = package == pkg_result
71 if override_suggestion:
72 package = pkg_result
74 if override_suggestion or keyword is None:
75 debian_pr_message = msg.get('X-Debian-PR-Message', '')
76 if debian_pr_message.startswith('transcript'):
77 keyword = 'bts-control'
78 else:
79 keyword = 'bts'
81 return (package, keyword)
84def _classify_dak_message(msg, package, keyword):
85 subject = msg.get('Subject', '').strip()
86 xdak = msg.get('X-DAK', '')
87 package = msg.get('Debian-Source', package)
88 if package:
89 package = package.strip()
90 action = msg.get('Debian-Archive-Action', '').strip()
91 architecture = msg.get('Debian-Architecture', '').strip()
92 body = get_message_body(msg)
94 if action == "accept":
95 if "source" in architecture:
96 keyword = 'upload-source'
97 if re.search(r'^Accepted', subject):
98 mail_news.create_news(msg, package, create_package=True)
99 else:
100 keyword = 'upload-binary'
101 else:
102 keyword = 'archive'
104 if xdak == 'dak rm':
105 # Find all lines giving information about removed source packages
106 re_rmline = re.compile(r"^\s*(\S+)\s*\|\s*(\S+)\s*\|\s*(.*)", re.M)
107 source_removals = re_rmline.findall(body)
108 removed_pkgver = {}
109 for pkgname, version, arch in source_removals:
110 removed_pkgver[pkgname] = (version, arch)
111 if package not in removed_pkgver:
112 package = _simplify_pkglist(list(removed_pkgver.keys()),
113 multi_allowed=False,
114 default=package)
115 if package in removed_pkgver and "source" in removed_pkgver[package][1]:
116 create_dak_rm_news(msg, package, version=removed_pkgver[package][0],
117 body=body)
119 return (package, keyword)
122def classify_message(msg, package, keyword):
123 """Classify incoming email messages with a package and a keyword."""
124 # Default values for git commit notifications
125 xgitrepo = msg.get('X-GitLab-Project-Path', msg.get('X-Git-Repo'))
126 if xgitrepo:
127 xgitrepo = xgitrepo.strip()
128 if not package: 128 ↛ 132line 128 didn't jump to line 132, because the condition on line 128 was never false
129 if xgitrepo.endswith('.git'):
130 xgitrepo = xgitrepo[:-4]
131 package = os.path.basename(xgitrepo)
132 if not keyword: 132 ↛ 135line 132 didn't jump to line 135, because the condition on line 132 was never false
133 keyword = 'vcs'
135 xloop = msg.get_all('X-Loop', ())
136 xdebian = msg.get_all('X-Debian', ())
137 testing_watch = msg.get('X-Testing-Watch-Package')
139 bts_match = 'owner@bugs.debian.org' in xloop
140 dak_match = 'DAK' in xdebian
141 buildd_match = 'buildd.debian.org' in xdebian
142 autoremovals_match = 'release.debian.org/autoremovals' in xdebian
144 if bts_match: # This is a mail of the Debian bug tracking system
145 package, keyword = _classify_bts_message(msg, package, keyword)
146 elif dak_match:
147 package, keyword = _classify_dak_message(msg, package, keyword)
148 elif buildd_match:
149 keyword = 'build'
150 package = msg.get('X-Debian-Package', package)
151 elif autoremovals_match:
152 keyword = 'summary'
153 package = msg.get('X-Debian-Package', package)
154 elif testing_watch:
155 package = testing_watch
156 keyword = 'summary'
157 mail_news.create_news(msg, package)
159 # Converts old PTS keywords into new ones
160 legacy_mapping = {
161 'katie-other': 'archive',
162 'buildd': 'build',
163 'ddtp': 'translation',
164 'cvs': 'vcs',
165 }
166 if keyword in legacy_mapping:
167 keyword = legacy_mapping[keyword]
168 if isinstance(package, str):
169 package = package.strip()
170 return (package, keyword)
173def add_new_headers(received_message, package_name, keyword, team):
174 """
175 Debian adds the following new headers:
176 - X-Debian-Package
177 - X-Debian
179 :param received_message: The original received package message
180 :type received_message: :py:class:`email.message.Message`
182 :param package_name: The name of the package for which the message was
183 intended
184 :type package_name: string
186 :param keyword: The keyword with which the message is tagged.
187 :type keyword: string
188 """
189 new_headers = [
190 ('X-Debian', 'tracker.debian.org'),
191 ]
192 if package_name: 192 ↛ 196line 192 didn't jump to line 196, because the condition on line 192 was never false
193 new_headers.append(('X-Debian-Package', package_name))
194 new_headers.append(
195 ('X-PTS-Package', package_name)) # for compat with old PTS
196 if keyword: 196 ↛ 199line 196 didn't jump to line 199, because the condition on line 196 was never false
197 new_headers.append(
198 ('X-PTS-Keyword', keyword)) # for compat with old PTS
199 return new_headers
202def approve_default_message(msg):
203 """
204 Debian approves a default message only if it has a X-Bugzilla-Product
205 header.
207 :param msg: The original received package message
208 :type msg: :py:class:`email.message.Message`
209 """
210 return 'X-Bugzilla-Product' in msg
213def get_pseudo_package_list():
214 """
215 Existing pseudo packages for Debian are obtained from
216 `BTS <https://bugs.debian.org/pseudo-packages.maintainers>`_
217 """
218 PSEUDO_PACKAGE_LIST_URL = (
219 'https://bugs.debian.org/pseudo-packages.maintainers'
220 )
221 cache = HttpCache(settings.DISTRO_TRACKER_CACHE_DIRECTORY)
222 if not cache.is_expired(PSEUDO_PACKAGE_LIST_URL): 222 ↛ 223line 222 didn't jump to line 223, because the condition on line 222 was never true
223 return
224 response, updated = cache.update(PSEUDO_PACKAGE_LIST_URL)
226 try:
227 response.raise_for_status()
228 except requests.exceptions.HTTPError:
229 raise PluginProcessingError()
231 if not updated: 231 ↛ 232line 231 didn't jump to line 232, because the condition on line 231 was never true
232 return
234 return [
235 line.split(None, 1)[0]
236 for line in response.text.splitlines()
237 ]
240def get_package_information_site_url(package_name, source_package=False,
241 repository=None, version=None):
242 """
243 Return a link pointing to more information about a package in a
244 given repository.
245 """
246 BASE_URL = 'https://packages.debian.org/'
247 PU_URL = 'https://release.debian.org/proposed-updates/'
248 SOURCE_PACKAGE_URL_TEMPLATES = {
249 'repository': BASE_URL + 'source/{repo}/{package}',
250 'no-repository': BASE_URL + 'src:{package}',
251 'pu': PU_URL + '{targetsuite}.html#{package}_{version}',
252 }
253 BINARY_PACKAGE_URL_TEMPLATES = {
254 'repository': BASE_URL + '{repo}/{package}',
255 'no-repository': BASE_URL + '{package}',
256 'pu': '',
257 }
259 params = {'package': package_name}
260 if repository:
261 suite = repository['suite'] or repository['codename']
262 if suite.endswith('proposed-updates'):
263 url_type = 'pu'
264 params['version'] = version
265 params['targetsuite'] = suite.replace('-proposed-updates', '')\
266 .replace('proposed-updates', 'stable')
267 else:
268 url_type = 'repository'
269 params['repo'] = suite
270 else:
271 url_type = 'no-repository'
273 if source_package:
274 template = SOURCE_PACKAGE_URL_TEMPLATES[url_type]
275 else:
276 template = BINARY_PACKAGE_URL_TEMPLATES[url_type]
278 return template.format(**params)
281def get_developer_information_url(developer_email):
282 """
283 Return a URL to extra information about a developer, by email address.
284 """
285 URL_TEMPLATE = 'https://qa.debian.org/developer.php?login={email}'
286 return URL_TEMPLATE.format(email=quote_plus(developer_email))
289def get_external_version_information_urls(package_name):
290 """
291 The function returns a list of external Web resources which provide
292 additional information about the versions of a package.
293 """
294 return [
295 {
296 'url': 'https://qa.debian.org/madison.php?package={package}'.format(
297 package=quote_plus(package_name)),
298 'description': 'more versions can be listed by madison',
299 },
300 {
301 'url': 'https://snapshot.debian.org/package/{package}/'.format(
302 package=package_name),
303 'description': 'old versions available from snapshot.debian.org',
304 }
305 ]
308def get_maintainer_extra(developer_email, package_name=None):
309 """
310 The function returns a list of additional items that are to be
311 included in the general panel next to the maintainer. This includes:
313 - Whether the maintainer agrees with lowthreshold NMU
314 - Whether the maintainer is a Debian Maintainer
315 """
316 developer = get_or_none(DebianContributor,
317 email__email__iexact=developer_email)
318 extra = []
319 _add_dmd_entry(extra, developer_email)
320 if developer and developer.agree_with_low_threshold_nmu: 320 ↛ 326line 320 didn't jump to line 326, because the condition on line 320 was never false
321 extra.append({
322 'display': 'LowNMU',
323 'description': 'maintainer agrees with Low Threshold NMU',
324 'link': 'https://wiki.debian.org/LowThresholdNmu',
325 })
326 _add_dm_entry(extra, developer, package_name)
327 return extra
330def get_uploader_extra(developer_email, package_name=None):
331 """
332 The function returns a list of additional items that are to be
333 included in the general panel next to an uploader. This includes:
335 - Whether the uploader is a DebianMaintainer
336 """
337 developer = get_or_none(DebianContributor,
338 email__email__iexact=developer_email)
340 extra = []
341 _add_dmd_entry(extra, developer_email)
342 _add_dm_entry(extra, developer, package_name)
343 return extra
346def _add_dmd_entry(extra, email):
347 extra.append({
348 'display': 'DMD',
349 'description': 'UDD\'s Debian Maintainer Dashboard',
350 'link': 'https://udd.debian.org/dmd/?{email}#todo'.format(
351 email=quote_plus(email)
352 )
353 })
356def _add_dm_entry(extra, developer, package_name):
357 if package_name and developer and developer.is_debian_maintainer:
358 if package_name in developer.allowed_packages: 358 ↛ exitline 358 didn't return from function '_add_dm_entry', because the condition on line 358 was never false
359 extra.append(
360 {
361 'display': 'DM',
362 'description': 'Debian Maintainer upload allowed',
363 'link': 'https://ftp-master.debian.org/dm.txt'
364 }
365 )
368def allow_package(stanza):
369 """
370 The function provides a way for vendors to exclude some packages from being
371 saved in the database.
373 In Debian's case, this is done for packages where the ``Extra-Source-Only``
374 is set since those packages are in the repository only for various
375 compliance reasons.
377 :param stanza: The raw package entry from a ``Sources`` file.
378 :type stanza: case-insensitive dict
379 """
380 return 'Extra-Source-Only' not in stanza
383def create_dak_rm_news(message, package, body=None, version=''):
384 """Create a :class:`News` out of a removal email sent by DAK."""
385 if not body: 385 ↛ 386line 385 didn't jump to line 386, because the condition on line 385 was never true
386 body = get_decoded_message_payload(message)
387 suite = re.search(r"have been removed from (\S+):", body).group(1)
388 title = "Removed {ver} from {suite}".format(ver=version, suite=suite)
389 return mail_news.create_news(message, package, title=title)
392def get_extra_versions(package):
393 """
394 :returns: The versions of the package found in the NEW queue.
395 """
396 try:
397 info = package.data.get(key=UpdateNewQueuePackages.DATA_KEY)
398 except PackageData.DoesNotExist:
399 return
401 version_url_template = 'https://ftp-master.debian.org/new/{pkg}_{ver}.html'
402 return [
403 {
404 'version': ver['version'],
405 'repository_shorthand': 'NEW/' + dist,
406 'version_link': version_url_template.format(
407 pkg=package.name, ver=ver['version']),
408 'repository_link': 'https://ftp-master.debian.org/new.html',
409 }
410 for dist, ver in info.value.items()
411 ]
414def pre_login(form):
415 """
416 If the user has a @debian.org email associated, don't let them log
417 in directly through local authentication.
418 """
419 username = form.cleaned_data.get('username')
420 if not username:
421 return
422 user_email = get_or_none(UserEmail, email__iexact=username)
423 emails = [username]
424 if user_email and user_email.user:
425 emails += [x.email for x in user_email.user.emails.all()]
426 if any(email.endswith('@debian.org') for email in emails):
427 raise forms.ValidationError(mark_safe(
428 "Your account has a @debian.org email address associated. "
429 "To log in to the package tracker, you must use a SSL client "
430 "certificate generated on "
431 "<a href='https://sso.debian.org/'>"
432 "sso.debian.org</a> (click on the link!)."))
435def post_logout(request, user, next_url=None):
436 """
437 If the user is authenticated via the SSO, sign them out at the SSO
438 level too.
439 """
440 if request.META.get('REMOTE_USER'):
441 if next_url is None:
442 next_url = 'https://' + settings.DISTRO_TRACKER_FQDN
443 elif next_url.startswith('/'):
444 next_url = 'https://' + settings.DISTRO_TRACKER_FQDN + next_url
445 return (
446 'https://sso.debian.org/cgi-bin/dacs/dacs_signout?' + urlencode({
447 'SIGNOUT_HANDLER': next_url
448 })
449 )
452def get_table_fields(table):
453 """
454 The function provides additional fields which should be displayed in
455 the team's packages table
456 """
457 return table.default_fields + [DebciTableField, UpstreamTableField]
460def additional_prefetch_related_lookups():
461 """
462 :returns: The list with additional lookups to be prefetched along with
463 default lookups defined by :class:`BaseTableField`
464 """
465 return [
466 Prefetch(
467 'action_items',
468 queryset=ActionItem.objects.filter(
469 item_type__type_name='vcswatch-warnings-and-errors'
470 ).prefetch_related('item_type'),
471 ),
472 Prefetch(
473 'data',
474 queryset=PackageData.objects.filter(key='vcswatch'),
475 to_attr='vcswatch_data'
476 ),
477 ]
480def get_vcs_data(package):
481 """
482 :returns: The dictionary with VCS Watch data to be displayed in
483 the template defined by :data:`DISTRO_TRACKER_VCS_TABLE_FIELD_TEMPLATE
484 <distro_tracker.project.local_settings.DISTRO_TRACKER_VCS_TABLE_FIELD_TEMPLATE>`
485 settings.
486 """
487 data = {}
488 try:
489 item = package.vcswatch_data[0]
490 data['changelog_version'] = item.value['changelog_version']
491 except IndexError:
492 # There is no vcs extra data for the package
493 pass
495 try:
496 item = package.action_items.all()[0]
497 data['action_item'] = item.to_dict()
498 data['action_item']['url'] = item.get_absolute_url()
499 except IndexError:
500 # There is no action item for the package
501 pass
502 return data
505def get_bug_display_manager_class():
506 """Return the class that knows how to display data about Debian bugs."""
507 return DebianBugDisplayManager
510def get_tables_for_team_page(team, limit):
511 """
512 The function must return a list of :class:`BasePackageTable` objects
513 to be displayed in the main page of teams.
515 :param team: The team for which the tables must be added.
516 :type package: :class:`Team <distro_tracker.core.models.Team>`
517 :param int limit: The number of packages to be displayed in the tables.
518 """
519 return [
520 create_table(slug='general', scope=team, limit=limit),
521 create_table(
522 slug='general', scope=team, limit=limit, tag='tag:rc-bugs'),
523 create_table(
524 slug='general', scope=team, limit=limit,
525 tag='tag:new-upstream-version'),
526 create_table(
527 slug='general', scope=team, limit=limit, tag='tag:bugs'),
528 create_table(
529 slug='general', scope=team, limit=limit,
530 tag='tag:debci-failures')
531 ]