Coverage for distro_tracker/vendor/debian/rules.py: 88%

2# See the COPYRIGHT file at the top-level directory of this distribution and

3# at https://deb.li/DTAuthors

5# This file is part of Distro Tracker. It is subject to the license terms

6# in the LICENSE file found in the top-level directory of this

7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker,

8# including this file, may be copied, modified, propagated, or distributed

9# except according to the terms contained in the LICENSE file.

10"""Debian specific rules for various Distro-Tracker hooks."""

12import os.path

13import re

14from urllib.parse import quote_plus

16from django import forms

17from django.conf import settings

18from django.db.models import Prefetch

19from django.utils.safestring import mark_safe

21import requests

23from distro_tracker.core.models import (

24 ActionItem,

25 PackageData,

26 UserEmail

27)

28from distro_tracker.core.package_tables import create_table

29from distro_tracker.core.utils import get_decoded_message_payload, get_or_none

30from distro_tracker.core.utils.email_messages import get_message_body

31from distro_tracker.core.utils.http import HttpCache

32from distro_tracker.debci_status.tracker_package_tables import DebciTableField

33from distro_tracker.mail import mail_news

34from distro_tracker.vendor.common import PluginProcessingError

35from distro_tracker.vendor.debian.tracker_tasks import UpdateNewQueuePackages

38from .models import DebianBugDisplayManager, DebianContributor

39from .tracker_package_tables import UpstreamTableField

42def _simplify_pkglist(pkglist, multi_allowed=True, default=None):

43 """Replace a single-list item by its sole item. A longer list is left

44 as-is (provided multi_allowed is True). An empty list returns the default

45 value."""

46 if len(pkglist) == 1 and pkglist[0]:

47 return pkglist[0]

48 elif len(pkglist) > 1 and multi_allowed:

49 return pkglist

50 return default

53def _classify_bts_message(msg, package, keyword):

54 bts_package = msg.get('X-Debian-PR-Source',

55 msg.get('X-Debian-PR-Package', ''))

56 pkglist = re.split(r'\s+', bts_package.strip())

57 # Don't override default package assignation when we find multiple package

58 # associated to the mail, otherwise we will send multiple copies of a mail

59 # that we already receive multiple times

60 multi_allowed = package is None

61 pkg_result = _simplify_pkglist(pkglist, multi_allowed=multi_allowed,

62 default=package)

64 # We override the package/keyword only...

65 if package is None: # When needed, because we don't have a suggestion

66 override_suggestion = True

67 else: # Or when package suggestion matches the one found in the header

68 override_suggestion = package == pkg_result

70 if override_suggestion:

71 package = pkg_result

73 if override_suggestion or keyword is None:

74 debian_pr_message = msg.get('X-Debian-PR-Message', '')

75 if debian_pr_message.startswith('transcript'):

76 keyword = 'bts-control'

77 else:

78 keyword = 'bts'

80 return (package, keyword)

83def _classify_dak_message(msg, package, keyword):

84 subject = msg.get('Subject', '').strip()

85 xdak = msg.get('X-DAK', '')

86 package = msg.get('Debian-Source', package)

87 if package:

88 package = package.strip()

89 action = msg.get('Debian-Archive-Action', '').strip()

90 architecture = msg.get('Debian-Architecture', '').strip()

91 body = get_message_body(msg)

93 if action == "accept":

94 if "source" in architecture:

95 keyword = 'upload-source'

96 if re.search(r'^Accepted', subject):

97 mail_news.create_news(msg, package, create_package=True)

98 else:

99 keyword = 'upload-binary'

100 else:

101 keyword = 'archive'

102

103 if xdak == 'dak rm':

104 # Find all lines giving information about removed source packages

105 re_rmline = re.compile(r"^\s*(\S+)\s*\|\s*(\S+)\s*\|\s*(.*)", re.M)

106 source_removals = re_rmline.findall(body)

107 removed_pkgver = {}

108 for pkgname, version, arch in source_removals:

109 removed_pkgver[pkgname] = (version, arch)

110 if package not in removed_pkgver:

111 package = _simplify_pkglist(list(removed_pkgver.keys()),

112 multi_allowed=False,

113 default=package)

114 if package in removed_pkgver and "source" in removed_pkgver[package][1]:

115 create_dak_rm_news(msg, package, version=removed_pkgver[package][0],

116 body=body)

117

118 return (package, keyword)

119

120

121def classify_message(msg, package, keyword):

122 """Classify incoming email messages with a package and a keyword."""

123 # Default values for git commit notifications

124 xgitrepo = msg.get('X-GitLab-Project-Path', msg.get('X-Git-Repo'))

125 if xgitrepo:

126 xgitrepo = xgitrepo.strip()

127 if not package: 127 ↛ 131line 127 didn't jump to line 131, because the condition on line 127 was never false

128 if xgitrepo.endswith('.git'):

129 xgitrepo = xgitrepo[:-4]

130 package = os.path.basename(xgitrepo)

131 if not keyword: 131 ↛ 134line 131 didn't jump to line 134, because the condition on line 131 was never false

132 keyword = 'vcs'

133

134 xloop = msg.get_all('X-Loop', ())

135 xdebian = msg.get_all('X-Debian', ())

136 testing_watch = msg.get('X-Testing-Watch-Package')

137

138 bts_match = 'owner@bugs.debian.org' in xloop

139 dak_match = 'DAK' in xdebian

140 buildd_match = 'buildd.debian.org' in xdebian

141 autoremovals_match = 'release.debian.org/autoremovals' in xdebian

142

143 if bts_match: # This is a mail of the Debian bug tracking system

144 package, keyword = _classify_bts_message(msg, package, keyword)

145 elif dak_match:

146 package, keyword = _classify_dak_message(msg, package, keyword)

147 elif buildd_match:

148 keyword = 'build'

149 package = msg.get('X-Debian-Package', package)

150 elif autoremovals_match:

151 keyword = 'summary'

152 package = msg.get('X-Debian-Package', package)

153 elif testing_watch:

154 package = testing_watch

155 keyword = 'summary'

156 mail_news.create_news(msg, package)

157

158 # Converts old PTS keywords into new ones

159 legacy_mapping = {

160 'katie-other': 'archive',

161 'buildd': 'build',

162 'ddtp': 'translation',

163 'cvs': 'vcs',

164 }

165 if keyword in legacy_mapping:

166 keyword = legacy_mapping[keyword]

167 if isinstance(package, str):

168 package = package.strip()

169 return (package, keyword)

170

171

172def add_new_headers(received_message, package_name, keyword, team):

173 """

174 Debian adds the following new headers:

175 - X-Debian-Package

176 - X-Debian

177

178 :param received_message: The original received package message

179 :type received_message: :py:class:`email.message.Message`

180

181 :param package_name: The name of the package for which the message was

182 intended

183 :type package_name: string

184

185 :param keyword: The keyword with which the message is tagged.

186 :type keyword: string

187 """

188 new_headers = [

189 ('X-Debian', 'tracker.debian.org'),

190 ]

191 if package_name: 191 ↛ 195line 191 didn't jump to line 195, because the condition on line 191 was never false

192 new_headers.append(('X-Debian-Package', package_name))

193 new_headers.append(

194 ('X-PTS-Package', package_name)) # for compat with old PTS

195 if keyword: 195 ↛ 198line 195 didn't jump to line 198, because the condition on line 195 was never false

196 new_headers.append(

197 ('X-PTS-Keyword', keyword)) # for compat with old PTS

198 return new_headers

199

200

201def approve_default_message(msg):

202 """

203 Debian approves a default message only if it has a X-Bugzilla-Product

204 header.

205

206 :param msg: The original received package message

207 :type msg: :py:class:`email.message.Message`

208 """

209 return 'X-Bugzilla-Product' in msg

210

211

212def get_pseudo_package_list():

213 """

214 Existing pseudo packages for Debian are obtained from

215 `BTS <https://bugs.debian.org/pseudo-packages.maintainers>`_

216 """

217 PSEUDO_PACKAGE_LIST_URL = (

218 'https://bugs.debian.org/pseudo-packages.maintainers'

219 )

220 cache = HttpCache(settings.DISTRO_TRACKER_CACHE_DIRECTORY)

221 if not cache.is_expired(PSEUDO_PACKAGE_LIST_URL): 221 ↛ 222line 221 didn't jump to line 222, because the condition on line 221 was never true

222 return

223 response, updated = cache.update(PSEUDO_PACKAGE_LIST_URL)

224

225 try:

226 response.raise_for_status()

227 except requests.exceptions.HTTPError:

228 raise PluginProcessingError()

229

230 if not updated: 230 ↛ 231line 230 didn't jump to line 231, because the condition on line 230 was never true

231 return

232

233 return [

234 line.split(None, 1)[0]

235 for line in response.text.splitlines()

236 ]

237

238

239def get_package_information_site_url(package_name, source_package=False,

240 repository=None, version=None):

241 """

242 Return a link pointing to more information about a package in a

243 given repository.

244 """

245 BASE_URL = 'https://packages.debian.org/'

246 PU_URL = 'https://release.debian.org/proposed-updates/'

247 SOURCE_PACKAGE_URL_TEMPLATES = {

248 'repository': BASE_URL + 'source/{repo}/{package}',

249 'no-repository': BASE_URL + 'src:{package}',

250 'pu': PU_URL + '{targetsuite}.html#{package}_{version}',

251 }

252 BINARY_PACKAGE_URL_TEMPLATES = {

253 'repository': BASE_URL + '{repo}/{package}',

254 'no-repository': BASE_URL + '{package}',

255 'pu': '',

256 }

257

258 params = {'package': package_name}

259 if repository:

260 suite = repository['suite'] or repository['codename']

261 if suite.endswith('proposed-updates'):

262 url_type = 'pu'

263 params['version'] = version

264 params['targetsuite'] = suite.replace('-proposed-updates', '')\

265 .replace('proposed-updates', 'stable')

266 else:

267 url_type = 'repository'

268 params['repo'] = suite

269 else:

270 url_type = 'no-repository'

271

272 if source_package:

273 template = SOURCE_PACKAGE_URL_TEMPLATES[url_type]

274 else:

275 template = BINARY_PACKAGE_URL_TEMPLATES[url_type]

276

277 return template.format(**params)

278

279

280def get_developer_information_url(developer_email):

281 """

282 Return a URL to extra information about a developer, by email address.

283 """

284 URL_TEMPLATE = 'https://qa.debian.org/developer.php?login={email}'

285 return URL_TEMPLATE.format(email=quote_plus(developer_email))

286

287

288def get_external_version_information_urls(package_name):

289 """

290 The function returns a list of external Web resources which provide

291 additional information about the versions of a package.

292 """

293 return [

294 {

295 'url': 'https://qa.debian.org/madison.php?package={package}'.format(

296 package=quote_plus(package_name)),

297 'description': 'more versions can be listed by madison',

298 },

299 {

300 'url': 'https://snapshot.debian.org/package/{package}/'.format(

301 package=package_name),

302 'description': 'old versions available from snapshot.debian.org',

303 }

304 ]

305

306

307def get_maintainer_extra(developer_email, package_name=None):

308 """

309 The function returns a list of additional items that are to be

310 included in the general panel next to the maintainer. This includes:

311

312 - Whether the maintainer agrees with lowthreshold NMU

313 - Whether the maintainer is a Debian Maintainer

314 """

315 developer = get_or_none(DebianContributor,

316 email__email__iexact=developer_email)

317 extra = []

318 _add_dmd_entry(extra, developer_email)

319 if developer and developer.agree_with_low_threshold_nmu: 319 ↛ 325line 319 didn't jump to line 325, because the condition on line 319 was never false

320 extra.append({

321 'display': 'LowNMU',

322 'description': 'maintainer agrees with Low Threshold NMU',

323 'link': 'https://wiki.debian.org/LowThresholdNmu',

324 })

325 _add_dm_entry(extra, developer, package_name)

326 return extra

327

328

329def get_uploader_extra(developer_email, package_name=None):

330 """

331 The function returns a list of additional items that are to be

332 included in the general panel next to an uploader. This includes:

333

334 - Whether the uploader is a DebianMaintainer

335 """

336 developer = get_or_none(DebianContributor,

337 email__email__iexact=developer_email)

338

339 extra = []

340 _add_dmd_entry(extra, developer_email)

341 _add_dm_entry(extra, developer, package_name)

342 return extra

343

344

345def _add_dmd_entry(extra, email):

346 extra.append({

347 'display': 'DMD',

348 'description': 'UDD\'s Debian Maintainer Dashboard',

349 'link': 'https://udd.debian.org/dmd/?{email}#todo'.format(

350 email=quote_plus(email)

351 )

352 })

353

354

355def _add_dm_entry(extra, developer, package_name):

356 if package_name and developer and developer.is_debian_maintainer:

357 if package_name in developer.allowed_packages: 357 ↛ exitline 357 didn't return from function '_add_dm_entry', because the condition on line 357 was never false

358 extra.append(

359 {

360 'display': 'DM',

361 'description': 'Debian Maintainer upload allowed',

362 'link': 'https://ftp-master.debian.org/dm.txt'

363 }

364 )

365

366

367def allow_package(stanza):

368 """

369 The function provides a way for vendors to exclude some packages from being

370 saved in the database.

371

372 In Debian's case, this is done for packages where the ``Extra-Source-Only``

373 is set since those packages are in the repository only for various

374 compliance reasons.

375

376 :param stanza: The raw package entry from a ``Sources`` file.

377 :type stanza: case-insensitive dict

378 """

379 return 'Extra-Source-Only' not in stanza

380

381

382def create_dak_rm_news(message, package, body=None, version=''):

383 """Create a :class:`News` out of a removal email sent by DAK."""

384 if not body: 384 ↛ 385line 384 didn't jump to line 385, because the condition on line 384 was never true

385 body = get_decoded_message_payload(message)

386 suite = re.search(r"have been removed from (\S+):", body).group(1)

387 title = "Removed {ver} from {suite}".format(ver=version, suite=suite)

388 return mail_news.create_news(message, package, title=title)

389

390

391def get_extra_versions(package):

392 """

393 :returns: The versions of the package found in the NEW queue.

394 """

395 try:

396 info = package.data.get(key=UpdateNewQueuePackages.DATA_KEY)

397 except PackageData.DoesNotExist:

398 return

399

400 version_url_template = 'https://ftp-master.debian.org/new/{pkg}_{ver}.html'

401 return [

402 {

403 'version': ver['version'],

404 'repository_shorthand': 'NEW/' + dist,

405 'version_link': version_url_template.format(

406 pkg=package.name, ver=ver['version']),

407 'repository_link': 'https://ftp-master.debian.org/new.html',

408 }

409 for dist, ver in info.value.items()

410 ]

411

412

413def pre_login(form):

414 """

415 If the user has a @debian.org email associated, don't let them log

416 in directly through local authentication.

417 """

418 username = form.cleaned_data.get('username')

419 if not username:

420 return

421 user_email = get_or_none(UserEmail, email__iexact=username)

422 emails = [username]

423 if user_email and user_email.user:

424 emails += [x.email for x in user_email.user.emails.all()]

425 if any(email.endswith('@debian.org') for email in emails):

426 raise forms.ValidationError(mark_safe(

427 "Your account has a @debian.org email address associated. "

428 "To log in to the package tracker, you must use SSO via your "

429 "account on <a href='https://salsa.debian.org/'>Salsa</a>."))

430

431

432def get_table_fields(table):

433 """

434 The function provides additional fields which should be displayed in

435 the team's packages table

436 """

437 return table.default_fields + [DebciTableField, UpstreamTableField]

438

439

440def additional_prefetch_related_lookups():

441 """

442 :returns: The list with additional lookups to be prefetched along with

443 default lookups defined by :class:`BaseTableField`

444 """

445 return [

446 Prefetch(

447 'action_items',

448 queryset=ActionItem.objects.filter(

449 item_type__type_name='vcswatch-warnings-and-errors'

450 ).prefetch_related('item_type'),

451 ),

452 Prefetch(

453 'data',

454 queryset=PackageData.objects.filter(key='vcswatch'),

455 to_attr='vcswatch_data'

456 ),

457 ]

458

459

460def get_vcs_data(package):

461 """

462 :returns: The dictionary with VCS Watch data to be displayed in

463 the template defined by :data:`DISTRO_TRACKER_VCS_TABLE_FIELD_TEMPLATE

464 <distro_tracker.project.local_settings.DISTRO_TRACKER_VCS_TABLE_FIELD_TEMPLATE>`

465 settings.

466 """

467 data = {}

468 try:

469 item = package.vcswatch_data[0]

470 data['changelog_version'] = item.value['changelog_version']

471 except IndexError:

472 # There is no vcs extra data for the package

473 pass

474

475 try:

476 item = package.action_items.all()[0]

477 data['action_item'] = item.to_dict()

478 data['action_item']['url'] = item.get_absolute_url()

479 except IndexError:

480 # There is no action item for the package

481 pass

482 return data

483

484

485def get_bug_display_manager_class():

486 """Return the class that knows how to display data about Debian bugs."""

487 return DebianBugDisplayManager

488

489

490def get_tables_for_team_page(team, limit):

491 """

492 The function must return a list of :class:`BasePackageTable` objects

493 to be displayed in the main page of teams.

494

495 :param team: The team for which the tables must be added.

496 :type package: :class:`Team <distro_tracker.core.models.Team>`

497 :param int limit: The number of packages to be displayed in the tables.

498 """

499 return [

500 create_table(slug='general', scope=team, limit=limit),

501 create_table(

502 slug='general', scope=team, limit=limit, tag='tag:rc-bugs'),

503 create_table(

504 slug='general', scope=team, limit=limit,

505 tag='tag:new-upstream-version'),

506 create_table(

507 slug='general', scope=team, limit=limit, tag='tag:bugs'),

508 create_table(

509 slug='general', scope=team, limit=limit,

510 tag='tag:debci-failures')

511 ]