1# Copyright 2013-2015 The Distro Tracker Developers 

2# See the COPYRIGHT file at the top-level directory of this distribution and 

3# at https://deb.li/DTAuthors 

4# 

5# This file is part of Distro Tracker. It is subject to the license terms 

6# in the LICENSE file found in the top-level directory of this 

7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker, 

8# including this file, may be copied, modified, propagated, or distributed 

9# except according to the terms contained in the LICENSE file. 

10"""Debian specific rules for various Distro-Tracker hooks.""" 

11 

12import os.path 

13import re 

14 

15from django import forms 

16from django.conf import settings 

17from django.db.models import Prefetch 

18from django.utils.http import urlencode, urlquote_plus 

19from django.utils.safestring import mark_safe 

20 

21import requests 

22 

23from distro_tracker.core.models import ( 

24 ActionItem, 

25 PackageData, 

26 UserEmail 

27) 

28from distro_tracker.core.package_tables import create_table 

29from distro_tracker.core.utils import get_decoded_message_payload, get_or_none 

30from distro_tracker.core.utils.http import HttpCache 

31from distro_tracker.debci_status.tracker_package_tables import DebciTableField 

32from distro_tracker.mail import mail_news 

33from distro_tracker.vendor.common import PluginProcessingError 

34from distro_tracker.vendor.debian.tracker_tasks import UpdateNewQueuePackages 

35 

36from .models import DebianBugDisplayManager, DebianContributor 

37from .tracker_package_tables import UpstreamTableField 

38 

39 

40def _simplify_pkglist(pkglist, multi_allowed=True, default=None): 

41 """Replace a single-list item by its sole item. A longer list is left 

42 as-is (provided multi_allowed is True). An empty list returns the default 

43 value.""" 

44 if len(pkglist) == 1 and pkglist[0]: 

45 return pkglist[0] 

46 elif len(pkglist) > 1 and multi_allowed: 

47 return pkglist 

48 return default 

49 

50 

51def _classify_bts_message(msg, package, keyword): 

52 bts_package = msg.get('X-Debian-PR-Source', 

53 msg.get('X-Debian-PR-Package', '')) 

54 pkglist = re.split(r'\s+', bts_package.strip()) 

55 # Don't override default package assignation when we find multiple package 

56 # associated to the mail, otherwise we will send multiple copies of a mail 

57 # that we already receive multiple times 

58 multi_allowed = package is None 

59 pkg_result = _simplify_pkglist(pkglist, multi_allowed=multi_allowed, 

60 default=package) 

61 

62 # We override the package/keyword only... 

63 if package is None: # When needed, because we don't have a suggestion 

64 override_suggestion = True 

65 else: # Or when package suggestion matches the one found in the header 

66 override_suggestion = package == pkg_result 

67 

68 if override_suggestion: 

69 package = pkg_result 

70 

71 if override_suggestion or keyword is None: 

72 debian_pr_message = msg.get('X-Debian-PR-Message', '') 

73 if debian_pr_message.startswith('transcript'): 

74 keyword = 'bts-control' 

75 else: 

76 keyword = 'bts' 

77 

78 return (package, keyword) 

79 

80 

81def _classify_dak_message(msg, package, keyword): 

82 package = msg.get('X-Debian-Package', package) 

83 subject = msg.get('Subject', '') 

84 xdak = msg.get('X-DAK', '') 

85 body = _get_message_body(msg) 

86 if re.search(r'^Accepted|ACCEPTED', subject): 

87 if re.search(r'^Accepted.*\(.*source.*\)', subject): 

88 mail_news.create_news(msg, package, create_package=True) 

89 if re.search(r'\.dsc\s*$', body, flags=re.MULTILINE): 

90 keyword = 'upload-source' 

91 else: 

92 keyword = 'upload-binary' 

93 else: 

94 keyword = 'archive' 

95 if xdak == 'dak rm': 

96 # Find all lines giving information about removed source packages 

97 re_rmline = re.compile(r"^\s*(\S+)\s*\|\s*(\S+)\s*\|\s*(.*)", re.M) 

98 source_removals = re_rmline.findall(body) 

99 removed_pkgver = {} 

100 for pkgname, version, arch in source_removals: 

101 removed_pkgver[pkgname] = (version, arch) 

102 if package not in removed_pkgver: 

103 package = _simplify_pkglist(list(removed_pkgver.keys()), 

104 multi_allowed=False, 

105 default=package) 

106 if package in removed_pkgver and "source" in removed_pkgver[package][1]: 

107 create_dak_rm_news(msg, package, version=removed_pkgver[package][0], 

108 body=body) 

109 

110 return (package, keyword) 

111 

112 

113def classify_message(msg, package, keyword): 

114 """Classify incoming email messages with a package and a keyword.""" 

115 # Default values for git commit notifications 

116 xgitrepo = msg.get('X-GitLab-Project-Path', msg.get('X-Git-Repo')) 

117 if xgitrepo: 

118 if not package: 118 ↛ 122line 118 didn't jump to line 122, because the condition on line 118 was never false

119 if xgitrepo.endswith('.git'): 

120 xgitrepo = xgitrepo[:-4] 

121 package = os.path.basename(xgitrepo) 

122 if not keyword: 122 ↛ 125line 122 didn't jump to line 125, because the condition on line 122 was never false

123 keyword = 'vcs' 

124 

125 xloop = msg.get_all('X-Loop', ()) 

126 xdebian = msg.get_all('X-Debian', ()) 

127 testing_watch = msg.get('X-Testing-Watch-Package') 

128 

129 bts_match = 'owner@bugs.debian.org' in xloop 

130 dak_match = 'DAK' in xdebian 

131 buildd_match = 'buildd.debian.org' in xdebian 

132 autoremovals_match = 'release.debian.org/autoremovals' in xdebian 

133 

134 if bts_match: # This is a mail of the Debian bug tracking system 

135 package, keyword = _classify_bts_message(msg, package, keyword) 

136 elif dak_match: 

137 package, keyword = _classify_dak_message(msg, package, keyword) 

138 elif buildd_match: 

139 keyword = 'build' 

140 package = msg.get('X-Debian-Package', package) 

141 elif autoremovals_match: 

142 keyword = 'summary' 

143 package = msg.get('X-Debian-Package', package) 

144 elif testing_watch: 

145 package = testing_watch 

146 keyword = 'summary' 

147 mail_news.create_news(msg, package) 

148 

149 # Converts old PTS keywords into new ones 

150 legacy_mapping = { 

151 'katie-other': 'archive', 

152 'buildd': 'build', 

153 'ddtp': 'translation', 

154 'cvs': 'vcs', 

155 } 

156 if keyword in legacy_mapping: 

157 keyword = legacy_mapping[keyword] 

158 return (package, keyword) 

159 

160 

161def add_new_headers(received_message, package_name, keyword, team): 

162 """ 

163 Debian adds the following new headers: 

164 - X-Debian-Package 

165 - X-Debian 

166 

167 :param received_message: The original received package message 

168 :type received_message: :py:class:`email.message.Message` 

169 

170 :param package_name: The name of the package for which the message was 

171 intended 

172 :type package_name: string 

173 

174 :param keyword: The keyword with which the message is tagged. 

175 :type keyword: string 

176 """ 

177 new_headers = [ 

178 ('X-Debian', 'tracker.debian.org'), 

179 ] 

180 if package_name: 180 ↛ 184line 180 didn't jump to line 184, because the condition on line 180 was never false

181 new_headers.append(('X-Debian-Package', package_name)) 

182 new_headers.append( 

183 ('X-PTS-Package', package_name)) # for compat with old PTS 

184 if keyword: 184 ↛ 187line 184 didn't jump to line 187, because the condition on line 184 was never false

185 new_headers.append( 

186 ('X-PTS-Keyword', keyword)) # for compat with old PTS 

187 return new_headers 

188 

189 

190def approve_default_message(msg): 

191 """ 

192 Debian approves a default message only if it has a X-Bugzilla-Product 

193 header. 

194 

195 :param msg: The original received package message 

196 :type msg: :py:class:`email.message.Message` 

197 """ 

198 return 'X-Bugzilla-Product' in msg 

199 

200 

201def _get_message_body(msg): 

202 """ 

203 Returns the message body, joining together all parts into one string. 

204 

205 :param msg: The original received package message 

206 :type msg: :py:class:`email.message.Message` 

207 """ 

208 return '\n'.join(get_decoded_message_payload(part) 

209 for part in msg.walk() if not part.is_multipart()) 

210 

211 

212def get_pseudo_package_list(): 

213 """ 

214 Existing pseudo packages for Debian are obtained from 

215 `BTS <https://bugs.debian.org/pseudo-packages.maintainers>`_ 

216 """ 

217 PSEUDO_PACKAGE_LIST_URL = ( 

218 'https://bugs.debian.org/pseudo-packages.maintainers' 

219 ) 

220 cache = HttpCache(settings.DISTRO_TRACKER_CACHE_DIRECTORY) 

221 if not cache.is_expired(PSEUDO_PACKAGE_LIST_URL): 221 ↛ 222line 221 didn't jump to line 222, because the condition on line 221 was never true

222 return 

223 response, updated = cache.update(PSEUDO_PACKAGE_LIST_URL) 

224 

225 try: 

226 response.raise_for_status() 

227 except requests.exceptions.HTTPError: 

228 raise PluginProcessingError() 

229 

230 if not updated: 230 ↛ 231line 230 didn't jump to line 231, because the condition on line 230 was never true

231 return 

232 

233 return [ 

234 line.split(None, 1)[0] 

235 for line in response.text.splitlines() 

236 ] 

237 

238 

239def get_package_information_site_url(package_name, source_package=False, 

240 repository=None, version=None): 

241 """ 

242 Return a link pointing to more information about a package in a 

243 given repository. 

244 """ 

245 BASE_URL = 'https://packages.debian.org/' 

246 PU_URL = 'https://release.debian.org/proposed-updates/' 

247 SOURCE_PACKAGE_URL_TEMPLATES = { 

248 'repository': BASE_URL + 'source/{repo}/{package}', 

249 'no-repository': BASE_URL + 'src:{package}', 

250 'pu': PU_URL + '{targetsuite}.html#{package}_{version}', 

251 } 

252 BINARY_PACKAGE_URL_TEMPLATES = { 

253 'repository': BASE_URL + '{repo}/{package}', 

254 'no-repository': BASE_URL + '{package}', 

255 'pu': '', 

256 } 

257 

258 params = {'package': package_name} 

259 if repository: 

260 suite = repository['suite'] or repository['codename'] 

261 if suite.endswith('proposed-updates'): 

262 url_type = 'pu' 

263 params['version'] = version 

264 params['targetsuite'] = suite.replace('-proposed-updates', '')\ 

265 .replace('proposed-updates', 'stable') 

266 else: 

267 url_type = 'repository' 

268 params['repo'] = suite 

269 else: 

270 url_type = 'no-repository' 

271 

272 if source_package: 

273 template = SOURCE_PACKAGE_URL_TEMPLATES[url_type] 

274 else: 

275 template = BINARY_PACKAGE_URL_TEMPLATES[url_type] 

276 

277 return template.format(**params) 

278 

279 

280def get_developer_information_url(developer_email): 

281 """ 

282 Return a URL to extra information about a developer, by email address. 

283 """ 

284 URL_TEMPLATE = 'https://qa.debian.org/developer.php?email={email}' 

285 return URL_TEMPLATE.format(email=urlquote_plus(developer_email)) 

286 

287 

288def get_external_version_information_urls(package_name): 

289 """ 

290 The function returns a list of external Web resources which provide 

291 additional information about the versions of a package. 

292 """ 

293 return [ 

294 { 

295 'url': 'https://qa.debian.org/madison.php?package={package}'.format( 

296 package=urlquote_plus(package_name)), 

297 'description': 'more versions can be listed by madison', 

298 }, 

299 { 

300 'url': 'https://snapshot.debian.org/package/{package}/'.format( 

301 package=package_name), 

302 'description': 'old versions available from snapshot.debian.org', 

303 } 

304 ] 

305 

306 

307def get_maintainer_extra(developer_email, package_name=None): 

308 """ 

309 The function returns a list of additional items that are to be 

310 included in the general panel next to the maintainer. This includes: 

311 

312 - Whether the maintainer agrees with lowthreshold NMU 

313 - Whether the maintainer is a Debian Maintainer 

314 """ 

315 developer = get_or_none(DebianContributor, 

316 email__email__iexact=developer_email) 

317 extra = [] 

318 _add_dmd_entry(extra, developer_email) 

319 if developer and developer.agree_with_low_threshold_nmu: 319 ↛ 325line 319 didn't jump to line 325, because the condition on line 319 was never false

320 extra.append({ 

321 'display': 'LowNMU', 

322 'description': 'maintainer agrees with Low Threshold NMU', 

323 'link': 'https://wiki.debian.org/LowThresholdNmu', 

324 }) 

325 _add_dm_entry(extra, developer, package_name) 

326 return extra 

327 

328 

329def get_uploader_extra(developer_email, package_name=None): 

330 """ 

331 The function returns a list of additional items that are to be 

332 included in the general panel next to an uploader. This includes: 

333 

334 - Whether the uploader is a DebianMaintainer 

335 """ 

336 developer = get_or_none(DebianContributor, 

337 email__email__iexact=developer_email) 

338 

339 extra = [] 

340 _add_dmd_entry(extra, developer_email) 

341 _add_dm_entry(extra, developer, package_name) 

342 return extra 

343 

344 

345def _add_dmd_entry(extra, email): 

346 extra.append({ 

347 'display': 'DMD', 

348 'description': 'UDD\'s Debian Maintainer Dashboard', 

349 'link': 'https://udd.debian.org/dmd/?{email}#todo'.format( 

350 email=urlquote_plus(email) 

351 ) 

352 }) 

353 

354 

355def _add_dm_entry(extra, developer, package_name): 

356 if package_name and developer and developer.is_debian_maintainer: 

357 if package_name in developer.allowed_packages: 357 ↛ exitline 357 didn't return from function '_add_dm_entry', because the condition on line 357 was never false

358 extra.append( 

359 { 

360 'display': 'DM', 

361 'description': 'Debian Maintainer upload allowed', 

362 'link': 'https://ftp-master.debian.org/dm.txt' 

363 } 

364 ) 

365 

366 

367def allow_package(stanza): 

368 """ 

369 The function provides a way for vendors to exclude some packages from being 

370 saved in the database. 

371 

372 In Debian's case, this is done for packages where the ``Extra-Source-Only`` 

373 is set since those packages are in the repository only for various 

374 compliance reasons. 

375 

376 :param stanza: The raw package entry from a ``Sources`` file. 

377 :type stanza: case-insensitive dict 

378 """ 

379 return 'Extra-Source-Only' not in stanza 

380 

381 

382def create_dak_rm_news(message, package, body=None, version=''): 

383 """Create a :class:`News` out of a removal email sent by DAK.""" 

384 if not body: 384 ↛ 385line 384 didn't jump to line 385, because the condition on line 384 was never true

385 body = get_decoded_message_payload(message) 

386 suite = re.search(r"have been removed from (\S+):", body).group(1) 

387 title = "Removed {ver} from {suite}".format(ver=version, suite=suite) 

388 return mail_news.create_news(message, package, title=title) 

389 

390 

391def get_extra_versions(package): 

392 """ 

393 :returns: The versions of the package found in the NEW queue. 

394 """ 

395 try: 

396 info = package.data.get(key=UpdateNewQueuePackages.DATA_KEY) 

397 except PackageData.DoesNotExist: 

398 return 

399 

400 version_url_template = 'https://ftp-master.debian.org/new/{pkg}_{ver}.html' 

401 return [ 

402 { 

403 'version': ver['version'], 

404 'repository_shorthand': 'NEW/' + dist, 

405 'version_link': version_url_template.format( 

406 pkg=package.name, ver=ver['version']), 

407 'repository_link': 'https://ftp-master.debian.org/new.html', 

408 } 

409 for dist, ver in info.value.items() 

410 ] 

411 

412 

413def pre_login(form): 

414 """ 

415 If the user has a @debian.org email associated, don't let them log 

416 in directly through local authentication. 

417 """ 

418 username = form.cleaned_data.get('username') 

419 if not username: 

420 return 

421 user_email = get_or_none(UserEmail, email__iexact=username) 

422 emails = [username] 

423 if user_email and user_email.user: 

424 emails += [x.email for x in user_email.user.emails.all()] 

425 if any(email.endswith('@debian.org') for email in emails): 

426 raise forms.ValidationError(mark_safe( 

427 "Your account has a @debian.org email address associated. " 

428 "To log in to the package tracker, you must use a SSL client " 

429 "certificate generated on " 

430 "<a href='https://sso.debian.org/'>" 

431 "sso.debian.org</a> (click on the link!).")) 

432 

433 

434def post_logout(request, user, next_url=None): 

435 """ 

436 If the user is authenticated via the SSO, sign them out at the SSO 

437 level too. 

438 """ 

439 if request.META.get('REMOTE_USER'): 

440 if next_url is None: 

441 next_url = 'https://' + settings.DISTRO_TRACKER_FQDN 

442 elif next_url.startswith('/'): 

443 next_url = 'https://' + settings.DISTRO_TRACKER_FQDN + next_url 

444 return ( 

445 'https://sso.debian.org/cgi-bin/dacs/dacs_signout?' + urlencode({ 

446 'SIGNOUT_HANDLER': next_url 

447 }) 

448 ) 

449 

450 

451def get_table_fields(table): 

452 """ 

453 The function provides additional fields which should be displayed in 

454 the team's packages table 

455 """ 

456 return table.default_fields + [DebciTableField, UpstreamTableField] 

457 

458 

459def additional_prefetch_related_lookups(): 

460 """ 

461 :returns: The list with additional lookups to be prefetched along with 

462 default lookups defined by :class:`BaseTableField` 

463 """ 

464 return [ 

465 Prefetch( 

466 'action_items', 

467 queryset=ActionItem.objects.filter( 

468 item_type__type_name='vcswatch-warnings-and-errors' 

469 ).prefetch_related('item_type'), 

470 ), 

471 Prefetch( 

472 'data', 

473 queryset=PackageData.objects.filter(key='vcswatch'), 

474 to_attr='vcswatch_data' 

475 ), 

476 ] 

477 

478 

479def get_vcs_data(package): 

480 """ 

481 :returns: The dictionary with VCS Watch data to be displayed in 

482 the template defined by :data:`DISTRO_TRACKER_VCS_TABLE_FIELD_TEMPLATE 

483 <distro_tracker.project.local_settings.DISTRO_TRACKER_VCS_TABLE_FIELD_TEMPLATE>` 

484 settings. 

485 """ 

486 data = {} 

487 try: 

488 item = package.vcswatch_data[0] 

489 data['changelog_version'] = item.value['changelog_version'] 

490 except IndexError: 

491 # There is no vcs extra data for the package 

492 pass 

493 

494 try: 

495 item = package.action_items.all()[0] 

496 data['action_item'] = item.to_dict() 

497 data['action_item']['url'] = item.get_absolute_url() 

498 except IndexError: 

499 # There is no action item for the package 

500 pass 

501 return data 

502 

503 

504def get_bug_display_manager_class(): 

505 """Return the class that knows how to display data about Debian bugs.""" 

506 return DebianBugDisplayManager 

507 

508 

509def get_tables_for_team_page(team, limit): 

510 """ 

511 The function must return a list of :class:`BasePackageTable` objects 

512 to be displayed in the main page of teams. 

513 

514 :param team: The team for which the tables must be added. 

515 :type package: :class:`Team <distro_tracker.core.models.Team>` 

516 :param int limit: The number of packages to be displayed in the tables. 

517 """ 

518 return [ 

519 create_table(slug='general', scope=team, limit=limit), 

520 create_table( 

521 slug='general', scope=team, limit=limit, tag='tag:rc-bugs'), 

522 create_table( 

523 slug='general', scope=team, limit=limit, 

524 tag='tag:new-upstream-version'), 

525 create_table( 

526 slug='general', scope=team, limit=limit, tag='tag:bugs'), 

527 create_table( 

528 slug='general', scope=team, limit=limit, 

529 tag='tag:debci-failures') 

530 ]