Coverage for distro_tracker/vendor/debian/rules.py: 84%

2# See the COPYRIGHT file at the top-level directory of this distribution and

3# at https://deb.li/DTAuthors

5# This file is part of Distro Tracker. It is subject to the license terms

6# in the LICENSE file found in the top-level directory of this

7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker,

8# including this file, may be copied, modified, propagated, or distributed

9# except according to the terms contained in the LICENSE file.

10"""Debian specific rules for various Distro-Tracker hooks."""

12import os.path

13import re

14from urllib.parse import quote_plus

16from django import forms

17from django.conf import settings

18from django.db.models import Prefetch

19from django.utils.http import urlencode

20from django.utils.safestring import mark_safe

22import requests

24from distro_tracker.core.models import (

25 ActionItem,

26 PackageData,

27 UserEmail

28)

29from distro_tracker.core.package_tables import create_table

30from distro_tracker.core.utils import get_decoded_message_payload, get_or_none

31from distro_tracker.core.utils.email_messages import get_message_body

32from distro_tracker.core.utils.http import HttpCache

33from distro_tracker.debci_status.tracker_package_tables import DebciTableField

34from distro_tracker.mail import mail_news

35from distro_tracker.vendor.common import PluginProcessingError

36from distro_tracker.vendor.debian.tracker_tasks import UpdateNewQueuePackages

39from .models import DebianBugDisplayManager, DebianContributor

40from .tracker_package_tables import UpstreamTableField

43def _simplify_pkglist(pkglist, multi_allowed=True, default=None):

44 """Replace a single-list item by its sole item. A longer list is left

45 as-is (provided multi_allowed is True). An empty list returns the default

46 value."""

47 if len(pkglist) == 1 and pkglist[0]:

48 return pkglist[0]

49 elif len(pkglist) > 1 and multi_allowed:

50 return pkglist

51 return default

54def _classify_bts_message(msg, package, keyword):

55 bts_package = msg.get('X-Debian-PR-Source',

56 msg.get('X-Debian-PR-Package', ''))

57 pkglist = re.split(r'\s+', bts_package.strip())

58 # Don't override default package assignation when we find multiple package

59 # associated to the mail, otherwise we will send multiple copies of a mail

60 # that we already receive multiple times

61 multi_allowed = package is None

62 pkg_result = _simplify_pkglist(pkglist, multi_allowed=multi_allowed,

63 default=package)

65 # We override the package/keyword only...

66 if package is None: # When needed, because we don't have a suggestion

67 override_suggestion = True

68 else: # Or when package suggestion matches the one found in the header

69 override_suggestion = package == pkg_result

71 if override_suggestion:

72 package = pkg_result

74 if override_suggestion or keyword is None:

75 debian_pr_message = msg.get('X-Debian-PR-Message', '')

76 if debian_pr_message.startswith('transcript'):

77 keyword = 'bts-control'

78 else:

79 keyword = 'bts'

81 return (package, keyword)

84def _classify_dak_message(msg, package, keyword):

85 subject = msg.get('Subject', '').strip()

86 xdak = msg.get('X-DAK', '')

87 package = msg.get('Debian-Source', package)

88 action = msg.get('Debian-Archive-Action', '')

89 architecture = msg.get('Debian-Architecture', '')

90 body = get_message_body(msg)

92 if action == "accept":

93 if "source" in architecture:

94 keyword = 'upload-source'

95 if re.search(r'^Accepted', subject):

96 mail_news.create_news(msg, package, create_package=True)

97 else:

98 keyword = 'upload-binary'

99 else:

100 keyword = 'archive'

101

102 if xdak == 'dak rm':

103 # Find all lines giving information about removed source packages

104 re_rmline = re.compile(r"^\s*(\S+)\s*\|\s*(\S+)\s*\|\s*(.*)", re.M)

105 source_removals = re_rmline.findall(body)

106 removed_pkgver = {}

107 for pkgname, version, arch in source_removals:

108 removed_pkgver[pkgname] = (version, arch)

109 if package not in removed_pkgver:

110 package = _simplify_pkglist(list(removed_pkgver.keys()),

111 multi_allowed=False,

112 default=package)

113 if package in removed_pkgver and "source" in removed_pkgver[package][1]:

114 create_dak_rm_news(msg, package, version=removed_pkgver[package][0],

115 body=body)

116

117 return (package, keyword)

118

119

120def classify_message(msg, package, keyword):

121 """Classify incoming email messages with a package and a keyword."""

122 # Default values for git commit notifications

123 xgitrepo = msg.get('X-GitLab-Project-Path', msg.get('X-Git-Repo'))

124 if xgitrepo:

125 if not package: 125 ↛ 129line 125 didn't jump to line 129, because the condition on line 125 was never false

126 if xgitrepo.endswith('.git'):

127 xgitrepo = xgitrepo[:-4]

128 package = os.path.basename(xgitrepo)

129 if not keyword: 129 ↛ 132line 129 didn't jump to line 132, because the condition on line 129 was never false

130 keyword = 'vcs'

131

132 xloop = msg.get_all('X-Loop', ())

133 xdebian = msg.get_all('X-Debian', ())

134 testing_watch = msg.get('X-Testing-Watch-Package')

135

136 bts_match = 'owner@bugs.debian.org' in xloop

137 dak_match = 'DAK' in xdebian

138 buildd_match = 'buildd.debian.org' in xdebian

139 autoremovals_match = 'release.debian.org/autoremovals' in xdebian

140

141 if bts_match: # This is a mail of the Debian bug tracking system

142 package, keyword = _classify_bts_message(msg, package, keyword)

143 elif dak_match:

144 package, keyword = _classify_dak_message(msg, package, keyword)

145 elif buildd_match:

146 keyword = 'build'

147 package = msg.get('X-Debian-Package', package)

148 elif autoremovals_match:

149 keyword = 'summary'

150 package = msg.get('X-Debian-Package', package)

151 elif testing_watch:

152 package = testing_watch

153 keyword = 'summary'

154 mail_news.create_news(msg, package)

155

156 # Converts old PTS keywords into new ones

157 legacy_mapping = {

158 'katie-other': 'archive',

159 'buildd': 'build',

160 'ddtp': 'translation',

161 'cvs': 'vcs',

162 }

163 if keyword in legacy_mapping:

164 keyword = legacy_mapping[keyword]

165 return (package, keyword)

166

167

168def add_new_headers(received_message, package_name, keyword, team):

169 """

170 Debian adds the following new headers:

171 - X-Debian-Package

172 - X-Debian

173

174 :param received_message: The original received package message

175 :type received_message: :py:class:`email.message.Message`

176

177 :param package_name: The name of the package for which the message was

178 intended

179 :type package_name: string

180

181 :param keyword: The keyword with which the message is tagged.

182 :type keyword: string

183 """

184 new_headers = [

185 ('X-Debian', 'tracker.debian.org'),

186 ]

187 if package_name: 187 ↛ 191line 187 didn't jump to line 191, because the condition on line 187 was never false

188 new_headers.append(('X-Debian-Package', package_name))

189 new_headers.append(

190 ('X-PTS-Package', package_name)) # for compat with old PTS

191 if keyword: 191 ↛ 194line 191 didn't jump to line 194, because the condition on line 191 was never false

192 new_headers.append(

193 ('X-PTS-Keyword', keyword)) # for compat with old PTS

194 return new_headers

195

196

197def approve_default_message(msg):

198 """

199 Debian approves a default message only if it has a X-Bugzilla-Product

200 header.

201

202 :param msg: The original received package message

203 :type msg: :py:class:`email.message.Message`

204 """

205 return 'X-Bugzilla-Product' in msg

206

207

208def get_pseudo_package_list():

209 """

210 Existing pseudo packages for Debian are obtained from

211 `BTS <https://bugs.debian.org/pseudo-packages.maintainers>`_

212 """

213 PSEUDO_PACKAGE_LIST_URL = (

214 'https://bugs.debian.org/pseudo-packages.maintainers'

215 )

216 cache = HttpCache(settings.DISTRO_TRACKER_CACHE_DIRECTORY)

217 if not cache.is_expired(PSEUDO_PACKAGE_LIST_URL): 217 ↛ 218line 217 didn't jump to line 218, because the condition on line 217 was never true

218 return

219 response, updated = cache.update(PSEUDO_PACKAGE_LIST_URL)

220

221 try:

222 response.raise_for_status()

223 except requests.exceptions.HTTPError:

224 raise PluginProcessingError()

225

226 if not updated: 226 ↛ 227line 226 didn't jump to line 227, because the condition on line 226 was never true

227 return

228

229 return [

230 line.split(None, 1)[0]

231 for line in response.text.splitlines()

232 ]

233

234

235def get_package_information_site_url(package_name, source_package=False,

236 repository=None, version=None):

237 """

238 Return a link pointing to more information about a package in a

239 given repository.

240 """

241 BASE_URL = 'https://packages.debian.org/'

242 PU_URL = 'https://release.debian.org/proposed-updates/'

243 SOURCE_PACKAGE_URL_TEMPLATES = {

244 'repository': BASE_URL + 'source/{repo}/{package}',

245 'no-repository': BASE_URL + 'src:{package}',

246 'pu': PU_URL + '{targetsuite}.html#{package}_{version}',

247 }

248 BINARY_PACKAGE_URL_TEMPLATES = {

249 'repository': BASE_URL + '{repo}/{package}',

250 'no-repository': BASE_URL + '{package}',

251 'pu': '',

252 }

253

254 params = {'package': package_name}

255 if repository:

256 suite = repository['suite'] or repository['codename']

257 if suite.endswith('proposed-updates'):

258 url_type = 'pu'

259 params['version'] = version

260 params['targetsuite'] = suite.replace('-proposed-updates', '')\

261 .replace('proposed-updates', 'stable')

262 else:

263 url_type = 'repository'

264 params['repo'] = suite

265 else:

266 url_type = 'no-repository'

267

268 if source_package:

269 template = SOURCE_PACKAGE_URL_TEMPLATES[url_type]

270 else:

271 template = BINARY_PACKAGE_URL_TEMPLATES[url_type]

272

273 return template.format(**params)

274

275

276def get_developer_information_url(developer_email):

277 """

278 Return a URL to extra information about a developer, by email address.

279 """

280 URL_TEMPLATE = 'https://qa.debian.org/developer.php?email={email}'

281 return URL_TEMPLATE.format(email=quote_plus(developer_email))

282

283

284def get_external_version_information_urls(package_name):

285 """

286 The function returns a list of external Web resources which provide

287 additional information about the versions of a package.

288 """

289 return [

290 {

291 'url': 'https://qa.debian.org/madison.php?package={package}'.format(

292 package=quote_plus(package_name)),

293 'description': 'more versions can be listed by madison',

294 },

295 {

296 'url': 'https://snapshot.debian.org/package/{package}/'.format(

297 package=package_name),

298 'description': 'old versions available from snapshot.debian.org',

299 }

300 ]

301

302

303def get_maintainer_extra(developer_email, package_name=None):

304 """

305 The function returns a list of additional items that are to be

306 included in the general panel next to the maintainer. This includes:

307

308 - Whether the maintainer agrees with lowthreshold NMU

309 - Whether the maintainer is a Debian Maintainer

310 """

311 developer = get_or_none(DebianContributor,

312 email__email__iexact=developer_email)

313 extra = []

314 _add_dmd_entry(extra, developer_email)

315 if developer and developer.agree_with_low_threshold_nmu: 315 ↛ 321line 315 didn't jump to line 321, because the condition on line 315 was never false

316 extra.append({

317 'display': 'LowNMU',

318 'description': 'maintainer agrees with Low Threshold NMU',

319 'link': 'https://wiki.debian.org/LowThresholdNmu',

320 })

321 _add_dm_entry(extra, developer, package_name)

322 return extra

323

324

325def get_uploader_extra(developer_email, package_name=None):

326 """

327 The function returns a list of additional items that are to be

328 included in the general panel next to an uploader. This includes:

329

330 - Whether the uploader is a DebianMaintainer

331 """

332 developer = get_or_none(DebianContributor,

333 email__email__iexact=developer_email)

334

335 extra = []

336 _add_dmd_entry(extra, developer_email)

337 _add_dm_entry(extra, developer, package_name)

338 return extra

339

340

341def _add_dmd_entry(extra, email):

342 extra.append({

343 'display': 'DMD',

344 'description': 'UDD\'s Debian Maintainer Dashboard',

345 'link': 'https://udd.debian.org/dmd/?{email}#todo'.format(

346 email=quote_plus(email)

347 )

348 })

349

350

351def _add_dm_entry(extra, developer, package_name):

352 if package_name and developer and developer.is_debian_maintainer:

353 if package_name in developer.allowed_packages: 353 ↛ exitline 353 didn't return from function '_add_dm_entry', because the condition on line 353 was never false

354 extra.append(

355 {

356 'display': 'DM',

357 'description': 'Debian Maintainer upload allowed',

358 'link': 'https://ftp-master.debian.org/dm.txt'

359 }

360 )

361

362

363def allow_package(stanza):

364 """

365 The function provides a way for vendors to exclude some packages from being