Coverage for distro_tracker/vendor/debian/tracker

2# See the COPYRIGHT file at the top-level directory of this distribution and

3# at https://deb.li/DTAuthors

5# This file is part of Distro Tracker. It is subject to the license terms

6# in the LICENSE file found in the top-level directory of this

7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker,

8# including this file, may be copied, modified, propagated, or distributed

9# except according to the terms contained in the LICENSE file.

11"""

12Debian-specific tasks.

13"""

15import collections

16import itertools

17import json

18import logging

19import os

20import re

21import urllib.parse

22import warnings

23from enum import Enum

25from bs4 import BeautifulSoup as soup, MarkupResemblesLocatorWarning

27from debian import deb822, debian_support

28from debian.debian_support import AptPkgVersion

30import debianbts

32from django.conf import settings

33from django.core.exceptions import ValidationError

34from django.db import transaction

35from django.db.models import Prefetch

36from django.utils.http import urlencode

38import yaml

40from distro_tracker.accounts.models import UserEmail

41from distro_tracker.core.models import (

42 ActionItem,

43 ActionItemType,

44 BinaryPackageBugStats,

45 BinaryPackageName,

46 BugDisplayManagerMixin,

47 PackageBugStats,

48 PackageData,

49 PackageName,

50 Repository,

51 SourcePackageDeps,

52 SourcePackageName

53)

54from distro_tracker.core.tasks import BaseTask

55from distro_tracker.core.tasks.mixins import ImportExternalData, PackageTagging

56from distro_tracker.core.tasks.schedulers import IntervalScheduler

57from distro_tracker.core.utils import get_or_none

58from distro_tracker.core.utils.http import get_resource_text

59from distro_tracker.core.utils.misc import get_data_checksum

60from distro_tracker.core.utils.packages import (

61 html_package_list,

62 package_url

63)

64from distro_tracker.vendor.debian.models import (

65 LintianStats,

66 PackageExcuses,

67 PackageTransition,

68 UbuntuPackage

69)

71from .models import DebianContributor

73logger = logging.getLogger(__name__)

74logger_input = logging.getLogger('distro_tracker.input')

76warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)

79class RetrieveDebianMaintainersTask(BaseTask):

80 """

81 Retrieves (and updates if necessary) a list of Debian Maintainers.

82 """

84 class Scheduler(IntervalScheduler):

85 interval = 3600 * 24

87 def execute_main(self):

88 url = "https://ftp-master.debian.org/dm.txt"

89 content = get_resource_text(url, force_update=self.force_update,

90 only_if_updated=True)

91 if content is None: 91 ↛ 93line 91 didn't jump to line 93, because the condition on line 91 was never true

92 # No need to do anything if the cached item was still not updated

93 return

95 maintainers = {}

96 lines = content.splitlines()

97 for stanza in deb822.Deb822.iter_paragraphs(lines):

98 if 'Uid' in stanza and 'Allow' in stanza: 98 ↛ 97line 98 didn't jump to line 97, because the condition on line 98 was never false

99 # Allow is a comma-separated string of 'package (DD fpr)' items,

100 # where DD fpr is the fingerprint of the DD that granted the

101 # permission

102 name, email = stanza['Uid'].rsplit(' ', 1)

103 email = email.strip('<>')

104 for pair in stanza['Allow'].split(','):

105 pair = pair.strip()

106 pkg, dd_fpr = pair.split()

107 maintainers.setdefault(email, [])

108 maintainers[email].append(pkg)

109

110 # Now update the developer information

111 with transaction.atomic():

112 # Reset all old maintainers first.

113 qs = DebianContributor.objects.filter(is_debian_maintainer=True)

114 qs.update(is_debian_maintainer=False)

115

116 for email, packages in maintainers.items():

117 try:

118 user_email, _ = UserEmail.objects.get_or_create(email=email)

119 except ValidationError:

120 logger_input.info('%s refers to invalid email "%s".',

121 url, email)

122 continue

123

124 contributor, _ = DebianContributor.objects.get_or_create(

125 email=user_email)

126

127 contributor.is_debian_maintainer = True

128 contributor.allowed_packages = packages

129 contributor.save()

130

131

132class RetrieveLowThresholdNmuTask(BaseTask):

133 """

134 Updates the list of Debian Maintainers which agree with the lowthreshold

135 NMU.

136 """

137

138 class Scheduler(IntervalScheduler):

139 interval = 3600 * 24

140

141 def _retrieve_emails(self):

142 """

143 Helper function which obtains the list of emails of maintainers that

144 agree with the lowthreshold NMU.

145 """

146 url = 'https://wiki.debian.org/LowThresholdNmu?action=raw'

147 content = get_resource_text(url, force_update=self.force_update,

148 only_if_updated=True)

149 if content is None: 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true

150 return

151

152 emails = []

153 devel_php_RE = re.compile(

154 r'https?://qa\.debian\.org/developer\.php\?login=([^\s&|]+)')

155 word_RE = re.compile(r'^\w+$')

156 for line in content.splitlines():

157 match = devel_php_RE.search(line)

158 while match: # look for several matches on the same line

159 email = None

160 login = match.group(1)

161 if word_RE.match(login):

162 email = login + '@debian.org'

163 elif login.find('@') >= 0: 163 ↛ 165line 163 didn't jump to line 165, because the condition on line 163 was never false

164 email = login

165 if email: 165 ↛ 167line 165 didn't jump to line 167, because the condition on line 165 was never false

166 emails.append(email)

167 line = line[match.end():]

168 match = devel_php_RE.search(line)

169 return emails

170

171 def execute_main(self):

172 emails = self._retrieve_emails()

173 with transaction.atomic():

174 # Reset all threshold flags first.

175 qs = DebianContributor.objects.filter(

176 agree_with_low_threshold_nmu=True)

177 qs.update(agree_with_low_threshold_nmu=False)

178

179 for email in emails:

180 try:

181 email, _ = UserEmail.objects.get_or_create(email=email)

182 except ValidationError:

183 logger_input.info(

184 'LowThresholdNmu refers to invalid email "%s".', email)

185 continue

186

187 contributor, _ = DebianContributor.objects.get_or_create(

188 email=email)

189

190 contributor.agree_with_low_threshold_nmu = True

191 contributor.save()

192

193

194class UpdatePackageBugStats(BaseTask, BugDisplayManagerMixin):

195 """

196 Updates the BTS bug stats for all packages (source, binary and pseudo).

197 Creates :class:`distro_tracker.core.ActionItem` instances for packages

198 which have bugs tagged help or patch.

199 """

200

201 class Scheduler(IntervalScheduler):

202 interval = 3600

203

204 PATCH_BUG_ACTION_ITEM_TYPE_NAME = 'debian-patch-bugs-warning'

205 HELP_BUG_ACTION_ITEM_TYPE_NAME = 'debian-help-bugs-warning'

206

207 PATCH_ITEM_SHORT_DESCRIPTION = (

208 '<a href="{url}">{count}</a> with a patch to review in the '

209 'bug tracker')

210 HELP_ITEM_SHORT_DESCRIPTION = (

211 '<a href="{url}">{count}</a> where the maintainer asks for help')

212 PATCH_ITEM_FULL_DESCRIPTION_TEMPLATE = 'debian/patch-bugs-action-item.html'

213 HELP_ITEM_FULL_DESCRIPTION_TEMPLATE = 'debian/help-bugs-action-item.html'

214

215 bug_categories = (

216 'rc',

217 'normal',

218 'wishlist',

219 'fixed',

220 'patch',

221 )

222

223 def initialize(self, *args, **kwargs):

224 super(UpdatePackageBugStats, self).initialize(*args, **kwargs)

225 # The :class:`distro_tracker.core.models.ActionItemType` instances which

226 # this task can create.

227 self.patch_item_type = ActionItemType.objects.create_or_update(

228 type_name=self.PATCH_BUG_ACTION_ITEM_TYPE_NAME,

229 full_description_template=self.PATCH_ITEM_FULL_DESCRIPTION_TEMPLATE)

230 self.help_item_type = ActionItemType.objects.create_or_update(

231 type_name=self.HELP_BUG_ACTION_ITEM_TYPE_NAME,

232 full_description_template=self.HELP_ITEM_FULL_DESCRIPTION_TEMPLATE)

233

234 def _get_tagged_bug_stats(self, tag, user=None):

235 """

236 Using the BTS interface, retrieves the statistics of bugs with a

237 particular tag.

238

239 :param tag: The tag for which the statistics are required.

240 :type tag: string

241 :param user: The email of the user who tagged the bug with the given

242 tag.

243 :type user: string

244

245 :returns: A dict mapping package names to the count of bugs with the

246 given tag.

247 """

248 debian_ca_bundle = '/etc/ssl/ca-debian/ca-certificates.crt'

249 if os.path.exists(debian_ca_bundle):

250 os.environ['SSL_CERT_FILE'] = debian_ca_bundle

251 if user:

252 bug_numbers = debianbts.get_usertag(user, tags=[tag]).get(tag, [])

253 else:

254 bug_numbers = debianbts.get_bugs(tag=tag)

255

256 # Match each retrieved bug ID to a package and then find the aggregate

257 # count for each package.

258 bug_stats = {}

259 bugs = debianbts.get_status(bug_numbers)

260 for bug in bugs:

261 if bug.done or bug.fixed_versions or bug.pending == 'done':

262 continue

263

264 bug_stats.setdefault(bug.package, 0)

265 bug_stats[bug.package] += 1

266

267 return bug_stats

268

269 def _extend_bug_stats(self, bug_stats, extra_stats, category_name):

270 """

271 Helper method which adds extra bug stats to an already existing list of

272 stats.

273

274 :param bug_stats: An already existing list of bug stats. Maps package

275 names to list of bug category descriptions.

276 :type bug_stats: dict

277 :param extra_stats: Extra bug stats which should be added to

278 ``bug_stats``. Maps package names to integers representing bug

279 counts.

280 :type extra_stats: dict

281 :param category_name: The name of the bug category which is being added

282 :type category_name: string

283 """

284 for package, count in extra_stats.items():

285 bug_stats.setdefault(package, [])

286 bug_stats[package].append({

287 'category_name': category_name,

288 'bug_count': count,

289 })

290

291 def _create_patch_bug_action_item(self, package, bug_stats):

292 """

293 Creates a :class:`distro_tracker.core.models.ActionItem` instance for

294 the given package if it contains any bugs tagged patch.

295

296 :param package: The package for which the action item should be

297 updated.

298 :type package: :class:`distro_tracker.core.models.PackageName`

299 :param bug_stats: A dictionary mapping category names to structures

300 describing those categories. Those structures should be

301 identical to the ones stored in the :class:`PackageBugStats`

302 instance.

303 :type bug_stats: dict

304 """

305 # Get the old action item, if any

306 action_item = package.get_action_item_for_type(

307 self.PATCH_BUG_ACTION_ITEM_TYPE_NAME)

308

309 if 'patch' not in bug_stats or bug_stats['patch']['bug_count'] == 0:

310 # Remove the old action item, since the package does not have any

311 # bugs tagged patch anymore.

312 if action_item is not None:

313 action_item.delete()

314 return

315

316 # If the package has bugs tagged patch, update the action item

317 if action_item is None:

318 action_item = ActionItem(

319 package=package,

320 item_type=self.patch_item_type)

321

322 bug_count = bug_stats['patch']['bug_count']

323 # Include the URL in the short description

324 url = self.bug_manager.get_bug_tracker_url(

325 package.name, 'source', 'patch')

326 if not url: 326 ↛ 327line 326 didn't jump to line 327, because the condition on line 326 was never true

327 url = ''

328 # Include the bug count in the short description

329 count = '{bug_count} bug'.format(bug_count=bug_count)

330 if bug_count > 1:

331 count += 's'

332 action_item.short_description = \

333 self.PATCH_ITEM_SHORT_DESCRIPTION.format(url=url, count=count)

334 # Set additional URLs and merged bug count in the extra data for a full

335 # description

336 action_item.extra_data = {

337 'bug_count': bug_count,

338 'merged_count': bug_stats['patch'].get('merged_count', 0),

339 'url': url,

340 'merged_url': self.bug_manager.get_bug_tracker_url(

341 package.name, 'source', 'patch-merged'),

342 }

343 action_item.save()

344

345 def _create_help_bug_action_item(self, package, bug_stats):

346 """

347 Creates a :class:`distro_tracker.core.models.ActionItem` instance for

348 the given package if it contains any bugs tagged help.

349

350 :param package: The package for which the action item should be

351 updated.

352 :type package: :class:`distro_tracker.core.models.PackageName`

353 :param bug_stats: A dictionary mapping category names to structures

354 describing those categories. Those structures should be

355 identical to the ones stored in the :class:`PackageBugStats`

356 instance.

357 :type bug_stats: dict

358 """

359 # Get the old action item, if any

360 action_item = package.get_action_item_for_type(

361 self.HELP_BUG_ACTION_ITEM_TYPE_NAME)

362

363 if 'help' not in bug_stats or bug_stats['help']['bug_count'] == 0:

364 # Remove the old action item, since the package does not have any

365 # bugs tagged patch anymore.

366 if action_item is not None:

367 action_item.delete()

368 return

369

370 # If the package has bugs tagged patch, update the action item

371 if action_item is None:

372 action_item = ActionItem(

373 package=package,

374 item_type=self.help_item_type)

375

376 bug_count = bug_stats['help']['bug_count']

377 # Include the URL in the short description

378 url = self.bug_manager.get_bug_tracker_url(

379 package.name, 'source', 'help')

380 if not url: 380 ↛ 381line 380 didn't jump to line 381, because the condition on line 380 was never true

381 url = ''

382 # Include the bug count in the short description

383 count = '{bug_count} bug'.format(bug_count=bug_count)

384 if bug_count > 1:

385 count += 's'

386 action_item.short_description = self.HELP_ITEM_SHORT_DESCRIPTION.format(

387 url=url, count=count)

388 # Set additional URLs and merged bug count in the extra data for a full

389 # description

390 action_item.extra_data = {

391 'bug_count': bug_count,

392 'url': url,

393 }

394 action_item.save()

395

396 def _create_action_items(self, package_bug_stats):

397 """

398 Method which creates a :class:`distro_tracker.core.models.ActionItem`

399 instance for a package based on the given package stats.

400

401 For now, an action item is created if the package either has bugs

402 tagged as help or patch.

403 """

404 # Transform the bug stats to a structure easier to pass to functions

405 # for particular bug-category action items.

406 bug_stats = {

407 category['category_name']: category

408 for category in package_bug_stats.stats

409 }

410 package = package_bug_stats.package

411 self._create_patch_bug_action_item(package, bug_stats)

412 self._create_help_bug_action_item(package, bug_stats)

413

414 def _get_udd_bug_stats(self):

415 url = 'https://udd.debian.org/cgi-bin/ddpo-bugs.cgi'

416 response_content = get_resource_text(url)

417 if not response_content:

418 return

419

420 # Each line in the response should be bug stats for a single package

421 bug_stats = {}

422 for line in response_content.splitlines():

423 line = line.strip()

424 try:

425 package_name, bug_counts = line.rsplit(':', 1)

426 if package_name.startswith('src:'):

427 src, package_name = package_name.split(':', 1)

428 # Merged counts are in parentheses so remove those before

429 # splitting the numbers

430 bug_counts = re.sub(r'[()]', ' ', bug_counts).split()

431 bug_counts = [int(count) for count in bug_counts]

432 except ValueError:

433 logger.warning(

434 'Failed to parse bug information for %s: %s',

435 package_name, bug_counts, exc_info=1)

436 continue

437

438 # Match the extracted counts with category names

439 bug_stats[package_name] = [

440 {

441 'category_name': category_name,

442 'bug_count': bug_count,

443 'merged_count': merged_count,

444 }

445 for category_name, (bug_count, merged_count) in zip(

446 self.bug_categories, zip(bug_counts[::2], bug_counts[1::2]))

447 ]

448

449 return bug_stats

450

451 def _remove_obsolete_action_items(self, package_names):

452 """

453 Removes action items for packages which no longer have any bug stats.

454 """

455 ActionItem.objects.delete_obsolete_items(

456 item_types=[self.patch_item_type, self.help_item_type],

457 non_obsolete_packages=package_names)

458

459 def update_source_and_pseudo_bugs(self):

460 """

461 Performs the update of bug statistics for source and pseudo packages.

462 """

463 # First get the bug stats exposed by the UDD.

464 bug_stats = self._get_udd_bug_stats()

465 if not bug_stats:

466 bug_stats = {}

467

468 # Add in help bugs from the BTS interface

469 try:

470 help_bugs = self._get_tagged_bug_stats('help')

471 self._extend_bug_stats(bug_stats, help_bugs, 'help')

472 except RuntimeError:

473 logger.exception("Could not get bugs tagged help")

474

475 # Add in newcomer bugs from the BTS interface

476 try:

477 newcomer_bugs = self._get_tagged_bug_stats('newcomer')

478 self._extend_bug_stats(bug_stats, newcomer_bugs, 'newcomer')

479 except RuntimeError:

480 logger.exception("Could not get bugs tagged newcomer")

481

482 with transaction.atomic():

483 # Clear previous stats

484 PackageBugStats.objects.all().delete()

485 self._remove_obsolete_action_items(bug_stats.keys())

486 # Get all packages which have updated stats, along with their

487 # action items in 2 DB queries.

488 packages = PackageName.objects.filter(name__in=bug_stats.keys())

489 packages.prefetch_related('action_items')

490

491 # Update stats and action items.

492 stats = []

493 for package in packages:

494 # Save the raw package bug stats

495 package_bug_stats = PackageBugStats(

496 package=package, stats=bug_stats[package.name])

497 stats.append(package_bug_stats)

498

499 # Add action items for the package.

500 self._create_action_items(package_bug_stats)

501

502 PackageBugStats.objects.bulk_create(stats)

503

504 def update_binary_bugs(self):

505 """

506 Performs the update of bug statistics for binary packages.

507 """

508 url = 'https://udd.debian.org/cgi-bin/bugs-binpkgs-pts.cgi'

509 response_content = get_resource_text(url)

510 if not response_content:

511 return

512

513 # Extract known binary package bug stats: each line is a separate pkg

514 bug_stats = {}

515 for line in response_content.splitlines():

516 package_name, bug_counts = line.split(None, 1)

517 bug_counts = bug_counts.split()

518 try:

519 bug_counts = [int(count) for count in bug_counts]

520 except ValueError:

521 logger.exception(

522 'Failed to parse bug information for %s: %s',

523 package_name, bug_counts)

524 continue

525

526 bug_stats[package_name] = [

527 {

528 'category_name': category_name,

529 'bug_count': bug_count,

530 }

531 for category_name, bug_count in zip(

532 self.bug_categories, bug_counts)

533 ]

534

535 with transaction.atomic():

536 # Clear previous stats

537 BinaryPackageBugStats.objects.all().delete()

538 packages = \

539 BinaryPackageName.objects.filter(name__in=bug_stats.keys())

540 # Create new stats in a single query

541 stats = [

542 BinaryPackageBugStats(package=package,

543 stats=bug_stats[package.name])

544 for package in packages

545 ]

546 BinaryPackageBugStats.objects.bulk_create(stats)

547

548 def execute_main(self):

549 # Stats for source and pseudo packages is retrieved from a different

550 # resource (with a different structure) than stats for binary packages.

551 self.update_source_and_pseudo_bugs()

552 self.update_binary_bugs()

553

554

555class UpdateLintianStatsTask(BaseTask):

556 """

557 Updates packages' lintian stats.

558 """

559

560 class Scheduler(IntervalScheduler):

561 interval = 3600 * 4

562

563 ACTION_ITEM_TYPE_NAME = 'lintian-warnings-and-errors'

564 ITEM_DESCRIPTION = 'lintian reports <a href="{url}">{report}</a>'

565 ITEM_FULL_DESCRIPTION_TEMPLATE = 'debian/lintian-action-item.html'

566

567 def initialize(self, *args, **kwargs):

568 super(UpdateLintianStatsTask, self).initialize(*args, **kwargs)

569 self.lintian_action_item_type = ActionItemType.objects.create_or_update(

570 type_name=self.ACTION_ITEM_TYPE_NAME,

571 full_description_template=self.ITEM_FULL_DESCRIPTION_TEMPLATE)

572

573 def get_lintian_stats(self):

574 url = 'https://udd.debian.org/lintian-qa-list.txt'

575 content = get_resource_text(url, force_update=self.force_update,

576 only_if_updated=True)

577 if content is None: 577 ↛ 578line 577 didn't jump to line 578, because the condition on line 577 was never true

578 return

579

580 all_stats = {}

581 categories = (

582 'errors',

583 'warnings',

584 'pedantics',

585 'experimentals',

586 'overriddens',

587 )

588 for line in content.splitlines():

589 package, stats = line.split(None, 1)

590 stats = stats.split()

591 try:

592 all_stats[package] = {

593 category: int(count)

594 for count, category in zip(stats, categories)

595 }

596 except ValueError:

597 logger.exception(

598 'Failed to parse lintian information for %s: %s',

599 package, line)

600 continue

601

602 return all_stats

603

604 def update_action_item(self, package, lintian_stats):

605 """

606 Updates the :class:`ActionItem` for the given package based on the

607 :class:`LintianStats <distro_tracker.vendor.debian.models.LintianStats`

608 given in ``package_stats``. If the package has errors or warnings an

609 :class:`ActionItem` is created.

610 """

611 package_stats = lintian_stats.stats

612 warnings, errors = (

613 package_stats.get('warnings'), package_stats.get('errors', 0))

614 # Get the old action item for this warning, if it exists.

615 lintian_action_item = package.get_action_item_for_type(

616 self.lintian_action_item_type.type_name)

617 if not warnings and not errors:

618 if lintian_action_item:

619 # If the item previously existed, delete it now since there

620 # are no longer any warnings/errors.

621 lintian_action_item.delete()

622 return

623

624 # The item didn't previously have an action item: create it now

625 if lintian_action_item is None:

626 lintian_action_item = ActionItem(

627 package=package,

628 item_type=self.lintian_action_item_type)

629

630 lintian_url = lintian_stats.get_lintian_url()

631 new_extra_data = {

632 'warnings': warnings,

633 'errors': errors,

634 'lintian_url': lintian_url,

635 }

636 if lintian_action_item.extra_data:

637 old_extra_data = lintian_action_item.extra_data

638 if (old_extra_data['warnings'] == warnings and

639 old_extra_data['errors'] == errors):

640 # No need to update

641 return

642

643 lintian_action_item.extra_data = new_extra_data

644

645 if errors and warnings:

646 report = '{} error{} and {} warning{}'.format(

647 errors,

648 's' if errors > 1 else '',

649 warnings,

650 's' if warnings > 1 else '')

651 elif errors:

652 report = '{} error{}'.format(

653 errors,

654 's' if errors > 1 else '')

655 elif warnings: 655 ↛ 660line 655 didn't jump to line 660, because the condition on line 655 was never false

656 report = '{} warning{}'.format(

657 warnings,

658 's' if warnings > 1 else '')

659

660 lintian_action_item.short_description = self.ITEM_DESCRIPTION.format(

661 url=lintian_url,

662 report=report)

663

664 # If there are errors make the item a high severity issue

665 if errors:

666 lintian_action_item.severity = ActionItem.SEVERITY_HIGH

667

668 lintian_action_item.save()

669

670 def execute_main(self):

671 all_lintian_stats = self.get_lintian_stats()

672 if not all_lintian_stats:

673 return

674

675 # Discard all old stats

676 LintianStats.objects.all().delete()

677

678 packages = PackageName.objects.filter(name__in=all_lintian_stats.keys())

679 packages.prefetch_related('action_items')

680 # Remove action items for packages which no longer have associated

681 # lintian data.

682 ActionItem.objects.delete_obsolete_items(

683 [self.lintian_action_item_type], all_lintian_stats.keys())

684

685 stats = []

686 for package in packages:

687 package_stats = all_lintian_stats[package.name]

688 # Save the raw lintian stats.

689 lintian_stats = LintianStats(package=package, stats=package_stats)

690 stats.append(lintian_stats)

691 # Create an ActionItem if there are errors or warnings

692 self.update_action_item(package, lintian_stats)

693

694 LintianStats.objects.bulk_create(stats)

695

696

697class UpdateAppStreamStatsTask(BaseTask):

698 """

699 Updates packages' AppStream issue hints data.

700 """

701

702 class Scheduler(IntervalScheduler):

703 interval = 3600 * 6

704

705 ACTION_ITEM_TYPE_NAME = 'appstream-issue-hints'

706 ITEM_DESCRIPTION = 'AppStream hints: {report} for {packageurllist}'

707 ITEM_FULL_DESCRIPTION_TEMPLATE = 'debian/appstream-action-item.html'

708

709 def initialize(self, *args, **kwargs):

710 super(UpdateAppStreamStatsTask, self).initialize(*args, **kwargs)

711 self.appstream_action_item_type = \

712 ActionItemType.objects.create_or_update(

713 type_name=self.ACTION_ITEM_TYPE_NAME,

714 full_description_template=self.ITEM_FULL_DESCRIPTION_TEMPLATE)

715 self._tag_severities = {}

716

717 def _load_tag_severities(self):

718 url = 'https://appstream.debian.org/hints/sid/hint-definitions.json'

719 json_data = get_resource_text(url, force_update=True)

720

721 data = json.loads(json_data)

722 for tag, info in data.items():

723 self._tag_severities[tag] = info['severity']

724

725 def _load_appstream_hint_stats(self, section, arch, all_stats={}):

726 url = 'https://appstream.debian.org/hints/sid/{}/Hints-{}.json.gz' \

727 .format(section, arch)

728 hints_json = get_resource_text(url, force_update=self.force_update)

729

730 hints = json.loads(hints_json)

731 for hint in hints:

732 pkid = hint['package']

733 parts = pkid.split('/')

734 package_name = parts[0]

735

736 # get the source package for this binary package name

737 src_pkgname = None

738 if SourcePackageName.objects.exists_with_name(package_name):

739 package = SourcePackageName.objects.get(name=package_name)

740 src_pkgname = package.name

741 elif BinaryPackageName.objects.exists_with_name(package_name):

742 bin_package = BinaryPackageName.objects.get(name=package_name)

743 package = bin_package.main_source_package_name

744 src_pkgname = package.name

745 else:

746 src_pkgname = package_name

747

748 if src_pkgname not in all_stats:

749 all_stats[src_pkgname] = {}

750 if package_name not in all_stats[src_pkgname]: 750 ↛ 753line 750 didn't jump to line 753, because the condition on line 750 was never false

751 all_stats[src_pkgname][package_name] = {}

752

753 for cid, h in hint['hints'].items():

754 for e in h:

755 severity = self._tag_severities[e['tag']]

756 if severity == "error":

757 sevkey = "errors"

758 elif severity == "warning":

759 sevkey = "warnings"

760 elif severity == "info": 760 ↛ 763line 760 didn't jump to line 763, because the condition on line 760 was never false

761 sevkey = "infos"

762 else:

763 continue

764 if sevkey not in all_stats[src_pkgname][package_name]:

765 all_stats[src_pkgname][package_name][sevkey] = 1

766 else:

767 all_stats[src_pkgname][package_name][sevkey] += 1

768

769 return all_stats

770

771 def _get_appstream_url(self, package, bin_pkgname):

772 """

773 Returns the AppStream URL for the given PackageName in :package.

774 """

775

776 src_package = get_or_none(SourcePackageName, pk=package.pk)

777 if not src_package: 777 ↛ 778line 777 didn't jump to line 778, because the condition on line 777 was never true

778 return '#'

779

780 if not src_package.main_version:

781 return '#'

782

783 component = 'main'

784 main_entry = src_package.main_entry

785 if main_entry: 785 ↛ 786line 785 didn't jump to line 786, because the condition on line 785 was never true

786 component = main_entry.component

787 if not component:

788 component = 'main'

789

790 return (

791 'https://appstream.debian.org/sid/{}/issues/{}.html'

792 .format(component, bin_pkgname)

793 )

794

795 def _create_final_stats_report(self, package, package_stats):

796 """

797 Returns a transformed statistics report to be stored in the database.

798 """

799

800 as_report = package_stats.copy()

801 for bin_package in list(as_report.keys()):

802 # we currently don't want to display info-type hints

803 as_report[bin_package].pop('infos', None)

804 if as_report[bin_package]: 804 ↛ 808line 804 didn't jump to line 808, because the condition on line 804 was never false

805 as_report[bin_package]['url'] = \

806 self._get_appstream_url(package, bin_package)

807 else:

808 as_report.pop(bin_package)

809 return as_report

810

811 def update_action_item(self, package, package_stats):

812 """

813 Updates the :class:`ActionItem` for the given package based on the

814 AppStream hint statistics given in ``package_stats``.

815 If the package has errors or warnings an

816 :class:`ActionItem` is created.

817 """

818

819 total_warnings = 0

820 total_errors = 0

821 packageurllist = []

822 for bin_pkgname, info in package_stats.items():

823 total_warnings += info.get('warnings', 0)

824 total_errors += info.get('errors', 0)

825 url = self._get_appstream_url(package, bin_pkgname)

826 packageurllist.append(f'<a href="{url}">{bin_pkgname}</a>')

827

828 # Get the old action item for this warning, if it exists.

829 appstream_action_item = package.get_action_item_for_type(

830 self.appstream_action_item_type.type_name)

831 if not total_warnings and not total_errors:

832 if appstream_action_item:

833 # If the item previously existed, delete it now since there

834 # are no longer any warnings/errors.

835 appstream_action_item.delete()

836 return

837

838 # The item didn't previously have an action item: create it now

839 if appstream_action_item is None:

840 appstream_action_item = ActionItem(

841 package=package,

842 item_type=self.appstream_action_item_type)

843

844 as_report = self._create_final_stats_report(package, package_stats)

845

846 if appstream_action_item.extra_data:

847 old_extra_data = appstream_action_item.extra_data

848 if old_extra_data == as_report:

849 # No need to update

850 return

851

852 appstream_action_item.extra_data = as_report

853

854 if total_errors and total_warnings:

855 short_report = '{} error{} and {} warning{}'.format(

856 total_errors,

857 's' if total_errors > 1 else '',

858 total_warnings,

859 's' if total_warnings > 1 else '')

860 elif total_errors:

861 short_report = '{} error{}'.format(

862 total_errors,

863 's' if total_errors > 1 else '')

864 elif total_warnings: 864 ↛ 869line 864 didn't jump to line 869

865 short_report = '{} warning{}'.format(

866 total_warnings,

867 's' if total_warnings > 1 else '')

868

869 appstream_action_item.short_description = \

870 self.ITEM_DESCRIPTION.format(packageurllist=",".join(

871 packageurllist), report=short_report)

872

873 # If there are errors make the item a high severity issue;

874 # otherwise, make sure to set the severity as normal in case the item

875 # existed already

876 if total_errors:

877 appstream_action_item.severity = ActionItem.SEVERITY_HIGH

878 else:

879 appstream_action_item.severity = ActionItem.SEVERITY_NORMAL

880

881 appstream_action_item.save()

882

883 def execute_main(self):

884 self._load_tag_severities()

885 all_stats = {}

886 repository = Repository.objects.get(default=True)

887 arch = "amd64"

888 for component in repository.components:

889 self._load_appstream_hint_stats(component, arch, all_stats)

890 if not all_stats: 890 ↛ 891line 890 didn't jump to line 891, because the condition on line 890 was never true

891 return

892

893 with transaction.atomic():

894 # Delete obsolete data

895 PackageData.objects.filter(key='appstream').delete()

896

897 packages = PackageName.objects.filter(name__in=all_stats.keys())

898 packages.prefetch_related('action_items')

899

900 stats = []

901 for package in packages:

902 package_stats = all_stats[package.name]

903 stats.append(

904 PackageData(

905 package=package,

906 key='appstream',

907 value=package_stats

908 )

909 )

910

911 # Create an ActionItem if there are errors or warnings

912 self.update_action_item(package, package_stats)

913

914 PackageData.objects.bulk_create(stats)

915 # Remove action items for packages which no longer have associated

916 # AppStream hints.

917 ActionItem.objects.delete_obsolete_items(

918 [self.appstream_action_item_type], all_stats.keys())

919

920

921class UpdateTransitionsTask(BaseTask):

922

923 class Scheduler(IntervalScheduler):

924 interval = 3600

925

926 REJECT_LIST_URL = 'https://ftp-master.debian.org/transitions.yaml'

927 PACKAGE_TRANSITION_LIST_URL = (

928 'https://release.debian.org/transitions/export/packages.yaml')

929

930 def _get_yaml_resource(self, url, **kwargs):

931 """

932 Gets the YAML resource at the given URL and returns it as a Python

933 object.

934 """

935 content = get_resource_text(url, **kwargs)

936 if content:

937 return yaml.safe_load(content)

938

939 def _add_reject_transitions(self, packages):

940 """

941 Adds the transitions which cause uploads to be rejected to the

942 given ``packages`` dict.

943 """

944 reject_list = self._get_yaml_resource(self.REJECT_LIST_URL)

945 for key, transition in reject_list.items():

946 for package in transition['packages']:

947 packages.setdefault(package, {})

948 packages[package].setdefault(key, {})

949 packages[package][key]['reject'] = True

950 packages[package][key]['status'] = 'ongoing'

951

952 def _add_package_transition_list(self, packages):

953 """

954 Adds the ongoing and planned transitions to the given ``packages``

955 dict.

956 """

957 package_transition_list = self._get_yaml_resource(

958 self.PACKAGE_TRANSITION_LIST_URL)

959

960 wanted_transition_statuses = ('ongoing', 'planned')

961 for package_info in package_transition_list:

962 package_name = package_info['name']

963 for transition_name, status in package_info['list']:

964 if status not in wanted_transition_statuses:

965 # Skip transitions with an unwanted status

966 continue

967

968 packages.setdefault(package_name, {})

969 packages[package_name].setdefault(transition_name, {})

970 packages[package_name][transition_name]['status'] = status

971

972 def execute_main(self):

973 # Update the relevant resources first

974 kwargs = {

975 'force_update': self.force_update,

976 'only_if_updated': True,

977 }

978 reject_list = self._get_yaml_resource(self.REJECT_LIST_URL, **kwargs)

979 package_transition_list = self._get_yaml_resource(

980 self.PACKAGE_TRANSITION_LIST_URL, **kwargs)

981

982 if reject_list is None and package_transition_list is None:

983 # Nothing to do - at least one needs to be updated...

984 return

985

986 package_transitions = {}

987 self._add_reject_transitions(package_transitions)

988 self._add_package_transition_list(package_transitions)

989

990 PackageTransition.objects.all().delete()

991 # Get the packages which have transitions

992 packages = PackageName.objects.filter(

993 name__in=package_transitions.keys())

994 transitions = []

995 for package in packages:

996 for transition_name, data in \

997 package_transitions[package.name].items():

998 transitions.append(PackageTransition(

999 package=package,

1000 transition_name=transition_name,

1001 status=data.get('status', None),

1002 reject=data.get('reject', False)))

1003

1004 PackageTransition.objects.bulk_create(transitions)

1005

1006

1007class UpdateExcusesTask(BaseTask):

1008

1009 class Scheduler(IntervalScheduler):

1010 interval = 3600

1011

1012 ACTION_ITEM_TYPE_NAME = 'debian-testing-migration'

1013 ITEM_DESCRIPTION = (

1014 "The package has not entered testing even though the delay is over")

1015 ITEM_FULL_DESCRIPTION_TEMPLATE = 'debian/testing-migration-action-item.html'

1016

1017 class AgeVerdict(Enum):

1018 PKG_OF_AGE = 0

1019 PKG_TOO_OLD = 1

1020 PKG_TOO_YOUNG = 2

1021 PKG_WO_POLICY = 3

1022

1023 def initialize(self, *args, **kwargs):

1024 super(UpdateExcusesTask, self).initialize(*args, **kwargs)

1025 self.action_item_type = ActionItemType.objects.create_or_update(

1026 type_name=self.ACTION_ITEM_TYPE_NAME,

1027 full_description_template=self.ITEM_FULL_DESCRIPTION_TEMPLATE)

1028

1029 def _adapt_excuse_links(self, excuse):

1030 """

1031 If the excuse contains any anchor links, convert them to links to Distro

1032 Tracker package pages. Return the original text unmodified, otherwise.

1033 """

1034 re_anchor_href = re.compile(r'^#(.*)$')

1035 html = soup(excuse, 'html.parser')

1036 for a_tag in html.findAll('a', {'href': True}):

1037 href = a_tag['href']

1038 match = re_anchor_href.match(href)

1039 if not match: 1039 ↛ 1040line 1039 didn't jump to line 1040, because the condition on line 1039 was never true

1040 continue

1041 package = match.group(1).split('/')[0]

1042 a_tag['href'] = package_url(package)

1043

1044 return str(html)

1045

1046 def _skip_excuses_item(self, item_text):

1047 if not item_text:

1048 return True

1049 # We ignore these excuses

1050 if "Section" in item_text or "Maintainer" in item_text:

1051 return True

1052 return False

1053

1054 def _check_age(self, source):

1055 """Checks the age of the package and compares it to the age requirement

1056 for migration"""

1057

1058 if 'policy_info' not in source or 'age' not in source['policy_info']: 1058 ↛ 1059line 1058 didn't jump to line 1059, because the condition on line 1058 was never true

1059 return (self.AgeVerdict.PKG_WO_POLICY, None, None)

1060

1061 age = source['policy_info']['age']['current-age']

1062 limit = source['policy_info']['age']['age-requirement']

1063 if age > limit:

1064 return (self.AgeVerdict.PKG_TOO_OLD, age, limit)

1065 elif age < limit: 1065 ↛ 1066line 1065 didn't jump to line 1066, because the condition on line 1065 was never true

1066 return (self.AgeVerdict.PKG_TOO_YOUNG, age, limit)

1067 else:

1068 return (self.AgeVerdict.PKG_OF_AGE, age, limit)

1069

1070 def _extract_problematic(self, source):

1071 verdict, age, limit = self._check_age(source)

1072

1073 if verdict == self.AgeVerdict.PKG_TOO_OLD:

1074 return (source['item-name'], {'age': age, 'limit': limit})

1075

1076 @staticmethod

1077 def _make_excuses_check_dependencies(source):

1078 """Checks the dependencies of the package (blocked-by and

1079 migrate-after) and returns a list to display."""

1080

1081 addendum = []

1082

1083 if 'dependencies' in source:

1084 blocked_by = source['dependencies'].get('blocked-by', [])

1085 after = source['dependencies'].get('migrate-after', [])

1086 after = [

1087 element

1088 for element in after

1089 if element not in blocked_by

1090 ]

1091 if blocked_by: 1091 ↛ 1092line 1091 didn't jump to line 1092, because the condition on line 1091 was never true

1092 addendum.append("Blocked by: %s" % (

1093 html_package_list(blocked_by),

1094 ))

1095 if after: 1095 ↛ 1100line 1095 didn't jump to line 1100, because the condition on line 1095 was never false

1096 addendum.append("Migrates after: %s" % (

1097 html_package_list(after),

1098 ))

1099

1100 return addendum

1101

1102 @staticmethod

1103 def _make_excuses_check_verdict(source):

1104 """Checks the migration policy verdict of the package and builds an

1105 excuses message depending on the result."""

1106

1107 addendum = []

1108

1109 if 'migration-policy-verdict' in source: 1109 ↛ 1110line 1109 didn't jump to line 1110, because the condition on line 1109 was never true

1110 verdict = source['migration-policy-verdict']

1111 if verdict == 'REJECTED_BLOCKED_BY_ANOTHER_ITEM':

1112 addendum.append("Migration status: Blocked. Can't migrate "

1113 "due to a non-migratable dependency. Check "

1114 "status below."

1115 )

1116

1117 return addendum

1118

1119 def _make_excuses(self, source):

1120 """Make the excuses list for a source item using the yaml data it

1121 contains"""

1122

1123 excuses = [

1124 self._adapt_excuse_links(excuse)

1125 for excuse in source['excuses']

1126 ]

1127

1128 # This is the place where we compute some additionnal

1129 # messages that should be added to excuses.

1130 addendum = []

1131

1132 addendum.extend(self._make_excuses_check_verdict(source))

1133 addendum.extend(self._make_excuses_check_dependencies(source))

1134

1135 excuses = addendum + excuses

1136

1137 if 'is-candidate' in source: 1137 ↛ 1141line 1137 didn't jump to line 1141, because the condition on line 1137 was never false

1138 if not source['is-candidate']: 1138 ↛ 1141line 1138 didn't jump to line 1141, because the condition on line 1138 was never false

1139 excuses.append("Not considered")

1140

1141 return (

1142 source['item-name'],

1143 excuses,

1144 )

1145

1146 def _get_excuses_and_problems(self, content):

1147 """

1148 Gets the excuses for each package.

1149 Also finds a list of packages which have not migrated to testing

1150 agter the necessary time has passed.

1151

1152 :returns: A two-tuple where the first element is a dict mapping

1153 package names to a list of excuses. The second element is a dict

1154 mapping packages names to a problem information. Problem information

1155 is a dict with the keys ``age`` and ``limit``.

1156 """

1157 if 'sources' not in content: 1157 ↛ 1158line 1157 didn't jump to line 1158, because the condition on line 1157 was never true

1158 logger.warning("Invalid format of excuses file")

1159 return

1160

1161 sources = content['sources']

1162 excuses = [

1163 self._make_excuses(source)

1164 for source in sources

1165 if '/' not in source['item-name']

1166 ]

1167 problems = [

1168 self._extract_problematic(source)

1169 for source in sources

1170 if '/' not in source['item-name']

1171 ]

1172 problematic = [p for p in problems if p]

1173 return dict(excuses), dict(problematic)

1174

1175 def _create_action_item(self, package, extra_data):

1176 """

1177 Creates a :class:`distro_tracker.core.models.ActionItem` for the given

1178 package including the given extra data. The item indicates that there is

1179 a problem with the package migrating to testing.

1180 """

1181 action_item = \

1182 package.get_action_item_for_type(self.ACTION_ITEM_TYPE_NAME)

1183 if action_item is None:

1184 action_item = ActionItem(

1185 package=package,

1186 item_type=self.action_item_type)

1187

1188 action_item.short_description = self.ITEM_DESCRIPTION

1189 if package.main_entry: 1189 ↛ 1190line 1189 didn't jump to line 1190, because the condition on line 1189 was never true

1190 query_string = urlencode({'package': package.name})

1191 extra_data['check_why_url'] = (

1192 'https://qa.debian.org/excuses.php'

1193 '?{query_string}'.format(query_string=query_string))

1194

1195 action_item.extra_data = extra_data

1196 action_item.save()

1197

1198 def _remove_obsolete_action_items(self, problematic):

1199 """

1200 Remove action items for packages which are no longer problematic.

1201 """

1202 ActionItem.objects.delete_obsolete_items(

1203 item_types=[self.action_item_type],

1204 non_obsolete_packages=problematic.keys())

1205

1206 def _get_excuses_yaml(self):

1207 """

1208 Function returning the content of excuses from debian-release

1209 :returns: a dict of excuses or ``None`` if the content in the

1210 cache is up to date.

1211 """

1212 url = 'https://release.debian.org/britney/excuses.yaml'

1213 content = get_resource_text(url, force_update=self.force_update,

1214 only_if_updated=True)

1215 if content is None:

1216 return

1217

1218 return yaml.safe_load(content)

1219

1220 def execute_main(self):

1221 content_lines = self._get_excuses_yaml()

1222 if not content_lines: 1222 ↛ 1223line 1222 didn't jump to line 1223, because the condition on line 1222 was never true

1223 return

1224

1225 result = self._get_excuses_and_problems(content_lines)

1226 if not result: 1226 ↛ 1227line 1226 didn't jump to line 1227, because the condition on line 1226 was never true

1227 return

1228 package_excuses, problematic = result

1229

1230 with transaction.atomic():

1231 # Remove stale excuses data and action items which are not still

1232 # problematic.

1233 self._remove_obsolete_action_items(problematic)

1234 PackageExcuses.objects.all().delete()

1235

1236 excuses = []

1237 packages = SourcePackageName.objects.filter(

1238 name__in=package_excuses.keys())

1239 packages.prefetch_related('action_items')

1240 for package in packages:

1241 excuse = PackageExcuses(

1242 package=package,

1243 excuses=package_excuses[package.name])

1244 excuses.append(excuse)

1245 if package.name in problematic:

1246 self._create_action_item(package, problematic[package.name])

1247

1248 # Create all excuses in a single query

1249 PackageExcuses.objects.bulk_create(excuses)

1250

1251

1252class DebianWatchFileScannerUpdate(BaseTask):

1253

1254 class Scheduler(IntervalScheduler):

1255 interval = 3600 * 6

1256

1257 ACTION_ITEM_TYPE_NAMES = (

1258 'new-upstream-version',

1259 'watch-failure',

1260 )

1261 ACTION_ITEM_TEMPLATES = {

1262 'new-upstream-version': "debian/new-upstream-version-action-item.html",

1263 'watch-failure': "debian/watch-failure-action-item.html",

1264 }

1265 ITEM_DESCRIPTIONS = {

1266 'new-upstream-version': lambda item: (

1267 'A new upstream version is available: '

1268 '<a href="{url}">{version}</a>'.format(

1269 url=item.extra_data['upstream_url'],

1270 version=item.extra_data['upstream_version'])),

1271 'watch-failure': lambda item: (

1272 'Problems while searching for a new upstream version'),

1273 }

1274 ITEM_SEVERITIES = {

1275 'new-upstream-version': ActionItem.SEVERITY_HIGH,

1276 'watch-failure': ActionItem.SEVERITY_HIGH,

1277 }

1278

1279 def initialize(self, *args, **kwargs):

1280 super(DebianWatchFileScannerUpdate, self).initialize(*args, **kwargs)

1281 self.action_item_types = {

1282 type_name: ActionItemType.objects.create_or_update(

1283 type_name=type_name,

1284 full_description_template=self.ACTION_ITEM_TEMPLATES.get(

1285 type_name, None))

1286 for type_name in self.ACTION_ITEM_TYPE_NAMES

1287 }

1288

1289 def _get_upstream_status_content(self):

1290 url = 'https://udd.debian.org/cgi-bin/upstream-status.json.cgi'

1291 return get_resource_text(url)

1292

1293 def _remove_obsolete_action_items(self, item_type_name,

1294 non_obsolete_packages):

1295 """

1296 Removes any existing :class:`ActionItem` with the given type name based

1297 on the list of package names which should still have the items based on

1298 the processed stats.

1299 """

1300 action_item_type = self.action_item_types[item_type_name]

1301 ActionItem.objects.delete_obsolete_items(

1302 item_types=[action_item_type],

1303 non_obsolete_packages=non_obsolete_packages)

1304

1305 def get_upstream_status_stats(self, stats):

1306 """

1307 Gets the stats from the downloaded data and puts them in the given

1308 ``stats`` dictionary.

1309 The keys of the dict are package names.

1310

1311 :returns: A a two-tuple where the first item is a list of packages

1312 which have new upstream versions and the second is a list of

1313 packages which have watch failures.

1314 """

1315 content = self._get_upstream_status_content()

1316 dehs_data = None

1317 if content:

1318 dehs_data = json.loads(content)

1319 if not dehs_data:

1320 return [], []

1321

1322 all_new_versions, all_failures = [], []

1323 for entry in dehs_data:

1324 package_name = entry['package']

1325 stats.setdefault(package_name, {})

1326 stats[package_name]['upstream_version'] = entry['upstream-version']

1327 stats[package_name]['upstream_url'] = entry['upstream-url']

1328 if 'status' in entry and ('Newer version' in entry['status'] or

1329 'newer package' in entry['status']):

1330 stats[package_name]['new-upstream-version'] = {

1331 'upstream_version': entry['upstream-version'],

1332 'upstream_url': entry['upstream-url'],

1333 }

1334 all_new_versions.append(package_name)

1335 if entry.get('warnings') or entry.get('errors'):

1336 msg = '{}\n{}'.format(

1337 entry.get('errors') or '',

1338 entry.get('warnings') or '',

1339 ).strip()

1340 stats[package_name]['watch-failure'] = {

1341 'warning': msg,

1342 }

1343 all_failures.append(package_name)

1344

1345 return all_new_versions, all_failures

1346

1347 def update_package_info(self, package, stats):

1348 """

1349 Updates upstream information of the given package based on the given

1350 stats. Upstream data is saved as a :class:`PackageData` within the

1351 `general` key

1352

1353 :param package: The package to which the upstream info should be

1354 associated.

1355 :type package: :class:`distro_tracker.core.models.PackageName`

1356 :param stats: The stats which are used to create the upstream info.

1357 :type stats: :class:`dict`

1358 """

1359 try:

1360 watch_data = package.watch_status[0]

1361 except IndexError:

1362 watch_data = PackageData(

1363 package=package,

1364 key='upstream-watch-status',

1365 )

1366

1367 watch_data.value = stats

1368 watch_data.save()

1369

1370 def update_action_item(self, item_type, package, stats):

1371 """

1372 Updates the action item of the given type for the given package based

1373 on the given stats.

1374

1375 The severity of the item is defined by the :attr:`ITEM_SEVERITIES` dict.

1376

1377 The short descriptions are created by passing the :class:`ActionItem`

1378 (with extra data already set) to the callables defined in

1379 :attr:`ITEM_DESCRIPTIONS`.

1380

1381 :param item_type: The type of the :class:`ActionItem` that should be

1382 updated.

1383 :type item_type: string

1384 :param package: The package to which this action item should be

1385 associated.

1386 :type package: :class:`distro_tracker.core.models.PackageName`

1387 :param stats: The stats which are used to create the action item.

1388 :type stats: :class:`dict`

1389 """

1390 action_item = package.get_action_item_for_type(item_type)

1391 if action_item is None:

1392 # Create an action item...

1393 action_item = ActionItem(

1394 package=package,

1395 item_type=self.action_item_types[item_type])

1396

1397 if item_type in self.ITEM_SEVERITIES: 1397 ↛ 1399line 1397 didn't jump to line 1399, because the condition on line 1397 was never false

1398 action_item.severity = self.ITEM_SEVERITIES[item_type]

1399 action_item.extra_data = stats

1400 action_item.short_description = \

1401 self.ITEM_DESCRIPTIONS[item_type](action_item)

1402

1403 action_item.save()

1404

1405 @transaction.atomic

1406 def execute_main(self):

1407 stats = {}

1408 new_upstream_version, failures = self.get_upstream_status_stats(stats)

1409 updated_packages_per_type = {

1410 'new-upstream-version': new_upstream_version,

1411 'watch-failure': failures,

1412 }

1413

1414 # Remove obsolete action items for each of the categories...

1415 for item_type, packages in updated_packages_per_type.items():

1416 self._remove_obsolete_action_items(item_type, packages)

1417

1418 packages = SourcePackageName.objects.filter(

1419 name__in=stats.keys())

1420 filter_qs = PackageData.objects.filter(key='upstream-watch-status')

1421 packages = packages.prefetch_related(

1422 'action_items',

1423 Prefetch('data', queryset=filter_qs, to_attr='watch_status')

1424 )

1425

1426 # Update action items for each package

1427 for package in packages:

1428 for type_name in self.ACTION_ITEM_TYPE_NAMES:

1429 if type_name in stats[package.name]:

1430 # method(package, stats[package.name][type_name])

1431 self.update_action_item(

1432 type_name, package, stats[package.name][type_name])

1433

1434 self.update_package_info(package, stats[package.name])

1435

1436

1437class UpdateSecurityIssuesTask(BaseTask):

1438

1439 class Scheduler(IntervalScheduler):

1440 interval = 3600 * 3

1441

1442 ACTION_ITEM_TYPE_NAME = 'debian-security-issue-in-{}'

1443 ACTION_ITEM_TEMPLATE = 'debian/security-issue-action-item.html'

1444 ITEM_DESCRIPTION_TEMPLATE = {

1445 'open': '<a href="{url}">{count} security {issue}</a> in {release}',

1446 'nodsa':

1447 '<a href="{url}">{count} low-priority security {issue}</a> '

1448 'in {release}',

1449 'none': 'No known security issue in {release}',

1450 }

1451 CVE_DATA_URL = 'https://security-tracker.debian.org/tracker/data/json'

1452 DISTRIBUTIONS_URL = (

1453 'https://security-tracker.debian.org/tracker/distributions.json'

1454 )

1455

1456 def initialize(self, *args, **kwargs):

1457 super(UpdateSecurityIssuesTask, self).initialize(*args, **kwargs)

1458 self._action_item_type = {}

1459 self._issues = None

1460 self._distributions = None

1461

1462 def action_item_type(self, release):

1463 return self._action_item_type.setdefault(

1464 release, ActionItemType.objects.create_or_update(

1465 type_name=self.ACTION_ITEM_TYPE_NAME.format(release),

1466 full_description_template=self.ACTION_ITEM_TEMPLATE))

1467

1468 def _get_distributions(self):

1469 if not self._distributions:

1470 content = get_resource_text(self.DISTRIBUTIONS_URL)

1471 self._distributions = json.loads(content)

1472 return self._distributions

1473

1474 def _get_support_status(self, release):

1475 """

1476 Return support status of a given release as documented by the

1477 security team in the security tracker.

1478 """

1479 return self._get_distributions().get(release, {}).get('support',

1480 'unknown')

1481

1482 def _get_issues_content(self):

1483 if self._issues: 1483 ↛ 1484line 1483 didn't jump to line 1484, because the condition on line 1483 was never true

1484 return self._issues

1485 content = get_resource_text(self.CVE_DATA_URL)

1486 if content: 1486 ↛ exitline 1486 didn't return from function '_get_issues_content', because the condition on line 1486 was never false

1487 self._issues = json.loads(content)

1488 return self._issues

1489

1490 @classmethod

1491 def _update_stats_with_nodsa_entry(cls, stats, nodsa_entry,

1492 entry_id, description):

1493 stats['nodsa'] += 1

1494

1495 nodsa_details = {'description': description,

1496 'nodsa': nodsa_entry.get('nodsa', ''),

1497 'nodsa_reason': nodsa_entry.get('nodsa_reason', '')

1498 }

1499

1500 nodsa_reason = nodsa_details['nodsa_reason']

1501 if nodsa_reason == '':

1502 nodsa_details['needs_triaging'] = True

1503 stats['nodsa_maintainer_to_handle_details'][entry_id] = \

1504 nodsa_details

1505 elif nodsa_reason == 'postponed': 1505 ↛ 1506line 1505 didn't jump to line 1506, because the condition on line 1505 was never true

1506 nodsa_details['fixed_via_stable_update'] = True

1507 stats['nodsa_maintainer_to_handle_details'][entry_id] = \

1508 nodsa_details

1509 elif nodsa_reason == 'ignored': 1509 ↛ exitline 1509 didn't return from function '_update_stats_with_nodsa_entry', because the condition on line 1509 was never false

1510 stats['nodsa_ignored_details'][entry_id] = nodsa_details

1511

1512 @classmethod

1513 def get_issues_summary(cls, issues):

1514 result = {}

1515 for issue_id, issue_data in issues.items():

1516 for release, data in issue_data['releases'].items():

1517 stats = result.setdefault(release, {

1518 'open': 0,

1519 'open_details': {},

1520 'nodsa': 0,

1521 'unimportant': 0,

1522 'next_point_update_details': {},

1523 'nodsa_maintainer_to_handle_details': {},

1524 'nodsa_ignored_details': {},

1525 })

1526 description = issue_data.get('description', '')

1527 if (data.get('status', '') == 'resolved' or

1528 data.get('urgency', '') == 'end-of-life'):

1529 continue

1530 elif data.get('urgency', '') == 'unimportant':

1531 stats['unimportant'] += 1

1532 elif data.get('next_point_update', False):

1533 stats['next_point_update_details'][issue_id] = \

1534 {'description': description}

1535 elif data.get('nodsa', False) is not False:

1536 cls._update_stats_with_nodsa_entry(stats,

1537 data, issue_id,

1538 description

1539 )

1540 else:

1541 stats['open'] += 1

1542 stats['open_details'][issue_id] = \

1543 {'description': description}

1544

1545 return result

1546

1547 @classmethod

1548 def get_issues_stats(cls, content):

1549 """

1550 Gets package issue stats from Debian's security tracker.

1551 """

1552 stats = {}

1553 for pkg, issues in content.items():

1554 stats[pkg] = cls.get_issues_summary(issues)

1555 return stats

1556

1557 def _get_short_description(self, key, action_item):

1558 count = action_item.extra_data['security_issues_count']

1559 url = 'https://security-tracker.debian.org/tracker/source-package/{}'

1560 return self.ITEM_DESCRIPTION_TEMPLATE[key].format(

1561 count=count,

1562 issue='issues' if count > 1 else 'issue',

1563 release=action_item.extra_data.get('release', 'sid'),

1564 url=url.format(action_item.package.name),

1565 )

1566

1567 def update_action_item(self, stats, action_item):

1568 """

1569 Updates the ``debian-security-issue`` action item based on the

1570 security issues.

1571 """

1572

1573 security_issues_count = stats['open'] + stats['nodsa']

1574 action_item.extra_data['security_issues_count'] = security_issues_count

1575 action_item.extra_data['support_status'] = (

1576 self._get_support_status(action_item.extra_data['release'])

1577 )

1578

1579 for base_key in ['open',

1580 'next_point_update',

1581 'nodsa_maintainer_to_handle',

1582 'nodsa_ignored']:

1583 details_key = base_key + '_details'

1584 count_key = base_key + '_count'

1585

1586 action_item.extra_data[details_key] = stats[details_key]

1587 action_item.extra_data[count_key] = len(stats[details_key])

1588

1589 # nodsa_next_point_update / nodsa_ignored_details are displayed

1590 # only if there is anything else to show

1591 nodsa_create_action = (stats['nodsa'] -

1592 len(stats['nodsa_ignored_details'])) > 0

1593

1594 if stats['open']:

1595 action_item.severity = ActionItem.SEVERITY_HIGH

1596 action_item.short_description = \

1597 self._get_short_description('open', action_item)

1598 elif nodsa_create_action:

1599 action_item.severity = ActionItem.SEVERITY_LOW

1600 action_item.short_description = \

1601 self._get_short_description('nodsa', action_item)

1602 else:

1603 action_item.severity = ActionItem.SEVERITY_WISHLIST

1604 action_item.short_description = \

1605 self._get_short_description('none', action_item)

1606

1607 @classmethod

1608 def generate_package_data(cls, issues):

1609 return {

1610 'details': issues,

1611 'stats': cls.get_issues_summary(issues),

1612 'checksum': get_data_checksum(issues)

1613 }

1614

1615 def want_action_item(self, pkgdata, release):

1616 stats = pkgdata.value.get('stats', {}).get(release)

1617 if stats is None: 1617 ↛ 1618line 1617 didn't jump to line 1618, because the condition on line 1617 was never true

1618 return False

1619

1620 supported_by = self._get_support_status(release)

1621 if supported_by == "end-of-life":

1622 return False

1623 elif supported_by == "security":

1624 count = stats.get('open', 0) + stats.get('nodsa', 0)

1625 else:

1626 count = stats.get('open', 0)

1627

1628 if count == 0:

1629 return False

1630

1631 return True

1632

1633 def process_pkg_action_items(self, pkgdata, existing_action_items):

1634 release_ai = {}

1635 to_add = []

1636 to_update = []

1637 to_drop = []

1638 global_stats = pkgdata.value.get('stats', {})

1639 for ai in existing_action_items:

1640 release = ai.extra_data['release']

1641 release_ai[release] = ai

1642 for release, stats in global_stats.items():

1643 ai = release_ai.get(release)

1644

1645 if self.want_action_item(pkgdata, release):

1646 if ai:

1647 to_update.append(ai)

1648 else:

1649 ai = ActionItem(

1650 item_type=self.action_item_type(release),

1651 package=pkgdata.package,

1652 extra_data={'release': release}

1653 )

1654 to_add.append(ai)

1655 self.update_action_item(stats, ai)

1656 else:

1657 if ai:

1658 to_drop.append(ai)

1659

1660 return to_add, to_update, to_drop

1661

1662 def execute_main(self):

1663 # Fetch all debian-security PackageData

1664 all_pkgdata = PackageData.objects.select_related(

1665 'package').filter(key='debian-security').only(

1666 'package__name', 'value')

1667

1668 all_data = {}

1669 packages = {}

1670 for pkgdata in all_pkgdata:

1671 all_data[pkgdata.package.name] = pkgdata

1672 packages[pkgdata.package.name] = pkgdata.package

1673

1674 # Fetch all debian-security ActionItems

1675 pkg_action_items = collections.defaultdict(lambda: [])

1676 all_action_items = ActionItem.objects.select_related(

1677 'package').filter(

1678 item_type__type_name__startswith='debian-security-issue-in-')

1679 for action_item in all_action_items:

1680 pkg_action_items[action_item.package.name].append(action_item)

1681

1682 # Check for changes on distributions.json

1683 distributions_checksum = get_data_checksum(self._get_distributions())

1684 if self.data.get('distributions_checksum') != distributions_checksum:

1685 # New distributions.json, force update all action items

1686 self.force_update = True

1687 self.data['distributions_checksum'] = distributions_checksum

1688

1689 # Scan the security tracker data

1690 content = self._get_issues_content()

1691 to_add = []

1692 to_update = []

1693 for pkgname, issues in content.items():

1694 if pkgname in all_data:

1695 # Check if we need to update the existing data

1696 checksum = get_data_checksum(issues)

1697 if not self.force_update and \

1698 all_data[pkgname].value.get('checksum', '') == checksum:

1699 continue

1700 # Update the data

1701 pkgdata = all_data[pkgname]

1702 pkgdata.value = self.generate_package_data(issues)

1703 to_update.append(pkgdata)

1704 else:

1705 # Add data for a new package

1706 package, _ = PackageName.objects.get_or_create(name=pkgname)

1707 to_add.append(

1708 PackageData(

1709 package=package,

1710 key='debian-security',

1711 value=self.generate_package_data(issues)

1712 )

1713 )

1714 # Process action items

1715 ai_to_add = []

1716 ai_to_update = []

1717 ai_to_drop = []

1718 for pkgdata in itertools.chain(to_add, to_update):

1719 add, update, drop = self.process_pkg_action_items(

1720 pkgdata, pkg_action_items[pkgdata.package.name])

1721 ai_to_add.extend(add)

1722 ai_to_update.extend(update)

1723 ai_to_drop.extend(drop)

1724 # Sync in database

1725 with transaction.atomic():

1726 # Delete obsolete data

1727 PackageData.objects.filter(

1728 key='debian-security').exclude(

1729 package__name__in=content.keys()).delete()

1730 ActionItem.objects.filter(

1731 item_type__type_name__startswith='debian-security-issue-in-'

1732 ).exclude(package__name__in=content.keys()).delete()

1733 ActionItem.objects.filter(

1734 item_type__type_name__startswith='debian-security-issue-in-',

1735 id__in=[ai.id for ai in ai_to_drop]).delete()

1736 # Add new entries

1737 PackageData.objects.bulk_create(to_add)

1738 ActionItem.objects.bulk_create(ai_to_add)

1739 # Update existing entries

1740 for pkgdata in to_update:

1741 pkgdata.save()

1742 for ai in ai_to_update:

1743 ai.save()

1744

1745

1746class UpdatePiuPartsTask(BaseTask):

1747 """

1748 Retrieves the piuparts stats for all the suites defined in the

1749 :data:`distro_tracker.project.local_settings.DISTRO_TRACKER_DEBIAN_PIUPARTS_SUITES`

1750 """

1751

1752 class Scheduler(IntervalScheduler):

1753 interval = 3600 * 3

1754

1755 ACTION_ITEM_TYPE_NAME = 'debian-piuparts-test-fail'

1756 ACTION_ITEM_TEMPLATE = 'debian/piuparts-action-item.html'

1757 ITEM_DESCRIPTION = 'piuparts found (un)installation error(s)'

1758

1759 def initialize(self, *args, **kwargs):

1760 super(UpdatePiuPartsTask, self).initialize(*args, **kwargs)

1761 self.action_item_type = ActionItemType.objects.create_or_update(

1762 type_name=self.ACTION_ITEM_TYPE_NAME,

1763 full_description_template=self.ACTION_ITEM_TEMPLATE)

1764

1765 def _get_piuparts_content(self, suite):

1766 """

1767 :returns: The content of the piuparts report for the given package

1768 or ``None`` if there is no data for the particular suite.

1769 """

1770 url = 'https://piuparts.debian.org/{suite}/sources.txt'

1771 return get_resource_text(url.format(suite=suite))

1772

1773 def get_piuparts_stats(self):

1774 suites = getattr(settings, 'DISTRO_TRACKER_DEBIAN_PIUPARTS_SUITES', [])

1775 failing_packages = {}

1776 for suite in suites:

1777 content = self._get_piuparts_content(suite)

1778 if content is None:

1779 logger.info("There is no piuparts for suite: %s", suite)

1780 continue

1781

1782 for line in content.splitlines():

1783 package_name, status = line.split(':', 1)

1784 package_name, status = package_name.strip(), status.strip()

1785 if status == 'fail':

1786 failing_packages.setdefault(package_name, [])

1787 failing_packages[package_name].append(suite)

1788

1789 return failing_packages

1790

1791 def create_action_item(self, package, suites):

1792 """

1793 Creates an :class:`ActionItem <distro_tracker.core.models.ActionItem>`

1794 instance for the package based on the list of suites in which the

1795 piuparts installation test failed.

1796 """

1797 action_item = package.get_action_item_for_type(self.action_item_type)

1798 if action_item is None:

1799 action_item = ActionItem(

1800 package=package,

1801 item_type=self.action_item_type,

1802 short_description=self.ITEM_DESCRIPTION)

1803

1804 if action_item.extra_data:

1805 existing_items = action_item.extra_data.get('suites', [])

1806 if list(sorted(existing_items)) == list(sorted(suites)):

1807 # No need to update this item

1808 return

1809 action_item.extra_data = {

1810 'suites': suites,

1811 }

1812 action_item.save()

1813

1814 def execute_main(self):

1815 failing_packages = self.get_piuparts_stats()

1816

1817 ActionItem.objects.delete_obsolete_items(

1818 item_types=[self.action_item_type],

1819 non_obsolete_packages=failing_packages.keys())

1820

1821 packages = SourcePackageName.objects.filter(

1822 name__in=failing_packages.keys())

1823 packages = packages.prefetch_related('action_items')

1824

1825 for package in packages:

1826 self.create_action_item(package, failing_packages[package.name])

1827

1828

1829class UpdateUbuntuStatsTask(BaseTask):

1830 """

1831 The task updates Ubuntu stats for packages. These stats are displayed in a

1832 separate panel.

1833 """

1834

1835 class Scheduler(IntervalScheduler):

1836 interval = 3600 * 3

1837

1838 def initialize(self, *args, **kwargs):

1839 super(UpdateUbuntuStatsTask, self).initialize(*args, **kwargs)

1840

1841 def _get_versions_content(self):

1842 url = 'https://udd.debian.org/cgi-bin/ubuntupackages.cgi'

1843 return get_resource_text(url)

1844

1845 def get_ubuntu_versions(self):

1846 """

1847 Retrieves the Ubuntu package versions.

1848

1849 :returns: A dict mapping package names to Ubuntu versions.

1850 """

1851 content = self._get_versions_content()

1852

1853 package_versions = {}

1854 for line in content.splitlines():

1855 package, version = line.split(' ', 1)

1856 version = version.strip()

1857 package_versions[package] = version

1858

1859 return package_versions

1860

1861 def _get_bug_stats_content(self):

1862 url = 'https://udd.debian.org/cgi-bin/ubuntubugs.cgi'

1863 return get_resource_text(url)

1864

1865 def get_ubuntu_bug_stats(self):

1866 """

1867 Retrieves the Ubuntu bug stats of a package. Bug stats contain the

1868 count of bugs and the count of patches.

1869

1870 :returns: A dict mapping package names to a dict of package stats.

1871 """

1872 content = self._get_bug_stats_content()

1873

1874 bug_stats = {}

1875 for line in content.splitlines():

1876 package_name, bug_count, patch_count = line.split("|", 2)

1877 try:

1878 bug_count, patch_count = int(bug_count), int(patch_count)

1879 except ValueError:

1880 continue

1881 bug_stats[package_name] = {

1882 'bug_count': bug_count,

1883 'patch_count': patch_count,

1884 }

1885

1886 return bug_stats

1887

1888 def _get_ubuntu_patch_diff_content(self):

1889 url = 'https://patches.ubuntu.com/PATCHES'

1890 return get_resource_text(url)

1891

1892 def get_ubuntu_patch_diffs(self):

1893 """

1894 Retrieves the Ubuntu patch diff information. The information consists

1895 of the diff URL and the version of the Ubuntu package to which the

1896 diff belongs to.

1897

1898 :returns: A dict mapping package names to diff information.

1899 """

1900 content = self._get_ubuntu_patch_diff_content()

1901

1902 patch_diffs = {}

1903 re_diff_version = re.compile(r'_(\S+)\.patch')

1904 for line in content.splitlines():

1905 package_name, diff_url = line.split(' ', 1)

1906 # Extract the version of the package from the diff url

1907 match = re_diff_version.search(diff_url)

1908 if not match: 1908 ↛ 1910line 1908 didn't jump to line 1910, because the condition on line 1908 was never true

1909 # Invalid URL: no version

1910 continue

1911 version = match.group(1)

1912 patch_diffs[package_name] = {

1913 'version': version,

1914 'diff_url': diff_url

1915 }

1916

1917 return patch_diffs

1918

1919 def execute_main(self):

1920 package_versions = self.get_ubuntu_versions()

1921 bug_stats = self.get_ubuntu_bug_stats()

1922 patch_diffs = self.get_ubuntu_patch_diffs()

1923

1924 obsolete_ubuntu_pkgs = UbuntuPackage.objects.exclude(

1925 package__name__in=package_versions.keys())

1926 obsolete_ubuntu_pkgs.delete()

1927

1928 packages = PackageName.objects.filter(name__in=package_versions.keys())

1929 packages = packages.prefetch_related('ubuntu_package')

1930

1931 for package in packages:

1932 version = package_versions[package.name]

1933 bugs = bug_stats.get(package.name, None)

1934 diff = patch_diffs.get(package.name, None)

1935

1936 try:

1937 ubuntu_package = package.ubuntu_package

1938 ubuntu_package.version = version

1939 ubuntu_package.bugs = bugs

1940 ubuntu_package.patch_diff = diff

1941 ubuntu_package.save()

1942 except UbuntuPackage.DoesNotExist:

1943 ubuntu_package = UbuntuPackage.objects.create(

1944 package=package,

1945 version=version,

1946 bugs=bugs,

1947 patch_diff=diff)

1948

1949

1950class UpdateWnppStatsTask(BaseTask):

1951 """

1952 The task updates the WNPP bugs for all packages.

1953 """

1954

1955 class Scheduler(IntervalScheduler):

1956 interval = 3600 * 3

1957

1958 ACTION_ITEM_TYPE_NAME = 'debian-wnpp-issue'

1959 ACTION_ITEM_TEMPLATE = 'debian/wnpp-action-item.html'

1960 ITEM_DESCRIPTION = '<a href="{url}">{wnpp_type}: {wnpp_msg}</a>'

1961

1962 def initialize(self, *args, **kwargs):

1963 super(UpdateWnppStatsTask, self).initialize(*args, **kwargs)

1964 self.action_item_type = ActionItemType.objects.create_or_update(

1965 type_name=self.ACTION_ITEM_TYPE_NAME,

1966 full_description_template=self.ACTION_ITEM_TEMPLATE)

1967

1968 def get_wnpp_stats(self):

1969 """

1970 Retrieves and parses the wnpp stats for all packages. WNPP stats

1971 include the WNPP type and the BTS bug id.

1972

1973 :returns: A dict mapping package names to wnpp stats.

1974 """

1975 url = 'https://qa.debian.org/data/bts/wnpp_rm'

1976 content = get_resource_text(url, only_if_updated=True)

1977 if content is None: 1977 ↛ 1978line 1977 didn't jump to line 1978, because the condition on line 1977 was never true

1978 return

1979

1980 wnpp_stats = {}

1981 for line in content.splitlines():

1982 line = line.strip()

1983 try:

1984 package_name, wnpp_type, bug_id = line.split('|')[0].split()

1985 bug_id = int(bug_id)

1986 except ValueError:

1987 # Badly formatted bug number

1988 continue

1989 # Strip the colon from the end of the package name

1990 package_name = package_name[:-1]

1991

1992 wnpp_stats[package_name] = {

1993 'wnpp_type': wnpp_type,

1994 'bug_id': bug_id,

1995 }

1996

1997 return wnpp_stats

1998

1999 def update_action_item(self, package, stats):

2000 """

2001 Creates an :class:`ActionItem <distro_tracker.core.models.ActionItem>`

2002 instance for the given type indicating that the package has a WNPP

2003 issue.

2004 """

2005 action_item = package.get_action_item_for_type(self.action_item_type)

2006 if not action_item:

2007 action_item = ActionItem(

2008 package=package,

2009 item_type=self.action_item_type)

2010

2011 # Check if the stats have actually been changed

2012 if action_item.extra_data:

2013 if action_item.extra_data.get('wnpp_info', None) == stats:

2014 # Nothing to do -- stll the same data

2015 return

2016

2017 # Update the data since something has changed

2018 try:

2019 release = package.main_entry.repository.suite or \

2020 package.main_entry.repository.codename

2021 except AttributeError:

2022 release = None

2023

2024 msgs = {

2025 'O': "This package has been orphaned and needs a maintainer.",

2026 'ITA': "Someone intends to adopt this package.",

2027 'RFA': "The maintainer wants to pass over package maintenance.",

2028 'RFH': "The maintainer is looking for help with this package.",

2029 'ITP': "Someone is planning to reintroduce this package.",

2030 'RFP': "There is a request to reintroduce this package.",

2031 'RM': "This package has been requested to be removed.",

2032 'RFS': "A sponsor is needed to update this package.",

2033 '?': "The WNPP database contains an entry for this package."

2034 }

2035 wnpp_type = stats['wnpp_type']

2036 try:

2037 wnpp_msg = msgs[wnpp_type]

2038 except KeyError:

2039 wnpp_msg = msgs['?']

2040

2041 action_item.short_description = self.ITEM_DESCRIPTION.format(

2042 url='https://bugs.debian.org/{}'.format(stats['bug_id']),

2043 wnpp_type=wnpp_type, wnpp_msg=wnpp_msg)

2044 action_item.extra_data = {

2045 'wnpp_info': stats,

2046 'release': release,

2047 }

2048 action_item.save()

2049

2050 def update_depneedsmaint_action_item(self, package_needs_maintainer, stats):

2051 short_description_template = \

2052 'Depends on packages which need a new maintainer'

2053 package_needs_maintainer.get_absolute_url()

2054 action_item_type = ActionItemType.objects.create_or_update(

2055 type_name='debian-depneedsmaint',

2056 full_description_template='debian/depneedsmaint-action-item.html')

2057 dependencies = SourcePackageDeps.objects.filter(

2058 dependency=package_needs_maintainer)

2059 for dependency in dependencies: 2059 ↛ 2060line 2059 didn't jump to line 2060, because the loop on line 2059 never started

2060 package = dependency.source

2061 action_item = package.get_action_item_for_type(action_item_type)

2062 if not action_item:

2063 action_item = ActionItem(

2064 package=package,

2065 item_type=action_item_type,

2066 extra_data={})

2067

2068 pkgdata = {

2069 'bug': stats['bug_id'],

2070 'details': dependency.details,

2071 }

2072

2073 if (action_item.extra_data.get(package_needs_maintainer.name, {}) ==

2074 pkgdata):

2075 # Nothing has changed

2076 continue

2077

2078 action_item.short_description = short_description_template

2079 action_item.extra_data[package_needs_maintainer.name] = pkgdata

2080

2081 action_item.save()

2082

2083 @transaction.atomic

2084 def execute_main(self):

2085 wnpp_stats = self.get_wnpp_stats()

2086 if wnpp_stats is None: 2086 ↛ 2088line 2086 didn't jump to line 2088, because the condition on line 2086 was never true

2087 # Nothing to do: cached content up to date

2088 return

2089

2090 ActionItem.objects.delete_obsolete_items(

2091 item_types=[self.action_item_type],

2092 non_obsolete_packages=wnpp_stats.keys())

2093 # Remove obsolete action items for packages whose dependencies need a

2094 # new maintainer.

2095 packages_need_maintainer = []

2096 for name, stats in wnpp_stats.items():

2097 if stats['wnpp_type'] in ('O', 'RFA'):

2098 packages_need_maintainer.append(name)

2099 packages_depneeds_maint = [

2100 package.name for package in SourcePackageName.objects.filter(

2101 source_dependencies__dependency__name__in=packages_need_maintainer) # noqa

2102 ]

2103 ActionItem.objects.delete_obsolete_items(

2104 item_types=[

2105 ActionItemType.objects.get_or_create(

2106 type_name='debian-depneedsmaint')[0],

2107 ],

2108 non_obsolete_packages=packages_depneeds_maint)

2109

2110 # Drop all reverse references

2111 for ai in ActionItem.objects.filter( 2111 ↛ 2113line 2111 didn't jump to line 2113, because the loop on line 2111 never started

2112 item_type__type_name='debian-depneedsmaint'):

2113 ai.extra_data = {}

2114 ai.save()

2115

2116 packages = SourcePackageName.objects.filter(name__in=wnpp_stats.keys())

2117 packages = packages.prefetch_related('action_items')

2118

2119 for package in packages:

2120 stats = wnpp_stats[package.name]

2121 self.update_action_item(package, stats)

2122 # Update action items for packages which depend on this one to

2123 # indicate that a dependency needs a new maintainer.

2124 if package.name in packages_need_maintainer:

2125 self.update_depneedsmaint_action_item(package, stats)

2126

2127

2128class UpdateNewQueuePackages(BaseTask):

2129 """

2130 Updates the versions of source packages found in the NEW queue.

2131 """

2132

2133 class Scheduler(IntervalScheduler):

2134 interval = 3600

2135

2136 DATA_KEY = 'debian-new-queue-info'

2137

2138 def initialize(self, *args, **kwargs):

2139 super(UpdateNewQueuePackages, self).initialize(*args, **kwargs)

2140

2141 def extract_package_info(self, content):

2142 """

2143 Extracts the package information from the content of the NEW queue.

2144

2145 :returns: A dict mapping package names to a dict mapping the

2146 distribution name in which the package is found to the version

2147 information for the most recent version of the package in the dist.

2148 """

2149 packages = {}

2150 for stanza in deb822.Deb822.iter_paragraphs(content.splitlines()):

2151 necessary_fields = ('Source', 'Queue', 'Version', 'Distribution')

2152 if not all(field in stanza for field in necessary_fields):

2153 continue

2154 if stanza['Queue'] != 'new': 2154 ↛ 2155line 2154 didn't jump to line 2155, because the condition on line 2154 was never true

2155 continue

2156

2157 versions = stanza['Version'].split()

2158 # Save only the most recent version

2159 version = max(versions, key=lambda x: AptPkgVersion(x))

2160

2161 package_name = stanza['Source']

2162 pkginfo = packages.setdefault(package_name, {})

2163 distribution = stanza['Distribution']

2164 if distribution in pkginfo:

2165 current_version = pkginfo[distribution]['version']

2166 if debian_support.version_compare(version, current_version) < 0:

2167 # The already saved version is more recent than this one.

2168 continue

2169

2170 pkginfo[distribution] = {

2171 'version': version,

2172 }

2173

2174 return packages

2175

2176 def _get_new_content(self):

2177 url = 'https://ftp-master.debian.org/new.822'

2178 return get_resource_text(url, force_update=self.force_update,

2179 only_if_updated=True)

2180

2181 def execute_main(self):

2182 content = self._get_new_content()

2183 if content is None: 2183 ↛ 2184line 2183 didn't jump to line 2184, because the condition on line 2183 was never true

2184 return

2185

2186 all_package_info = self.extract_package_info(content)

2187

2188 packages = SourcePackageName.objects.filter(

2189 name__in=all_package_info.keys())

2190

2191 with transaction.atomic():

2192 # Drop old entries

2193 PackageData.objects.filter(key=self.DATA_KEY).delete()

2194 # Prepare current entries

2195 data = []

2196 for package in packages:

2197 new_queue_info = PackageData(

2198 key=self.DATA_KEY,

2199 package=package,

2200 value=all_package_info[package.name])

2201 data.append(new_queue_info)

2202 # Bulk create them

2203 PackageData.objects.bulk_create(data)

2204

2205

2206class UpdateAutoRemovalsStatsTask(BaseTask):

2207 """

2208 A task for updating autoremovals information on all packages.

2209 """

2210

2211 class Scheduler(IntervalScheduler):

2212 interval = 3600

2213

2214 ACTION_ITEM_TYPE_NAME = 'debian-autoremoval'

2215 ACTION_ITEM_TEMPLATE = 'debian/autoremoval-action-item.html'

2216 ITEM_DESCRIPTION = ('Marked for autoremoval on {removal_date}' +

2217 '{dependencies}: {bugs}')

2218

2219 def initialize(self, *args, **kwargs):

2220 super(UpdateAutoRemovalsStatsTask, self).initialize(*args, **kwargs)

2221 self.action_item_type = ActionItemType.objects.create_or_update(

2222 type_name=self.ACTION_ITEM_TYPE_NAME,

2223 full_description_template=self.ACTION_ITEM_TEMPLATE)

2224

2225 def get_autoremovals_stats(self):

2226 """

2227 Retrieves and parses the autoremoval stats for all packages.

2228 Autoremoval stats include the BTS bugs id.

2229

2230 :returns: A dict mapping package names to autoremoval stats.

2231 """

2232 content = get_resource_text(

2233 'https://udd.debian.org/cgi-bin/autoremovals.yaml.cgi',

2234 force_update=self.force_update,

2235 only_if_updated=True

2236 )

2237 if content: 2237 ↛ exitline 2237 didn't return from function 'get_autoremovals_stats', because the condition on line 2237 was never false

2238 return yaml.safe_load(content)

2239

2240 def update_action_item(self, package, stats):

2241 """

2242 Creates an :class:`ActionItem <distro_tracker.core.models.ActionItem>`

2243 instance for the given type indicating that the package has an

2244 autoremoval issue.

2245 """

2246 action_item = package.get_action_item_for_type(self.action_item_type)

2247 if not action_item: 2247 ↛ 2253line 2247 didn't jump to line 2253, because the condition on line 2247 was never false

2248 action_item = ActionItem(

2249 package=package,

2250 item_type=self.action_item_type,

2251 severity=ActionItem.SEVERITY_HIGH)

2252

2253 bugs_dependencies = stats.get('bugs_dependencies', [])

2254 buggy_dependencies = stats.get('buggy_dependencies', [])

2255 reverse_dependencies = stats.get('rdeps', [])

2256 all_bugs = stats['bugs'] + bugs_dependencies

2257 link = '<a href="https://bugs.debian.org/{}">#{}</a>'

2258 removal_date = stats['removal_date'].strftime('%d %B')

2259 if isinstance(removal_date, bytes): 2259 ↛ 2260line 2259 didn't jump to line 2260, because the condition on line 2259 was never true

2260 removal_date = removal_date.decode('utf-8', 'ignore')

2261

2262 action_item.short_description = self.ITEM_DESCRIPTION.format(

2263 removal_date=removal_date,

2264 dependencies=(' due to ' + html_package_list(

2265 buggy_dependencies) if buggy_dependencies else ''),

2266 bugs=', '.join(link.format(bug, bug) for bug in all_bugs))

2267

2268 # datetime objects are not JSON-serializable, convert them ourselves

2269 for key in stats.keys():

2270 if hasattr(stats[key], 'strftime'):

2271 stats[key] = stats[key].strftime('%a %d %b %Y')

2272

2273 action_item.extra_data = {

2274 'stats': stats,

2275 'removal_date': stats['removal_date'],

2276 'version': stats.get('version', ''),

2277 'bugs': ', '.join(link.format(bug, bug) for bug in stats['bugs']),

2278 'bugs_dependencies': ', '.join(

2279 link.format(bug, bug) for bug in bugs_dependencies),

2280 'buggy_dependencies':

2281 html_package_list(buggy_dependencies),

2282 'reverse_dependencies':

2283 html_package_list(reverse_dependencies),

2284 'number_rdeps': len(reverse_dependencies)}

2285 action_item.save()

2286

2287 def execute_main(self):

2288 autoremovals_stats = self.get_autoremovals_stats()

2289 if autoremovals_stats is None: 2289 ↛ 2291line 2289 didn't jump to line 2291, because the condition on line 2289 was never true

2290 # Nothing to do: cached content up to date

2291 return

2292

2293 ActionItem.objects.delete_obsolete_items(

2294 item_types=[self.action_item_type],

2295 non_obsolete_packages=autoremovals_stats.keys())

2296

2297 packages = SourcePackageName.objects.filter(

2298 name__in=autoremovals_stats.keys())

2299 packages = packages.prefetch_related('action_items')

2300

2301 for package in packages:

2302 self.update_action_item(package, autoremovals_stats[package.name])

2303

2304

2305class UpdatePackageScreenshotsTask(BaseTask):

2306 """

2307 Check if a screenshot exists on screenshots.debian.net, and add a

2308 key to PackageData if it does.

2309 """

2310

2311 class Scheduler(IntervalScheduler):

2312 interval = 3600 * 24

2313

2314 DATA_KEY = 'screenshots'

2315

2316 def _get_screenshots(self):

2317 url = 'https://screenshots.debian.net/json/packages'

2318 content = get_resource_text(url, force_update=self.force_update,

2319 only_if_updated=True)

2320 if content is None: 2320 ↛ 2321line 2320 didn't jump to line 2321, because the condition on line 2320 was never true

2321 return

2322

2323 data = json.loads(content)

2324 return data

2325

2326 def execute_main(self):

2327 content = self._get_screenshots()

2328 if content is None: 2328 ↛ 2329line 2328 didn't jump to line 2329, because the condition on line 2328 was never true

2329 return

2330

2331 packages_with_screenshots = []

2332 for item in content['packages']:

2333 try:

2334 package = SourcePackageName.objects.get(name=item['name'])

2335 packages_with_screenshots.append(package)

2336 except SourcePackageName.DoesNotExist:

2337 pass

2338

2339 with transaction.atomic():

2340 PackageData.objects.filter(key='screenshots').delete()

2341

2342 data = []

2343 for package in packages_with_screenshots:

2344 try:

2345 screenshot_info = package.data.get(key=self.DATA_KEY)

2346 screenshot_info.value['screenshots'] = 'true'

2347 except PackageData.DoesNotExist:

2348 screenshot_info = PackageData(

2349 key=self.DATA_KEY,

2350 package=package,

2351 value={'screenshots': 'true'})

2352

2353 data.append(screenshot_info)

2354

2355 PackageData.objects.bulk_create(data)

2356

2357

2358class UpdateBuildReproducibilityTask(BaseTask):

2359

2360 class Scheduler(IntervalScheduler):

2361 interval = 3600 * 6

2362

2363 BASE_URL = 'https://tests.reproducible-builds.org'

2364 ACTION_ITEM_TYPE_NAME = 'debian-build-reproducibility'

2365 ACTION_ITEM_TEMPLATE = 'debian/build-reproducibility-action-item.html'

2366 ITEM_DESCRIPTION = {

2367 'blacklisted': '<a href="{url}">Blacklisted</a> from build '

2368 'reproducibility testing',

2369 'FTBFS': '<a href="{url}">Fails to build</a> during reproducibility '

2370 'testing',

2371 'reproducible': None,

2372 'FTBR': '<a href="{url}">Does not build reproducibly</a> '

2373 'during testing',

2374 '404': None,

2375 'not for us': None,

2376 }

2377

2378 def initialize(self, *args, **kwargs):

2379 super(UpdateBuildReproducibilityTask, self).initialize(*args, **kwargs)

2380 self.action_item_type = ActionItemType.objects.create_or_update(

2381 type_name=self.ACTION_ITEM_TYPE_NAME,

2382 full_description_template=self.ACTION_ITEM_TEMPLATE)

2383

2384 def get_build_reproducibility(self):

2385 url = '{}/debian/reproducible-tracker.json'.format(self.BASE_URL)

2386 content = get_resource_text(url, force_update=self.force_update,

2387 only_if_updated=True)

2388 if content is None: 2388 ↛ 2389line 2388 didn't jump to line 2389, because the condition on line 2388 was never true

2389 return

2390

2391 reproducibilities = json.loads(content)

2392 packages = {}

2393 for item in reproducibilities:

2394 package = item['package']

2395 status = item['status']

2396 missing = package not in packages

2397 important = self.ITEM_DESCRIPTION.get(status) is not None

2398 if important or missing: 2398 ↛ 2393line 2398 didn't jump to line 2393, because the condition on line 2398 was never false

2399 packages[package] = status

2400

2401 return packages

2402

2403 def update_action_item(self, package, status):

2404 description = self.ITEM_DESCRIPTION.get(status)

2405

2406 if not description: # Not worth an action item

2407 return False

2408

2409 action_item = package.get_action_item_for_type(

2410 self.action_item_type.type_name)

2411 if action_item is None: 2411 ↛ 2417line 2411 didn't jump to line 2417, because the condition on line 2411 was never false

2412 action_item = ActionItem(

2413 package=package,

2414 item_type=self.action_item_type,

2415 severity=ActionItem.SEVERITY_NORMAL)

2416

2417 url = "{}/debian/rb-pkg/{}.html".format(self.BASE_URL, package.name)

2418 action_item.short_description = description.format(url=url)

2419 action_item.save()

2420 return True

2421

2422 def execute_main(self):

2423 reproducibilities = self.get_build_reproducibility()

2424 if reproducibilities is None: 2424 ↛ 2425line 2424 didn't jump to line 2425, because the condition on line 2424 was never true

2425 return

2426

2427 with transaction.atomic():

2428 PackageData.objects.filter(key='reproducibility').delete()

2429

2430 packages = []

2431 data = []

2432

2433 for name, status in reproducibilities.items():

2434 try:

2435 package = SourcePackageName.objects.get(name=name)

2436 if self.update_action_item(package, status):

2437 packages.append(package)

2438 except SourcePackageName.DoesNotExist:

2439 continue

2440

2441 reproducibility_info = PackageData(

2442 key='reproducibility',

2443 package=package,

2444 value={'reproducibility': status})

2445 data.append(reproducibility_info)

2446

2447 ActionItem.objects.delete_obsolete_items([self.action_item_type],

2448 packages)

2449 PackageData.objects.bulk_create(data)

2450

2451

2452class MultiArchHintsTask(BaseTask):

2453

2454 class Scheduler(IntervalScheduler):

2455 interval = 3600 * 6

2456

2457 ACTIONS_WEB = 'https://wiki.debian.org/MultiArch/Hints'

2458 ACTIONS_URL = 'https://dedup.debian.net/static/multiarch-hints.yaml'

2459 ACTION_ITEM_TYPE_NAME = 'debian-multiarch-hints'

2460 ACTION_ITEM_TEMPLATE = 'debian/multiarch-hints.html'

2461 ACTION_ITEM_DESCRIPTION = \

2462 '<a href="{link}">Multiarch hinter</a> reports {count} issue(s)'

2463

2464 def initialize(self, *args, **kwargs):

2465 super(MultiArchHintsTask, self).initialize(*args, **kwargs)

2466 self.action_item_type = ActionItemType.objects.create_or_update(

2467 type_name=self.ACTION_ITEM_TYPE_NAME,

2468 full_description_template=self.ACTION_ITEM_TEMPLATE)

2469 self.SEVERITIES = {}

2470 for value, name in ActionItem.SEVERITIES:

2471 self.SEVERITIES[name] = value

2472

2473 def get_data(self):

2474 data = get_resource_text(self.ACTIONS_URL)

2475 if data:

2476 return yaml.safe_load(data)

2477

2478 def get_packages(self):

2479 data = self.get_data()

2480 if data is None:

2481 return

2482 if data['format'] != 'multiarch-hints-1.0':

2483 return None

2484 data = data['hints']

2485 packages = collections.defaultdict(dict)

2486 for item in data:

2487 if 'source' not in item:

2488 continue

2489 package = item['source']

2490 wishlist = ActionItem.SEVERITY_WISHLIST

2491 severity = self.SEVERITIES.get(item['severity'], wishlist)

2492 pkg_severity = packages[package].get('severity', wishlist)

2493 packages[package]['severity'] = max(severity, pkg_severity)

2494 packages[package].setdefault('hints', []).append(

2495 (item['description'], item['link']))

2496 return packages

2497

2498 def update_action_item(self, package, severity, description, extra_data):

2499 action_item = package.get_action_item_for_type(

2500 self.action_item_type.type_name)

2501 if action_item is None:

2502 action_item = ActionItem(

2503 package=package,

2504 item_type=self.action_item_type)

2505 action_item.severity = severity

2506 action_item.short_description = description

2507 action_item.extra_data = extra_data

2508 action_item.save()

2509

2510 def execute_main(self):

2511 packages = self.get_packages()

2512 if not packages:

2513 return

2514

2515 with transaction.atomic():

2516 for name, data in packages.items():

2517 try:

2518 package = SourcePackageName.objects.get(name=name)

2519 except SourcePackageName.DoesNotExist:

2520 continue

2521

2522 description = self.ACTION_ITEM_DESCRIPTION.format(

2523 count=len(data['hints']), link=self.ACTIONS_WEB)

2524 self.update_action_item(package, data['severity'], description,

2525 data['hints'])

2526

2527 ActionItem.objects.delete_obsolete_items([self.action_item_type],

2528 packages.keys())

2529

2530

2531class UpdateVcsWatchTask(BaseTask):

2532 """

2533 Updates packages' vcswatch stats.

2534 """

2535

2536 class Scheduler(IntervalScheduler):

2537 interval = 3600

2538

2539 VCSWATCH_ACTION_ITEM_TYPE_NAME = 'vcswatch-warnings-and-errors'

2540 VCSWATCH_ACTION_ITEM_TEMPLATE = 'debian/vcswatch-action-item.html'

2541 SALSA_MR_ACTION_ITEM_TYPE_NAME = 'salsa-merge-requests'

2542 SALSA_MR_ACTION_ITEM_TEMPLATE = 'debian/salsa-mr-action-item.html'

2543 SALSA_MR_SHORT_DESCRIPTION = '<a href="{url}">{count_str}</a> in Salsa'

2544 VCSWATCH_URL = 'https://qa.debian.org/cgi-bin/vcswatch?package=%(package)s'

2545 VCSWATCH_DATA_URL = 'https://qa.debian.org/data/vcswatch/vcswatch.json.gz'

2546

2547 VCSWATCH_STATUS_DICT = {

2548 "NEW": {

2549 "description":

2550 '<a href="{vcswatch_url}">version in VCS is newer</a> than in '

2551 'repository, is it time to upload?',

2552 "severity": ActionItem.SEVERITY_NORMAL,

2553 },

2554 "COMMITS": {

2555 "description":

2556 '<a href="{vcswatch_url}">{commits} new commit{commits_s}</a> '

2557 'since last upload, is it time to release?',

2558 "severity": ActionItem.SEVERITY_NORMAL,

2559 },

2560 "OLD": {

2561 'description':

2562 'The <a href="{vcswatch_url}">VCS repository is not up to '

2563 'date</a>, push the missing commits.',

2564 "severity": ActionItem.SEVERITY_HIGH,

2565 },

2566 "UNREL": {

2567 "description":

2568 'The <a href="{vcswatch_url}">VCS repository is not up to '

2569 'date</a>, push the missing commits.',

2570 "severity": ActionItem.SEVERITY_HIGH,

2571 },

2572 "ERROR": {

2573 "description":

2574 '<a href="{vcswatch_url}">Failed to analyze the VCS '

2575 'repository</a>. Please troubleshoot and fix the issue.',

2576 "severity": ActionItem.SEVERITY_HIGH,

2577 },

2578 "DEFAULT": {

2579 "description":

2580 '<a href="{url}">Unexpected status</a> ({status}) reported by '

2581 'VcsWatch.',

2582 "severity": ActionItem.SEVERITY_HIGH,

2583 },

2584 }

2585

2586 def initialize(self, *args, **kwargs):

2587 super(UpdateVcsWatchTask, self).initialize(*args, **kwargs)

2588 self.vcswatch_ai_type = ActionItemType.objects.create_or_update(

2589 type_name=self.VCSWATCH_ACTION_ITEM_TYPE_NAME,

2590 full_description_template=self.VCSWATCH_ACTION_ITEM_TEMPLATE

2591 )

2592 self.salsa_mr_ai_type = ActionItemType.objects.create_or_update(

2593 type_name=self.SALSA_MR_ACTION_ITEM_TYPE_NAME,

2594 full_description_template=self.SALSA_MR_ACTION_ITEM_TEMPLATE

2595 )

2596

2597 def get_vcswatch_data(self):

2598 text = get_resource_text(self.VCSWATCH_DATA_URL)

2599

2600 if text is None: 2600 ↛ 2601line 2600 didn't jump to line 2601, because the condition on line 2600 was never true

2601 return

2602

2603 # There's some text, let's load!

2604 data = json.loads(text)

2605

2606 out = {}

2607 # This allows to save a lot of list search later.

2608 for entry in data:

2609 out[entry[u'package']] = entry

2610

2611 return out

2612

2613 def clean_package_info(self, package_infos_without_watch, todo):

2614 """Takes a list of :class:`PackageData` which do not

2615 have a watch entry and cleans it. Then schedule in todo what

2616 to do with them.

2617 """

2618 for package_info in package_infos_without_watch:

2619 if 'QA' in package_info.value: 2619 ↛ 2618line 2619 didn't jump to line 2618, because the condition on line 2619 was never false

2620 package_info.value.pop('QA')

2621 if (list(package_info.value.keys()) == ['checksum'] or

2622 not package_info.value.keys()):

2623 todo['drop']['package_infos'].append(package_info)

2624 else:

2625 package_info.value['checksum'] = get_data_checksum(

2626 package_info.value

2627 )

2628 todo['update']['package_infos'].append(package_info)

2629

2630 def update_action_item(self, package, vcswatch_data, action_item, todo):

2631 """

2632 For a given :class:`ActionItem` and a given vcswatch data, updates

2633 properly the todo dict if required.

2634

2635 Returns dependingly on what has been done. If something is to

2636 be updated, returns True, if nothing is to be updated, returns

2637 False. If the calling loop should `continue`, returns `None`.

2638

2639 :rtype: bool or `None`

2640 """

2641

2642 package_status = vcswatch_data['status']

2643

2644 if package_status == "OK":

2645 # Everything is fine, let's purge the action item. Not the

2646 # package extracted info as its QA url is still relevant.

2647 if action_item: 2647 ↛ 2651line 2647 didn't jump to line 2651, because the condition on line 2647 was never false

2648 todo['drop']['action_items'].append(action_item)

2649

2650 # Nothing more to do!

2651 return False

2652

2653 # NOT BEFORE "OK" check!!

2654 if package_status not in self.VCSWATCH_STATUS_DICT: 2654 ↛ 2655line 2654 didn't jump to line 2655, because the condition on line 2654 was never true

2655 package_status = "DEFAULT"

2656

2657 # If we are here, then something is not OK. Let's check if we

2658 # already had some intel regarding the current package status.

2659 if action_item is None:

2660 action_item = ActionItem(

2661 package=package,

2662 item_type=self.vcswatch_ai_type)

2663 todo['add']['action_items'].append(action_item)

2664 else:

2665 todo['update']['action_items'].append(action_item)

2666

2667 # Computes the watch URL

2668 vcswatch_url = self.VCSWATCH_URL % {'package': package.name}

2669

2670 if action_item.extra_data:

2671 extra_data = action_item.extra_data

2672 else:

2673 extra_data = {}

2674

2675 # Fetches the long description and severity from

2676 # the VCSWATCH_STATUS_DICT dict.

2677 action_item.severity = \

2678 self.VCSWATCH_STATUS_DICT[package_status]['severity']

2679

2680 nb_commits = int(vcswatch_data["commits"] or 0)

2681

2682 # The new data

2683 new_extra_data = {

2684 'vcswatch_url': vcswatch_url,

2685 }

2686 new_extra_data.update(vcswatch_data)

2687

2688 extra_data_match = all([

2689 new_extra_data[key] == extra_data.get(key, None)

2690 for key in new_extra_data

2691 ])

2692

2693 # If everything is fine and we are not forcing the update

2694 # then we proceed to the next package.

2695 if extra_data_match and not self.force_update: 2695 ↛ 2697line 2695 didn't jump to line 2697, because the condition on line 2695 was never true

2696 # Remove from the todolist

2697 todo['update']['action_items'].remove(action_item)

2698 return False

2699 else:

2700 # Report for short description of the :class:`ActionItem`

2701 desc = self.VCSWATCH_STATUS_DICT[package_status]['description']

2702 commits_s = 's' if nb_commits != 1 else ''

2703 action_item.short_description = \

2704 desc.format(commits_s=commits_s, **new_extra_data)

2705 action_item.extra_data = new_extra_data

2706 return True

2707

2708 def update_package_info(self, package, vcswatch_data, package_info, key,

2709 todo):

2710 # Same thing with PackageData

2711 if package_info is None:

2712 package_info = PackageData(

2713 package=package,

2714 key=key,

2715 )

2716 todo['add']['package_infos'].append(package_info)

2717 else:

2718 todo['update']['package_infos'].append(package_info)

2719

2720 # Computes the watch URL

2721 vcswatch_url = self.VCSWATCH_URL % {'package': package.name}

2722

2723 new_value = dict(package_info.value)

2724 if key == 'vcs_extra_links':

2725 new_value['QA'] = vcswatch_url

2726 elif key == 'vcswatch': 2726 ↛ 2736line 2726 didn't jump to line 2736, because the condition on line 2726 was never false

2727 if 'package_version' in vcswatch_data: 2727 ↛ 2729line 2727 didn't jump to line 2729, because the condition on line 2727 was never false

2728 new_value['package_version'] = vcswatch_data['package_version']

2729 if 'changelog_version' in vcswatch_data: 2729 ↛ 2732line 2729 didn't jump to line 2732, because the condition on line 2729 was never false

2730 new_value['changelog_version'] = vcswatch_data[

2731 'changelog_version']

2732 if 'changelog_distribution' in vcswatch_data: 2732 ↛ 2736line 2732 didn't jump to line 2736, because the condition on line 2732 was never false

2733 new_value['changelog_distribution'] = vcswatch_data[

2734 'changelog_distribution']

2735

2736 new_value['checksum'] = get_data_checksum(new_value)

2737

2738 package_info_match = (

2739 new_value['checksum'] == package_info.value.get('checksum', None)

2740 )

2741

2742 if package_info_match and not self.force_update:

2743 todo['update']['package_infos'].remove(package_info)

2744 return False

2745 else:

2746 package_info.value = new_value

2747 return True

2748

2749 def update_packages_item(self, packages, vcswatch_datas):

2750 """Generates the lists of :class:`ActionItem` to be added,

2751 deleted or updated regarding the status of their packages.

2752

2753 Categories of statuses are:

2754 {u'COMMITS', u'ERROR', u'NEW', u'OK', u'OLD', u'UNREL'}

2755

2756 Basically, it fetches all info from :class:`PackageData`

2757 with key='vcs', the ones without data matching vcswatch_datas are

2758 stored in one variable that's iterated through directly, and if

2759 there was something before, it is purged. Then, all entries in

2760 that queryset that have no relevant intel anymore are scheduled

2761 to be deleted. The others are only updated.

2762

2763 All :class:`PackageData` matching vcswatch_datas

2764 are stored in another variable. The same is done with the list of

2765 :class:`ActionItem` that match this task type.

2766

2767 Then, it iterates on all vcswatch_datas' packages and it tries to

2768 determine if there are any news, if so, it updates apopriately the

2769 prospective :class:`ActionItem` and :class:`PackageData`,

2770 and schedule them to be updated. If no data was existent, then

2771 it creates them and schedule them to be added to the database.

2772

2773 At the end, this function returns a dict of all instances of

2774 :class:`ActionItem` and :class:`PackageData` stored

2775 in subdicts depending on their class and what is to be done

2776 with them.

2777

2778 :rtype: dict

2779

2780 """

2781

2782 todo = {

2783 'drop': {

2784 'action_items': [],

2785 'package_infos': [],

2786 },

2787 'update': {

2788 'action_items': [],

2789 'package_infos': [],

2790 },

2791 'add': {

2792 'action_items': [],

2793 'package_infos': [],

2794 },

2795 }

2796

2797 package_info_keys = ['vcs_extra_links', 'vcswatch']

2798 package_infos = {}

2799 for key in package_info_keys:

2800 # Fetches all PackageData with a given key for packages having

2801 # a vcswatch key. As the pair (package, key) is unique, there is a

2802 # bijection between these data, and we fetch them classifying them

2803 # by package name.

2804 for package_info in PackageData.objects.select_related(

2805 'package').filter(key=key).only('package__name', 'value'):

2806 if package_info.package.name not in package_infos:

2807 package_infos[package_info.package.name] = {}

2808 package_infos[package_info.package.name][key] = package_info

2809

2810 # As :class:`PackageData` key=vcs_extra_links is shared, we

2811 # have to clean up those with vcs watch_url that aren't in vcs_data

2812 package_infos_without_watch = PackageData.objects.filter(

2813 key='vcs_extra_links').exclude(

2814 package__name__in=vcswatch_datas.keys()).only('value')

2815

2816 # Do the actual clean.

2817 self.clean_package_info(package_infos_without_watch, todo)

2818

2819 # Fetches all :class:`ActionItem` for packages concerned by a vcswatch

2820 # action.

2821 action_items = {

2822 action_item.package.name: action_item

2823 for action_item in ActionItem.objects.select_related(

2824 'package'

2825 ).filter(item_type=self.vcswatch_ai_type)

2826 }

2827

2828 for package in packages:

2829 # Get the vcswatch_data from the whole vcswatch_datas

2830 vcswatch_data = vcswatch_datas[package.name]

2831

2832 # Get the old action item for this warning, if it exists.

2833 action_item = action_items.get(package.name, None)

2834

2835 # Updates the :class:`ActionItem`. If _continue is None,

2836 # then there is nothing more to do with this package.

2837 # If it is False, then no update is pending for the

2838 # :class:`ActionItem`, else there is an update

2839 # to do.

2840 _ai_continue = self.update_action_item(

2841 package,

2842 vcswatch_data,

2843 action_item,

2844 todo)

2845

2846 _pi_continue = False

2847 for key in package_info_keys:

2848 try:

2849 package_info = package_infos[package.name][key]

2850 except KeyError:

2851 package_info = None

2852

2853 _pi_continue |= self.update_package_info(

2854 package,

2855 vcswatch_data,

2856 package_info,

2857 key,

2858 todo)

2859

2860 if not _ai_continue and not _pi_continue:

2861 continue

2862

2863 return todo

2864

2865 def update_action_item_for_salsa_mrs(self, package, vcswatch_data,

2866 action_item, todo):

2867 """

2868 For a given :class:`ActionItem` and a given vcswatch data, updates

2869 properly the todo dict if required.

2870

2871 :rtype: `None`

2872 """

2873

2874 try:

2875 parsed_url = urllib.parse.urlparse(vcswatch_data['url'])

2876 except (KeyError, ValueError):

2877 is_salsa = False

2878 else:

2879 is_salsa = parsed_url.netloc == 'salsa.debian.org'

2880 merge_requests = vcswatch_data.get('merge_requests', 0)

2881 need_ai = is_salsa and merge_requests

2882

2883 if not need_ai:

2884 # There are no open Salsa MRs (or we can't check) so remove the AI

2885 if action_item:

2886 todo['drop'].append(action_item)

2887

2888 # Nothing more to do!

2889 return

2890

2891 if action_item is None:

2892 action_item = ActionItem(

2893 package=package,

2894 item_type=self.salsa_mr_ai_type)

2895 todo['add'].append(action_item)

2896 else:

2897 todo['update'].append(action_item)

2898

2899 # Computes the Salsa MR URL

2900 salsa_path = parsed_url.path.split(' ')[0]

2901 if salsa_path.endswith('.git'): 2901 ↛ 2903line 2901 didn't jump to line 2903, because the condition on line 2901 was never false

2902 salsa_path = salsa_path[:-4]

2903 url = 'https://salsa.debian.org{}/-/merge_requests'.format(salsa_path)

2904

2905 if action_item.extra_data:

2906 extra_data = action_item.extra_data

2907 else:

2908 extra_data = {}

2909

2910 new_extra_data = {

2911 'count': merge_requests,

2912 'url': url,

2913 }

2914

2915 extra_data_match = all([

2916 new_extra_data[key] == extra_data.get(key, None)

2917 for key in new_extra_data

2918 ])

2919

2920 # If everything is fine and we are not forcing the update

2921 # then we proceed to the next package.

2922 if extra_data_match and not self.force_update: 2922 ↛ 2924line 2922 didn't jump to line 2924, because the condition on line 2922 was never true

2923 # Remove from the todolist

2924 todo['update'].remove(action_item)

2925 else:

2926 # Report for short description of the :class:`ActionItem`

2927 count_str = '{} open merge request{}'.format(

2928 merge_requests,

2929 's' if merge_requests != 1 else '')

2930 action_item.short_description = \

2931 self.SALSA_MR_SHORT_DESCRIPTION.format(

2932 count_str=count_str,

2933 **new_extra_data)

2934 action_item.severity = ActionItem.SEVERITY_NORMAL

2935 action_item.extra_data = new_extra_data

2936

2937 def update_packages_item_for_salsa_mrs(self, packages, vcswatch_datas):

2938 """Generates the lists of :class:`ActionItem` to be added,

2939 deleted or updated regarding open Salsa MRs for their packages.

2940

2941 At the end, this function returns a dict of all instances of

2942 :class:`ActionItem` stored in subdicts depending on their class

2943 and what is to be done with them.

2944

2945 :rtype: dict

2946

2947 """

2948

2949 todo = {

2950 'drop': [],

2951 'update': [],

2952 'add': [],

2953 }

2954

2955 # Fetches all :class:`ActionItem` for packages concerned by a salsa mr

2956 # action.

2957 action_items = {

2958 action_item.package.name: action_item

2959 for action_item in ActionItem.objects.select_related(

2960 'package'

2961 ).filter(item_type=self.salsa_mr_ai_type)

2962 }

2963

2964 for package in packages:

2965 # Get the vcswatch_data from the whole vcswatch_datas

2966 vcswatch_data = vcswatch_datas[package.name]

2967

2968 # Get the old action item for this warning, if it exists.

2969 action_item = action_items.get(package.name, None)

2970

2971 # Updates the :class:`ActionItem`.

2972 self.update_action_item_for_salsa_mrs(

2973 package,

2974 vcswatch_data,

2975 action_item,

2976 todo)

2977

2978 return todo

2979

2980 def execute_main(self):

2981 # Get the actual vcswatch json file from qa.debian.org

2982 vcs_data = self.get_vcswatch_data()

2983

2984 # Only fetch the packages that are in the json dict.

2985 packages = PackageName.objects.filter(name__in=vcs_data.keys())

2986

2987 # Faster than fetching the action items one by one in a loop

2988 # when handling each package.

2989 packages.prefetch_related('action_items')

2990

2991 # Determine wether something is to be kept or dropped.

2992 todo = self.update_packages_item(packages, vcs_data)

2993 todo_salsa_mrs = self.update_packages_item_for_salsa_mrs(

2994 packages,

2995 vcs_data)

2996

2997 with transaction.atomic():

2998 # Delete the :class:`ActionItem` that are osbolete, and also

2999 # the :class:`PackageData` of the same.

3000 ActionItem.objects.delete_obsolete_items(

3001 [self.vcswatch_ai_type, self.salsa_mr_ai_type],

3002 vcs_data.keys())

3003 PackageData.objects.filter(

3004 key='vcs_extra_links',

3005 id__in=[

3006 package_info.id

3007 for package_info in todo['drop']['package_infos']

3008 ]

3009 ).delete()

3010

3011 # Then delete the :class:`ActionItem` that are to be deleted.

3012 ActionItem.objects.filter(

3013 item_type__type_name=self.vcswatch_ai_type.type_name,

3014 id__in=[

3015 action_item.id

3016 for action_item in todo['drop']['action_items']

3017 ]

3018 ).delete()

3019 ActionItem.objects.filter(

3020 item_type__type_name=self.salsa_mr_ai_type.type_name,

3021 id__in=[

3022 action_item.id

3023 for action_item in todo_salsa_mrs['drop']

3024 ]

3025 ).delete()

3026

3027 # Then bulk_create the :class:`ActionItem` to add and the

3028 # :class:`PackageData`

3029 ActionItem.objects.bulk_create(todo['add']['action_items'])

3030 PackageData.objects.bulk_create(todo['add']['package_infos'])

3031 ActionItem.objects.bulk_create(todo_salsa_mrs['add'])

3032

3033 # Update existing entries

3034 for action_item in todo['update']['action_items']:

3035 action_item.save()

3036 for package_info in todo['update']['package_infos']:

3037 package_info.save()

3038 for action_item in todo_salsa_mrs['update']:

3039 action_item.save()

3040

3041

3042class TagPackagesWithRcBugs(BaseTask, PackageTagging):

3043 """

3044 Performs an update of 'rc-bugs' tag for packages.

3045 """

3046

3047 class Scheduler(IntervalScheduler):

3048 interval = 3600

3049

3050 TAG_NAME = 'tag:rc-bugs'

3051 TAG_DISPLAY_NAME = 'rc bugs'

3052 TAG_COLOR_TYPE = 'danger'

3053 TAG_DESCRIPTION = 'The package has Release Critical bugs'

3054 TAG_TABLE_TITLE = 'Packages with RC bugs'

3055

3056 def packages_to_tag(self):

3057 all_bug_stats = PackageBugStats.objects.prefetch_related('package')

3058 packages_list = []

3059 for bug_stats in all_bug_stats:

3060 categories = bug_stats.stats

3061 found = False

3062 for category in categories: 3062 ↛ 3059line 3062 didn't jump to line 3059, because the loop on line 3062 didn't complete

3063 if found:

3064 break

3065 if category['category_name'] == 'rc': 3065 ↛ 3062line 3065 didn't jump to line 3062, because the condition on line 3065 was never false

3066 found = True

3067 if category['bug_count'] > 0:

3068 packages_list.append(bug_stats.package)

3069 return packages_list

3070

3071

3072class TagPackagesWithNewUpstreamVersion(BaseTask, PackageTagging):

3073 """

3074 Performs an update of 'new-upstream-version' tag for packages.

3075 """

3076

3077 class Scheduler(IntervalScheduler):

3078 interval = 3600 * 3

3079

3080 TAG_NAME = 'tag:new-upstream-version'

3081 TAG_DISPLAY_NAME = 'new upstream version'

3082 TAG_COLOR_TYPE = 'warning'

3083 TAG_DESCRIPTION = 'The upstream has a newer version available'

3084 TAG_TABLE_TITLE = 'Newer upstream version'

3085

3086 def packages_to_tag(self):

3087 try:

3088 action_type = ActionItemType.objects.get(

3089 type_name='new-upstream-version')

3090 except ActionItemType.DoesNotExist:

3091 return []

3092

3093 packages_list = []

3094 items = action_type.action_items.prefetch_related('package')

3095 for item in items:

3096 packages_list.append(item.package)

3097 return packages_list

3098

3099

3100class UpdateDependencySatisfactionTask(BaseTask):

3101 """

3102 Fetches binary package installability results from qa.debian.org/dose

3103 """

3104

3105 class Scheduler(IntervalScheduler):

3106 interval = 3600 * 3

3107

3108 BASE_URL = 'https://qa.debian.org/dose/debcheck/unstable_main/latest'

3109 ACTION_ITEM_TYPE_NAME = 'debian-dependency-satisfaction'

3110 ACTION_ITEM_TEMPLATE = 'debian/dependency-satisfaction-action-item.html'

3111

3112 def __init__(self, force_update=False, *args, **kwargs):

3113 super(UpdateDependencySatisfactionTask, self).__init__(*args, **kwargs)

3114 self.force_update = force_update

3115 self.action_item_type = ActionItemType.objects.create_or_update(

3116 type_name=self.ACTION_ITEM_TYPE_NAME,

3117 full_description_template=self.ACTION_ITEM_TEMPLATE)

3118

3119 def set_parameters(self, parameters):

3120 if 'force_update' in parameters:

3121 self.force_update = parameters['force_update']

3122

3123 def get_dependency_satisfaction(self):

3124 url = '{}/each.txt'.format(self.BASE_URL)

3125 content = get_resource_text(url, force_update=self.force_update,

3126 only_if_updated=True)

3127 if content is None: 3127 ↛ 3128line 3127 didn't jump to line 3128, because the condition on line 3127 was never true

3128 return

3129

3130 dep_sats = collections.defaultdict(set)

3131 for i, line in enumerate(content.splitlines()):

3132 binpkg_name, ver, isnative, anchor, expl, arches = line.split('#')

3133 try:

3134 bin_package = BinaryPackageName.objects.get(name=binpkg_name)

3135 srcpkg_name = bin_package.main_source_package_name

3136 except BinaryPackageName.DoesNotExist:

3137 continue

3138 arches = set([arch.strip() for arch in arches.split()])

3139 # TODO: retrieve this list programmatically, either from

3140 # https://api.ftp-master.debian.org/suite/testing

3141 # or from the Architecture field in the Release file

3142 # for testing (both lists should be equal).

3143 arches = arches.intersection(

3144 {'amd64', 'arm64', 'armel', 'armhf', 'i386', 'loong64', 'mips',

3145 'mips64el', 'mipsel', 'ppc64el', 'riscv64', 's390x'})

3146 # only report problems for release architectures

3147 if not arches:

3148 continue

3149 # if the package is arch:all, only report problems on amd64

3150 if isnative != "True":

3151 arches = arches.intersection({"amd64"})

3152 if not arches:

3153 continue

3154 dep_sats[srcpkg_name].add(

3155 (binpkg_name, ver, tuple(arches), expl, anchor))

3156 # turn sets into lists

3157 dep_sats = dict([(k, list(v)) for k, v in dep_sats.items()])

3158 return dep_sats

3159

3160 def update_action_item(self, package, unsats):

3161 action_item = package.get_action_item_for_type(

3162 self.action_item_type.type_name)

3163 if action_item is None: 3163 ↛ 3168line 3163 didn't jump to line 3168

3164 action_item = ActionItem(

3165 package=package,

3166 item_type=self.action_item_type,

3167 severity=ActionItem.SEVERITY_HIGH)

3168 action_item.short_description = \

3169 "{count} binary package{plural} {have} unsatisfiable " \

3170 "dependencies".format(

3171 count=len(unsats),

3172 plural='' if len(unsats) == 1 else 's',

3173 have='has' if len(unsats) == 1 else 'have',

3174 )

3175 action_item.extra_data = {

3176 'unsats': unsats,

3177 'base_url': '{}/packages/'.format(self.BASE_URL),

3178 }

3179 action_item.save()

3180

3181 def execute(self):

3182 dep_sats = self.get_dependency_satisfaction()

3183 if dep_sats is None: 3183 ↛ 3184line 3183 didn't jump to line 3184, because the condition on line 3183 was never true

3184 return

3185

3186 with transaction.atomic():

3187 PackageData.objects.filter(key='dependency_satisfaction').delete()

3188

3189 packages = []

3190 pkgdata_list = []

3191

3192 for name, unsats in dep_sats.items():

3193 try:

3194 package = SourcePackageName.objects.get(name=name)

3195 packages.append(package)

3196 self.update_action_item(package, unsats)

3197 except SourcePackageName.DoesNotExist:

3198 continue

3199

3200 dep_sat_info = PackageData(

3201 key='dependency_satisfaction',

3202 package=package,

3203 value={'dependency_satisfaction': unsats})

3204 pkgdata_list.append(dep_sat_info)

3205

3206 ActionItem.objects.delete_obsolete_items([self.action_item_type],

3207 packages)

3208 PackageData.objects.bulk_create(pkgdata_list)

3209

3210

3211class UpdateBuildDependencySatisfactionTask(BaseTask):

3212 """

3213 Fetches source package installability results from qa.debian.org/dose

3214 """

3215

3216 class Scheduler(IntervalScheduler):

3217 interval = 3600 * 3

3218

3219 BASE_URL = 'https://qa.debian.org/dose/debcheck/src_unstable_main/latest'

3220 ACTION_ITEM_TYPE_NAME = 'debian-builddependency-satisfaction'

3221 ACTION_ITEM_TEMPLATE = \

3222 'debian/builddependency-satisfaction-action-item.html'

3223

3224 def __init__(self, *args, **kwargs):

3225 super(UpdateBuildDependencySatisfactionTask, self).__init__(*args,

3226 **kwargs)

3227 self.action_item_type = ActionItemType.objects.create_or_update(

3228 type_name=self.ACTION_ITEM_TYPE_NAME,

3229 full_description_template=self.ACTION_ITEM_TEMPLATE)

3230

3231 def get_dependency_satisfaction(self):

3232 url = '{}/each.txt'.format(self.BASE_URL)

3233 content = get_resource_text(url, force_update=self.force_update,

3234 only_if_updated=True)

3235 if content is None: 3235 ↛ 3236line 3235 didn't jump to line 3236, because the condition on line 3235 was never true

3236 return

3237

3238 dep_sats = collections.defaultdict(set)

3239 for i, line in enumerate(content.splitlines()):

3240 srcpkg_name, ver, isnative, anchor, expl, arches = line.split('#')

3241 arches = set([arch.strip() for arch in arches.split()])

3242 # TODO: retrieve this list programmatically, either from

3243 # https://api.ftp-master.debian.org/suite/testing

3244 # or from the Architecture field in the Release file

3245 # for testing (both lists should be equal).

3246 arches = arches.intersection(

3247 {'amd64', 'arm64', 'armel', 'armhf', 'i386', 'loong64', 'mips',

3248 'mips64el', 'mipsel', 'ppc64el', 'riscv64', 's390x'})

3249 # only report problems for release architectures

3250 if not arches:

3251 continue

3252 # if the source package only builds arch:all binary packages, only

3253 # report problems on amd64

3254 if isnative != "True":

3255 arches = arches.intersection({"amd64"})

3256 if not arches:

3257 continue

3258 dep_sats[srcpkg_name].add(

3259 (srcpkg_name, tuple(arches), expl, anchor))

3260 # turn sets into lists

3261 dep_sats = dict([(k, list(v)) for k, v in dep_sats.items()])

3262 return dep_sats

3263

3264 def update_action_item(self, package, unsats):

3265 action_item = package.get_action_item_for_type(

3266 self.action_item_type.type_name)

3267 if action_item is None: 3267 ↛ 3272line 3267 didn't jump to line 3272

3268 action_item = ActionItem(

3269 package=package,

3270 item_type=self.action_item_type,

3271 severity=ActionItem.SEVERITY_HIGH)

3272 action_item.short_description = \

3273 "source package has {count} unsatisfiable " \

3274 "build dependenc{plural}".format(

3275 count=len(unsats),

3276 plural='y' if len(unsats) == 1 else 'ies',

3277 )

3278 action_item.extra_data = {

3279 'unsats': unsats,

3280 'base_url': '{}/packages/'.format(self.BASE_URL),

3281 }

3282 action_item.save()

3283

3284 def execute(self):

3285 dep_sats = self.get_dependency_satisfaction()

3286 if dep_sats is None: 3286 ↛ 3287line 3286 didn't jump to line 3287, because the condition on line 3286 was never true

3287 return

3288

3289 with transaction.atomic():

3290 PackageData.objects.filter(

3291 key='builddependency_satisfaction').delete()

3292

3293 packages = []

3294 pkgdata_list = []

3295

3296 for name, unsats in dep_sats.items():

3297 try:

3298 package = SourcePackageName.objects.get(name=name)

3299 packages.append(package)

3300 self.update_action_item(package, unsats)

3301 except SourcePackageName.DoesNotExist:

3302 continue

3303

3304 dep_sat_info = PackageData(

3305 key='builddependency_satisfaction',

3306 package=package,

3307 value={'builddependency_satisfaction': unsats})

3308 pkgdata_list.append(dep_sat_info)

3309

3310 ActionItem.objects.delete_obsolete_items([self.action_item_type],

3311 packages)

3312 PackageData.objects.bulk_create(pkgdata_list)

3313

3314

3315class UpdateDl10nStatsTask(BaseTask):

3316 """

3317 Updates packages' l10n statistics.

3318 """

3319

3320 class Scheduler(IntervalScheduler):

3321 interval = 3600 * 6

3322

3323 ACTION_ITEM_TYPE_NAME = 'dl10n'

3324 ITEM_DESCRIPTION = \

3325 '<a href="{url}">Issues</a> found with some translations'

3326 ITEM_FULL_DESCRIPTION_TEMPLATE = 'debian/dl10n-action-item.html'

3327

3328 def initialize(self, *args, **kwargs):

3329 super(UpdateDl10nStatsTask, self).initialize(*args, **kwargs)

3330 self.l10n_action_item_type = \

3331 ActionItemType.objects.create_or_update(

3332 type_name=self.ACTION_ITEM_TYPE_NAME,

3333 full_description_template=self.ITEM_FULL_DESCRIPTION_TEMPLATE)

3334

3335 def _load_l10n_stats(self):

3336 url = 'https://i18n.debian.org/l10n-pkg-status/pkglist'

3337 content = get_resource_text(url, force_update=self.force_update,

3338 only_if_updated=True)

3339 if content is None: 3339 ↛ 3340line 3339 didn't jump to line 3340, because the condition on line 3339 was never true

3340 return

3341

3342 def parse_score(score):

3343 if score == '-':

3344 return None

3345 return int(score)

3346

3347 all_stats = {}

3348

3349 # The format of the file is (copied from its header):

3350 # <package> <version> (<comma sperated scores>) <link> <todo>

3351 line_re = re.compile(

3352 r'^([^\s]+) ([^\s]+) $([^)]+)$ ([^\s]+) ([^\s]+)')

3353 for line in content.splitlines():

3354 if not line or line.startswith('#'): 3354 ↛ 3355line 3354 didn't jump to line 3355, because the condition on line 3354 was never true

3355 continue

3356 match = line_re.search(line)

3357 if not match: 3357 ↛ 3358line 3357 didn't jump to line 3358, because the condition on line 3357 was never true

3358 logger.warning('Failed to parse l10n pkglist line: %s', line)

3359 continue

3360

3361 src_pkgname = match.group(1)

3362 try:

3363 scores = match.group(3).split(',')

3364 score_debian = parse_score(scores[0])

3365 score_other = parse_score(scores[1])

3366 # <todo> is a "0" or "1" string, so convert through int to get

3367 # a proper bool

3368 todo = bool(int(match.group(5)))

3369 except (IndexError, ValueError):

3370 logger.warning(

3371 'Failed to parse l10n scores: %s',

3372 line, exc_info=1)

3373 continue

3374 link = match.group(4)

3375 if not score_debian and not score_other: 3375 ↛ 3376line 3375 didn't jump to line 3376, because the condition on line 3375 was never true

3376 continue

3377

3378 all_stats[src_pkgname] = {

3379 'score_debian': score_debian,

3380 'score_other': score_other,

3381 'link': link,

3382 'todo': todo,

3383 }

3384

3385 return all_stats

3386

3387 def update_action_item(self, package, package_stats):

3388 todo = package_stats['todo']

3389

3390 # Get the old action item, if it exists.

3391 l10n_action_item = package.get_action_item_for_type(

3392 self.l10n_action_item_type.type_name)

3393 if not todo:

3394 if l10n_action_item:

3395 # If the item previously existed, delete it now since there

3396 # are no longer any warnings/errors.

3397 l10n_action_item.delete()

3398 return

3399

3400 # The item didn't previously have an action item: create it now

3401 if l10n_action_item is None:

3402 desc = self.ITEM_DESCRIPTION.format(url=package_stats['link'])

3403 l10n_action_item = ActionItem(

3404 package=package,

3405 item_type=self.l10n_action_item_type,

3406 severity=ActionItem.SEVERITY_LOW,

3407 short_description=desc)

3408

3409 if l10n_action_item.extra_data:

3410 old_extra_data = l10n_action_item.extra_data

3411 if old_extra_data == package_stats: 3411 ↛ 3413line 3411 didn't jump to line 3413, because the condition on line 3411 was never true

3412 # No need to update

3413 return

3414

3415 l10n_action_item.extra_data = package_stats

3416

3417 l10n_action_item.save()

3418

3419 def execute_main(self):

3420 stats = self._load_l10n_stats()

3421 if not stats: 3421 ↛ 3422line 3421 didn't jump to line 3422, because the condition on line 3421 was never true

3422 return

3423

3424 with transaction.atomic():

3425 PackageData.objects.filter(key='dl10n').delete()

3426

3427 packages = []

3428 pkgdata_list = []

3429

3430 for name, stat in stats.items():

3431 try:

3432 package = SourcePackageName.objects.get(name=name)

3433 packages.append(package)

3434 self.update_action_item(package, stat)

3435 except SourcePackageName.DoesNotExist:

3436 continue

3437

3438 dl10n_stat = PackageData(

3439 key='dl10n',

3440 package=package,

3441 value=stat)

3442 pkgdata_list.append(dl10n_stat)

3443

3444 ActionItem.objects.delete_obsolete_items(

3445 [self.l10n_action_item_type], packages)

3446 PackageData.objects.bulk_create(pkgdata_list)

3447

3448

3449class UpdateDebianPatchesTask(BaseTask, ImportExternalData):

3450 """

3451 Import statistics about Debian patches from UDD.

3452 """

3453

3454 class Scheduler(IntervalScheduler):

3455 interval = 3600 * 6

3456

3457 data_url = 'https://udd.debian.org/patches.cgi?json=1'

3458 action_item_types = [

3459 {

3460 'type_name': 'debian-patches',

3461 'full_description_template':

3462 'debian/debian-patches-action-item.html',

3463 },

3464 ]

3465

3466 def generate_package_data(self):

3467 pkgdata = {}

3468 for entry in self.external_data:

3469 source = entry.get('source')

3470 if source: 3470 ↛ 3468line 3470 didn't jump to line 3468, because the condition on line 3470 was never false

3471 data = entry.copy()

3472 data['url'] = self._generate_url(entry)

3473 pkgdata[source] = data

3474

3475 return [

3476 ('debian-patches', pkgdata),

3477 ]

3478

3479 @staticmethod

3480 def _generate_url(entry):

3481 query_string = urlencode({

3482 'src': entry.get('source'),

3483 'version': entry.get('version'),

3484 })

3485 return f"https://udd.debian.org/patches.cgi?{query_string}"

3486

3487 def generate_action_items(self):

3488 pkgdata = {}

3489 for entry in self.external_data:

3490 # Skip invalid entries and those without (problematic) patches

3491 source = entry.get('source')

3492 forwarded_invalid = entry.get('forwarded_invalid', 0)

3493 forwarded_no = entry.get('forwarded_no', 0)

3494 if not source: 3494 ↛ 3495line 3494 didn't jump to line 3495, because the condition on line 3494 was never true

3495 continue # Invalid, no source package data

3496 if entry.get('status') != 'patches':

3497 continue # No patch at all

3498 if forwarded_invalid == 0 and forwarded_no == 0:

3499 continue # No problematic patch

3500

3501 # Build the parameterers for the action item

3502 severity = ActionItem.SEVERITY_LOW

3503 desc = ''

3504 url = self._generate_url(entry)

3505

3506 if forwarded_invalid:

3507 severity = ActionItem.SEVERITY_HIGH

3508 count = f"{forwarded_invalid} patch"

3509 if forwarded_invalid > 1:

3510 count += 'es'

3511 count = f'<a href="{url}">{count}</a>'

3512 desc += f"{count} with invalid metadata"

3513

3514 if forwarded_no:

3515 if desc:

3516 desc += ', '

3517 count = f"{forwarded_no} patch"

3518 if forwarded_no > 1:

3519 count += 'es'

3520 count = f'<a href="{url}">{count}</a>'

3521 desc += f"{count} to forward upstream"

3522

3523 extra_data = entry.copy()

3524 extra_data['url'] = url

3525

3526 # Record the action item parameters

3527 pkgdata[source] = {

3528 'short_description': f"debian/patches: {desc}",

3529 'severity': severity,

3530 'extra_data': extra_data,

3531 }

3532

3533 return [

3534 ('debian-patches', pkgdata),

3535 ]

Coverage for distro_tracker/vendor/debian/tracker_tasks.py: 84%

1716 statements