1# Copyright 2013 The Distro Tracker Developers 

2# See the COPYRIGHT file at the top-level directory of this distribution and 

3# at https://deb.li/DTAuthors 

4# 

5# This file is part of Distro Tracker. It is subject to the license terms 

6# in the LICENSE file found in the top-level directory of this 

7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker, 

8# including this file, may be copied, modified, propagated, or distributed 

9# except according to the terms contained in the LICENSE file. 

10""" 

11Implements the Distro Tracker tasks necessary for interesting package source 

12files. 

13""" 

14import logging 

15import os 

16 

17from django.core.files import File 

18 

19from distro_tracker.core.models import ExtractedSourceFile 

20from distro_tracker.core.tasks import BaseTask 

21from distro_tracker.core.tasks.mixins import ProcessSourcePackage 

22from distro_tracker.core.tasks.schedulers import IntervalScheduler 

23from distro_tracker.core.utils.packages import AptCache 

24 

25logger = logging.getLogger('distro_tracker.core.tasks') 

26 

27 

28class ExtractSourcePackageFiles(BaseTask, ProcessSourcePackage): 

29 """ 

30 A task which extracts some files from a new source package version. 

31 The extracted files are: 

32 

33 - debian/changelog 

34 - debian/copyright 

35 - debian/rules 

36 - debian/control 

37 - debian/watch 

38 """ 

39 

40 class Scheduler(IntervalScheduler): 

41 interval = 3600 

42 

43 ALL_FILES_TO_EXTRACT = ( 

44 'changelog', 

45 'copyright', 

46 'rules', 

47 'control', 

48 'watch', 

49 ) 

50 

51 def items_extend_queryset(self, queryset): 

52 return queryset.prefetch_related('extracted_source_files') 

53 

54 def extract_files(self, source_package, files_to_extract=None): 

55 """ 

56 Extract files for just the given source package. 

57 

58 :type source_package: :class:`SourcePackage 

59 <distro_tracker.core.models.SourcePackage>` 

60 :type files_to_extract: An iterable of file names which should be 

61 extracted 

62 """ 

63 if not hasattr(self, 'cache'): 63 ↛ 66line 63 didn't jump to line 66, because the condition on line 63 was never false

64 self.cache = AptCache() 

65 

66 source_directory = self.cache.retrieve_source( 

67 source_package.source_package_name.name, 

68 source_package.version, 

69 debian_directory_only=True) 

70 debian_directory = os.path.join(source_directory, 'debian') 

71 

72 if files_to_extract is None: 72 ↛ 73line 72 didn't jump to line 73, because the condition on line 72 was never true

73 files_to_extract = self.ALL_FILES_TO_EXTRACT 

74 

75 for file_name in files_to_extract: 

76 file_path = os.path.join(debian_directory, file_name) 

77 if not os.path.exists(file_path): 

78 continue 

79 with open(file_path, 'rb') as f: 

80 extracted_file = File(f) 

81 ExtractedSourceFile.objects.create( 

82 source_package=source_package, 

83 extracted_file=extracted_file, 

84 name=file_name) 

85 

86 def execute_main(self): 

87 # First remove all source files which are no longer to be included. 

88 qs = ExtractedSourceFile.objects.exclude( 

89 name__in=self.ALL_FILES_TO_EXTRACT) 

90 qs.delete() 

91 

92 # Process pending items 

93 for srcpkg in self.items_to_process(): 

94 # Save what has been processed when it takes long enough that we 

95 # had to extend the lock 

96 if self.extend_lock(): 96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true

97 self.save_data() 

98 

99 extracted_files = [ 

100 extracted_file.name 

101 for extracted_file in srcpkg.extracted_source_files.all() 

102 ] 

103 files_to_extract = [ 

104 file_name 

105 for file_name in self.ALL_FILES_TO_EXTRACT 

106 if file_name not in extracted_files 

107 ] 

108 if files_to_extract: 108 ↛ 117line 108 didn't jump to line 117, because the condition on line 108 was never false

109 try: 

110 self.extract_files(srcpkg, files_to_extract) 

111 self.item_mark_processed(srcpkg) 

112 except Exception: 

113 logger.exception( 

114 'Problem extracting source files for %s version %s', 

115 srcpkg, srcpkg.version) 

116 else: 

117 self.item_mark_processed(srcpkg) 

118 

119 # TODO: remove extracted files associated to vanished source packages