1# Copyright 2013-2016 The Distro Tracker Developers
2# See the COPYRIGHT file at the top-level directory of this distribution and
3# at https://deb.li/DTAuthors
4#
5# This file is part of Distro Tracker. It is subject to the license terms
6# in the LICENSE file found in the top-level directory of this
7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker,
8# including this file, may be copied, modified, propagated, or distributed
9# except according to the terms contained in the LICENSE file.
10"""
11Implements the processing of received package messages in order to dispatch
12them to subscribers.
13"""
14import logging
15import re
16from copy import deepcopy
17from datetime import datetime
19from django.conf import settings
20from django.core.mail import EmailMessage, get_connection
21from django.utils import timezone
23from distro_tracker import vendor
24from distro_tracker.core.models import Keyword, PackageName, Team
25from distro_tracker.core.utils import (
26 distro_tracker_render_to_string,
27 extract_email_address_from_header,
28 get_decoded_message_payload,
29 get_or_none,
30 verp
31)
32from distro_tracker.core.utils.email_messages import (
33 CustomEmailMessage,
34 patch_message_for_django_compat
35)
36from distro_tracker.mail.models import UserEmailBounceStats
38DISTRO_TRACKER_CONTROL_EMAIL = settings.DISTRO_TRACKER_CONTROL_EMAIL
39DISTRO_TRACKER_FQDN = settings.DISTRO_TRACKER_FQDN
41logger = logging.getLogger(__name__)
44class SkipMessage(Exception):
45 """This exception can be raised by the vendor provided classify_message()
46 to tell the dispatch code to skip processing this message being processed.
47 The mail is then silently dropped."""
50def _get_logdata(msg, package, keyword, team):
51 return {
52 'from': extract_email_address_from_header(msg.get('From', '')),
53 'msgid': msg.get('Message-ID', 'no-msgid-present@localhost'),
54 'package': package or '<unknown>',
55 'keyword': keyword or '<unknown>',
56 'team': team or '<unknown>',
57 }
60def _must_discard(msg, logdata):
61 # Check loop
62 dispatch_email = 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)
63 if dispatch_email in msg.get_all('X-Loop', ()):
64 # Bad X-Loop, discard the message
65 logger.info('dispatch :: discarded %(msgid)s due to X-Loop', logdata)
66 return True
67 return False
70def process(msg, package=None, keyword=None):
71 """
72 Dispatches received messages by identifying where they should
73 be sent and then by forwarding them.
75 :param msg: The received message
76 :type msg: :py:class:`email.message.Message`
78 :param str package: The package to which the message was sent.
80 :param str keyword: The keyword under which the message must be dispatched.
81 """
82 logdata = _get_logdata(msg, package, keyword, None)
83 logger.info("dispatch :: received from %(from)s :: %(msgid)s",
84 logdata)
85 try:
86 package, keyword = classify_message(msg, package, keyword)
87 except SkipMessage:
88 logger.info('dispatch :: skipping %(msgid)s', logdata)
89 return
91 if package is None:
92 logger.warning('dispatch :: no package identified for %(msgid)s',
93 logdata)
94 return
96 if _must_discard(msg, logdata):
97 return
99 if isinstance(package, (list, set)):
100 for pkg in package:
101 forward(msg, pkg, keyword)
102 else:
103 forward(msg, package, keyword)
106def forward(msg, package, keyword):
107 """
108 Forwards a received message to the various subscribers of the
109 given package/keyword combination.
111 :param msg: The received message
112 :type msg: :py:class:`email.message.Message`
114 :param str package: The package name.
116 :param str keyword: The keyword under which the message must be forwarded.
117 """
118 logdata = _get_logdata(msg, package, keyword, None)
120 logger.info("dispatch :: forward to %(package)s %(keyword)s :: %(msgid)s",
121 logdata)
123 # Default keywords require special approvement
124 if keyword == 'default' and not approved_default(msg):
125 logger.info('dispatch :: discarded non-approved message %(msgid)s',
126 logdata)
127 return
129 # Now send the message to subscribers
130 add_new_headers(msg, package_name=package, keyword=keyword)
131 send_to_subscribers(msg, package, keyword)
132 send_to_teams(msg, package, keyword)
135def process_for_team(msg, team_slug):
136 """Dispatch a message sent to a team."""
137 logdata = _get_logdata(msg, None, None, team_slug)
138 logger.info("dispatch :: received for team %(team)s "
139 "from %(from)s :: %(msgid)s", logdata)
141 if _must_discard(msg, logdata):
142 return
144 try:
145 team = Team.objects.get(slug=team_slug)
146 except Team.DoesNotExist:
147 logger.info("dispatch :: discarded %(msgid)s for team %(team)s "
148 "since team doesn't exist", logdata)
149 return
151 package, keyword = classify_message(msg)
152 if package:
153 logger.info("dispatch :: discarded %(msgid)s for team %(team)s "
154 "as an automatic mail", logdata)
155 return
157 forward_to_team(msg, team)
160def forward_to_team(msg, team):
161 """Forward a message to a team, adding headers as required."""
162 logdata = _get_logdata(msg, None, None, team.slug)
163 logger.info("dispatch :: forward to team %(team)s :: %(msgid)s",
164 logdata)
166 add_new_headers(msg, keyword="contact", team=team.slug)
167 send_to_team(msg, team, keyword="contact")
170def classify_message(msg, package=None, keyword=None):
171 """
172 Analyzes a message to identify what package it is about and
173 what keyword is appropriate.
175 :param msg: The received message
176 :type msg: :py:class:`email.message.Message`
178 :param str package: The suggested package name.
180 :param str keyword: The suggested keyword under which the message can be
181 forwarded.
183 """
184 if package is None:
185 package = msg.get('X-Distro-Tracker-Package')
186 if keyword is None:
187 keyword = msg.get('X-Distro-Tracker-Keyword')
189 result, implemented = vendor.call('classify_message', msg,
190 package=package, keyword=keyword)
191 if implemented:
192 package, keyword = result
193 if package and keyword is None:
194 keyword = 'default'
195 return (package, keyword)
198def approved_default(msg):
199 """
200 The function checks whether a message tagged with the default keyword should
201 be approved, meaning that it gets forwarded to subscribers.
203 :param msg: The received package message
204 :type msg: :py:class:`email.message.Message` or an equivalent interface
205 object
206 """
207 if 'X-Distro-Tracker-Approved' in msg:
208 return True
210 approved, implemented = vendor.call('approve_default_message', msg)
211 if implemented: 211 ↛ 214line 211 didn't jump to line 214, because the condition on line 211 was never false
212 return approved
213 else:
214 return False
217def add_new_headers(received_message, package_name=None, keyword=None,
218 team=None):
219 """
220 The function adds new distro-tracker specific headers to the received
221 message. This is used before forwarding the message to subscribers.
223 The headers added by this function are used regardless whether the
224 message is forwarded due to direct package subscriptions or a team
225 subscription.
227 :param received_message: The received package message
228 :type received_message: :py:class:`email.message.Message` or an equivalent
229 interface object
231 :param package_name: The name of the package for which this message was
232 intended.
233 :type package_name: string
235 :param keyword: The keyword with which the message should be tagged
236 :type keyword: string
237 """
238 new_headers = [
239 ('X-Loop', 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)),
240 ]
241 if keyword: 241 ↛ 243line 241 didn't jump to line 243, because the condition on line 241 was never false
242 new_headers.append(('X-Distro-Tracker-Keyword', keyword))
243 if package_name:
244 new_headers.extend([
245 ('X-Distro-Tracker-Package', package_name),
246 ('List-Id', '<{}.{}>'.format(package_name, DISTRO_TRACKER_FQDN)),
247 ])
248 if team:
249 new_headers.append(('X-Distro-Tracker-Team', team))
251 extra_vendor_headers, implemented = vendor.call(
252 'add_new_headers', received_message, package_name, keyword, team)
253 if implemented:
254 new_headers.extend(extra_vendor_headers)
256 for header_name, header_value in new_headers:
257 received_message[header_name] = header_value
260def add_direct_subscription_headers(received_message, package_name, keyword):
261 """
262 The function adds headers to the received message which are specific for
263 messages to be sent to users that are directly subscribed to the package.
264 """
265 new_headers = [
266 ('Precedence', 'list'),
267 ('List-Unsubscribe',
268 '<mailto:{control_email}?body=unsubscribe%20{package}>'.format(
269 control_email=DISTRO_TRACKER_CONTROL_EMAIL,
270 package=package_name)),
271 ]
272 for header_name, header_value in new_headers:
273 received_message[header_name] = header_value
276def add_team_membership_headers(received_message, keyword, team):
277 """
278 The function adds headers to the received message which are specific for
279 messages to be sent to users that are members of a team.
280 """
281 received_message['X-Distro-Tracker-Team'] = team.slug
284def send_to_teams(received_message, package_name, keyword):
285 """
286 Sends the given email message to all members of each team that has the
287 given package.
289 The message is only sent to those users who have not muted the team
290 and have the given keyword in teir set of keywords for the team
291 membership.
293 :param received_message: The modified received package message to be sent
294 to the subscribers.
295 :type received_message: :py:class:`email.message.Message` or an equivalent
296 interface object
298 :param package_name: The name of the package for which this message was
299 intended.
300 :type package_name: string
302 :param keyword: The keyword with which the message should be tagged
303 :type keyword: string
304 """
305 keyword = get_or_none(Keyword, name=keyword)
306 package = get_or_none(PackageName, name=package_name)
307 if not keyword or not package:
308 return
309 # Get all teams that have the given package
310 teams = Team.objects.filter(packages=package)
311 teams = teams.prefetch_related('team_membership_set')
313 for team in teams:
314 send_to_team(received_message, team, keyword, package.name)
317def send_to_team(received_message, team, keyword, package_name=None):
318 """Send a message to a team."""
319 keyword = get_or_none(Keyword, name=keyword)
320 package = get_or_none(PackageName, name=package_name)
321 date = timezone.now().date()
322 messages_to_send = []
323 logger.info('dispatch :: sending to team %s', team.slug)
324 team_message = deepcopy(received_message)
325 add_team_membership_headers(team_message, keyword.name, team)
327 # Send the message to each member of the team
328 for membership in team.team_membership_set.all():
329 # Do not send messages to muted memberships
330 if membership.is_muted(package):
331 continue
332 # Do not send the message if the user has disabled the keyword
333 if keyword not in membership.get_keywords(package):
334 continue
336 messages_to_send.append(prepare_message(
337 team_message, membership.user_email.email, date))
339 send_messages(messages_to_send, date)
342def send_to_subscribers(received_message, package_name, keyword):
343 """
344 Sends the given email message to all subscribers of the package with the
345 given name and those that accept messages tagged with the given keyword.
347 :param received_message: The modified received package message to be sent
348 to the subscribers.
349 :type received_message: :py:class:`email.message.Message` or an equivalent
350 interface object
352 :param package_name: The name of the package for which this message was
353 intended.
354 :type package_name: string
356 :param keyword: The keyword with which the message should be tagged
357 :type keyword: string
358 """
359 # Make a copy of the message to be sent and add any headers which are
360 # specific for users that are directly subscribed to the package.
361 received_message = deepcopy(received_message)
362 add_direct_subscription_headers(received_message, package_name, keyword)
363 package = get_or_none(PackageName, name=package_name)
364 if not package:
365 return
366 # Build a list of all messages to be sent
367 date = timezone.now().date()
368 messages_to_send = [
369 prepare_message(received_message,
370 subscription.email_settings.user_email.email,
371 date)
372 for subscription in package.subscription_set.all_active(keyword)
373 ]
374 send_messages(messages_to_send, date)
377def send_messages(messages_to_send, date):
378 """
379 Sends all the given email messages over a single SMTP connection.
380 """
381 connection = get_connection()
382 connection.send_messages(messages_to_send)
384 for message in messages_to_send:
385 logger.info("dispatch => %s", message.to[0])
386 UserEmailBounceStats.objects.add_sent_for_user(email=message.to[0],
387 date=date)
390def prepare_message(received_message, to_email, date):
391 """
392 Converts a message which is to be sent to a subscriber to a
393 :py:class:`CustomEmailMessage
394 <distro_tracker.core.utils.email_messages.CustomEmailMessage>`
395 so that it can be sent out using Django's API.
396 It also sets the required evelope-to value in order to track the bounce for
397 the message.
399 :param received_message: The modified received package message to be sent
400 to the subscribers.
401 :type received_message: :py:class:`email.message.Message` or an equivalent
402 interface object
404 :param to_email: The email of the subscriber to whom the message is to be
405 sent
406 :type to_email: string
408 :param date: The date which should be used as the message's sent date.
409 :type date: :py:class:`datetime.datetime`
410 """
411 bounce_address = 'bounces+{date}@{distro_tracker_fqdn}'.format(
412 date=date.strftime('%Y%m%d'),
413 distro_tracker_fqdn=DISTRO_TRACKER_FQDN)
414 message = CustomEmailMessage(
415 msg=patch_message_for_django_compat(received_message),
416 from_email=verp.encode(bounce_address, to_email),
417 to=[to_email])
418 return message
421def bounce_is_for_spam(message):
422 """Return True if the bounce has been generated by spam, False otherwise."""
423 spam_bounce_re = [
424 # Google blocks executables files
425 # 552-5.7.0 This message was blocked because its content presents a[...]
426 # 552-5.7.0 security issue. Please visit
427 # 552-5.7.0 https://support.google.com/mail/?p=BlockedMessage to [...]
428 # 552 5.7.0 message content and attachment content guidelines. [...]
429 r"552-5.7.0 This message was blocked",
430 # host ...: 550 High probability of spam
431 # host ...: 554 5.7.1 Message rejected because it contains malware
432 # 550 Executable files are not allowed in compressed files.
433 r"55[0-9][- ].*(?:spam|malware|virus|[Ee]xecutable files)",
434 ]
435 # XXX: Handle delivery report properly
436 for part in message.walk():
437 if not part or part.is_multipart(): 437 ↛ 438line 437 didn't jump to line 438, because the condition on line 437 was never true
438 continue
439 text = get_decoded_message_payload(part)
440 if text is None:
441 continue
442 for line in text.splitlines()[0:15]: 442 ↛ 436line 442 didn't jump to line 436, because the loop on line 442 didn't complete
443 for rule in spam_bounce_re:
444 if re.search(rule, line):
445 return True
447 return False
450def handle_bounces(sent_to_address, message):
451 """
452 Handles a received bounce message.
454 :param sent_to_address: The envelope-to (return path) address to which the
455 bounced email was returned.
456 :type sent_to_address: string
457 """
458 try:
459 bounce_email, user_email = verp.decode(sent_to_address)
460 except ValueError:
461 logger.warning('bounces :: no VERP data to extract from %s',
462 sent_to_address)
463 return
464 match = re.match(r'^bounces\+(\d{8})@' + DISTRO_TRACKER_FQDN, bounce_email)
465 if not match: 465 ↛ 466line 465 didn't jump to line 466, because the condition on line 465 was never true
466 logger.warning('bounces :: invalid address %s', bounce_email)
467 return
468 try:
469 date = datetime.strptime(match.group(1), '%Y%m%d')
470 except ValueError:
471 logger.warning('bounces :: invalid date in address %s', bounce_email)
472 return
474 logger.info('bounces :: received one for %s/%s', user_email, date)
475 try:
476 user = UserEmailBounceStats.objects.get(email__iexact=user_email)
477 except UserEmailBounceStats.DoesNotExist:
478 logger.warning('bounces :: unknown user email %s', user_email)
479 return
481 if bounce_is_for_spam(message):
482 logger.info('bounces :: discarded spam bounce for %s/%s',
483 user_email, date)
484 return
486 UserEmailBounceStats.objects.add_bounce_for_user(email=user_email,
487 date=date)
489 if user.has_too_many_bounces():
490 logger.info('bounces => %s has too many bounces', user_email)
492 packages = list(user.emailsettings.packagename_set.all())
493 teams = [m.team for m in user.membership_set.all()]
494 email_body = distro_tracker_render_to_string(
495 'dispatch/unsubscribed-due-to-bounces-email.txt', {
496 'email': user_email,
497 'packages': packages,
498 'teams': teams,
499 })
500 EmailMessage(
501 subject='All your package subscriptions have been cancelled',
502 from_email=settings.DISTRO_TRACKER_BOUNCES_LIKELY_SPAM_EMAIL,
503 to=[user_email],
504 cc=[settings.DISTRO_TRACKER_CONTACT_EMAIL],
505 body=email_body,
506 headers={
507 'From': settings.DISTRO_TRACKER_CONTACT_EMAIL,
508 },
509 ).send()
511 user.emailsettings.unsubscribe_all()
512 for package in packages:
513 logger.info('bounces :: removed %s from %s', user_email,
514 package.name)
515 user.membership_set.all().update(muted=True)
516 for team in teams:
517 logger.info('bounces :: muted membership of %s in team %s',
518 user_email, team.slug)