1# Copyright 2013-2016 The Distro Tracker Developers
2# See the COPYRIGHT file at the top-level directory of this distribution and
3# at https://deb.li/DTAuthors
4#
5# This file is part of Distro Tracker. It is subject to the license terms
6# in the LICENSE file found in the top-level directory of this
7# distribution and at https://deb.li/DTLicense. No part of Distro Tracker,
8# including this file, may be copied, modified, propagated, or distributed
9# except according to the terms contained in the LICENSE file.
10"""
11Implements the processing of received package messages in order to dispatch
12them to subscribers.
13"""
14import logging
15import re
16from copy import deepcopy
17from datetime import datetime
19from django.conf import settings
20from django.core.mail import EmailMessage, get_connection
21from django.utils import timezone
23from distro_tracker import vendor
24from distro_tracker.core.models import Keyword, PackageName, Team
25from distro_tracker.core.utils import (
26 distro_tracker_render_to_string,
27 extract_email_address_from_header,
28 get_decoded_message_payload,
29 get_or_none,
30 verp
31)
32from distro_tracker.core.utils.email_messages import (
33 CustomEmailMessage,
34 patch_message_for_django_compat
35)
36from distro_tracker.mail.models import UserEmailBounceStats
38DISTRO_TRACKER_CONTROL_EMAIL = settings.DISTRO_TRACKER_CONTROL_EMAIL
39DISTRO_TRACKER_FQDN = settings.DISTRO_TRACKER_FQDN
41logger = logging.getLogger(__name__)
44class SkipMessage(Exception):
45 """This exception can be raised by the vendor provided classify_message()
46 to tell the dispatch code to skip processing this message being processed.
47 The mail is then silently dropped."""
50def _get_logdata(msg, package, keyword, team):
51 return {
52 'from': extract_email_address_from_header(msg.get('From', '')),
53 'msgid': msg.get('Message-ID', 'no-msgid-present@localhost'),
54 'package': package or '<unknown>',
55 'keyword': keyword or '<unknown>',
56 'team': team or '<unknown>',
57 }
60def _must_discard(msg, logdata):
61 # Check loop
62 dispatch_email = 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)
63 if dispatch_email in msg.get_all('X-Loop', ()):
64 # Bad X-Loop, discard the message
65 logger.info('dispatch :: discarded %(msgid)s due to X-Loop', logdata)
66 return True
67 return False
70def process(msg, package=None, keyword=None):
71 """
72 Dispatches received messages by identifying where they should
73 be sent and then by forwarding them.
75 :param msg: The received message
76 :type msg: :py:class:`email.message.Message`
78 :param str package: The package to which the message was sent.
80 :param str keyword: The keyword under which the message must be dispatched.
81 """
82 logdata = _get_logdata(msg, package, keyword, None)
83 logger.info("dispatch :: received from %(from)s :: %(msgid)s",
84 logdata)
85 try:
86 package, keyword = classify_message(msg, package, keyword)
87 except SkipMessage:
88 logger.info('dispatch :: skipping %(msgid)s', logdata)
89 return
91 if package is None:
92 logger.warning('dispatch :: no package identified for %(msgid)s',
93 logdata)
94 return
96 if _must_discard(msg, logdata):
97 return
99 if isinstance(package, (list, set)):
100 for pkg in package:
101 forward(msg, pkg, keyword)
102 else:
103 forward(msg, package, keyword)
106def forward(msg, package, keyword):
107 """
108 Forwards a received message to the various subscribers of the
109 given package/keyword combination.
111 :param msg: The received message
112 :type msg: :py:class:`email.message.Message`
114 :param str package: The package name.
116 :param str keyword: The keyword under which the message must be forwarded.
117 """
118 logdata = _get_logdata(msg, package, keyword, None)
120 logger.info("dispatch :: forward to %(package)s %(keyword)s :: %(msgid)s",
121 logdata)
123 # Default keywords require special approvement
124 if keyword == 'default' and not approved_default(msg):
125 logger.info('dispatch :: discarded non-approved message %(msgid)s',
126 logdata)
127 return
129 # Now send the message to subscribers
130 add_new_headers(msg, package_name=package, keyword=keyword)
131 send_to_subscribers(msg, package, keyword)
132 send_to_teams(msg, package, keyword)
135def process_for_team(msg, team_slug):
136 """Dispatch a message sent to a team."""
137 logdata = _get_logdata(msg, None, None, team_slug)
138 logger.info("dispatch :: received for team %(team)s "
139 "from %(from)s :: %(msgid)s", logdata)
141 if _must_discard(msg, logdata):
142 return
144 try:
145 team = Team.objects.get(slug=team_slug)
146 except Team.DoesNotExist:
147 logger.info("dispatch :: discarded %(msgid)s for team %(team)s "
148 "since team doesn't exist", logdata)
149 return
151 package, keyword = classify_message(msg)
152 if package:
153 logger.info("dispatch :: discarded %(msgid)s for team %(team)s "
154 "as an automatic mail", logdata)
155 return
157 forward_to_team(msg, team)
160def forward_to_team(msg, team):
161 """Forward a message to a team, adding headers as required."""
162 logdata = _get_logdata(msg, None, None, team.slug)
163 logger.info("dispatch :: forward to team %(team)s :: %(msgid)s",
164 logdata)
166 add_new_headers(msg, keyword="contact", team=team.slug)
167 send_to_team(msg, team, keyword="contact")
170def classify_message(msg, package=None, keyword=None):
171 """
172 Analyzes a message to identify what package it is about and
173 what keyword is appropriate.
175 :param msg: The received message
176 :type msg: :py:class:`email.message.Message`
178 :param str package: The suggested package name.
180 :param str keyword: The suggested keyword under which the message can be
181 forwarded.
183 """
184 if package is None:
185 package = msg.get('X-Distro-Tracker-Package')
186 if keyword is None:
187 keyword = msg.get('X-Distro-Tracker-Keyword')
189 result, implemented = vendor.call('classify_message', msg,
190 package=package, keyword=keyword)
191 if implemented:
192 package, keyword = result
193 if package and keyword is None:
194 keyword = 'default'
195 return (package, keyword)
198def approved_default(msg):
199 """
200 The function checks whether a message tagged with the default keyword should
201 be approved, meaning that it gets forwarded to subscribers.
203 :param msg: The received package message
204 :type msg: :py:class:`email.message.Message` or an equivalent interface
205 object
206 """
207 if 'X-Distro-Tracker-Approved' in msg:
208 return True
210 approved, implemented = vendor.call('approve_default_message', msg)
211 if implemented: 211 ↛ 214line 211 didn't jump to line 214, because the condition on line 211 was never false
212 return approved
213 else:
214 return False
217def add_new_headers(received_message, package_name=None, keyword=None,
218 team=None):
219 """
220 The function adds new distro-tracker specific headers to the received
221 message. This is used before forwarding the message to subscribers.
223 The headers added by this function are used regardless whether the
224 message is forwarded due to direct package subscriptions or a team
225 subscription.
227 :param received_message: The received package message
228 :type received_message: :py:class:`email.message.Message` or an equivalent
229 interface object
231 :param package_name: The name of the package for which this message was
232 intended.
233 :type package_name: string
235 :param keyword: The keyword with which the message should be tagged
236 :type keyword: string
237 """
238 new_headers = [
239 ('X-Loop', 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)),
240 ]
241 if keyword: 241 ↛ 243line 241 didn't jump to line 243, because the condition on line 241 was never false
242 new_headers.append(('X-Distro-Tracker-Keyword', keyword))
243 if package_name:
244 new_headers.extend([
245 ('X-Distro-Tracker-Package', package_name),
246 ('List-Id', '<{}.{}>'.format(package_name, DISTRO_TRACKER_FQDN)),
247 ])
248 if team:
249 new_headers.append(('X-Distro-Tracker-Team', team))
251 extra_vendor_headers, implemented = vendor.call(
252 'add_new_headers', received_message, package_name, keyword, team)
253 if implemented:
254 new_headers.extend(extra_vendor_headers)
256 for header_name, header_value in new_headers:
257 received_message[header_name] = header_value
260def add_direct_subscription_headers(received_message, package_name, keyword):
261 """
262 The function adds headers to the received message which are specific for
263 messages to be sent to users that are directly subscribed to the package.
264 """
265 new_headers = [
266 ('Precedence', 'list'),
267 ('List-Unsubscribe',
268 '<mailto:{control_email}?body=unsubscribe%20{package}>'.format(
269 control_email=DISTRO_TRACKER_CONTROL_EMAIL,
270 package=package_name)),
271 ]
272 for header_name, header_value in new_headers:
273 received_message[header_name] = header_value
276def add_team_membership_headers(received_message, keyword, team):
277 """
278 The function adds headers to the received message which are specific for
279 messages to be sent to users that are members of a team.
280 """
281 received_message['X-Distro-Tracker-Team'] = team.slug
284def send_to_teams(received_message, package_name, keyword):
285 """
286 Sends the given email message to all members of each team that has the
287 given package.
289 The message is only sent to those users who have not muted the team
290 and have the given keyword in teir set of keywords for the team
291 membership.
293 :param received_message: The modified received package message to be sent
294 to the subscribers.
295 :type received_message: :py:class:`email.message.Message` or an equivalent
296 interface object
298 :param package_name: The name of the package for which this message was
299 intended.
300 :type package_name: string
302 :param keyword: The keyword with which the message should be tagged
303 :type keyword: string
304 """
305 keyword = get_or_none(Keyword, name=keyword)
306 package = get_or_none(PackageName, name=package_name)
307 if not keyword or not package:
308 return
309 # Get all teams that have the given package
310 teams = Team.objects.filter(packages=package)
311 teams = teams.prefetch_related('team_membership_set')
313 for team in teams:
314 send_to_team(received_message, team, keyword, package.name)
317def send_to_team(received_message, team, keyword, package_name=None):
318 """Send a message to a team."""
319 keyword = get_or_none(Keyword, name=keyword)
320 package = get_or_none(PackageName, name=package_name)
321 date = timezone.now().date()
322 messages_to_send = []
323 logger.info('dispatch :: sending to team %s', team.slug)
324 team_message = deepcopy(received_message)
325 add_team_membership_headers(team_message, keyword.name, team)
327 # Send the message to each member of the team
328 for membership in team.team_membership_set.all():
329 # Do not send messages to muted memberships
330 if membership.is_muted(package):
331 continue
332 # Do not send the message if the user has disabled the keyword
333 if keyword not in membership.get_keywords(package):
334 continue
336 messages_to_send.append(prepare_message(
337 team_message, membership.user_email.email, date))
339 send_messages(messages_to_send, date)
342def send_to_subscribers(received_message, package_name, keyword):
343 """
344 Sends the given email message to all subscribers of the package with the
345 given name and those that accept messages tagged with the given keyword.
347 :param received_message: The modified received package message to be sent
348 to the subscribers.
349 :type received_message: :py:class:`email.message.Message` or an equivalent
350 interface object
352 :param package_name: The name of the package for which this message was
353 intended.
354 :type package_name: string
356 :param keyword: The keyword with which the message should be tagged
357 :type keyword: string
358 """
359 # Make a copy of the message to be sent and add any headers which are
360 # specific for users that are directly subscribed to the package.
361 received_message = deepcopy(received_message)
362 add_direct_subscription_headers(received_message, package_name, keyword)
363 package = get_or_none(PackageName, name=package_name)
364 if not package:
365 return
366 # Build a list of all messages to be sent
367 date = timezone.now().date()
368 messages_to_send = [
369 prepare_message(received_message,
370 subscription.email_settings.user_email.email,
371 date)
372 for subscription in package.subscription_set.all_active(keyword)
373 ]
374 send_messages(messages_to_send, date)
377def send_messages(messages_to_send, date):
378 """
379 Sends all the given email messages over a single SMTP connection.
380 """
381 connection = get_connection()
382 connection.send_messages(messages_to_send)
384 for message in messages_to_send:
385 logger.info("dispatch => %s", message.to[0])
386 UserEmailBounceStats.objects.add_sent_for_user(email=message.to[0],
387 date=date)
390def prepare_message(received_message, to_email, date):
391 """
392 Converts a message which is to be sent to a subscriber to a
393 :py:class:`CustomEmailMessage
394 <distro_tracker.core.utils.email_messages.CustomEmailMessage>`
395 so that it can be sent out using Django's API.
396 It also sets the required evelope-to value in order to track the bounce for
397 the message.
399 :param received_message: The modified received package message to be sent
400 to the subscribers.
401 :type received_message: :py:class:`email.message.Message` or an equivalent
402 interface object
404 :param to_email: The email of the subscriber to whom the message is to be
405 sent
406 :type to_email: string
408 :param date: The date which should be used as the message's sent date.
409 :type date: :py:class:`datetime.datetime`
410 """
411 bounce_address = 'bounces+{date}@{distro_tracker_fqdn}'.format(
412 date=date.strftime('%Y%m%d'),
413 distro_tracker_fqdn=DISTRO_TRACKER_FQDN)
414 message = CustomEmailMessage(
415 msg=patch_message_for_django_compat(received_message),
416 from_email=verp.encode(bounce_address, to_email),
417 to=[to_email])
418 return message
421def bounce_is_for_spam(message):
422 """Return True if the bounce has been generated by spam, False otherwise."""
423 spam_bounce_re = [
424 # Google blocks executables files
425 # 552-5.7.0 This message was blocked because its content presents a[...]
426 # 552-5.7.0 security issue. Please visit
427 # 552-5.7.0 https://support.google.com/mail/?p=BlockedMessage to [...]
428 # 552 5.7.0 message content and attachment content guidelines. [...]
429 r"552-5.7.0 This message was blocked",
430 # host ...: 550 High probability of spam
431 # host ...: 554 5.7.1 Message rejected because it contains malware
432 # 550 Executable files are not allowed in compressed files.
433 # 554 5.7.1 Spam message rejected
434 r"55[0-9][- ].*(?:[Ss]pam|malware|virus|[Ee]xecutable files)",
436 ]
437 # XXX: Handle delivery report properly
438 for part in message.walk():
439 if not part or part.is_multipart(): 439 ↛ 440line 439 didn't jump to line 440, because the condition on line 439 was never true
440 continue
441 text = get_decoded_message_payload(part)
442 if text is None:
443 continue
444 for line in text.splitlines()[0:15]: 444 ↛ 438line 444 didn't jump to line 438, because the loop on line 444 didn't complete
445 for rule in spam_bounce_re:
446 if re.search(rule, line):
447 return True
449 return False
452def handle_bounces(sent_to_address, message):
453 """
454 Handles a received bounce message.
456 :param sent_to_address: The envelope-to (return path) address to which the
457 bounced email was returned.
458 :type sent_to_address: string
459 """
460 try:
461 bounce_email, user_email = verp.decode(sent_to_address)
462 except ValueError:
463 logger.warning('bounces :: no VERP data to extract from %s',
464 sent_to_address)
465 return
466 match = re.match(r'^bounces\+(\d{8})@' + DISTRO_TRACKER_FQDN, bounce_email)
467 if not match: 467 ↛ 468line 467 didn't jump to line 468, because the condition on line 467 was never true
468 logger.warning('bounces :: invalid address %s', bounce_email)
469 return
470 try:
471 date = datetime.strptime(match.group(1), '%Y%m%d')
472 except ValueError:
473 logger.warning('bounces :: invalid date in address %s', bounce_email)
474 return
476 logger.info('bounces :: received one for %s/%s', user_email, date)
477 try:
478 user = UserEmailBounceStats.objects.get(email__iexact=user_email)
479 except UserEmailBounceStats.DoesNotExist:
480 logger.warning('bounces :: unknown user email %s', user_email)
481 return
483 if bounce_is_for_spam(message):
484 logger.info('bounces :: discarded spam bounce for %s/%s',
485 user_email, date)
486 return
488 UserEmailBounceStats.objects.add_bounce_for_user(email=user_email,
489 date=date)
491 if user.has_too_many_bounces():
492 logger.info('bounces => %s has too many bounces', user_email)
494 packages = list(user.emailsettings.packagename_set.all())
495 teams = [m.team for m in user.membership_set.all()]
496 email_body = distro_tracker_render_to_string(
497 'dispatch/unsubscribed-due-to-bounces-email.txt', {
498 'email': user_email,
499 'packages': packages,
500 'teams': teams,
501 })
502 EmailMessage(
503 subject='All your package subscriptions have been cancelled',
504 from_email=settings.DISTRO_TRACKER_BOUNCES_LIKELY_SPAM_EMAIL,
505 to=[user_email],
506 cc=[settings.DISTRO_TRACKER_CONTACT_EMAIL],
507 body=email_body,
508 headers={
509 'From': settings.DISTRO_TRACKER_CONTACT_EMAIL,
510 },
511 ).send()
513 user.emailsettings.unsubscribe_all()
514 for package in packages:
515 logger.info('bounces :: removed %s from %s', user_email,
516 package.name)
517 user.membership_set.all().update(muted=True)
518 for team in teams:
519 logger.info('bounces :: muted membership of %s in team %s',
520 user_email, team.slug)