ZEN-35430: break out of email retry loop on success

Author: StevePC

src/Products/ZenModel/actions.py

@@ -434,86 +434,96 @@ def executeBatch(self, notification, signal, targets):
         original_fst = signal.event.first_seen_time
         original_sct = signal.event.status_change_time
         for target_timezone, targets in tz_targets.iteritems():
-            # Convert timestamp to user timezone
-            signal.event.last_seen_time = self._adjustToTimezone(
-                original_lst, target_timezone)
-            signal.event.first_seen_time = self._adjustToTimezone(
-                original_fst, target_timezone)
-            signal.event.status_change_time =  self._adjustToTimezone(
-                original_sct, target_timezone)
-
-            data = self._signalToContextDict(signal, notification)
-
-            # Check for email recipients in the event
-            details = data['evt'].details
-            mail_targets = details.get('recipients', '')
-            mail_targets = [x.strip() for x in mail_targets.split(',') if x.strip()]
-            if len(mail_targets) > 0:
-                log.debug("Adding recipients defined in the event %s", mail_targets)
-                targets |= set(mail_targets)
-
-            skipfails = notification.content.get('skipfails', False)
-
-            if signal.clear:
-                log.debug("Generating a notification at enabled 'Send Clear' option when event was closed")
-                subject = processTalSource(notification.content['clear_subject_format'], skipfails, **data)
-                body = processTalSource(notification.content['clear_body_format'], skipfails, **data)
-            else:
-                subject = processTalSource(notification.content['subject_format'], skipfails, **data)
-                body = processTalSource(notification.content['body_format'], skipfails, **data)
-
-            log.debug('Sending this subject: %s', subject)
-
-            # Find all time strings in body and add timezone to it
-            body = re.sub(r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{3})',
-                          r'\1 ({})'.format(target_timezone), body)
-
-            plain_body = self._encodeBody(self._stripTags(body)) \
-                if self.stripBodyTags else self._encodeBody(body)
-
-            email_message = plain_body
-
-            if notification.content['body_content_type'] == 'html':
-                email_message = MIMEMultipart('related')
-                email_message_alternative = MIMEMultipart('alternative')
-                email_message_alternative.attach(plain_body)
-                html_body = self._encodeBody(body)
-                html_body.set_type('text/html')
-                email_message_alternative.attach(html_body)
-
-                email_message.attach(email_message_alternative)
-                log.debug('Sending this body: %s', body)
-            else:
-                log.debug('Sending this body: %s', plain_body)
-
-            host = notification.content['host']
-            port = notification.content['port']
-            user = notification.content['user']
-            password = notification.content['password']
-            useTls = notification.content['useTls']
-            email_from = notification.content['email_from']
-
-            email_message['Subject'] = subject
-            email_message['From'] = email_from
-            email_message['To'] = ','.join(targets)
-            email_message['Date'] = formatdate(None, True)
-
-            result, errorMsg = sendEmail(
-                email_message,
-                host, port,
-                useTls,
-                user, password
-            )
-
-            if result:
-                log.debug("Notification '%s' sent emails to: %s",
-                         notification.id, targets)
-            else:
-                raise ActionExecutionException(
-                    "Notification '%s' FAILED to send emails to %s: %s" %
-                    (notification.id, targets, errorMsg)
+            num_of_attempts = 3
+            for attempt in xrange(1, num_of_attempts+1):
+                # Convert timestamp to user timezone
+                signal.event.last_seen_time = self._adjustToTimezone(
+                    original_lst, target_timezone)
+                signal.event.first_seen_time = self._adjustToTimezone(
+                    original_fst, target_timezone)
+                signal.event.status_change_time =  self._adjustToTimezone(
+                    original_sct, target_timezone)
+
+                data = self._signalToContextDict(signal, notification)
+
+                # Check for email recipients in the event
+                details = data['evt'].details
+                mail_targets = details.get('recipients', '')
+                mail_targets = [x.strip() for x in mail_targets.split(',') if x.strip()]
+                if len(mail_targets) > 0:
+                    log.debug("Adding recipients defined in the event %s", mail_targets)
+                    targets |= set(mail_targets)
+
+                skipfails = notification.content.get('skipfails', False)
+
+                if signal.clear:
+                    log.debug("Generating a notification at enabled 'Send Clear' option when event was closed")
+                    subject = processTalSource(notification.content['clear_subject_format'], skipfails, **data)
+                    body = processTalSource(notification.content['clear_body_format'], skipfails, **data)
+                else:
+                    subject = processTalSource(notification.content['subject_format'], skipfails, **data)
+                    body = processTalSource(notification.content['body_format'], skipfails, **data)
+
+                log.debug('Sending this subject: %s', subject)
+
+                # Find all time strings in body and add timezone to it
+                body = re.sub(r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{3})',
+                              r'\1 ({})'.format(target_timezone), body)
+
+                plain_body = self._encodeBody(self._stripTags(body)) \
+                    if self.stripBodyTags else self._encodeBody(body)
+
+                email_message = plain_body
+
+                if notification.content['body_content_type'] == 'html':
+                    email_message = MIMEMultipart('related')
+                    email_message_alternative = MIMEMultipart('alternative')
+                    email_message_alternative.attach(plain_body)
+                    html_body = self._encodeBody(body)
+                    html_body.set_type('text/html')
+                    email_message_alternative.attach(html_body)
+
+                    email_message.attach(email_message_alternative)
+                    log.debug('Sending this body: %s', body)
+                else:
+                    log.debug('Sending this body: %s', plain_body)
+
+                host = notification.content['host']
+                port = notification.content['port']
+                user = notification.content['user']
+                password = notification.content['password']
+                useTls = notification.content['useTls']
+                email_from = notification.content['email_from']
+
+                email_message['Subject'] = subject
+                email_message['From'] = email_from
+                email_message['To'] = ','.join(targets)
+                email_message['Date'] = formatdate(None, True)
+
+                result, errorMsg = sendEmail(
+                    email_message,
+                    host, port,
+                    useTls,
+                    user, password
                 )
 
+                if result:
+                    log.debug("Notification '%s' sent emails to: %s",
+                             notification.id, targets)
+                    break
+                elif "socket.error" in errorMsg:
+                    if attempt < num_of_attempts:
+                        current_timeout = attempt * 5
+                    log.info('socket.error: %s. Still have time to try again. Try in %d seconds again...' %
+                             (errorMsg, current_timeout))
+                    time.sleep(current_timeout)
+                    continue
+                else:
+                    raise ActionExecutionException(
+                        "Notification '%s' FAILED to send emails to %s: %s" %
+                        (notification.id, targets, errorMsg)
+                    )
+
     def getActionableTargets(self, target):
         """
         @param target: This is an object that implements the IProvidesEmailAddresses