Welcome and farewell mails: own text part and Auto-Submitted header (junk-folder work, part 2)

Both template mails also land in Junk at some recipients. The templates are already image-free
and inline-styled, so the remaining technical signals were the missing text/plain part (the
provider derived one) and the missing automation header. HtmlToText renders the effective
template (default or operator override) as the text part — head, style, script and comments
dropped, blocks and list items as lines, links with their URL, entities decoded — and
EmailService sends every mail with Auto-Submitted: auto-generated / X-Auto-Response-Suppress,
shared with the alert mails. Wording changes to the customer-visible templates are deliberately
not part of this commit.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

src/Backend/AutopilotMonitor.Functions.Tests/EmailServiceTests.cs

@@ -113,7 +113,12 @@ public sealed class EmailServiceTests
         Assert.Equal("to", message.GetProperty("to")[0].GetProperty("type").GetString());
         Assert.Equal(EmailTemplates.PreviewApprovedSubject, message.GetProperty("subject").GetString());
         Assert.Contains("contoso.invalid", message.GetProperty("html").GetString());
-        Assert.True(message.GetProperty("auto_text").GetBoolean());
+        // A text part of our own (HtmlToText) instead of the provider's derived one, and the
+        // RFC 3834 header on every automated mail.
+        Assert.False(message.GetProperty("auto_text").GetBoolean());
+        Assert.Contains("contoso.invalid", message.GetProperty("text").GetString());
+        Assert.DoesNotContain("<", message.GetProperty("text").GetString());
+        Assert.Equal("auto-generated", message.GetProperty("headers").GetProperty("Auto-Submitted").GetString());
         Assert.False(message.GetProperty("track_opens").GetBoolean());
         Assert.False(message.GetProperty("track_clicks").GetBoolean());
         Assert.Equal("welcome", message.GetProperty("tags")[0].GetString());

src/Backend/AutopilotMonitor.Functions.Tests/HtmlToTextTests.cs

@@ -0,0 +1,54 @@
+using AutopilotMonitor.Functions.Services;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+/// <summary>The text/plain twin of the HTML mails (HtmlToText): what it keeps, what it drops, how it breaks lines.</summary>
+public class HtmlToTextTests
+{
+    [Fact]
+    public void Drops_head_style_script_and_comments_and_keeps_the_prose()
+    {
+        const string html = "<!DOCTYPE html><html><head><title>t</title><style>p{color:red}</style></head>" +
+                            "<body><!-- note --><script>alert(1)</script><p>Hello <strong>world</strong></p></body></html>";
+        Assert.Equal("Hello world", HtmlToText.Convert(html));
+    }
+
+    [Fact]
+    public void Turns_blocks_lists_and_breaks_into_lines_and_decodes_entities()
+    {
+        const string html = "<h2>Welcome</h2><p>Great &ndash; access for <strong>contoso.invalid</strong> is ready.<br>Second line.</p>" +
+                            "<ul><li>One &amp; two</li><li>Three</li></ul><table><tr><td>a</td><td>b</td></tr></table>";
+        var text = HtmlToText.Convert(html);
+        Assert.Equal("Welcome\nGreat – access for contoso.invalid is ready.\nSecond line.\n\n- One & two\n- Three\na b", text);
+    }
+
+    [Fact]
+    public void Keeps_link_targets_and_collapses_whitespace()
+    {
+        const string html = "<p>You can now <a href=\"https://portal.example.invalid/\" target=\"_blank\" style=\"color:red\">sign   in</a>   and start.</p>" +
+                            "<p><a href=\"https://docs.example.invalid\">https://docs.example.invalid</a></p>" +
+                            "<p><a href=\"mailto:x@example.invalid\">write us</a> or <a href=\"https://x.invalid/a?b=1&amp;c=2\">this</a></p>";
+        var text = HtmlToText.Convert(html);
+        Assert.Equal("You can now sign in (https://portal.example.invalid/) and start.\nhttps://docs.example.invalid\nwrite us or this (https://x.invalid/a?b=1&c=2)", text);
+    }
+
+    [Fact]
+    public void Survives_empty_and_broken_markup()
+    {
+        Assert.Equal(string.Empty, HtmlToText.Convert(null));
+        Assert.Equal(string.Empty, HtmlToText.Convert("   "));
+        Assert.Equal("unterminated bold", HtmlToText.Convert("<p>unterminated <b>bold"));
+        Assert.Equal("x", HtmlToText.Convert("<a href='nolink'>x</a>"));
+    }
+
+    [Fact]
+    public void Renders_the_default_welcome_template_as_readable_text()
+    {
+        var text = HtmlToText.Convert(EmailTemplates.GetPreviewApprovedHtml("contoso.invalid"));
+        Assert.Contains("Welcome to Autopilot Monitor", text);
+        Assert.Contains("contoso.invalid", text);
+        Assert.DoesNotContain("<", text);
+        Assert.DoesNotContain("style=", text);
+        Assert.Contains("https://", text);   // the links survive as URLs
+    }
+}

src/Backend/AutopilotMonitor.Functions/Services/EmailService.cs

@@ -42,6 +42,16 @@ public class EmailService : IEmailService, IOffboardFarewellEmailSender
     public const string DefaultFromAddress = "noreply@autopilotmonitor.com";
     public const string DefaultFromName = "Autopilot Monitor";
 
+    /// <summary>
+    /// RFC 3834: every mail the platform sends is machine-generated — auto-responders stay quiet,
+    /// and filters read the message as a notification rather than as someone pretending to write.
+    /// </summary>
+    public static readonly IReadOnlyDictionary<string, string> AutomatedHeaders = new Dictionary<string, string>
+    {
+        ["Auto-Submitted"] = "auto-generated",
+        ["X-Auto-Response-Suppress"] = "All",
+    };
+
     private static readonly JsonSerializerOptions JsonOptions = new()
     {
         DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
@@ -85,12 +95,9 @@ public class EmailService : IEmailService, IOffboardFarewellEmailSender
             return false;
         }
 
-        var sent = (await SendViaMandrillAsync(
-            toEmail,
-            EmailTemplates.PreviewApprovedSubject,
-            await _templates.GetHtmlAsync(EmailTemplateKind.Welcome, domainName, ct),
-            tag: "welcome",
-            ct)).Accepted;
+        var html = await _templates.GetHtmlAsync(EmailTemplateKind.Welcome, domainName, ct);
+        var sent = (await SendViaMandrillAsync(toEmail, EmailTemplates.PreviewApprovedSubject, html, "welcome", ct,
+            HtmlToText.Convert(html), AutomatedHeaders)).Accepted;
 
         if (sent)
         {
@@ -129,12 +136,9 @@ public class EmailService : IEmailService, IOffboardFarewellEmailSender
             return false;
         }
 
-        var sent = (await SendViaMandrillAsync(
-            toEmail,
-            EmailTemplates.OffboardingFarewellSubject,
-            await _templates.GetHtmlAsync(EmailTemplateKind.Farewell, domainName, ct),
-            tag: "offboarding-farewell",
-            ct)).Accepted;
+        var html = await _templates.GetHtmlAsync(EmailTemplateKind.Farewell, domainName, ct);
+        var sent = (await SendViaMandrillAsync(toEmail, EmailTemplates.OffboardingFarewellSubject, html, "offboarding-farewell", ct,
+            HtmlToText.Convert(html), AutomatedHeaders)).Accepted;
 
         if (sent)
         {
@@ -159,7 +163,8 @@ public class EmailService : IEmailService, IOffboardFarewellEmailSender
             ? await _templates.GetHtmlAsync(kind, domainName, ct)
             : EmailTemplateService.Render(draftHtml, domainName);
 
-        var sent = (await SendViaMandrillAsync(toEmail, EmailTemplateService.Subject(kind), html, tag: "test", ct)).Accepted;
+        var sent = (await SendViaMandrillAsync(toEmail, EmailTemplateService.Subject(kind), html, "test", ct,
+            HtmlToText.Convert(html), AutomatedHeaders)).Accepted;
         if (sent)
             _logger.LogInformation("{Kind} test email sent to {ToEmail} (draft={IsDraft})", kind, toEmail, draftHtml is not null);
         return sent;
@@ -208,8 +213,8 @@ public class EmailService : IEmailService, IOffboardFarewellEmailSender
                     To = new[] { new MandrillRecipient { Email = toEmail, Type = "to" } },
                     Subject = subject,
                     Html = html,
-                    // A text part of our own (the alert mails) beats the provider's derived one; the
-                    // templates still let the provider derive it.
+                    // A text part of our own beats the provider's derived one (a balanced multipart is
+                    // a content signal filters weigh); auto_text stays as the fallback for callers without one.
                     Text = text,
                     AutoText = text is null,
                     Headers = headers is { Count: > 0 } ? new Dictionary<string, string>(headers) : null,

src/Backend/AutopilotMonitor.Functions/Services/HtmlToText.cs

@@ -0,0 +1,57 @@
+using System.Net;
+using System.Text.RegularExpressions;
+
+namespace AutopilotMonitor.Functions.Services;
+
+/// <summary>
+/// The plain-text twin of an HTML mail, good enough for the text/plain part of a multipart
+/// message: a balanced multipart with a real text part is one of the strongest content signals
+/// a mail filter has, and a derived one by the provider is weaker than our own. Conservative on
+/// purpose — head, style, script and comments go, block boundaries become line breaks, list
+/// items get a dash, links keep their URL in parentheses, entities are decoded, whitespace is
+/// collapsed. Never throws on odd markup; the worst case is slightly ugly text.
+/// </summary>
+public static class HtmlToText
+{
+    private const RegexOptions Opts = RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled;
+
+    private static readonly Regex Dropped = new(@"<!--.*?-->|<head\b.*?</head>|<style\b.*?</style>|<script\b.*?</script>", Opts);
+    private static readonly Regex Anchor = new(@"<a\b[^>]*?href\s*=\s*([""'])(?<url>.*?)\1[^>]*>(?<text>.*?)</a>", Opts);
+    private static readonly Regex LineBreak = new(@"<br\s*/?>", Opts);
+    private static readonly Regex ListItem = new(@"<li\b[^>]*>", Opts);
+    // A list item ends where the next one starts ("\n- "), so </li> adds no line of its own.
+    private static readonly Regex BlockEnd = new(@"</(p|div|tr|h[1-6]|table|ul|ol|blockquote|section|article|header|footer)\s*>", Opts);
+    private static readonly Regex CellEnd = new(@"</t[dh]\s*>", Opts);
+    private static readonly Regex AnyTag = new(@"<[^>]+>", Opts);
+    private static readonly Regex Spaces = new(@"[ \t ]+", RegexOptions.Compiled);
+    private static readonly Regex BlankLines = new(@"\n{3,}", RegexOptions.Compiled);
+
+    public static string Convert(string? html)
+    {
+        if (string.IsNullOrWhiteSpace(html))
+            return string.Empty;
+
+        var text = Dropped.Replace(html, string.Empty);
+        text = Anchor.Replace(text, m =>
+        {
+            var url = WebUtility.HtmlDecode(m.Groups["url"].Value).Trim();
+            var label = Spaces.Replace(WebUtility.HtmlDecode(AnyTag.Replace(m.Groups["text"].Value, string.Empty)), " ").Trim();
+            if (!url.StartsWith("http://", StringComparison.OrdinalIgnoreCase) && !url.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+                return label;
+            if (label.Length == 0 || string.Equals(label, url, StringComparison.OrdinalIgnoreCase))
+                return url;
+            return $"{label} ({url})";
+        });
+        text = LineBreak.Replace(text, "\n");
+        text = ListItem.Replace(text, "\n- ");
+        text = BlockEnd.Replace(text, "\n");
+        text = CellEnd.Replace(text, " ");
+        text = AnyTag.Replace(text, string.Empty);
+        text = WebUtility.HtmlDecode(text);
+        text = text.Replace("\r\n", "\n").Replace('\r', '\n');
+
+        var lines = text.Split('\n').Select(l => Spaces.Replace(l, " ").Trim());
+        var joined = string.Join("\n", lines).Trim();
+        return BlankLines.Replace(joined, "\n\n");
+    }
+}

src/Backend/AutopilotMonitor.Functions/Services/Notifications/EmailNotificationService.cs

@@ -18,15 +18,8 @@ public class EmailNotificationService : IEmailChannelSender
     public const int MaxRecipients = 5;
     public const string Tag = "alert";
 
-    /// <summary>
-    /// Declares the mail as machine-generated (RFC 3834): auto-responders stay quiet, and filters
-    /// read it as a notification rather than as someone pretending to write a letter.
-    /// </summary>
-    public static readonly IReadOnlyDictionary<string, string> Headers = new Dictionary<string, string>
-    {
-        ["Auto-Submitted"] = "auto-generated",
-        ["X-Auto-Response-Suppress"] = "All",
-    };
+    /// <summary>The platform's automated-mail headers (RFC 3834), shared with the template mails.</summary>
+    public static IReadOnlyDictionary<string, string> Headers => EmailService.AutomatedHeaders;
 
     private readonly EmailService _email;
     private readonly ILogger<EmailNotificationService> _logger;