Changeset 6192

Timestamp:

01/30/10 10:24:04 (3 years ago)

Author:

MichaelTempest

Message:

Item8384: Give the option of extracting the HTML portion from emails.
The default behaviour is supposed to be unchanged (i.e. extract the text portion).

Location:

trunk/MailInContrib

Files:

• data/System/MailInContrib.txt (modified) (2 diffs)
• lib/Foswiki/Contrib/MailInContrib.pm (modified) (5 diffs)
• lib/Foswiki/Contrib/MailInContrib/Config.spec (modified) (1 diff)
• test/unit/MailInContrib/MailInContribMimeTests.pm (added)
• test/unit/MailInContrib/MailInContribSuite.pm (modified) (1 diff)

trunk/MailInContrib/data/System/MailInContrib.txt

@@ -62 +62 @@
 !MailInContribUserTemplate and edit to taste.
 
+---+ Which part of the mail is added
+
+The module defaults to extracts the plain-text portion of the mail; the HTML portion (if present) is discarded. 
+When the module extracts the plain-text portion, it also discards inline images associated with the HTML (because, whilst they are part of the mail, they are not _attachments_).
+
+%X% *Experimental:*
+You can configure the module to extract the HTML portion instead, in which case the plain-text portion is discarded. The module then also extracts and attaches inline images.
+
 ---+ How the contributor is identified
 The user identity is used for access control checks on the target topic, so you can use Foswiki access controls to protect target topics.
@@ -104 +112 @@
 |  Release: | %$RELEASE% |
 | Change History: | |
+| _not yet released_ | Foswikitask:Item8384: Support HTML email |
 |  3 Feb 2009 | Foswikitask:Item456: ported Foswikitask:Item4955: config options moved to 'Mail and Proxies' Foswikitask:Item5829: force new revision on each submission Foswikitask:Item5843: extract target topic from CC: field - general improvement to rules for extracting the target topic |
 |  10 Mar 2005 | 1.000 Initial version |

trunk/MailInContrib/lib/Foswiki/Contrib/MailInContrib.pm

@@ -147 +147 @@
     $box->{onSuccess}  ||= 'log';
 
+        # Copy the valid domain pattern for external resource URLs (img, script, style)
+        $this->{validUrlPattern} = $box->{validUrlPattern};
+
     # Load the mail templates
     Foswiki::Func::loadTemplate('MailInContrib');
@@ -296 +299 @@
                 my $body        = '';
 
-                _extract( $mail, \$body, \@attachments );
+                                unless ($this->{validUrlPattern}) {
+                                        # Default to files attached to this wiki and files in the mail message
+                                        my $puburl = Foswiki::Func::expandCommonVariables(
+                                                '%PUBURL%',
+                                                $topic, $web
+                                        );
+                                        $this->{validUrlPattern} = qr/cid:|\Q$puburl/;
+                                }
+
+                $this->_extract( $mail, \$body, \@attachments, $box );
 
                 print "Received mail from $sender for $web.$topic\n";
@@ -337 +349 @@
             matching => sub {
                 my $test = shift;
-                if ( defined $kill{ $test->header('Message-ID') } ) {
-                    print STDERR "Delete ", $test->header('Message-ID'), "\n"
+                                my $message_id = $test->header('Message-ID');
+                if ( defined $message_id and defined $kill{ $message_id } ) {
+                    print STDERR "Delete $message_id\n"
                       if $this->{debug};
                     return 1;
@@ -368 +381 @@
 }
 
+sub _extract {
+    my ( $this, $mime, $text, $attach, $box ) = @_;
+        $box->{content}{type} ||= '';
+    if ($box->{content}{type} =~ /html/) {
+        $this->_extractHtmlAndAttachments($mime, $text, $attach, $box->{content});
+    }
+    else {
+        _extractPlainTextAndAttachments($mime, $text, $attach);
+    }
+}
+
+sub _extractHtmlAndAttachments {
+    my ( $this, $mime, $text, $attach, $options ) = @_;
+        my $ct = $mime->content_type || 'text/plain';
+    my $dp = $mime->header('Content-Disposition') || 'inline';
+        print STDERR "\nContent-type: $ct\n" if $this->{debug};
+        if ($ct =~ m[multipart/mixed]) {
+                $this->_extractMultipartMixed($mime, $text, $attach, $options);
+        }
+        elsif ($ct =~ m[multipart/alternative]) {
+                $this->_extractMultipartAlternative($mime, $text, $attach, $options);
+        }
+        elsif ( $ct =~ m[multipart/related] ) {
+                my $found;
+                $found = _extractMultipartHtml($mime, $text, $attach, $options);
+                print STDERR "Found multipart/related HTML\n" if $found and $this->{debug};
+                if (not $found)
+                {
+                        print STDERR "Cannot find HTML. Extracting plain text\n" if $this->{debug};
+                        _extractPlainTextAndAttachments($mime, $text, $attach);
+                }
+        }
+        elsif ( $ct =~ m[text/html] and $dp =~ /inline/ ) {
+                print STDERR "Extracting text/html\n" if $this->{debug};
+                _extractPlainHtml($mime, $text, $options);
+        }
+        else {
+                print STDERR "Extracting plain text and attachments\n" if $this->{debug};
+                _extractPlainTextAndAttachments($mime, $text, $attach);
+        }
+}
+
+sub _extractMultipartMixed {
+    my ( $this, $mime, $text, $attach, $options ) = @_;
+    foreach my $part ( grep { $_ != $mime } $mime->parts() ) {
+                print STDERR "Multipart/mixed: Recursing\n" if $this->{debug};
+                $this->_extractHtmlAndAttachments($part, $text, $attach, $options);
+        }
+}
+
+sub _extractMultipartAlternative {
+    my ( $this, $mime, $text, $attach, $options ) = @_;
+
+        print STDERR "Multipart/alternative\n" if $this->{debug};
+        # See what alternatives are available
+        my @alternates = map +{ 
+                mime => $_, 
+                ct => $_->content_type || 'text/plain', 
+          }, grep { $_ != $mime } $mime->parts();
+
+        my ($multipartRelatedAlternate) = grep { $_->{ct} =~ m[multipart/related] } @alternates;
+        my ($htmlAlternate) = grep { $_->{ct} =~ m[text/html] } @alternates;
+
+        # Pick one
+        my $found;
+        if ($multipartRelatedAlternate and $options->{type} !~ /plain/) {
+                $found = $this->_extractMultipartHtml($multipartRelatedAlternate->{mime}, $text, $attach, $options);
+                print STDERR "Found multipart/related HTML\n" if $found and $this->{debug};
+        }
+        if ($htmlAlternate and not $found) {
+                $found = $this->_extractPlainHtml($htmlAlternate->{mime}, $text, $options);
+                print STDERR "Found text/html\n" if $found and $this->{debug};
+        }
+        if (not $found)
+        {
+                print STDERR "Cannot find HTML - Extracting plain text\n" if $this->{debug};
+                _extractPlainTextAndAttachments($mime, $text, $attach);
+        }
+}
+
+sub _extractMultipartHtml {
+    my ( $this, $mime, $text, $attach, $options ) = @_;
+        my @bits = map +{ 
+                mime => $_, 
+                ct => $_->content_type || 'text/plain', 
+        dp => $_->header('Content-Disposition') || 'inline'
+          }, grep { $_ != $mime } $mime->parts();
+        my ($htmlBit) = grep { $_->{ct} =~ m[text/html] and $_->{dp} =~ /inline/ } @bits;
+        return unless $htmlBit; # Not found
+
+        my $html = $this->_extractAndTrimHtml($htmlBit->{mime});
+        return unless $html;
+    for my $bit (grep { $_ != $htmlBit } @bits)
+        {
+                my $filename = $bit->{mime}->filename();
+                ($filename) = Foswiki::Sandbox::sanitizeAttachmentName( $bit->{mime}->filename() ) if defined $filename;
+                my $cid = $bit->{mime}->header('Content-ID') || '';
+                my $cid_used = '';
+                print STDERR "cid:[$cid]\n" if $cid and $this->{debug};
+                if ($cid =~ /^\s*<?((.*?)\@.*?)>?\s*$/) {
+                        $cid = $1;
+                        ($filename) = Foswiki::Sandbox::sanitizeAttachmentName($2);
+                        $cid_used = ($html =~ s{"cid:\Q$cid\E"}{"%ATTACHURLPATH%/$filename"});
+                }
+                if ( $filename and ($bit->{dp} !~ /inline/ or ($cid and $cid_used) ) ) {
+                        push(
+                                @$attach,
+                                {
+                                        payload  => $bit->{mime}->body(),
+                                        filename => $filename
+                                }
+                        );
+                }
+        }
+        $$text .= "<literal><div class=\"foswikiMailInContribHtml\">$html</div></literal>\n";
+        return 1;
+}
+
+sub _extractPlainHtml {
+    my ( $this, $mime, $text, $options ) = @_;
+        my $html = $this->_extractAndTrimHtml($mime);
+        return unless $html;
+        $$text .= "<literal><div class=\"foswikiMailInContribPlainHtml\">$html</div></literal>\n";
+        return 1;
+}
+
+sub _extractAndTrimHtml {
+    my ($this, $mime) = @_;
+        return unless $mime;
+        my $html = $mime->body();
+        return unless $html;
+
+        # Remove anything outside the body tag, and change the body tag into a div tag
+        # It is better to keep the body tag as a tag (and not just discard it altogether)
+        # because that tag sometimes has attributes that should be retained.
+        $html =~ s{.*<body([^>]*>.*)</body>.*}{<div$1</div>}is;
+
+        # Remove tags that point to external sites
+        my $validUrlPattern = $this->{validUrlPattern};
+        $html =~ s{<(script|style|img)          # opening tag
+                       [^>]+                        # whitespace or attributes
+                           \bsrc=                       # attribute that contains a URL that could be used as e.g. a webbug
+                           (['"])                       # opening quote
+                           (?!$validUrlPattern)         # Zero-width negative lookahead for valid URLs
+                                                        # URLs that don't match this pattern might be evil
+                           [^>]+?                       # the URL itself
+                           \2                           # closing quote that matches the opening quote
+                           [^>]*                        # Any other attributes or whitespace
+                           (?:
+                             >.*?</\1>                  # End of tag, content, and closing tag
+                           |                            #   or
+                                 />                         # End of tag, and tag does not have content
+                           )
+                          }{<em>External link removed</em>}isgx if $validUrlPattern;
+
+        return unless $html =~ /\S/;
+    return $html;
+}
+
+
 # Extract plain text and attachments from the MIME
-sub _extract {
+sub _extractPlainTextAndAttachments {
     my ( $mime, $text, $attach ) = @_;
 
@@ -388 +561 @@
         }
         elsif ( $part != $mime ) {
-            _extract( $part, $text, $attach );
+            _extractPlainTextAndAttachments( $part, $text, $attach );
         }
     }

trunk/MailInContrib/lib/Foswiki/Contrib/MailInContrib/Config.spec

@@ -56 +56 @@
 #    <li> error - treat this as an error (overrides all other options)</li>
 #    <li> spam - save the mail in the spambox topic.
-#    Note: if you clear this, then Foswiki will simply ignore the mail..</li>
+#    Note: if you clear this, then Foswiki will simply ignore the mail.</li>
 #   </ul>
 #  </li>
-#  <li> spambox - optional required of onNoTopic = spam. Name of the topic
+#  <li> spambox - optional, required if onNoTopic = spam. Name of the topic
 #   where you want to save mails that don't have a valid web.topic. You must
 #   specify a full web.topicname
+#  </li>
+#  <li> content - optional, defaults to "extract plain text". 
+#   Specifies what part of the mail to extract and how to process it.
+#   It takes a number of fields:
+#   <ul>
+#    <li> type - specifies type of content to extract. 
+#       Available options:
+#     <ul>
+#      <li> text - extract the plain-text portion </li>
+#      <li> html - extract the HTML portion, by preference 
+#        - reverts to the plain-text if the mail does not contain HTML 
+#      </li>
+#     </ul>
+#    </li>
+#   </ul>
 #  </li>
 # </ul>

trunk/MailInContrib/test/unit/MailInContrib/MailInContribSuite.pm

@@ -8 +8 @@
 sub name { 'MailInContribSuite' }
 
-sub include_tests { qw(MailInContribTests) }
+sub include_tests { qw(MailInContribTests MailInContribMimeTests) }
 
 1;