Handle unparsed email contents as binary.

I think I was wrong in a83b379fd2d676172855825d0592937b234371e2 in
assuming that all email gets properly encoded for transfer. Looking
at the mail gem load method
https://github.com/mikel/mail/blob/b159e0a542962fdd5e292a48cfffa560d7cf412e/lib/mail/mail.rb#L175a,
it reads raw email content from a file in binary mode. So this commit
makes both reading and writing the raw_email a binary mode operation
and adds a data_as_text method for displaying the data in the admin
interface that coerces it to valid utf-8.

4 files changed, 59 insertions, 14 deletions

diff --git a/app/models/raw_email.rb b/app/models/raw_email.rb
index 3b466cb81..907d3c7a0 100644
--- a/app/models/raw_email.rb
+++ b/app/models/raw_email.rb
@@ -39,11 +39,26 @@ class RawEmail < ActiveRecord::Base
 
     def data=(d)
         FileUtils.mkdir_p(directory) unless File.exists?(directory)
-        File.atomic_write(filepath) { |file| file.write(d) }
+        File.atomic_write(filepath) do |file|
+            file.binmode
+            file.write(d)
+        end
     end
 
     def data
-        File.open(filepath, "r").read
+        File.open(filepath, "rb").read
+    end
+
+    def data_as_text
+        text = data
+        if text.respond_to?(:encoding)
+            text = text.encode("UTF-8", :invalid => :replace,
+                                        :undef => :replace,
+                                        :replace => "")
+        else
+            text = Iconv.conv('UTF-8//IGNORE', 'UTF-8', text)
+        end
+        text
     end
 
     def destroy_file_representation!
diff --git a/app/views/admin_raw_email/show.html.erb b/app/views/admin_raw_email/show.html.erb
index f88b00ef0..1de719544 100644
--- a/app/views/admin_raw_email/show.html.erb
+++ b/app/views/admin_raw_email/show.html.erb
@@ -59,5 +59,8 @@
 
 <p><%= link_to "Download",  admin_raw_email_path(@raw_email, :format => 'txt') %></p>
 
-<pre><%=h(@raw_email.data).gsub(/\n/, '<br>').html_safe %></pre>
+<h2>Preview</h2>
+
+For an exact rendering of this email, use the "Download" link.
+<pre><%=h(@raw_email.data_as_text).gsub(/\n/, '<br>').html_safe %></pre>
 
diff --git a/spec/factories/raw_emails.rb b/spec/factories/raw_emails.rb
index 30fb24c37..b271515d2 100644
--- a/spec/factories/raw_emails.rb
+++ b/spec/factories/raw_emails.rb
@@ -1,5 +1,3 @@
 FactoryGirl.define do
-
     factory :raw_email
-
 end
diff --git a/spec/models/raw_email_spec.rb b/spec/models/raw_email_spec.rb
index cbd7d71a5..8e0d3b457 100644
--- a/spec/models/raw_email_spec.rb
+++ b/spec/models/raw_email_spec.rb
@@ -1,3 +1,4 @@
+# -*- encoding : utf-8 -*-
 # == Schema Information
 #
 # Table name: raw_emails
@@ -9,18 +10,46 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe RawEmail do
 
+    def roundtrip_data(raw_email, data)
+        raw_email.data = data
+        raw_email.save!
+        raw_email.reload
+        raw_email.data
+    end
+
     describe :data do
 
         it 'roundtrips data unchanged' do
-            raw_email = RawEmail.new
-            incoming_message = mock_model(IncomingMessage)
-            info_request = mock_model(InfoRequest)
-            incoming_message.stub!(:info_request).and_return(info_request)
-            raw_email.stub!(:incoming_message).and_return(incoming_message)
-            raw_email.data = "Hello, world!"
-            raw_email.save!
-            raw_email.reload
-            raw_email.data.should == "Hello, world!"
+            raw_email = FactoryGirl.create(:incoming_message).raw_email
+            data = roundtrip_data(raw_email, "Hello, world!")
+            data.should == "Hello, world!"
+        end
+
+        it 'returns an unchanged binary string with a valid encoding if the data is non-ascii and non-utf-8' do
+            raw_email = FactoryGirl.create(:incoming_message).raw_email
+            data = roundtrip_data(raw_email, "\xA0")
+
+            if data.respond_to?(:encoding)
+                data.encoding.to_s.should == 'ASCII-8BIT'
+                data.valid_encoding?.should be_true
+                data = data.force_encoding('UTF-8')
+            end
+            data.should == "\xA0"
+        end
+
+    end
+
+    describe :data_as_text do
+
+        it 'returns a utf-8 string with a valid encoding if the data is non-ascii and non-utf8' do
+            raw_email = FactoryGirl.create(:incoming_message).raw_email
+            roundtrip_data(raw_email, "\xA0ccc")
+            data_as_text = raw_email.data_as_text
+            data_as_text.should == "ccc"
+            if data_as_text.respond_to?(:encoding)
+                data_as_text.encoding.to_s.should == 'UTF-8'
+                data_as_text.valid_encoding?.should be_true
+            end
         end
 
     end