diff options
Diffstat (limited to 'lib')
25 files changed, 589 insertions, 969 deletions
diff --git a/lib/activesupport_cache_extensions.rb b/lib/activesupport_cache_extensions.rb index f15d72894..2791d5996 100644 --- a/lib/activesupport_cache_extensions.rb +++ b/lib/activesupport_cache_extensions.rb @@ -2,7 +2,7 @@ # Extensions / fixes to ActiveSupport::Cache # # Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. -# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ +# Email: hello@mysociety.org; WWW: http://www.mysociety.org/ # Monkeypatch! ./activesupport/lib/active_support/cache/file_store.rb diff --git a/lib/alaveteli_external_command.rb b/lib/alaveteli_external_command.rb index 5e9a7ee83..fbdee8a62 100644 --- a/lib/alaveteli_external_command.rb +++ b/lib/alaveteli_external_command.rb @@ -22,14 +22,14 @@ module AlaveteliExternalCommand program_path = program_name else found = false - Configuration::utility_search_path.each do |d| + AlaveteliConfiguration::utility_search_path.each do |d| program_path = File.join(d, program_name) if File.file? program_path and File.executable? program_path found = true break end end - raise "Could not find #{program_name} in any of #{Configuration::utility_search_path.join(', ')}" if !found + raise "Could not find #{program_name} in any of #{AlaveteliConfiguration::utility_search_path.join(', ')}" if !found end xc = ExternalCommand.new(program_path, *args) diff --git a/lib/configuration.rb b/lib/configuration.rb index 4a0e0339b..88890856b 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -1,64 +1,77 @@ +require File.dirname(__FILE__) + '/../commonlib/rblib/config' + +# Load intial mySociety config +if ENV["RAILS_ENV"] == "test" + MySociety::Config.set_file(File.join(File.dirname(__FILE__), '..', 'config', 'test'), true) +else + MySociety::Config.set_file(File.join(File.dirname(__FILE__), '..', 'config', 'general'), true) +end +MySociety::Config.load_default + # Configuration values with defaults # TODO: Make this return different values depending on the current rails environment -module Configuration - DEFAULTS = { - :ADMIN_PASSWORD => '', - :ADMIN_USERNAME => '', - :AVAILABLE_LOCALES => '', - :BLACKHOLE_PREFIX => 'do-not-reply-to-this-address', - :BLOG_FEED => '', - :CONTACT_EMAIL => 'contact@localhost', - :CONTACT_NAME => 'Alaveteli', - :COOKIE_STORE_SESSION_SECRET => 'this default is insecure as code is open source, please override for live sites in config/general; this will do for local development', - :DEBUG_RECORD_MEMORY => false, - :DEFAULT_LOCALE => '', - :DISABLE_EMERGENCY_USER => false, - :DOMAIN => 'localhost:3000', - :EXCEPTION_NOTIFICATIONS_FROM => '', - :EXCEPTION_NOTIFICATIONS_TO => '', - :FORCE_REGISTRATION_ON_NEW_REQUEST => false, - :FORCE_SSL => true, - :FORWARD_NONBOUNCE_RESPONSES_TO => 'user-support@localhost', - :FRONTPAGE_PUBLICBODY_EXAMPLES => '', - :GA_CODE => '', - :GAZE_URL => '', - :HTML_TO_PDF_COMMAND => '', - :INCLUDE_DEFAULT_LOCALE_IN_URLS => true, - :INCOMING_EMAIL_DOMAIN => 'localhost', - :INCOMING_EMAIL_PREFIX => '', - :INCOMING_EMAIL_SECRET => 'dummysecret', - :ISO_COUNTRY_CODE => 'GB', - :MAX_REQUESTS_PER_USER_PER_DAY => '', - :MTA_LOG_TYPE => 'exim', - :NEW_RESPONSE_REMINDER_AFTER_DAYS => [3, 10, 24], - :OVERRIDE_ALL_PUBLIC_BODY_REQUEST_EMAILS => '', - :RAW_EMAILS_LOCATION => 'files/raw_emails', - :READ_ONLY => '', - :RECAPTCHA_PRIVATE_KEY => 'x', - :RECAPTCHA_PUBLIC_KEY => 'x', - :REPLY_LATE_AFTER_DAYS => 20, - :REPLY_VERY_LATE_AFTER_DAYS => 40, - :SITE_NAME => 'Alaveteli', - :SKIP_ADMIN_AUTH => false, - :SPECIAL_REPLY_VERY_LATE_AFTER_DAYS => 60, - :THEME_BRANCH => false, - :THEME_URL => "", - :THEME_URLS => [], - :TIME_ZONE => "UTC", - :TRACK_SENDER_EMAIL => 'contact@localhost', - :TRACK_SENDER_NAME => 'Alaveteli', - :TWITTER_USERNAME => '', - :TWITTER_WIDGET_ID => false, - :USE_DEFAULT_BROWSER_LANGUAGE => true, - :USE_GHOSTSCRIPT_COMPRESSION => false, - :UTILITY_SEARCH_PATH => ["/usr/bin", "/usr/local/bin"], - :VARNISH_HOST => '', - :WORKING_OR_CALENDAR_DAYS => 'working', - } +module AlaveteliConfiguration + if !const_defined?(:DEFAULTS) + + DEFAULTS = { + :ADMIN_PASSWORD => '', + :ADMIN_USERNAME => '', + :AVAILABLE_LOCALES => '', + :BLACKHOLE_PREFIX => 'do-not-reply-to-this-address', + :BLOG_FEED => '', + :CONTACT_EMAIL => 'contact@localhost', + :CONTACT_NAME => 'Alaveteli', + :COOKIE_STORE_SESSION_SECRET => 'this default is insecure as code is open source, please override for live sites in config/general; this will do for local development', + :DEBUG_RECORD_MEMORY => false, + :DEFAULT_LOCALE => '', + :DISABLE_EMERGENCY_USER => false, + :DOMAIN => 'localhost:3000', + :EXCEPTION_NOTIFICATIONS_FROM => '', + :EXCEPTION_NOTIFICATIONS_TO => '', + :FORCE_REGISTRATION_ON_NEW_REQUEST => false, + :FORCE_SSL => true, + :FORWARD_NONBOUNCE_RESPONSES_TO => 'user-support@localhost', + :FRONTPAGE_PUBLICBODY_EXAMPLES => '', + :GA_CODE => '', + :GAZE_URL => '', + :HTML_TO_PDF_COMMAND => '', + :INCLUDE_DEFAULT_LOCALE_IN_URLS => true, + :INCOMING_EMAIL_DOMAIN => 'localhost', + :INCOMING_EMAIL_PREFIX => '', + :INCOMING_EMAIL_SECRET => 'dummysecret', + :ISO_COUNTRY_CODE => 'GB', + :MAX_REQUESTS_PER_USER_PER_DAY => '', + :MTA_LOG_TYPE => 'exim', + :NEW_RESPONSE_REMINDER_AFTER_DAYS => [3, 10, 24], + :OVERRIDE_ALL_PUBLIC_BODY_REQUEST_EMAILS => '', + :RAW_EMAILS_LOCATION => 'files/raw_emails', + :READ_ONLY => '', + :RECAPTCHA_PRIVATE_KEY => 'x', + :RECAPTCHA_PUBLIC_KEY => 'x', + :REPLY_LATE_AFTER_DAYS => 20, + :REPLY_VERY_LATE_AFTER_DAYS => 40, + :SITE_NAME => 'Alaveteli', + :SKIP_ADMIN_AUTH => false, + :SPECIAL_REPLY_VERY_LATE_AFTER_DAYS => 60, + :THEME_BRANCH => false, + :THEME_URL => "", + :THEME_URLS => [], + :TIME_ZONE => "UTC", + :TRACK_SENDER_EMAIL => 'contact@localhost', + :TRACK_SENDER_NAME => 'Alaveteli', + :TWITTER_USERNAME => '', + :TWITTER_WIDGET_ID => false, + :USE_DEFAULT_BROWSER_LANGUAGE => true, + :USE_GHOSTSCRIPT_COMPRESSION => false, + :UTILITY_SEARCH_PATH => ["/usr/bin", "/usr/local/bin"], + :VARNISH_HOST => '', + :WORKING_OR_CALENDAR_DAYS => 'working', + } + end - def Configuration.method_missing(name) + def AlaveteliConfiguration.method_missing(name) key = name.to_s.upcase if DEFAULTS.has_key?(key.to_sym) MySociety::Config.get(key, DEFAULTS[key.to_sym]) diff --git a/lib/cookie_store_with_line_break_fix.rb b/lib/cookie_store_with_line_break_fix.rb deleted file mode 100644 index dc623fbd0..000000000 --- a/lib/cookie_store_with_line_break_fix.rb +++ /dev/null @@ -1,19 +0,0 @@ -# See https://makandracards.com/makandra/9443-rails-2-s-cookiestore-produces-invalid-cookie-data-causing-tests-to-break - -# Should be able to remove this when we upgrade to Rails 3 - -module ActionController - module Session - CookieStore.class_eval do - - def call_with_line_break_fix(*args) - status, headers, body = call_without_line_break_fix(*args) - headers['Set-Cookie'].gsub! "\n\n", "\n" if headers['Set-Cookie'].present? - [ status, headers, body ] - end - - alias_method_chain :call, :line_break_fix - - end - end -end
\ No newline at end of file diff --git a/lib/google_translate.rb b/lib/google_translate.rb deleted file mode 100644 index 369e1de3b..000000000 --- a/lib/google_translate.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'rubygems' -require 'net/http' -require 'open-uri' -require 'cgi' -require 'json' - -def detect_language(request, translate_string) - google_api_key = '' - user_ip = URI.encode(request.env['REMOTE_ADDR']) - translate_string = URI.encode(translate_string) - url = "http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q=#{translate_string}&userip=#{user_ip}" - if google_api_key != '' - url += "&key=#{google_api_key}" - end - response = Net::HTTP.get_response(URI.parse(url)) - result = JSON.parse(response.body) - result['responseData']['language'] -end diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index 0a12ab3bb..561946980 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -1,4 +1,35 @@ require 'mail' +require 'mapi/msg' +require 'mapi/convert' + +module Mail + class Message + + # The behaviour of the 'to' and 'cc' methods have changed + # between TMail and Mail; this monkey-patching restores the + # TMail behaviour. The key difference is that when there's an + # invalid address, e.g. '<foo@example.org', Mail returns the + # string as an ActiveSupport::Multibyte::Chars, whereas + # previously TMail would return nil. + + alias_method :old_to, :to + alias_method :old_cc, :cc + + def clean_addresses(old_method, val) + old_result = self.send(old_method, val) + old_result.class == Mail::AddressContainer ? old_result : nil + end + + def to(val = nil) + self.clean_addresses :old_to, val + end + + def cc(val = nil) + self.clean_addresses :old_cc, val + end + + end +end module MailHandler module Backends @@ -38,7 +69,11 @@ module MailHandler # Get the body of a mail part def get_part_body(part) - part.body.decoded + decoded = part.body.decoded + if part.content_type =~ /^text\// + decoded = convert_string_to_utf8_or_binary decoded, part.charset + end + decoded end # Return the first from field if any @@ -60,7 +95,7 @@ module MailHandler def get_from_address(mail) first_from = first_from(mail) if first_from - if first_from.is_a?(String) + if first_from.is_a?(ActiveSupport::Multibyte::Chars) return nil else return first_from.address @@ -74,7 +109,7 @@ module MailHandler def get_from_name(mail) first_from = first_from(mail) if first_from - if first_from.is_a?(String) + if first_from.is_a?(ActiveSupport::Multibyte::Chars) return nil else return first_from.display_name ? eval(%Q{"#{first_from.display_name}"}) : nil @@ -85,7 +120,7 @@ module MailHandler end def get_all_addresses(mail) - envelope_to = mail['envelope-to'] ? [mail['envelope-to'].value] : [] + envelope_to = mail['envelope-to'] ? [mail['envelope-to'].value.to_s] : [] ((mail.to || []) + (mail.cc || []) + (envelope_to || [])).uniq @@ -141,9 +176,14 @@ module MailHandler end elsif get_content_type(part) == 'application/ms-tnef' # A set of attachments in a TNEF file - part.rfc822_attachment = mail_from_tnef(part.body.decoded) - if part.rfc822_attachment.nil? - # Attached mail didn't parse, so treat as binary + begin + part.rfc822_attachment = mail_from_tnef(part.body.decoded) + if part.rfc822_attachment.nil? + # Attached mail didn't parse, so treat as binary + part.content_type = 'application/octet-stream' + end + rescue TNEFParsingError + part.rfc822_attachment = nil part.content_type = 'application/octet-stream' end end @@ -160,8 +200,11 @@ module MailHandler part.parts.each{ |sub_part| expand_and_normalize_parts(sub_part, parent_mail) } else part_filename = get_part_file_name(part) - charset = part.charset # save this, because overwriting content_type also resets charset - + if part.has_charset? + original_charset = part.charset # save this, because overwriting content_type also resets charset + else + original_charset = nil + end # Don't allow nil content_types if get_content_type(part).nil? part.content_type = 'application/octet-stream' @@ -180,7 +223,9 @@ module MailHandler # Use standard content types for Word documents etc. part.content_type = normalise_content_type(get_content_type(part)) decode_attached_part(part, parent_mail) - part.charset = charset + if original_charset + part.charset = original_charset + end end end @@ -228,8 +273,15 @@ module MailHandler def _get_attachment_leaves_recursive(part, within_rfc822_attachment, parent_mail) leaves_found = [] if part.multipart? - raise "no parts on multipart mail" if part.parts.size == 0 - if part.sub_type == 'alternative' + if part.parts.size == 0 + # This is typically caused by a missing final + # MIME boundary, in which case the text of the + # message (including the opening MIME + # boundary) is in part.body, so just add this + # part as a leaf and treat it as text/plain: + part.content_type = "text/plain" + leaves_found += [part] + elsif part.sub_type == 'alternative' best_part = choose_best_alternative(part) leaves_found += _get_attachment_leaves_recursive(best_part, within_rfc822_attachment, @@ -315,8 +367,10 @@ module MailHandler end def address_from_string(string) - Mail::Address.new(string).address + mail = Mail.new + mail.from = string + mail.from[0] end end end -end
\ No newline at end of file +end diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index f756abd1a..322c49bb5 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -64,4 +64,68 @@ module Mail end.join(";\r\n\s") end end -end
\ No newline at end of file + + # HACK: Backport encoding fixes for Ruby 1.8 from Mail 2.5 + # Can be removed when we no longer support Ruby 1.8 + class Ruby18 + def Ruby18.b_value_decode(str) + match = str.match(/\=\?(.+)?\?[Bb]\?(.+)?\?\=/m) + if match + encoding = match[1] + str = Ruby18.decode_base64(match[2]) + # Adding and removing trailing spaces is a workaround + # for Iconv.conv throwing an exception if it finds an + # invalid character at the end of the string, even + # with UTF-8//IGNORE: + # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/ + str = Iconv.conv('UTF-8//IGNORE', fix_encoding(encoding), str + " ")[0...-4] + end + str + end + + def Ruby18.q_value_decode(str) + match = str.match(/\=\?(.+)?\?[Qq]\?(.+)?\?\=/m) + if match + encoding = match[1] + string = match[2].gsub(/_/, '=20') + # Remove trailing = if it exists in a Q encoding + string = string.sub(/\=$/, '') + str = Encodings::QuotedPrintable.decode(string) + # Adding and removing trailing spaces is a workaround + # for Iconv.conv throwing an exception if it finds an + # invalid character at the end of the string, even + # with UTF-8//IGNORE: + # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/ + str = Iconv.conv('UTF-8//IGNORE', fix_encoding(encoding), str + " ")[0...-4] + end + str + end + + private + + def Ruby18.fix_encoding(encoding) + case encoding.upcase + when 'UTF8' + 'UTF-8' + else + encoding + end + end + end + class Ruby19 + + def Ruby19.q_value_decode(str) + match = str.match(/\=\?(.+)?\?[Qq]\?(.+)?\?\=/m) + if match + encoding = match[1] + str = Encodings::QuotedPrintable.decode(match[2].gsub(/_/, '=20')) + # Backport line from mail 2.5 to strip a trailing = character + # Remove trailing = if it exists in a Q encoding + str = str.sub(/\=$/, '') + str.force_encoding(fix_encoding(encoding)) + end + decoded = str.encode("utf-8", :invalid => :replace, :replace => "") + decoded.valid_encoding? ? decoded : decoded.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8") + end + end +end diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb deleted file mode 100644 index 1e241f261..000000000 --- a/lib/mail_handler/backends/tmail_backend.rb +++ /dev/null @@ -1,288 +0,0 @@ -module MailHandler - module Backends - module TmailBackend - - def backend() - 'TMail' - end - - # Turn raw data into a structured TMail::Mail object - # Documentation at http://i.loveruby.net/en/projects/tmail/doc/ - def mail_from_raw_email(data) - # Hack round bug in TMail's MIME decoding. - # Report of TMail bug: - # http://rubyforge.org/tracker/index.php?func=detail&aid=21810&group_id=4512&atid=17370 - copy_of_raw_data = data.gsub(/; boundary=\s+"/im,'; boundary="') - TMail::Mail.parse(copy_of_raw_data) - end - - # Extracts all attachments from the given TNEF file as a TMail::Mail object - def mail_from_tnef(content) - main = TMail::Mail.new - main.set_content_type 'multipart', 'mixed', { 'boundary' => TMail.new_boundary } - tnef_attachments(content).each do |attachment| - tmail_attachment = TMail::Mail.new - tmail_attachment['content-location'] = attachment[:filename] - tmail_attachment.body = attachment[:content] - main.parts << tmail_attachment - end - main - end - - # Return a copy of the file name for the mail part - def get_part_file_name(mail_part) - part_file_name = TMail::Mail.get_part_file_name(mail_part) - if part_file_name.nil? - return nil - end - part_file_name = part_file_name.dup - return part_file_name - end - - # Get the body of a mail part - def get_part_body(mail_part) - mail_part.body - end - - # Return the first from address if any - def get_from_address(mail) - if mail.from_addrs.nil? || mail.from_addrs.size == 0 - return nil - end - mail.from_addrs[0].spec - end - - # Return the first from name if any - def get_from_name(mail) - mail.from_name_if_present - end - - def get_all_addresses(mail) - ((mail.to || []) + - (mail.cc || []) + - (mail.envelope_to || [])).uniq - end - - def empty_return_path?(mail) - return false if mail['return-path'].nil? - return true if mail['return-path'].addr.to_s == '<>' - return false - end - - def get_auto_submitted(mail) - mail['auto-submitted'] ? mail['auto-submitted'].body : nil - end - - def get_content_type(part) - part.content_type - end - - def get_header_string(header, mail) - mail.header_string(header) - end - - # Number the attachments in depth first tree order, for use in URLs. - # XXX This fills in part.rfc822_attachment and part.url_part_number within - # all the parts of the email (see monkeypatches in lib/mail_handler/tmail_extensions and - # lib/mail_handler/mail_extensions for how these attributes are added). ensure_parts_counted - # must be called before using the attributes. - def ensure_parts_counted(mail) - mail.count_parts_count = 0 - _count_parts_recursive(mail, mail) - # we carry on using these numeric ids for attachments uudecoded from within text parts - mail.count_first_uudecode_count = mail.count_parts_count - end - def _count_parts_recursive(part, mail) - if part.multipart? - part.parts.each do |p| - _count_parts_recursive(p, mail) - end - else - part_filename = get_part_file_name(part) - begin - if part.content_type == 'message/rfc822' - # An email attached as text - # e.g. http://www.whatdotheyknow.com/request/64/response/102 - part.rfc822_attachment = mail_from_raw_email(part.body) - elsif part.content_type == 'application/vnd.ms-outlook' || part_filename && AlaveteliFileTypes.filename_to_mimetype(part_filename) == 'application/vnd.ms-outlook' - # An email attached as an Outlook file - # e.g. http://www.whatdotheyknow.com/request/chinese_names_for_british_politi - msg = Mapi::Msg.open(StringIO.new(part.body)) - part.rfc822_attachment = mail_from_raw_email(msg.to_mime.to_s) - elsif part.content_type == 'application/ms-tnef' - # A set of attachments in a TNEF file - part.rfc822_attachment = mail_from_tnef(part.body) - end - rescue - # If attached mail doesn't parse, treat it as text part - part.rfc822_attachment = nil - else - unless part.rfc822_attachment.nil? - _count_parts_recursive(part.rfc822_attachment, mail) - end - end - if part.rfc822_attachment.nil? - mail.count_parts_count += 1 - part.url_part_number = mail.count_parts_count - end - end - end - - def get_attachment_attributes(mail) - leaves = get_attachment_leaves(mail) - # XXX we have to call ensure_parts_counted after get_attachment_leaves - # which is really messy. - ensure_parts_counted(mail) - attachment_attributes = [] - for leaf in leaves - body = get_part_body(leaf) - # As leaf.body causes MIME decoding which uses lots of RAM, do garbage collection here - # to prevent excess memory use. XXX not really sure if this helps reduce - # peak RAM use overall. Anyway, maybe there is something better to do than this. - GC.start - if leaf.within_rfc822_attachment - within_rfc822_subject = leaf.within_rfc822_attachment.subject - # Test to see if we are in the first part of the attached - # RFC822 message and it is text, if so add headers. - # XXX should probably use hunting algorithm to find main text part, rather than - # just expect it to be first. This will do for now though. - if leaf.within_rfc822_attachment == leaf && leaf.content_type == 'text/plain' - headers = "" - for header in [ 'Date', 'Subject', 'From', 'To', 'Cc' ] - if leaf.within_rfc822_attachment.header.include?(header.downcase) - header_value = leaf.within_rfc822_attachment.header[header.downcase] - if !header_value.blank? - headers = headers + header + ": " + header_value.to_s + "\n" - end - end - end - # XXX call _convert_part_body_to_text here, but need to get charset somehow - # e.g. http://www.whatdotheyknow.com/request/1593/response/3088/attach/4/Freedom%20of%20Information%20request%20-%20car%20oval%20sticker:%20Article%2020,%20Convention%20on%20Road%20Traffic%201949.txt - body = headers + "\n" + body - - # This is quick way of getting all headers, but instead we only add some a) to - # make it more usable, b) as at least one authority accidentally leaked security - # information into a header. - #attachment.body = leaf.within_rfc822_attachment.port.to_s - end - end - attachment_attributes << {:url_part_number => leaf.url_part_number, - :content_type => get_content_type(leaf), - :filename => get_part_file_name(leaf), - :charset => leaf.charset, - :within_rfc822_subject => within_rfc822_subject, - :body => body, - :hexdigest => Digest::MD5.hexdigest(body) } - end - attachment_attributes - end - - # (This risks losing info if the unchosen alternative is the only one to contain - # useful info, but let's worry about that another time) - def get_attachment_leaves(mail) - return _get_attachment_leaves_recursive(mail, mail) - end - def _get_attachment_leaves_recursive(curr_mail, parent_mail, within_rfc822_attachment = nil) - leaves_found = [] - if curr_mail.multipart? - if curr_mail.parts.size == 0 - raise "no parts on multipart mail" - end - - if curr_mail.sub_type == 'alternative' - # Choose best part from alternatives - best_part = nil - # Take the last text/plain one, or else the first one - curr_mail.parts.each do |m| - if not best_part - best_part = m - elsif m.content_type == 'text/plain' - best_part = m - end - end - # Take an HTML one as even higher priority. (They tend - # to render better than text/plain, e.g. don't wrap links here: - # http://www.whatdotheyknow.com/request/amount_and_cost_of_freedom_of_in#incoming-72238 ) - curr_mail.parts.each do |m| - if m.content_type == 'text/html' - best_part = m - end - end - leaves_found += _get_attachment_leaves_recursive(best_part, parent_mail, within_rfc822_attachment) - else - # Add all parts - curr_mail.parts.each do |m| - leaves_found += _get_attachment_leaves_recursive(m, parent_mail, within_rfc822_attachment) - end - end - else - # XXX Yuck. this section alters various content_types. That puts - # it into conflict with ensure_parts_counted which it has to be - # called both before and after. It will fail with cases of - # attachments of attachments etc. - charset = curr_mail.charset # save this, because overwriting content_type also resets charset - # Don't allow nil content_types - if curr_mail.content_type.nil? - curr_mail.content_type = 'application/octet-stream' - end - # PDFs often come with this mime type, fix it up for view code - if curr_mail.content_type == 'application/octet-stream' - part_file_name = get_part_file_name(curr_mail) - part_body = get_part_body(curr_mail) - calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(part_file_name, part_body) - if calc_mime - curr_mail.content_type = calc_mime - end - end - - # Use standard content types for Word documents etc. - curr_mail.content_type = normalise_content_type(curr_mail.content_type) - if curr_mail.content_type == 'message/rfc822' - ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable - if curr_mail.rfc822_attachment.nil? - # Attached mail didn't parse, so treat as text - curr_mail.content_type = 'text/plain' - end - end - if curr_mail.content_type == 'application/vnd.ms-outlook' || curr_mail.content_type == 'application/ms-tnef' - ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable - if curr_mail.rfc822_attachment.nil? - # Attached mail didn't parse, so treat as binary - curr_mail.content_type = 'application/octet-stream' - end - end - # If the part is an attachment of email - if curr_mail.content_type == 'message/rfc822' || curr_mail.content_type == 'application/vnd.ms-outlook' || curr_mail.content_type == 'application/ms-tnef' - ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable - leaves_found += _get_attachment_leaves_recursive(curr_mail.rfc822_attachment, parent_mail, curr_mail.rfc822_attachment) - else - # Store leaf - curr_mail.within_rfc822_attachment = within_rfc822_attachment - leaves_found += [curr_mail] - end - # restore original charset - curr_mail.charset = charset - end - return leaves_found - end - - - def address_from_name_and_email(name, email) - if !MySociety::Validate.is_valid_email(email) - raise "invalid email " + email + " passed to address_from_name_and_email" - end - if name.nil? - return TMail::Address.parse(email).to_s - end - # Botch an always quoted RFC address, then parse it - name = name.gsub(/(["\\])/, "\\\\\\1") - TMail::Address.parse('"' + name + '" <' + email + '>').to_s - end - - def address_from_string(string) - TMail::Address.parse(string).address - end - - end - end -end
\ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_extensions.rb b/lib/mail_handler/backends/tmail_extensions.rb deleted file mode 100644 index 3576a8eca..000000000 --- a/lib/mail_handler/backends/tmail_extensions.rb +++ /dev/null @@ -1,138 +0,0 @@ -# lib/tmail_extensions.rb: -# Extensions / fixes to TMail. -# -# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. -# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ - -require 'racc/parser' -require 'tmail' -require 'tmail/scanner' -require 'tmail/utils' -require 'tmail/interface' - -# Monkeypatch! - -# These mainly used in app/models/incoming_message.rb -module TMail - class Mail - # Monkeypatch! Adding some extra members to store extra info in. - - attr_accessor :url_part_number - attr_accessor :rfc822_attachment # when a whole email message is attached as text - attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly) - attr_accessor :count_parts_count - attr_accessor :count_first_uudecode_count - - # Monkeypatch! (check to see if this becomes a standard function in - # TMail::Mail, then use that, whatever it is called) - def Mail.get_part_file_name(part) - file_name = (part['content-location'] && - part['content-location'].body) || - part.sub_header("content-type", "name") || - part.sub_header("content-disposition", "filename") - file_name = file_name.strip if file_name - file_name - end - - # Monkeypatch! Return the name part of from address, or nil if there isn't one - def from_name_if_present - if self.from && self.from_addrs[0].name - return TMail::Unquoter.unquote_and_convert_to(self.from_addrs[0].name, "utf-8") - else - return nil - end - end - - # Monkeypatch! Generalisation of To:, Cc: - def envelope_to(default = nil) - # XXX assumes only one envelope-to, and no parsing needed - val = self.header_string('envelope-to') - return val ? [val,] : [] - end - - # Monkeypatch! - # Bug fix to this function - is for message in humberside-police-odd-mime-type.email - # Which was originally: https://secure.mysociety.org/admin/foi/request/show_raw_email/11209 - # See test in spec/lib/tmail_extensions.rb - def set_content_type( str, sub = nil, param = nil ) - if sub - main, sub = str, sub - else - main, sub = str.split(%r</>, 2) - raise ArgumentError, "sub type missing: #{str.inspect}" unless sub - end - if h = @header['content-type'] - h.main_type = main - h.sub_type = sub - h.params.clear if !h.params.nil? # XXX this if statement is the fix # XXX disabled until works with test - else - store 'Content-Type', "#{main}/#{sub}" - end - @header['content-type'].params.replace param if param - str - end - # Need to make sure this alias calls the Monkeypatch too - alias content_type= set_content_type - - end - - module TextUtils - # Monkeypatch! Much more aggressive list of characters to cause quoting - # than in normal TMail. e.g. Have found real cases where @ needs quoting. - # We list characters to allow, rather than characters not to allow. - NEW_PHRASE_UNSAFE=/[^A-Za-z0-9!#\$%&'*+\-\/=?^_`{|}~ ]/n - def quote_phrase( str ) - (NEW_PHRASE_UNSAFE === str) ? dquote(str) : str - end - end -end - -# Monkeypatch! TMail 1.2.7.1 will parse only one address out of a list of addresses with -# unquoted display parts https://github.com/mikel/tmail/issues#issue/9 - this monkeypatch -# fixes this issue. -module TMail - - class Parser < Racc::Parser - -module_eval <<'..end lib/tmail/parser.y modeval..id2dd1c7d21d', 'lib/tmail/parser.y', 340 - - def self.special_quote_address(str) #:nodoc: - # Takes a string which is an address and adds quotation marks to special - # edge case methods that the RACC parser can not handle. - # - # Right now just handles two edge cases: - # - # Full stop as the last character of the display name: - # Mikel L. <mikel@me.com> - # Returns: - # "Mikel L." <mikel@me.com> - # - # Unquoted @ symbol in the display name: - # mikel@me.com <mikel@me.com> - # Returns: - # "mikel@me.com" <mikel@me.com> - # - # Any other address not matching these patterns just gets returned as is. - case - # This handles the missing "" in an older version of Apple Mail.app - # around the display name when the display name contains a '@' - # like 'mikel@me.com <mikel@me.com>' - # Just quotes it to: '"mikel@me.com" <mikel@me.com>' - when str =~ /\A([^"][^<]+@[^>]+[^"])\s(<.*?>)\Z/ - return "\"#{$1}\" #{$2}" - # This handles cases where 'Mikel A. <mikel@me.com>' which is a trailing - # full stop before the address section. Just quotes it to - # '"Mikel A." <mikel@me.com>' - when str =~ /\A(.*?\.)\s(<.*?>)\s*\Z/ - return "\"#{$1}\" #{$2}" - else - str - end - end - -..end lib/tmail/parser.y modeval..id2dd1c7d21d - end # class Parser - -end # module TMail - - diff --git a/lib/mail_handler/mail_handler.rb b/lib/mail_handler/mail_handler.rb index cd5abfab7..9c955cccd 100644 --- a/lib/mail_handler/mail_handler.rb +++ b/lib/mail_handler/mail_handler.rb @@ -3,16 +3,12 @@ require 'tmpdir' module MailHandler - if RUBY_VERSION.to_f >= 1.9 - require 'mail' - require 'backends/mail_extensions' - require 'backends/mail_backend' - include Backends::MailBackend - else - require 'action_mailer' - require 'backends/tmail_extensions' - require 'backends/tmail_backend' - include Backends::TmailBackend + require 'mail' + require 'backends/mail_extensions' + require 'backends/mail_backend' + include Backends::MailBackend + + class TNEFParsingError < StandardError end # Returns a set of attachments from the given TNEF contents @@ -21,14 +17,14 @@ module MailHandler def tnef_attachments(content) attachments = [] Dir.mktmpdir do |dir| - IO.popen("#{`which tnef`.chomp} -K -C #{dir}", "wb") do |f| + IO.popen("tnef -K -C #{dir} 2> /dev/null", "wb") do |f| f.write(content) f.close if $?.signaled? raise IOError, "tnef exited with signal #{$?.termsig}" end if $?.exited? && $?.exitstatus != 0 - raise IOError, "tnef exited with status #{$?.exitstatus}" + raise TNEFParsingError, "tnef exited with status #{$?.exitstatus}" end end found = 0 @@ -41,7 +37,7 @@ module MailHandler end end if found == 0 - raise IOError, "tnef produced no attachments" + raise TNEFParsingError, "tnef produced no attachments" end end attachments diff --git a/lib/no_constraint_disabling.rb b/lib/no_constraint_disabling.rb new file mode 100644 index 000000000..d515a959a --- /dev/null +++ b/lib/no_constraint_disabling.rb @@ -0,0 +1,110 @@ +# In order to work around the problem of the database use not having +# the permission to disable referential integrity when loading fixtures, +# we redefine disable_referential_integrity so that it doesn't try to +# disable foreign key constraints, and redefine the +# ActiveRecord::Fixtures.create_fixtures method to pay attention to the order +# which fixture tables are passed so that foreign key constraints won't be +# violated. The only lines that are changed from the initial definition +# are those between the "***" comments +require 'active_record/fixtures' +require 'active_record/connection_adapters/postgresql_adapter' +module ActiveRecord + module ConnectionAdapters + class PostgreSQLAdapter < AbstractAdapter + def disable_referential_integrity(&block) + transaction { + yield + } + end + end + end +end + +module ActiveRecord + class Fixtures + + def self.create_fixtures(fixtures_directory, table_names, class_names = {}) + table_names = [table_names].flatten.map { |n| n.to_s } + table_names.each { |n| + class_names[n.tr('/', '_').to_sym] = n.classify if n.include?('/') + } + + # FIXME: Apparently JK uses this. + connection = block_given? ? yield : ActiveRecord::Base.connection + + files_to_read = table_names.reject { |table_name| + fixture_is_cached?(connection, table_name) + } + + unless files_to_read.empty? + connection.disable_referential_integrity do + fixtures_map = {} + + fixture_files = files_to_read.map do |path| + table_name = path.tr '/', '_' + + fixtures_map[path] = ActiveRecord::Fixtures.new( + connection, + table_name, + class_names[table_name.to_sym] || table_name.classify, + File.join(fixtures_directory, path)) + end + + all_loaded_fixtures.update(fixtures_map) + + connection.transaction(:requires_new => true) do + # Patch - replace this... + # *** + # fixture_files.each do |ff| + # conn = ff.model_class.respond_to?(:connection) ? ff.model_class.connection : connection + # table_rows = ff.table_rows + # + # table_rows.keys.each do |table| + # conn.delete "DELETE FROM #{conn.quote_table_name(table)}", 'Fixture Delete' + # end + # + # table_rows.each do |table_name,rows| + # rows.each do |row| + # conn.insert_fixture(row, table_name) + # end + # end + # end + # *** + # ... with this + fixture_files.reverse.each do |ff| + conn = ff.model_class.respond_to?(:connection) ? ff.model_class.connection : connection + table_rows = ff.table_rows + + table_rows.keys.each do |table| + conn.delete "DELETE FROM #{conn.quote_table_name(table)}", 'Fixture Delete' + end + end + + fixture_files.each do |ff| + conn = ff.model_class.respond_to?(:connection) ? ff.model_class.connection : connection + table_rows = ff.table_rows + table_rows.each do |table_name,rows| + rows.each do |row| + conn.insert_fixture(row, table_name) + end + end + end + # *** + + # Cap primary key sequences to max(pk). + if connection.respond_to?(:reset_pk_sequence!) + table_names.each do |table_name| + connection.reset_pk_sequence!(table_name.tr('/', '_')) + end + end + end + + cache_fixtures(connection, fixtures_map) + end + end + cached_fixtures(connection, table_names) + end + + end + +end diff --git a/lib/normalize_string.rb b/lib/normalize_string.rb new file mode 100644 index 000000000..f02b18ee0 --- /dev/null +++ b/lib/normalize_string.rb @@ -0,0 +1,86 @@ +require 'iconv' unless RUBY_VERSION.to_f >= 1.9 +require 'charlock_holmes' + +class EncodingNormalizationError < StandardError +end + +def normalize_string_to_utf8(s, suggested_character_encoding=nil) + + # Make a list of encodings to try: + to_try = [] + + guessed_encoding = CharlockHolmes::EncodingDetector.detect(s)[:encoding] + guessed_encoding ||= '' + + # It's reasonably common for windows-1252 text to be mislabelled + # as ISO-8859-1, so try that first if charlock_holmes guessed + # that. However, it can also easily misidentify UTF-8 strings as + # ISO-8859-1 so we don't want to go with the guess by default... + to_try.push guessed_encoding if guessed_encoding.downcase == 'windows-1252' + + to_try.push suggested_character_encoding if suggested_character_encoding + to_try.push 'UTF-8' + to_try.push guessed_encoding + + to_try.each do |from_encoding| + if RUBY_VERSION.to_f >= 1.9 + begin + s.force_encoding from_encoding + return s.encode('UTF-8') if s.valid_encoding? + rescue ArgumentError + # We get this is there are invalid bytes when + # interpreted as from_encoding at the point of + # the encode('UTF-8'); move onto the next one... + end + else + to_encoding = 'UTF-8' + begin + converted = Iconv.conv 'UTF-8', from_encoding, s + return converted + rescue Iconv::Failure + # We get this is there are invalid bytes when + # interpreted as from_encoding at the point of + # the Iconv.iconv; move onto the next one... + end + end + end + raise EncodingNormalizationError, "Couldn't find a valid character encoding for the string" + +end + +def convert_string_to_utf8_or_binary(s, suggested_character_encoding=nil) + # This function exists to help to keep consistent with the + # behaviour of earlier versions of Alaveteli: in the code as it + # is, there are situations where it's expected that we generally + # have a UTF-8 encoded string, but if the source data was + # unintepretable under any character encoding, the string may be + # binary data (i.e. invalid UTF-8). Such a string would then be + # mangled into valid UTF-8 by _sanitize_text for the purposes of + # display. + + # This seems unsatisfactory to me - two better alternatives would + # be either: (a) to mangle the data into valid UTF-8 in this + # method or (b) to treat the 'text/*' attachment as + # 'application/octet-stream' instead. However, for the purposes + # of the transition to Ruby 1.9 and/or Rails 3 we just want the + # behaviour to be as similar as possible. + + begin + result = normalize_string_to_utf8 s, suggested_character_encoding + rescue EncodingNormalizationError + result = s + s.force_encoding 'ASCII-8BIT' if RUBY_VERSION.to_f >= 1.9 + end + result +end + +def log_text_details(message, text) + if RUBY_VERSION.to_f >= 1.9 + STDERR.puts "#{message}, we have text: #{text}, of class #{text.class} and encoding #{text.encoding}" + else + STDERR.puts "#{message}, we have text: #{text}, of class #{text.class}" + end + filename = "/var/tmp/#{Digest::MD5.hexdigest(text)}.txt" + File.open(filename, "wb") { |f| f.write text } + STDERR.puts "#{message}, the filename is: #{filename}" +end diff --git a/lib/old_rubygems_patch.rb b/lib/old_rubygems_patch.rb deleted file mode 100644 index 3001a7381..000000000 --- a/lib/old_rubygems_patch.rb +++ /dev/null @@ -1,46 +0,0 @@ -if File.exist? File.join(File.dirname(__FILE__),'..','vendor','rails','railties','lib','rails','gem_dependency.rb') - require File.join(File.dirname(__FILE__),'..','vendor','rails','railties','lib','rails','gem_dependency.rb') -else - require 'rails/gem_dependency' -end - -module Rails - class GemDependency < Gem::Dependency - - # This definition of the requirement method is a patch - if !method_defined?(:requirement) - def requirement - req = version_requirements - end - end - - def add_load_paths - self.class.add_frozen_gem_path - return if @loaded || @load_paths_added - if framework_gem? - @load_paths_added = @loaded = @frozen = true - return - end - - begin - dep = Gem::Dependency.new(name, requirement) - spec = Gem.source_index.find { |_,s| s.satisfies_requirement?(dep) }.last - spec.activate # a way that exists - rescue - begin - gem self.name, self.requirement # < 1.8 unhappy way - # This second rescue is a patch - fall back to passing Rails::GemDependency to gem - # for older rubygems - rescue ArgumentError - gem self - end - end - - @spec = Gem.loaded_specs[name] - @frozen = @spec.loaded_from.include?(self.class.unpacked_path) if @spec - @load_paths_added = true - rescue Gem::LoadError - end - end - -end diff --git a/lib/patches/fixtures_constraint_disabling.rb b/lib/patches/fixtures_constraint_disabling.rb deleted file mode 100644 index 7d97e81f7..000000000 --- a/lib/patches/fixtures_constraint_disabling.rb +++ /dev/null @@ -1,21 +0,0 @@ -# An alternative way of disabling foreign keys in fixture loading in Postgres and -# does not require superuser permissions -# http://kopongo.com/2008/7/25/postgres-ri_constrainttrigger-error -require 'active_record/connection_adapters/postgresql_adapter' -module ActiveRecord - module ConnectionAdapters - class PostgreSQLAdapter < AbstractAdapter - def disable_referential_integrity(&block) - transaction { - begin - execute "SET CONSTRAINTS ALL DEFERRED" - yield - ensure - execute "SET CONSTRAINTS ALL IMMEDIATE" - end - } - end - end - end -end - diff --git a/lib/public_body_categories.rb b/lib/public_body_categories.rb index c6f0a6690..7f548b130 100644 --- a/lib/public_body_categories.rb +++ b/lib/public_body_categories.rb @@ -2,7 +2,7 @@ # Categorisations of public bodies. # # Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. -# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ +# Email: hello@mysociety.org; WWW: http://www.mysociety.org/ class PublicBodyCategories diff --git a/lib/rack_quote_monkeypatch.rb b/lib/rack_quote_monkeypatch.rb deleted file mode 100644 index b477ac0cb..000000000 --- a/lib/rack_quote_monkeypatch.rb +++ /dev/null @@ -1,65 +0,0 @@ -# There's a bug in Rack 1.1.x which is fixed in Rack 1.2, but our -# current version of Rails won't use that. So for now, monkeypatch, -# This can be dropped when we move to Rails 3. -# -# See https://github.com/mysociety/alaveteli/issues/38 for Alaveteli -# bug report -# -# More info about the monkeypatch: -# http://thewebfellas.com/blog/2010/7/15/rails-2-3-8-rack-1-1-and-the-curious-case-of-the-missing-quotes - -module Rack - module Utils - def parse_query(qs, d = nil) - params = {} - - (qs || '').split(d ? /[#{d}] */n : DEFAULT_SEP).each do |p| - k, v = p.split('=', 2).map { |x| unescape(x) } - if cur = params[k] - if cur.class == Array - params[k] << v - else - params[k] = [cur, v] - end - else - params[k] = v - end - end - - return params - end - module_function :parse_query - - def normalize_params(params, name, v = nil) - name =~ %r(\A[\[\]]*([^\[\]]+)\]*) - k = $1 || '' - after = $' || '' - - return if k.empty? - - if after == "" - params[k] = v - elsif after == "[]" - params[k] ||= [] - raise TypeError, "expected Array (got #{params[k].class.name}) for param `#{k}'" unless params[k].is_a?(Array) - params[k] << v - elsif after =~ %r(^\[\]\[([^\[\]]+)\]$) || after =~ %r(^\[\](.+)$) - child_key = $1 - params[k] ||= [] - raise TypeError, "expected Array (got #{params[k].class.name}) for param `#{k}'" unless params[k].is_a?(Array) - if params[k].last.is_a?(Hash) && !params[k].last.key?(child_key) - normalize_params(params[k].last, child_key, v) - else - params[k] << normalize_params({}, child_key, v) - end - else - params[k] ||= {} - raise TypeError, "expected Hash (got #{params[k].class.name}) for param `#{k}'" unless params[k].is_a?(Hash) - params[k] = normalize_params(params[k], after, v) - end - - return params - end - module_function :normalize_params - end -end diff --git a/lib/sendmail_return_path.rb b/lib/sendmail_return_path.rb deleted file mode 100644 index 23c4d4376..000000000 --- a/lib/sendmail_return_path.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Monkeypatch! -# Grrr, semantics of smtp and sendmail send should be the same with regard to setting return path - -# See test in spec/lib/sendmail_return_path_spec.rb - -module ActionMailer - class Base - def perform_delivery_sendmail(mail) - sender = (mail['return-path'] && mail['return-path'].spec) || mail.from.first - - sendmail_args = sendmail_settings[:arguments].dup - sendmail_args += " -f \"#{sender}\"" - - IO.popen("#{sendmail_settings[:location]} #{sendmail_args}","w+") do |sm| - sm.print(mail.encoded.gsub(/\r/, '')) - sm.flush - end - end - end -end - diff --git a/lib/tasks/.gitkeep b/lib/tasks/.gitkeep new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/lib/tasks/.gitkeep diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index c73c2584e..ace7205ae 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -1,7 +1,3 @@ -# Rails won't automatically load rakefiles from gems - see -# http://stackoverflow.com/questions/1878640/including-rake-tasks-in-gems -Dir["#{Gem.searcher.find('gettext_i18n_rails').full_gem_path}/lib/tasks/**/*.rake"].each { |ext| load ext } - namespace :gettext do desc 'Rewrite .po files into a consistent msgmerge format' diff --git a/lib/tasks/rspec.rake b/lib/tasks/rspec.rake deleted file mode 100644 index d4fd4a9ff..000000000 --- a/lib/tasks/rspec.rake +++ /dev/null @@ -1,148 +0,0 @@ -rspec_gem_dir = nil -Dir["#{Rails.root}/vendor/gems/*"].each do |subdir| - rspec_gem_dir = subdir if subdir.gsub("#{Rails.root}/vendor/gems/","") =~ /^(\w+-)?rspec-(\d+)/ && File.exist?("#{subdir}/lib/spec/rake/spectask.rb") -end -rspec_plugin_dir = File.expand_path(File.dirname(__FILE__) + '/../../vendor/plugins/rspec') - -if rspec_gem_dir && (test ?d, rspec_plugin_dir) - raise "\n#{'*'*50}\nYou have rspec installed in both vendor/gems and vendor/plugins\nPlease pick one and dispose of the other.\n#{'*'*50}\n\n" -end - -if rspec_gem_dir - $LOAD_PATH.unshift("#{rspec_gem_dir}/lib") -elsif File.exist?(rspec_plugin_dir) - $LOAD_PATH.unshift("#{rspec_plugin_dir}/lib") -end - -# Don't load rspec if running "rake gems:*" -unless ARGV.any? {|a| a =~ /^gems/} - -begin - require 'spec/rake/spectask' -rescue MissingSourceFile - module Spec - module Rake - class SpecTask - if defined?(::Rake::DSL) - include ::Rake::DSL - end - def initialize(name) - task name do - # if rspec-rails is a configured gem, this will output helpful material and exit ... - require File.expand_path(File.join(File.dirname(__FILE__),"..","..","config","environment")) - - # ... otherwise, do this: - raise <<-MSG - -#{"*" * 80} -* You are trying to run an rspec rake task defined in -* #{__FILE__}, -* but rspec can not be found in vendor/gems, vendor/plugins or system gems. -#{"*" * 80} -MSG - end - end - end - end - end -end - -Rake.application.instance_variable_get('@tasks').delete('default') - -spec_prereq = File.exist?(File.join(Rails.root, 'config', 'database.yml')) ? "db:test:prepare" : :noop -task :noop do -end - -task :default => :spec -task :stats => "spec:statsetup" -task :test => ['spec'] -task :cruise => ['spec'] - -desc "Run all specs in spec directory (excluding plugin specs)" -Spec::Rake::SpecTask.new(:spec => spec_prereq) do |t| - t.spec_opts = ['--options', "\"#{Rails.root}/spec/spec.opts\""] - t.spec_files = FileList['spec/**/*_spec.rb'] -end - -namespace :spec do - desc "Run all specs in spec directory with RCov (excluding plugin specs)" - Spec::Rake::SpecTask.new(:rcov) do |t| - t.spec_opts = ['--options', "\"#{Rails.root}/spec/spec.opts\""] - t.spec_files = FileList['spec/**/*_spec.rb'] - t.rcov = true - t.rcov_opts = lambda do - IO.readlines("#{Rails.root}/spec/rcov.opts").map {|l| l.chomp.split " "}.flatten - end - end - - desc "Print Specdoc for all specs (excluding plugin specs)" - Spec::Rake::SpecTask.new(:doc) do |t| - t.spec_opts = ["--format", "specdoc", "--dry-run"] - t.spec_files = FileList['spec/**/*_spec.rb'] - end - - desc "Print Specdoc for all plugin examples" - Spec::Rake::SpecTask.new(:plugin_doc) do |t| - t.spec_opts = ["--format", "specdoc", "--dry-run"] - t.spec_files = FileList['vendor/plugins/**/spec/**/*_spec.rb'].exclude('vendor/plugins/rspec/*') - end - - [:models, :controllers, :views, :helpers, :lib, :integration].each do |sub| - desc "Run the code examples in spec/#{sub}" - Spec::Rake::SpecTask.new(sub => spec_prereq) do |t| - t.spec_opts = ['--options', "\"#{Rails.root}/spec/spec.opts\""] - t.spec_files = FileList["spec/#{sub}/**/*_spec.rb"] - end - end - - desc "Run the code examples in vendor/plugins (except RSpec's own)" - Spec::Rake::SpecTask.new(:plugins => spec_prereq) do |t| - t.spec_opts = ['--options', "\"#{Rails.root}/spec/spec.opts\""] - t.spec_files = FileList['vendor/plugins/**/spec/**/*_spec.rb'].exclude('vendor/plugins/rspec/*').exclude("vendor/plugins/rspec-rails/*") - end - - namespace :plugins do - desc "Runs the examples for rspec_on_rails" - Spec::Rake::SpecTask.new(:rspec_on_rails) do |t| - t.spec_opts = ['--options', "\"#{Rails.root}/spec/spec.opts\""] - t.spec_files = FileList['vendor/plugins/rspec-rails/spec/**/*_spec.rb'] - end - end - - # Setup specs for stats - task :statsetup do - require 'code_statistics' - ::STATS_DIRECTORIES << %w(Model\ specs spec/models) if File.exist?('spec/models') - ::STATS_DIRECTORIES << %w(View\ specs spec/views) if File.exist?('spec/views') - ::STATS_DIRECTORIES << %w(Controller\ specs spec/controllers) if File.exist?('spec/controllers') - ::STATS_DIRECTORIES << %w(Helper\ specs spec/helpers) if File.exist?('spec/helpers') - ::STATS_DIRECTORIES << %w(Library\ specs spec/lib) if File.exist?('spec/lib') - ::STATS_DIRECTORIES << %w(Routing\ specs spec/routing) if File.exist?('spec/routing') - ::STATS_DIRECTORIES << %w(Integration\ specs spec/integration) if File.exist?('spec/integration') - ::CodeStatistics::TEST_TYPES << "Model specs" if File.exist?('spec/models') - ::CodeStatistics::TEST_TYPES << "View specs" if File.exist?('spec/views') - ::CodeStatistics::TEST_TYPES << "Controller specs" if File.exist?('spec/controllers') - ::CodeStatistics::TEST_TYPES << "Helper specs" if File.exist?('spec/helpers') - ::CodeStatistics::TEST_TYPES << "Library specs" if File.exist?('spec/lib') - ::CodeStatistics::TEST_TYPES << "Routing specs" if File.exist?('spec/routing') - ::CodeStatistics::TEST_TYPES << "Integration specs" if File.exist?('spec/integration') - end - - namespace :db do - namespace :fixtures do - desc "Load fixtures (from spec/fixtures) into the current environment's database. Load specific fixtures using FIXTURES=x,y. Load from subdirectory in test/fixtures using FIXTURES_DIR=z." - task :load => :environment do - ActiveRecord::Base.establish_connection(Rails.env) - base_dir = File.join(Rails.root, 'spec', 'fixtures') - fixtures_dir = ENV['FIXTURES_DIR'] ? File.join(base_dir, ENV['FIXTURES_DIR']) : base_dir - - require 'active_record/fixtures' - (ENV['FIXTURES'] ? ENV['FIXTURES'].split(/,/).map {|f| File.join(fixtures_dir, f) } : Dir.glob(File.join(fixtures_dir, '*.{yml,csv}'))).each do |fixture_file| - Fixtures.create_fixtures(File.dirname(fixture_file), File.basename(fixture_file, '.*')) - end - end - end - end -end - -end diff --git a/lib/tasks/temp.rake b/lib/tasks/temp.rake index e49a84ecb..f0085b5e1 100644 --- a/lib/tasks/temp.rake +++ b/lib/tasks/temp.rake @@ -50,4 +50,154 @@ namespace :temp do end end + desc 'Create a CSV file of a random selection of raw emails, for comparing hexdigests' + task :random_attachments_hexdigests => :environment do + + # The idea is to run this under the Rail 2 codebase, where + # Tmail was used to extract the attachements, and the task + # will output all of those file paths in a CSV file, and a + # list of the raw email files in another. The latter file is + # useful so that one can easily tar up the emails with: + # + # tar cvz -T raw-email-files -f raw_emails.tar.gz + # + # Then you can switch to the Rails 3 codebase, where + # attachment parsing is done via + # recompute_attachments_hexdigests + + require 'csv' + + File.open('raw-email-files', 'w') do |f| + CSV.open('attachment-hexdigests.csv', 'w') do |csv| + csv << ['filepath', 'i', 'url_part_number', 'hexdigest'] + IncomingMessage.all(:order => 'RANDOM()', :limit => 1000).each do |incoming_message| + # raw_email.filepath fails unless the + # incoming_message has an associated request + next unless incoming_message.info_request + raw_email = incoming_message.raw_email + f.puts raw_email.filepath + incoming_message.foi_attachments.each_with_index do |attachment, i| + csv << [raw_email.filepath, i, attachment.url_part_number, attachment.hexdigest] + end + end + end + end + + end + + + desc 'Check the hexdigests of attachments in emails on disk' + task :recompute_attachments_hexdigests => :environment do + + require 'csv' + require 'digest/md5' + + OldAttachment = Struct.new :filename, :attachment_index, :url_part_number, :hexdigest + + filename_to_attachments = Hash.new {|h,k| h[k] = []} + + header_line = true + CSV.foreach('attachment-hexdigests.csv') do |filename, attachment_index, url_part_number, hexdigest| + if header_line + header_line = false + else + filename_to_attachments[filename].push OldAttachment.new filename, attachment_index, url_part_number, hexdigest + end + end + + total_attachments = 0 + attachments_with_different_hexdigest = 0 + files_with_different_numbers_of_attachments = 0 + no_tnef_attachments = 0 + no_parts_in_multipart = 0 + + multipart_error = "no parts on multipart mail" + tnef_error = "tnef produced no attachments" + + # Now check each file: + filename_to_attachments.each do |filename, old_attachments| + + # Currently it doesn't seem to be possible to reuse the + # attachment parsing code in Alaveteli without saving + # objects to the database, so reproduce what it does: + + raw_email = nil + File.open(filename) do |f| + raw_email = f.read + end + mail = MailHandler.mail_from_raw_email(raw_email) + + begin + attachment_attributes = MailHandler.get_attachment_attributes(mail) + rescue IOError => e + if e.message == tnef_error + puts "#{filename} #{tnef_error}" + no_tnef_attachments += 1 + next + else + raise + end + rescue Exception => e + if e.message == multipart_error + puts "#{filename} #{multipart_error}" + no_parts_in_multipart += 1 + next + else + raise + end + end + + if attachment_attributes.length != old_attachments.length + puts "#{filename} the number of old attachments #{old_attachments.length} didn't match the number of new attachments #{attachment_attributes.length}" + files_with_different_numbers_of_attachments += 1 + else + old_attachments.each_with_index do |old_attachment, i| + total_attachments += 1 + attrs = attachment_attributes[i] + old_hexdigest = old_attachment.hexdigest + new_hexdigest = attrs[:hexdigest] + new_content_type = attrs[:content_type] + old_url_part_number = old_attachment.url_part_number.to_i + new_url_part_number = attrs[:url_part_number] + if old_url_part_number != new_url_part_number + puts "#{i} #{filename} old_url_part_number #{old_url_part_number}, new_url_part_number #{new_url_part_number}" + end + if old_hexdigest != new_hexdigest + body = attrs[:body] + # First, if the content type is one of + # text/plain, text/html or application/rtf try + # changing CRLF to LF and calculating a new + # digest - we generally don't worry about + # these changes: + new_converted_hexdigest = nil + if ["text/plain", "text/html", "application/rtf"].include? new_content_type + converted_body = body.gsub /\r\n/, "\n" + new_converted_hexdigest = Digest::MD5.hexdigest converted_body + puts "new_converted_hexdigest is #{new_converted_hexdigest}" + end + if (! new_converted_hexdigest) || (old_hexdigest != new_converted_hexdigest) + puts "#{i} #{filename} old_hexdigest #{old_hexdigest} wasn't the same as new_hexdigest #{new_hexdigest}" + puts " body was of length #{body.length}" + puts " content type was: #{new_content_type}" + path = "/tmp/#{new_hexdigest}" + f = File.new path, "w" + f.write body + f.close + puts " wrote body to #{path}" + attachments_with_different_hexdigest += 1 + end + end + end + end + + end + + puts "total_attachments: #{total_attachments}" + puts "attachments_with_different_hexdigest: #{attachments_with_different_hexdigest}" + puts "files_with_different_numbers_of_attachments: #{files_with_different_numbers_of_attachments}" + puts "no_tnef_attachments: #{no_tnef_attachments}" + puts "no_parts_in_multipart: #{no_parts_in_multipart}" + + end + end diff --git a/lib/tasks/themes.rake b/lib/tasks/themes.rake index 14aa15551..cbd3d123e 100644 --- a/lib/tasks/themes.rake +++ b/lib/tasks/themes.rake @@ -31,7 +31,7 @@ namespace :themes do if system(clone_command) Dir.chdir install_path do # First try to checkout a specific branch of the theme - tag_checked_out = checkout_remote_branch(Configuration::theme_branch) if Configuration::theme_branch + tag_checked_out = checkout_remote_branch(AlaveteliConfiguration::theme_branch) if AlaveteliConfiguration::theme_branch if !tag_checked_out # try to checkout a tag exactly matching ALAVETELI VERSION tag_checked_out = checkout_tag(ALAVETELI_VERSION) @@ -94,10 +94,10 @@ namespace :themes do desc "Install themes specified in the config file's THEME_URLS" task :install => :environment do verbose = true - Configuration::theme_urls.each{ |theme_url| install_theme(theme_url, verbose) } - if ! Configuration::theme_url.blank? + AlaveteliConfiguration::theme_urls.each{ |theme_url| install_theme(theme_url, verbose) } + if ! AlaveteliConfiguration::theme_url.blank? # Old version of the above, for backwards compatibility - install_theme(Configuration::theme_url, verbose, deprecated=true) + install_theme(AlaveteliConfiguration::theme_url, verbose, deprecated=true) end end -end
\ No newline at end of file +end diff --git a/lib/tasks/translation.rake b/lib/tasks/translation.rake index ff07fc6f6..351faef2c 100644 --- a/lib/tasks/translation.rake +++ b/lib/tasks/translation.rake @@ -42,14 +42,14 @@ namespace :translation do output_file = File.open(File.join(ENV['DIR'], 'message_preview.txt'), 'w') # outgoing mailer - request_email = OutgoingMailer.create_initial_request(info_request, initial_request) + request_email = OutgoingMailer.initial_request(info_request, initial_request) write_email(request_email, 'Initial Request', output_file) - followup_email = OutgoingMailer.create_followup(info_request, follow_up, nil) + followup_email = OutgoingMailer.followup(info_request, follow_up, nil) write_email(followup_email, 'Follow up', output_file) # contact mailer - contact_email = ContactMailer.create_to_admin_message(info_request.user_name, + contact_email = ContactMailer.to_admin_message(info_request.user_name, info_request.user.email, 'A test message', 'Hello!', @@ -59,20 +59,20 @@ namespace :translation do write_email(contact_email, 'Contact email (to admin)', output_file) - user_contact_email = ContactMailer.create_user_message(info_request.user, + user_contact_email = ContactMailer.user_message(info_request.user, info_request.user, 'http://www.example.com/user', 'A test message', 'Hello!') write_email(user_contact_email, 'Contact email (user to user)', output_file) - admin_contact_email = ContactMailer.create_from_admin_message(info_request.user, + admin_contact_email = ContactMailer.from_admin_message(info_request.user, 'A test message', 'Hello!') write_email(admin_contact_email, 'Contact email (admin to user)', output_file) # request mailer - fake_response_email = RequestMailer.create_fake_response(info_request, + fake_response_email = RequestMailer.fake_response(info_request, info_request.user, "test body", "attachment.txt", @@ -89,54 +89,54 @@ namespace :translation do response_mail = MailHandler.mail_from_raw_email(content) response_mail.from = "authority@example.com" - stopped_responses_email = RequestMailer.create_stopped_responses(info_request, + stopped_responses_email = RequestMailer.stopped_responses(info_request, response_mail, content) write_email(stopped_responses_email, 'Bounce if someone sends email to a request that has had responses stopped', output_file) - requires_admin_email = RequestMailer.create_requires_admin(info_request) + requires_admin_email = RequestMailer.requires_admin(info_request) write_email(requires_admin_email, 'Drawing admin attention to a response', output_file) - new_response_email = RequestMailer.create_new_response(info_request, incoming_message) + new_response_email = RequestMailer.new_response(info_request, incoming_message) write_email(new_response_email, 'Telling the requester that a new response has arrived', output_file) - overdue_alert_email = RequestMailer.create_overdue_alert(info_request, info_request.user) + overdue_alert_email = RequestMailer.overdue_alert(info_request, info_request.user) write_email(overdue_alert_email, 'Telling the requester that the public body is late in replying', output_file) - very_overdue_alert_email = RequestMailer.create_very_overdue_alert(info_request, info_request.user) + very_overdue_alert_email = RequestMailer.very_overdue_alert(info_request, info_request.user) write_email(very_overdue_alert_email, 'Telling the requester that the public body is very late in replying', output_file) - response_reminder_alert_email = RequestMailer.create_new_response_reminder_alert(info_request, + response_reminder_alert_email = RequestMailer.new_response_reminder_alert(info_request, incoming_message) write_email(response_reminder_alert_email, 'Telling the requester that they need to say if the new response contains info or not', output_file) - old_unclassified_email = RequestMailer.create_old_unclassified_updated(info_request) + old_unclassified_email = RequestMailer.old_unclassified_updated(info_request) write_email(old_unclassified_email, 'Telling the requester that someone updated their old unclassified request', output_file) - not_clarified_alert_email = RequestMailer.create_not_clarified_alert(info_request, incoming_message) + not_clarified_alert_email = RequestMailer.not_clarified_alert(info_request, incoming_message) write_email(not_clarified_alert_email, 'Telling the requester that they need to clarify their request', output_file) - comment_on_alert_email = RequestMailer.create_comment_on_alert(info_request, comment) + comment_on_alert_email = RequestMailer.comment_on_alert(info_request, comment) write_email(comment_on_alert_email, 'Telling requester that somebody added an annotation to their request', output_file) - comment_on_alert_plural_email = RequestMailer.create_comment_on_alert_plural(info_request, 2, comment) + comment_on_alert_plural_email = RequestMailer.comment_on_alert_plural(info_request, 2, comment) write_email(comment_on_alert_plural_email, 'Telling requester that somebody added multiple annotations to their request', output_file) @@ -149,38 +149,38 @@ namespace :translation do nil, 100, 1) - event_digest_email = TrackMailer.create_event_digest(info_request.user, + event_digest_email = TrackMailer.event_digest(info_request.user, [[track_thing, xapian_object.results, xapian_object]]) write_email(event_digest_email, 'Alerts on things the user is tracking', output_file) # user mailer - site_name = Configuration::site_name + site_name = AlaveteliConfiguration::site_name reasons = { :web => "", :email => _("Then you can sign in to {{site_name}}", :site_name => site_name), :email_subject => _("Confirm your account on {{site_name}}", :site_name => site_name) } - confirm_login_email = UserMailer.create_confirm_login(info_request.user, + confirm_login_email = UserMailer.confirm_login(info_request.user, reasons, 'http://www.example.com') write_email(confirm_login_email, 'Confirm a user login', output_file) - already_registered_email = UserMailer.create_already_registered(info_request.user, + already_registered_email = UserMailer.already_registered(info_request.user, reasons, 'http://www.example.com') write_email(already_registered_email, 'Tell a user they are already registered', output_file) new_email = 'new_email@example.com' - changeemail_confirm_email = UserMailer.create_changeemail_confirm(info_request.user, + changeemail_confirm_email = UserMailer.changeemail_confirm(info_request.user, new_email, 'http://www.example.com') write_email(changeemail_confirm_email, 'Confirm that the user wants to change their email', output_file) - changeemail_already_used = UserMailer.create_changeemail_already_used('old_email@example.com', + changeemail_already_used = UserMailer.changeemail_already_used('old_email@example.com', new_email) write_email(changeemail_already_used, 'Tell a user that the email they want to change to is already used', @@ -189,4 +189,4 @@ namespace :translation do output_file.close end -end
\ No newline at end of file +end diff --git a/lib/timezone_fixes.rb b/lib/timezone_fixes.rb deleted file mode 100644 index 1bf326ccd..000000000 --- a/lib/timezone_fixes.rb +++ /dev/null @@ -1,26 +0,0 @@ -# Taken from -# https://rails.lighthouseapp.com/projects/8994/tickets/2946 -# http://github.com/rails/rails/commit/6f97ad07ded847f29159baf71050c63f04282170 - -# Otherwise times get stored wrong during British Summer Time - -# Hopefully fixed in later Rails. There is a test in spec/lib/timezone_fixes_spec.rb - -# This fix is applied in Rails 3.x. So, should be possible to remove this then! - -# Monkeypatch! -module ActiveRecord - module ConnectionAdapters # :nodoc: - module Quoting - def quoted_date(value) - if value.acts_like?(:time) - zone_conversion_method = ActiveRecord::Base.default_timezone == :utc ? :getutc : :getlocal - value.respond_to?(zone_conversion_method) ? value.send(zone_conversion_method) : value - else - value - end.to_s(:db) - end - end - end -end - diff --git a/lib/willpaginate_extension.rb b/lib/willpaginate_extension.rb deleted file mode 100644 index 3cdb0ae60..000000000 --- a/lib/willpaginate_extension.rb +++ /dev/null @@ -1,59 +0,0 @@ -# this extension is loaded in environment.rb -module WillPaginateExtension - class LinkRenderer < WillPaginate::LinkRenderer - def page_link(page, text, attributes = {}) - # Hack for admin pages, when proxied via https on mySociety servers, they - # need a relative URL. - url = url_for(page) - if url.match(/\/admin.*(\?.*)/) - url = $1 - end - # Hack around our type-ahead search magic - if url.match(/\/body\/search_ahead/) - url.sub!("/body/search_ahead", "/select_authority") - end - @template.link_to text, url, attributes - end - - # Returns URL params for +page_link_or_span+, taking the current GET params - # and <tt>:params</tt> option into account. - def url_for(page) - page_one = page == 1 - unless @url_string and !page_one - @url_params = {} - # page links should preserve GET parameters - stringified_merge @url_params, @template.params if @template.request.get? - stringified_merge @url_params, @options[:params] if @options[:params] - if complex = param_name.index(/[^\w-]/) - page_param = parse_query_parameters("#{param_name}=#{page}") - - stringified_merge @url_params, page_param - else - @url_params[param_name] = page_one ? 1 : 2 - end - # the following line makes pagination work on our specially munged search page - combined = @template.request.path_parameters["combined"] - @url_params["combined"] = combined if !combined.nil? - url = @template.url_for(@url_params) - return url if page_one - - if complex - @url_string = url.sub(%r!((?:\?|&)#{CGI.escape param_name}=)#{page}!, "\\1\0") - return url - else - @url_string = url - @url_params[param_name] = 3 - @template.url_for(@url_params).split(//).each_with_index do |char, i| - if char == '3' and url[i, 1] == '2' - @url_string[i] = "\0" - break - end - end - end - end - # finally! - @url_string.sub "\0", page.to_s - end - - end -end |