Messaging: Bring back Unicode Stripping

Forward port from previous versions of mms.apk
Original author: David van Tonder (http://review.cyanogenmod.org/#/c/53477/)

Change-Id: Iba1faa51fad4077ee56957a0412cde6cc58a6083

2 files changed, 160 insertions, 2 deletions

diff --git a/src/com/android/messaging/ui/conversation/ComposeMessageView.java b/src/com/android/messaging/ui/conversation/ComposeMessageView.java
index 17f8f74..e666f2a 100644
--- a/src/com/android/messaging/ui/conversation/ComposeMessageView.java
+++ b/src/com/android/messaging/ui/conversation/ComposeMessageView.java
@@ -17,9 +17,11 @@ package com.android.messaging.ui.conversation;
 
 import android.content.Context;
 import android.content.res.Resources;
+import android.content.SharedPreferences;
 import android.graphics.Rect;
 import android.net.Uri;
 import android.os.Bundle;
+import android.preference.PreferenceManager;
 import android.support.v7.app.ActionBar;
 import android.text.Editable;
 import android.text.Html;
@@ -68,6 +70,7 @@ import com.android.messaging.util.LogUtil;
 import com.android.messaging.util.MediaUtil;
 import com.android.messaging.util.OsUtil;
 import com.android.messaging.util.UiUtils;
+import com.android.messaging.util.UnicodeFilter;
 
 import java.util.Collection;
 import java.util.List;
@@ -120,6 +123,7 @@ public class ComposeMessageView extends LinearLayout
     private ImageButton mDeleteSubjectButton;
     private AttachmentPreview mAttachmentPreview;
     private ImageButton mAttachMediaButton;
+    private UnicodeFilter mUnicodeFilter;
 
     private final Binding<DraftMessageData> mBinding;
     private IComposeMessageViewHost mHost;
@@ -158,6 +162,18 @@ public class ComposeMessageView extends LinearLayout
         super(new ContextThemeWrapper(context, R.style.ColorAccentBlueOverrideStyle), attrs);
         mOriginalContext = context;
         mBinding = BindingBase.createBinding(this);
+
+        String unicodeIntactValue = context.getString(R.string.unicode_stripping_leave_intact_value);
+        String unicodePrefKey = context.getString(R.string.unicode_stripping_pref_key);
+        SharedPreferences prefs = context.getSharedPreferences(BuglePrefs.SHARED_PREFERENCES_NAME,
+                Context.MODE_PRIVATE);
+        String unicodeStripping = prefs.getString(unicodePrefKey, unicodeIntactValue);
+        if (!unicodeIntactValue.equals(unicodeStripping)) {
+            String unicodeNonEncodableValue = context.getString(
+                    R.string.unicode_stripping_non_encodable_value);
+            boolean stripNonEncodableOnly = unicodeNonEncodableValue.equals(unicodeStripping);
+            mUnicodeFilter = new UnicodeFilter(stripNonEncodableOnly);
+        }
     }
 
     /**
@@ -368,6 +384,13 @@ public class ComposeMessageView extends LinearLayout
         return false;
     }
 
+    private CharSequence stripUnicodeIfRequested(CharSequence text) {
+        if (mUnicodeFilter != null) {
+            text = mUnicodeFilter.filter(text);
+        }
+        return text;
+    }
+
     private void sendMessageInternal(final boolean checkMessageSize) {
         LogUtil.i(LogUtil.BUGLE_TAG, "UI initiated message sending in conversation " +
                 mBinding.getData().getConversationId());
@@ -379,7 +402,7 @@ public class ComposeMessageView extends LinearLayout
         // Check the host for pre-conditions about any action.
         if (mHost.isReadyForAction()) {
             mInputManager.showHideSimSelector(false /* show */, true /* animate */);
-            final String messageToSend = mComposeEditText.getText().toString();
+            String messageToSend = stripUnicodeIfRequested(mComposeEditText.getText()).toString();
             mBinding.getData().setMessageText(messageToSend);
             final String subject = mComposeSubjectText.getText().toString();
             mBinding.getData().setMessageSubject(subject);
@@ -844,8 +867,10 @@ public class ComposeMessageView extends LinearLayout
     }
 
     @Override
-    public void onTextChanged(final CharSequence s, final int start, final int before,
+    public void onTextChanged(CharSequence s, final int start, final int before,
             final int count) {
+        // strip unicode for counting characters
+        s = stripUnicodeIfRequested(s);
         final BugleActionBarActivity activity = (mOriginalContext instanceof BugleActionBarActivity)
                 ? (BugleActionBarActivity) mOriginalContext : null;
         if (activity != null && activity.getIsDestroyed()) {
diff --git a/src/com/android/messaging/util/UnicodeFilter.java b/src/com/android/messaging/util/UnicodeFilter.java
new file mode 100644
index 0000000..9dc54e4
--- /dev/null
+++ b/src/com/android/messaging/util/UnicodeFilter.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2012-2015 The CyanogenMod Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.messaging.util;
+
+import android.text.SpannableString;
+import android.text.Spanned;
+import android.text.TextUtils;
+
+import com.android.internal.telephony.GsmAlphabet;
+import com.android.internal.telephony.EncodeException;
+
+import java.text.Normalizer;
+import java.util.regex.Pattern;
+
+/**
+ * Attempts to substitute characters that cannot be encoded in the limited
+ * GSM 03.38 character set. In many cases this will prevent sending a message
+ * containing characters that would switch the message from 7-bit GSM
+ * encoding (160 char limit) to 16-bit Unicode encoding (70 char limit).
+ */
+public class UnicodeFilter {
+    private static final Pattern DIACRITICS_PATTERN =
+            Pattern.compile("\\p{InCombiningDiacriticalMarks}");
+
+    private boolean mStripNonDecodableOnly;
+
+    public UnicodeFilter(boolean stripNonDecodableOnly) {
+        mStripNonDecodableOnly = stripNonDecodableOnly;
+    }
+
+    public CharSequence filter(CharSequence source) {
+        StringBuilder output = new StringBuilder(source);
+        final int sourceLength = source.length();
+
+        for (int i = 0; i < sourceLength; i++) {
+            char c = source.charAt(i);
+            boolean canEncodeInGsm;
+
+            try {
+                GsmAlphabet.charToGsm(c, true);
+                canEncodeInGsm = true;
+            } catch (EncodeException e) {
+                canEncodeInGsm = false;
+            }
+
+            // Character requires Unicode, try to replace it
+            if (!mStripNonDecodableOnly || !canEncodeInGsm) {
+                String s = String.valueOf(c);
+
+                // Try normalizing the character into Unicode NFKD form and
+                // stripping out diacritic mark characters.
+                s = Normalizer.normalize(s, Normalizer.Form.NFKD);
+                s = DIACRITICS_PATTERN.matcher(s).replaceAll("");
+
+                // Special case characters that don't get stripped by the
+                // above technique.
+                s = s.replace("Œ", "OE");
+                s = s.replace("œ", "oe");
+                s = s.replace("Ł", "L");
+                s = s.replace("ł", "l");
+                s = s.replace("Đ", "DJ");
+                s = s.replace("đ", "dj");
+                s = s.replace("Α", "A");
+                s = s.replace("Β", "B");
+                s = s.replace("Ε", "E");
+                s = s.replace("Ζ", "Z");
+                s = s.replace("Η", "H");
+                s = s.replace("Ι", "I");
+                s = s.replace("Κ", "K");
+                s = s.replace("Μ", "M");
+                s = s.replace("Ν", "N");
+                s = s.replace("Ο", "O");
+                s = s.replace("Ρ", "P");
+                s = s.replace("Τ", "T");
+                s = s.replace("Υ", "Y");
+                s = s.replace("Χ", "X");
+                s = s.replace("α", "A");
+                s = s.replace("β", "B");
+                s = s.replace("γ", "Γ");
+                s = s.replace("δ", "Δ");
+                s = s.replace("ε", "E");
+                s = s.replace("ζ", "Z");
+                s = s.replace("η", "H");
+                s = s.replace("θ", "Θ");
+                s = s.replace("ι", "I");
+                s = s.replace("κ", "K");
+                s = s.replace("λ", "Λ");
+                s = s.replace("μ", "M");
+                s = s.replace("ν", "N");
+                s = s.replace("ξ", "Ξ");
+                s = s.replace("ο", "O");
+                s = s.replace("π", "Π");
+                s = s.replace("ρ", "P");
+                s = s.replace("σ", "Σ");
+                s = s.replace("τ", "T");
+                s = s.replace("υ", "Y");
+                s = s.replace("φ", "Φ");
+                s = s.replace("χ", "X");
+                s = s.replace("ψ", "Ψ");
+                s = s.replace("ω", "Ω");
+                s = s.replace("ς", "Σ");
+
+                output.replace(i, i + 1, s);
+            }
+        }
+
+        // Source is a spanned string, so copy the spans from it
+        if (source instanceof Spanned) {
+            SpannableString spannedoutput = new SpannableString(output);
+            TextUtils.copySpansFrom(
+                    (Spanned) source, 0, sourceLength, null, spannedoutput, 0);
+
+            return spannedoutput;
+        }
+
+        // Source is a vanilla charsequence, so return output as-is
+        return output.toString();
+    }
+}