From 6d15657e4a3826d4d47d5358f1dde211484527e9 Mon Sep 17 00:00:00 2001 From: Raph Levien Date: Tue, 8 Sep 2015 18:19:53 -0700 Subject: Add line breaks to email addresses and URLs This change adds accceptable line breaks according to sections 7.42 (Dividing URLs and e-mail addresses) and 14.12 (URLs or DOIs and line breaks) of the Chicago Manual of Style (16th ed.). In general, these place breaks before punctuation symbols, and suppresses them after hyphens. Bug: 20126487 Bug: 20566159 Change-Id: I2d07d516b920a506a2f718c38fb435c5eb1ee1f8 --- include/minikin/WordBreaker.h | 2 +- libs/minikin/WordBreaker.cpp | 65 +++++++++++++++------ tests/WordBreakerTests.cpp | 130 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 175 insertions(+), 22 deletions(-) diff --git a/include/minikin/WordBreaker.h b/include/minikin/WordBreaker.h index 8c00502..c4aa151 100644 --- a/include/minikin/WordBreaker.h +++ b/include/minikin/WordBreaker.h @@ -63,7 +63,7 @@ private: // state for the email address / url detector ssize_t mScanOffset; - bool mSuppressHyphen; + bool mInEmailOrUrl; }; } // namespace diff --git a/libs/minikin/WordBreaker.cpp b/libs/minikin/WordBreaker.cpp index f438cd5..edac993 100644 --- a/libs/minikin/WordBreaker.cpp +++ b/libs/minikin/WordBreaker.cpp @@ -43,7 +43,7 @@ void WordBreaker::setText(const uint16_t* data, size_t size) { mLast = 0; mCurrent = 0; mScanOffset = 0; - mSuppressHyphen = false; + mInEmailOrUrl = false; UErrorCode status = U_ZERO_ERROR; utext_openUChars(&mUText, data, size, &status); mBreakIterator->setText(&mUText, status); @@ -62,6 +62,17 @@ enum ScanState { SAW_COLON_SLASH_SLASH, }; +// Chicago Manual of Style recommends breaking after these characters in URLs and email addresses +static bool breakAfter(uint16_t c) { + return c == ':' || c == '=' || c == '&'; +} + +// Chicago Manual of Style recommends breaking before these characters in URLs and email addresses +static bool breakBefore(uint16_t c) { + return c == '~' || c == '.' || c == ',' || c == '-' || c == '_' || c == '?' || c == '#' + || c == '%' || c == '=' || c == '&'; +} + ssize_t WordBreaker::next() { mLast = mCurrent; @@ -88,23 +99,45 @@ ssize_t WordBreaker::next() { } } if (state == SAW_AT || state == SAW_COLON_SLASH_SLASH) { - // no line breaks in entire email address or url - // TODO: refine this according to Chicago Manual of Style rules - while (i < mTextSize && mText[i] == ' ') { - i++; + if (!mBreakIterator->isBoundary(i)) { + i = mBreakIterator->following(i); } - mCurrent = i; - mSuppressHyphen = true; - // Setting mIteratorWasReset will cause next break to be computed following - // mCurrent, rather than following the current ICU iterator location. + mInEmailOrUrl = true; mIteratorWasReset = true; - if (mBreakIterator->isBoundary(mCurrent)) { - return mCurrent; - } } else { - mScanOffset = i; - mSuppressHyphen = false; + mInEmailOrUrl = false; + } + mScanOffset = i; + } + + if (mInEmailOrUrl) { + // special rules for email addresses and URL's as per Chicago Manual of Style (16th ed.) + uint16_t lastChar = mText[mLast]; + ssize_t i; + for (i = mLast + 1; i < mScanOffset; i++) { + if (breakAfter(lastChar)) { + break; + } + // break after double slash + if (lastChar == '/' && i >= mLast + 2 && mText[i - 2] == '/') { + break; + } + uint16_t thisChar = mText[i]; + // never break after hyphen + if (lastChar != '-') { + if (breakBefore(thisChar)) { + break; + } + // break before single slash + if (thisChar == '/' && lastChar != '/' && + !(i + 1 < mScanOffset && mText[i + 1] == '/')) { + break; + } + } + lastChar = thisChar; } + mCurrent = i; + return mCurrent; } int32_t result; @@ -122,7 +155,7 @@ ssize_t WordBreaker::next() { } ssize_t WordBreaker::wordStart() const { - if (mSuppressHyphen) { + if (mInEmailOrUrl) { return mLast; } ssize_t result = mLast; @@ -142,7 +175,7 @@ ssize_t WordBreaker::wordStart() const { } ssize_t WordBreaker::wordEnd() const { - if (mSuppressHyphen) { + if (mInEmailOrUrl) { return mLast; } ssize_t result = mCurrent; diff --git a/tests/WordBreakerTests.cpp b/tests/WordBreakerTests.cpp index 4111a1b..284b02c 100644 --- a/tests/WordBreakerTests.cpp +++ b/tests/WordBreakerTests.cpp @@ -85,7 +85,9 @@ TEST_F(WordBreakerTest, email) { breaker.setLocale(icu::Locale::getEnglish()); breaker.setText(buf, NELEM(buf)); EXPECT_EQ(0, breaker.current()); - EXPECT_EQ(16, breaker.next()); // after "foo@example.com " + EXPECT_EQ(11, breaker.next()); // after "foo@example" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(16, breaker.next()); // after ".com " EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end EXPECT_EQ(16, breaker.wordStart()); // "x" @@ -99,13 +101,19 @@ TEST_F(WordBreakerTest, mailto) { breaker.setLocale(icu::Locale::getEnglish()); breaker.setText(buf, NELEM(buf)); EXPECT_EQ(0, breaker.current()); - EXPECT_EQ(23, breaker.next()); // after "mailto:foo@example.com " + EXPECT_EQ(7, breaker.next()); // after "mailto:" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(18, breaker.next()); // after "foo@example" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(23, breaker.next()); // after ".com " EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end EXPECT_EQ(23, breaker.wordStart()); // "x" EXPECT_EQ(24, breaker.wordEnd()); } +// The current logic always places a line break after a detected email address or URL +// and an immediately following non-ASCII character. TEST_F(WordBreakerTest, emailNonAscii) { uint16_t buf[] = {'f', 'o', 'o', '@', 'e', 'x', 'a', 'm', 'p', 'l', 'e', '.', 'c', 'o', 'm', 0x4E00}; @@ -113,7 +121,9 @@ TEST_F(WordBreakerTest, emailNonAscii) { breaker.setLocale(icu::Locale::getEnglish()); breaker.setText(buf, NELEM(buf)); EXPECT_EQ(0, breaker.current()); - EXPECT_EQ(15, breaker.next()); // after "foo@example.com" + EXPECT_EQ(11, breaker.next()); // after "foo@example" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(15, breaker.next()); // after ".com" EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end EXPECT_EQ(15, breaker.wordStart()); // "一" @@ -127,13 +137,31 @@ TEST_F(WordBreakerTest, emailCombining) { breaker.setLocale(icu::Locale::getEnglish()); breaker.setText(buf, NELEM(buf)); EXPECT_EQ(0, breaker.current()); - EXPECT_EQ(17, breaker.next()); // after "foo@example.com̃" + EXPECT_EQ(11, breaker.next()); // after "foo@example" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(17, breaker.next()); // after ".com̃ " EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end EXPECT_EQ(17, breaker.wordStart()); // "x" EXPECT_EQ(18, breaker.wordEnd()); } +TEST_F(WordBreakerTest, lonelyAt) { + uint16_t buf[] = {'a', ' ', '@', ' ', 'b'}; + WordBreaker breaker; + breaker.setLocale(icu::Locale::getEnglish()); + breaker.setText(buf, NELEM(buf)); + EXPECT_EQ(0, breaker.current()); + EXPECT_EQ(2, breaker.next()); // after "a " + EXPECT_EQ(0, breaker.wordStart()); // "a" + EXPECT_EQ(1, breaker.wordEnd()); + EXPECT_EQ(4, breaker.next()); // after "@ " + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end + EXPECT_EQ(4, breaker.wordStart()); // "b" + EXPECT_EQ(5, breaker.wordEnd()); +} + TEST_F(WordBreakerTest, url) { uint16_t buf[] = {'h', 't', 't', 'p', ':', '/', '/', 'e', 'x', 'a', 'm', 'p', 'l', 'e', '.', 'c', 'o', 'm', ' ', 'x'}; @@ -141,9 +169,101 @@ TEST_F(WordBreakerTest, url) { breaker.setLocale(icu::Locale::getEnglish()); breaker.setText(buf, NELEM(buf)); EXPECT_EQ(0, breaker.current()); - EXPECT_EQ(19, breaker.next()); // after "http://example.com " + EXPECT_EQ(5, breaker.next()); // after "http:" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(7, breaker.next()); // after "//" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(14, breaker.next()); // after "example" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(19, breaker.next()); // after ".com " EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end EXPECT_EQ(19, breaker.wordStart()); // "x" EXPECT_EQ(20, breaker.wordEnd()); } + +// Breaks according to section 14.12 of Chicago Manual of Style, *URLs or DOIs and line breaks* +TEST_F(WordBreakerTest, urlBreakChars) { + uint16_t buf[] = {'h', 't', 't', 'p', ':', '/', '/', 'a', '.', 'b', '/', '~', 'c', ',', 'd', + '-', 'e', '?', 'f', '=', 'g', '&', 'h', '#', 'i', '%', 'j', '_', 'k', '/', 'l'}; + WordBreaker breaker; + breaker.setLocale(icu::Locale::getEnglish()); + breaker.setText(buf, NELEM(buf)); + EXPECT_EQ(0, breaker.current()); + EXPECT_EQ(5, breaker.next()); // after "http:" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(7, breaker.next()); // after "//" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(8, breaker.next()); // after "a" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(10, breaker.next()); // after ".b" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(11, breaker.next()); // after "/" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(13, breaker.next()); // after "~c" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(15, breaker.next()); // after ",d" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(17, breaker.next()); // after "-e" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(19, breaker.next()); // after "?f" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(20, breaker.next()); // after "=" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(21, breaker.next()); // after "g" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(22, breaker.next()); // after "&" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(23, breaker.next()); // after "h" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(25, breaker.next()); // after "#i" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(27, breaker.next()); // after "%j" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(29, breaker.next()); // after "_k" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); +} + +TEST_F(WordBreakerTest, urlNoHyphenBreak) { + uint16_t buf[] = {'h', 't', 't', 'p', ':', '/', '/', 'a', '-', '/', 'b'}; + WordBreaker breaker; + breaker.setLocale(icu::Locale::getEnglish()); + breaker.setText(buf, NELEM(buf)); + EXPECT_EQ(0, breaker.current()); + EXPECT_EQ(5, breaker.next()); // after "http:" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(7, breaker.next()); // after "//" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(8, breaker.next()); // after "a" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); +} + +TEST_F(WordBreakerTest, urlEndsWithSlash) { + uint16_t buf[] = {'h', 't', 't', 'p', ':', '/', '/', 'a', '/'}; + WordBreaker breaker; + breaker.setLocale(icu::Locale::getEnglish()); + breaker.setText(buf, NELEM(buf)); + EXPECT_EQ(0, breaker.current()); + EXPECT_EQ(5, breaker.next()); // after "http:" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(7, breaker.next()); // after "//" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ(8, breaker.next()); // after "a" + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); + EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); +} + +TEST_F(WordBreakerTest, emailStartsWithSlash) { + uint16_t buf[] = {'/', 'a', '@', 'b'}; + WordBreaker breaker; + breaker.setLocale(icu::Locale::getEnglish()); + breaker.setText(buf, NELEM(buf)); + EXPECT_EQ(0, breaker.current()); + EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end + EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); +} -- cgit v1.2.3