//! Functionality for finding words. //! //! In order to wrap text, we need to know where the legal break //! points are, i.e., where the words of the text are. This means that //! we need to define what a "word" is. //! //! A simple approach is to simply split the text on whitespace, but //! this does not work for East-Asian languages such as Chinese or //! Japanese where there are no spaces between words. Breaking a long //! sequence of emojis is another example where line breaks might be //! wanted even if there are no whitespace to be found. //! //! The [`WordSeparator`] enum is responsible for determining where //! there words are in a line of text. Please refer to the enum and //! its variants for more information.
/// Describes where words occur in a line of text. /// /// The simplest approach is say that words are separated by one or /// more ASCII spaces (`' '`). This works for Western languages /// without emojis. A more complex approach is to use the Unicode line /// breaking algorithm, which finds break points in non-ASCII text. /// /// The line breaks occur between words, please see /// [`WordSplitter`](crate::WordSplitter) for options of how to handle /// hyphenation of individual words. /// /// # Examples /// /// ``` /// use textwrap::core::Word; /// use textwrap::WordSeparator::AsciiSpace; /// /// let words = AsciiSpace.find_words("Hello World!").collect::<Vec<_>>(); /// assert_eq!(words, vec![Word::from("Hello "), Word::from("World!")]); /// ``` #[derive(Clone, Copy)] pubenum WordSeparator { /// Find words by splitting on runs of `' '` characters. /// /// # Examples /// /// ``` /// use textwrap::core::Word; /// use textwrap::WordSeparator::AsciiSpace; /// /// let words = AsciiSpace.find_words("Hello World!").collect::<Vec<_>>(); /// assert_eq!(words, vec![Word::from("Hello "), /// Word::from("World!")]); /// ```
AsciiSpace,
/// Split `line` into words using Unicode break properties. /// /// This word separator uses the Unicode line breaking algorithm /// described in [Unicode Standard Annex /// #14](https://www.unicode.org/reports/tr14/) to find legal places /// to break lines. There is a small difference in that the U+002D /// (Hyphen-Minus) and U+00AD (Soft Hyphen) don’t create a line break: /// to allow a line break at a hyphen, use /// [`WordSplitter::HyphenSplitter`](crate::WordSplitter::HyphenSplitter). /// Soft hyphens are not currently supported. /// /// # Examples /// /// Unlike [`WordSeparator::AsciiSpace`], the Unicode line /// breaking algorithm will find line break opportunities between /// some characters with no intervening whitespace: /// /// ``` /// #[cfg(feature = "unicode-linebreak")] { /// use textwrap::core::Word; /// use textwrap::WordSeparator::UnicodeBreakProperties; /// /// assert_eq!(UnicodeBreakProperties.find_words("Emojis: ").collect::<Vec<_>>(), /// vec![Word::from("Emojis: "), /// Word::from(""), /// Word::from("")]); /// /// assert_eq!(UnicodeBreakProperties.find_words("CJK: 你好").collect::<Vec<_>>(), /// vec![Word::from("CJK: "), /// Word::from("你"), /// Word::from("好")]); /// } /// ``` /// /// A U+2060 (Word Joiner) character can be inserted if you want to /// manually override the defaults and keep the characters together: /// /// ``` /// #[cfg(feature = "unicode-linebreak")] { /// use textwrap::core::Word; /// use textwrap::WordSeparator::UnicodeBreakProperties; /// /// assert_eq!(UnicodeBreakProperties.find_words("Emojis: \u{2060}").collect::<Vec<_>>(), /// vec![Word::from("Emojis: "), /// Word::from("\u{2060}")]); /// } /// ``` /// /// The Unicode line breaking algorithm will also automatically /// suppress break breaks around certain punctuation characters:: /// /// ``` /// #[cfg(feature = "unicode-linebreak")] { /// use textwrap::core::Word; /// use textwrap::WordSeparator::UnicodeBreakProperties; /// /// assert_eq!(UnicodeBreakProperties.find_words("[ foo ] bar !").collect::<Vec<_>>(), /// vec![Word::from("[ foo ] "), /// Word::from("bar !")]); /// } /// ``` #[cfg(feature = "unicode-linebreak")]
UnicodeBreakProperties,
/// Find words using a custom word separator
Custom(fn(line: &str) -> Box<dyn Iterator<Item = Word<'_>> + '_>),
}
impl WordSeparator { /// Create a new word separator. /// /// The best available algorithm is used by default, i.e., /// [`WordSeparator::UnicodeBreakProperties`] if available, /// otherwise [`WordSeparator::AsciiSpace`]. pubconstfn new() -> Self { #[cfg(feature = "unicode-linebreak")]
{
WordSeparator::UnicodeBreakProperties
}
/// Soft hyphen, also knows as a “shy hyphen”. Should show up as ‘-’ /// if a line is broken at this point, and otherwise be invisible. /// Textwrap does not currently support breaking words at soft /// hyphens. #[cfg(feature = "unicode-linebreak")] const SHY: char = '\u{00ad}';
/// Find words in line. ANSI escape sequences are ignored in `line`. #[cfg(feature = "unicode-linebreak")] fn find_words_unicode_break_properties<'a>(
line: &'a str,
) -> Box<dyn Iterator<Item = Word<'a>> + 'a> { // Construct an iterator over (original index, stripped index) // tuples. We find the Unicode linebreaks on a stripped string, // but we need the original indices so we can form words based on // the original string. letmut last_stripped_idx = 0; letmut char_indices = line.char_indices(); letmut idx_map = std::iter::from_fn(move || match char_indices.next() {
Some((orig_idx, ch)) => { let stripped_idx = last_stripped_idx; if !skip_ansi_escape_sequence(ch, &mut char_indices.by_ref().map(|(_, ch)| ch)) {
last_stripped_idx += ch.len_utf8();
}
Some((orig_idx, stripped_idx))
}
None => None,
});
let stripped = strip_ansi_escape_sequences(line); letmut opportunities = unicode_linebreak::linebreaks(&stripped)
.filter(|(idx, _)| { #[allow(clippy::match_like_matches_macro)] match &stripped[..*idx].chars().next_back() { // We suppress breaks at ‘-’ since we want to control // this via the WordSplitter.
Some('-') => false, // Soft hyphens are currently not supported since we // require all `Word` fragments to be continuous in // the input string.
Some(SHY) => false, // Other breaks should be fine!
_ => true,
}
})
.collect::<Vec<_>>()
.into_iter();
// Remove final break opportunity, we will add it below using // &line[start..]; This ensures that we correctly include a // trailing ANSI escape sequence.
opportunities.next_back();
letmut start = 0; Box::new(std::iter::from_fn(move || { for (idx, _) in opportunities.by_ref() { iflet Some((orig_idx, _)) = idx_map.find(|&(_, stripped_idx)| stripped_idx == idx) { let word = Word::from(&line[start..orig_idx]);
start = orig_idx; return Some(word);
}
}
if start < line.len() { let word = Word::from(&line[start..]);
start = line.len(); return Some(word);
}
None
}))
}
#[cfg(test)] mod tests { usesuper::WordSeparator::*; usesuper::*;
// Like assert_eq!, but the left expression is an iterator.
macro_rules! assert_iter_eq {
($left:expr, $right:expr) => {
assert_eq!($left.collect::<Vec<_>>(), $right);
};
}
#[test] fn find_words_color_inside_word() { let text = "foo\u{1b}[0m\u{1b}[32mbar\u{1b}[0mbaz";
assert_iter_eq!(AsciiSpace.find_words(text), vec![Word::from(text)]);
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.