usecrate::complex::*; usecrate::indices::*; usecrate::provider::*; usecrate::rule_segmenter::*; use alloc::string::String; use alloc::vec; use alloc::vec::Vec; use core::char; use icu_locale_core::subtags::{language, Language}; use icu_locale_core::LanguageIdentifier; use icu_provider::prelude::*; use utf8_iter::Utf8CharIndices;
/// An enum specifies the strictness of line-breaking rules. It can be passed as /// an argument when creating a line segmenter. /// /// Each enum value has the same meaning with respect to the `line-break` /// property values in the CSS Text spec. See the details in /// <https://drafts.csswg.org/css-text-3/#line-break-property>. #[non_exhaustive] #[derive(Copy, Clone, PartialEq, Eq, Debug, Default)] pubenum LineBreakStrictness { /// Breaks text using the least restrictive set of line-breaking rules. /// Typically used for short lines, such as in newspapers. /// <https://drafts.csswg.org/css-text-3/#valdef-line-break-loose>
Loose,
/// Breaks text assuming there is a soft wrap opportunity around every /// typographic character unit, disregarding any prohibition against line /// breaks. See more details in /// <https://drafts.csswg.org/css-text-3/#valdef-line-break-anywhere>.
Anywhere,
}
/// An enum specifies the line break opportunities between letters. It can be /// passed as an argument when creating a line segmenter. /// /// Each enum value has the same meaning with respect to the `word-break` /// property values in the CSS Text spec. See the details in /// <https://drafts.csswg.org/css-text-3/#word-break-property> #[non_exhaustive] #[derive(Copy, Clone, PartialEq, Eq, Debug, Default)] pubenum LineBreakWordOption { /// Words break according to their customary rules. See the details in /// <https://drafts.csswg.org/css-text-3/#valdef-word-break-normal>. #[default]
Normal,
/// Options to tailor line-breaking behavior. #[non_exhaustive] #[derive(Copy, Clone, PartialEq, Eq, Debug, Default)] pubstruct LineBreakOptions<'a> { /// Strictness of line-breaking rules. See [`LineBreakStrictness`]. /// /// Default is [`LineBreakStrictness::Strict`] pub strictness: Option<LineBreakStrictness>,
/// Line break opportunities between letters. See [`LineBreakWordOption`]. /// /// Default is [`LineBreakStrictness::Normal`] pub word_option: Option<LineBreakWordOption>,
/// Content locale for line segmenter /// /// This allows more break opportunities when `LineBreakStrictness` is /// `Normal` or `Loose`. See /// <https://drafts.csswg.org/css-text-3/#line-break-property> for details. /// This option has no effect in Latin-1 mode. pub content_locale: Option<&'a LanguageIdentifier>,
}
/// Supports loading line break data, and creating line break iterators for different string /// encodings. /// /// Most segmentation methods live on [`LineSegmenterBorrowed`], which can be obtained via /// [`LineSegmenter::new_auto()`] (etc) or [`LineSegmenter::as_borrowed()`]. /// /// The segmenter returns mandatory breaks (as defined by [definition LD7][LD7] of /// Unicode Standard Annex #14, _Unicode Line Breaking Algorithm_) as well as /// line break opportunities ([definition LD3][LD3]). /// It does not distinguish them. Callers requiring that distinction can check /// the Line_Break property of the code point preceding the break against those /// listed in rules [LB4][LB4] and [LB5][LB5], special-casing the end of text /// according to [LB3][LB3]. /// /// For consistency with the grapheme, word, and sentence segmenters, there is /// always a breakpoint returned at index 0, but this breakpoint is not a /// meaningful line break opportunity. /// /// [LD3]: https://www.unicode.org/reports/tr14/#LD3 /// [LD7]: https://www.unicode.org/reports/tr14/#LD7 /// [LB3]: https://www.unicode.org/reports/tr14/#LB3 /// [LB4]: https://www.unicode.org/reports/tr14/#LB4 /// [LB5]: https://www.unicode.org/reports/tr14/#LB5 /// /// ```rust /// # use icu::segmenter::LineSegmenter; /// # /// # let segmenter = LineSegmenter::new_auto(Default::default()); /// # /// let text = "Summary\r\nThis annex…"; /// let breakpoints: Vec<usize> = segmenter.segment_str(text).collect(); /// // 9 and 22 are mandatory breaks, 14 is a line break opportunity. /// assert_eq!(&breakpoints, &[0, 9, 14, 22]); /// /// // There is a break opportunity between emoji, but not within the ZWJ sequence ��️��. /// let flag_equation = "��️➕������️\u{200D}��"; /// let possible_first_lines: Vec<&str> = /// segmenter.segment_str(flag_equation).skip(1).map(|i| &flag_equation[..i]).collect(); /// assert_eq!( /// &possible_first_lines, /// &[ /// "��️", /// "��️➕", /// "��️➕��", /// "��️➕����", /// "��️➕������️��" /// ] /// ); /// ``` /// /// # Examples /// /// Segment a string with default options: /// /// ```rust /// use icu::segmenter::LineSegmenter; /// /// let segmenter = LineSegmenter::new_auto(Default::default()); /// /// let breakpoints: Vec<usize> = /// segmenter.segment_str("Hello World").collect(); /// assert_eq!(&breakpoints, &[0, 6, 11]); /// ``` /// /// Segment a string with CSS option overrides: /// /// ```rust /// use icu::segmenter::options::{ /// LineBreakOptions, LineBreakStrictness, LineBreakWordOption, /// }; /// use icu::segmenter::LineSegmenter; /// /// let mut options = LineBreakOptions::default(); /// options.strictness = Some(LineBreakStrictness::Strict); /// options.word_option = Some(LineBreakWordOption::BreakAll); /// options.content_locale = None; /// let segmenter = LineSegmenter::new_auto(options); /// /// let breakpoints: Vec<usize> = /// segmenter.segment_str("Hello World").collect(); /// assert_eq!(&breakpoints, &[0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11]); /// ``` /// /// Segment a Latin1 byte string: /// /// ```rust /// use icu::segmenter::LineSegmenter; /// /// let segmenter = LineSegmenter::new_auto(Default::default()); /// /// let breakpoints: Vec<usize> = /// segmenter.segment_latin1(b"Hello World").collect(); /// assert_eq!(&breakpoints, &[0, 6, 11]); /// ``` /// /// Separate mandatory breaks from the break opportunities: /// /// ```rust /// use icu::properties::{props::LineBreak, CodePointMapData}; /// use icu::segmenter::LineSegmenter; /// /// # let segmenter = LineSegmenter::new_auto(Default::default()); /// # /// let text = "Summary\r\nThis annex…"; /// /// let mandatory_breaks: Vec<usize> = segmenter /// .segment_str(text) /// .filter(|&i| { /// text[..i].chars().next_back().is_some_and(|c| { /// matches!( /// CodePointMapData::<LineBreak>::new().get(c), /// LineBreak::MandatoryBreak /// | LineBreak::CarriageReturn /// | LineBreak::LineFeed /// | LineBreak::NextLine /// ) || i == text.len() /// }) /// }) /// .collect(); /// assert_eq!(&mandatory_breaks, &[9, 22]); /// ``` #[derive(Debug)] pubstruct LineSegmenter {
options: ResolvedLineBreakOptions,
payload: DataPayload<SegmenterBreakLineV1>,
complex: ComplexPayloads,
}
/// Segments a string into lines (borrowed version). /// /// See [`LineSegmenter`] for examples. #[derive(Clone, Debug, Copy)] pubstruct LineSegmenterBorrowed<'data> {
options: ResolvedLineBreakOptions,
data: &'data RuleBreakData<'data>,
complex: ComplexPayloadsBorrowed<'data>,
}
impl LineSegmenter { /// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and /// the best available compiled data for complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The current behavior, which is subject to change, is to use the LSTM model when available. /// /// See also [`Self::new_auto`]. /// /// ✨ *Enabled with the `compiled_data` and `auto` Cargo features.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "auto")] #[cfg(feature = "compiled_data")] pubfn new_auto(options: LineBreakOptions) -> LineSegmenterBorrowed<'static> { Self::new_lstm(options)
}
/// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and /// compiled LSTM data for complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The LSTM, or Long Term Short Memory, is a machine learning model. It is smaller than /// the full dictionary but more expensive during segmentation (inference). /// /// See also [`Self::new_lstm`]. /// /// ✨ *Enabled with the `compiled_data` and `lstm` Cargo features.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "lstm")] #[cfg(feature = "compiled_data")] pubfn new_lstm(options: LineBreakOptions) -> LineSegmenterBorrowed<'static> {
LineSegmenterBorrowed {
options: options.resolve(),
data: crate::provider::Baked::SINGLETON_SEGMENTER_BREAK_LINE_V1,
complex: ComplexPayloadsBorrowed::new_lstm(),
}
}
/// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and /// compiled dictionary data for complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The dictionary model uses a list of words to determine appropriate breakpoints. It is /// faster than the LSTM model but requires more data. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "compiled_data")] pubfn new_dictionary(options: LineBreakOptions) -> LineSegmenterBorrowed<'static> {
LineSegmenterBorrowed {
options: options.resolve(),
data: crate::provider::Baked::SINGLETON_SEGMENTER_BREAK_LINE_V1, // Line segmenter doesn't need to load CJ dictionary because UAX 14 rules handles CJK // characters [1]. Southeast Asian languages however require complex context analysis // [2]. // // [1]: https://www.unicode.org/reports/tr14/#ID // [2]: https://www.unicode.org/reports/tr14/#SA
complex: ComplexPayloadsBorrowed::new_southeast_asian(),
}
}
/// Constructs a borrowed version of this type for more efficient querying. /// /// Most useful methods for segmentation are on this type. pubfn as_borrowed(&self) -> LineSegmenterBorrowed<'_> {
LineSegmenterBorrowed {
options: self.options,
data: self.payload.get(),
complex: self.complex.as_borrowed(),
}
}
}
impl<'data> LineSegmenterBorrowed<'data> { /// Creates a line break iterator for an `str` (a UTF-8 string). /// /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string. pubfn segment_str<'s>(self, input: &'s str) -> LineBreakIterator<'data, 's, Utf8> {
LineBreakIterator {
iter: input.char_indices(),
len: input.len(),
current_pos_data: None,
result_cache: Vec::new(),
data: self.data,
options: self.options,
complex: self.complex,
}
} /// Creates a line break iterator for a potentially ill-formed UTF8 string /// /// Invalid characters are treated as REPLACEMENT CHARACTER /// /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string. pubfn segment_utf8<'s>( self,
input: &'s [u8],
) -> LineBreakIterator<'data, 's, PotentiallyIllFormedUtf8> {
LineBreakIterator {
iter: Utf8CharIndices::new(input),
len: input.len(),
current_pos_data: None,
result_cache: Vec::new(),
data: self.data,
options: self.options,
complex: self.complex,
}
} /// Creates a line break iterator for a Latin-1 (8-bit) string. /// /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string. pubfn segment_latin1<'s>(self, input: &'s [u8]) -> LineBreakIterator<'data, 's, Latin1> {
LineBreakIterator {
iter: Latin1Indices::new(input),
len: input.len(),
current_pos_data: None,
result_cache: Vec::new(),
data: self.data,
options: self.options,
complex: self.complex,
}
}
/// Creates a line break iterator for a UTF-16 string. /// /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string. pubfn segment_utf16<'s>(self, input: &'s [u16]) -> LineBreakIterator<'data, 's, Utf16> {
LineBreakIterator {
iter: Utf16Indices::new(input),
len: input.len(),
current_pos_data: None,
result_cache: Vec::new(),
data: self.data,
options: self.options,
complex: self.complex,
}
}
}
impl LineSegmenterBorrowed<'static> { /// Cheaply converts a [`LineSegmenterBorrowed<'static>`] into a [`LineSegmenter`]. /// /// Note: Due to branching and indirection, using [`LineSegmenter`] might inhibit some /// compile-time optimizations that are possible with [`LineSegmenterBorrowed`]. pubfn static_to_owned(self) -> LineSegmenter {
LineSegmenter {
payload: DataPayload::from_static_ref(self.data),
complex: self.complex.static_to_owned(),
options: self.options,
}
}
}
impl RuleBreakData<'_> { fn get_linebreak_property_utf32_with_rule(
&self,
codepoint: u32,
strictness: LineBreakStrictness,
word_option: LineBreakWordOption,
) -> u8 { // Note: Default value is 0 == UNKNOWN let prop = self.property_table.get32(codepoint);
if word_option == LineBreakWordOption::BreakAll
|| strictness == LineBreakStrictness::Loose
|| strictness == LineBreakStrictness::Normal
{ returnmatch prop {
CJ => ID, // All CJ's General_Category is Other_Letter (Lo).
_ => prop,
};
}
#[inline] fn get_break_state_from_table(&self, left: u8, right: u8) -> BreakState { let idx = (left as usize) * (self.property_count as usize) + (right as usize); // We use unwrap_or to fall back to the base case and prevent panics on bad data. self.break_state_table.get(idx).unwrap_or(BreakState::Keep)
}
// breaks before certain centered punctuation marks: if right_codepoint == 0x30FB
|| right_codepoint == 0xFF1A
|| right_codepoint == 0xFF1B
|| right_codepoint == 0xFF65
|| right_codepoint == 0x203C
|| (0x2047..=0x2049).contains(&right_codepoint)
{ return Some(ja_zh);
}
} elseif right_prop == IN { // breaks between inseparable characters such as U+2025, U+2026 i.e. characters with the Unicode Line Break property IN return Some(true);
} elseif right_prop == EX { // breaks before certain centered punctuation marks: if right_codepoint == 0xFF01 || right_codepoint == 0xFF1F { return Some(ja_zh);
}
}
// breaks before suffixes: // Characters with the Unicode Line Break property PO and the East Asian Width property if right_prop == PO_EAW { return Some(ja_zh);
} // breaks after prefixes: // Characters with the Unicode Line Break property PR and the East Asian Width property if left_prop == PR_EAW { return Some(ja_zh);
}
None
}
/// A trait allowing for LineBreakIterator to be generalized to multiple string iteration methods. /// /// This is implemented by ICU4X for several common string types. /// /// <div class="stab unstable"> /// �� This trait is sealed; it cannot be implemented by user code. If an API requests an item that implements this /// trait, please consider using a type from the implementors listed below. /// </div> pubtrait LineBreakType: crate::private::Sealed + Sized + RuleBreakType { #[doc(hidden)] fn use_complex_breaking(iterator: &LineBreakIterator<'_, '_, Self>, c: Self::CharType) -> bool;
/// Implements the [`Iterator`] trait over the line break opportunities of the given string. /// /// Lifetimes: /// /// - `'l` = lifetime of the [`LineSegmenter`] object from which this iterator was created /// - `'s` = lifetime of the string being segmented /// /// The [`Iterator::Item`] is an [`usize`] representing index of a code unit /// _after_ the break (for a break at the end of text, this index is the length /// of the [`str`] or array of code units). /// /// For examples of use, see [`LineSegmenter`]. #[derive(Debug)] pubstruct LineBreakIterator<'data, 's, Y: LineBreakType> {
iter: Y::IterAttr<'s>,
len: usize,
current_pos_data: Option<(usize, Y::CharType)>,
result_cache: Vec<usize>,
data: &'data RuleBreakData<'data>,
options: ResolvedLineBreakOptions,
complex: ComplexPayloadsBorrowed<'data>,
}
impl<Y: LineBreakType> Iterator for LineBreakIterator<'_, '_, Y> { type Item = usize;
// If we have break point cache by previous run, return this result iflet Some(&first_pos) = self.result_cache.first() { letmut i = 0; loop { if i == first_pos { self.result_cache = self.result_cache.iter().skip(1).map(|r| r - i).collect(); returnself.get_current_position();
}
i += self.get_current_codepoint().map_or(0, Y::char_len); self.advance_iter(); ifself.is_eof() { self.result_cache.clear(); return Some(self.len);
}
}
}
// The state prior to a sequence of CM and ZWJ affected by rule LB9. letmut lb9_left: Option<u8> = None; // Whether LB9 was applied to a ZWJ, so that breaks at the current // position must be suppressed. letmut lb8a_after_lb9 = false;
let Some(right_codepoint) = self.get_current_codepoint() else { return Some(self.len);
}; let right_prop = self.get_linebreak_property(right_codepoint); // NOTE(egg): The special-casing of `LineBreakStrictness::Anywhere` allows us to pass // a test, but eventually that option should just be simplified to call the extended // grapheme cluster segmenter. if (right_prop == CM
|| (right_prop == ZWJ && self.options.strictness != LineBreakStrictness::Anywhere))
&& left_prop != BK
&& left_prop != CR
&& left_prop != LF
&& left_prop != NL
&& left_prop != SP
&& left_prop != ZW
{
lb9_left = Some(left_prop);
lb8a_after_lb9 = right_prop == ZWJ; continue;
} else {
lb9_left = None;
lb8a_after_lb9 = false;
}
// CSS word-break property handling match (self.options.word_option, left_prop, right_prop) {
(LineBreakWordOption::BreakAll, AL | NU | SA, _) => {
left_prop = ID;
} // typographic letter units shouldn't be break
(
LineBreakWordOption::KeepAll,
AI | AL | ID | NU | HY | H2 | H3 | JL | JV | JT | CJ,
AI | AL | ID | NU | HY | H2 | H3 | JL | JV | JT | CJ,
) => { continue;
}
_ => (),
}
// CSS line-break property handling matchself.options.strictness {
LineBreakStrictness::Normal => { ifself.is_break_by_normal(right_codepoint) && !after_zwj { returnself.get_current_position();
}
}
LineBreakStrictness::Loose => { iflet Some(breakable) = is_break_utf32_by_loose(
right_codepoint.into(),
left_prop,
right_prop, self.options.ja_zh,
) { if breakable && !after_zwj { returnself.get_current_position();
} continue;
}
}
LineBreakStrictness::Anywhere => { // TODO(egg): My reading of the CSS standard is that this // should break around extended grapheme clusters, not at // arbitrary code points, so this seems wrong. returnself.get_current_position();
}
_ => (),
};
// UAX14 doesn't have Thai etc, so use another way. ifself.options.word_option != LineBreakWordOption::BreakAll
&& Y::use_complex_breaking(self, left_codepoint)
&& Y::use_complex_breaking(self, right_codepoint)
{ let result = Y::line_handle_complex_language(self, left_codepoint); if result.is_some() { return result;
} // I may have to fetch text until non-SA character?.
}
// If break_state is equals or grater than 0, it is alias of property. matchself.data.get_break_state_from_table(left_prop, right_prop) {
BreakState::Break | BreakState::NoMatch => { if after_zwj { continue;
} else { returnself.get_current_position();
}
}
BreakState::Keep => continue,
BreakState::Index(mut index) | BreakState::Intermediate(mut index) => { letmut previous_iter = self.iter.clone(); letmut previous_pos_data = self.current_pos_data; letmut previous_is_after_zwj = after_zwj;
// Since we are building up a state in this inner loop, we do not // need an analogue of lb9_left; continuing the inner loop preserves // `index` which is the current state, and thus implements the // “treat as” rule. letmut left_prop_pre_lb9 = right_prop;
// current state isn't resolved due to intermediating. // Example, [AK] [AS] is processing LB28a, but if not matched after fetching // data, we should break after [AK]. let is_intermediate_rule_no_match = if lb8a_after_lb9 { // left was ZWJ so we don't break between ZWJ. true
} else {
index > self.data.last_codepoint_property
};
loop { self.advance_iter(); let after_zwj = left_prop_pre_lb9 == ZWJ;
let previous_break_state_is_cp_prop =
index <= self.data.last_codepoint_property;
let Some(prop) = self.get_current_linebreak_property() else { // Reached EOF. But we are analyzing multiple characters now, so next break may be previous point. let break_state = self
.data
.get_break_state_from_table(index, self.data.eot_property); if break_state == BreakState::NoMatch { self.iter = previous_iter; self.current_pos_data = previous_pos_data; if previous_is_after_zwj { // Do not break [AK] [ZWJ] ÷ [AS] (eot). continue'a;
} else { returnself.get_current_position();
}
} // EOF return Some(self.len);
};
#[inline] fn check_eof(&mutself) -> StringBoundaryPosType { ifself.is_eof() { self.advance_iter(); ifself.is_eof() { ifself.len == 0 { // Empty string. Since `self.current_pos_data` is always going to be empty, // we never read `self.len` except for here, so we can use it to mark that // we have already returned the single empty-string breakpoint. self.len = 1;
StringBoundaryPosType::Start
} else {
StringBoundaryPosType::End
}
} else {
StringBoundaryPosType::Start
}
} else {
StringBoundaryPosType::Middle
}
}
fn line_handle_complex_language(
iter: &mut LineBreakIterator<'_, '_, Self>,
left_codepoint: char,
) -> Option<usize> {
line_handle_complex_language_utf8(iter, left_codepoint)
}
} /// line_handle_complex_language impl for UTF8 iterators fn line_handle_complex_language_utf8<T>(
iter: &mut LineBreakIterator<'_, '_, T>,
left_codepoint: char,
) -> Option<usize> where
T: LineBreakType<CharType = char>,
{ // word segmenter doesn't define break rules for some languages such as Thai. let start_iter = iter.iter.clone(); let start_point = iter.current_pos_data; letmut s = String::new();
s.push(left_codepoint); loop {
debug_assert!(!iter.is_eof());
s.push(iter.get_current_codepoint()?);
iter.advance_iter(); iflet Some(current_codepoint) = iter.get_current_codepoint() { if !T::use_complex_breaking(iter, current_codepoint) { break;
}
} else { // EOF break;
}
}
// Restore iterator to move to head of complex string
iter.iter = start_iter;
iter.current_pos_data = start_point; let breaks = iter.complex.complex_language_segment_str(&s);
iter.result_cache = breaks; let first_pos = *iter.result_cache.first()?; letmut i = left_codepoint.len_utf8(); loop { if i == first_pos { // Re-calculate breaking offset
iter.result_cache = iter.result_cache.iter().skip(1).map(|r| r - i).collect(); return iter.get_current_position();
}
debug_assert!(
i < first_pos, "we should always arrive at first_pos: near index {:?}",
iter.get_current_position()
);
i += iter.get_current_codepoint().map_or(0, T::char_len);
iter.advance_iter(); if iter.is_eof() {
iter.result_cache.clear(); return Some(iter.len);
}
}
}
impl LineBreakType for Latin1 { fn get_linebreak_property_with_rule(iterator: &LineBreakIterator<Self>, c: u8) -> u8 { // No CJ on Latin1 // Note: Default value is 0 == UNKNOWN
iterator.data.property_table.get32(c as u32)
}
fn line_handle_complex_language(
iterator: &mut LineBreakIterator<Self>,
left_codepoint: Self::CharType,
) -> Option<usize> { // word segmenter doesn't define break rules for some languages such as Thai. let start_iter = iterator.iter.clone(); let start_point = iterator.current_pos_data; letmut s = vec![left_codepoint as u16]; loop {
debug_assert!(!iterator.is_eof());
s.push(iterator.get_current_codepoint()? as u16);
iterator.advance_iter(); iflet Some(current_codepoint) = iterator.get_current_codepoint() { if !Self::use_complex_breaking(iterator, current_codepoint) { break;
}
} else { // EOF break;
}
}
// Restore iterator to move to head of complex string
iterator.iter = start_iter;
iterator.current_pos_data = start_point; let breaks = iterator.complex.complex_language_segment_utf16(&s);
iterator.result_cache = breaks; // result_cache vector is utf-16 index that is in BMP. let first_pos = *iterator.result_cache.first()?; letmut i = 1; loop { if i == first_pos { // Re-calculate breaking offset
iterator.result_cache = iterator
.result_cache
.iter()
.skip(1)
.map(|r| r - i)
.collect(); return iterator.get_current_position();
}
debug_assert!(
i < first_pos, "we should always arrive at first_pos: near index {:?}",
iterator.get_current_position()
);
i += 1;
iterator.advance_iter(); if iterator.is_eof() {
iterator.result_cache.clear(); return Some(iterator.len);
}
}
}
}
#[cfg(test)] #[cfg(feature = "serde")] mod tests { usesuper::*; usecrate::LineSegmenter;
#[test] fn linebreak_property() { let payload =
DataProvider::<SegmenterBreakLineV1>::load(&crate::provider::Baked, Default::default())
.expect("Loading should succeed!")
.payload;
let get_linebreak_property = |codepoint| {
payload.get().get_linebreak_property_utf32_with_rule(
codepoint as u32,
LineBreakStrictness::Strict,
LineBreakWordOption::Normal,
)
};
// LB15 used to prevent the break at 6, but has been removed in Unicode 15.1.
iter = segmenter.segment_str("abc\u{0022} (def");
assert_eq!(Some(0), iter.next());
assert_eq!(Some(6), iter.next());
assert_eq!(Some(10), iter.next());
assert_eq!(None, iter.next());
let segmenter = LineSegmenter::new_lstm(Default::default()); let breaks: Vec<usize> = segmenter.segment_str(TEST_STR).collect(); // LSTM model breaks more characters, but it is better to return [30].
assert_eq!(breaks, [0, 12, 18, 30, TEST_STR.len()], "Burmese test");
let utf16: Vec<u16> = TEST_STR.encode_utf16().collect(); let breaks: Vec<usize> = segmenter.segment_utf16(&utf16).collect(); // LSTM model breaks more characters, but it is better to return [10].
assert_eq!(breaks, [0, 4, 6, 10, utf16.len()], "Burmese utf-16 test");
}
let segmenter = LineSegmenter::new_lstm(Default::default()); let breaks: Vec<usize> = segmenter.segment_str(TEST_STR).collect(); // Note: LSTM finds a break at '12' that the dictionary does not find
assert_eq!(breaks, [0, 12, 21, 30, 39, TEST_STR.len()], "Lao test");
