// Macro to check/initialize the registry. ONLY USE WITHIN // MUTEX. Avoids function call when registry is initialized. #define HAVE_REGISTRY(status) (registry!=0 || initializeRegistry(status))
/** *Copyconstructor.
*/
Transliterator::Transliterator(const Transliterator& other) :
UObject(other), ID(other.ID), filter(nullptr),
maximumContextLength(other.maximumContextLength)
{ // NUL-terminate the ID string, which is a non-aliased copy.
ID.append(static_cast<char16_t>(0));
ID.truncate(ID.length()-1);
if (other.filter != nullptr) { // We own the filter, so we must have our own copy
filter = other.filter->clone();
}
}
if (index.limit > 0 &&
U16_IS_LEAD(text.charAt(index.limit - 1))) { // Oops, there is a dangling lead surrogate in the buffer. // This will break most transliterators, since they will // assume it is part of a pair. Don't transliterate until // more text comes in. return;
}
filteredTransliterate(text, index, true, true);
#if0 // TODO // I CAN'T DO what I'm attempting below now that the Kleene star // operator is supported. For example, in the rule
// ([:Lu:]+) { x } > $1;
// what is the maximum context length? getMaximumContextLength() // will return 1, but this is just the length of the ante context // part of the pattern string -- 1 character, which is a standin // for a Quantifier, which contains a StringMatcher, which // contains a UnicodeSet.
// There is a complicated way to make this work again, and that's // to add a "maximum left context" protocol into the // UnicodeMatcher hierarchy. At present I'm not convinced this is // worth it.
// ---
// The purpose of the code below is to keep the context small // while doing incremental transliteration. When part of the left // context (between contextStart and start) is no longer needed, // we try to advance contextStart past that portion. We use the // maximum context length to do so.
int32_t newCS = index.start;
int32_t n = getMaximumContextLength(); while (newCS > originalStart && n-- > 0) {
--newCS;
newCS -= U16_LENGTH(text.char32At(newCS)) - 1;
}
index.contextStart = uprv_max(newCS, originalStart); #endif
}
/** *Thismethodbreaksuptheinputtextintorunsofunfiltered *characters.Itpasseseachsuchrunto *<subclass>.handleTransliterate().Subclassesthatcanhandlethe *filterlogicmoreefficientlythemselvesmayoverridethismethod. * *Alltransliterationcallsinthisclassgothroughthismethod.
*/ void Transliterator::filteredTransliterate(Replaceable& text,
UTransPosition& index,
UBool incremental,
UBool rollback) const { // Short circuit path for transliterators with no filter in // non-incremental mode. if (filter == nullptr && !rollback) {
handleTransliterate(text, index, incremental); return;
}
//---------------------------------------------------------------------- // This method processes text in two groupings: // // RUNS -- A run is a contiguous group of characters which are contained // in the filter for this transliterator (filter.contains(ch) == true). // Text outside of runs may appear as context but it is not modified. // The start and limit Position values are narrowed to each run. // // PASSES (incremental only) -- To make incremental mode work correctly, // each run is broken up into n passes, where n is the length (in code // points) of the run. Each pass contains the first n characters. If a // pass is completely transliterated, it is committed, and further passes // include characters after the committed text. If a pass is blocked, // and does not transliterate completely, then this method rolls back // the changes made during the pass, extends the pass by one code point, // and tries again. //----------------------------------------------------------------------
// globalLimit is the limit value for the entire operation. We // set index.limit to the end of each unfiltered run before // calling handleTransliterate(), so we need to maintain the real // value of index.limit here. After each transliteration, we // update globalLimit for insertions or deletions that have // happened.
int32_t globalLimit = index.limit;
// If there is a non-null filter, then break the input text up. Say the // input text has the form: // xxxabcxxdefxx // where 'x' represents a filtered character (filter.contains('x') == // false). Then we break this up into: // xxxabc xxdef xx // Each pass through the loop consumes a run of filtered // characters (which are ignored) and a subsequent run of // unfiltered characters (which are transliterated).
for (;;) {
if (filter != nullptr) { // Narrow the range to be transliterated to the first segment // of unfiltered characters at or after index.start.
// Advance past filtered chars
UChar32 c; while (index.start < globalLimit &&
!filter->contains(c=text.char32At(index.start))) {
index.start += U16_LENGTH(c);
}
// Find the end of this run of unfiltered chars
index.limit = index.start; while (index.limit < globalLimit &&
filter->contains(c=text.char32At(index.limit))) {
index.limit += U16_LENGTH(c);
}
}
// Check to see if the unfiltered run is empty. This only // happens at the end of the string when all the remaining // characters are filtered. if (index.limit == index.start) { // assert(index.start == globalLimit); break;
}
// Is this run incremental? If there is additional // filtered text (if limit < globalLimit) then we pass in // an incremental value of false to force the subclass to // complete the transliteration for this run.
UBool isIncrementalRun =
(index.limit < globalLimit ? false : incremental);
int32_t delta;
// Implement rollback. To understand the need for rollback, // consider the following transliterator: // // "t" is "a > A;" // "u" is "A > b;" // "v" is a compound of "t; NFD; u" with a filter [:Ll:] // // Now apply "c" to the input text "a". The result is "b". But if // the transliteration is done incrementally, then the NFD holds // things up after "t" has already transformed "a" to "A". When // finishTransliterate() is called, "A" is _not_ processed because // it gets excluded by the [:Ll:] filter, and the end result is "A" // -- incorrect. The problem is that the filter is applied to a // partially-transliterated result, when we only want it to apply to // input text. Although this example hinges on a compound // transliterator containing NFD and a specific filter, it can // actually happen with any transliterator which may do a partial // transformation in incremental mode into characters outside its // filter. // // To handle this, when in incremental mode we supply characters to // handleTransliterate() in several passes. Each pass adds one more // input character to the input text. That is, for input "ABCD", we // first try "A", then "AB", then "ABC", and finally "ABCD". If at // any point we block (upon return, start < limit) then we roll // back. If at any point we complete the run (upon return start == // limit) then we commit that run.
// Make a rollback copy at the end of the string
int32_t rollbackOrigin = text.length();
text.copy(runStart, runLimit, rollbackOrigin);
// Variables reflecting the commitment of completely // transliterated text. passStart is the runStart, advanced // past committed text. rollbackStart is the rollbackOrigin, // advanced past rollback text that corresponds to committed // text.
int32_t passStart = runStart;
int32_t rollbackStart = rollbackOrigin;
// The limit for each pass; we advance by one code point with // each iteration.
int32_t passLimit = index.start;
// Total length, in 16-bit code units, of uncommitted text. // This is the length to be rolled back.
int32_t uncommittedLength = 0;
// Total delta (change in length) for all passes
int32_t totalDelta = 0;
// PASS MAIN LOOP -- Start with a single character, and extend // the text by one character at a time. Roll back partial // transliterations and commit complete transliterations. for (;;) { // Length of additional code point, either one or two
int32_t charLength = U16_LENGTH(text.char32At(passLimit));
passLimit += charLength; if (passLimit > runLimit) { break;
}
uncommittedLength += charLength;
index.limit = passLimit;
// Delegate to subclass for actual transliteration. Upon // return, start will be updated to point after the // transliterated text, and limit and contextLimit will be // adjusted for length changes.
handleTransliterate(text, index, true);
delta = index.limit - passLimit; // change in length
// We failed to completely transliterate this pass. // Roll back the text. Indices remain unchanged; reset // them where necessary. if (index.start != index.limit) { // Find the rollbackStart, adjusted for length changes // and the deletion of partially transliterated text.
int32_t rs = rollbackStart + delta - (index.limit - passStart);
// Delete the partially transliterated text
text.handleReplaceBetween(passStart, index.limit, UnicodeString());
// Copy the rollback text back
text.copy(rs, rs + uncommittedLength, passStart);
// Restore indices to their original values
index.start = passStart;
index.limit = passLimit;
index.contextLimit -= delta;
}
// We did completely transliterate this pass. Update the // commit indices to record how far we got. Adjust indices // for length change. else { // Move the pass indices past the committed text.
passStart = passLimit = index.start;
// Adjust the rollbackStart for length changes and move // it past the committed text. All characters we've // processed to this point are committed now, so zero // out the uncommittedLength.
rollbackStart += delta + uncommittedLength;
uncommittedLength = 0;
// Adjust indices for length changes.
runLimit += delta;
totalDelta += delta;
}
}
// Adjust overall limit and rollbackOrigin for insertions and // deletions. Don't need to worry about contextLimit because // handleTransliterate() maintains that.
rollbackOrigin += totalDelta;
globalLimit += totalDelta;
// Delete the rollback copy
text.handleReplaceBetween(rollbackOrigin, rollbackOrigin + runLength, UnicodeString());
// Move start past committed text
index.start = passStart;
}
else { // Delegate to subclass for actual transliteration.
int32_t limit = index.limit;
handleTransliterate(text, index, isIncrementalRun);
delta = index.limit - limit; // change in length
// In a properly written transliterator, start == limit after // handleTransliterate() returns when incremental is false. // Catch cases where the subclass doesn't do this, and throw // an exception. (Just pinning start to limit is a bad idea, // because what's probably happening is that the subclass // isn't transliterating all the way to the end, and it should // in non-incremental mode.) if (!incremental && index.start != index.limit) { // We can't throw an exception, so just fudge things
index.start = index.limit;
}
// Adjust overall limit for insertions/deletions. Don't need // to worry about contextLimit because handleTransliterate() // maintains that.
globalLimit += delta;
}
if (filter == nullptr || isIncrementalRun) { break;
}
// If we did completely transliterate this // run, then repeat with the next unfiltered run.
}
// Start is valid where it is. Limit needs to be put back where // it was, modulo adjustments for deletions/insertions.
index.limit = globalLimit;
}
// Try to retrieve a UnicodeString from the bundle.
UnicodeString resString = bundle.getStringEx(key, status);
if (U_SUCCESS(status) && resString.length() != 0) { return result = resString; // [sic] assign & return
}
#if !UCONFIG_NO_FORMATTING // We have failed to get a name from the locale data. This is // typical, since most transliterators will not have localized // name data. The next step is to retrieve the MessageFormat // pattern from the locale data and to use it to synthesize the // name from the ID.
status = U_ZERO_ERROR;
resString = bundle.getStringEx(RB_DISPLAY_NAME_PATTERN, status);
if (U_SUCCESS(status) && resString.length() != 0) {
MessageFormat msg(resString, inLocale, status); // Suspend checking status until later...
// We pass either 2 or 3 Formattable objects to msg.
Formattable args[3];
int32_t nargs;
args[0].setLong(2); // # of args to follow
args[1].setString(source);
args[2].setString(target);
nargs = 3;
// Use display names for the scripts, if they exist
UnicodeString s;
length = static_cast<int32_t>(uprv_strlen(RB_SCRIPT_DISPLAY_NAME_PREFIX)); for (int j=1; j<=2; ++j) {
status = U_ZERO_ERROR;
uprv_strcpy(key, RB_SCRIPT_DISPLAY_NAME_PREFIX);
args[j].getString(s); if (uprv_isInvariantUString(s.getBuffer(), s.length())) {
s.extract(0, sizeof(key) - length - 1, key + length, static_cast<int32_t>(sizeof(key)) - length - 1, US_INV);
resString = bundle.getStringEx(key, status);
if (U_SUCCESS(status)) {
args[j] = resString;
}
}
}
status = U_ZERO_ERROR;
FieldPosition pos; // ignored by msg
msg.format(args, nargs, result, pos, status); if (U_SUCCESS(status)) {
result.append(variant); return result;
}
} #endif
}
// We should not reach this point unless there is something // wrong with the build or the RB_DISPLAY_NAME_PATTERN has // been deleted from the root RB_LOCALE_ELEMENTS resource.
result = ID; return result;
}
UnicodeString canonID;
UVector list(status); if (U_FAILURE(status)) { return nullptr;
}
UnicodeSet* globalFilter = nullptr; // TODO add code for parseError...currently unused, but // later may be used by parsing code... if (!TransliteratorIDParser::parseCompoundID(ID, dir, canonID, list, globalFilter)) {
status = U_INVALID_ID; delete globalFilter; return nullptr;
}
LocalPointer<UnicodeSet> lpGlobalFilter(globalFilter);
TransliteratorIDParser::instantiateList(list, status); if (U_FAILURE(status)) { return nullptr;
}
U_ASSERT(list.size() > 0);
Transliterator* t = nullptr;
if (list.size() > 1 || canonID.indexOf(ID_DELIM) >= 0) { // [NOTE: If it's a compoundID, we instantiate a CompoundTransliterator even if it only // has one child transliterator. This is so that toRules() will return the right thing // (without any inactive ID), but our main ID still comes out correct. That is, if we // instantiate "(Lower);Latin-Greek;", we want the rules to come out as "::Latin-Greek;" // even though the ID is "(Lower);Latin-Greek;".
t = new CompoundTransliterator(list, parseError, status);
} else {
t = static_cast<Transliterator*>(list.elementAt(0));
} // Check null pointer if (t != nullptr) {
t->setID(canonID); if (lpGlobalFilter.isValid()) {
t->adoptFilter(lpGlobalFilter.orphan());
}
} elseif (U_SUCCESS(status)) {
status = U_MEMORY_ALLOCATION_ERROR;
} return t;
}
umtx_lock(®istryMutex); if (HAVE_REGISTRY(ec)) {
t = registry->get(id, alias, ec);
}
umtx_unlock(®istryMutex);
if (U_FAILURE(ec)) { delete t; delete alias; return nullptr;
}
// We may have not gotten a transliterator: Because we can't // instantiate a transliterator from inside TransliteratorRegistry:: // get() (that would deadlock), we sometimes pass back an alias. This // contains the data we need to finish the instantiation outside the // registry mutex. The alias may, in turn, generate another alias, so // we handle aliases in a loop. The max times through the loop is two. // [alan] while (alias != nullptr) {
U_ASSERT(t==0); // Rule-based aliases are handled with TransliteratorAlias:: // parse(), followed by TransliteratorRegistry::reget(). // Other aliases are handled with TransliteratorAlias::create(). if (alias->isRuleBased()) { // Step 1. parse
TransliteratorParser parser(ec);
alias->parse(parser, pe, ec); delete alias;
alias = nullptr;
// Step 2. reget
umtx_lock(®istryMutex); if (HAVE_REGISTRY(ec)) {
t = registry->reget(id, parser, alias, ec);
}
umtx_unlock(®istryMutex);
// Step 3. Loop back around!
} else {
t = alias->create(pe, ec); delete alias;
alias = nullptr; break;
} if (U_FAILURE(ec)) { delete t; delete alias;
t = nullptr; break;
}
}
// NOTE: The logic here matches that in TransliteratorRegistry. if (parser.idBlockVector.size() == 0 && parser.dataVector.size() == 0) {
t = new NullTransliterator();
} elseif (parser.idBlockVector.size() == 0 && parser.dataVector.size() == 1) {
t = new RuleBasedTransliterator(ID, static_cast<TransliterationRuleData*>(parser.dataVector.orphanElementAt(0)), true);
} elseif (parser.idBlockVector.size() == 1 && parser.dataVector.size() == 0) { // idBlock, no data -- this is an alias. The ID has // been munged from reverse into forward mode, if // necessary, so instantiate the ID in the forward // direction. if (parser.compoundFilter != nullptr) {
UnicodeString filterPattern;
parser.compoundFilter->toPattern(filterPattern, false);
t = createInstance(filterPattern + UnicodeString(ID_DELIM)
+ *static_cast<UnicodeString*>(parser.idBlockVector.elementAt(0)), UTRANS_FORWARD, parseError, status);
} else
t = createInstance(*static_cast<UnicodeString*>(parser.idBlockVector.elementAt(0)), UTRANS_FORWARD, parseError, status);
if (t != nullptr) {
t->setID(ID);
}
} else {
UVector transliterators(status); // TODO ICU-21701 missing U_FAILURE check here. // Error and nullptr checking through this whole block looks suspect.
int32_t passNumber = 1;
for (int32_t i = 0; i < limit; i++) { if (i < parser.idBlockVector.size()) {
UnicodeString* idBlock = static_cast<UnicodeString*>(parser.idBlockVector.elementAt(i)); if (!idBlock->isEmpty()) {
Transliterator* temp = createInstance(*idBlock, UTRANS_FORWARD, parseError, status); if (U_FAILURE(status)) { delete temp; return nullptr;
} if (temp != nullptr && typeid(*temp) != typeid(NullTransliterator)) {
transliterators.addElement(temp, status); if (U_FAILURE(status)) { delete temp; return nullptr;
}
} else { delete temp;
}
}
} if (!parser.dataVector.isEmpty()) {
TransliterationRuleData* data = static_cast<TransliterationRuleData*>(parser.dataVector.orphanElementAt(0)); // TODO: Should passNumber be turned into a decimal-string representation (1 -> "1")?
RuleBasedTransliterator* temprbt = new RuleBasedTransliterator(UnicodeString(CompoundTransliterator::PASS_STRING) + UnicodeString(passNumber++),
data, true); // Check if nullptr before adding it to transliterators to avoid future usage of nullptr pointer. if (temprbt == nullptr) { if (U_SUCCESS(status)) {
status = U_MEMORY_ALLOCATION_ERROR;
} return t;
}
transliterators.addElement(temprbt, status); if (U_FAILURE(status)) { delete temprbt; return t;
} // TODO: ICU-21701 the transliterators vector will leak its contents if anything goes wrong. // Under normal operation, the CompoundTransliterator constructor adopts the // the contents of the vector.
}
}
t = new CompoundTransliterator(transliterators, passNumber - 1, parseError, status); // Null pointer check if (t != nullptr) {
t->setID(ID);
t->adoptFilter(parser.orphanCompoundFilter());
}
} if (U_SUCCESS(status) && t == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
} return t;
}
UnicodeString& Transliterator::toRules(UnicodeString& rulesSource,
UBool escapeUnprintable) const { // The base class implementation of toRules munges the ID into // the correct format. That is: foo => ::foo if (escapeUnprintable) {
rulesSource.truncate(0);
UnicodeString id = getID(); for (int32_t i=0; i<id.length();) {
UChar32 c = id.char32At(i); if (!ICU_Utility::escapeUnprintable(rulesSource, c)) {
rulesSource.append(c);
}
i += U16_LENGTH(c);
}
} else {
rulesSource = getID();
} // KEEP in sync with rbt_pars
rulesSource.insert(0, UNICODE_STRING_SIMPLE("::"));
rulesSource.append(ID_DELIM); return rulesSource;
}
// For public consumption void U_EXPORT2 Transliterator::registerFactory(const UnicodeString& id,
Transliterator::Factory factory,
Transliterator::Token context) {
Mutex lock(®istryMutex);
UErrorCode ec = U_ZERO_ERROR; if (HAVE_REGISTRY(ec)) {
_registerFactory(id, factory, context);
}
}
// To be called only by Transliterator subclasses that are called // to register themselves by initializeRegistry(). void Transliterator::_registerFactory(const UnicodeString& id,
Transliterator::Factory factory,
Transliterator::Token context) {
UErrorCode ec = U_ZERO_ERROR;
registry->put(id, factory, context, true, ec);
}
// To be called only by Transliterator subclasses that are called // to register themselves by initializeRegistry(). void Transliterator::_registerSpecialInverse(const UnicodeString& target, const UnicodeString& inverseTarget,
UBool bidirectional) {
UErrorCode status = U_ZERO_ERROR;
TransliteratorIDParser::registerSpecialInverse(target, inverseTarget, bidirectional, status);
}
RemoveTransliterator::registerIDs(); // Must be within mutex
EscapeTransliterator::registerIDs();
UnescapeTransliterator::registerIDs();
NormalizationTransliterator::registerIDs();
AnyTransliterator::registerIDs();
¤ Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.0.58Bemerkung:
(vorverarbeitet am 2026-09-28)
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
