// TODO(VIXL): Make VIXL compile cleanly with -Wshadow, -Wdeprecated-declarations. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include"aarch64/disasm-aarch64.h" #include"aarch64/macro-assembler-aarch64.h" #pragma GCC diagnostic pop
namespace art HIDDEN {
namespace arm64 {
using helpers::CPURegisterFrom; using helpers::DRegisterFrom; using helpers::HeapOperand; using helpers::LocationFrom; using helpers::Int64FromLocation; using helpers::InputCPURegisterAt; using helpers::InputCPURegisterOrZeroRegAt; using helpers::OperandFrom; using helpers::RegisterFrom; using helpers::SRegisterFrom; using helpers::WRegisterFrom; using helpers::XRegisterFrom; using helpers::HRegisterFrom; using helpers::InputRegisterAt; using helpers::OutputRegister;
__ Bind(GetEntryLabel()); // The source range and destination pointer were initialized before entering the slow-path.
vixl::aarch64::Label slow_copy_loop;
__ Bind(&slow_copy_loop);
__ Ldr(tmp_reg, MemOperand(src_curr_addr, element_size, PostIndex));
codegen->GetAssembler()->MaybeUnpoisonHeapReference(tmp_reg); // TODO: Inline the mark bit check before calling the runtime? // tmp_reg = ReadBarrier::Mark(tmp_reg); // No need to save live registers; it's taken care of by the // entrypoint. Also, there is no need to update the stack mask, // as this runtime call will not trigger a garbage collection. // (See ReadBarrierMarkSlowPathARM64::EmitNativeCode for more // explanations.)
DCHECK_NE(tmp_.reg(), LR);
DCHECK_NE(tmp_.reg(), WSP);
DCHECK_NE(tmp_.reg(), WZR); // IP0 is used internally by the ReadBarrierMarkRegX entry point // as a temporary (and not preserved). It thus cannot be used by // any live register in this slow path.
DCHECK_NE(LocationFrom(src_curr_addr).reg(), IP0);
DCHECK_NE(LocationFrom(dst_curr_addr).reg(), IP0);
DCHECK_NE(LocationFrom(src_stop_addr).reg(), IP0);
DCHECK_NE(tmp_.reg(), IP0);
DCHECK(0 <= tmp_.reg() && tmp_.reg() < kNumberOfWRegisters) << tmp_.reg(); // TODO: Load the entrypoint once before the loop, instead of // loading it at every iteration.
int32_t entry_point_offset =
Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(tmp_.reg()); // This runtime call does not require a stack map.
codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
codegen->GetAssembler()->MaybePoisonHeapReference(tmp_reg);
__ Str(tmp_reg, MemOperand(dst_curr_addr, element_size, PostIndex));
__ Cmp(src_curr_addr, src_stop_addr);
__ B(&slow_copy_loop, ne);
__ B(GetExitLabel());
}
// The MethodHandle.invokeExact intrinsic sets up arguments to match the target method call. If we // need to go to the slow path, we call art_quick_invoke_polymorphic_with_hidden_receiver, which // expects the MethodHandle object in w0 (in place of the actual ArtMethod). class InvokePolymorphicSlowPathARM64 : public SlowPathCodeARM64 { public:
InvokePolymorphicSlowPathARM64(HInstruction* instruction, Register method_handle)
: SlowPathCodeARM64(instruction), method_handle_(method_handle) {
DCHECK(instruction->IsInvokePolymorphic());
}
staticvoid GenMathRound(HInvoke* invoke, bool is_double, vixl::aarch64::MacroAssembler* masm) { // Java 8 API definition for Math.round(): // Return the closest long or int to the argument, with ties rounding to positive infinity. // // There is no single instruction in ARMv8 that can support the above definition. // We choose to use FCVTAS here, because it has closest semantic. // FCVTAS performs rounding to nearest integer, ties away from zero. // For most inputs (positive values, zero or NaN), this instruction is enough. // We only need a few handling code after FCVTAS if the input is negative half value. // // The reason why we didn't choose FCVTPS instruction here is that // although it performs rounding toward positive infinity, it doesn't perform rounding to nearest. // For example, FCVTPS(-1.9) = -1 and FCVTPS(1.1) = 2. // If we were using this instruction, for most inputs, more handling code would be needed.
LocationSummary* l = invoke->GetLocations();
VRegister in_reg = is_double ? DRegisterFrom(l->InAt(0)) : SRegisterFrom(l->InAt(0));
VRegister tmp_fp = is_double ? DRegisterFrom(l->GetTemp(0)) : SRegisterFrom(l->GetTemp(0)); Register out_reg = is_double ? XRegisterFrom(l->Out()) : WRegisterFrom(l->Out());
vixl::aarch64::Label done;
// Round to nearest integer, ties away from zero.
__ Fcvtas(out_reg, in_reg);
// For positive values, zero or NaN inputs, rounding is done.
__ Tbz(out_reg, out_reg.GetSizeInBits() - 1, &done);
// Handle input < 0 cases. // If input is negative but not a tie, previous result (round to nearest) is valid. // If input is a negative tie, out_reg += 1.
__ Frinta(tmp_fp, in_reg);
__ Fsub(tmp_fp, in_reg, tmp_fp);
__ Fcmp(tmp_fp, 0.5);
__ Cinc(out_reg, out_reg, eq);
static constexpr int kOffsetIndex = 2; static constexpr int kValueIndex = 3; Register base = WRegisterFrom(locations->InAt(1)); // Object pointer.
Location offset = locations->InAt(kOffsetIndex); // Long offset.
CPURegister value = InputCPURegisterOrZeroRegAt(invoke, kValueIndex);
CPURegister source = value;
MemOperand mem_op; if (offset.IsConstant()) {
mem_op = MemOperand(base.X(), Int64FromLocation(offset));
} else {
mem_op = MemOperand(base.X(), XRegisterFrom(offset));
}
{ // We use a block to end the scratch scope before the write barrier, thus // freeing the temporary registers so they can be used in `MarkGCCard`.
UseScratchRegisterScope temps(masm);
staticvoid EmitLoadExclusive(CodeGeneratorARM64* codegen,
DataType::Type type, Register ptr, Register old_value, bool use_load_acquire) {
Arm64Assembler* assembler = codegen->GetAssembler();
MacroAssembler* masm = assembler->GetVIXLAssembler(); switch (type) { case DataType::Type::kBool: case DataType::Type::kUint8: case DataType::Type::kInt8: if (use_load_acquire) {
__ Ldaxrb(old_value, MemOperand(ptr));
} else {
__ Ldxrb(old_value, MemOperand(ptr));
} break; case DataType::Type::kUint16: case DataType::Type::kInt16: if (use_load_acquire) {
__ Ldaxrh(old_value, MemOperand(ptr));
} else {
__ Ldxrh(old_value, MemOperand(ptr));
} break; case DataType::Type::kInt32: case DataType::Type::kInt64: case DataType::Type::kReference: if (use_load_acquire) {
__ Ldaxr(old_value, MemOperand(ptr));
} else {
__ Ldxr(old_value, MemOperand(ptr));
} break; default:
LOG(FATAL) << "Unexpected type: " << type;
UNREACHABLE();
} switch (type) { case DataType::Type::kInt8:
__ Sxtb(old_value, old_value); break; case DataType::Type::kInt16:
__ Sxth(old_value, old_value); break; case DataType::Type::kReference:
assembler->MaybeUnpoisonHeapReference(old_value); break; default: break;
}
}
staticvoid EmitStoreExclusive(CodeGeneratorARM64* codegen,
DataType::Type type, Register ptr, Register store_result, Register new_value, bool use_store_release) {
Arm64Assembler* assembler = codegen->GetAssembler();
MacroAssembler* masm = assembler->GetVIXLAssembler(); if (type == DataType::Type::kReference) {
assembler->MaybePoisonHeapReference(new_value);
} switch (type) { case DataType::Type::kBool: case DataType::Type::kUint8: case DataType::Type::kInt8: if (use_store_release) {
__ Stlxrb(store_result, new_value, MemOperand(ptr));
} else {
__ Stxrb(store_result, new_value, MemOperand(ptr));
} break; case DataType::Type::kUint16: case DataType::Type::kInt16: if (use_store_release) {
__ Stlxrh(store_result, new_value, MemOperand(ptr));
} else {
__ Stxrh(store_result, new_value, MemOperand(ptr));
} break; case DataType::Type::kInt32: case DataType::Type::kInt64: case DataType::Type::kReference: if (use_store_release) {
__ Stlxr(store_result, new_value, MemOperand(ptr));
} else {
__ Stxr(store_result, new_value, MemOperand(ptr));
} break; default:
LOG(FATAL) << "Unexpected type: " << type;
UNREACHABLE();
} if (type == DataType::Type::kReference) {
assembler->MaybeUnpoisonHeapReference(new_value);
}
}
staticvoid GenerateCompareAndSet(CodeGeneratorARM64* codegen,
DataType::Type type,
std::memory_order order, bool strong, bool use_lse,
vixl::aarch64::Label* cmp_failure, Register ptr, Register new_value, Register old_value, Register store_result, Register expected, Register expected2) { // The `expected2` is valid only for reference slow path and represents the unmarked old value // from the main path attempt to emit CAS when the marked old value matched `expected`.
DCHECK_IMPLIES(expected2.IsValid(), type == DataType::Type::kReference);
DCHECK_IMPLIES(expected2.IsValid(), !use_lse);
DCHECK(ptr.IsX());
DCHECK_EQ(new_value.IsX(), type == DataType::Type::kInt64);
DCHECK_EQ(old_value.IsX(), type == DataType::Type::kInt64);
DCHECK(store_result.IsW());
DCHECK_EQ(expected.IsX(), type == DataType::Type::kInt64);
DCHECK_IMPLIES(expected2.IsValid(), expected2.IsW());
// Compare-and-set, using LSE atomics if available. // // Without LSE, the code is a standard `ldxr`/`stxr` loop for strong CAS: // loop: // ldxr old_value, [ptr] // cmp old_value, expected // b.ne failure // stxr store_result, new_value, [ptr] // cbnz store_result, loop // For weak CAS, there is no loop and the `stxr` result is returned. // // With LSE, the code is: // mov old_value, expected // cas old_value, new_value, [ptr] // cmp old_value, expected // For strong CAS, the final Z flag from the `cmp` is used by the caller to determine // the result. For weak CAS, the result is computed with a `cset` and returned in // `store_result`. // // `expected2` is used for an additional comparison if valid.
__ Cmp(old_value, expected); if (expected2.IsValid()) {
__ Ccmp(old_value, expected2, ZFlag, ne);
} // If the comparison failed, the Z flag is cleared as we branch to the `cmp_failure` label. // If the comparison succeeded, the Z flag is set and remains set after the end of the // code emitted here, unless we retry the whole operation. if (cmp_failure != nullptr) {
__ B(cmp_failure, ne);
}
if (use_lse) { if (!strong) {
__ Cset(store_result, eq);
}
} else {
EmitStoreExclusive(codegen, type, ptr, store_result, new_value, use_store_release); if (strong) {
__ Cbnz(store_result, &loop_head);
} else { // Flip the `store_result` register to indicate success by 1 and failure by 0.
__ Eor(store_result, store_result, 1);
}
}
}
class ReadBarrierCasSlowPathARM64 : public SlowPathCodeARM64 { public:
ReadBarrierCasSlowPathARM64(HInvoke* invoke,
std::memory_order order, bool strong, Register base, Register offset, Register expected, Register new_value, Register old_value, Register old_value_temp, Register store_result, bool update_old_value,
CodeGeneratorARM64* arm64_codegen)
: SlowPathCodeARM64(invoke),
order_(order),
strong_(strong),
base_(base),
offset_(offset),
expected_(expected),
new_value_(new_value),
old_value_(old_value),
old_value_temp_(old_value_temp),
store_result_(store_result),
update_old_value_(update_old_value),
mark_old_value_slow_path_(nullptr),
update_old_value_slow_path_(nullptr) { if (!kUseBakerReadBarrier) { // We need to add the slow path now, it is too late when emitting slow path code.
mark_old_value_slow_path_ = arm64_codegen->AddReadBarrierSlowPath(
invoke,
Location::CoreRegister(old_value_temp.GetCode()),
Location::CoreRegister(old_value.GetCode()),
Location::CoreRegister(base.GetCode()), /*offset=*/ 0u, /*index=*/ Location::CoreRegister(offset.GetCode())); if (update_old_value_) {
update_old_value_slow_path_ = arm64_codegen->AddReadBarrierSlowPath(
invoke,
Location::CoreRegister(old_value.GetCode()),
Location::CoreRegister(old_value_temp.GetCode()),
Location::CoreRegister(base.GetCode()), /*offset=*/ 0u, /*index=*/ Location::CoreRegister(offset.GetCode()));
}
}
}
// Check if we need to finalize the boolean result in the slow path (weak CAS, returns // success, main path uses LSE and thus skips result finalization at the exit label). bool emit_success_csel = !strong_ && !update_old_value_ && arm64_codegen->ShouldUseLSE();
__ Bind(GetEntryLabel());
// Mark the `old_value_` from the main path and compare with `expected_`. if (kUseBakerReadBarrier) {
DCHECK(mark_old_value_slow_path_ == nullptr);
arm64_codegen->GenerateIntrinsicMoveWithBakerReadBarrier(old_value_temp_, old_value_);
} else {
DCHECK(mark_old_value_slow_path_ != nullptr);
__ B(mark_old_value_slow_path_->GetEntryLabel());
__ Bind(mark_old_value_slow_path_->GetExitLabel());
}
__ Cmp(old_value_temp_, expected_); if (update_old_value_) { // Update the old value if we're going to return from the slow path.
__ Csel(old_value_, old_value_temp_, old_value_, ne);
}
// If the main path uses LSE, it skips the CSEL at the exit label. We must ensure that // all failure paths from the slow path also finalize the result register to 0.
vixl::aarch64::Label success_csel_label;
__ B(emit_success_csel ? &success_csel_label : GetExitLabel(), ne);
// The `old_value` we have read did not match `expected` (which is always a to-space // reference) but after the read barrier the marked to-space value matched, so the // `old_value` must be a from-space reference to the same object. Do the same CAS loop // as the main path but check for both `expected` and the unmarked old value // representing the to-space and from-space references for the same object.
// Recalculate the `tmp_ptr` from main path clobbered by the read barrier above.
__ Add(tmp_ptr, base_.X(), Operand(offset_));
vixl::aarch64::Label mark_old_value; // By using 'success_csel_label' as the failure label for GenerateCompareAndSet, // we ensure that a comparison mismatch will branch to the CSEL which sets the result to 0.
vixl::aarch64::Label* cmp_failure = emit_success_csel
? &success_csel_label
: (update_old_value_ ? &mark_old_value : GetExitLabel());
GenerateCompareAndSet(arm64_codegen,
DataType::Type::kReference,
order_,
strong_, /*use_lse=*/ false, // Cannot use LSE - comparing with two values.
cmp_failure,
tmp_ptr,
new_value_, /*old_value=*/ old_value_temp_,
store_result,
expected_, /*expected2=*/ old_value_); if (update_old_value_) { // To reach this point, the `old_value_temp_` must be either a from-space or a to-space // reference of the `expected_` object. Update the `old_value_` to the to-space reference.
__ Mov(old_value_, expected_);
}
// Z=true from the CMP+CCMP in GenerateCompareAndSet() above indicates comparison success. // For strong CAS, that's the overall success. For weak CAS, the code also needs to check // the `store_result` from the STLXR to determine the success. // With LSE enabled, the main path skips the CSEL, so we emit it here to ensure // 'store_result' is correctly finalized for both fallthrough (success) and // branch-to-label (failure) paths. if (emit_success_csel) {
__ Bind(&success_csel_label);
__ Csel(store_result, store_result, wzr, eq);
}
__ B(GetExitLabel());
if (update_old_value_) {
__ Bind(&mark_old_value); if (kUseBakerReadBarrier) {
DCHECK(update_old_value_slow_path_ == nullptr);
arm64_codegen->GenerateIntrinsicMoveWithBakerReadBarrier(old_value_, old_value_temp_);
} else { // Note: We could redirect the `failure` above directly to the entry label and bind // the exit label in the main path, but the main path would need to access the // `update_old_value_slow_path_`. To keep the code simple, keep the extra jumps.
DCHECK(update_old_value_slow_path_ != nullptr);
__ B(update_old_value_slow_path_->GetEntryLabel());
__ Bind(update_old_value_slow_path_->GetExitLabel());
}
__ B(GetExitLabel());
}
}
Register out = WRegisterFrom(locations->Out()); // Boolean result. Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. Register expected = RegisterFrom(locations->InAt(3), type); // Expected. Register new_value = RegisterFrom(locations->InAt(4), type); // New value.
// This needs to be before the temp registers, as MarkGCCard also uses VIXL temps. if (type == DataType::Type::kReference) { // Mark card for object assuming new value is stored. bool new_value_can_be_null = true; // TODO: Worth finding out this information?
codegen->MaybeMarkGCCard(base, new_value, new_value_can_be_null);
}
UseScratchRegisterScope temps(masm); Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory. Register old_value; // Value in memory.
if (type == DataType::Type::kReference && codegen->EmitReadBarrier()) { // We need to store the `old_value` in a non-scratch register to make sure // the read barrier in the slow path does not clobber it.
old_value = WRegisterFrom(locations->GetTemp(0)); // The old value from main path. // The `old_value_temp` is used first for the marked `old_value` and then for the unmarked // reloaded old value for subsequent CAS in the slow path. It cannot be a scratch register. Register old_value_temp = WRegisterFrom(locations->GetTemp(1));
ReadBarrierCasSlowPathARM64* slow_path = new (codegen->GetScopedAllocator()) ReadBarrierCasSlowPathARM64(
invoke,
std::memory_order_seq_cst, /*strong=*/ true,
base,
offset,
expected,
new_value,
old_value,
old_value_temp, /*store_result=*/ Register(), // Use a scratch register. /*update_old_value=*/ false,
codegen);
codegen->AddSlowPath(slow_path);
exit_loop = slow_path->GetExitLabel();
cmp_failure = slow_path->GetEntryLabel();
} else {
old_value = temps.AcquireSameSizeAs(new_value); if (use_lse) {
cmp_failure = nullptr;
}
}
void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCompareAndSetInt(HInvoke* invoke) {
CreateUnsafeCASLocations(allocator_, invoke, codegen_);
} void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCompareAndSetLong(HInvoke* invoke) {
CreateUnsafeCASLocations(allocator_, invoke, codegen_);
} void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { // The only supported read barrier implementation is the Baker-style read barriers. if (codegen_->EmitNonBakerReadBarrier()) { return;
}
CreateUnsafeCASLocations(allocator_, invoke, codegen_); if (codegen_->EmitReadBarrier()) { // We need two non-scratch temporary registers for read barrier.
LocationSummary* locations = invoke->GetLocations(); if (kUseBakerReadBarrier) {
locations->AddRegisterTemps(2);
} else { // To preserve the old value across the non-Baker read barrier // slow path, use a fixed callee-save register.
constexpr int first_callee_save = CTZ(kArm64CalleeSaveRefSpills);
locations->AddTemp(Location::CoreRegister(first_callee_save)); // To reduce the number of moves, request x0 as the second temporary.
DCHECK(InvokeRuntimeCallingConvention().GetReturnLocation(DataType::Type::kReference).Equals(
Location::CoreRegister(x0.GetCode())));
locations->AddTemp(Location::CoreRegister(x0.GetCode()));
}
}
}
// Request another temporary register for methods that don't return a value.
DataType::Type return_type = invoke->GetType(); constbool is_void = return_type == DataType::Type::kVoid; if (is_void) {
locations->AddTemp(Location::RequiresCoreRegister());
} else {
locations->SetOut(Location::RequiresCoreRegister(), Location::kOutputOverlap);
}
}
staticvoid GenUnsafeGetAndUpdate(HInvoke* invoke,
DataType::Type type,
CodeGeneratorARM64* codegen,
GetAndUpdateOp get_and_update_op) { // Currently only used for these GetAndUpdateOp. Might be fine for other ops but double check // before using.
DCHECK(get_and_update_op == GetAndUpdateOp::kAdd || get_and_update_op == GetAndUpdateOp::kSet);
DataType::Type return_type = invoke->GetType(); constbool is_void = return_type == DataType::Type::kVoid; // We use a temporary for void methods, as we don't return the value.
Location out_or_temp_loc =
is_void ? locations->GetTemp(locations->GetTempCount() - 1u) : locations->Out(); Register out_or_temp = RegisterFrom(out_or_temp_loc, type); // Result. Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. Register arg = RegisterFrom(locations->InAt(3), type); // New value or addend. Register tmp_ptr = XRegisterFrom(locations->GetTemp(0)); // Pointer to actual memory.
// This needs to be before the temp registers, as MarkGCCard also uses VIXL temps. if (type == DataType::Type::kReference) {
DCHECK(get_and_update_op == GetAndUpdateOp::kSet); // Mark card for object as a new value shall be stored. bool new_value_can_be_null = true; // TODO: Worth finding out this information?
codegen->MaybeMarkGCCard(base, /*value=*/arg, new_value_can_be_null);
}
// Get offsets of count and value fields within a string object. const int32_t count_offset = mirror::String::CountOffset().Int32Value(); const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
// Note that the null check must have been done earlier.
DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
// Take slow path and throw if input can be and is null.
SlowPathCodeARM64* slow_path = nullptr; constbool can_slow_path = invoke->InputAt(1)->CanBeNull(); if (can_slow_path) {
slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
codegen_->AddSlowPath(slow_path);
__ Cbz(arg, slow_path->GetEntryLabel());
}
// Reference equality check, return 0 if same reference.
__ Subs(out, str, arg);
__ B(&end, eq);
if (mirror::kUseStringCompression) { // Load `count` fields of this and argument strings.
__ Ldr(temp3, HeapOperand(str, count_offset));
__ Ldr(temp2, HeapOperand(arg, count_offset)); // Clean out compression flag from lengths.
__ Lsr(temp0, temp3, 1u);
__ Lsr(temp1, temp2, 1u);
} else { // Load lengths of this and argument strings.
__ Ldr(temp0, HeapOperand(str, count_offset));
__ Ldr(temp1, HeapOperand(arg, count_offset));
} // out = length diff.
__ Subs(out, temp0, temp1); // temp0 = min(len(str), len(arg)).
__ Csel(temp0, temp1, temp0, ge); // Shorter string is empty?
__ Cbz(temp0, &end);
if (mirror::kUseStringCompression) { // Check if both strings using same compression style to use this comparison loop.
__ Eor(temp2, temp2, Operand(temp3)); // Interleave with compression flag extraction which is needed for both paths // and also set flags which is needed only for the different compressions path.
__ Ands(temp3.W(), temp3.W(), Operand(1));
__ Tbnz(temp2, 0, &different_compression); // Does not use flags.
} // Store offset of string value in preparation for comparison loop.
__ Mov(temp1, value_offset); if (mirror::kUseStringCompression) { // For string compression, calculate the number of bytes to compare (not chars). // This could in theory exceed INT32_MAX, so treat temp0 as unsigned.
__ Lsl(temp0, temp0, temp3);
}
// Assertions that must hold in order to compare strings 8 bytes at a time.
DCHECK_ALIGNED(value_offset, 8);
static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
// Promote temp2 to an X reg, ready for LDR.
temp2 = temp2.X();
// Loop to compare 4x16-bit characters at a time (ok because of string data alignment).
__ Bind(&loop);
__ Ldr(temp4, MemOperand(str.X(), temp1.X()));
__ Ldr(temp2, MemOperand(arg.X(), temp1.X()));
__ Cmp(temp4, temp2);
__ B(ne, &find_char_diff);
__ Add(temp1, temp1, char_size * 4); // With string compression, we have compared 8 bytes, otherwise 4 chars.
__ Subs(temp0, temp0, (mirror::kUseStringCompression) ? 8 : 4);
__ B(&loop, hi);
__ B(&end);
// Promote temp1 to an X reg, ready for EOR.
temp1 = temp1.X();
// Find the single character difference.
__ Bind(&find_char_diff); // Get the bit position of the first character that differs.
__ Eor(temp1, temp2, temp4);
__ Rbit(temp1, temp1);
__ Clz(temp1, temp1);
// If the number of chars remaining <= the index where the difference occurs (0-3), then // the difference occurs outside the remaining string data, so just return length diff (out). // Unlike ARM, we're doing the comparison in one go here, without the subtraction at the // find_char_diff_2nd_cmp path, so it doesn't matter whether the comparison is signed or // unsigned when string compression is disabled. // When it's enabled, the comparison must be unsigned.
__ Cmp(temp0, Operand(temp1.W(), LSR, (mirror::kUseStringCompression) ? 3 : 4));
__ B(ls, &end);
// Extract the characters and calculate the difference. if (mirror:: kUseStringCompression) {
__ Bic(temp1, temp1, 0x7);
__ Bic(temp1, temp1, Operand(temp3.X(), LSL, 3u));
} else {
__ Bic(temp1, temp1, 0xf);
}
__ Lsr(temp2, temp2, temp1);
__ Lsr(temp4, temp4, temp1); if (mirror::kUseStringCompression) { // Prioritize the case of compressed strings and calculate such result first.
__ Uxtb(temp1, temp4);
__ Sub(out, temp1.W(), Operand(temp2.W(), UXTB));
__ Tbz(temp3, 0u, &end); // If actually compressed, we're done.
}
__ Uxth(temp4, temp4);
__ Sub(out, temp4.W(), Operand(temp2.W(), UXTH));
if (mirror::kUseStringCompression) {
__ B(&end);
__ Bind(&different_compression);
// `temp1` will hold the compressed data pointer, `temp2` the uncompressed data pointer. // Note that flags have been set by the `str` compression flag extraction to `temp3` // before branching to the `different_compression` label.
__ Csel(temp1, str, arg, eq); // Pointer to the compressed string.
__ Csel(temp2, str, arg, ne); // Pointer to the uncompressed string.
// We want to free up the temp3, currently holding `str` compression flag, for comparison. // So, we move it to the bottom bit of the iteration count `temp0` which we then need to treat // as unsigned. Start by freeing the bit with a LSL and continue further down by a SUB which // will allow `subs temp0, #2; bhi different_compression_loop` to serve as the loop condition.
__ Lsl(temp0, temp0, 1u);
// Adjust temp1 and temp2 from string pointers to data pointers.
__ Add(temp1, temp1, Operand(value_offset));
__ Add(temp2, temp2, Operand(value_offset));
// Complete the move of the compression flag.
__ Sub(temp0, temp0, Operand(temp3));
if (can_slow_path) {
__ Bind(slow_path->GetExitLabel());
}
}
// The cut off for unrolling the loop in String.equals() intrinsic for const strings. // The normal loop plus the pre-header is 9 instructions without string compression and 12 // instructions with string compression. We can compare up to 8 bytes in 4 instructions // (LDR+LDR+CMP+BNE) and up to 16 bytes in 5 instructions (LDP+LDP+CMP+CCMP+BNE). Allow up // to 10 instructions for the unrolled loop.
constexpr size_t kShortConstStringEqualsCutoffInBytes = 32;
// For the generic implementation and for long const strings we need a temporary. // We do not need it for short const strings, up to 8 bytes, see code generation below.
uint32_t const_string_length = 0u; constchar* const_string = GetConstString(invoke->InputAt(0), &const_string_length); if (const_string == nullptr) {
const_string = GetConstString(invoke->InputAt(1), &const_string_length);
} bool is_compressed =
mirror::kUseStringCompression &&
const_string != nullptr &&
mirror::String::DexFileStringAllASCII(const_string, const_string_length); if (const_string == nullptr || const_string_length > (is_compressed ? 8u : 4u)) {
locations->AddTemp(Location::RequiresCoreRegister());
}
// TODO: If the String.equals() is used only for an immediately following HIf, we can // mark it as emitted-at-use-site and emit branches directly to the appropriate blocks. // Then we shall need an extra temporary register instead of the output register.
locations->SetOut(Location::RequiresCoreRegister(), Location::kOutputOverlap);
}
// Get offsets of count, value, and class fields within a string object. const int32_t count_offset = mirror::String::CountOffset().Int32Value(); const int32_t value_offset = mirror::String::ValueOffset().Int32Value(); const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
// Note that the null check must have been done earlier.
DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
StringEqualsOptimizations optimizations(invoke); if (!optimizations.GetArgumentNotNull()) { // Check if input is null, return false if it is.
__ Cbz(arg, &return_false);
}
// Reference equality check, return true if same reference.
__ Cmp(str, arg);
__ B(&return_true, eq);
if (!optimizations.GetArgumentIsString()) { // Instanceof check for the argument by comparing class fields. // All string objects must have the same type since String cannot be subclassed. // Receiver must be a string object, so its class field is equal to all strings' class fields. // If the argument is a string object, its class field must be equal to receiver's class field. // // As the String class is expected to be non-movable, we can read the class // field from String.equals' arguments without read barriers.
AssertNonMovableStringClass(); // /* HeapReference<Class> */ temp = str->klass_
__ Ldr(temp, MemOperand(str.X(), class_offset)); // /* HeapReference<Class> */ temp1 = arg->klass_
__ Ldr(temp1, MemOperand(arg.X(), class_offset)); // Also, because we use the previously loaded class references only in the // following comparison, we don't need to unpoison them.
__ Cmp(temp, temp1);
__ B(&return_false, ne);
}
// Check if one of the inputs is a const string. Do not special-case both strings // being const, such cases should be handled by constant folding if needed.
uint32_t const_string_length = 0u; constchar* const_string = GetConstString(invoke->InputAt(0), &const_string_length); if (const_string == nullptr) {
const_string = GetConstString(invoke->InputAt(1), &const_string_length); if (const_string != nullptr) {
std::swap(str, arg); // Make sure the const string is in `str`.
}
} bool is_compressed =
mirror::kUseStringCompression &&
const_string != nullptr &&
mirror::String::DexFileStringAllASCII(const_string, const_string_length);
if (const_string != nullptr) { // Load `count` field of the argument string and check if it matches the const string. // Also compares the compression style, if differs return false.
__ Ldr(temp, MemOperand(arg.X(), count_offset)); // Temporarily release temp1 as we may not be able to embed the flagged count in CMP immediate.
scratch_scope.Release(temp1);
__ Cmp(temp, Operand(mirror::String::GetFlaggedCount(const_string_length, is_compressed)));
temp1 = scratch_scope.AcquireW();
__ B(&return_false, ne);
} else { // Load `count` fields of this and argument strings.
__ Ldr(temp, MemOperand(str.X(), count_offset));
__ Ldr(temp1, MemOperand(arg.X(), count_offset)); // Check if `count` fields are equal, return false if they're not. // Also compares the compression style, if differs return false.
__ Cmp(temp, temp1);
__ B(&return_false, ne);
}
// Assertions that must hold in order to compare strings 8 bytes at a time. // Ok to do this because strings are zero-padded to kObjectAlignment.
DCHECK_ALIGNED(value_offset, 8);
static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
if (const_string != nullptr &&
const_string_length <= (is_compressed ? kShortConstStringEqualsCutoffInBytes
: kShortConstStringEqualsCutoffInBytes / 2u)) { // Load and compare the contents. Though we know the contents of the short const string // at compile time, materializing constants may be more code than loading from memory.
int32_t offset = value_offset;
size_t remaining_bytes =
RoundUp(is_compressed ? const_string_length : const_string_length * 2u, 8u);
temp = temp.X();
temp1 = temp1.X(); while (remaining_bytes > sizeof(uint64_t)) { Register temp2 = XRegisterFrom(locations->GetTemp(0));
__ Ldp(temp, temp1, MemOperand(str.X(), offset));
__ Ldp(temp2, out, MemOperand(arg.X(), offset));
__ Cmp(temp, temp2);
__ Ccmp(temp1, out, NoFlag, eq);
__ B(&return_false, ne);
offset += 2u * sizeof(uint64_t);
remaining_bytes -= 2u * sizeof(uint64_t);
} if (remaining_bytes != 0u) {
__ Ldr(temp, MemOperand(str.X(), offset));
__ Ldr(temp1, MemOperand(arg.X(), offset));
__ Cmp(temp, temp1);
__ B(&return_false, ne);
}
} else { // Return true if both strings are empty. Even with string compression `count == 0` means empty.
static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, "Expecting 0=compressed, 1=uncompressed");
__ Cbz(temp, &return_true);
if (mirror::kUseStringCompression) { // For string compression, calculate the number of bytes to compare (not chars). // This could in theory exceed INT32_MAX, so treat temp as unsigned.
__ And(temp1, temp, Operand(1)); // Extract compression flag.
__ Lsr(temp, temp, 1u); // Extract length.
__ Lsl(temp, temp, temp1); // Calculate number of bytes to compare.
}
// Store offset of string value in preparation for comparison loop
__ Mov(temp1, value_offset);
temp1 = temp1.X(); Register temp2 = XRegisterFrom(locations->GetTemp(0)); // Loop to compare strings 8 bytes at a time starting at the front of the string.
__ Bind(&loop);
__ Ldr(out, MemOperand(str.X(), temp1));
__ Ldr(temp2, MemOperand(arg.X(), temp1));
__ Add(temp1, temp1, Operand(sizeof(uint64_t)));
__ Cmp(out, temp2);
__ B(&return_false, ne); // With string compression, we have compared 8 bytes, otherwise 4 chars.
__ Sub(temp, temp, Operand(mirror::kUseStringCompression ? 8 : 4), SetFlags);
__ B(&loop, hi);
}
// Return true and exit the function. // If loop does not result in returning false, we return true.
__ Bind(&return_true);
__ Mov(out, 1);
__ B(&end);
// Return false and exit the function.
__ Bind(&return_false);
__ Mov(out, 0);
__ Bind(&end);
}
// Note that the null check must have been done earlier.
DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
// Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, // or directly dispatch for a large constant, or omit slow-path for a small constant or a char.
SlowPathCodeARM64* slow_path = nullptr;
HInstruction* code_point = invoke->InputAt(1); if (code_point->IsIntConstant()) { if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) > 0xFFFFU) { // Always needs the slow-path. We could directly dispatch to it, but this case should be // rare, so for simplicity just put the full slow-path down and branch unconditionally.
slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
codegen->AddSlowPath(slow_path);
__ B(slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel()); return;
}
} elseif (code_point->GetType() != DataType::Type::kUint16) { Register char_reg = WRegisterFrom(locations->InAt(1));
__ Tst(char_reg, 0xFFFF0000);
slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
codegen->AddSlowPath(slow_path);
__ B(ne, slow_path->GetEntryLabel());
}
if (slow_path != nullptr) {
__ Bind(slow_path->GetExitLabel());
}
}
void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
LocationSummary* locations = LocationSummary::Create(
allocator_, invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's // best to align the inputs accordingly.
InvokeRuntimeCallingConvention calling_convention;
locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kInt32));
// Need to send start_index=0.
locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
}
void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
LocationSummary* locations = LocationSummary::Create(
allocator_, invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's // best to align the inputs accordingly.
InvokeRuntimeCallingConvention calling_convention;
locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kInt32));
}
void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) { // No need to emit code checking whether `locations->InAt(2)` is a null // pointer, as callers of the native method // // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) // // all include a null check on `data` before calling that method.
codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke);
CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>();
}
// Do the copy.
vixl::aarch64::Label loop;
vixl::aarch64::Label remainder;
// Save repairing the value of num_chr on the < 8 character path.
__ Subs(tmp1, num_chr, 8);
__ B(lt, &remainder);
// Keep the result of the earlier subs, we are going to fetch at least 8 characters.
__ Mov(num_chr, tmp1);
// Main loop used for longer fetches loads and stores 8x16-bit characters at a time. // (Unaligned addresses are acceptable here and not worth inlining extra code to rectify.)
__ Bind(&loop);
__ Ldp(tmp1, tmp2, MemOperand(src_ptr, char_size * 8, PostIndex));
__ Subs(num_chr, num_chr, 8);
__ Stp(tmp1, tmp2, MemOperand(dst_ptr, char_size * 8, PostIndex));
__ B(ge, &loop);
__ Adds(num_chr, num_chr, 8);
__ B(eq, &done);
// Main loop for < 8 character case and remainder handling. Loads and stores one // 16-bit Java character at a time.
__ Bind(&remainder);
__ Ldrh(tmp1, MemOperand(src_ptr, char_size, PostIndex));
__ Subs(num_chr, num_chr, 1);
__ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex));
__ B(gt, &remainder);
__ B(&done);
if (mirror::kUseStringCompression) { // For compressed strings, acquire a SIMD temporary register.
VRegister vtmp1 = temps.AcquireVRegisterOfSize(kQRegSize); const size_t c_char_size = DataType::Size(DataType::Type::kInt8);
DCHECK_EQ(c_char_size, 1u);
__ Bind(&compressed_string_preloop);
__ Add(src_ptr, src_ptr, Operand(srcBegin));
// Save repairing the value of num_chr on the < 8 character path.
__ Subs(tmp1, num_chr, 8);
__ B(lt, &compressed_string_remainder);
// Keep the result of the earlier subs, we are going to fetch at least 8 characters.
__ Mov(num_chr, tmp1);
// Main loop for compressed src, copying 8 characters (8-bit) to (16-bit) at a time. // Uses SIMD instructions.
__ Bind(&compressed_string_vector_loop);
__ Ld1(vtmp1.V8B(), MemOperand(src_ptr, c_char_size * 8, PostIndex));
__ Subs(num_chr, num_chr, 8);
__ Uxtl(vtmp1.V8H(), vtmp1.V8B());
__ St1(vtmp1.V8H(), MemOperand(dst_ptr, char_size * 8, PostIndex));
__ B(ge, &compressed_string_vector_loop);
__ Adds(num_chr, num_chr, 8);
__ B(eq, &done);
// Loop for < 8 character case and remainder handling with a compressed src. // Copies 1 character (8-bit) to (16-bit) at a time.
__ Bind(&compressed_string_remainder);
__ Ldrb(tmp1, MemOperand(src_ptr, c_char_size, PostIndex));
__ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex));
__ Subs(num_chr, num_chr, Operand(1));
__ B(gt, &compressed_string_remainder);
}
// This value is in bytes and greater than ARRAYCOPY_SHORT_XXX_ARRAY_THRESHOLD // in libcore, so if we choose to jump to the slow path we will end up // in the native implementation. static constexpr int32_t kSystemArrayCopyPrimThreshold = 384;
staticvoid CheckSystemArrayCopyPosition(MacroAssembler* masm, Register array,
Location pos,
Location length,
SlowPathCodeARM64* slow_path, Register temp, bool length_is_array_length, bool position_sign_checked) { const int32_t length_offset = mirror::Array::LengthOffset().Int32Value(); if (pos.IsConstant()) {
int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue(); if (pos_const == 0) { if (!length_is_array_length) { // Check that length(array) >= length.
__ Ldr(temp, MemOperand(array, length_offset));
__ Cmp(temp, OperandFrom(length, DataType::Type::kInt32));
__ B(slow_path->GetEntryLabel(), lt);
}
} else { // Calculate length(array) - pos. // Both operands are known to be non-negative `int32_t`, so the difference cannot underflow // as `int32_t`. If the result is negative, the B.LT below shall go to the slow path.
__ Ldr(temp, MemOperand(array, length_offset));
__ Sub(temp, temp, pos_const);
// Check that (length(array) - pos) >= length.
__ Cmp(temp, OperandFrom(length, DataType::Type::kInt32));
__ B(slow_path->GetEntryLabel(), lt);
}
} elseif (length_is_array_length) { // The only way the copy can succeed is if pos is zero.
__ Cbnz(WRegisterFrom(pos), slow_path->GetEntryLabel());
} else { // Check that pos >= 0. Register pos_reg = WRegisterFrom(pos); if (!position_sign_checked) {
__ Tbnz(pos_reg, pos_reg.GetSizeInBits() - 1, slow_path->GetEntryLabel());
}
// Calculate length(array) - pos. // Both operands are known to be non-negative `int32_t`, so the difference cannot underflow // as `int32_t`. If the result is negative, the B.LT below shall go to the slow path.
__ Ldr(temp, MemOperand(array, length_offset));
__ Sub(temp, temp, pos_reg);
// If source and destination are the same, then the copied arrays may overlap. // For overlapping arrays we can only guarantee correctness if `src_pos >= dst_pos`, otherwise // copying the elements at the beginning of source array may clobber the elements at the end. if (!optimizations.GetSourcePositionIsDestinationPosition()) { if (src_pos.IsConstant()) {
int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); if (dest_pos.IsConstant()) {
int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); if (optimizations.GetDestinationIsSource()) { // Checked when building locations.
DCHECK_GE(src_pos_constant, dest_pos_constant);
} elseif (src_pos_constant < dest_pos_constant) {
__ Cmp(src, dest);
__ B(slow_path->GetEntryLabel(), eq);
}
} else { if (!optimizations.GetDestinationIsSource()) {
__ Cmp(src, dest);
__ B(&conditions_on_positions_validated, ne);
}
__ Cmp(WRegisterFrom(dest_pos), src_pos_constant);
__ B(slow_path->GetEntryLabel(), gt);
}
} else { if (!optimizations.GetDestinationIsSource()) {
__ Cmp(src, dest);
__ B(&conditions_on_positions_validated, ne);
}
__ Cmp(RegisterFrom(src_pos, invoke->InputAt(1)->GetType()),
OperandFrom(dest_pos, invoke->InputAt(3)->GetType()));
__ B(slow_path->GetEntryLabel(), lt);
}
}
__ Bind(&conditions_on_positions_validated);
if (!optimizations.GetSourceIsNotNull()) { // Bail out if the source is null.
__ Cbz(src, slow_path->GetEntryLabel());
}
if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) { // Bail out if the destination is null.
__ Cbz(dest, slow_path->GetEntryLabel());
}
// We have already checked in the LocationsBuilder for the constant case. if (!length.IsConstant()) { // Merge the following two comparisons into one: // If the length is negative, bail out (delegate to libcore's native implementation). // If the length >= 128 then (currently) prefer native implementation.
__ Cmp(WRegisterFrom(length), copy_threshold);
__ B(slow_path->GetEntryLabel(), hs);
} else { // We have already checked in the LocationsBuilder for the constant case.
DCHECK_GE(length.GetConstant()->AsIntConstant()->GetValue(), 0);
DCHECK_LE(length.GetConstant()->AsIntConstant()->GetValue(), copy_threshold);
}
}
SlowPathCodeARM64* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
codegen->AddSlowPath(slow_path);
// Check that source and position are different, or if they are the same check that copy // direction is backward. Also check for null pointers.
int32_t copy_threshold = kSystemArrayCopyPrimThreshold / DataType::Size(type);
CheckSystemArrayCopyNullOrOverlap(
invoke, masm, slow_path, src, dst, src_pos, dst_pos, length, copy_threshold);
// Check that source position is within bounds.
CheckSystemArrayCopyPosition(masm,
src,
src_pos,
length,
slow_path,
src_curr_addr, /*length_is_array_length=*/ false, /*position_sign_checked=*/ false);
// Check that destination position is within bounds.
CheckSystemArrayCopyPosition(masm,
dst,
dst_pos,
length,
slow_path,
src_curr_addr, /*length_is_array_length=*/ false, /*position_sign_checked=*/ false);
// Iterate over the arrays and do a raw copy of the chars. const int32_t element_size = DataType::Size(type);
UseScratchRegisterScope temps(masm);
// We split processing of the array in two parts: head and tail. // A first loop handles the head by copying a block of characters per // iteration (see: chars_per_block). // A second loop handles the tail by copying the remaining characters. // If the copy length is not constant, we copy them one-by-one. // If the copy length is constant, we optimize by always unrolling the tail // loop, and also unrolling the head loop when the copy length is small (see: // unroll_threshold). // // Both loops are inverted for better performance, meaning they are // implemented as conditional do-while loops. // Here, the loop condition is first checked to determine if there are // sufficient chars to run an iteration, then we enter the do-while: an // iteration is performed followed by a conditional branch only if another // iteration is necessary. As opposed to a standard while-loop, this inversion // can save some branching (e.g. we don't branch back to the initial condition // at the end of every iteration only to potentially immediately branch // again). // // A full block of chars is subtracted and added before and after the head // loop, respectively. This ensures that any remaining length after each // head loop iteration means there is a full block remaining, reducing the // number of conditional checks required on every iteration.
constexpr int32_t max_stride_in_bytes = 8;
constexpr int32_t unroll_threshold = 2 * max_stride_in_bytes;
DCHECK_EQ(max_stride_in_bytes % element_size, 0);
int32_t elements_per_block = max_stride_in_bytes / element_size;
vixl::aarch64::Label loop1, loop2, pre_loop2, done;
// We choose to use the native implementation for longer copy lengths. static constexpr int32_t kSystemArrayCopyThreshold = 128;
void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopy(HInvoke* invoke) { // The only read barrier implementation supporting the // SystemArrayCopy intrinsic is the Baker-style read barriers. if (codegen_->EmitNonBakerReadBarrier()) { return;
}
constexpr size_t kInitialNumTemps = 2u; // We need at least two temps.
LocationSummary* locations = CodeGenerator::CreateSystemArrayCopyLocationSummary(
invoke, kSystemArrayCopyThreshold, kInitialNumTemps); if (locations != nullptr) {
locations->SetInAt(1, LocationForSystemArrayCopyInput(invoke->InputAt(1)));
locations->SetInAt(3, LocationForSystemArrayCopyInput(invoke->InputAt(3)));
locations->SetInAt(4, LocationForSystemArrayCopyInput(invoke->InputAt(4))); if (codegen_->EmitBakerReadBarrier()) { // Temporary register IP0, obtained from the VIXL scratch register // pool, cannot be used in ReadBarrierSystemArrayCopySlowPathARM64 // (because that register is clobbered by ReadBarrierMarkRegX // entry points). It cannot be used in calls to // CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier // either. For these reasons, get a third extra temporary register // from the register allocator.
locations->AddTemp(Location::RequiresCoreRegister());
} else { // Cases other than Baker read barriers: the third temporary will // be acquired from the VIXL scratch register pool.
}
}
}
void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) { // The only read barrier implementation supporting the // SystemArrayCopy intrinsic is the Baker-style read barriers.
DCHECK_IMPLIES(codegen_->EmitReadBarrier(), kUseBakerReadBarrier);
SlowPathCodeARM64* intrinsic_slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
codegen_->AddSlowPath(intrinsic_slow_path);
// Check that source and position are different, or if they are the same check that copy // direction is backward. Also check for null pointers.
CheckSystemArrayCopyNullOrOverlap(invoke,
masm,
intrinsic_slow_path,
src,
dest,
src_pos,
dest_pos,
length,
kSystemArrayCopyThreshold);
// Check that source position is within bounds.
CheckSystemArrayCopyPosition(masm,
src,
src_pos,
length,
intrinsic_slow_path,
temp1,
optimizations.GetCountIsSourceLength(), /*position_sign_checked=*/ false);
// Check that destination position is within bounds. bool dest_position_sign_checked = optimizations.GetSourcePositionIsDestinationPosition();
CheckSystemArrayCopyPosition(masm,
dest,
dest_pos,
length,
intrinsic_slow_path,
temp1,
optimizations.GetCountIsDestinationLength(),
dest_position_sign_checked);
auto check_non_primitive_array_class = [&](Register klass, Register temp) { // No read barrier is needed for reading a chain of constant references for comparing // with null, or for reading a constant primitive value, see `ReadBarrierOption`. // /* HeapReference<Class> */ temp = klass->component_type_
__ Ldr(temp, HeapOperand(klass, component_offset));
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp); // Check that the component type is not null.
__ Cbz(temp, intrinsic_slow_path->GetEntryLabel()); // Check that the component type is not a primitive. // /* uint16_t */ temp = static_cast<uint16>(klass->primitive_type_);
__ Ldrh(temp, HeapOperand(temp, primitive_offset));
static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
__ Cbnz(temp, intrinsic_slow_path->GetEntryLabel());
};
if (!optimizations.GetDoesNotNeedTypeCheck()) { // Check whether all elements of the source array are assignable to the component // type of the destination array. We do two checks: the classes are the same, // or the destination is Object[]. If none of these checks succeed, we go to the // slow path.
if (codegen_->EmitBakerReadBarrier()) {
Location temp3_loc = locations->GetTemp(2); // /* HeapReference<Class> */ temp1 = dest->klass_
codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
temp1_loc,
dest.W(),
class_offset,
temp3_loc, /* needs_null_check= */ false, /* use_load_acquire= */ false); // Register `temp1` is not trashed by the read barrier emitted // by GenerateFieldLoadWithBakerReadBarrier below, as that // method produces a call to a ReadBarrierMarkRegX entry point, // which saves all potentially live registers, including // temporaries such a `temp1`. // /* HeapReference<Class> */ temp2 = src->klass_
codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
temp2_loc,
src.W(),
class_offset,
temp3_loc, /* needs_null_check= */ false, /* use_load_acquire= */ false);
} else { // /* HeapReference<Class> */ temp1 = dest->klass_
__ Ldr(temp1, MemOperand(dest, class_offset));
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1); // /* HeapReference<Class> */ temp2 = src->klass_
__ Ldr(temp2, MemOperand(src, class_offset));
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp2);
}
__ Cmp(temp1, temp2); if (optimizations.GetDestinationIsTypedObjectArray()) {
DCHECK(optimizations.GetDestinationIsNonPrimitiveArray());
vixl::aarch64::Label do_copy; // For class match, we can skip the source type check regardless of the optimization flag.
__ B(&do_copy, eq); // No read barrier is needed for reading a chain of constant references // for comparing with null, see `ReadBarrierOption`. // /* HeapReference<Class> */ temp1 = temp1->component_type_
__ Ldr(temp1, HeapOperand(temp1, component_offset));
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1); // /* HeapReference<Class> */ temp1 = temp1->super_class_
__ Ldr(temp1, HeapOperand(temp1, super_offset)); // No need to unpoison the result, we're comparing against null.
__ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); // Bail out if the source is not a non primitive array. if (!optimizations.GetSourceIsNonPrimitiveArray()) {
check_non_primitive_array_class(temp2, temp2);
}
__ Bind(&do_copy);
} else {
DCHECK(!optimizations.GetDestinationIsTypedObjectArray()); // For class match, we can skip the array type check completely if at least one of source // and destination is known to be a non primitive array, otherwise one check is enough.
__ B(intrinsic_slow_path->GetEntryLabel(), ne); if (!optimizations.GetDestinationIsNonPrimitiveArray() &&
!optimizations.GetSourceIsNonPrimitiveArray()) {
check_non_primitive_array_class(temp2, temp2);
}
}
} elseif (!optimizations.GetSourceIsNonPrimitiveArray()) {
DCHECK(optimizations.GetDestinationIsNonPrimitiveArray()); // Bail out if the source is not a non primitive array. // No read barrier is needed for reading a chain of constant references for comparing // with null, or for reading a constant primitive value, see `ReadBarrierOption`. // /* HeapReference<Class> */ temp2 = src->klass_
__ Ldr(temp2, MemOperand(src, class_offset));
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp2);
check_non_primitive_array_class(temp2, temp2);
}
if (length.IsConstant() && length.GetConstant()->AsIntConstant()->GetValue() == 0) { // Null constant length: not need to emit the loop code at all.
} else {
vixl::aarch64::Label skip_copy_and_write_barrier; if (length.IsCoreRegister()) { // Don't enter the copy loop if the length is null.
__ Cbz(WRegisterFrom(length), &skip_copy_and_write_barrier);
}
{ // We use a block to end the scratch scope before the write barrier, thus // freeing the temporary registers so they can be used in `MarkGCCard`.
UseScratchRegisterScope temps(masm); bool emit_rb = codegen_->EmitBakerReadBarrier(); Register temp3; Register tmp; if (emit_rb) {
temp3 = WRegisterFrom(locations->GetTemp(2)); // Make sure `tmp` is not IP0, as it is clobbered by ReadBarrierMarkRegX entry points // in ReadBarrierSystemArrayCopySlowPathARM64. Explicitly allocate the register IP1.
DCHECK(temps.IsAvailable(ip1));
temps.Exclude(ip1);
tmp = ip1.W();
} else {
temp3 = temps.AcquireW();
tmp = temps.AcquireW();
}
SlowPathCodeARM64* read_barrier_slow_path = nullptr; if (emit_rb) { // TODO: Also convert this intrinsic to the IsGcMarking strategy?
// SystemArrayCopy implementation for Baker read barriers (see // also CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier): // // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // // Slow-path copy. // do { // *dest_ptr++ = MaybePoison(ReadBarrier::Mark(MaybeUnpoison(*src_ptr++))); // } while (src_ptr != end_ptr) // } else { // // Fast-path copy. // do { // *dest_ptr++ = *src_ptr++; // } while (src_ptr != end_ptr) // }
// /* int32_t */ monitor = src->monitor_
__ Ldr(tmp, HeapOperand(src.W(), monitor_offset)); // /* LockWord */ lock_word = LockWord(monitor)
static_assert(sizeof(LockWord) == sizeof(int32_t), "art::LockWord and int32_t have different sizes.");
// Introduce a dependency on the lock_word including rb_state, // to prevent load-load reordering, and without using // a memory barrier (which would be more expensive). // `src` is unchanged by this operation, but its value now depends // on `tmp`.
__ Add(src.X(), src.X(), Operand(tmp.X(), LSR, 32));
// Slow path used to copy array when `src` is gray.
read_barrier_slow_path = new (codegen_->GetScopedAllocator()) ReadBarrierSystemArrayCopySlowPathARM64(
invoke, LocationFrom(tmp));
codegen_->AddSlowPath(read_barrier_slow_path);
}
// Compute base source address, base destination address, and end // source address for System.arraycopy* intrinsics in `src_base`, // `dst_base` and `src_end` respectively. // Note that `src_curr_addr` is computed from from `src` (and // `src_pos`) here, and thus honors the artificial dependency // of `src` on `tmp`.
GenSystemArrayCopyAddresses(masm,
type,
src,
src_pos,
dest,
dest_pos,
length,
src_curr_addr,
dst_curr_addr,
src_stop_addr);
if (emit_rb) { // Given the numeric representation, it's enough to check the low bit of the rb_state.
static_assert(ReadBarrier::NonGrayState() == 0, "Expecting non-gray to have value 0");
static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
__ Tbnz(tmp, LockWord::kReadBarrierStateShift, read_barrier_slow_path->GetEntryLabel());
}
// Iterate over the arrays and do a raw copy of the objects. We don't need to // poison/unpoison.
vixl::aarch64::Label loop;
__ Bind(&loop);
__ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex));
__ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex));
__ Cmp(src_curr_addr, src_stop_addr);
__ B(&loop, ne);
if (emit_rb) {
DCHECK(read_barrier_slow_path != nullptr);
__ Bind(read_barrier_slow_path->GetExitLabel());
}
}
// We only need one card marking on the destination array.
codegen_->MarkGCCard(dest.W());
if (is64bit) {
infinity = Operand(kPositiveInfinityDouble);
tst_mask = MaskLeastSignificant<uint64_t>(63);
out = XRegisterFrom(locations->Out());
} else {
infinity = Operand(kPositiveInfinityFloat);
tst_mask = MaskLeastSignificant<uint32_t>(31);
out = WRegisterFrom(locations->Out());
}
MoveFPToInt(locations, is64bit, masm); // Checks whether exponent bits are all 1 and fraction bits are all 0.
__ Eor(out, out, infinity); // TST bitmask is used to mask out the sign bit: either 0x7fffffff or 0x7fffffffffffffff // depending on is64bit.
__ Tst(out, tst_mask);
__ Cset(out, eq);
}
Register out = RegisterFrom(locations->Out(), DataType::Type::kReference);
UseScratchRegisterScope temps(masm); Register temp = temps.AcquireW(); auto allocate_instance = [&]() {
DCHECK(out.X().Is(InvokeRuntimeCallingConvention().GetRegisterAt(0)));
codegen_->LoadIntrinsicDeclaringClass(out, invoke);
codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke);
CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
}; if (invoke->InputAt(0)->IsIntConstant()) {
int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); if (static_cast<uint32_t>(value - info.low) < info.length) { // Just embed the object in the code.
DCHECK_NE(info.value_boot_image_reference, ValueOfInfo::kInvalidReference);
codegen_->LoadBootImageAddress(out, info.value_boot_image_reference);
} else {
DCHECK(locations->CanCall()); // Allocate and initialize a new object. // TODO: If we JIT, we could allocate the object now, and store it in the // JIT object table.
allocate_instance();
__ Mov(temp.W(), value);
codegen_->Store(type, temp.W(), HeapOperand(out.W(), info.value_offset)); // Class pointer and `value` final field stores require a barrier before publication.
codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore);
}
} else {
DCHECK(locations->CanCall()); Register in = RegisterFrom(locations->InAt(0), DataType::Type::kInt32); // Check bounds of our cache.
__ Add(out.W(), in.W(), -info.low);
__ Cmp(out.W(), info.length);
vixl::aarch64::Label allocate, done;
__ B(&allocate, hs); // If the value is within the bounds, load the object directly from the array.
codegen_->LoadBootImageAddress(temp, info.array_data_boot_image_reference);
MemOperand source = HeapOperand(
temp, out.X(), LSL, DataType::SizeShift(DataType::Type::kReference));
codegen_->Load(DataType::Type::kReference, out, source);
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(out);
__ B(&done);
__ Bind(&allocate); // Otherwise allocate and initialize a new object.
allocate_instance();
codegen_->Store(type, in.W(), HeapOperand(out.W(), info.value_offset)); // Class pointer and `value` final field stores require a barrier before publication.
codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore);
__ Bind(&done);
}
}
if (codegen_->EmitReadBarrier()) {
DCHECK(kUseBakerReadBarrier);
vixl::aarch64::Label calculate_result;
// If the GC is not marking, the comparison result is final.
__ Cbz(mr, &calculate_result);
__ B(&calculate_result, eq); // ZF set if taken.
// Check if the loaded reference is null.
__ Cbz(tmp, &calculate_result); // ZF clear if taken.
// For correct memory visibility, we need a barrier before loading the lock word.
codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
// Load the lockword and check if it is a forwarding address.
static_assert(LockWord::kStateShift == 30u);
static_assert(LockWord::kStateForwardingAddress == 3u);
__ Ldr(tmp, HeapOperand(tmp, monitor_offset));
__ Cmp(tmp, Operand(0xc0000000));
__ B(&calculate_result, lo); // ZF clear if taken.
// Extract the forwarding address and compare with `other`.
__ Cmp(other, Operand(tmp, LSL, LockWord::kForwardingAddressShift));
__ Bind(&calculate_result);
}
// Convert ZF into the Boolean result.
__ Cset(out, eq);
}
// Lower the invoke of CRC32.update(int crc, int b). void IntrinsicCodeGeneratorARM64::VisitCRC32Update(HInvoke* invoke) {
DCHECK(codegen_->GetInstructionSetFeatures().HasCRC());
MacroAssembler* masm = GetVIXLAssembler();
Register crc = InputRegisterAt(invoke, 0); Register val = InputRegisterAt(invoke, 1); Register out = OutputRegister(invoke);
// The general algorithm of the CRC32 calculation is: // crc = ~crc // result = crc32_for_byte(crc, b) // crc = ~result // It is directly lowered to three instructions.
// Generate code using CRC32 instructions which calculates // a CRC32 value of a byte. // // Parameters: // masm - VIXL macro assembler // crc - a register holding an initial CRC value // ptr - a register holding a memory address of bytes // length - a register holding a number of bytes to process // out - a register to put a result of calculation staticvoid GenerateCodeForCalculationCRC32ValueOfBytes(MacroAssembler* masm, constRegister& crc, constRegister& ptr, constRegister& length, constRegister& out) { // The algorithm of CRC32 of bytes is: // crc = ~crc // process a few first bytes to make the array 8-byte aligned // while array has 8 bytes do: // crc = crc32_of_8bytes(crc, 8_bytes(array)) // if array has 4 bytes: // crc = crc32_of_4bytes(crc, 4_bytes(array)) // if array has 2 bytes: // crc = crc32_of_2bytes(crc, 2_bytes(array)) // if array has a byte: // crc = crc32_of_byte(crc, 1_byte(array)) // crc = ~crc
// Use VIXL scratch registers as the VIXL macro assembler won't use them in // instructions below.
UseScratchRegisterScope temps(masm); Register len = temps.AcquireW(); Register array_elem = temps.AcquireW();
__ Bind(&aligned8);
__ Subs(len, len, 8); // If len < 8 go to process data by 4 bytes, 2 bytes and a byte.
__ B(&process_4bytes, lo);
// The main loop processing data by 8 bytes.
__ Bind(&loop);
__ Ldr(array_elem.X(), MemOperand(ptr, 8, PostIndex));
__ Subs(len, len, 8);
__ Crc32x(out, out, array_elem.X()); // if len >= 8, process the next 8 bytes.
__ B(&loop, hs);
// Process the data which is less than 8 bytes. // The code generated below works with values of len // which come in the range [-8, 0]. // The first three bits are used to detect whether 4 bytes or 2 bytes or // a byte can be processed. // The checking order is from bit 2 to bit 0: // bit 2 is set: at least 4 bytes available // bit 1 is set: at least 2 bytes available // bit 0 is set: at least a byte available
__ Bind(&process_4bytes); // Goto process_2bytes if less than four bytes available
__ Tbz(len, 2, &process_2bytes);
__ Ldr(array_elem, MemOperand(ptr, 4, PostIndex));
__ Crc32w(out, out, array_elem);
__ Bind(&process_2bytes); // Goto process_1bytes if less than two bytes available
__ Tbz(len, 1, &process_1byte);
__ Ldrh(array_elem, MemOperand(ptr, 2, PostIndex));
__ Crc32h(out, out, array_elem);
__ Bind(&process_1byte); // Goto done if no bytes available
__ Tbz(len, 0, &done);
__ Ldrb(array_elem, MemOperand(ptr));
__ Crc32b(out, out, array_elem);
__ Bind(&done);
__ Mvn(out, out);
}
// The threshold for sizes of arrays to use the library provided implementation // of CRC32.updateBytes instead of the intrinsic. static constexpr int32_t kCRC32UpdateBytesThreshold = 64 * 1024;
void IntrinsicLocationsBuilderARM64::VisitCRC32UpdateBytes(HInvoke* invoke) { if (!codegen_->GetInstructionSetFeatures().HasCRC()) { return;
}
// Lower the invoke of CRC32.updateBytes(int crc, byte[] b, int off, int len) // // Note: The intrinsic is not used if len exceeds a threshold. void IntrinsicCodeGeneratorARM64::VisitCRC32UpdateBytes(HInvoke* invoke) {
DCHECK(codegen_->GetInstructionSetFeatures().HasCRC());
// Lower the invoke of CRC32.updateByteBuffer(int crc, long addr, int off, int len) // // There is no need to generate code checking if addr is 0. // The method updateByteBuffer is a private method of java.util.zip.CRC32. // This guarantees no calls outside of the CRC32 class. // An address of DirectBuffer is always passed to the call of updateByteBuffer. // It might be an implementation of an empty DirectBuffer which can use a zero // address but it must have the length to be zero. The current generated code // correctly works with the zero length. void IntrinsicCodeGeneratorARM64::VisitCRC32UpdateByteBuffer(HInvoke* invoke) {
DCHECK(codegen_->GetInstructionSetFeatures().HasCRC());
void IntrinsicLocationsBuilderARM64::VisitFP16Rint(HInvoke* invoke) { if (!codegen_->GetInstructionSetFeatures().HasFP16()) { return;
}
CreateIntToIntLocations(allocator_, invoke);
}
void IntrinsicCodeGeneratorARM64::VisitFP16Rint(HInvoke* invoke) {
MacroAssembler* masm = GetVIXLAssembler(); auto roundOp = [masm](const VRegister& out, const VRegister& in) {
__ Frintn(out, in); // Round to nearest, with ties to even
};
GenerateFP16Round(invoke, codegen_, masm, roundOp);
}
void FP16ComparisonLocations(HInvoke* invoke,
ArenaAllocator* allocator_, const CodeGeneratorARM64* codegen_, int requiredTemps) { if (!codegen_->GetInstructionSetFeatures().HasFP16()) { return;
}
CreateIntIntToIntLocations(allocator_, invoke); for (int i = 0; i < requiredTemps; i++) {
invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister());
}
}
// The normal cases for this method are: // - in0 > in1 => out = 1 // - in0 < in1 => out = -1 // - in0 == in1 => out = 0 // +/-Infinity are ordered by default so are handled by the normal case. // There are two special cases that Fcmp is insufficient for distinguishing: // - in0 and in1 are +0 and -0 => +0 > -0 so compare encoding instead of value // - in0 or in1 is NaN => manually compare with in0 and in1 separately
__ Fcmp(in0, in1);
__ B(eq, &equal); // in0==in1 or +0 -0 case.
__ B(vc, &normal); // in0 and in1 are ordered (not NaN).
// Either of the inputs is NaN. // NaN is equal to itself and greater than any other number so: // - if only in0 is NaN => return 1 // - if only in1 is NaN => return -1 // - if both in0 and in1 are NaN => return 0
__ Fcmp(in0, 0.0);
__ Mov(out, -1);
__ B(vc, &end); // in0 != NaN => out = -1.
__ Fcmp(in1, 0.0);
__ Cset(out, vc); // if in1 != NaN => out = 1, otherwise both are NaNs => out = 0.
__ B(&end);
// in0 == in1 or if one of the inputs is +0 and the other is -0.
__ Bind(&equal); // Compare encoding of in0 and in1 as the denormal fraction of single precision float. // Reverse operand order because -0 > +0 when compared as S registers. // The instruction Fmov(Hregister, Wregister) zero extends the Hregister. // Therefore the value of bits[127:16] will not matter when doing the // below Fcmp as they are set to 0.
__ Fcmp(in1.S(), in0.S());
__ Bind(&normal);
__ Cset(out, gt); // if in0 > in1 => out = 1, otherwise out = 0. // Note: could be from equals path or original comparison
__ Csinv(out, out, wzr, pl); // if in0 >= in1 out=out, otherwise out=-1.
// The normal cases for this method are: // - in0.h == in1.h => out = in0 or in1 // - in0.h <cond> in1.h => out = in0 // - in0.h <!cond> in1.h => out = in1 // +/-Infinity are ordered by default so are handled by the normal case. // There are two special cases that Fcmp is insufficient for distinguishing: // - in0 and in1 are +0 and -0 => +0 > -0 so compare encoding instead of value // - in0 or in1 is NaN => out = NaN
__ Fmov(half0, in0);
__ Fmov(half1, in1);
__ Fcmp(half0, half1);
__ B(eq, &equal); // half0 = half1 or +0/-0 case.
__ Csel(out, in0, in1, cond); // if half0 <cond> half1 => out = in0, otherwise out = in1.
__ B(vc, &end); // None of the inputs were NaN.
// Atleast one input was NaN.
__ Mov(out, kFP16NaN); // out=NaN.
__ B(&end);
// in0 == in1 or if one of the inputs is +0 and the other is -0.
__ Bind(&equal); // Fcmp cannot normally distinguish +0 and -0 so compare encoding. // Encoding is compared as the denormal fraction of a Single. // Note: encoding of -0 > encoding of +0 despite +0 > -0 so in0 and in1 are swapped. // Note: The instruction Fmov(Hregister, Wregister) zero extends the Hregister.
__ Fcmp(half1.S(), half0.S());
__ Csel(out, in0, in1, cond); // if half0 <cond> half1 => out = in0, otherwise out = in1.
// Check if divisor is zero, bail to managed implementation to handle.
SlowPathCodeARM64* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
codegen->AddSlowPath(slow_path);
__ Cbz(divisor, slow_path->GetEntryLabel());
Register x = RegisterFrom(locations->InAt(0), type); Register y = RegisterFrom(locations->InAt(1), type); Register out = RegisterFrom(locations->Out(), type);
VRegister n = helpers::InputFPRegisterAt(invoke, 0);
VRegister m = helpers::InputFPRegisterAt(invoke, 1);
VRegister a = helpers::InputFPRegisterAt(invoke, 2);
VRegister out = helpers::OutputFPRegister(invoke);
vixl::aarch64::Label byte_array_view_check_label_;
vixl::aarch64::Label native_byte_order_label_; // Shared parameter for all VarHandle intrinsics.
std::memory_order order_; // Extra arguments for GenerateVarHandleCompareAndSetOrExchange(). bool return_success_; bool strong_; // Extra argument for GenerateVarHandleGetAndUpdate().
GetAndUpdateOp get_and_update_op_;
};
// Check access mode and the primitive type from VarHandle.varType. // Check reference arguments against the VarHandle.varType; for references this is a subclass // check without read barrier, so it can have false negatives which we handle in the slow path. staticvoid GenerateVarHandleAccessModeAndVarTypeChecks(HInvoke* invoke,
CodeGeneratorARM64* codegen,
SlowPathCodeARM64* slow_path,
DataType::Type type) {
mirror::VarHandle::AccessMode access_mode =
mirror::VarHandle::GetAccessModeByIntrinsic(invoke->GetIntrinsic());
Primitive::Type primitive_type = DataTypeToPrimitive(type);
// Check that the operation is permitted and the primitive type of varhandle.varType. // We do not need a read barrier when loading a reference only for loading constant // primitive field through the reference. Use LDP to load the fields together.
DCHECK_EQ(var_type_offset.Int32Value() + 4, access_mode_bit_mask_offset.Int32Value());
__ Ldp(var_type_no_rb, temp2, HeapOperand(varhandle, var_type_offset.Int32Value()));
codegen->GetAssembler()->MaybeUnpoisonHeapReference(var_type_no_rb);
__ Tbz(temp2, static_cast<uint32_t>(access_mode), slow_path->GetEntryLabel());
__ Ldrh(temp2, HeapOperand(var_type_no_rb, primitive_type_offset.Int32Value())); if (primitive_type == Primitive::kPrimNot) {
static_assert(Primitive::kPrimNot == 0);
__ Cbnz(temp2, slow_path->GetEntryLabel());
} else {
__ Cmp(temp2, static_cast<uint16_t>(primitive_type));
__ B(slow_path->GetEntryLabel(), ne);
}
temps.Release(temp2);
if (type == DataType::Type::kReference) { // Check reference arguments against the varType. // False negatives due to varType being an interface or array type // or due to the missing read barrier are handled by the slow path.
size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke);
uint32_t arguments_start = /* VarHandle object */ 1u + expected_coordinates_count;
uint32_t number_of_arguments = invoke->GetNumberOfArguments(); for (size_t arg_index = arguments_start; arg_index != number_of_arguments; ++arg_index) {
HInstruction* arg = invoke->InputAt(arg_index);
DCHECK_EQ(arg->GetType(), DataType::Type::kReference); if (!arg->IsNullConstant()) { Register arg_reg = WRegisterFrom(invoke->GetLocations()->InAt(arg_index));
GenerateSubTypeObjectCheckNoReadBarrier(codegen, slow_path, arg_reg, var_type_no_rb);
}
}
}
}
// Check that the VarHandle references a static field by checking that coordinateType0 == null. // Do not emit read barrier (or unpoison the reference) for comparing to null.
__ Ldr(temp, HeapOperand(varhandle, coordinate_type0_offset.Int32Value()));
__ Cbnz(temp, slow_path->GetEntryLabel());
}
// Check that the VarHandle references an instance field by checking that // coordinateType1 == null. coordinateType0 should not be null, but this is handled by the // type compatibility check with the source object's type, which will fail for null.
DCHECK_EQ(coordinate_type0_offset.Int32Value() + 4, coordinate_type1_offset.Int32Value());
__ Ldp(temp, temp2, HeapOperand(varhandle, coordinate_type0_offset.Int32Value()));
codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp); // No need for read barrier or unpoisoning of coordinateType1 for comparison with null.
__ Cbnz(temp2, slow_path->GetEntryLabel());
// Check that the object has the correct type. // We deliberately avoid the read barrier, letting the slow path handle the false negatives.
temps.Release(temp2); // Needed by GenerateSubTypeObjectCheckNoReadBarrier().
GenerateSubTypeObjectCheckNoReadBarrier(
codegen, slow_path, object, temp, /*object_can_be_null=*/ false);
}
}
// Check that the VarHandle references an array, byte array view or ByteBuffer by checking // that coordinateType1 != null. If that's true, coordinateType1 shall be int.class and // coordinateType0 shall not be null but we do not explicitly verify that.
DCHECK_EQ(coordinate_type0_offset.Int32Value() + 4, coordinate_type1_offset.Int32Value());
__ Ldp(temp, temp2, HeapOperand(varhandle, coordinate_type0_offset.Int32Value()));
codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp); // No need for read barrier or unpoisoning of coordinateType1 for comparison with null.
__ Cbz(temp2, slow_path->GetEntryLabel());
// Check object class against componentType0. // // This is an exact check and we defer other cases to the runtime. This includes // conversion to array of superclass references, which is valid but subsequently // requires all update operations to check that the value can indeed be stored. // We do not want to perform such extra checks in the intrinsified code. // // We do this check without read barrier, so there can be false negatives which we // defer to the slow path. There shall be no false negatives for array classes in the // boot image (including Object[] and primitive arrays) because they are non-movable.
__ Ldr(temp2, HeapOperand(object, class_offset.Int32Value()));
codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp2);
__ Cmp(temp, temp2);
__ B(slow_path->GetEntryLabel(), ne);
// Check that the coordinateType0 is an array type. We do not need a read barrier // for loading constant reference fields (or chains of them) for comparison with null, // nor for finally loading a constant primitive field (primitive type) below.
__ Ldr(temp2, HeapOperand(temp, component_type_offset.Int32Value()));
codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp2);
__ Cbz(temp2, slow_path->GetEntryLabel());
// Check that the array component type matches the primitive type.
__ Ldrh(temp2, HeapOperand(temp2, primitive_type_offset.Int32Value())); if (primitive_type == Primitive::kPrimNot) {
static_assert(Primitive::kPrimNot == 0);
__ Cbnz(temp2, slow_path->GetEntryLabel());
} else { // With the exception of `kPrimNot` (handled above), `kPrimByte` and `kPrimBoolean`, // we shall check for a byte array view in the slow path. // The check requires the ByteArrayViewVarHandle.class to be in the boot image, // so we cannot emit that if we're JITting without boot image. bool boot_image_available =
codegen->GetCompilerOptions().IsBootImage() ||
!Runtime::Current()->GetHeap()->GetBootImageSpaces().empty(); bool can_be_view = (DataType::Size(value_type) != 1u) && boot_image_available;
vixl::aarch64::Label* slow_path_label =
can_be_view ? slow_path->GetByteArrayViewCheckLabel() : slow_path->GetEntryLabel();
__ Cmp(temp2, static_cast<uint16_t>(primitive_type));
__ B(slow_path_label, ne);
}
// Check for array index out of bounds.
__ Ldr(temp, HeapOperand(object, array_length_offset.Int32Value()));
__ Cmp(index, temp);
__ B(slow_path->GetEntryLabel(), hs);
}
VarHandleTarget target; // The temporary allocated for loading the offset.
target.offset = WRegisterFrom(locations->GetTemp(0u)); // The reference to the object that holds the value to operate on.
target.object = (expected_coordinates_count == 0u)
? WRegisterFrom(locations->GetTemp(1u))
: InputRegisterAt(invoke, 1); return target;
}
if (expected_coordinates_count <= 1u) { if (VarHandleOptimizations(invoke).GetUseKnownImageVarHandle()) {
ScopedObjectAccess soa(Thread::Current());
ArtField* target_field = GetImageVarHandleField(invoke); if (expected_coordinates_count == 0u) {
ObjPtr<mirror::Class> declaring_class = target_field->GetDeclaringClass(); if (Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(declaring_class)) {
uint32_t boot_image_offset = CodeGenerator::GetBootImageOffset(declaring_class);
codegen->LoadBootImageRelRoEntry(target.object, boot_image_offset);
} else {
codegen->LoadTypeForBootImageIntrinsic(
target.object,
TypeReference(&declaring_class->GetDexFile(), declaring_class->GetDexTypeIndex()));
}
}
__ Mov(target.offset, target_field->GetOffset().Uint32Value());
} else { // For static fields, we need to fill the `target.object` with the declaring class, // so we can use `target.object` as temporary for the `ArtField*`. For instance fields, // we do not need the declaring class, so we can forget the `ArtField*` when // we load the `target.offset`, so use the `target.offset` to hold the `ArtField*`. Register field = (expected_coordinates_count == 0) ? target.object : target.offset;
ArenaAllocator* allocator = codegen->GetGraph()->GetAllocator();
LocationSummary* locations =
LocationSummary::Create(allocator, invoke, LocationSummary::kCallOnSlowPath, kIntrinsified);
locations->SetInAt(0, Location::RequiresCoreRegister()); // Require coordinates in registers. These are the object holding the value // to operate on (except for static fields) and index (for arrays and views). for (size_t i = 0; i != expected_coordinates_count; ++i) {
locations->SetInAt(/* VarHandle object */ 1u + i, Location::RequiresCoreRegister());
} if (return_type != DataType::Type::kVoid) { if (DataType::IsFloatingPointType(return_type)) {
locations->SetOut(Location::RequiresFpuRegister());
} else {
locations->SetOut(Location::RequiresCoreRegister());
}
}
uint32_t arguments_start = /* VarHandle object */ 1u + expected_coordinates_count;
uint32_t number_of_arguments = invoke->GetNumberOfArguments(); for (size_t arg_index = arguments_start; arg_index != number_of_arguments; ++arg_index) {
HInstruction* arg = invoke->InputAt(arg_index); if (IsZeroBitPattern(arg)) {
locations->SetInAt(arg_index, Location::ConstantLocation(arg));
} elseif (DataType::IsFloatingPointType(arg->GetType())) {
locations->SetInAt(arg_index, Location::RequiresFpuRegister());
} else {
locations->SetInAt(arg_index, Location::RequiresCoreRegister());
}
}
// Add a temporary for offset. if (codegen->EmitNonBakerReadBarrier() &&
GetExpectedVarHandleCoordinatesCount(invoke) == 0u) { // For static fields. // To preserve the offset value across the non-Baker read barrier slow path // for loading the declaring class, use a fixed callee-save register.
constexpr int first_callee_save = CTZ(kArm64CalleeSaveRefSpills);
locations->AddTemp(Location::CoreRegister(first_callee_save));
} else {
locations->AddTemp(Location::RequiresCoreRegister());
} if (expected_coordinates_count == 0u) { // Add a temporary to hold the declaring class.
locations->AddTemp(Location::RequiresCoreRegister());
}
if (codegen->EmitNonBakerReadBarrier() &&
invoke->GetType() == DataType::Type::kReference &&
invoke->GetIntrinsic() != Intrinsics::kVarHandleGet &&
invoke->GetIntrinsic() != Intrinsics::kVarHandleGetOpaque) { // Unsupported for non-Baker read barrier because the artReadBarrierSlow() ignores // the passed reference and reloads it from the field. This gets the memory visibility // wrong for Acquire/Volatile operations. b/173104084 return;
}
uint32_t number_of_arguments = invoke->GetNumberOfArguments();
DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1u); if (value_type == DataType::Type::kReference && codegen->EmitNonBakerReadBarrier()) { // Unsupported for non-Baker read barrier because the artReadBarrierSlow() ignores // the passed reference and reloads it from the field. This breaks the read barriers // in slow path in different ways. The marked old value may not actually be a to-space // reference to the same object as `old_value`, breaking slow path assumptions. And // for CompareAndExchange, marking the old value after comparison failure may actually // return the reference to `expected`, erroneously indicating success even though we // did not set the new value. (And it also gets the memory visibility wrong.) b/173104084 return;
}
if (codegen->EmitNonBakerReadBarrier()) { // We need callee-save registers for both the class object and offset instead of // the temporaries reserved in CreateVarHandleCommonLocations().
static_assert(POPCOUNT(kArm64CalleeSaveRefSpills) >= 2u);
uint32_t first_callee_save = CTZ(kArm64CalleeSaveRefSpills);
uint32_t second_callee_save = CTZ(kArm64CalleeSaveRefSpills ^ (1u << first_callee_save)); if (GetExpectedVarHandleCoordinatesCount(invoke) == 0u) { // For static fields.
DCHECK_EQ(locations->GetTempCount(), 2u);
DCHECK(locations->GetTemp(0u).Equals(Location::RequiresCoreRegister()));
DCHECK(locations->GetTemp(1u).Equals(Location::CoreRegister(first_callee_save)));
locations->SetTempAt(0u, Location::CoreRegister(second_callee_save));
} else {
DCHECK_EQ(locations->GetTempCount(), 1u);
DCHECK(locations->GetTemp(0u).Equals(Location::RequiresCoreRegister()));
locations->SetTempAt(0u, Location::CoreRegister(first_callee_save));
}
}
size_t old_temp_count = locations->GetTempCount();
DCHECK_EQ(old_temp_count, (GetExpectedVarHandleCoordinatesCount(invoke) == 0) ? 2u : 1u); if (!return_success) { if (DataType::IsFloatingPointType(value_type)) { // Add a temporary for old value and exclusive store result if floating point // `expected` and/or `new_value` take scratch registers.
size_t available_scratch_registers =
(IsZeroBitPattern(invoke->InputAt(number_of_arguments - 1u)) ? 1u : 0u) +
(IsZeroBitPattern(invoke->InputAt(number_of_arguments - 2u)) ? 1u : 0u);
size_t temps_needed = /* pointer, old value, store result */ 3u - available_scratch_registers; // We can reuse the declaring class (if present) and offset temporary. if (temps_needed > old_temp_count) {
locations->AddRegisterTemps(temps_needed - old_temp_count);
}
} elseif ((value_type != DataType::Type::kReference && DataType::Size(value_type) != 1u) &&
!IsZeroBitPattern(invoke->InputAt(number_of_arguments - 2u)) &&
!IsZeroBitPattern(invoke->InputAt(number_of_arguments - 1u)) &&
GetExpectedVarHandleCoordinatesCount(invoke) == 2u) { // Allocate a normal temporary for store result in the non-native byte order path // because scratch registers are used by the byte-swapped `expected` and `new_value`.
DCHECK_EQ(old_temp_count, 1u);
locations->AddTemp(Location::RequiresCoreRegister());
}
} if (value_type == DataType::Type::kReference && codegen->EmitReadBarrier()) { // Add a temporary for the `old_value_temp` in slow path.
locations->AddTemp(Location::RequiresCoreRegister());
}
}
// This needs to be before the temp registers, as MarkGCCard also uses VIXL temps. if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(new_value_index))) { // Mark card for object assuming new value is stored. bool new_value_can_be_null = true; // TODO: Worth finding out this information?
codegen->MaybeMarkGCCard(target.object, new_value.W(), new_value_can_be_null);
}
// Reuse the `offset` temporary for the pointer to the target location, // except for references that need the offset for the read barrier.
UseScratchRegisterScope temps(masm); Register tmp_ptr = target.offset.X(); if (value_type == DataType::Type::kReference && codegen->EmitReadBarrier()) {
tmp_ptr = temps.AcquireX();
}
__ Add(tmp_ptr, target.object.X(), target.offset.X());
// Move floating point values to scratch registers. // Note that float/double CAS uses bitwise comparison, rather than the operator==. Register expected_reg = MoveToTempIfFpRegister(expected, value_type, masm, &temps); Register new_value_reg = MoveToTempIfFpRegister(new_value, value_type, masm, &temps); bool is_fp = DataType::IsFloatingPointType(value_type);
DataType::Type cas_type = is_fp
? ((value_type == DataType::Type::kFloat64) ? DataType::Type::kInt64 : DataType::Type::kInt32)
: value_type; // Avoid sign extension in the CAS loop by zero-extending `expected` before the loop. This adds // one instruction for CompareAndExchange as we shall need to sign-extend the returned value. if (value_type == DataType::Type::kInt16 && !expected.IsZero()) { Register temp = temps.AcquireW();
__ Uxth(temp, expected_reg);
expected_reg = temp;
cas_type = DataType::Type::kUint16;
} elseif (value_type == DataType::Type::kInt8 && !expected.IsZero()) { Register temp = temps.AcquireW();
__ Uxtb(temp, expected_reg);
expected_reg = temp;
cas_type = DataType::Type::kUint8;
}
if (byte_swap) { // Do the byte swap and move values to scratch registers if needed. // Non-zero FP values and non-zero `expected` for `kInt16` are already in scratch registers.
DCHECK_NE(value_type, DataType::Type::kInt8); if (!expected.IsZero()) { bool is_scratch = is_fp || (value_type == DataType::Type::kInt16); Register temp = is_scratch ? expected_reg : temps.AcquireSameSizeAs(expected_reg);
GenerateReverseBytes(masm, cas_type, expected_reg, temp);
expected_reg = temp;
} if (!new_value.IsZero()) { Register temp = is_fp ? new_value_reg : temps.AcquireSameSizeAs(new_value_reg);
GenerateReverseBytes(masm, cas_type, new_value_reg, temp);
new_value_reg = temp;
}
}
// Prepare registers for old value and the result of the exclusive store. Register old_value; Register store_result; if (return_success) { // Use the output register for both old value and exclusive store result.
old_value = (cas_type == DataType::Type::kInt64) ? out.X() : out.W();
store_result = out.W();
} elseif (DataType::IsFloatingPointType(value_type)) { // We need two temporary registers but we have already used scratch registers for // holding the expected and new value unless they are zero bit pattern (+0.0f or // +0.0). We have allocated sufficient normal temporaries to handle that.
size_t next_temp = 1u; if (expected.IsZero()) {
old_value = (cas_type == DataType::Type::kInt64) ? temps.AcquireX() : temps.AcquireW();
} else {
Location temp = locations->GetTemp(next_temp);
++next_temp;
old_value = (cas_type == DataType::Type::kInt64) ? XRegisterFrom(temp) : WRegisterFrom(temp);
}
store_result =
new_value.IsZero() ? temps.AcquireW() : WRegisterFrom(locations->GetTemp(next_temp));
DCHECK(!old_value.Is(tmp_ptr));
DCHECK(!store_result.Is(tmp_ptr));
} else { // Use the output register for the old value.
old_value = (cas_type == DataType::Type::kInt64) ? out.X() : out.W(); // Use scratch register for the store result, except when we have used up // scratch registers for byte-swapped `expected` and `new_value`. // In that case, we have allocated a normal temporary.
store_result = (byte_swap && !expected.IsZero() && !new_value.IsZero())
? WRegisterFrom(locations->GetTemp(1))
: temps.AcquireW();
DCHECK(!store_result.Is(tmp_ptr));
}
bool use_lse = codegen->ShouldUseLSE(); if (value_type == DataType::Type::kReference && codegen->EmitReadBarrier()) { // The `old_value_temp` is used first for the marked `old_value` and then for the unmarked // reloaded old value for subsequent CAS in the slow path. It cannot be a scratch register.
size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); Register old_value_temp =
WRegisterFrom(locations->GetTemp((expected_coordinates_count == 0u) ? 2u : 1u)); // For strong CAS, use a scratch register for the store result in slow path. // For weak CAS, we need to check the store result, so store it in `store_result`. Register slow_path_store_result = strong ? Register() : store_result;
ReadBarrierCasSlowPathARM64* rb_slow_path = new (codegen->GetScopedAllocator()) ReadBarrierCasSlowPathARM64(
invoke,
order,
strong,
target.object,
target.offset.X(),
expected_reg,
new_value_reg,
old_value,
old_value_temp,
slow_path_store_result, /*update_old_value=*/ !return_success,
codegen);
codegen->AddSlowPath(rb_slow_path);
exit_loop = rb_slow_path->GetExitLabel();
cmp_failure = rb_slow_path->GetEntryLabel();
} elseif (use_lse) {
cmp_failure = nullptr;
}
if (return_success) { if (strong) {
__ Cset(out.W(), eq);
} elseif (use_lse) { // The result from `GenerateCompareAndSet()` is already final with LSE for the common case. // The slow path must ensure it finalizes the result if it's taken.
DCHECK(store_result.Is(out));
} else { // On success, the Z flag is set and the store result is 1, see GenerateCompareAndSet(). // On failure, either the Z flag is clear or the store result is 0. // Determine the final success value with a CSEL.
__ Csel(out.W(), store_result, wzr, eq);
}
} elseif (byte_swap) { // Also handles moving to FP registers.
GenerateReverseBytes(masm, value_type, old_value, out);
} elseif (DataType::IsFloatingPointType(value_type)) {
__ Fmov((value_type == DataType::Type::kFloat64) ? out.D() : out.S(), old_value);
} elseif (value_type == DataType::Type::kInt8) {
__ Sxtb(out.W(), old_value);
} elseif (value_type == DataType::Type::kInt16) {
__ Sxth(out.W(), old_value);
}
if (slow_path != nullptr) {
DCHECK(!byte_swap);
__ Bind(slow_path->GetExitLabel());
}
}
// Get the type from the shorty as the invokes may not return a value.
uint32_t arg_index = invoke->GetNumberOfArguments() - 1;
DataType::Type value_type = GetDataTypeFromShorty(invoke, arg_index); if (value_type == DataType::Type::kReference && codegen->EmitNonBakerReadBarrier()) { // Unsupported for non-Baker read barrier because the artReadBarrierSlow() ignores // the passed reference and reloads it from the field, thus seeing the new value // that we have just stored. (And it also gets the memory visibility wrong.) b/173104084 return;
}
DCHECK_EQ(old_temp_count, (GetExpectedVarHandleCoordinatesCount(invoke) == 0) ? 2u : 1u); if (DataType::IsFloatingPointType(value_type)) { if (get_and_update_op == GetAndUpdateOp::kAdd) { // For ADD, do not use ZR for zero bit pattern (+0.0f or +0.0).
locations->SetInAt(invoke->GetNumberOfArguments() - 1u, Location::RequiresFpuRegister());
} else {
DCHECK(get_and_update_op == GetAndUpdateOp::kSet); // We can reuse the declaring class temporary if present. if (old_temp_count == 1u &&
!IsZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { // Add a temporary for `old_value` if floating point `new_value` takes a scratch register.
locations->AddTemp(Location::RequiresCoreRegister());
}
}
} // We need a temporary for the byte-swap path for bitwise operations unless the argument is a // zero which does not need a byte-swap. We can reuse the declaring class temporary if present. if (old_temp_count == 1u &&
(get_and_update_op != GetAndUpdateOp::kSet && get_and_update_op != GetAndUpdateOp::kAdd) &&
GetExpectedVarHandleCoordinatesCount(invoke) == 2u &&
!IsZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { if (value_type != DataType::Type::kReference && DataType::Size(value_type) != 1u) {
locations->AddTemp(Location::RequiresCoreRegister());
}
}
// Request another temporary register for methods that don't return a value. // For the non-void case, we already set `out` in `CreateVarHandleCommonLocations`.
DataType::Type return_type = invoke->GetType(); constbool is_void = return_type == DataType::Type::kVoid;
DCHECK_IMPLIES(!is_void, return_type == value_type); if (is_void) { if (DataType::IsFloatingPointType(value_type)) {
locations->AddTemp(Location::RequiresFpuRegister());
} else {
locations->AddTemp(Location::RequiresCoreRegister());
}
}
}
staticvoid GenerateVarHandleGetAndUpdate(HInvoke* invoke,
CodeGeneratorARM64* codegen,
GetAndUpdateOp get_and_update_op,
std::memory_order order, bool byte_swap = false) { // Get the type from the shorty as the invokes may not return a value.
uint32_t arg_index = invoke->GetNumberOfArguments() - 1;
DataType::Type value_type = GetDataTypeFromShorty(invoke, arg_index); bool is_fp = DataType::IsFloatingPointType(value_type);
MacroAssembler* masm = codegen->GetVIXLAssembler();
LocationSummary* locations = invoke->GetLocations();
CPURegister arg = (is_fp && get_and_update_op == GetAndUpdateOp::kAdd)
? InputCPURegisterAt(invoke, arg_index)
: InputCPURegisterOrZeroRegAt(invoke, arg_index);
DataType::Type return_type = invoke->GetType(); constbool is_void = return_type == DataType::Type::kVoid;
DCHECK_IMPLIES(!is_void, return_type == value_type); // We use a temporary for void methods, as we don't return the value.
CPURegister out_or_temp =
is_void ? CPURegisterFrom(locations->GetTemp(locations->GetTempCount() - 1u), value_type) :
helpers::OutputCPURegister(invoke);
// This needs to be before the temp registers, as MarkGCCard also uses VIXL temps. if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(arg_index))) {
DCHECK(get_and_update_op == GetAndUpdateOp::kSet); // Mark card for object, the new value shall be stored. bool new_value_can_be_null = true; // TODO: Worth finding out this information?
codegen->MaybeMarkGCCard(target.object, arg.W(), new_value_can_be_null);
}
// Reuse the `target.offset` temporary for the pointer to the target location, // except for references that need the offset for the non-Baker read barrier.
UseScratchRegisterScope temps(masm); Register tmp_ptr = target.offset.X(); if (value_type == DataType::Type::kReference && codegen->EmitNonBakerReadBarrier()) {
tmp_ptr = temps.AcquireX();
}
__ Add(tmp_ptr, target.object.X(), target.offset.X());
// The load/store type is never floating point.
DataType::Type load_store_type = is_fp
? ((value_type == DataType::Type::kFloat32) ? DataType::Type::kInt32 : DataType::Type::kInt64)
: value_type; // Avoid sign extension in the CAS loop. Sign-extend after the loop. // Note: Using unsigned values yields the same value to store (we do not store higher bits). if (value_type == DataType::Type::kInt8) {
load_store_type = DataType::Type::kUint8;
} elseif (value_type == DataType::Type::kInt16) {
load_store_type = DataType::Type::kUint16;
}
// Prepare register for old value.
CPURegister old_value = out_or_temp; if (get_and_update_op == GetAndUpdateOp::kSet) { // For floating point GetAndSet, do the GenerateGetAndUpdate() with core registers, // rather than moving between core and FP registers in the loop.
arg = MoveToTempIfFpRegister(arg, value_type, masm, &temps); if (is_fp) { // `old_value` needs to be a core register for `GenerateGetAndUpdate`. If the argument // is zero bit pattern (+0.0f or +0.0), we can use a scratch register. Otherwise it's used // by the argument, and we should use a temporary that has been allocated for nonzero case.
old_value = arg.IsZero()
? (old_value.IsD() ? temps.AcquireX() : temps.AcquireW())
: CPURegisterFrom(locations->GetTemp(1u), load_store_type);
} elseif (value_type == DataType::Type::kReference && codegen->EmitBakerReadBarrier()) { // Load the old value initially to a scratch register. // We shall move it to `out` later with a read barrier.
old_value = temps.AcquireW();
}
}
if (byte_swap) {
DCHECK_NE(value_type, DataType::Type::kReference);
DCHECK_NE(DataType::Size(value_type), 1u); if (get_and_update_op == GetAndUpdateOp::kAdd) { // We need to do the byte swapping in the CAS loop for GetAndAdd.
get_and_update_op = GetAndUpdateOp::kAddWithByteSwap;
} elseif (!arg.IsZero()) { // For other operations, avoid byte swap inside the CAS loop by providing an adjusted `arg`. // For GetAndSet use a scratch register; FP argument is already in a scratch register. // For bitwise operations GenerateGetAndUpdate() needs both scratch registers; // we have allocated a normal temporary to handle that.
CPURegister temp = (get_and_update_op == GetAndUpdateOp::kSet)
? (is_fp ? arg : (arg.Is64Bits() ? temps.AcquireX() : temps.AcquireW()))
: CPURegisterFrom(locations->GetTemp(1u), load_store_type);
GenerateReverseBytes(masm, load_store_type, arg, temp);
arg = temp;
}
}
// The main path checked that the coordinateType0 is an array class that matches // the class of the actual coordinate argument but it does not match the value type. // Check if the `varhandle` references a ByteArrayViewVarHandle instance.
__ Ldr(temp, HeapOperand(varhandle, class_offset.Int32Value()));
codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp);
codegen->LoadClassRootForIntrinsic(temp2, ClassRoot::kJavaLangInvokeByteArrayViewVarHandle);
__ Cmp(temp, temp2);
__ B(GetEntryLabel(), ne);
// Check for array index out of bounds.
__ Ldr(temp, HeapOperand(object, array_length_offset.Int32Value()));
__ Subs(temp, temp, index);
__ Ccmp(temp, size, NoFlag, hs); // If SUBS yields LO (C=false), keep the C flag clear.
__ B(GetEntryLabel(), lo);
// Construct the target.
__ Add(target.offset, index, data_offset.Int32Value());
// Alignment check. For unaligned access, go to the runtime.
DCHECK(IsPowerOfTwo(size)); if (size == 2u) {
__ Tbnz(target.offset, 0, GetEntryLabel());
} else {
__ Tst(target.offset, size - 1u);
__ B(GetEntryLabel(), ne);
}
// Byte order check. For native byte order return to the main path. if (access_mode_template == mirror::VarHandle::AccessModeTemplate::kSet &&
IsZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { // There is no reason to differentiate between native byte order and byte-swap // for setting a zero bit pattern. Just return to the main path.
__ B(GetNativeByteOrderLabel()); return;
}
__ Ldrb(temp, HeapOperand(varhandle, native_byte_order_offset.Int32Value()));
__ Cbnz(temp, GetNativeByteOrderLabel());
}
switch (access_mode_template) { case mirror::VarHandle::AccessModeTemplate::kGet:
GenerateVarHandleGet(invoke, codegen, order_, /*byte_swap=*/ true); break; case mirror::VarHandle::AccessModeTemplate::kSet:
GenerateVarHandleSet(invoke, codegen, order_, /*byte_swap=*/ true); break; case mirror::VarHandle::AccessModeTemplate::kCompareAndSet: case mirror::VarHandle::AccessModeTemplate::kCompareAndExchange:
GenerateVarHandleCompareAndSetOrExchange(
invoke, codegen, order_, return_success_, strong_, /*byte_swap=*/ true); break; case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate:
GenerateVarHandleGetAndUpdate(
invoke, codegen, get_and_update_op_, order_, /*byte_swap=*/ true); break;
}
__ B(GetExitLabel());
}
for (uint32_t i = 1; i < number_of_args; ++i) {
locations->SetInAt(i, calling_convention.GetNextLocation(invoke->InputAt(i)->GetType()));
}
// Passing MethodHandle object as the last parameter: accessors implementation rely on it.
DCHECK_EQ(invoke->InputAt(0)->GetType(), DataType::Type::kReference);
Location receiver_mh_loc = calling_convention.GetNextLocation(DataType::Type::kReference);
locations->SetInAt(0, receiver_mh_loc);
if (invoke->AsInvokePolymorphic()->NeedsCallSiteTypeCheck()) { // The last input is MethodType object corresponding to the call-site.
locations->SetInAt(number_of_args, Location::RequiresCoreRegister());
}
if (invoke->AsInvokePolymorphic()->NeedsCallSiteTypeCheck()) { Register call_site_type = InputRegisterAt(invoke, invoke->GetNumberOfArguments());
// Call site should match with MethodHandle's type.
__ Ldr(temp, HeapOperand(method_handle.W(), mirror::MethodHandle::MethodTypeOffset()));
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp);
__ Cmp(call_site_type, temp);
__ B(ne, slow_path->GetEntryLabel());
}
if (invoke->AsInvokePolymorphic()->CanTargetInstanceMethod()) { Register receiver = InputRegisterAt(invoke, 1);
// Receiver shouldn't be null for all the following cases.
__ Cbz(receiver, slow_path->GetEntryLabel());
__ Cmp(method_handle_kind, Operand(mirror::MethodHandle::Kind::kInvokeDirect)); // No dispatch is needed for invoke-direct.
__ B(eq, &execute_target_method);
// Skip virtual dispatch if `method` is private.
__ Ldr(temp, MemOperand(method, ArtMethod::AccessFlagsOffset().Int32Value()));
__ And(temp, temp, Operand(kAccPrivate));
__ Cbnz(temp, &execute_target_method);
Register receiver_class = WRegisterFrom(locations->GetTemp(3)); // If method is defined in the receiver's class, execute it as it is.
__ Ldr(temp, MemOperand(method, ArtMethod::DeclaringClassOffset().Int32Value()));
__ Ldr(receiver_class, HeapOperand(receiver.W(), mirror::Object::ClassOffset().Int32Value()));
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(receiver_class.W()); // `receiver_class` is read w/o read barriers: false negatives go through virtual dispatch.
__ Cmp(temp, receiver_class);
__ B(eq, &execute_target_method);
// MethodIndex is uint16_t.
__ Ldrh(temp, MemOperand(method, ArtMethod::MethodIndexOffset().Int32Value()));
// Re-using receiver class register to store vtable.
constexpr uint32_t vtable_offset =
mirror::Class::EmbeddedVTableOffset(art::PointerSize::k64).Int32Value();
__ Add(receiver_class.X(), receiver_class.X(), vtable_offset);
__ Ldr(method, MemOperand(receiver_class.X(), temp, Extend::UXTW, 3u));
__ B(&execute_target_method);
// Skip virtual dispatch if `method` is private. // Re-using method_handle_kind to store access flags. Register access_flags = WRegisterFrom(locations->GetTemp(4));
__ Ldr(access_flags, MemOperand(method, ArtMethod::AccessFlagsOffset().Int32Value()));
__ And(temp, access_flags, Operand(kAccPrivate));
__ Cbnz(temp, &execute_target_method);
// The register x15 is used as the hidden argument for `art_quick_imt_conflict_trampoline`, // so make sure VIXL is not using it as a scratch register.
DCHECK(!UseScratchRegisterScope(GetVIXLAssembler()).IsAvailable(x15));
// Get IMT index. // Not doing default conflict check as IMT index is set for all method which have // kAccAbstract bit.
__ And(temp, access_flags, Operand(kAccAbstract));
__ Cbz(temp, &get_imt_index_from_method_index);
// imt_index is uint16_t
__ Ldrh(temp, MemOperand(method, ArtMethod::ImtIndexOffset().Int32Value()));
__ B(&do_imt_dispatch);
__ Bind(&do_imt_dispatch); // Re-using `method` to store receiver class and ImTableEntry.
__ Ldr(method.W(), HeapOperand(receiver.W(), mirror::Object::ClassOffset()));
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(method.W());
¤ Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.0.151Bemerkung:
(Wie Sie bei der Firma Beratungs- und Dienstleistungen beauftragen können 2026-06-29)
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
