From 45a0105de3ec46347495d764888fd8cd93635727 Mon Sep 17 00:00:00 2001 From: Liu Yu Date: Thu, 15 Dec 2022 15:00:55 +0800 Subject: [PATCH] [SpiderMonkey] Support loongarch64. --- build/autoconf/config.guess | 3 + build/autoconf/config.sub | 1 + build/moz.configure/init.configure | 3 + config/check_macroassembler_style.py | 4 +- .../icu/source/i18n/double-conversion-utils.h | 2 +- js/moz.configure | 12 +- js/src/builtin/TestingFunctions.cpp | 18 + js/src/ctypes/libffi/config.guess | 3 + js/src/ctypes/libffi/config.sub | 2 + .../imported/regexp-macro-assembler.h | 1 + js/src/jit-test/tests/ion/bug1433496.js | 2 +- .../wasm/simd/wormhole-gating-baseline.js | 2 +- .../tests/wasm/simd/wormhole-gating-ion.js | 2 +- js/src/jit/AtomicOperations.h | 3 +- js/src/jit/BaselineBailouts.cpp | 7 +- js/src/jit/BaselineCodeGen.cpp | 2 + js/src/jit/CodeGenerator.cpp | 3 +- js/src/jit/CodeGenerator.h | 2 + js/src/jit/FlushICache.h | 5 +- js/src/jit/GenerateOpcodeFiles.py | 1 + js/src/jit/JitFrames.cpp | 9 + js/src/jit/JitOptions.cpp | 3 +- js/src/jit/LIR.h | 2 + js/src/jit/Label.h | 4 +- js/src/jit/Lowering.h | 2 + js/src/jit/MacroAssembler-inl.h | 9 + js/src/jit/MacroAssembler.cpp | 12 +- js/src/jit/MacroAssembler.h | 272 +- js/src/jit/MoveEmitter.h | 2 + js/src/jit/RegisterAllocator.h | 2 +- js/src/jit/RegisterSets.h | 2 + js/src/jit/Registers.h | 2 + js/src/jit/SharedICHelpers-inl.h | 2 + js/src/jit/SharedICHelpers.h | 2 + js/src/jit/SharedICRegisters.h | 2 + js/src/jit/Simulator.h | 2 + js/src/jit/loong64/Architecture-loong64.cpp | 83 + js/src/jit/loong64/Architecture-loong64.h | 520 ++ js/src/jit/loong64/Assembler-loong64.cpp | 2479 +++++++ js/src/jit/loong64/Assembler-loong64.h | 1956 ++++++ js/src/jit/loong64/AtomicOperations-loong64.h | 196 + js/src/jit/loong64/Bailouts-loong64.cpp | 68 + js/src/jit/loong64/CodeGenerator-loong64.cpp | 2839 ++++++++ js/src/jit/loong64/CodeGenerator-loong64.h | 198 + js/src/jit/loong64/LIR-loong64.h | 399 ++ js/src/jit/loong64/Lowering-loong64.cpp | 1056 +++ js/src/jit/loong64/Lowering-loong64.h | 120 + .../jit/loong64/MacroAssembler-loong64-inl.h | 1904 ++++++ js/src/jit/loong64/MacroAssembler-loong64.cpp | 5335 +++++++++++++++ js/src/jit/loong64/MacroAssembler-loong64.h | 1025 +++ js/src/jit/loong64/MoveEmitter-loong64.cpp | 327 + js/src/jit/loong64/MoveEmitter-loong64.h | 76 + .../jit/loong64/SharedICHelpers-loong64-inl.h | 111 + js/src/jit/loong64/SharedICHelpers-loong64.h | 104 + .../jit/loong64/SharedICRegisters-loong64.h | 48 + js/src/jit/loong64/Simulator-loong64.cpp | 6045 +++++++++++++++++ js/src/jit/loong64/Simulator-loong64.h | 651 ++ js/src/jit/loong64/Trampoline-loong64.cpp | 1274 ++++ js/src/jit/moz.build | 14 + js/src/jit/shared/Assembler-shared.h | 7 +- .../AtomicOperations-feeling-lucky-gcc.h | 7 +- js/src/jit/shared/CodeGenerator-shared.cpp | 6 +- js/src/jit/shared/CodeGenerator-shared.h | 1 + .../IonAssemblerBufferWithConstantPools.h | 5 +- js/src/jit/shared/Lowering-shared-inl.h | 3 +- js/src/jsapi-tests/testJitABIcalls.cpp | 3 + js/src/jsapi-tests/testsJit.cpp | 4 +- js/src/old-configure.in | 4 +- js/src/shell/js.cpp | 24 + js/src/tests/non262/Array/regress-157652.js | 2 +- js/src/tests/non262/Array/regress-330812.js | 2 +- js/src/tests/non262/regress/regress-422348.js | 2 +- js/src/util/Poison.h | 2 + js/src/vm/ArrayBufferObject.h | 2 +- js/src/wasm/WasmBaselineCompile.cpp | 122 +- js/src/wasm/WasmCompile.cpp | 4 + js/src/wasm/WasmFrameIter.cpp | 64 +- js/src/wasm/WasmGC.cpp | 3 + js/src/wasm/WasmIonCompile.cpp | 7 +- js/src/wasm/WasmSignalHandlers.cpp | 41 +- js/src/wasm/WasmStubs.cpp | 15 +- js/xpconnect/src/XPCJSContext.cpp | 2 +- memory/build/mozjemalloc.cpp | 5 +- .../double-conversion/utils.h | 2 +- mfbt/tests/TestPoisonArea.cpp | 3 + modules/libpref/init/StaticPrefList.yaml | 4 +- .../mozbuild/mozbuild/configure/constants.py | 2 + 87 files changed, 27369 insertions(+), 214 deletions(-) create mode 100644 js/src/jit/loong64/Architecture-loong64.cpp create mode 100644 js/src/jit/loong64/Architecture-loong64.h create mode 100644 js/src/jit/loong64/Assembler-loong64.cpp create mode 100644 js/src/jit/loong64/Assembler-loong64.h create mode 100644 js/src/jit/loong64/AtomicOperations-loong64.h create mode 100644 js/src/jit/loong64/Bailouts-loong64.cpp create mode 100644 js/src/jit/loong64/CodeGenerator-loong64.cpp create mode 100644 js/src/jit/loong64/CodeGenerator-loong64.h create mode 100644 js/src/jit/loong64/LIR-loong64.h create mode 100644 js/src/jit/loong64/Lowering-loong64.cpp create mode 100644 js/src/jit/loong64/Lowering-loong64.h create mode 100644 js/src/jit/loong64/MacroAssembler-loong64-inl.h create mode 100644 js/src/jit/loong64/MacroAssembler-loong64.cpp create mode 100644 js/src/jit/loong64/MacroAssembler-loong64.h create mode 100644 js/src/jit/loong64/MoveEmitter-loong64.cpp create mode 100644 js/src/jit/loong64/MoveEmitter-loong64.h create mode 100644 js/src/jit/loong64/SharedICHelpers-loong64-inl.h create mode 100644 js/src/jit/loong64/SharedICHelpers-loong64.h create mode 100644 js/src/jit/loong64/SharedICRegisters-loong64.h create mode 100644 js/src/jit/loong64/Simulator-loong64.cpp create mode 100644 js/src/jit/loong64/Simulator-loong64.h create mode 100644 js/src/jit/loong64/Trampoline-loong64.cpp diff --git a/build/autoconf/config.guess b/build/autoconf/config.guess index e94095c5fb..9148e5b3da 100755 --- a/build/autoconf/config.guess +++ b/build/autoconf/config.guess @@ -985,6 +985,9 @@ EOF k1om:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; + loongarch64:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; m32r*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; diff --git a/build/autoconf/config.sub b/build/autoconf/config.sub index 3d9a8dc3d5..3b86ae23c5 100755 --- a/build/autoconf/config.sub +++ b/build/autoconf/config.sub @@ -1185,6 +1185,7 @@ case $cpu-$vendor in | k1om \ | le32 | le64 \ | lm32 \ + | loongarch32 | loongarch64 | loongarchx32 \ | m32c | m32r | m32rle \ | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ diff --git a/build/moz.configure/init.configure b/build/moz.configure/init.configure index 3a164c6558..96c3a22f9c 100644 --- a/build/moz.configure/init.configure +++ b/build/moz.configure/init.configure @@ -762,6 +762,9 @@ def split_triplet(triplet, allow_msvc=False, allow_wasi=False): elif cpu in ("mips64", "mips64el"): canonical_cpu = "mips64" endianness = "little" if "el" in cpu else "big" + elif cpu == "loongarch64": + canonical_cpu = "loongarch64" + endianness = "little" elif cpu.startswith("aarch64"): canonical_cpu = "aarch64" endianness = "little" diff --git a/config/check_macroassembler_style.py b/config/check_macroassembler_style.py index 0d040a939b..259343bdeb 100644 --- a/config/check_macroassembler_style.py +++ b/config/check_macroassembler_style.py @@ -30,8 +30,8 @@ import sys architecture_independent = set(["generic"]) all_unsupported_architectures_names = set(["mips32", "mips64", "mips_shared"]) -all_architecture_names = set(["x86", "x64", "arm", "arm64"]) -all_shared_architecture_names = set(["x86_shared", "arm", "arm64"]) +all_architecture_names = set(["x86", "x64", "arm", "arm64", "loong64"]) +all_shared_architecture_names = set(["x86_shared", "arm", "arm64", "loong64"]) reBeforeArg = "(?<=[(,\s])" reArgType = "(?P[\w\s:*&]+)" diff --git a/intl/icu/source/i18n/double-conversion-utils.h b/intl/icu/source/i18n/double-conversion-utils.h index c937463647..ae3a2d9fdc 100644 --- a/intl/icu/source/i18n/double-conversion-utils.h +++ b/intl/icu/source/i18n/double-conversion-utils.h @@ -117,7 +117,7 @@ int main(int argc, char** argv) { #if defined(_M_X64) || defined(__x86_64__) || \ defined(__ARMEL__) || defined(__avr32__) || defined(_M_ARM) || defined(_M_ARM64) || \ defined(__hppa__) || defined(__ia64__) || \ - defined(__mips__) || \ + defined(__mips__) || defined(__loongarch64) || \ defined(__nios2__) || defined(__ghs) || \ defined(__powerpc__) || defined(__ppc__) || defined(__ppc64__) || \ defined(_POWER) || defined(_ARCH_PPC) || defined(_ARCH_PPC64) || \ diff --git a/js/moz.configure b/js/moz.configure index 3c3d0d4359..4cfdee8742 100644 --- a/js/moz.configure +++ b/js/moz.configure @@ -142,7 +142,7 @@ set_define("NO_RUST_PANIC_HOOK", True, when="NO_RUST_PANIC_HOOK") # ======================================================= @depends(target) def jit_default(target): - if target.cpu in ("x86", "x86_64", "arm", "aarch64", "mips32", "mips64"): + if target.cpu in ("x86", "x86_64", "arm", "aarch64", "mips32", "mips64", "loongarch64"): return True return False @@ -162,7 +162,7 @@ def report_deprecated(value): # ======================================================= option( "--enable-simulator", - choices=("arm", "arm64", "mips32", "mips64"), + choices=("arm", "arm64", "mips32", "mips64", "loong64"), nargs=1, help="Enable a JIT code simulator for the specified architecture", ) @@ -179,7 +179,7 @@ def simulator(jit_enabled, simulator_enabled, target): if target.cpu != "x86": die("The %s simulator only works on x86." % sim_cpu) - if sim_cpu in ("arm64", "mips64"): + if sim_cpu in ("arm64", "mips64", "loong64"): if target.cpu != "x86_64": die("The %s simulator only works on x86-64." % sim_cpu) @@ -191,11 +191,13 @@ set_config("JS_SIMULATOR_ARM", simulator.arm) set_config("JS_SIMULATOR_ARM64", simulator.arm64) set_config("JS_SIMULATOR_MIPS32", simulator.mips32) set_config("JS_SIMULATOR_MIPS64", simulator.mips64) +set_config("JS_SIMULATOR_LOONG64", simulator.loong64) set_define("JS_SIMULATOR", depends_if(simulator)(lambda x: True)) set_define("JS_SIMULATOR_ARM", simulator.arm) set_define("JS_SIMULATOR_ARM64", simulator.arm64) set_define("JS_SIMULATOR_MIPS32", simulator.mips32) set_define("JS_SIMULATOR_MIPS64", simulator.mips64) +set_define("JS_SIMULATOR_LOONG64", simulator.loong64) @depends("--enable-jit", simulator, target) @@ -208,6 +210,8 @@ def jit_codegen(jit_enabled, simulator, target): if target.cpu == "aarch64": return namespace(arm64=True) + elif target.cpu == "loongarch64": + return namespace(loong64=True) elif target.cpu == "x86_64": return namespace(x64=True) @@ -219,6 +223,7 @@ set_config("JS_CODEGEN_ARM", jit_codegen.arm) set_config("JS_CODEGEN_ARM64", jit_codegen.arm64) set_config("JS_CODEGEN_MIPS32", jit_codegen.mips32) set_config("JS_CODEGEN_MIPS64", jit_codegen.mips64) +set_config("JS_CODEGEN_LOONG64", jit_codegen.loong64) set_config("JS_CODEGEN_X86", jit_codegen.x86) set_config("JS_CODEGEN_X64", jit_codegen.x64) set_define("JS_CODEGEN_NONE", jit_codegen.none) @@ -226,6 +231,7 @@ set_define("JS_CODEGEN_ARM", jit_codegen.arm) set_define("JS_CODEGEN_ARM64", jit_codegen.arm64) set_define("JS_CODEGEN_MIPS32", jit_codegen.mips32) set_define("JS_CODEGEN_MIPS64", jit_codegen.mips64) +set_define("JS_CODEGEN_LOONG64", jit_codegen.loong64) set_define("JS_CODEGEN_X86", jit_codegen.x86) set_define("JS_CODEGEN_X64", jit_codegen.x64) diff --git a/js/src/builtin/TestingFunctions.cpp b/js/src/builtin/TestingFunctions.cpp index 006824eb15..ad0c247438 100644 --- a/js/src/builtin/TestingFunctions.cpp +++ b/js/src/builtin/TestingFunctions.cpp @@ -417,6 +417,24 @@ static bool GetBuildConfiguration(JSContext* cx, unsigned argc, Value* vp) { return false; } +#ifdef JS_CODEGEN_LOONG64 + value = BooleanValue(true); +#else + value = BooleanValue(false); +#endif + if (!JS_SetProperty(cx, info, "loong64", value)) { + return false; + } + +#ifdef JS_SIMULATOR_LOONG64 + value = BooleanValue(true); +#else + value = BooleanValue(false); +#endif + if (!JS_SetProperty(cx, info, "loong64-simulator", value)) { + return false; + } + #ifdef MOZ_ASAN value = BooleanValue(true); #else diff --git a/js/src/ctypes/libffi/config.guess b/js/src/ctypes/libffi/config.guess index faa63aa942..c8913e5d70 100644 --- a/js/src/ctypes/libffi/config.guess +++ b/js/src/ctypes/libffi/config.guess @@ -976,6 +976,9 @@ EOF k1om:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; + loongarch64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; m32r*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; diff --git a/js/src/ctypes/libffi/config.sub b/js/src/ctypes/libffi/config.sub index 40ea5dfe11..50e2b3fb2e 100644 --- a/js/src/ctypes/libffi/config.sub +++ b/js/src/ctypes/libffi/config.sub @@ -268,6 +268,7 @@ case $basic_machine in | k1om \ | le32 | le64 \ | lm32 \ + | loongarch64 \ | m32c | m32r | m32rle | m68000 | m68k | m88k \ | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ | mips | mipsbe | mipseb | mipsel | mipsle \ @@ -394,6 +395,7 @@ case $basic_machine in | k1om-* \ | le32-* | le64-* \ | lm32-* \ + | loongarch64-* \ | m32c-* | m32r-* | m32rle-* \ | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ diff --git a/js/src/irregexp/imported/regexp-macro-assembler.h b/js/src/irregexp/imported/regexp-macro-assembler.h index 2f11d1e7ea..f183f88d9a 100644 --- a/js/src/irregexp/imported/regexp-macro-assembler.h +++ b/js/src/irregexp/imported/regexp-macro-assembler.h @@ -45,6 +45,7 @@ class RegExpMacroAssembler { V(ARM64) \ V(MIPS) \ V(RISCV) \ + V(LOONG) \ V(S390) \ V(PPC) \ V(X64) \ diff --git a/js/src/jit-test/tests/ion/bug1433496.js b/js/src/jit-test/tests/ion/bug1433496.js index e5ef04c8e0..cb16eb97a0 100644 --- a/js/src/jit-test/tests/ion/bug1433496.js +++ b/js/src/jit-test/tests/ion/bug1433496.js @@ -1,4 +1,4 @@ -// |jit-test| --spectre-mitigations=on; skip-if: getBuildConfiguration()['mips32'] || getBuildConfiguration()['mips64'] +// |jit-test| --spectre-mitigations=on; skip-if: getBuildConfiguration()['mips64-simulator'] function f() { return arguments[arguments.length]; } diff --git a/js/src/jit-test/tests/wasm/simd/wormhole-gating-baseline.js b/js/src/jit-test/tests/wasm/simd/wormhole-gating-baseline.js index 43c91d2b99..d8a4c863b1 100644 --- a/js/src/jit-test/tests/wasm/simd/wormhole-gating-baseline.js +++ b/js/src/jit-test/tests/wasm/simd/wormhole-gating-baseline.js @@ -11,7 +11,7 @@ function nativeIntel() { var conf = getBuildConfiguration(); if (!(conf.x64 || conf.x86)) return false; - if (conf.arm64 || conf.arm || conf.mips32 || conf.mips64) + if (conf.arm64 || conf.arm || conf.mips32 || conf.mips64 || conf.loong64) return false; return true; } diff --git a/js/src/jit-test/tests/wasm/simd/wormhole-gating-ion.js b/js/src/jit-test/tests/wasm/simd/wormhole-gating-ion.js index 5e37d2f933..eae213d79a 100644 --- a/js/src/jit-test/tests/wasm/simd/wormhole-gating-ion.js +++ b/js/src/jit-test/tests/wasm/simd/wormhole-gating-ion.js @@ -11,7 +11,7 @@ function nativeIntel() { var conf = getBuildConfiguration(); if (!(conf.x64 || conf.x86)) return false; - if (conf.arm64 || conf.arm || conf.mips32 || conf.mips64) + if (conf.arm64 || conf.arm || conf.mips32 || conf.mips64 || conf.loong64) return false; return true; } diff --git a/js/src/jit/AtomicOperations.h b/js/src/jit/AtomicOperations.h index f4a5727d05..c0d24597f5 100644 --- a/js/src/jit/AtomicOperations.h +++ b/js/src/jit/AtomicOperations.h @@ -382,7 +382,8 @@ constexpr inline bool AtomicOperations::isLockfreeJS(int32_t size) { defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ defined(__PPC64LE__) || defined(__alpha__) || defined(__hppa__) || \ defined(__sh__) || defined(__s390__) || defined(__s390x__) || \ - defined(__m68k__) || defined(__riscv) || defined(__wasi__) + defined(__m68k__) || defined(__riscv) || defined(__wasi__) || \ + defined(__loongarch64) # include "jit/shared/AtomicOperations-feeling-lucky.h" #else # error "No AtomicOperations support provided for this platform" diff --git a/js/src/jit/BaselineBailouts.cpp b/js/src/jit/BaselineBailouts.cpp index bca1427f93..d809844c5d 100644 --- a/js/src/jit/BaselineBailouts.cpp +++ b/js/src/jit/BaselineBailouts.cpp @@ -20,6 +20,7 @@ #include "jit/JitFrames.h" #include "jit/JitRuntime.h" #include "jit/JitSpewer.h" +#include "jit/loong64/Simulator-loong64.h" #include "jit/mips32/Simulator-mips32.h" #include "jit/mips64/Simulator-mips64.h" #include "jit/Recover.h" @@ -461,9 +462,9 @@ class MOZ_STACK_CLASS BaselineStackBuilder { return virtualPointerAtStackOffset(priorOffset); #elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ - defined(JS_CODEGEN_X64) - // On X64, ARM, ARM64, and MIPS, the frame pointer save location depends on - // the caller of the rectifier frame. + defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_LOONG64) + // On X64, ARM, ARM64, MIPS and LoongArch, the frame pointer save location + // depends on the caller of the rectifier frame. BufferPointer priorFrame = pointerAtStackOffset(priorOffset); FrameType priorType = priorFrame->prevType(); diff --git a/js/src/jit/BaselineCodeGen.cpp b/js/src/jit/BaselineCodeGen.cpp index 7089f5e300..98c4b8747c 100644 --- a/js/src/jit/BaselineCodeGen.cpp +++ b/js/src/jit/BaselineCodeGen.cpp @@ -525,6 +525,8 @@ bool BaselineCodeGen::emitOutOfLinePostBarrierSlot() { masm.push(lr); #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) masm.push(ra); +#elif defined(JS_CODEGEN_LOONG64) + masm.push(ra); #endif masm.pushValue(R0); diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index 795505dc36..f0cab12b97 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -10967,7 +10967,8 @@ void CodeGenerator::visitOutOfLineStoreElementHole( // condition flags sticking from the incoming branch. // Also note: this branch does not need Spectre mitigations, doing that for // the capacity check below is sufficient. -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) // Had to reimplement for MIPS because there are no flags. Address initLength(elements, ObjectElements::offsetOfInitializedLength()); masm.branch32(Assembler::NotEqual, initLength, indexReg, ool->callStub()); diff --git a/js/src/jit/CodeGenerator.h b/js/src/jit/CodeGenerator.h index 5321978fc2..e1d81beb64 100644 --- a/js/src/jit/CodeGenerator.h +++ b/js/src/jit/CodeGenerator.h @@ -25,6 +25,8 @@ # include "jit/mips32/CodeGenerator-mips32.h" #elif defined(JS_CODEGEN_MIPS64) # include "jit/mips64/CodeGenerator-mips64.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/CodeGenerator-loong64.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/CodeGenerator-none.h" #else diff --git a/js/src/jit/FlushICache.h b/js/src/jit/FlushICache.h index fe66080df5..42b1fb045c 100644 --- a/js/src/jit/FlushICache.h +++ b/js/src/jit/FlushICache.h @@ -23,8 +23,9 @@ inline void FlushICache(void* code, size_t size, // No-op. Code and data caches are coherent on x86 and x64. } -#elif (defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)) || \ - (defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)) +#elif (defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)) || \ + (defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)) || \ + defined(JS_CODEGEN_LOONG64) extern void FlushICache(void* code, size_t size, bool codeIsThreadLocal = true); diff --git a/js/src/jit/GenerateOpcodeFiles.py b/js/src/jit/GenerateOpcodeFiles.py index 41e7a67a1a..b84c9f0b0b 100644 --- a/js/src/jit/GenerateOpcodeFiles.py +++ b/js/src/jit/GenerateOpcodeFiles.py @@ -33,6 +33,7 @@ def get_opcodes(inputs, pat): match = pat.match(line) if match: op = match.group("name") + print("opcode:", op) if op in ops_set: raise Exception("Duplicate opcode {} in {}".format(op, inputfile)) ops.append(op) diff --git a/js/src/jit/JitFrames.cpp b/js/src/jit/JitFrames.cpp index 879a2b64fd..29b1ffe6ca 100644 --- a/js/src/jit/JitFrames.cpp +++ b/js/src/jit/JitFrames.cpp @@ -2227,6 +2227,15 @@ MachineState MachineState::FromBailout(RegisterDump::GPRArray& regs, &fpregs[i]); // No SIMD support in bailouts, SIMD is internal to wasm } +#elif defined(JS_CODEGEN_LOONG64) + for (unsigned i = 0; i < FloatRegisters::TotalPhys; i++) { + machine.setRegisterLocation( + FloatRegister(FloatRegisters::Encoding(i), FloatRegisters::Single), + &fpregs[i]); + machine.setRegisterLocation( + FloatRegister(FloatRegisters::Encoding(i), FloatRegisters::Double), + &fpregs[i]); + } #elif defined(JS_CODEGEN_NONE) MOZ_CRASH(); diff --git a/js/src/jit/JitOptions.cpp b/js/src/jit/JitOptions.cpp index de13777fc3..e28cc88e12 100644 --- a/js/src/jit/JitOptions.cpp +++ b/js/src/jit/JitOptions.cpp @@ -246,7 +246,8 @@ DefaultJitOptions::DefaultJitOptions() { } } -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) SET_DEFAULT(spectreIndexMasking, false); SET_DEFAULT(spectreObjectMitigations, false); SET_DEFAULT(spectreStringMitigations, false); diff --git a/js/src/jit/LIR.h b/js/src/jit/LIR.h index 024bd798ca..c164181318 100644 --- a/js/src/jit/LIR.h +++ b/js/src/jit/LIR.h @@ -1937,6 +1937,8 @@ AnyRegister LAllocation::toRegister() const { # include "jit/arm/LIR-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/LIR-arm64.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/LIR-loong64.h" #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) # if defined(JS_CODEGEN_MIPS32) # include "jit/mips32/LIR-mips32.h" diff --git a/js/src/jit/Label.h b/js/src/jit/Label.h index a8f93de378..cd7f4290ef 100644 --- a/js/src/jit/Label.h +++ b/js/src/jit/Label.h @@ -26,7 +26,9 @@ struct LabelBase { void operator=(const LabelBase& label) = delete; -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +// TODO:LOONG64 Recheck INVALID_OFFSET. +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) public: #endif static const uint32_t INVALID_OFFSET = 0x7fffffff; // UINT31_MAX. diff --git a/js/src/jit/Lowering.h b/js/src/jit/Lowering.h index 979687da85..4558f4f3cf 100644 --- a/js/src/jit/Lowering.h +++ b/js/src/jit/Lowering.h @@ -23,6 +23,8 @@ # include "jit/mips32/Lowering-mips32.h" #elif defined(JS_CODEGEN_MIPS64) # include "jit/mips64/Lowering-mips64.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/Lowering-loong64.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/Lowering-none.h" #else diff --git a/js/src/jit/MacroAssembler-inl.h b/js/src/jit/MacroAssembler-inl.h index cf16cdf0a7..d7ad948cf5 100644 --- a/js/src/jit/MacroAssembler-inl.h +++ b/js/src/jit/MacroAssembler-inl.h @@ -35,6 +35,8 @@ # include "jit/mips32/MacroAssembler-mips32-inl.h" #elif defined(JS_CODEGEN_MIPS64) # include "jit/mips64/MacroAssembler-mips64-inl.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/MacroAssembler-loong64-inl.h" #elif !defined(JS_CODEGEN_NONE) # error "Unknown architecture!" #endif @@ -1003,6 +1005,13 @@ void MacroAssembler::assertCanonicalInt32(Register r) { branchPtr(Assembler::BelowOrEqual, r, ImmWord(UINT32_MAX), &ok); breakpoint(); bind(&ok); +# elif defined(JS_CODEGEN_LOONG64) + Label ok; + ScratchRegisterScope scratch(asMasm()); + move32SignExtendToPtr(r, scratch); + branchPtr(Assembler::Equal, r, scratch, &ok); + breakpoint(); + bind(&ok); # else MOZ_CRASH("IMPLEMENT ME"); # endif diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp index 2a3aeec607..7c4c4a309e 100644 --- a/js/src/jit/MacroAssembler.cpp +++ b/js/src/jit/MacroAssembler.cpp @@ -2391,8 +2391,9 @@ void MacroAssembler::outOfLineTruncateSlow(FloatRegister src, Register dest, } int32_t framePushedAfterTls = framePushed(); -#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) ScratchDoubleScope fpscratch(*this); if (widenFloatToDouble) { convertFloat32ToDouble(src, fpscratch); @@ -2429,8 +2430,9 @@ void MacroAssembler::outOfLineTruncateSlow(FloatRegister src, Register dest, } storeCallInt32Result(dest); -#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) // Nothing #elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) if (widenFloatToDouble) { @@ -4049,6 +4051,8 @@ void MacroAssembler::emitPreBarrierFastPath(JSRuntime* rt, MIRType type, ma_sll(temp1, temp1, temp3); #elif JS_CODEGEN_MIPS64 ma_dsll(temp1, temp1, temp3); +#elif JS_CODEGEN_LOONG64 + as_sll_d(temp1, temp1, temp3); #elif JS_CODEGEN_NONE MOZ_CRASH(); #else diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h index e2d53d5cef..17c96dae60 100644 --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -25,6 +25,8 @@ # include "jit/mips32/MacroAssembler-mips32.h" #elif defined(JS_CODEGEN_MIPS64) # include "jit/mips64/MacroAssembler-mips64.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/MacroAssembler-loong64.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/MacroAssembler-none.h" #else @@ -92,8 +94,8 @@ // } // ////}}} check_macroassembler_style -#define ALL_ARCH mips32, mips64, arm, arm64, x86, x64 -#define ALL_SHARED_ARCH arm, arm64, x86_shared, mips_shared +#define ALL_ARCH mips32, mips64, arm, arm64, x86, x64, loong64 +#define ALL_SHARED_ARCH arm, arm64, loong64, x86_shared, mips_shared // * How this macro works: // @@ -139,6 +141,7 @@ #define DEFINED_ON_mips32 #define DEFINED_ON_mips64 #define DEFINED_ON_mips_shared +#define DEFINED_ON_loong64 #define DEFINED_ON_none // Specialize for each architecture. @@ -168,6 +171,9 @@ # define DEFINED_ON_mips64 define # undef DEFINED_ON_mips_shared # define DEFINED_ON_mips_shared define +#elif defined(JS_CODEGEN_LOONG64) +# undef DEFINED_ON_loong64 +# define DEFINED_ON_loong64 define #elif defined(JS_CODEGEN_NONE) # undef DEFINED_ON_none # define DEFINED_ON_none crash @@ -291,7 +297,8 @@ struct AllocSiteInput // accordance with platform-specific rules, as follows. // // - On x64 and arm64, the upper bits are zero -// - On mips64 the upper bits are the sign extension of the lower bits +// - On mips64 and loongarch64 the upper bits are the sign extension of the +// lower bits // - (On risc-v we have no rule, having no port yet. Sign extension is the most // likely rule, but "unpredictable" is an option.) // @@ -304,8 +311,8 @@ struct AllocSiteInput // Additionally, the inputs to many 32-bit operations must be properly // sign-extended to avoid "unpredictable" behavior, and our simulators check // that inputs conform. -// - (32-bit operations on risc-v sign-extend, much as mips, but appear to -// ignore the upper bits of the inputs.) +// - (32-bit operations on risc-v and loongarch64 sign-extend, much as mips, but +// appear to ignore the upper bits of the inputs.) // // The upshot of these invariants is, among other things, that: // @@ -484,10 +491,10 @@ class MacroAssembler : public MacroAssemblerSpecific { // The size of the area used by PushRegsInMask. size_t PushRegsInMaskSizeInBytes(LiveRegisterSet set) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); void PushRegsInMask(LiveRegisterSet set) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); void PushRegsInMask(LiveGeneralRegisterSet set); // Like PushRegsInMask, but instead of pushing the registers, store them to @@ -498,12 +505,12 @@ class MacroAssembler : public MacroAssemblerSpecific { // must point to either the lowest address in the save area, or some address // below that. void storeRegsInMask(LiveRegisterSet set, Address dest, Register scratch) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); void PopRegsInMask(LiveRegisterSet set); void PopRegsInMask(LiveGeneralRegisterSet set); void PopRegsInMaskIgnore(LiveRegisterSet set, LiveRegisterSet ignore) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); // =============================================================== // Stack manipulation functions -- single registers/values. @@ -536,7 +543,7 @@ class MacroAssembler : public MacroAssemblerSpecific { void Pop(FloatRegister t) PER_SHARED_ARCH; void Pop(const ValueOperand& val) PER_SHARED_ARCH; void PopFlags() DEFINED_ON(x86_shared); - void PopStackPtr() DEFINED_ON(arm, mips_shared, x86_shared); + void PopStackPtr() DEFINED_ON(arm, mips_shared, x86_shared, loong64); void popRooted(VMFunctionData::RootType rootType, Register cellReg, const ValueOperand& valueReg); @@ -594,8 +601,8 @@ class MacroAssembler : public MacroAssemblerSpecific { void callAndPushReturnAddress(Label* label) DEFINED_ON(x86_shared); // These do not adjust framePushed(). - void pushReturnAddress() DEFINED_ON(mips_shared, arm, arm64); - void popReturnAddress() DEFINED_ON(mips_shared, arm, arm64); + void pushReturnAddress() DEFINED_ON(mips_shared, arm, arm64, loong64); + void popReturnAddress() DEFINED_ON(mips_shared, arm, arm64, loong64); // Useful for dealing with two-valued returns. void moveRegPair(Register src0, Register src1, Register dst0, Register dst1, @@ -626,10 +633,10 @@ class MacroAssembler : public MacroAssemblerSpecific { // Note: "Near" applies to ARM64 where the target must be within 1 MB (this is // release-asserted). CodeOffset moveNearAddressWithPatch(Register dest) - DEFINED_ON(x86, x64, arm, arm64, mips_shared); + DEFINED_ON(x86, x64, arm, arm64, loong64, mips_shared); static void patchNearAddressMove(CodeLocationLabel loc, CodeLocationLabel target) - DEFINED_ON(x86, x64, arm, arm64, mips_shared); + DEFINED_ON(x86, x64, arm, arm64, loong64, mips_shared); public: // =============================================================== @@ -988,10 +995,11 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void xorPtr(Imm32 imm, Register dest) PER_ARCH; inline void and64(const Operand& src, Register64 dest) - DEFINED_ON(x64, mips64); - inline void or64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64); + DEFINED_ON(x64, mips64, loong64); + inline void or64(const Operand& src, Register64 dest) + DEFINED_ON(x64, mips64, loong64); inline void xor64(const Operand& src, Register64 dest) - DEFINED_ON(x64, mips64); + DEFINED_ON(x64, mips64, loong64); // =============================================================== // Swap instructions @@ -1025,17 +1033,17 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void addPtr(ImmWord imm, Register dest) PER_ARCH; inline void addPtr(ImmPtr imm, Register dest); inline void addPtr(Imm32 imm, const Address& dest) - DEFINED_ON(mips_shared, arm, arm64, x86, x64); + DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64); inline void addPtr(Imm32 imm, const AbsoluteAddress& dest) DEFINED_ON(x86, x64); inline void addPtr(const Address& src, Register dest) - DEFINED_ON(mips_shared, arm, arm64, x86, x64); + DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64); inline void add64(Register64 src, Register64 dest) PER_ARCH; inline void add64(Imm32 imm, Register64 dest) PER_ARCH; inline void add64(Imm64 imm, Register64 dest) PER_ARCH; inline void add64(const Operand& src, Register64 dest) - DEFINED_ON(x64, mips64); + DEFINED_ON(x64, mips64, loong64); inline void addFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; @@ -1054,16 +1062,16 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void subPtr(Register src, Register dest) PER_ARCH; inline void subPtr(Register src, const Address& dest) - DEFINED_ON(mips_shared, arm, arm64, x86, x64); + DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64); inline void subPtr(Imm32 imm, Register dest) PER_ARCH; inline void subPtr(ImmWord imm, Register dest) DEFINED_ON(x64); inline void subPtr(const Address& addr, Register dest) - DEFINED_ON(mips_shared, arm, arm64, x86, x64); + DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64); inline void sub64(Register64 src, Register64 dest) PER_ARCH; inline void sub64(Imm64 imm, Register64 dest) PER_ARCH; inline void sub64(const Operand& src, Register64 dest) - DEFINED_ON(x64, mips64); + DEFINED_ON(x64, mips64, loong64); inline void subFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; @@ -1078,10 +1086,10 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void mul64(const Operand& src, const Register64& dest) DEFINED_ON(x64); inline void mul64(const Operand& src, const Register64& dest, - const Register temp) DEFINED_ON(x64, mips64); + const Register temp) DEFINED_ON(x64, mips64, loong64); inline void mul64(Imm64 imm, const Register64& dest) PER_ARCH; inline void mul64(Imm64 imm, const Register64& dest, const Register temp) - DEFINED_ON(x86, x64, arm, mips32, mips64); + DEFINED_ON(x86, x64, arm, mips32, mips64, loong64); inline void mul64(const Register64& src, const Register64& dest, const Register temp) PER_ARCH; inline void mul64(const Register64& src1, const Register64& src2, @@ -1095,7 +1103,7 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void mulDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; inline void mulDoublePtr(ImmPtr imm, Register temp, FloatRegister dest) - DEFINED_ON(mips_shared, arm, arm64, x86, x64); + DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64); // Perform an integer division, returning the integer part rounded toward // zero. rhs must not be zero, and the division must not overflow. @@ -1122,7 +1130,7 @@ class MacroAssembler : public MacroAssemblerSpecific { // rhs is preserved, srdDest is clobbered. void flexibleRemainder32(Register rhs, Register srcDest, bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) - DEFINED_ON(mips_shared, arm, arm64, x86_shared); + DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64); // Perform an integer division, returning the integer part rounded toward // zero. rhs must not be zero, and the division must not overflow. @@ -1133,7 +1141,7 @@ class MacroAssembler : public MacroAssemblerSpecific { // rhs is preserved, srdDest is clobbered. void flexibleQuotient32(Register rhs, Register srcDest, bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) - DEFINED_ON(mips_shared, arm, arm64, x86_shared); + DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64); // Perform an integer division, returning the integer part rounded toward // zero. rhs must not be zero, and the division must not overflow. The @@ -1146,7 +1154,7 @@ class MacroAssembler : public MacroAssemblerSpecific { void flexibleDivMod32(Register rhs, Register srcDest, Register remOutput, bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) - DEFINED_ON(mips_shared, arm, arm64, x86_shared); + DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64); inline void divFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; inline void divDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; @@ -1347,7 +1355,7 @@ class MacroAssembler : public MacroAssemblerSpecific { template inline void cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64); template inline void cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) PER_ARCH; @@ -1372,10 +1380,10 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void branch32(Condition cond, const AbsoluteAddress& lhs, Register rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void branch32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void branch32(Condition cond, const BaseIndex& lhs, Register rhs, Label* label) DEFINED_ON(arm, x86_shared); @@ -1389,7 +1397,7 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void branch32(Condition cond, wasm::SymbolicAddress lhs, Imm32 rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); // The supported condition are Equal, NotEqual, LessThan(orEqual), // GreaterThan(orEqual), Below(orEqual) and Above(orEqual). When a fail label @@ -1438,14 +1446,14 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, Register rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, ImmWord rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void branchPtr(Condition cond, wasm::SymbolicAddress lhs, Register rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); // Given a pointer to a GC Cell, retrieve the StoreBuffer pointer from its // chunk header, or nullptr if it is in the tenured heap. @@ -1453,7 +1461,7 @@ class MacroAssembler : public MacroAssemblerSpecific { void branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); void branchPtrInNurseryChunk(Condition cond, const Address& address, Register temp, Label* label) DEFINED_ON(x86); void branchValueIsNurseryCell(Condition cond, const Address& address, @@ -1475,10 +1483,10 @@ class MacroAssembler : public MacroAssemblerSpecific { // x64 variants will do this only in the int64_t range. inline void branchTruncateFloat32MaybeModUint32(FloatRegister src, Register dest, Label* fail) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void branchTruncateDoubleMaybeModUint32(FloatRegister src, Register dest, Label* fail) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); // Truncate a double/float32 to intptr and when it doesn't fit jump to the // failure label. @@ -1491,10 +1499,10 @@ class MacroAssembler : public MacroAssemblerSpecific { // failure label. inline void branchTruncateFloat32ToInt32(FloatRegister src, Register dest, Label* fail) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void branchTruncateDoubleToInt32(FloatRegister src, Register dest, Label* fail) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void branchDouble(DoubleCondition cond, FloatRegister lhs, FloatRegister rhs, Label* label) PER_SHARED_ARCH; @@ -1548,7 +1556,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTest32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); template inline void branchTestPtr(Condition cond, Register lhs, Register rhs, @@ -1694,7 +1702,7 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void branchTestInt32(Condition cond, Register tag, Label* label) PER_SHARED_ARCH; inline void branchTestDouble(Condition cond, Register tag, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestNumber(Condition cond, Register tag, Label* label) PER_SHARED_ARCH; inline void branchTestBoolean(Condition cond, Register tag, @@ -1726,7 +1734,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestUndefined(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestInt32(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1734,7 +1742,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestInt32(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestDouble(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1742,11 +1750,11 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestDouble(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestNumber(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestBoolean(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1754,7 +1762,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestBoolean(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestString(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1762,7 +1770,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestString(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestSymbol(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1770,7 +1778,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestSymbol(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestBigInt(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1778,7 +1786,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestBigInt(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestNull(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1786,7 +1794,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestNull(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); // Clobbers the ScratchReg on x64. inline void branchTestObject(Condition cond, const Address& address, @@ -1795,7 +1803,7 @@ class MacroAssembler : public MacroAssemblerSpecific { Label* label) PER_SHARED_ARCH; inline void branchTestObject(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestGCThing(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1806,7 +1814,7 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void branchTestPrimitive(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestMagic(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; @@ -1815,7 +1823,7 @@ class MacroAssembler : public MacroAssemblerSpecific { template inline void branchTestMagic(Condition cond, const ValueOperand& value, L label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) PER_ARCH; @@ -1833,17 +1841,17 @@ class MacroAssembler : public MacroAssemblerSpecific { // The type of the value should match the type of the method. inline void branchTestInt32Truthy(bool truthy, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestDoubleTruthy(bool truthy, FloatRegister reg, Label* label) PER_SHARED_ARCH; inline void branchTestBooleanTruthy(bool truthy, const ValueOperand& value, Label* label) PER_ARCH; inline void branchTestStringTruthy(bool truthy, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); inline void branchTestBigIntTruthy(bool truthy, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); // Create an unconditional branch to the address given as argument. inline void branchToComputedAddress(const BaseIndex& address) PER_ARCH; @@ -1858,7 +1866,7 @@ class MacroAssembler : public MacroAssemblerSpecific { template void branchValueIsNurseryCellImpl(Condition cond, const T& value, Register temp, Label* label) - DEFINED_ON(arm64, x64, mips64); + DEFINED_ON(arm64, x64, mips64, loong64); template inline void branchTestUndefinedImpl(Condition cond, const T& t, Label* label) @@ -1928,11 +1936,11 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void cmp32Move32(Condition cond, Register lhs, Register rhs, Register src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86_shared); + DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared); inline void cmp32Move32(Condition cond, Register lhs, const Address& rhs, Register src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86_shared); + DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared); inline void cmpPtrMovePtr(Condition cond, Register lhs, Register rhs, Register src, Register dest) PER_ARCH; @@ -1942,36 +1950,36 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void cmp32Load32(Condition cond, Register lhs, const Address& rhs, const Address& src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86_shared); + DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared); inline void cmp32Load32(Condition cond, Register lhs, Register rhs, const Address& src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86_shared); + DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared); inline void cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs, const Address& src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, loong64, mips_shared, x86, x64); inline void cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs, Register src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, loong64, mips_shared, x86, x64); inline void test32LoadPtr(Condition cond, const Address& addr, Imm32 mask, const Address& src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, loong64, mips_shared, x86, x64); inline void test32MovePtr(Condition cond, const Address& addr, Imm32 mask, Register src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, loong64, mips_shared, x86, x64); // Conditional move for Spectre mitigations. inline void spectreMovePtr(Condition cond, Register src, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); // Zeroes dest if the condition is true. inline void spectreZeroRegister(Condition cond, Register scratch, Register dest) - DEFINED_ON(arm, arm64, mips_shared, x86_shared); + DEFINED_ON(arm, arm64, mips_shared, x86_shared, loong64); // Performs a bounds check and zeroes the index register if out-of-bounds // (to mitigate Spectre). @@ -1983,17 +1991,17 @@ class MacroAssembler : public MacroAssemblerSpecific { public: inline void spectreBoundsCheck32(Register index, Register length, Register maybeScratch, Label* failure) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void spectreBoundsCheck32(Register index, const Address& length, Register maybeScratch, Label* failure) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void spectreBoundsCheckPtr(Register index, Register length, Register maybeScratch, Label* failure) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); inline void spectreBoundsCheckPtr(Register index, const Address& length, Register maybeScratch, Label* failure) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64); // ======================================================================== // Canonicalization primitives. @@ -2007,10 +2015,10 @@ class MacroAssembler : public MacroAssemblerSpecific { // ======================================================================== // Memory access primitives. inline void storeUncanonicalizedDouble(FloatRegister src, const Address& dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64); inline void storeUncanonicalizedDouble(FloatRegister src, const BaseIndex& dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64); inline void storeUncanonicalizedDouble(FloatRegister src, const Operand& dest) DEFINED_ON(x86_shared); @@ -2024,10 +2032,10 @@ class MacroAssembler : public MacroAssemblerSpecific { inline void storeUncanonicalizedFloat32(FloatRegister src, const Address& dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64); inline void storeUncanonicalizedFloat32(FloatRegister src, const BaseIndex& dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64); inline void storeUncanonicalizedFloat32(FloatRegister src, const Operand& dest) DEFINED_ON(x86_shared); @@ -3473,12 +3481,13 @@ class MacroAssembler : public MacroAssemblerSpecific { // ======================================================================== // Convert floating point. - // temp required on x86 and x64; must be undefined on mips64. + // TODO:LOONG64 Recheck convert + // temp required on x86 and x64; must be undefined on mips64 and loong64. void convertUInt64ToFloat32(Register64 src, FloatRegister dest, Register temp) - DEFINED_ON(arm64, mips64, x64, x86); + DEFINED_ON(arm64, mips64, loong64, x64, x86); void convertInt64ToFloat32(Register64 src, FloatRegister dest) - DEFINED_ON(arm64, mips64, x64, x86); + DEFINED_ON(arm64, mips64, loong64, x64, x86); bool convertUInt64ToDoubleNeedsTemp() PER_ARCH; @@ -3519,19 +3528,19 @@ class MacroAssembler : public MacroAssemblerSpecific { void wasmBoundsCheck32(Condition cond, Register index, Register boundsCheckLimit, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); void wasmBoundsCheck32(Condition cond, Register index, Address boundsCheckLimit, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared); void wasmBoundsCheck64(Condition cond, Register64 index, Register64 boundsCheckLimit, Label* label) - DEFINED_ON(arm64, mips64, x64); + DEFINED_ON(arm64, mips64, x64, loong64); void wasmBoundsCheck64(Condition cond, Register64 index, Address boundsCheckLimit, Label* label) - DEFINED_ON(arm64, mips64, x64); + DEFINED_ON(arm64, mips64, x64, loong64); // Each wasm load/store instruction appends its own wasm::Trap::OutOfBounds. void wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr, @@ -3543,7 +3552,7 @@ class MacroAssembler : public MacroAssemblerSpecific { void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Operand dstAddr) DEFINED_ON(x86); - // For all the ARM/MIPS wasmLoad and wasmStore functions below, `ptr` + // For all the ARM/MIPS/LOONG64 wasmLoad and wasmStore functions below, `ptr` // MUST equal `ptrScratch`, and that register will be updated based on // conditions listed below (where it is only mentioned as `ptr`). @@ -3551,16 +3560,16 @@ class MacroAssembler : public MacroAssemblerSpecific { // Scalar::Int64. void wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr, Register ptrScratch, AnyRegister output) - DEFINED_ON(arm, mips_shared); + DEFINED_ON(arm, loong64, mips_shared); void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr, Register ptrScratch, Register64 output) - DEFINED_ON(arm, mips32, mips64); + DEFINED_ON(arm, mips32, mips64, loong64); void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Register memoryBase, Register ptr, Register ptrScratch) - DEFINED_ON(arm, mips_shared); + DEFINED_ON(arm, loong64, mips_shared); void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase, Register ptr, Register ptrScratch) - DEFINED_ON(arm, mips32, mips64); + DEFINED_ON(arm, mips32, mips64, loong64); // These accept general memoryBase + ptr + offset (in `access`); the offset is // always smaller than the guard region. They will insert an additional add @@ -3625,7 +3634,7 @@ class MacroAssembler : public MacroAssemblerSpecific { void oolWasmTruncateCheckF64ToI32(FloatRegister input, Register output, TruncFlags flags, wasm::BytecodeOffset off, Label* rejoin) - DEFINED_ON(arm, arm64, x86_shared, mips_shared); + DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64); void wasmTruncateFloat32ToUInt32(FloatRegister input, Register output, bool isSaturating, Label* oolEntry) PER_ARCH; @@ -3635,35 +3644,35 @@ class MacroAssembler : public MacroAssemblerSpecific { void oolWasmTruncateCheckF32ToI32(FloatRegister input, Register output, TruncFlags flags, wasm::BytecodeOffset off, Label* rejoin) - DEFINED_ON(arm, arm64, x86_shared, mips_shared); + DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64); // The truncate-to-int64 methods will always bind the `oolRejoin` label // after the last emitted instruction. void wasmTruncateDoubleToInt64(FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) - DEFINED_ON(arm64, x86, x64, mips64); + DEFINED_ON(arm64, x86, x64, mips64, loong64); void wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) - DEFINED_ON(arm64, x86, x64, mips64); + DEFINED_ON(arm64, x86, x64, mips64, loong64); void oolWasmTruncateCheckF64ToI64(FloatRegister input, Register64 output, TruncFlags flags, wasm::BytecodeOffset off, Label* rejoin) - DEFINED_ON(arm, arm64, x86_shared, mips_shared); + DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64); void wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) - DEFINED_ON(arm64, x86, x64, mips64); + DEFINED_ON(arm64, x86, x64, mips64, loong64); void wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) - DEFINED_ON(arm64, x86, x64, mips64); + DEFINED_ON(arm64, x86, x64, mips64, loong64); void oolWasmTruncateCheckF32ToI64(FloatRegister input, Register64 output, TruncFlags flags, wasm::BytecodeOffset off, Label* rejoin) - DEFINED_ON(arm, arm64, x86_shared, mips_shared); + DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64); // This function takes care of loading the callee's TLS and pinned regs but // it is the caller's responsibility to save/restore TLS or pinned regs. @@ -3740,13 +3749,13 @@ class MacroAssembler : public MacroAssemblerSpecific { const Address& mem, Register expected, Register replacement, Register valueTemp, Register offsetTemp, Register maskTemp, Register output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); void compareExchange(Scalar::Type type, const Synchronization& sync, const BaseIndex& mem, Register expected, Register replacement, Register valueTemp, Register offsetTemp, Register maskTemp, Register output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); // x86: `expected` and `output` must be edx:eax; `replacement` is ecx:ebx. // x64: `output` must be rax. @@ -3756,12 +3765,12 @@ class MacroAssembler : public MacroAssemblerSpecific { void compareExchange64(const Synchronization& sync, const Address& mem, Register64 expected, Register64 replacement, Register64 output) - DEFINED_ON(arm, arm64, x64, x86, mips64); + DEFINED_ON(arm, arm64, x64, x86, mips64, loong64); void compareExchange64(const Synchronization& sync, const BaseIndex& mem, Register64 expected, Register64 replacement, Register64 output) - DEFINED_ON(arm, arm64, x64, x86, mips64); + DEFINED_ON(arm, arm64, x64, x86, mips64, loong64); // Exchange with memory. Return the value initially in memory. // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit @@ -3778,12 +3787,12 @@ class MacroAssembler : public MacroAssemblerSpecific { void atomicExchange(Scalar::Type type, const Synchronization& sync, const Address& mem, Register value, Register valueTemp, Register offsetTemp, Register maskTemp, Register output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); void atomicExchange(Scalar::Type type, const Synchronization& sync, const BaseIndex& mem, Register value, Register valueTemp, Register offsetTemp, Register maskTemp, Register output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); // x86: `value` must be ecx:ebx; `output` must be edx:eax. // ARM: `value` and `output` must be distinct and (even,odd) pairs. @@ -3791,11 +3800,11 @@ class MacroAssembler : public MacroAssemblerSpecific { void atomicExchange64(const Synchronization& sync, const Address& mem, Register64 value, Register64 output) - DEFINED_ON(arm, arm64, x64, x86, mips64); + DEFINED_ON(arm, arm64, x64, x86, mips64, loong64); void atomicExchange64(const Synchronization& sync, const BaseIndex& mem, Register64 value, Register64 output) - DEFINED_ON(arm, arm64, x64, x86, mips64); + DEFINED_ON(arm, arm64, x64, x86, mips64, loong64); // Read-modify-write with memory. Return the value in memory before the // operation. @@ -3831,12 +3840,12 @@ class MacroAssembler : public MacroAssemblerSpecific { void atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, Register value, const Address& mem, Register valueTemp, Register offsetTemp, Register maskTemp, - Register output) DEFINED_ON(mips_shared); + Register output) DEFINED_ON(mips_shared, loong64); void atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, Register value, const BaseIndex& mem, Register valueTemp, Register offsetTemp, Register maskTemp, - Register output) DEFINED_ON(mips_shared); + Register output) DEFINED_ON(mips_shared, loong64); // x86: // `temp` must be ecx:ebx; `output` must be edx:eax. @@ -3850,7 +3859,8 @@ class MacroAssembler : public MacroAssemblerSpecific { void atomicFetchOp64(const Synchronization& sync, AtomicOp op, Register64 value, const Address& mem, Register64 temp, - Register64 output) DEFINED_ON(arm, arm64, x64, mips64); + Register64 output) + DEFINED_ON(arm, arm64, x64, mips64, loong64); void atomicFetchOp64(const Synchronization& sync, AtomicOp op, const Address& value, const Address& mem, @@ -3858,7 +3868,8 @@ class MacroAssembler : public MacroAssemblerSpecific { void atomicFetchOp64(const Synchronization& sync, AtomicOp op, Register64 value, const BaseIndex& mem, Register64 temp, - Register64 output) DEFINED_ON(arm, arm64, x64, mips64); + Register64 output) + DEFINED_ON(arm, arm64, x64, mips64, loong64); void atomicFetchOp64(const Synchronization& sync, AtomicOp op, const Address& value, const BaseIndex& mem, @@ -3876,14 +3887,14 @@ class MacroAssembler : public MacroAssemblerSpecific { void atomicEffectOp64(const Synchronization& sync, AtomicOp op, Register64 value, const Address& mem, Register64 temp) - DEFINED_ON(arm, arm64, mips64); + DEFINED_ON(arm, arm64, mips64, loong64); void atomicEffectOp64(const Synchronization& sync, AtomicOp op, Register64 value, const BaseIndex& mem) DEFINED_ON(x64); void atomicEffectOp64(const Synchronization& sync, AtomicOp op, Register64 value, const BaseIndex& mem, Register64 temp) - DEFINED_ON(arm, arm64, mips64); + DEFINED_ON(arm, arm64, mips64, loong64); // 64-bit atomic load. On 64-bit systems, use regular load with // Synchronization::Load, not this method. @@ -3935,13 +3946,13 @@ class MacroAssembler : public MacroAssemblerSpecific { const Address& mem, Register expected, Register replacement, Register valueTemp, Register offsetTemp, Register maskTemp, - Register output) DEFINED_ON(mips_shared); + Register output) DEFINED_ON(mips_shared, loong64); void wasmCompareExchange(const wasm::MemoryAccessDesc& access, const BaseIndex& mem, Register expected, Register replacement, Register valueTemp, Register offsetTemp, Register maskTemp, - Register output) DEFINED_ON(mips_shared); + Register output) DEFINED_ON(mips_shared, loong64); void wasmAtomicExchange(const wasm::MemoryAccessDesc& access, const Address& mem, Register value, Register output) @@ -3955,13 +3966,13 @@ class MacroAssembler : public MacroAssemblerSpecific { const Address& mem, Register value, Register valueTemp, Register offsetTemp, Register maskTemp, Register output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); void wasmAtomicExchange(const wasm::MemoryAccessDesc& access, const BaseIndex& mem, Register value, Register valueTemp, Register offsetTemp, Register maskTemp, Register output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Register value, const Address& mem, Register temp, @@ -3982,13 +3993,13 @@ class MacroAssembler : public MacroAssemblerSpecific { void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Register value, const Address& mem, Register valueTemp, Register offsetTemp, Register maskTemp, - Register output) DEFINED_ON(mips_shared); + Register output) DEFINED_ON(mips_shared, loong64); void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Register value, const BaseIndex& mem, Register valueTemp, Register offsetTemp, Register maskTemp, Register output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); // Read-modify-write with memory. Return no value. // @@ -4014,12 +4025,12 @@ class MacroAssembler : public MacroAssemblerSpecific { void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Register value, const Address& mem, Register valueTemp, Register offsetTemp, - Register maskTemp) DEFINED_ON(mips_shared); + Register maskTemp) DEFINED_ON(mips_shared, loong64); void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Register value, const BaseIndex& mem, Register valueTemp, Register offsetTemp, - Register maskTemp) DEFINED_ON(mips_shared); + Register maskTemp) DEFINED_ON(mips_shared, loong64); // 64-bit wide operations. @@ -4079,12 +4090,12 @@ class MacroAssembler : public MacroAssemblerSpecific { void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op, Register64 value, const Address& mem, Register64 temp, Register64 output) - DEFINED_ON(arm, arm64, mips32, mips64, x64); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x64); void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op, Register64 value, const BaseIndex& mem, Register64 temp, Register64 output) - DEFINED_ON(arm, arm64, mips32, mips64, x64); + DEFINED_ON(arm, arm64, mips32, mips64, loong64, x64); void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op, const Address& value, const Address& mem, @@ -4136,13 +4147,13 @@ class MacroAssembler : public MacroAssemblerSpecific { const Address& mem, Register expected, Register replacement, Register valueTemp, Register offsetTemp, Register maskTemp, Register temp, - AnyRegister output) DEFINED_ON(mips_shared); + AnyRegister output) DEFINED_ON(mips_shared, loong64); void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync, const BaseIndex& mem, Register expected, Register replacement, Register valueTemp, Register offsetTemp, Register maskTemp, Register temp, - AnyRegister output) DEFINED_ON(mips_shared); + AnyRegister output) DEFINED_ON(mips_shared, loong64); void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync, const Address& mem, Register value, Register temp, @@ -4155,13 +4166,13 @@ class MacroAssembler : public MacroAssemblerSpecific { void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync, const Address& mem, Register value, Register valueTemp, Register offsetTemp, Register maskTemp, Register temp, - AnyRegister output) DEFINED_ON(mips_shared); + AnyRegister output) DEFINED_ON(mips_shared, loong64); void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync, const BaseIndex& mem, Register value, Register valueTemp, Register offsetTemp, Register maskTemp, Register temp, AnyRegister output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, Register value, const Address& mem, @@ -4187,13 +4198,13 @@ class MacroAssembler : public MacroAssemblerSpecific { AtomicOp op, Register value, const Address& mem, Register valueTemp, Register offsetTemp, Register maskTemp, Register temp, AnyRegister output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, Register value, const BaseIndex& mem, Register valueTemp, Register offsetTemp, Register maskTemp, Register temp, AnyRegister output) - DEFINED_ON(mips_shared); + DEFINED_ON(mips_shared, loong64); void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, Register value, const Address& mem, @@ -4214,12 +4225,12 @@ class MacroAssembler : public MacroAssemblerSpecific { void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, Register value, const Address& mem, Register valueTemp, Register offsetTemp, - Register maskTemp) DEFINED_ON(mips_shared); + Register maskTemp) DEFINED_ON(mips_shared, loong64); void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, Register value, const BaseIndex& mem, Register valueTemp, Register offsetTemp, - Register maskTemp) DEFINED_ON(mips_shared); + Register maskTemp) DEFINED_ON(mips_shared, loong64); void atomicIsLockFreeJS(Register value, Register output); @@ -4808,7 +4819,8 @@ class MacroAssembler : public MacroAssemblerSpecific { template inline void addStackPtrTo(T t); - void subFromStackPtr(Imm32 imm32) DEFINED_ON(mips32, mips64, arm, x86, x64); + void subFromStackPtr(Imm32 imm32) + DEFINED_ON(mips32, mips64, loong64, arm, x86, x64); void subFromStackPtr(Register reg); template diff --git a/js/src/jit/MoveEmitter.h b/js/src/jit/MoveEmitter.h index 6c62c0561a..a51cbc100a 100644 --- a/js/src/jit/MoveEmitter.h +++ b/js/src/jit/MoveEmitter.h @@ -17,6 +17,8 @@ # include "jit/mips32/MoveEmitter-mips32.h" #elif defined(JS_CODEGEN_MIPS64) # include "jit/mips64/MoveEmitter-mips64.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/MoveEmitter-loong64.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/MoveEmitter-none.h" #else diff --git a/js/src/jit/RegisterAllocator.h b/js/src/jit/RegisterAllocator.h index 51f18b2148..50db79703a 100644 --- a/js/src/jit/RegisterAllocator.h +++ b/js/src/jit/RegisterAllocator.h @@ -316,7 +316,7 @@ class RegisterAllocator { static void takeWasmRegisters(TakeableSet& regs) { #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) regs.take(HeapReg); #endif regs.take(FramePointer); diff --git a/js/src/jit/RegisterSets.h b/js/src/jit/RegisterSets.h index b2b9d4c10c..f2262566b4 100644 --- a/js/src/jit/RegisterSets.h +++ b/js/src/jit/RegisterSets.h @@ -1313,6 +1313,8 @@ inline LiveGeneralRegisterSet SavedNonVolatileRegisters( result.add(Register::FromCode(Registers::lr)); #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) result.add(Register::FromCode(Registers::ra)); +#elif defined(JS_CODEGEN_LOONG64) + result.add(Register::FromCode(Registers::ra)); #endif return result; diff --git a/js/src/jit/Registers.h b/js/src/jit/Registers.h index 67c8661004..51d9b4be91 100644 --- a/js/src/jit/Registers.h +++ b/js/src/jit/Registers.h @@ -20,6 +20,8 @@ # include "jit/mips32/Architecture-mips32.h" #elif defined(JS_CODEGEN_MIPS64) # include "jit/mips64/Architecture-mips64.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/Architecture-loong64.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/Architecture-none.h" #else diff --git a/js/src/jit/SharedICHelpers-inl.h b/js/src/jit/SharedICHelpers-inl.h index 901c80cdd8..60a77956f0 100644 --- a/js/src/jit/SharedICHelpers-inl.h +++ b/js/src/jit/SharedICHelpers-inl.h @@ -17,6 +17,8 @@ # include "jit/arm64/SharedICHelpers-arm64-inl.h" #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) # include "jit/mips-shared/SharedICHelpers-mips-shared-inl.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/SharedICHelpers-loong64-inl.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/SharedICHelpers-none-inl.h" #else diff --git a/js/src/jit/SharedICHelpers.h b/js/src/jit/SharedICHelpers.h index 563cae3ccf..da8378ebae 100644 --- a/js/src/jit/SharedICHelpers.h +++ b/js/src/jit/SharedICHelpers.h @@ -17,6 +17,8 @@ # include "jit/arm64/SharedICHelpers-arm64.h" #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) # include "jit/mips-shared/SharedICHelpers-mips-shared.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/SharedICHelpers-loong64.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/SharedICHelpers-none.h" #else diff --git a/js/src/jit/SharedICRegisters.h b/js/src/jit/SharedICRegisters.h index c87e5f8408..e29f21c28d 100644 --- a/js/src/jit/SharedICRegisters.h +++ b/js/src/jit/SharedICRegisters.h @@ -19,6 +19,8 @@ # include "jit/mips32/SharedICRegisters-mips32.h" #elif defined(JS_CODEGEN_MIPS64) # include "jit/mips64/SharedICRegisters-mips64.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/SharedICRegisters-loong64.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/SharedICRegisters-none.h" #else diff --git a/js/src/jit/Simulator.h b/js/src/jit/Simulator.h index 2f61685145..babe2b9c19 100644 --- a/js/src/jit/Simulator.h +++ b/js/src/jit/Simulator.h @@ -15,6 +15,8 @@ # include "jit/mips32/Simulator-mips32.h" #elif defined(JS_SIMULATOR_MIPS64) # include "jit/mips64/Simulator-mips64.h" +#elif defined(JS_SIMULATOR_LOONG64) +# include "jit/loong64/Simulator-loong64.h" #elif defined(JS_SIMULATOR) # error "Unexpected simulator platform" #endif diff --git a/js/src/jit/loong64/Architecture-loong64.cpp b/js/src/jit/loong64/Architecture-loong64.cpp new file mode 100644 index 0000000000..9882c50221 --- /dev/null +++ b/js/src/jit/loong64/Architecture-loong64.cpp @@ -0,0 +1,83 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loong64/Architecture-loong64.h" + +#include "jit/FlushICache.h" // js::jit::FlushICache +#include "jit/loong64/Simulator-loong64.h" +#include "jit/RegisterSets.h" + +namespace js { +namespace jit { + +Registers::Code Registers::FromName(const char* name) { + // TODO:LOONG64 Registers::FromName alias? + for (size_t i = 0; i < Total; i++) { + if (strcmp(GetName(i), name) == 0) { + return Code(i); + } + } + + return Invalid; +} + +FloatRegisters::Code FloatRegisters::FromName(const char* name) { + for (size_t i = 0; i < Total; i++) { + if (strcmp(GetName(i), name) == 0) { + return Code(i); + } + } + + return Invalid; +} + +FloatRegisterSet FloatRegister::ReduceSetForPush(const FloatRegisterSet& s) { +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + LiveFloatRegisterSet ret; + for (FloatRegisterIterator iter(s); iter.more(); ++iter) { + ret.addUnchecked(FromCode((*iter).encoding())); + } + return ret.set(); +} + +uint32_t FloatRegister::GetPushSizeInBytes(const FloatRegisterSet& s) { +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + return s.size() * sizeof(double); +} + +uint32_t FloatRegister::getRegisterDumpOffsetInBytes() { +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + return encoding() * sizeof(double); +} + +uint32_t GetLOONG64Flags() { return 0; } + +void FlushICache(void* code, size_t size, bool codeIsThreadLocal) { +#if defined(JS_SIMULATOR) + js::jit::SimulatorProcess::FlushICache(code, size); + +#elif defined(__GNUC__) + intptr_t end = reinterpret_cast(code) + size; + __builtin___clear_cache(reinterpret_cast(code), + reinterpret_cast(end)); + +#else + _flush_cache(reinterpret_cast(code), size, BCACHE); + +#endif +} + +} // namespace jit +} // namespace js diff --git a/js/src/jit/loong64/Architecture-loong64.h b/js/src/jit/loong64/Architecture-loong64.h new file mode 100644 index 0000000000..ff40333f94 --- /dev/null +++ b/js/src/jit/loong64/Architecture-loong64.h @@ -0,0 +1,520 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_Architecture_loong64_h +#define jit_loong64_Architecture_loong64_h + +#include "mozilla/MathAlgorithms.h" + +#include "jit/shared/Architecture-shared.h" + +#include "js/Utility.h" + +namespace js { +namespace jit { + +// LoongArch64 has 32 64-bit integer registers, r0 though r31. +// The program counter is not accessible as a register. +// SIMD and scalar floating-point registers share a register bank. +// Floating-point registers are f0 through f31. +// 128 bit SIMD registers are vr0 through vr31. +// e.g., f0 is the bottom 64 bits of vr0. + +// LoongArch64 INT Register Convention: +// Name Alias Usage +// $r0 $zero Constant zero +// $r1 $ra Return address +// $r2 $tp TLS +// $r3 $sp Stack pointer +// $r4-$r11 $a0-$a7 Argument registers +// $r4-$r5 $v0-$v1 Return values +// $r12-$r20 $t0-$t8 Temporary registers +// $r21 $x Reserved +// $r22 $fp Frame pointer +// $r23-$r31 $s0-$s8 Callee-saved registers + +// LoongArch64 FP Register Convention: +// Name Alias Usage +// $f0-$f7 $fa0-$fa7 Argument registers +// $f0-$f1 $fv0-$fv1 Return values +// $f8-f23 $ft0-$ft15 Temporary registers +// $f24-$f31 $fs0-$fs7 Callee-saved registers + +class Registers { + public: + enum RegisterID { + r0 = 0, + r1, + r2, + r3, + r4, + r5, + r6, + r7, + r8, + r9, + r10, + r11, + r12, + r13, + r14, + r15, + r16, + r17, + r18, + r19, + r20, + r21, + r22, + r23, + r24, + r25, + r26, + r27, + r28, + r29, + r30, + r31, + zero = r0, + ra = r1, + tp = r2, + sp = r3, + a0 = r4, + a1 = r5, + a2 = r6, + a3 = r7, + a4 = r8, + a5 = r9, + a6 = r10, + a7 = r11, + t0 = r12, + t1 = r13, + t2 = r14, + t3 = r15, + t4 = r16, + t5 = r17, + t6 = r18, + t7 = r19, + t8 = r20, + rx = r21, + fp = r22, + s0 = r23, + s1 = r24, + s2 = r25, + s3 = r26, + s4 = r27, + s5 = r28, + s6 = r29, + s7 = r30, + s8 = r31, + }; + typedef uint8_t Code; + typedef uint32_t Encoding; + typedef uint32_t SetType; + + static const Code StackPointer = sp; + static const Code Invalid = 0xFF; + + // Content spilled during bailouts. + union RegisterContent { + uintptr_t r; + }; + + static uint32_t SetSize(SetType x) { + static_assert(sizeof(SetType) == 4, "SetType must be 32 bits"); + return mozilla::CountPopulation32(x); + } + static uint32_t FirstBit(SetType x) { + return mozilla::CountTrailingZeroes32(x); + } + static uint32_t LastBit(SetType x) { + return 31 - mozilla::CountLeadingZeroes32(x); + } + + static const char* GetName(uint32_t code) { + static const char* const Names[] = { + "zero", "ra", "tp", "sp", "a0", "a1", "a2", "a3", "a4", "a5", "a6", + "a7", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "rx", + "fp", "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8"}; + static_assert(Total == sizeof(Names) / sizeof(Names[0]), + "Table is the correct size"); + if (code >= Total) { + return "invalid"; + } + return Names[code]; + } + + static Code FromName(const char* name); + + static const uint32_t Total = 32; + static const uint32_t TotalPhys = 32; + static const uint32_t Allocatable = + 23; // No named special-function registers. + + static const SetType AllMask = 0xFFFFFFFF; + static const SetType NoneMask = 0x0; + + static const SetType ArgRegMask = + (1 << Registers::a0) | (1 << Registers::a1) | (1 << Registers::a2) | + (1 << Registers::a3) | (1 << Registers::a4) | (1 << Registers::a5) | + (1 << Registers::a6) | (1 << Registers::a7); + + static const SetType VolatileMask = + (1 << Registers::a0) | (1 << Registers::a1) | (1 << Registers::a2) | + (1 << Registers::a3) | (1 << Registers::a4) | (1 << Registers::a5) | + (1 << Registers::a6) | (1 << Registers::a7) | (1 << Registers::t0) | + (1 << Registers::t1) | (1 << Registers::t2) | (1 << Registers::t3) | + (1 << Registers::t4) | (1 << Registers::t5) | (1 << Registers::t6) | + (1 << Registers::t7) | (1 << Registers::t8); + + // We use this constant to save registers when entering functions. This + // is why $ra is added here even though it is not "Non Volatile". + static const SetType NonVolatileMask = + (1 << Registers::ra) | (1 << Registers::fp) | (1 << Registers::s0) | + (1 << Registers::s1) | (1 << Registers::s2) | (1 << Registers::s3) | + (1 << Registers::s4) | (1 << Registers::s5) | (1 << Registers::s6) | + (1 << Registers::s7) | (1 << Registers::s8); + + // TODO:LOONG64 SingleByteRegs + static const SetType SingleByteRegs = VolatileMask | NonVolatileMask; + + static const SetType NonAllocatableMask = + (1 << Registers::zero) | // Always be zero. + (1 << Registers::t6) | // First scratch register. + (1 << Registers::t7) | // Second scratch register. + (1 << Registers::t8) | // Third scratch register. + (1 << Registers::rx) | // Reserved Register. + (1 << Registers::ra) | (1 << Registers::tp) | (1 << Registers::sp); + + static const SetType WrapperMask = VolatileMask; + + // Registers returned from a JS -> JS call. + static const SetType JSCallMask = (1 << Registers::a2); + + // Registers returned from a JS -> C call. + static const SetType CallMask = (1 << Registers::a0); + + static const SetType AllocatableMask = AllMask & ~NonAllocatableMask; +}; + +// Smallest integer type that can hold a register bitmask. +typedef uint32_t PackedRegisterMask; + +template +class TypedRegisterSet; + +class FloatRegisters { + public: + enum FPRegisterID { + f0 = 0, + f1, + f2, + f3, + f4, + f5, + f6, + f7, + f8, + f9, + f10, + f11, + f12, + f13, + f14, + f15, + f16, + f17, + f18, + f19, + f20, + f21, + f22, + f23, + f24, + f25, + f26, + f27, + f28, + f29, + f30, + f31, // Scratch register. + }; + + // Eight bits: (invalid << 7) | (kind << 5) | encoding + typedef uint8_t Code; + typedef FPRegisterID Encoding; + typedef uint64_t SetType; + + enum Kind : uint8_t { Double, Single, NumTypes }; + + static constexpr Code Invalid = 0x80; + + static const char* GetName(uint32_t code) { + static const char* const Names[] = { + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"}; + static_assert(TotalPhys == sizeof(Names) / sizeof(Names[0]), + "Table is the correct size"); + if (code >= Total) { + return "invalid"; + } + return Names[code]; + } + + static Code FromName(const char* name); + + static const uint32_t TotalPhys = 32; + // TODO:LOONG64 Recheck + static const uint32_t Total = TotalPhys * NumTypes; + static const uint32_t Allocatable = 31; // Without f31, the scratch register. + + static_assert(sizeof(SetType) * 8 >= Total, + "SetType should be large enough to enumerate all registers."); + + // Magic values which are used to duplicate a mask of physical register for + // a specific type of register. A multiplication is used to copy and shift + // the bits of the physical register mask. + static const SetType SpreadSingle = SetType(1) + << (uint32_t(Single) * TotalPhys); + static const SetType SpreadDouble = SetType(1) + << (uint32_t(Double) * TotalPhys); + static const SetType Spread = SpreadSingle | SpreadDouble; + + static const SetType AllPhysMask = ((SetType(1) << TotalPhys) - 1); + static const SetType AllMask = AllPhysMask * Spread; + static const SetType AllSingleMask = AllPhysMask * SpreadSingle; + static const SetType AllDoubleMask = AllPhysMask * SpreadDouble; + static const SetType NoneMask = SetType(0); + + // f31 is the ScratchFloatReg. + static const SetType NonVolatileMask = + SetType((1 << FloatRegisters::f24) | (1 << FloatRegisters::f25) | + (1 << FloatRegisters::f26) | (1 << FloatRegisters::f27) | + (1 << FloatRegisters::f28) | (1 << FloatRegisters::f29) | + (1 << FloatRegisters::f30)) * + Spread; + + static const SetType VolatileMask = AllMask & ~NonVolatileMask; + + static const SetType WrapperMask = VolatileMask; + + // f31 is the ScratchFloatReg. + static const SetType NonAllocatableMask = + (SetType(1) << FloatRegisters::f31) * Spread; + + static const SetType AllocatableMask = AllMask & ~NonAllocatableMask; + + // Content spilled during bailouts. + union RegisterContent { + float s; + double d; + }; + + static constexpr Encoding encoding(Code c) { + // assert() not available in constexpr function. + // assert(c < Total); + return Encoding(c & 31); + } + + static constexpr Kind kind(Code c) { + // assert() not available in constexpr function. + // assert(c < Total && ((c >> 5) & 3) < NumTypes); + return Kind((c >> 5) & 3); + } + + static constexpr Code fromParts(uint32_t encoding, uint32_t kind, + uint32_t invalid) { + return Code((invalid << 7) | (kind << 5) | encoding); + } +}; + +static const uint32_t ShadowStackSpace = 0; + +// When our only strategy for far jumps is to encode the offset directly, and +// not insert any jump islands during assembly for even further jumps, then the +// architecture restricts us to -2^27 .. 2^27-4, to fit into a signed 28-bit +// value. We further reduce this range to allow the far-jump inserting code to +// have some breathing room. +static const uint32_t JumpImmediateRange = ((1 << 27) - (20 * 1024 * 1024)); + +struct FloatRegister { + typedef FloatRegisters Codes; + typedef size_t Code; + typedef Codes::Encoding Encoding; + typedef Codes::SetType SetType; + + static uint32_t SetSize(SetType x) { + static_assert(sizeof(SetType) == 8, "SetType must be 64 bits"); + x |= x >> FloatRegisters::TotalPhys; + x &= FloatRegisters::AllPhysMask; + return mozilla::CountPopulation32(x); + } + + static uint32_t FirstBit(SetType x) { + static_assert(sizeof(SetType) == 8, "SetType"); + return mozilla::CountTrailingZeroes64(x); + } + static uint32_t LastBit(SetType x) { + static_assert(sizeof(SetType) == 8, "SetType"); + return 63 - mozilla::CountLeadingZeroes64(x); + } + + private: + // These fields only hold valid values: an invalid register is always + // represented as a valid encoding and kind with the invalid_ bit set. + uint8_t encoding_; // 32 encodings + uint8_t kind_; // Double, Single; more later + bool invalid_; + + typedef Codes::Kind Kind; + + public: + constexpr FloatRegister(Encoding encoding, Kind kind) + : encoding_(encoding), kind_(kind), invalid_(false) { + // assert(uint32_t(encoding) < Codes::TotalPhys); + } + + constexpr FloatRegister() + : encoding_(0), kind_(FloatRegisters::Double), invalid_(true) {} + + static FloatRegister FromCode(uint32_t i) { + MOZ_ASSERT(i < Codes::Total); + return FloatRegister(FloatRegisters::encoding(i), FloatRegisters::kind(i)); + } + + bool isSingle() const { + MOZ_ASSERT(!invalid_); + return kind_ == FloatRegisters::Single; + } + bool isDouble() const { + MOZ_ASSERT(!invalid_); + return kind_ == FloatRegisters::Double; + } + bool isSimd128() const { + MOZ_ASSERT(!invalid_); + return false; + } + bool isInvalid() const { return invalid_; } + + FloatRegister asSingle() const { + MOZ_ASSERT(!invalid_); + return FloatRegister(Encoding(encoding_), FloatRegisters::Single); + } + FloatRegister asDouble() const { + MOZ_ASSERT(!invalid_); + return FloatRegister(Encoding(encoding_), FloatRegisters::Double); + } + FloatRegister asSimd128() const { MOZ_CRASH(); } + + constexpr uint32_t size() const { + MOZ_ASSERT(!invalid_); + if (kind_ == FloatRegisters::Double) { + return sizeof(double); + } + MOZ_ASSERT(kind_ == FloatRegisters::Single); + return sizeof(float); + } + + constexpr Code code() const { + // assert(!invalid_); + return Codes::fromParts(encoding_, kind_, invalid_); + } + + constexpr Encoding encoding() const { + MOZ_ASSERT(!invalid_); + return Encoding(encoding_); + } + + const char* name() const { return FloatRegisters::GetName(code()); } + bool volatile_() const { + MOZ_ASSERT(!invalid_); + return !!((SetType(1) << code()) & FloatRegisters::VolatileMask); + } + constexpr bool operator!=(FloatRegister other) const { + return code() != other.code(); + } + constexpr bool operator==(FloatRegister other) const { + return code() == other.code(); + } + + bool aliases(FloatRegister other) const { + return other.encoding_ == encoding_; + } + // Ensure that two floating point registers' types are equivalent. + bool equiv(FloatRegister other) const { + MOZ_ASSERT(!invalid_); + return kind_ == other.kind_; + } + + uint32_t numAliased() const { return Codes::NumTypes; } + uint32_t numAlignedAliased() { return numAliased(); } + + FloatRegister aliased(uint32_t aliasIdx) { + MOZ_ASSERT(!invalid_); + MOZ_ASSERT(aliasIdx < numAliased()); + return FloatRegister(Encoding(encoding_), + Kind((aliasIdx + kind_) % numAliased())); + } + FloatRegister alignedAliased(uint32_t aliasIdx) { + MOZ_ASSERT(aliasIdx < numAliased()); + return aliased(aliasIdx); + } + SetType alignedOrDominatedAliasedSet() const { + return Codes::Spread << encoding_; + } + + static constexpr RegTypeName DefaultType = RegTypeName::Float64; + + template + static SetType LiveAsIndexableSet(SetType s) { + return SetType(0); + } + + template + static SetType AllocatableAsIndexableSet(SetType s) { + static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable"); + return LiveAsIndexableSet(s); + } + + static TypedRegisterSet ReduceSetForPush( + const TypedRegisterSet& s); + static uint32_t GetPushSizeInBytes(const TypedRegisterSet& s); + uint32_t getRegisterDumpOffsetInBytes(); +}; + +template <> +inline FloatRegister::SetType +FloatRegister::LiveAsIndexableSet(SetType set) { + return set & FloatRegisters::AllSingleMask; +} + +template <> +inline FloatRegister::SetType +FloatRegister::LiveAsIndexableSet(SetType set) { + return set & FloatRegisters::AllDoubleMask; +} + +template <> +inline FloatRegister::SetType +FloatRegister::LiveAsIndexableSet(SetType set) { + return set; +} + +// LoongArch doesn't have double registers that cannot be treated as float32. +inline bool hasUnaliasedDouble() { return false; } + +// LoongArch doesn't have double registers that alias multiple floats. +inline bool hasMultiAlias() { return false; } + +// TODO:LOONG64 GetLOONG64Flags +uint32_t GetLOONG64Flags(); + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_Architecture_loong64_h */ diff --git a/js/src/jit/loong64/Assembler-loong64.cpp b/js/src/jit/loong64/Assembler-loong64.cpp new file mode 100644 index 0000000000..6561f484a9 --- /dev/null +++ b/js/src/jit/loong64/Assembler-loong64.cpp @@ -0,0 +1,2479 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loong64/Assembler-loong64.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/Maybe.h" + +#include "gc/Marking.h" +#include "jit/AutoWritableJitCode.h" +#include "jit/ExecutableAllocator.h" +#include "vm/Realm.h" + +using mozilla::DebugOnly; + +using namespace js; +using namespace js::jit; + +// Encode a standard register when it is being used as rd, the rs, and +// an extra register(rt). These should never be called with an InvalidReg. +uint32_t js::jit::RK(Register r) { + MOZ_ASSERT((r.code() & ~RegMask) == 0); + return r.code() << RKShift; +} + +uint32_t js::jit::RJ(Register r) { + MOZ_ASSERT((r.code() & ~RegMask) == 0); + return r.code() << RJShift; +} + +uint32_t js::jit::RD(Register r) { + MOZ_ASSERT((r.code() & ~RegMask) == 0); + return r.code() << RDShift; +} + +uint32_t js::jit::SA2(uint32_t value) { + MOZ_ASSERT(value < 4); + return (value & SA2Mask) << SAShift; +} + +uint32_t js::jit::SA3(uint32_t value) { + MOZ_ASSERT(value < 8); + return (value & SA3Mask) << SAShift; +} + +Register js::jit::toRK(Instruction& i) { + return Register::FromCode((i.encode() & RKMask) >> RKShift); +} + +Register js::jit::toRJ(Instruction& i) { + return Register::FromCode((i.encode() & RJMask) >> RJShift); +} + +Register js::jit::toRD(Instruction& i) { + return Register::FromCode((i.encode() & RDMask) >> RDShift); +} + +Register js::jit::toR(Instruction& i) { + return Register::FromCode(i.encode() & RegMask); +} + +void InstImm::extractImm16(BOffImm16* dest) { *dest = BOffImm16(*this); } + +void AssemblerLOONG64::finish() { + MOZ_ASSERT(!isFinished); + isFinished = true; +} + +bool AssemblerLOONG64::appendRawCode(const uint8_t* code, size_t numBytes) { + return m_buffer.appendRawCode(code, numBytes); +} + +bool AssemblerLOONG64::reserve(size_t size) { + // This buffer uses fixed-size chunks so there's no point in reserving + // now vs. on-demand. + return !oom(); +} + +bool AssemblerLOONG64::swapBuffer(wasm::Bytes& bytes) { + // For now, specialize to the one use case. As long as wasm::Bytes is a + // Vector, not a linked-list of chunks, there's not much we can do other + // than copy. + MOZ_ASSERT(bytes.empty()); + if (!bytes.resize(bytesNeeded())) { + return false; + } + m_buffer.executableCopy(bytes.begin()); + return true; +} + +void AssemblerLOONG64::copyJumpRelocationTable(uint8_t* dest) { + if (jumpRelocations_.length()) { + memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length()); + } +} + +void AssemblerLOONG64::copyDataRelocationTable(uint8_t* dest) { + if (dataRelocations_.length()) { + memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length()); + } +} + +AssemblerLOONG64::Condition AssemblerLOONG64::InvertCondition(Condition cond) { + switch (cond) { + case Equal: + return NotEqual; + case NotEqual: + return Equal; + case Zero: + return NonZero; + case NonZero: + return Zero; + case LessThan: + return GreaterThanOrEqual; + case LessThanOrEqual: + return GreaterThan; + case GreaterThan: + return LessThanOrEqual; + case GreaterThanOrEqual: + return LessThan; + case Above: + return BelowOrEqual; + case AboveOrEqual: + return Below; + case Below: + return AboveOrEqual; + case BelowOrEqual: + return Above; + case Signed: + return NotSigned; + case NotSigned: + return Signed; + default: + MOZ_CRASH("unexpected condition"); + } +} + +AssemblerLOONG64::DoubleCondition AssemblerLOONG64::InvertCondition( + DoubleCondition cond) { + switch (cond) { + case DoubleOrdered: + return DoubleUnordered; + case DoubleEqual: + return DoubleNotEqualOrUnordered; + case DoubleNotEqual: + return DoubleEqualOrUnordered; + case DoubleGreaterThan: + return DoubleLessThanOrEqualOrUnordered; + case DoubleGreaterThanOrEqual: + return DoubleLessThanOrUnordered; + case DoubleLessThan: + return DoubleGreaterThanOrEqualOrUnordered; + case DoubleLessThanOrEqual: + return DoubleGreaterThanOrUnordered; + case DoubleUnordered: + return DoubleOrdered; + case DoubleEqualOrUnordered: + return DoubleNotEqual; + case DoubleNotEqualOrUnordered: + return DoubleEqual; + case DoubleGreaterThanOrUnordered: + return DoubleLessThanOrEqual; + case DoubleGreaterThanOrEqualOrUnordered: + return DoubleLessThan; + case DoubleLessThanOrUnordered: + return DoubleGreaterThanOrEqual; + case DoubleLessThanOrEqualOrUnordered: + return DoubleGreaterThan; + default: + MOZ_CRASH("unexpected condition"); + } +} + +AssemblerLOONG64::Condition AssemblerLOONG64::InvertCmpCondition( + Condition cond) { + switch (cond) { + case Equal: + case NotEqual: + return cond; + case LessThan: + return GreaterThan; + case LessThanOrEqual: + return GreaterThanOrEqual; + case GreaterThan: + return LessThanOrEqual; + case GreaterThanOrEqual: + return LessThan; + case Above: + return Below; + case AboveOrEqual: + return BelowOrEqual; + case Below: + return Above; + case BelowOrEqual: + return AboveOrEqual; + default: + MOZ_CRASH("no meaningful swapped-operand condition"); + } +} + +BOffImm16::BOffImm16(InstImm inst) + : data((inst.encode() >> Imm16Shift) & Imm16Mask) {} + +Instruction* BOffImm16::getDest(Instruction* src) const { + return &src[(((int32_t)data << 16) >> 16) + 1]; +} + +bool AssemblerLOONG64::oom() const { + return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() || + dataRelocations_.oom(); +} + +// Size of the instruction stream, in bytes. +size_t AssemblerLOONG64::size() const { return m_buffer.size(); } + +// Size of the relocation table, in bytes. +size_t AssemblerLOONG64::jumpRelocationTableBytes() const { + return jumpRelocations_.length(); +} + +size_t AssemblerLOONG64::dataRelocationTableBytes() const { + return dataRelocations_.length(); +} + +// Size of the data table, in bytes. +size_t AssemblerLOONG64::bytesNeeded() const { + return size() + jumpRelocationTableBytes() + dataRelocationTableBytes(); +} + +// write a blob of binary into the instruction stream +BufferOffset AssemblerLOONG64::writeInst(uint32_t x, uint32_t* dest) { + if (dest == nullptr) { + return m_buffer.putInt(x); + } + + WriteInstStatic(x, dest); + return BufferOffset(); +} + +void AssemblerLOONG64::WriteInstStatic(uint32_t x, uint32_t* dest) { + MOZ_ASSERT(dest != nullptr); + *dest = x; +} + +BufferOffset AssemblerLOONG64::haltingAlign(int alignment) { + // TODO: Implement a proper halting align. + return nopAlign(alignment); +} + +BufferOffset AssemblerLOONG64::nopAlign(int alignment) { + BufferOffset ret; + MOZ_ASSERT(m_buffer.isAligned(4)); + if (alignment == 8) { + if (!m_buffer.isAligned(alignment)) { + BufferOffset tmp = as_nop(); + if (!ret.assigned()) { + ret = tmp; + } + } + } else { + MOZ_ASSERT((alignment & (alignment - 1)) == 0); + while (size() & (alignment - 1)) { + BufferOffset tmp = as_nop(); + if (!ret.assigned()) { + ret = tmp; + } + } + } + return ret; +} + +// Logical operations. +BufferOffset AssemblerLOONG64::as_and(Register rd, Register rj, Register rk) { + spew("and %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_and, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_or(Register rd, Register rj, Register rk) { + spew("or %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_or, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_xor(Register rd, Register rj, Register rk) { + spew("xor %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_xor, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_nor(Register rd, Register rj, Register rk) { + spew("nor %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_nor, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_andn(Register rd, Register rj, Register rk) { + spew("andn %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_andn, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_orn(Register rd, Register rj, Register rk) { + spew("orn %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_orn, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_andi(Register rd, Register rj, int32_t ui12) { + MOZ_ASSERT(is_uintN(ui12, 12)); + spew("andi %3s,%3s,0x%x", rd.name(), rj.name(), ui12); + return writeInst(InstImm(op_andi, ui12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_ori(Register rd, Register rj, int32_t ui12) { + MOZ_ASSERT(is_uintN(ui12, 12)); + spew("ori %3s,%3s,0x%x", rd.name(), rj.name(), ui12); + return writeInst(InstImm(op_ori, ui12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_xori(Register rd, Register rj, int32_t ui12) { + MOZ_ASSERT(is_uintN(ui12, 12)); + spew("xori %3s,%3s,0x%x", rd.name(), rj.name(), ui12); + return writeInst(InstImm(op_xori, ui12, rj, rd, 12).encode()); +} + +// Branch and jump instructions +BufferOffset AssemblerLOONG64::as_b(JOffImm26 off) { + spew("b %d", off.decode()); + return writeInst(InstJump(op_b, off).encode()); +} + +BufferOffset AssemblerLOONG64::as_bl(JOffImm26 off) { + spew("bl %d", off.decode()); + return writeInst(InstJump(op_bl, off).encode()); +} + +BufferOffset AssemblerLOONG64::as_jirl(Register rd, Register rj, + BOffImm16 off) { + spew("jirl %3s, %3s, %d", rd.name(), rj.name(), off.decode()); + return writeInst(InstImm(op_jirl, off, rj, rd).encode()); +} + +InstImm AssemblerLOONG64::getBranchCode(JumpOrCall jumpOrCall) { + // jirl or beq + if (jumpOrCall == BranchIsCall) { + return InstImm(op_jirl, BOffImm16(0), zero, ra); + } + + return InstImm(op_beq, BOffImm16(0), zero, zero); +} + +InstImm AssemblerLOONG64::getBranchCode(Register rj, Register rd, Condition c) { + // beq, bne + MOZ_ASSERT(c == AssemblerLOONG64::Equal || c == AssemblerLOONG64::NotEqual); + return InstImm(c == AssemblerLOONG64::Equal ? op_beq : op_bne, BOffImm16(0), + rj, rd); +} + +InstImm AssemblerLOONG64::getBranchCode(Register rj, Condition c) { + // beq, bne, blt, bge + switch (c) { + case AssemblerLOONG64::Equal: + case AssemblerLOONG64::Zero: + case AssemblerLOONG64::BelowOrEqual: + return InstImm(op_beq, BOffImm16(0), rj, zero); + case AssemblerLOONG64::NotEqual: + case AssemblerLOONG64::NonZero: + case AssemblerLOONG64::Above: + return InstImm(op_bne, BOffImm16(0), rj, zero); + case AssemblerLOONG64::GreaterThan: + return InstImm(op_blt, BOffImm16(0), zero, rj); + case AssemblerLOONG64::GreaterThanOrEqual: + case AssemblerLOONG64::NotSigned: + return InstImm(op_bge, BOffImm16(0), rj, zero); + case AssemblerLOONG64::LessThan: + case AssemblerLOONG64::Signed: + return InstImm(op_blt, BOffImm16(0), rj, zero); + case AssemblerLOONG64::LessThanOrEqual: + return InstImm(op_bge, BOffImm16(0), zero, rj); + default: + MOZ_CRASH("Condition not supported."); + } +} + +// Code semantics must conform to compareFloatingpoint +InstImm AssemblerLOONG64::getBranchCode(FPConditionBit cj) { + return InstImm(op_bcz, 0, cj, true); // bcnez +} + +// Arithmetic instructions +BufferOffset AssemblerLOONG64::as_add_w(Register rd, Register rj, Register rk) { + spew("add_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_add_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_add_d(Register rd, Register rj, Register rk) { + spew("add_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_add_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_sub_w(Register rd, Register rj, Register rk) { + spew("sub_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sub_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_sub_d(Register rd, Register rj, Register rk) { + spew("sub_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sub_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_addi_w(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("addi_w %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_addi_w, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_addi_d(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("addi_d %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_addi_d, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_addu16i_d(Register rd, Register rj, + int32_t si16) { + MOZ_ASSERT(Imm16::IsInSignedRange(si16)); + spew("addu16i_d %3s,%3s,0x%x", rd.name(), rj.name(), si16); + return writeInst(InstImm(op_addu16i_d, Imm16(si16), rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_alsl_w(Register rd, Register rj, Register rk, + uint32_t sa2) { + MOZ_ASSERT(sa2 < 4); + spew("alsl_w %3s,%3s,0x%x", rd.name(), rj.name(), sa2); + return writeInst(InstReg(op_alsl_w, sa2, rk, rj, rd, 2).encode()); +} + +BufferOffset AssemblerLOONG64::as_alsl_wu(Register rd, Register rj, Register rk, + uint32_t sa2) { + MOZ_ASSERT(sa2 < 4); + spew("alsl_wu %3s,%3s,0x%x", rd.name(), rj.name(), sa2); + return writeInst(InstReg(op_alsl_wu, sa2, rk, rj, rd, 2).encode()); +} + +BufferOffset AssemblerLOONG64::as_alsl_d(Register rd, Register rj, Register rk, + uint32_t sa2) { + MOZ_ASSERT(sa2 < 4); + spew("alsl_d %3s,%3s,%3s,0x%x", rd.name(), rj.name(), rk.name(), sa2); + return writeInst(InstReg(op_alsl_d, sa2, rk, rj, rd, 2).encode()); +} + +BufferOffset AssemblerLOONG64::as_lu12i_w(Register rd, int32_t si20) { + spew("lu12i_w %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_lu12i_w, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONG64::as_lu32i_d(Register rd, int32_t si20) { + spew("lu32i_d %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_lu32i_d, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONG64::as_lu52i_d(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_uintN(si12, 12)); + spew("lu52i_d %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_lu52i_d, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_slt(Register rd, Register rj, Register rk) { + spew("slt %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_slt, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_sltu(Register rd, Register rj, Register rk) { + spew("sltu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sltu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_slti(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("slti %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_slti, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_sltui(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("sltui %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_sltui, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_pcaddi(Register rd, int32_t si20) { + spew("pcaddi %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_pcaddi, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONG64::as_pcaddu12i(Register rd, int32_t si20) { + spew("pcaddu12i %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_pcaddu12i, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONG64::as_pcaddu18i(Register rd, int32_t si20) { + spew("pcaddu18i %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_pcaddu18i, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONG64::as_pcalau12i(Register rd, int32_t si20) { + spew("pcalau12i %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_pcalau12i, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONG64::as_mul_w(Register rd, Register rj, Register rk) { + spew("mul_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mul_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mulh_w(Register rd, Register rj, + Register rk) { + spew("mulh_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulh_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mulh_wu(Register rd, Register rj, + Register rk) { + spew("mulh_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulh_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mul_d(Register rd, Register rj, Register rk) { + spew("mul_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mul_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mulh_d(Register rd, Register rj, + Register rk) { + spew("mulh_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulh_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mulh_du(Register rd, Register rj, + Register rk) { + spew("mulh_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulh_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mulw_d_w(Register rd, Register rj, + Register rk) { + spew("mulw_d_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulw_d_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mulw_d_wu(Register rd, Register rj, + Register rk) { + spew("mulw_d_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulw_d_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_div_w(Register rd, Register rj, Register rk) { + spew("div_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_div_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mod_w(Register rd, Register rj, Register rk) { + spew("mod_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mod_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_div_wu(Register rd, Register rj, + Register rk) { + spew("div_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_div_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mod_wu(Register rd, Register rj, + Register rk) { + spew("mod_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mod_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_div_d(Register rd, Register rj, Register rk) { + spew("div_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_div_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mod_d(Register rd, Register rj, Register rk) { + spew("mod_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mod_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_div_du(Register rd, Register rj, + Register rk) { + spew("div_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_div_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_mod_du(Register rd, Register rj, + Register rk) { + spew("mod_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mod_du, rk, rj, rd).encode()); +} + +// Shift instructions +BufferOffset AssemblerLOONG64::as_sll_w(Register rd, Register rj, Register rk) { + spew("sll_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sll_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_srl_w(Register rd, Register rj, Register rk) { + spew("srl_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_srl_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_sra_w(Register rd, Register rj, Register rk) { + spew("sra_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sra_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_rotr_w(Register rd, Register rj, + Register rk) { + spew("rotr_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_rotr_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_slli_w(Register rd, Register rj, + int32_t ui5) { + MOZ_ASSERT(is_uintN(ui5, 5)); + spew("slli_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5); + return writeInst(InstImm(op_slli_w, ui5, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONG64::as_srli_w(Register rd, Register rj, + int32_t ui5) { + MOZ_ASSERT(is_uintN(ui5, 5)); + spew("srli_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5); + return writeInst(InstImm(op_srli_w, ui5, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONG64::as_srai_w(Register rd, Register rj, + int32_t ui5) { + MOZ_ASSERT(is_uintN(ui5, 5)); + spew("srai_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5); + return writeInst(InstImm(op_srai_w, ui5, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONG64::as_rotri_w(Register rd, Register rj, + int32_t ui5) { + MOZ_ASSERT(is_uintN(ui5, 5)); + spew("rotri_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5); + return writeInst(InstImm(op_rotri_w, ui5, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONG64::as_sll_d(Register rd, Register rj, Register rk) { + spew("sll_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sll_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_srl_d(Register rd, Register rj, Register rk) { + spew("srl_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_srl_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_sra_d(Register rd, Register rj, Register rk) { + spew("sra_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sra_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_rotr_d(Register rd, Register rj, + Register rk) { + spew("rotr_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_rotr_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_slli_d(Register rd, Register rj, + int32_t ui6) { + MOZ_ASSERT(is_uintN(ui6, 6)); + spew("slli_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6); + return writeInst(InstImm(op_slli_d, ui6, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONG64::as_srli_d(Register rd, Register rj, + int32_t ui6) { + MOZ_ASSERT(is_uintN(ui6, 6)); + spew("srli_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6); + return writeInst(InstImm(op_srli_d, ui6, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONG64::as_srai_d(Register rd, Register rj, + int32_t ui6) { + MOZ_ASSERT(is_uintN(ui6, 6)); + spew("srai_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6); + return writeInst(InstImm(op_srai_d, ui6, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONG64::as_rotri_d(Register rd, Register rj, + int32_t ui6) { + MOZ_ASSERT(is_uintN(ui6, 6)); + spew("rotri_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6); + return writeInst(InstImm(op_rotri_d, ui6, rj, rd, 6).encode()); +} + +// Bit operation instrucitons +BufferOffset AssemblerLOONG64::as_ext_w_b(Register rd, Register rj) { + spew("ext_w_b %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_ext_w_b, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ext_w_h(Register rd, Register rj) { + spew("ext_w_h %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_ext_w_h, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_clo_w(Register rd, Register rj) { + spew("clo_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_clo_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_clz_w(Register rd, Register rj) { + spew("clz_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_clz_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_cto_w(Register rd, Register rj) { + spew("cto_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_cto_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ctz_w(Register rd, Register rj) { + spew("ctz_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_ctz_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_clo_d(Register rd, Register rj) { + spew("clo_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_clo_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_clz_d(Register rd, Register rj) { + spew("clz_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_clz_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_cto_d(Register rd, Register rj) { + spew("cto_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_cto_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ctz_d(Register rd, Register rj) { + spew("ctz_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_ctz_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_bytepick_w(Register rd, Register rj, + Register rk, int32_t sa2) { + MOZ_ASSERT(sa2 < 4); + spew("bytepick_w %3s,%3s,%3s, 0x%x", rd.name(), rj.name(), rk.name(), sa2); + return writeInst(InstReg(op_bytepick_w, sa2, rk, rj, rd, 2).encode()); +} + +BufferOffset AssemblerLOONG64::as_bytepick_d(Register rd, Register rj, + Register rk, int32_t sa3) { + MOZ_ASSERT(sa3 < 8); + spew("bytepick_d %3s,%3s,%3s, 0x%x", rd.name(), rj.name(), rk.name(), sa3); + return writeInst(InstReg(op_bytepick_d, sa3, rk, rj, rd, 3).encode()); +} + +BufferOffset AssemblerLOONG64::as_revb_2h(Register rd, Register rj) { + spew("revb_2h %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revb_2h, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_revb_4h(Register rd, Register rj) { + spew("revb_4h %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revb_4h, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_revb_2w(Register rd, Register rj) { + spew("revb_2w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revb_2w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_revb_d(Register rd, Register rj) { + spew("revb_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revb_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_revh_2w(Register rd, Register rj) { + spew("revh_2w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revh_2w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_revh_d(Register rd, Register rj) { + spew("revh_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revh_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_bitrev_4b(Register rd, Register rj) { + spew("bitrev_4b %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_bitrev_4b, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_bitrev_8b(Register rd, Register rj) { + spew("bitrev_8b %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_bitrev_8b, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_bitrev_w(Register rd, Register rj) { + spew("bitrev_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_bitrev_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_bitrev_d(Register rd, Register rj) { + spew("bitrev_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_bitrev_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_bstrins_w(Register rd, Register rj, + int32_t msbw, int32_t lsbw) { + MOZ_ASSERT(lsbw <= msbw); + spew("bstrins_w %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbw, lsbw); + return writeInst(InstImm(op_bstr_w, msbw, lsbw, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONG64::as_bstrins_d(Register rd, Register rj, + int32_t msbd, int32_t lsbd) { + MOZ_ASSERT(lsbd <= msbd); + spew("bstrins_d %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbd, lsbd); + return writeInst(InstImm(op_bstrins_d, msbd, lsbd, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONG64::as_bstrpick_w(Register rd, Register rj, + int32_t msbw, int32_t lsbw) { + MOZ_ASSERT(lsbw <= msbw); + spew("bstrpick_w %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbw, lsbw); + return writeInst(InstImm(op_bstr_w, msbw, lsbw, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_bstrpick_d(Register rd, Register rj, + int32_t msbd, int32_t lsbd) { + MOZ_ASSERT(lsbd <= msbd); + spew("bstrpick_d %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbd, lsbd); + return writeInst(InstImm(op_bstrpick_d, msbd, lsbd, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONG64::as_maskeqz(Register rd, Register rj, + Register rk) { + spew("maskeqz %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_maskeqz, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_masknez(Register rd, Register rj, + Register rk) { + spew("masknez %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_masknez, rk, rj, rd).encode()); +} + +// Load and store instructions +BufferOffset AssemblerLOONG64::as_ld_b(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_b %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_b, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_ld_h(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_h %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_h, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_ld_w(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_w %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_w, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_ld_d(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_d %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_d, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_ld_bu(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_bu %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_bu, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_ld_hu(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_hu %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_hu, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_ld_wu(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_wu %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_wu, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_st_b(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("st_b %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_st_b, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_st_h(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("st_h %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_st_h, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_st_w(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("st_w %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_st_w, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_st_d(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("st_d %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_st_d, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldx_b(Register rd, Register rj, Register rk) { + spew("ldx_b %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_b, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldx_h(Register rd, Register rj, Register rk) { + spew("ldx_h %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_h, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldx_w(Register rd, Register rj, Register rk) { + spew("ldx_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldx_d(Register rd, Register rj, Register rk) { + spew("ldx_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldx_bu(Register rd, Register rj, + Register rk) { + spew("ldx_bu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_bu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldx_hu(Register rd, Register rj, + Register rk) { + spew("ldx_hu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_hu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldx_wu(Register rd, Register rj, + Register rk) { + spew("ldx_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_b, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_stx_b(Register rd, Register rj, Register rk) { + spew("stx_b %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_stx_b, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_stx_h(Register rd, Register rj, Register rk) { + spew("stx_h %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_stx_h, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_stx_w(Register rd, Register rj, Register rk) { + spew("stx_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_stx_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_stx_d(Register rd, Register rj, Register rk) { + spew("stx_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_stx_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldptr_w(Register rd, Register rj, + int32_t si14) { + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + spew("ldptr_w %3s,%3s,0x%x", rd.name(), rj.name(), si14); + return writeInst(InstImm(op_ldptr_w, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONG64::as_ldptr_d(Register rd, Register rj, + int32_t si14) { + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + spew("ldptr_d %3s,%3s,0x%x", rd.name(), rj.name(), si14); + return writeInst(InstImm(op_ldptr_d, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONG64::as_stptr_w(Register rd, Register rj, + int32_t si14) { + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + spew("stptr_w %3s,%3s,0x%x", rd.name(), rj.name(), si14); + return writeInst(InstImm(op_stptr_w, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONG64::as_stptr_d(Register rd, Register rj, + int32_t si14) { + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + spew("stptr_d %3s,%3s,0x%x", rd.name(), rj.name(), si14); + return writeInst(InstImm(op_stptr_d, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONG64::as_preld(int32_t hint, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("preld 0x%x,%3s,0x%x", hint, rj.name(), si12); + return writeInst(InstImm(op_preld, si12, rj, hint).encode()); +} + +// Atomic instructions +BufferOffset AssemblerLOONG64::as_amswap_w(Register rd, Register rj, + Register rk) { + spew("amswap_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amswap_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amswap_d(Register rd, Register rj, + Register rk) { + spew("amswap_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amswap_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amadd_w(Register rd, Register rj, + Register rk) { + spew("amadd_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amadd_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amadd_d(Register rd, Register rj, + Register rk) { + spew("amadd_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amadd_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amand_w(Register rd, Register rj, + Register rk) { + spew("amand_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amand_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amand_d(Register rd, Register rj, + Register rk) { + spew("amand_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amand_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amor_w(Register rd, Register rj, + Register rk) { + spew("amor_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amor_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amor_d(Register rd, Register rj, + Register rk) { + spew("amor_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amor_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amxor_w(Register rd, Register rj, + Register rk) { + spew("amxor_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amxor_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amxor_d(Register rd, Register rj, + Register rk) { + spew("amxor_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amxor_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammax_w(Register rd, Register rj, + Register rk) { + spew("ammax_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammax_d(Register rd, Register rj, + Register rk) { + spew("ammax_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammin_w(Register rd, Register rj, + Register rk) { + spew("ammin_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammin_d(Register rd, Register rj, + Register rk) { + spew("ammin_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammax_wu(Register rd, Register rj, + Register rk) { + spew("ammax_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammax_du(Register rd, Register rj, + Register rk) { + spew("ammax_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammin_wu(Register rd, Register rj, + Register rk) { + spew("ammin_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammin_du(Register rd, Register rj, + Register rk) { + spew("ammin_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amswap_db_w(Register rd, Register rj, + Register rk) { + spew("amswap_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amswap_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amswap_db_d(Register rd, Register rj, + Register rk) { + spew("amswap_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amswap_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amadd_db_w(Register rd, Register rj, + Register rk) { + spew("amadd_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amadd_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amadd_db_d(Register rd, Register rj, + Register rk) { + spew("amadd_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amadd_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amand_db_w(Register rd, Register rj, + Register rk) { + spew("amand_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amand_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amand_db_d(Register rd, Register rj, + Register rk) { + spew("amand_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amand_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amor_db_w(Register rd, Register rj, + Register rk) { + spew("amor_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amor_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amor_db_d(Register rd, Register rj, + Register rk) { + spew("amor_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amor_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amxor_db_w(Register rd, Register rj, + Register rk) { + spew("amxor_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amxor_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_amxor_db_d(Register rd, Register rj, + Register rk) { + spew("amxor_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amxor_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammax_db_w(Register rd, Register rj, + Register rk) { + spew("ammax_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammax_db_d(Register rd, Register rj, + Register rk) { + spew("ammax_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammin_db_w(Register rd, Register rj, + Register rk) { + spew("ammin_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammin_db_d(Register rd, Register rj, + Register rk) { + spew("ammin_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammax_db_wu(Register rd, Register rj, + Register rk) { + spew("ammax_db_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_db_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammax_db_du(Register rd, Register rj, + Register rk) { + spew("ammax_db_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_db_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammin_db_wu(Register rd, Register rj, + Register rk) { + spew("ammin_db_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_db_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ammin_db_du(Register rd, Register rj, + Register rk) { + spew("ammin_db_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_db_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ll_w(Register rd, Register rj, int32_t si14) { + spew("ll_w %3s,%3s,0x%x", rd.name(), rj.name(), si14); + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + return writeInst(InstImm(op_ll_w, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONG64::as_ll_d(Register rd, Register rj, int32_t si14) { + spew("ll_d %3s,%3s,0x%x", rd.name(), rj.name(), si14); + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + return writeInst(InstImm(op_ll_d, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONG64::as_sc_w(Register rd, Register rj, int32_t si14) { + spew("sc_w %3s,%3s,0x%x", rd.name(), rj.name(), si14); + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + return writeInst(InstImm(op_sc_w, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONG64::as_sc_d(Register rd, Register rj, int32_t si14) { + spew("sc_d %3s,%3s,0x%x", rd.name(), rj.name(), si14); + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + return writeInst(InstImm(op_sc_d, si14 >> 2, rj, rd, 14).encode()); +} + +// Barrier instructions +BufferOffset AssemblerLOONG64::as_dbar(int32_t hint) { + MOZ_ASSERT(is_uintN(hint, 15)); + spew("dbar 0x%x", hint); + return writeInst(InstImm(op_dbar, hint).encode()); +} + +BufferOffset AssemblerLOONG64::as_ibar(int32_t hint) { + MOZ_ASSERT(is_uintN(hint, 15)); + spew("ibar 0x%x", hint); + return writeInst(InstImm(op_ibar, hint).encode()); +} + +/* =============================================================== */ + +// FP Arithmetic instructions +BufferOffset AssemblerLOONG64::as_fadd_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fadd_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fadd_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fadd_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fadd_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fadd_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fsub_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fsub_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fsub_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fsub_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fsub_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fsub_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmul_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmul_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmul_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmul_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmul_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmul_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fdiv_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fdiv_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fdiv_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fdiv_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fdiv_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fdiv_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmadd_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fmadd_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fmadd_s, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmadd_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fmadd_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fmadd_d, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmsub_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fmsub_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fmsub_s, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmsub_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fmsub_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fmsub_d, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fnmadd_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fnmadd_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fnmadd_s, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fnmadd_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fnmadd_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fnmadd_d, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fnmsub_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fnmsub_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fnmsub_s, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fnmsub_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fnmsub_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fnmsub_d, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmax_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmax_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmax_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmax_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmax_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmax_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmin_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmin_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmin_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmin_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmin_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmin_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmaxa_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmaxa_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmaxa_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmaxa_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmaxa_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmaxa_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmina_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmina_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmina_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmina_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmina_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmina_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fabs_s(FloatRegister fd, FloatRegister fj) { + spew("fabs_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fabs_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fabs_d(FloatRegister fd, FloatRegister fj) { + spew("fabs_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fabs_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fneg_s(FloatRegister fd, FloatRegister fj) { + spew("fneg_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fneg_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fneg_d(FloatRegister fd, FloatRegister fj) { + spew("fneg_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fneg_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fsqrt_s(FloatRegister fd, FloatRegister fj) { + spew("fsqrt_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fsqrt_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fsqrt_d(FloatRegister fd, FloatRegister fj) { + spew("fsqrt_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fsqrt_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fcopysign_s(FloatRegister fd, + FloatRegister fj, + FloatRegister fk) { + spew("fcopysign_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fcopysign_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fcopysign_d(FloatRegister fd, + FloatRegister fj, + FloatRegister fk) { + spew("fcopysign_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fcopysign_d, fk, fj, fd).encode()); +} + +// FP compare instructions +// fcmp.cond.s and fcmp.cond.d instructions +BufferOffset AssemblerLOONG64::as_fcmp_cor(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cor_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, COR, fk, fj, cd).encode()); + } else { + spew("fcmp_cor_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, COR, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_ceq(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_ceq_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CEQ, fk, fj, cd).encode()); + } else { + spew("fcmp_ceq_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CEQ, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_cne(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cne_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CNE, fk, fj, cd).encode()); + } else { + spew("fcmp_cne_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CNE, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_cle(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cle_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CLE, fk, fj, cd).encode()); + } else { + spew("fcmp_cle_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CLE, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_clt(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_clt_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CLT, fk, fj, cd).encode()); + } else { + spew("fcmp_clt_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CLT, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_cun(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cun_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CUN, fk, fj, cd).encode()); + } else { + spew("fcmp_cun_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CUN, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_cueq(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cueq_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CUEQ, fk, fj, cd).encode()); + } else { + spew("fcmp_cueq_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CUEQ, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_cune(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cune_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CUNE, fk, fj, cd).encode()); + } else { + spew("fcmp_cune_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CUNE, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_cule(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cule_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CULE, fk, fj, cd).encode()); + } else { + spew("fcmp_cule_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CULE, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONG64::as_fcmp_cult(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cult_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CULT, fk, fj, cd).encode()); + } else { + spew("fcmp_cult_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CULT, fk, fj, cd).encode()); + } +} + +// FP conversion instructions +BufferOffset AssemblerLOONG64::as_fcvt_s_d(FloatRegister fd, FloatRegister fj) { + spew("fcvt_s_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fcvt_s_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fcvt_d_s(FloatRegister fd, FloatRegister fj) { + spew("fcvt_d_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fcvt_d_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ffint_s_w(FloatRegister fd, + FloatRegister fj) { + spew("ffint_s_w %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ffint_s_w, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ffint_s_l(FloatRegister fd, + FloatRegister fj) { + spew("ffint_s_l %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ffint_s_l, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ffint_d_w(FloatRegister fd, + FloatRegister fj) { + spew("ffint_d_w %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ffint_d_w, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ffint_d_l(FloatRegister fd, + FloatRegister fj) { + spew("ffint_d_l %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ffint_d_l, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftint_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftint_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftint_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftint_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftint_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftint_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftint_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftint_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftint_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftint_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftint_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftint_l_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrm_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrm_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrm_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrm_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrm_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrm_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrm_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrm_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrm_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrm_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrm_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrm_l_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrp_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrp_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrp_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrp_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrp_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrp_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrp_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrp_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrp_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrp_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrp_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrp_l_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrz_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrz_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrz_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrz_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrz_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrz_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrz_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrz_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrz_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrz_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrz_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrz_l_d, fj, fd).encode()); +} +BufferOffset AssemblerLOONG64::as_ftintrne_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrne_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrne_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrne_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrne_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrne_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrne_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrne_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrne_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_ftintrne_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrne_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrne_l_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_frint_s(FloatRegister fd, FloatRegister fj) { + spew("frint_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_frint_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_frint_d(FloatRegister fd, FloatRegister fj) { + spew("frint_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_frint_d, fj, fd).encode()); +} + +// FP mov instructions +BufferOffset AssemblerLOONG64::as_fmov_s(FloatRegister fd, FloatRegister fj) { + spew("fmov_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fmov_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fmov_d(FloatRegister fd, FloatRegister fj) { + spew("fmov_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fmov_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fsel(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FPConditionBit ca) { + spew("fsel %3s,%3s,%3s,%d", fd.name(), fj.name(), fk.name(), ca); + return writeInst(InstReg(op_fsel, ca, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movgr2fr_w(FloatRegister fd, Register rj) { + spew("movgr2fr_w %3s,%3s", fd.name(), rj.name()); + return writeInst(InstReg(op_movgr2fr_w, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movgr2fr_d(FloatRegister fd, Register rj) { + spew("movgr2fr_d %3s,%3s", fd.name(), rj.name()); + return writeInst(InstReg(op_movgr2fr_d, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movgr2frh_w(FloatRegister fd, Register rj) { + spew("movgr2frh_w %3s,%3s", fd.name(), rj.name()); + return writeInst(InstReg(op_movgr2frh_w, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movfr2gr_s(Register rd, FloatRegister fj) { + spew("movfr2gr_s %3s,%3s", rd.name(), fj.name()); + return writeInst(InstReg(op_movfr2gr_s, fj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movfr2gr_d(Register rd, FloatRegister fj) { + spew("movfr2gr_d %3s,%3s", rd.name(), fj.name()); + return writeInst(InstReg(op_movfr2gr_d, fj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movfrh2gr_s(Register rd, FloatRegister fj) { + spew("movfrh2gr_s %3s,%3s", rd.name(), fj.name()); + return writeInst(InstReg(op_movfrh2gr_s, fj, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movgr2fcsr(Register rj) { + spew("movgr2fcsr %3s", rj.name()); + return writeInst(InstReg(op_movgr2fcsr, rj, FCSR).encode()); +} + +BufferOffset AssemblerLOONG64::as_movfcsr2gr(Register rd) { + spew("movfcsr2gr %3s", rd.name()); + return writeInst(InstReg(op_movfcsr2gr, FCSR, rd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movfr2cf(FPConditionBit cd, + FloatRegister fj) { + spew("movfr2cf %d,%3s", cd, fj.name()); + return writeInst(InstReg(op_movfr2cf, fj, cd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movcf2fr(FloatRegister fd, + FPConditionBit cj) { + spew("movcf2fr %3s,%d", fd.name(), cj); + return writeInst(InstReg(op_movcf2fr, cj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movgr2cf(FPConditionBit cd, Register rj) { + spew("movgr2cf %d,%3s", cd, rj.name()); + return writeInst(InstReg(op_movgr2cf, rj, cd).encode()); +} + +BufferOffset AssemblerLOONG64::as_movcf2gr(Register rd, FPConditionBit cj) { + spew("movcf2gr %3s,%d", rd.name(), cj); + return writeInst(InstReg(op_movcf2gr, cj, rd).encode()); +} + +// FP load/store instructions +BufferOffset AssemblerLOONG64::as_fld_s(FloatRegister fd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("fld_s %3s,%3s,0x%x", fd.name(), rj.name(), si12); + return writeInst(InstImm(op_fld_s, si12, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fld_d(FloatRegister fd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("fld_d %3s,%3s,0x%x", fd.name(), rj.name(), si12); + return writeInst(InstImm(op_fld_d, si12, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fst_s(FloatRegister fd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("fst_s %3s,%3s,0x%x", fd.name(), rj.name(), si12); + return writeInst(InstImm(op_fst_s, si12, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fst_d(FloatRegister fd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("fst_d %3s,%3s,0x%x", fd.name(), rj.name(), si12); + return writeInst(InstImm(op_fst_d, si12, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fldx_s(FloatRegister fd, Register rj, + Register rk) { + spew("fldx_s %3s,%3s,%3s", fd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_fldx_s, rk, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fldx_d(FloatRegister fd, Register rj, + Register rk) { + spew("fldx_d %3s,%3s,%3s", fd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_fldx_d, rk, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fstx_s(FloatRegister fd, Register rj, + Register rk) { + spew("fstx_s %3s,%3s,%3s", fd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_fstx_s, rk, rj, fd).encode()); +} + +BufferOffset AssemblerLOONG64::as_fstx_d(FloatRegister fd, Register rj, + Register rk) { + spew("fstx_d %3s,%3s,%3s", fd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_fstx_d, rk, rj, fd).encode()); +} + +/* ========================================================================= */ + +void AssemblerLOONG64::bind(Label* label, BufferOffset boff) { + spew(".set Llabel %p", label); + // If our caller didn't give us an explicit target to bind to + // then we want to bind to the location of the next instruction + BufferOffset dest = boff.assigned() ? boff : nextOffset(); + if (label->used()) { + int32_t next; + + // A used label holds a link to branch that uses it. + BufferOffset b(label); + do { + // Even a 0 offset may be invalid if we're out of memory. + if (oom()) { + return; + } + + Instruction* inst = editSrc(b); + + // Second word holds a pointer to the next branch in label's chain. + next = inst[1].encode(); + bind(reinterpret_cast(inst), b.getOffset(), dest.getOffset()); + + b = BufferOffset(next); + } while (next != LabelBase::INVALID_OFFSET); + } + label->bind(dest.getOffset()); +} + +void AssemblerLOONG64::retarget(Label* label, Label* target) { + spew("retarget %p -> %p", label, target); + if (label->used() && !oom()) { + if (target->bound()) { + bind(label, BufferOffset(target)); + } else if (target->used()) { + // The target is not bound but used. Prepend label's branch list + // onto target's. + int32_t next; + BufferOffset labelBranchOffset(label); + + // Find the head of the use chain for label. + do { + Instruction* inst = editSrc(labelBranchOffset); + + // Second word holds a pointer to the next branch in chain. + next = inst[1].encode(); + labelBranchOffset = BufferOffset(next); + } while (next != LabelBase::INVALID_OFFSET); + + // Then patch the head of label's use chain to the tail of + // target's use chain, prepending the entire use chain of target. + Instruction* inst = editSrc(labelBranchOffset); + int32_t prev = target->offset(); + target->use(label->offset()); + inst[1].setData(prev); + } else { + // The target is unbound and unused. We can just take the head of + // the list hanging off of label, and dump that into target. + target->use(label->offset()); + } + } + label->reset(); +} + +void dbg_break() {} + +void AssemblerLOONG64::as_break(uint32_t code) { + MOZ_ASSERT(code <= MAX_BREAK_CODE); + spew("break %d", code); + writeInst(InstImm(op_break, code).encode()); +} + +// This just stomps over memory with 32 bits of raw data. Its purpose is to +// overwrite the call of JITed code with 32 bits worth of an offset. This will +// is only meant to function on code that has been invalidated, so it should +// be totally safe. Since that instruction will never be executed again, a +// ICache flush should not be necessary +void AssemblerLOONG64::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) { + // Raw is going to be the return address. + uint32_t* raw = (uint32_t*)label.raw(); + // Overwrite the 4 bytes before the return address, which will + // end up being the call instruction. + *(raw - 1) = imm.value; +} + +uint8_t* AssemblerLOONG64::NextInstruction(uint8_t* inst_, uint32_t* count) { + Instruction* inst = reinterpret_cast(inst_); + if (count != nullptr) { + *count += sizeof(Instruction); + } + return reinterpret_cast(inst->next()); +} + +void AssemblerLOONG64::ToggleToJmp(CodeLocationLabel inst_) { + InstImm* inst = (InstImm*)inst_.raw(); + + MOZ_ASSERT(inst->extractBitField(31, 26) == (uint32_t)op_addu16i_d >> 26); + // We converted beq to addu16i_d, so now we restore it. + inst->setOpcode(op_beq, 6); +} + +void AssemblerLOONG64::ToggleToCmp(CodeLocationLabel inst_) { + InstImm* inst = (InstImm*)inst_.raw(); + + // toggledJump is allways used for short jumps. + MOZ_ASSERT(inst->extractBitField(31, 26) == (uint32_t)op_beq >> 26); + // Replace "beq $zero, $zero, offset" with "andu16i_d $zero, $zero, offset" + inst->setOpcode(op_addu16i_d, 6); +} + +// Since there are no pools in LA implementation, this should be simple. +Instruction* Instruction::next() { return this + 1; } + +InstImm AssemblerLOONG64::invertBranch(InstImm branch, BOffImm16 skipOffset) { + uint32_t rj = 0; + OpcodeField opcode = (OpcodeField)((branch.extractBitField(31, 26)) << 26); + switch (opcode) { + case op_beq: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bne, 6); + return branch; + case op_bne: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_beq, 6); + return branch; + case op_bge: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_blt, 6); + return branch; + case op_bgeu: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bltu, 6); + return branch; + case op_blt: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bge, 6); + return branch; + case op_bltu: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bgeu, 6); + return branch; + case op_beqz: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bnez, 6); + return branch; + case op_bnez: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_beqz, 6); + return branch; + case op_bcz: + branch.setBOffImm16(skipOffset); + rj = branch.extractRJ(); + if (rj & 0x8) { + branch.setRJ(rj & 0x17); + } else { + branch.setRJ(rj | 0x8); + } + return branch; + default: + MOZ_CRASH("Error creating long branch."); + } +} + +#ifdef JS_JITSPEW +void AssemblerLOONG64::decodeBranchInstAndSpew(InstImm branch) { + OpcodeField opcode = (OpcodeField)((branch.extractBitField(31, 26)) << 26); + uint32_t rd_id; + uint32_t rj_id; + uint32_t cj_id; + uint32_t immi = branch.extractImm16Value(); + switch (opcode) { + case op_beq: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("beq 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_bne: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bne 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_bge: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bge 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_bgeu: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bgeu 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_blt: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("blt 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_bltu: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bltu 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_beqz: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("beqz 0x%x,%3s,0x%x", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), rd_id); + break; + case op_bnez: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bnez 0x%x,%3s,0x%x", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), rd_id); + break; + case op_bcz: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + cj_id = branch.extractBitField(CJShift + CJBits - 1, CJShift); + if (rj_id & 0x8) { + spew("bcnez 0x%x,FCC%d,0x%x", (int32_t(immi << 18) >> 16) + 4, cj_id, + rd_id); + } else { + spew("bceqz 0x%x,FCC%d,0x%x", (int32_t(immi << 18) >> 16) + 4, cj_id, + rd_id); + } + break; + case op_jirl: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("beqz 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + default: + MOZ_CRASH("Error disassemble branch."); + } +} +#endif + +ABIArg ABIArgGenerator::next(MIRType type) { + // TODO:LOONG64 ABIArg + switch (type) { + case MIRType::Int32: + case MIRType::Int64: + case MIRType::Pointer: + case MIRType::RefOrNull: + case MIRType::StackResults: { + if (intRegIndex_ == NumIntArgRegs) { + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(uintptr_t); + break; + } + // a0 == r4 + current_ = ABIArg(Register::FromCode(intRegIndex_ + 4)); + intRegIndex_++; + break; + } + case MIRType::Float32: + case MIRType::Double: { + if (floatRegIndex_ == NumFloatArgRegs) { + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(double); + break; + } + current_ = ABIArg(FloatRegister(FloatRegisters::Encoding(floatRegIndex_), + type == MIRType::Double + ? FloatRegisters::Double + : FloatRegisters::Single)); + floatRegIndex_++; + break; + } + case MIRType::Simd128: { + MOZ_CRASH("LoongArch does not support simd yet."); + break; + } + default: + MOZ_CRASH("Unexpected argument type"); + } + return current_; +} + +uint32_t js::jit::RK(FloatRegister r) { return r.encoding() << RKShift; } + +uint32_t js::jit::RD(FloatRegister r) { return r.encoding() << RDShift; } + +uint32_t js::jit::RJ(FloatRegister r) { return r.encoding() << RJShift; } + +uint32_t js::jit::FA(FloatRegister r) { return r.encoding() << FAShift; } + +uint32_t js::jit::SA2(FloatRegister r) { + return (r.encoding() & SA2Mask) << SAShift; +} + +uint32_t js::jit::SA3(FloatRegister r) { + return (r.encoding() & SA3Mask) << SAShift; +} + +void Assembler::executableCopy(uint8_t* buffer) { + MOZ_ASSERT(isFinished); + m_buffer.executableCopy(buffer); +} + +uintptr_t Assembler::GetPointer(uint8_t* instPtr) { + Instruction* inst = (Instruction*)instPtr; + return Assembler::ExtractLoad64Value(inst); +} + +static JitCode* CodeFromJump(Instruction* jump) { + uint8_t* target = (uint8_t*)Assembler::ExtractLoad64Value(jump); + return JitCode::FromExecutable(target); +} + +void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + while (reader.more()) { + JitCode* child = + CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned())); + TraceManuallyBarrieredEdge(trc, &child, "rel32"); + } +} + +static void TraceOneDataRelocation(JSTracer* trc, + mozilla::Maybe& awjc, + JitCode* code, Instruction* inst) { + void* ptr = (void*)Assembler::ExtractLoad64Value(inst); + void* prior = ptr; + + // Data relocations can be for Values or for raw pointers. If a Value is + // zero-tagged, we can trace it as if it were a raw pointer. If a Value + // is not zero-tagged, we have to interpret it as a Value to ensure that the + // tag bits are masked off to recover the actual pointer. + uintptr_t word = reinterpret_cast(ptr); + if (word >> JSVAL_TAG_SHIFT) { + // This relocation is a Value with a non-zero tag. + Value v = Value::fromRawBits(word); + TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value"); + ptr = (void*)v.bitsAsPunboxPointer(); + } else { + // This relocation is a raw pointer or a Value with a zero tag. + // No barrier needed since these are constants. + TraceManuallyBarrieredGenericPointerEdge( + trc, reinterpret_cast(&ptr), "jit-masm-ptr"); + } + + if (ptr != prior) { + if (awjc.isNothing()) { + awjc.emplace(code); + } + Assembler::UpdateLoad64Value(inst, uint64_t(ptr)); + } +} + +/* static */ +void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + mozilla::Maybe awjc; + while (reader.more()) { + size_t offset = reader.readUnsigned(); + Instruction* inst = (Instruction*)(code->raw() + offset); + TraceOneDataRelocation(trc, awjc, code, inst); + } +} + +void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) { + if (label.patchAt().bound()) { + auto mode = label.linkMode(); + intptr_t offset = label.patchAt().offset(); + intptr_t target = label.target().offset(); + + if (mode == CodeLabel::RawPointer) { + *reinterpret_cast(rawCode + offset) = rawCode + target; + } else { + MOZ_ASSERT(mode == CodeLabel::MoveImmediate || + mode == CodeLabel::JumpImmediate); + Instruction* inst = (Instruction*)(rawCode + offset); + Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + target)); + } + } +} + +void Assembler::bind(InstImm* inst, uintptr_t branch, uintptr_t target) { + int64_t offset = target - branch; + InstImm inst_jirl = InstImm(op_jirl, BOffImm16(0), zero, ra); + InstImm inst_beq = InstImm(op_beq, BOffImm16(0), zero, zero); + + // If encoded offset is 4, then the jump must be short + if (BOffImm16(inst[0]).decode() == 4) { + // TODO //hwj + MOZ_ASSERT(BOffImm16::IsInRange(offset)); + inst[0].setBOffImm16(BOffImm16(offset)); + inst[1].makeNop(); // because before set INVALID_OFFSET + return; + } + + // Generate the long jump for calls because return address has to be the + // address after the reserved block. + if (inst[0].encode() == inst_jirl.encode()) { + addLongJump(BufferOffset(branch), BufferOffset(target)); + Assembler::WriteLoad64Instructions(inst, ScratchRegister, + LabelBase::INVALID_OFFSET); + inst[3].makeNop(); // There are 1 nop. + inst[4] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); + return; + } + + if (BOffImm16::IsInRange(offset)) { + // Skip trailing nops . + bool skipNops = (inst[0].encode() != inst_jirl.encode() && + inst[0].encode() != inst_beq.encode()); + + inst[0].setBOffImm16(BOffImm16(offset)); + inst[1].makeNop(); + + if (skipNops) { + inst[2] = InstImm(op_bge, BOffImm16(3 * sizeof(uint32_t)), zero, zero); + // There are 2 nops after this + } + return; + } + + if (inst[0].encode() == inst_beq.encode()) { + // Handle long unconditional jump. Only four 4 instruction. + addLongJump(BufferOffset(branch), BufferOffset(target)); + Assembler::WriteLoad64Instructions(inst, ScratchRegister, + LabelBase::INVALID_OFFSET); + inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, zero); + } else { + // Handle long conditional jump. + inst[0] = invertBranch(inst[0], BOffImm16(5 * sizeof(uint32_t))); + // No need for a "nop" here because we can clobber scratch. + addLongJump(BufferOffset(branch + sizeof(uint32_t)), BufferOffset(target)); + Assembler::WriteLoad64Instructions(&inst[1], ScratchRegister, + LabelBase::INVALID_OFFSET); + inst[4] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, zero); + } +} + +void Assembler::processCodeLabels(uint8_t* rawCode) { + for (const CodeLabel& label : codeLabels_) { + Bind(rawCode, label); + } +} + +uint32_t Assembler::PatchWrite_NearCallSize() { + // Load an address needs 3 instructions, and a jump. + return (3 + 1) * sizeof(uint32_t); +} + +void Assembler::PatchWrite_NearCall(CodeLocationLabel start, + CodeLocationLabel toCall) { + Instruction* inst = (Instruction*)start.raw(); + uint8_t* dest = toCall.raw(); + + // Overwrite whatever instruction used to be here with a call. + // Always use long jump for two reasons: + // - Jump has to be the same size because of PatchWrite_NearCallSize. + // - Return address has to be at the end of replaced block. + // Short jump wouldn't be more efficient. + Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)dest); + inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); +} + +uint64_t Assembler::ExtractLoad64Value(Instruction* inst0) { + InstImm* i0 = (InstImm*)inst0; + InstImm* i1 = (InstImm*)i0->next(); + InstImm* i2 = (InstImm*)i1->next(); + InstImm* i3 = (InstImm*)i2->next(); + + MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25)); + MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22)); + MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25)); + + if ((i3->extractBitField(31, 22)) == ((uint32_t)op_lu52i_d >> 22)) { + // Li64 + uint64_t value = + (uint64_t(i0->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift)) + << 12) | + (uint64_t( + i1->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift))) | + (uint64_t(i2->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift)) + << 32) | + (uint64_t(i3->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift)) + << 52); + return value; + } else { + // Li48 + uint64_t value = + (uint64_t(i0->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift)) + << 12) | + (uint64_t( + i1->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift))) | + (uint64_t(i2->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift)) + << 32); + + return uint64_t((int64_t(value) << 16) >> 16); + } +} + +void Assembler::UpdateLoad64Value(Instruction* inst0, uint64_t value) { + // Todo: with ma_liPatchable + InstImm* i0 = (InstImm*)inst0; + InstImm* i1 = (InstImm*)i0->next(); + InstImm* i2 = (InstImm*)i1->next(); + InstImm* i3 = (InstImm*)i2->next(); + + MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25)); + MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22)); + MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25)); + + if ((i3->extractBitField(31, 22)) == ((uint32_t)op_lu52i_d >> 22)) { + // Li64 + *i0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff), + Register::FromCode(i0->extractRD()), false); + *i1 = InstImm(op_ori, (int32_t)(value & 0xfff), + Register::FromCode(i1->extractRJ()), + Register::FromCode(i1->extractRD()), 12); + *i2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff), + Register::FromCode(i2->extractRD()), false); + *i3 = InstImm(op_lu52i_d, (int32_t)((value >> 52) & 0xfff), + Register::FromCode(i3->extractRJ()), + Register::FromCode(i3->extractRD()), 12); + } else { + // Li48 + *i0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff), + Register::FromCode(i0->extractRD()), false); + *i1 = InstImm(op_ori, (int32_t)(value & 0xfff), + Register::FromCode(i1->extractRJ()), + Register::FromCode(i1->extractRD()), 12); + *i2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff), + Register::FromCode(i2->extractRD()), false); + } +} + +void Assembler::WriteLoad64Instructions(Instruction* inst0, Register reg, + uint64_t value) { + Instruction* inst1 = inst0->next(); + Instruction* inst2 = inst1->next(); + *inst0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff), reg, false); + *inst1 = InstImm(op_ori, (int32_t)(value & 0xfff), reg, reg, 12); + *inst2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff), reg, false); +} + +void Assembler::PatchDataWithValueCheck(CodeLocationLabel label, + ImmPtr newValue, ImmPtr expectedValue) { + PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value), + PatchedImmPtr(expectedValue.value)); +} + +void Assembler::PatchDataWithValueCheck(CodeLocationLabel label, + PatchedImmPtr newValue, + PatchedImmPtr expectedValue) { + Instruction* inst = (Instruction*)label.raw(); + + // Extract old Value + DebugOnly value = Assembler::ExtractLoad64Value(inst); + MOZ_ASSERT(value == uint64_t(expectedValue.value)); + + // Replace with new value + Assembler::UpdateLoad64Value(inst, uint64_t(newValue.value)); +} + +uint64_t Assembler::ExtractInstructionImmediate(uint8_t* code) { + InstImm* inst = (InstImm*)code; + return Assembler::ExtractLoad64Value(inst); +} + +void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) { + Instruction* inst = (Instruction*)inst_.raw(); + InstImm* i0 = (InstImm*)inst; + InstImm* i1 = (InstImm*)i0->next(); + InstImm* i2 = (InstImm*)i1->next(); + Instruction* i3 = (Instruction*)i2->next(); + + MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25)); + MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22)); + MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25)); + + if (enabled) { + MOZ_ASSERT((i3->extractBitField(31, 25)) != ((uint32_t)op_lu12i_w >> 25)); + InstImm jirl = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); + *i3 = jirl; + } else { + InstNOP nop; + *i3 = nop; + } +} diff --git a/js/src/jit/loong64/Assembler-loong64.h b/js/src/jit/loong64/Assembler-loong64.h new file mode 100644 index 0000000000..8024f3063d --- /dev/null +++ b/js/src/jit/loong64/Assembler-loong64.h @@ -0,0 +1,1956 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_Assembler_loong64_h +#define jit_loong64_Assembler_loong64_h + +#include "mozilla/Sprintf.h" +#include + +#include "jit/CompactBuffer.h" +#include "jit/JitCode.h" +#include "jit/JitSpewer.h" +#include "jit/loong64/Architecture-loong64.h" +#include "jit/shared/Assembler-shared.h" +#include "jit/shared/Disassembler-shared.h" +#include "jit/shared/IonAssemblerBuffer.h" +#include "wasm/WasmTypes.h" + +namespace js { +namespace jit { + +static constexpr Register zero{Registers::zero}; +static constexpr Register ra{Registers::ra}; +static constexpr Register tp{Registers::tp}; +static constexpr Register sp{Registers::sp}; +static constexpr Register a0{Registers::a0}; +static constexpr Register a1{Registers::a1}; +static constexpr Register a2{Registers::a2}; +static constexpr Register a3{Registers::a3}; +static constexpr Register a4{Registers::a4}; +static constexpr Register a5{Registers::a5}; +static constexpr Register a6{Registers::a6}; +static constexpr Register a7{Registers::a7}; +static constexpr Register t0{Registers::t0}; +static constexpr Register t1{Registers::t1}; +static constexpr Register t2{Registers::t2}; +static constexpr Register t3{Registers::t3}; +static constexpr Register t4{Registers::t4}; +static constexpr Register t5{Registers::t5}; +static constexpr Register t6{Registers::t6}; +static constexpr Register t7{Registers::t7}; +static constexpr Register t8{Registers::t8}; +static constexpr Register rx{Registers::rx}; +static constexpr Register fp{Registers::fp}; +static constexpr Register s0{Registers::s0}; +static constexpr Register s1{Registers::s1}; +static constexpr Register s2{Registers::s2}; +static constexpr Register s3{Registers::s3}; +static constexpr Register s4{Registers::s4}; +static constexpr Register s5{Registers::s5}; +static constexpr Register s6{Registers::s6}; +static constexpr Register s7{Registers::s7}; +static constexpr Register s8{Registers::s8}; + +static constexpr Register StackPointer = sp; +static constexpr Register FramePointer = fp; +static constexpr Register JSReturnReg = a2; + +static constexpr Register ScratchRegister = t6; +static constexpr Register SecondScratchReg = t7; + +// Helper classes for ScratchRegister usage. Asserts that only one piece +// of code thinks it has exclusive ownership of each scratch register. +struct ScratchRegisterScope : public AutoRegisterScope { + explicit ScratchRegisterScope(MacroAssembler& masm) + : AutoRegisterScope(masm, ScratchRegister) {} +}; + +struct SecondScratchRegisterScope : public AutoRegisterScope { + explicit SecondScratchRegisterScope(MacroAssembler& masm) + : AutoRegisterScope(masm, SecondScratchReg) {} +}; + +static constexpr Register InvalidReg{Registers::Invalid}; +static constexpr FloatRegister InvalidFloatReg = {}; + +// Use arg reg from EnterJIT function as OsrFrameReg. +static constexpr Register OsrFrameReg = a3; +static constexpr Register CallTempReg0 = t0; +static constexpr Register CallTempReg1 = t1; +static constexpr Register CallTempReg2 = t2; +static constexpr Register CallTempReg3 = t3; + +static constexpr Register PreBarrierReg = a1; + +static constexpr Register ReturnReg = a0; +static constexpr Register64 ReturnReg64(ReturnReg); + +static constexpr Register IntArgReg0 = a0; +static constexpr Register IntArgReg1 = a1; +static constexpr Register IntArgReg2 = a2; +static constexpr Register IntArgReg3 = a3; +static constexpr Register IntArgReg4 = a4; +static constexpr Register IntArgReg5 = a5; +static constexpr Register IntArgReg6 = a6; +static constexpr Register IntArgReg7 = a7; +static constexpr Register HeapReg = s7; + +static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg; +static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg; + +static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg); + +// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand). +static constexpr Register RegExpMatcherRegExpReg = CallTempReg0; +static constexpr Register RegExpMatcherStringReg = CallTempReg1; +static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2; + +// Registerd used in RegExpTester instruction (do not use ReturnReg). +static constexpr Register RegExpTesterRegExpReg = CallTempReg0; +static constexpr Register RegExpTesterStringReg = CallTempReg1; +static constexpr Register RegExpTesterLastIndexReg = CallTempReg2; + +// TODO:LOONG64 Recheck +static constexpr Register JSReturnReg_Type = a3; +static constexpr Register JSReturnReg_Data = a2; + +static constexpr FloatRegister NANReg = {FloatRegisters::f14, + FloatRegisters::Single}; + +/* clang-format off */ +// TODO:LOONG64 Instruction Types +// LA instruction types +// +---------------------------------------------------------------+ +// | 6 | 5 | 5 | 5 | 5 | 6 | +// +---------------------------------------------------------------+ +// Register type | Opcode | Rs | Rt | Rd | Sa | Function | +// +---------------------------------------------------------------+ +// | 6 | 5 | 5 | 16 | +// +---------------------------------------------------------------+ +// Immediate type | Opcode | Rs | Rt | 2's complement constant | +// +---------------------------------------------------------------+ +// | 6 | 26 | +// +---------------------------------------------------------------+ +// Jump type | Opcode | jump_target | +// +---------------------------------------------------------------+ +// 31 bit bit 0 +/* clang-format on */ + +// LA instruction encoding constants. +static const uint32_t RKShift = 10; +static const uint32_t RKBits = 5; +static const uint32_t RJShift = 5; +static const uint32_t RJBits = 5; +static const uint32_t RDShift = 0; +static const uint32_t RDBits = 5; +static const uint32_t SAShift = 15; +static const uint32_t SA2Bits = 2; +static const uint32_t SA3Bits = 3; +static const uint32_t Imm5Shift = 10; +static const uint32_t Imm5Bits = 5; +static const uint32_t Imm6Shift = 10; +static const uint32_t Imm6Bits = 6; +static const uint32_t Imm12Shift = 10; +static const uint32_t Imm12Bits = 12; +static const uint32_t Imm14Shift = 10; +static const uint32_t Imm14Bits = 14; +static const uint32_t Imm15Shift = 0; +static const uint32_t Imm15Bits = 15; +static const uint32_t Imm16Shift = 10; +static const uint32_t Imm16Bits = 16; +static const uint32_t Imm20Shift = 5; +static const uint32_t Imm20Bits = 20; +static const uint32_t Imm21Shift = 0; +static const uint32_t Imm21Bits = 21; +static const uint32_t Imm26Shift = 0; +static const uint32_t Imm26Bits = 26; +static const uint32_t ImmFieldShift = 2; +static const uint32_t CODEShift = 0; +static const uint32_t CODEBits = 15; + +static const uint32_t MSBWShift = 16; +static const uint32_t MSBWBits = 5; +static const uint32_t MSBDShift = 16; +static const uint32_t MSBDBits = 6; + +static const uint32_t LSBBits = 6; +static const uint32_t LSBWShift = 10; +static const uint32_t LSBWBits = 5; +static const uint32_t LSBDShift = 10; +static const uint32_t LSBDBits = 6; + +static const uint32_t FKShift = 10; +static const uint32_t FKBits = 5; +static const uint32_t FJShift = 5; +static const uint32_t FJBits = 5; +static const uint32_t FDShift = 0; +static const uint32_t FDBits = 5; +static const uint32_t FAShift = 15; +static const uint32_t FABits = 5; +static const uint32_t CAShift = 15; +static const uint32_t CABits = 3; +static const uint32_t CDShift = 0; +static const uint32_t CDBits = 3; +static const uint32_t CJShift = 5; +static const uint32_t CJBits = 3; +static const uint32_t CONDShift = 15; +static const uint32_t CONDBits = 5; + +// LA instruction field bit masks. +static const uint32_t Imm5Mask = (1 << Imm5Bits) - 1; +static const uint32_t Imm6Mask = (1 << Imm6Bits) - 1; +static const uint32_t Imm12Mask = (1 << Imm12Bits) - 1; +static const uint32_t Imm14Mask = (1 << Imm14Bits) - 1; +static const uint32_t Imm15Mask = (1 << Imm15Bits) - 1; +static const uint32_t Imm16Mask = (1 << Imm16Bits) - 1; +static const uint32_t BOffImm16Mask = ((1 << Imm16Bits) - 1) << Imm16Shift; +static const uint32_t BOffImm21Mask = ((1 << Imm21Bits) - 1) << Imm21Shift; +static const uint32_t BOffImm26Mask = ((1 << Imm26Bits) - 1) << Imm26Shift; +static const uint32_t Imm20Mask = (1 << Imm20Bits) - 1; +static const uint32_t Imm21Mask = (1 << Imm21Bits) - 1; +static const uint32_t Imm26Mask = (1 << Imm26Bits) - 1; +static const uint32_t CONDMask = (1 << CONDBits) - 1; +static const uint32_t MSBWMask = (1 << MSBWBits) - 1; +static const uint32_t MSBDMask = (1 << MSBDBits) - 1; +static const uint32_t LSBMask = (1 << LSBBits) - 1; +static const uint32_t LSBWMask = (1 << LSBWBits) - 1; +static const uint32_t LSBDMask = (1 << LSBDBits) - 1; +static const uint32_t RKMask = (1 << RKBits) - 1; +static const uint32_t RJMask = (1 << RJBits) - 1; +static const uint32_t RDMask = (1 << RDBits) - 1; +static const uint32_t SA2Mask = (1 << SA2Bits) - 1; +static const uint32_t SA3Mask = (1 << SA3Bits) - 1; +static const uint32_t RegMask = Registers::Total - 1; + +// TODO:LOONG64 Change to syscall? +static const uint32_t MAX_BREAK_CODE = 1024 - 1; +static const uint32_t WASM_TRAP = 6; // BRK_OVERFLOW + +class Instruction; +class InstReg; +class InstImm; +class InstJump; + +uint32_t RK(Register r); +uint32_t RK(FloatRegister r); +uint32_t RJ(Register r); +uint32_t RJ(FloatRegister r); +uint32_t RD(Register r); +uint32_t RD(FloatRegister r); +uint32_t FA(FloatRegister r); +uint32_t SA2(uint32_t value); +uint32_t SA2(FloatRegister r); +uint32_t SA3(uint32_t value); +uint32_t SA3(FloatRegister r); + +Register toRK(Instruction& i); +Register toRJ(Instruction& i); +Register toRD(Instruction& i); +Register toR(Instruction& i); + +// LA enums for instruction fields +enum OpcodeField { + op_beqz = 0x10U << 26, + op_bnez = 0x11U << 26, + op_bcz = 0x12U << 26, // bceqz & bcnez + op_jirl = 0x13U << 26, + op_b = 0x14U << 26, + op_bl = 0x15U << 26, + op_beq = 0x16U << 26, + op_bne = 0x17U << 26, + op_blt = 0x18U << 26, + op_bge = 0x19U << 26, + op_bltu = 0x1aU << 26, + op_bgeu = 0x1bU << 26, + + op_addu16i_d = 0x4U << 26, + + op_lu12i_w = 0xaU << 25, + op_lu32i_d = 0xbU << 25, + op_pcaddi = 0xcU << 25, + op_pcalau12i = 0xdU << 25, + op_pcaddu12i = 0xeU << 25, + op_pcaddu18i = 0xfU << 25, + op_ll_w = 0x20U << 24, + op_sc_w = 0x21U << 24, + op_ll_d = 0x22U << 24, + op_sc_d = 0x23U << 24, + op_ldptr_w = 0x24U << 24, + op_stptr_w = 0x25U << 24, + op_ldptr_d = 0x26U << 24, + op_stptr_d = 0x27U << 24, + op_bstrins_d = 0x2U << 22, + op_bstrpick_d = 0x3U << 22, + op_slti = 0x8U << 22, + op_sltui = 0x9U << 22, + op_addi_w = 0xaU << 22, + op_addi_d = 0xbU << 22, + op_lu52i_d = 0xcU << 22, + op_andi = 0xdU << 22, + op_ori = 0xeU << 22, + op_xori = 0xfU << 22, + op_ld_b = 0xa0U << 22, + op_ld_h = 0xa1U << 22, + op_ld_w = 0xa2U << 22, + op_ld_d = 0xa3U << 22, + op_st_b = 0xa4U << 22, + op_st_h = 0xa5U << 22, + op_st_w = 0xa6U << 22, + op_st_d = 0xa7U << 22, + op_ld_bu = 0xa8U << 22, + op_ld_hu = 0xa9U << 22, + op_ld_wu = 0xaaU << 22, + op_preld = 0xabU << 22, + op_fld_s = 0xacU << 22, + op_fst_s = 0xadU << 22, + op_fld_d = 0xaeU << 22, + op_fst_d = 0xafU << 22, + op_bstr_w = 0x3U << 21, // BSTRINS_W & BSTRPICK_W + op_fmadd_s = 0x81U << 20, + op_fmadd_d = 0x82U << 20, + op_fmsub_s = 0x85U << 20, + op_fmsub_d = 0x86U << 20, + op_fnmadd_s = 0x89U << 20, + op_fnmadd_d = 0x8aU << 20, + op_fnmsub_s = 0x8dU << 20, + op_fnmsub_d = 0x8eU << 20, + op_fcmp_cond_s = 0xc1U << 20, + op_fcmp_cond_d = 0xc2U << 20, + + op_bytepick_d = 0x3U << 18, + op_fsel = 0x340U << 18, + + op_bytepick_w = 0x4U << 17, + op_alsl_w = 0x2U << 17, + op_alsl_wu = 0x3U << 17, + op_alsl_d = 0x16U << 17, + + op_slli_d = 0x41U << 16, + op_srli_d = 0x45U << 16, + op_srai_d = 0x49U << 16, + + op_slli_w = 0x81U << 15, + op_srli_w = 0x89U << 15, + op_srai_w = 0x91U << 15, + op_add_w = 0x20U << 15, + op_add_d = 0x21U << 15, + op_sub_w = 0x22U << 15, + op_sub_d = 0x23U << 15, + op_slt = 0x24U << 15, + op_sltu = 0x25U << 15, + op_maskeqz = 0x26U << 15, + op_masknez = 0x27U << 15, + op_nor = 0x28U << 15, + op_and = 0x29U << 15, + op_or = 0x2aU << 15, + op_xor = 0x2bU << 15, + op_orn = 0x2cU << 15, + op_andn = 0x2dU << 15, + op_sll_w = 0x2eU << 15, + op_srl_w = 0x2fU << 15, + op_sra_w = 0x30U << 15, + op_sll_d = 0x31U << 15, + op_srl_d = 0x32U << 15, + op_sra_d = 0x33U << 15, + op_rotr_w = 0x36U << 15, + op_rotr_d = 0x37U << 15, + op_rotri_w = 0x99U << 15, + op_rotri_d = 0x4DU << 16, + op_mul_w = 0x38U << 15, + op_mulh_w = 0x39U << 15, + op_mulh_wu = 0x3aU << 15, + op_mul_d = 0x3bU << 15, + op_mulh_d = 0x3cU << 15, + op_mulh_du = 0x3dU << 15, + op_mulw_d_w = 0x3eU << 15, + op_mulw_d_wu = 0x3fU << 15, + op_div_w = 0x40U << 15, + op_mod_w = 0x41U << 15, + op_div_wu = 0x42U << 15, + op_mod_wu = 0x43U << 15, + op_div_d = 0x44U << 15, + op_mod_d = 0x45U << 15, + op_div_du = 0x46U << 15, + op_mod_du = 0x47U << 15, + op_break = 0x54U << 15, + op_fadd_s = 0x201U << 15, + op_fadd_d = 0x202U << 15, + op_fsub_s = 0x205U << 15, + op_fsub_d = 0x206U << 15, + op_fmul_s = 0x209U << 15, + op_fmul_d = 0x20aU << 15, + op_fdiv_s = 0x20dU << 15, + op_fdiv_d = 0x20eU << 15, + op_fmax_s = 0x211U << 15, + op_fmax_d = 0x212U << 15, + op_fmin_s = 0x215U << 15, + op_fmin_d = 0x216U << 15, + op_fmaxa_s = 0x219U << 15, + op_fmaxa_d = 0x21aU << 15, + op_fmina_s = 0x21dU << 15, + op_fmina_d = 0x21eU << 15, + op_fcopysign_s = 0x225U << 15, + op_fcopysign_d = 0x226U << 15, + op_ldx_b = 0x7000U << 15, + op_ldx_h = 0x7008U << 15, + op_ldx_w = 0x7010U << 15, + op_ldx_d = 0x7018U << 15, + op_stx_b = 0x7020U << 15, + op_stx_h = 0x7028U << 15, + op_stx_w = 0x7030U << 15, + op_stx_d = 0x7038U << 15, + op_ldx_bu = 0x7040U << 15, + op_ldx_hu = 0x7048U << 15, + op_ldx_wu = 0x7050U << 15, + op_fldx_s = 0x7060U << 15, + op_fldx_d = 0x7068U << 15, + op_fstx_s = 0x7070U << 15, + op_fstx_d = 0x7078U << 15, + op_amswap_w = 0x70c0U << 15, + op_amswap_d = 0x70c1U << 15, + op_amadd_w = 0x70c2U << 15, + op_amadd_d = 0x70c3U << 15, + op_amand_w = 0x70c4U << 15, + op_amand_d = 0x70c5U << 15, + op_amor_w = 0x70c6U << 15, + op_amor_d = 0x70c7U << 15, + op_amxor_w = 0x70c8U << 15, + op_amxor_d = 0x70c9U << 15, + op_ammax_w = 0x70caU << 15, + op_ammax_d = 0x70cbU << 15, + op_ammin_w = 0x70ccU << 15, + op_ammin_d = 0x70cdU << 15, + op_ammax_wu = 0x70ceU << 15, + op_ammax_du = 0x70cfU << 15, + op_ammin_wu = 0x70d0U << 15, + op_ammin_du = 0x70d1U << 15, + op_amswap_db_w = 0x70d2U << 15, + op_amswap_db_d = 0x70d3U << 15, + op_amadd_db_w = 0x70d4U << 15, + op_amadd_db_d = 0x70d5U << 15, + op_amand_db_w = 0x70d6U << 15, + op_amand_db_d = 0x70d7U << 15, + op_amor_db_w = 0x70d8U << 15, + op_amor_db_d = 0x70d9U << 15, + op_amxor_db_w = 0x70daU << 15, + op_amxor_db_d = 0x70dbU << 15, + op_ammax_db_w = 0x70dcU << 15, + op_ammax_db_d = 0x70ddU << 15, + op_ammin_db_w = 0x70deU << 15, + op_ammin_db_d = 0x70dfU << 15, + op_ammax_db_wu = 0x70e0U << 15, + op_ammax_db_du = 0x70e1U << 15, + op_ammin_db_wu = 0x70e2U << 15, + op_ammin_db_du = 0x70e3U << 15, + op_dbar = 0x70e4U << 15, + op_ibar = 0x70e5U << 15, + op_clo_w = 0x4U << 10, + op_clz_w = 0x5U << 10, + op_cto_w = 0x6U << 10, + op_ctz_w = 0x7U << 10, + op_clo_d = 0x8U << 10, + op_clz_d = 0x9U << 10, + op_cto_d = 0xaU << 10, + op_ctz_d = 0xbU << 10, + op_revb_2h = 0xcU << 10, + op_revb_4h = 0xdU << 10, + op_revb_2w = 0xeU << 10, + op_revb_d = 0xfU << 10, + op_revh_2w = 0x10U << 10, + op_revh_d = 0x11U << 10, + op_bitrev_4b = 0x12U << 10, + op_bitrev_8b = 0x13U << 10, + op_bitrev_w = 0x14U << 10, + op_bitrev_d = 0x15U << 10, + op_ext_w_h = 0x16U << 10, + op_ext_w_b = 0x17U << 10, + op_fabs_s = 0x4501U << 10, + op_fabs_d = 0x4502U << 10, + op_fneg_s = 0x4505U << 10, + op_fneg_d = 0x4506U << 10, + op_fsqrt_s = 0x4511U << 10, + op_fsqrt_d = 0x4512U << 10, + op_fmov_s = 0x4525U << 10, + op_fmov_d = 0x4526U << 10, + op_movgr2fr_w = 0x4529U << 10, + op_movgr2fr_d = 0x452aU << 10, + op_movgr2frh_w = 0x452bU << 10, + op_movfr2gr_s = 0x452dU << 10, + op_movfr2gr_d = 0x452eU << 10, + op_movfrh2gr_s = 0x452fU << 10, + op_movgr2fcsr = 0x4530U << 10, + op_movfcsr2gr = 0x4532U << 10, + op_movfr2cf = 0x4534U << 10, + op_movgr2cf = 0x4536U << 10, + op_fcvt_s_d = 0x4646U << 10, + op_fcvt_d_s = 0x4649U << 10, + op_ftintrm_w_s = 0x4681U << 10, + op_ftintrm_w_d = 0x4682U << 10, + op_ftintrm_l_s = 0x4689U << 10, + op_ftintrm_l_d = 0x468aU << 10, + op_ftintrp_w_s = 0x4691U << 10, + op_ftintrp_w_d = 0x4692U << 10, + op_ftintrp_l_s = 0x4699U << 10, + op_ftintrp_l_d = 0x469aU << 10, + op_ftintrz_w_s = 0x46a1U << 10, + op_ftintrz_w_d = 0x46a2U << 10, + op_ftintrz_l_s = 0x46a9U << 10, + op_ftintrz_l_d = 0x46aaU << 10, + op_ftintrne_w_s = 0x46b1U << 10, + op_ftintrne_w_d = 0x46b2U << 10, + op_ftintrne_l_s = 0x46b9U << 10, + op_ftintrne_l_d = 0x46baU << 10, + op_ftint_w_s = 0x46c1U << 10, + op_ftint_w_d = 0x46c2U << 10, + op_ftint_l_s = 0x46c9U << 10, + op_ftint_l_d = 0x46caU << 10, + op_ffint_s_w = 0x4744U << 10, + op_ffint_s_l = 0x4746U << 10, + op_ffint_d_w = 0x4748U << 10, + op_ffint_d_l = 0x474aU << 10, + op_frint_s = 0x4791U << 10, + op_frint_d = 0x4792U << 10, + op_movcf2fr = 0x114d4U << 8, + op_movcf2gr = 0x114dcU << 8, +}; + +class Operand; + +// A BOffImm16 is a 16 bit immediate that is used for branches. +class BOffImm16 { + uint32_t data; + + public: + uint32_t encode() { + MOZ_ASSERT(!isInvalid()); + return data; + } + int32_t decode() { + MOZ_ASSERT(!isInvalid()); + return (int32_t(data << 18) >> 16); + } + + explicit BOffImm16(int offset) : data((offset) >> 2 & Imm16Mask) { + MOZ_ASSERT((offset & 0x3) == 0); + MOZ_ASSERT(IsInRange(offset)); + } + static bool IsInRange(int offset) { + if ((offset) < int(unsigned(INT16_MIN) << 2)) { + return false; + } + if ((offset) > (INT16_MAX << 2)) { + return false; + } + return true; + } + static const uint32_t INVALID = 0x00020000; + BOffImm16() : data(INVALID) {} + + bool isInvalid() { return data == INVALID; } + Instruction* getDest(Instruction* src) const; + + BOffImm16(InstImm inst); +}; + +// A JOffImm26 is a 26 bit immediate that is used for unconditional jumps. +class JOffImm26 { + uint32_t data; + + public: + uint32_t encode() { + MOZ_ASSERT(!isInvalid()); + return data; + } + int32_t decode() { + MOZ_ASSERT(!isInvalid()); + return (int32_t(data << 8) >> 6); + } + + explicit JOffImm26(int offset) : data((offset) >> 2 & Imm26Mask) { + MOZ_ASSERT((offset & 0x3) == 0); + MOZ_ASSERT(IsInRange(offset)); + } + static bool IsInRange(int offset) { + if ((offset) < -536870912) { + return false; + } + if ((offset) > 536870908) { + return false; + } + return true; + } + static const uint32_t INVALID = 0x20000000; + JOffImm26() : data(INVALID) {} + + bool isInvalid() { return data == INVALID; } + Instruction* getDest(Instruction* src); +}; + +class Imm16 { + uint16_t value; + + public: + Imm16(); + Imm16(uint32_t imm) : value(imm) {} + uint32_t encode() { return value; } + int32_t decodeSigned() { return value; } + uint32_t decodeUnsigned() { return value; } + + static bool IsInSignedRange(int32_t imm) { + return imm >= INT16_MIN && imm <= INT16_MAX; + } + + static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT16_MAX; } +}; + +class Imm8 { + uint8_t value; + + public: + Imm8(); + Imm8(uint32_t imm) : value(imm) {} + uint32_t encode(uint32_t shift) { return value << shift; } + int32_t decodeSigned() { return value; } + uint32_t decodeUnsigned() { return value; } + static bool IsInSignedRange(int32_t imm) { + return imm >= INT8_MIN && imm <= INT8_MAX; + } + static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT8_MAX; } + static Imm8 Lower(Imm16 imm) { return Imm8(imm.decodeSigned() & 0xff); } + static Imm8 Upper(Imm16 imm) { + return Imm8((imm.decodeSigned() >> 8) & 0xff); + } +}; + +class Operand { + public: + enum Tag { REG, FREG, MEM }; + + private: + Tag tag : 3; + uint32_t reg : 5; + int32_t offset; + + public: + Operand(Register reg_) : tag(REG), reg(reg_.code()) {} + + Operand(FloatRegister freg) : tag(FREG), reg(freg.code()) {} + + Operand(Register base, Imm32 off) + : tag(MEM), reg(base.code()), offset(off.value) {} + + Operand(Register base, int32_t off) + : tag(MEM), reg(base.code()), offset(off) {} + + Operand(const Address& addr) + : tag(MEM), reg(addr.base.code()), offset(addr.offset) {} + + Tag getTag() const { return tag; } + + Register toReg() const { + MOZ_ASSERT(tag == REG); + return Register::FromCode(reg); + } + + FloatRegister toFReg() const { + MOZ_ASSERT(tag == FREG); + return FloatRegister::FromCode(reg); + } + + void toAddr(Register* r, Imm32* dest) const { + MOZ_ASSERT(tag == MEM); + *r = Register::FromCode(reg); + *dest = Imm32(offset); + } + Address toAddress() const { + MOZ_ASSERT(tag == MEM); + return Address(Register::FromCode(reg), offset); + } + int32_t disp() const { + MOZ_ASSERT(tag == MEM); + return offset; + } + + int32_t base() const { + MOZ_ASSERT(tag == MEM); + return reg; + } + Register baseReg() const { + MOZ_ASSERT(tag == MEM); + return Register::FromCode(reg); + } +}; + +// int check. +inline bool is_intN(int32_t x, unsigned n) { + MOZ_ASSERT((0 < n) && (n < 64)); + int32_t limit = static_cast(1) << (n - 1); + return (-limit <= x) && (x < limit); +} + +inline bool is_uintN(int32_t x, unsigned n) { + MOZ_ASSERT((0 < n) && (n < (sizeof(x) * 8))); + return !(x >> n); +} + +inline Imm32 Imm64::firstHalf() const { return low(); } + +inline Imm32 Imm64::secondHalf() const { return hi(); } + +static constexpr int32_t SliceSize = 1024; +typedef js::jit::AssemblerBuffer LABuffer; + +class LABufferWithExecutableCopy : public LABuffer { + public: + void executableCopy(uint8_t* buffer) { + if (this->oom()) { + return; + } + + for (Slice* cur = head; cur != nullptr; cur = cur->getNext()) { + memcpy(buffer, &cur->instructions, cur->length()); + buffer += cur->length(); + } + } + + bool appendRawCode(const uint8_t* code, size_t numBytes) { + if (this->oom()) { + return false; + } + while (numBytes > SliceSize) { + this->putBytes(SliceSize, code); + numBytes -= SliceSize; + code += SliceSize; + } + this->putBytes(numBytes, code); + return !this->oom(); + } +}; + +class AssemblerLOONG64 : public AssemblerShared { + public: + enum Condition { + Equal, + NotEqual, + Above, + AboveOrEqual, + Below, + BelowOrEqual, + GreaterThan, // TODO + GreaterThanOrEqual, // TODO + GreaterThanOrEqual_Signed, + GreaterThanOrEqual_NotSigned, + LessThan, // TODO + LessThan_Signed, + LessThan_NotSigned, + LessThanOrEqual, // TODO + Overflow, + CarrySet, + CarryClear, + Signed, + NotSigned, + Zero, + NonZero, + Always, + }; + + enum DoubleCondition { + // These conditions will only evaluate to true if the comparison is ordered + // - i.e. neither operand is NaN. + DoubleOrdered, + DoubleEqual, + DoubleNotEqual, + DoubleGreaterThan, + DoubleGreaterThanOrEqual, + DoubleLessThan, + DoubleLessThanOrEqual, + // If either operand is NaN, these conditions always evaluate to true. + DoubleUnordered, + DoubleEqualOrUnordered, + DoubleNotEqualOrUnordered, + DoubleGreaterThanOrUnordered, + DoubleGreaterThanOrEqualOrUnordered, + DoubleLessThanOrUnordered, + DoubleLessThanOrEqualOrUnordered + }; + + enum FPUCondition { + kNoFPUCondition = -1, + + CAF = 0x00, // False. + SAF = 0x01, // False. + CLT = 0x02, // Less Than quiet + SLT = 0x03, // Less Than signaling + CEQ = 0x04, + SEQ = 0x05, + CLE = 0x06, + SLE = 0x07, + CUN = 0x08, + SUN = 0x09, + CULT = 0x0a, + SULT = 0x0b, + CUEQ = 0x0c, + SUEQ = 0x0d, + CULE = 0x0e, + SULE = 0x0f, + CNE = 0x10, + SNE = 0x11, + COR = 0x14, + SOR = 0x15, + CUNE = 0x18, + SUNE = 0x19, + }; + + enum FPConditionBit { FCC0 = 0, FCC1, FFC2, FCC3, FCC4, FCC5, FCC6, FCC7 }; + + enum FPControl { FCSR = 0 }; + + enum FCSRBit { CauseI = 24, CauseU, CauseO, CauseZ, CauseV }; + + enum FloatFormat { SingleFloat, DoubleFloat }; + + enum JumpOrCall { BranchIsJump, BranchIsCall }; + + enum FloatTestKind { TestForTrue, TestForFalse }; + + // :( this should be protected, but since CodeGenerator + // wants to use it, It needs to go out here :( + + BufferOffset nextOffset() { return m_buffer.nextOffset(); } + + protected: + Instruction* editSrc(BufferOffset bo) { return m_buffer.getInst(bo); } + + // structure for fixing up pc-relative loads/jumps when a the machine code + // gets moved (executable copy, gc, etc.) + struct RelativePatch { + // the offset within the code buffer where the value is loaded that + // we want to fix-up + BufferOffset offset; + void* target; + RelocationKind kind; + + RelativePatch(BufferOffset offset, void* target, RelocationKind kind) + : offset(offset), target(target), kind(kind) {} + }; + + js::Vector jumps_; + + CompactBufferWriter jumpRelocations_; + CompactBufferWriter dataRelocations_; + + LABufferWithExecutableCopy m_buffer; + +#ifdef JS_JITSPEW + Sprinter* printer; +#endif + + public: + AssemblerLOONG64() + : m_buffer(), +#ifdef JS_JITSPEW + printer(nullptr), +#endif + isFinished(false) { + } + + static Condition InvertCondition(Condition cond); + static DoubleCondition InvertCondition(DoubleCondition cond); + // This is changing the condition codes for cmp a, b to the same codes for cmp + // b, a. + static Condition InvertCmpCondition(Condition cond); + + // As opposed to x86/x64 version, the data relocation has to be executed + // before to recover the pointer, and not after. + void writeDataRelocation(ImmGCPtr ptr) { + // Raw GC pointer relocations and Value relocations both end up in + // TraceOneDataRelocation. + if (ptr.value) { + if (gc::IsInsideNursery(ptr.value)) { + embedsNurseryPointers_ = true; + } + dataRelocations_.writeUnsigned(nextOffset().getOffset()); + } + } + + void assertNoGCThings() const { +#ifdef DEBUG + MOZ_ASSERT(dataRelocations_.length() == 0); + for (auto& j : jumps_) { + MOZ_ASSERT(j.kind == RelocationKind::HARDCODED); + } +#endif + } + + public: + void setUnlimitedBuffer() { m_buffer.setUnlimited(); } + bool oom() const; + + void setPrinter(Sprinter* sp) { +#ifdef JS_JITSPEW + printer = sp; +#endif + } + +#ifdef JS_JITSPEW + inline void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) { + if (MOZ_UNLIKELY(printer || JitSpewEnabled(JitSpew_Codegen))) { + va_list va; + va_start(va, fmt); + spew(fmt, va); + va_end(va); + } + } + + void decodeBranchInstAndSpew(InstImm branch); +#else + MOZ_ALWAYS_INLINE void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) {} +#endif + +#ifdef JS_JITSPEW + MOZ_COLD void spew(const char* fmt, va_list va) MOZ_FORMAT_PRINTF(2, 0) { + // Buffer to hold the formatted string. Note that this may contain + // '%' characters, so do not pass it directly to printf functions. + char buf[200]; + + int i = VsprintfLiteral(buf, fmt, va); + if (i > -1) { + if (printer) { + printer->printf("%s\n", buf); + } + js::jit::JitSpew(js::jit::JitSpew_Codegen, "%s", buf); + } + } +#endif + + Register getStackPointer() const { return StackPointer; } + + protected: + bool isFinished; + + public: + void finish(); + bool appendRawCode(const uint8_t* code, size_t numBytes); + bool reserve(size_t size); + bool swapBuffer(wasm::Bytes& bytes); + void executableCopy(void* buffer); + void copyJumpRelocationTable(uint8_t* dest); + void copyDataRelocationTable(uint8_t* dest); + + // Size of the instruction stream, in bytes. + size_t size() const; + // Size of the jump relocation table, in bytes. + size_t jumpRelocationTableBytes() const; + size_t dataRelocationTableBytes() const; + + // Size of the data table, in bytes. + size_t bytesNeeded() const; + + // Write a blob of binary into the instruction stream *OR* + // into a destination address. If dest is nullptr (the default), then the + // instruction gets written into the instruction stream. If dest is not null + // it is interpreted as a pointer to the location that we want the + // instruction to be written. + BufferOffset writeInst(uint32_t x, uint32_t* dest = nullptr); + // A static variant for the cases where we don't want to have an assembler + // object at all. Normally, you would use the dummy (nullptr) object. + static void WriteInstStatic(uint32_t x, uint32_t* dest); + + public: + BufferOffset haltingAlign(int alignment); + BufferOffset nopAlign(int alignment); + BufferOffset as_nop() { return as_andi(zero, zero, 0); } + + // Branch and jump instructions + BufferOffset as_b(JOffImm26 off); + BufferOffset as_bl(JOffImm26 off); + BufferOffset as_jirl(Register rd, Register rj, BOffImm16 off); + + InstImm getBranchCode(JumpOrCall jumpOrCall); // b, bl + InstImm getBranchCode(Register rd, Register rj, + Condition c); // beq, bne, bge, bgeu, blt, bltu + InstImm getBranchCode(Register rj, Condition c); // beqz, bnez + InstImm getBranchCode(FPConditionBit cj); // bceqz, bcnez + + // Arithmetic instructions + BufferOffset as_add_w(Register rd, Register rj, Register rk); + BufferOffset as_add_d(Register rd, Register rj, Register rk); + BufferOffset as_sub_w(Register rd, Register rj, Register rk); + BufferOffset as_sub_d(Register rd, Register rj, Register rk); + + BufferOffset as_addi_w(Register rd, Register rj, int32_t si12); + BufferOffset as_addi_d(Register rd, Register rj, int32_t si12); + BufferOffset as_addu16i_d(Register rd, Register rj, int32_t si16); + + BufferOffset as_alsl_w(Register rd, Register rj, Register rk, uint32_t sa2); + BufferOffset as_alsl_wu(Register rd, Register rj, Register rk, uint32_t sa2); + BufferOffset as_alsl_d(Register rd, Register rj, Register rk, uint32_t sa2); + + BufferOffset as_lu12i_w(Register rd, int32_t si20); + BufferOffset as_lu32i_d(Register rd, int32_t si20); + BufferOffset as_lu52i_d(Register rd, Register rj, int32_t si12); + + BufferOffset as_slt(Register rd, Register rj, Register rk); + BufferOffset as_sltu(Register rd, Register rj, Register rk); + BufferOffset as_slti(Register rd, Register rj, int32_t si12); + BufferOffset as_sltui(Register rd, Register rj, int32_t si12); + + BufferOffset as_pcaddi(Register rd, int32_t si20); + BufferOffset as_pcaddu12i(Register rd, int32_t si20); + BufferOffset as_pcaddu18i(Register rd, int32_t si20); + BufferOffset as_pcalau12i(Register rd, int32_t si20); + + BufferOffset as_mul_w(Register rd, Register rj, Register rk); + BufferOffset as_mulh_w(Register rd, Register rj, Register rk); + BufferOffset as_mulh_wu(Register rd, Register rj, Register rk); + BufferOffset as_mul_d(Register rd, Register rj, Register rk); + BufferOffset as_mulh_d(Register rd, Register rj, Register rk); + BufferOffset as_mulh_du(Register rd, Register rj, Register rk); + + BufferOffset as_mulw_d_w(Register rd, Register rj, Register rk); + BufferOffset as_mulw_d_wu(Register rd, Register rj, Register rk); + + BufferOffset as_div_w(Register rd, Register rj, Register rk); + BufferOffset as_mod_w(Register rd, Register rj, Register rk); + BufferOffset as_div_wu(Register rd, Register rj, Register rk); + BufferOffset as_mod_wu(Register rd, Register rj, Register rk); + BufferOffset as_div_d(Register rd, Register rj, Register rk); + BufferOffset as_mod_d(Register rd, Register rj, Register rk); + BufferOffset as_div_du(Register rd, Register rj, Register rk); + BufferOffset as_mod_du(Register rd, Register rj, Register rk); + + // Logical instructions + BufferOffset as_and(Register rd, Register rj, Register rk); + BufferOffset as_or(Register rd, Register rj, Register rk); + BufferOffset as_xor(Register rd, Register rj, Register rk); + BufferOffset as_nor(Register rd, Register rj, Register rk); + BufferOffset as_andn(Register rd, Register rj, Register rk); + BufferOffset as_orn(Register rd, Register rj, Register rk); + + BufferOffset as_andi(Register rd, Register rj, int32_t ui12); + BufferOffset as_ori(Register rd, Register rj, int32_t ui12); + BufferOffset as_xori(Register rd, Register rj, int32_t ui12); + + // Shift instructions + BufferOffset as_sll_w(Register rd, Register rj, Register rk); + BufferOffset as_srl_w(Register rd, Register rj, Register rk); + BufferOffset as_sra_w(Register rd, Register rj, Register rk); + BufferOffset as_rotr_w(Register rd, Register rj, Register rk); + + BufferOffset as_slli_w(Register rd, Register rj, int32_t ui5); + BufferOffset as_srli_w(Register rd, Register rj, int32_t ui5); + BufferOffset as_srai_w(Register rd, Register rj, int32_t ui5); + BufferOffset as_rotri_w(Register rd, Register rj, int32_t ui5); + + BufferOffset as_sll_d(Register rd, Register rj, Register rk); + BufferOffset as_srl_d(Register rd, Register rj, Register rk); + BufferOffset as_sra_d(Register rd, Register rj, Register rk); + BufferOffset as_rotr_d(Register rd, Register rj, Register rk); + + BufferOffset as_slli_d(Register rd, Register rj, int32_t ui6); + BufferOffset as_srli_d(Register rd, Register rj, int32_t ui6); + BufferOffset as_srai_d(Register rd, Register rj, int32_t ui6); + BufferOffset as_rotri_d(Register rd, Register rj, int32_t ui6); + + // Bit operation instrucitons + BufferOffset as_ext_w_b(Register rd, Register rj); + BufferOffset as_ext_w_h(Register rd, Register rj); + + BufferOffset as_clo_w(Register rd, Register rj); + BufferOffset as_clz_w(Register rd, Register rj); + BufferOffset as_cto_w(Register rd, Register rj); + BufferOffset as_ctz_w(Register rd, Register rj); + BufferOffset as_clo_d(Register rd, Register rj); + BufferOffset as_clz_d(Register rd, Register rj); + BufferOffset as_cto_d(Register rd, Register rj); + BufferOffset as_ctz_d(Register rd, Register rj); + + BufferOffset as_bytepick_w(Register rd, Register rj, Register rk, + int32_t sa2); + BufferOffset as_bytepick_d(Register rd, Register rj, Register rk, + int32_t sa3); + + BufferOffset as_revb_2h(Register rd, Register rj); + BufferOffset as_revb_4h(Register rd, Register rj); + BufferOffset as_revb_2w(Register rd, Register rj); + BufferOffset as_revb_d(Register rd, Register rj); + + BufferOffset as_revh_2w(Register rd, Register rj); + BufferOffset as_revh_d(Register rd, Register rj); + + BufferOffset as_bitrev_4b(Register rd, Register rj); + BufferOffset as_bitrev_8b(Register rd, Register rj); + + BufferOffset as_bitrev_w(Register rd, Register rj); + BufferOffset as_bitrev_d(Register rd, Register rj); + + BufferOffset as_bstrins_w(Register rd, Register rj, int32_t msbw, + int32_t lsbw); + BufferOffset as_bstrins_d(Register rd, Register rj, int32_t msbd, + int32_t lsbd); + BufferOffset as_bstrpick_w(Register rd, Register rj, int32_t msbw, + int32_t lsbw); + BufferOffset as_bstrpick_d(Register rd, Register rj, int32_t msbd, + int32_t lsbd); + + BufferOffset as_maskeqz(Register rd, Register rj, Register rk); + BufferOffset as_masknez(Register rd, Register rj, Register rk); + + // Load and store instructions + BufferOffset as_ld_b(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_h(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_w(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_d(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_bu(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_hu(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_wu(Register rd, Register rj, int32_t si12); + BufferOffset as_st_b(Register rd, Register rj, int32_t si12); + BufferOffset as_st_h(Register rd, Register rj, int32_t si12); + BufferOffset as_st_w(Register rd, Register rj, int32_t si12); + BufferOffset as_st_d(Register rd, Register rj, int32_t si12); + + BufferOffset as_ldx_b(Register rd, Register rj, Register rk); + BufferOffset as_ldx_h(Register rd, Register rj, Register rk); + BufferOffset as_ldx_w(Register rd, Register rj, Register rk); + BufferOffset as_ldx_d(Register rd, Register rj, Register rk); + BufferOffset as_ldx_bu(Register rd, Register rj, Register rk); + BufferOffset as_ldx_hu(Register rd, Register rj, Register rk); + BufferOffset as_ldx_wu(Register rd, Register rj, Register rk); + BufferOffset as_stx_b(Register rd, Register rj, Register rk); + BufferOffset as_stx_h(Register rd, Register rj, Register rk); + BufferOffset as_stx_w(Register rd, Register rj, Register rk); + BufferOffset as_stx_d(Register rd, Register rj, Register rk); + + BufferOffset as_ldptr_w(Register rd, Register rj, int32_t si14); + BufferOffset as_ldptr_d(Register rd, Register rj, int32_t si14); + BufferOffset as_stptr_w(Register rd, Register rj, int32_t si14); + BufferOffset as_stptr_d(Register rd, Register rj, int32_t si14); + + BufferOffset as_preld(int32_t hint, Register rj, int32_t si12); + + // Atomic instructions + BufferOffset as_amswap_w(Register rd, Register rj, Register rk); + BufferOffset as_amswap_d(Register rd, Register rj, Register rk); + BufferOffset as_amadd_w(Register rd, Register rj, Register rk); + BufferOffset as_amadd_d(Register rd, Register rj, Register rk); + BufferOffset as_amand_w(Register rd, Register rj, Register rk); + BufferOffset as_amand_d(Register rd, Register rj, Register rk); + BufferOffset as_amor_w(Register rd, Register rj, Register rk); + BufferOffset as_amor_d(Register rd, Register rj, Register rk); + BufferOffset as_amxor_w(Register rd, Register rj, Register rk); + BufferOffset as_amxor_d(Register rd, Register rj, Register rk); + BufferOffset as_ammax_w(Register rd, Register rj, Register rk); + BufferOffset as_ammax_d(Register rd, Register rj, Register rk); + BufferOffset as_ammin_w(Register rd, Register rj, Register rk); + BufferOffset as_ammin_d(Register rd, Register rj, Register rk); + BufferOffset as_ammax_wu(Register rd, Register rj, Register rk); + BufferOffset as_ammax_du(Register rd, Register rj, Register rk); + BufferOffset as_ammin_wu(Register rd, Register rj, Register rk); + BufferOffset as_ammin_du(Register rd, Register rj, Register rk); + + BufferOffset as_amswap_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amswap_db_d(Register rd, Register rj, Register rk); + BufferOffset as_amadd_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amadd_db_d(Register rd, Register rj, Register rk); + BufferOffset as_amand_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amand_db_d(Register rd, Register rj, Register rk); + BufferOffset as_amor_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amor_db_d(Register rd, Register rj, Register rk); + BufferOffset as_amxor_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amxor_db_d(Register rd, Register rj, Register rk); + BufferOffset as_ammax_db_w(Register rd, Register rj, Register rk); + BufferOffset as_ammax_db_d(Register rd, Register rj, Register rk); + BufferOffset as_ammin_db_w(Register rd, Register rj, Register rk); + BufferOffset as_ammin_db_d(Register rd, Register rj, Register rk); + BufferOffset as_ammax_db_wu(Register rd, Register rj, Register rk); + BufferOffset as_ammax_db_du(Register rd, Register rj, Register rk); + BufferOffset as_ammin_db_wu(Register rd, Register rj, Register rk); + BufferOffset as_ammin_db_du(Register rd, Register rj, Register rk); + + BufferOffset as_ll_w(Register rd, Register rj, int32_t si14); + BufferOffset as_ll_d(Register rd, Register rj, int32_t si14); + BufferOffset as_sc_w(Register rd, Register rj, int32_t si14); + BufferOffset as_sc_d(Register rd, Register rj, int32_t si14); + + // Barrier instructions + BufferOffset as_dbar(int32_t hint); + BufferOffset as_ibar(int32_t hint); + + // FP Arithmetic instructions + BufferOffset as_fadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmul_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmul_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fdiv_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fdiv_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + + BufferOffset as_fmadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fmadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fmsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fmsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fnmadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fnmadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fnmsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fnmsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + + BufferOffset as_fmax_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmax_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmin_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmin_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + + BufferOffset as_fmaxa_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmaxa_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmina_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmina_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + + BufferOffset as_fabs_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_fabs_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_fneg_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_fneg_d(FloatRegister fd, FloatRegister fj); + + BufferOffset as_fsqrt_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_fsqrt_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_fcopysign_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk); + BufferOffset as_fcopysign_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk); + + // FP compare instructions + // FP compare instructions (fcmp.cond.s fcmp.cond.d) + BufferOffset as_fcmp_cor(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_ceq(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cne(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cle(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_clt(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cun(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cueq(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cune(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cule(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cult(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + + // FP conversion instructions + BufferOffset as_fcvt_s_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_fcvt_d_s(FloatRegister fd, FloatRegister fj); + + BufferOffset as_ffint_s_w(FloatRegister fd, FloatRegister fj); + BufferOffset as_ffint_s_l(FloatRegister fd, FloatRegister fj); + BufferOffset as_ffint_d_w(FloatRegister fd, FloatRegister fj); + BufferOffset as_ffint_d_l(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftint_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftint_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftint_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftint_l_d(FloatRegister fd, FloatRegister fj); + + BufferOffset as_ftintrm_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrm_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrm_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrm_l_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrp_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrp_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrp_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrp_l_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrz_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrz_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrz_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrz_l_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrne_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrne_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrne_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrne_l_d(FloatRegister fd, FloatRegister fj); + + BufferOffset as_frint_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_frint_d(FloatRegister fd, FloatRegister fj); + + // FP mov instructions + BufferOffset as_fmov_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_fmov_d(FloatRegister fd, FloatRegister fj); + + BufferOffset as_fsel(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FPConditionBit ca); + + BufferOffset as_movgr2fr_w(FloatRegister fd, Register rj); + BufferOffset as_movgr2fr_d(FloatRegister fd, Register rj); + BufferOffset as_movgr2frh_w(FloatRegister fd, Register rj); + + BufferOffset as_movfr2gr_s(Register rd, FloatRegister fj); + BufferOffset as_movfr2gr_d(Register rd, FloatRegister fj); + BufferOffset as_movfrh2gr_s(Register rd, FloatRegister fj); + + BufferOffset as_movgr2fcsr(Register rj); + BufferOffset as_movfcsr2gr(Register rd); + + BufferOffset as_movfr2cf(FPConditionBit cd, FloatRegister fj); + BufferOffset as_movcf2fr(FloatRegister fd, FPConditionBit cj); + + BufferOffset as_movgr2cf(FPConditionBit cd, Register rj); + BufferOffset as_movcf2gr(Register rd, FPConditionBit cj); + + // FP load/store instructions + BufferOffset as_fld_s(FloatRegister fd, Register rj, int32_t si12); + BufferOffset as_fld_d(FloatRegister fd, Register rj, int32_t si12); + BufferOffset as_fst_s(FloatRegister fd, Register rj, int32_t si12); + BufferOffset as_fst_d(FloatRegister fd, Register rj, int32_t si12); + + BufferOffset as_fldx_s(FloatRegister fd, Register rj, Register rk); + BufferOffset as_fldx_d(FloatRegister fd, Register rj, Register rk); + BufferOffset as_fstx_s(FloatRegister fd, Register rj, Register rk); + BufferOffset as_fstx_d(FloatRegister fd, Register rj, Register rk); + + // label operations + void bind(Label* label, BufferOffset boff = BufferOffset()); + virtual void bind(InstImm* inst, uintptr_t branch, uintptr_t target) = 0; + void bind(CodeLabel* label) { label->target()->bind(currentOffset()); } + uint32_t currentOffset() { return nextOffset().getOffset(); } + void retarget(Label* label, Label* target); + + void call(Label* label); + void call(void* target); + + void as_break(uint32_t code); + + public: + static bool SupportsFloatingPoint() { +#if defined(__loongarch_hard_float) || defined(JS_SIMULATOR_LOONG64) + return true; +#else + return false; +#endif + } + static bool SupportsUnalignedAccesses() { return true; } + static bool SupportsFastUnalignedAccesses() { return true; } + + static bool HasRoundInstruction(RoundingMode mode) { return false; } + + protected: + InstImm invertBranch(InstImm branch, BOffImm16 skipOffset); + void addPendingJump(BufferOffset src, ImmPtr target, RelocationKind kind) { + enoughMemory_ &= jumps_.append(RelativePatch(src, target.value, kind)); + if (kind == RelocationKind::JITCODE) { + jumpRelocations_.writeUnsigned(src.getOffset()); + } + } + + void addLongJump(BufferOffset src, BufferOffset dst) { + CodeLabel cl; + cl.patchAt()->bind(src.getOffset()); + cl.target()->bind(dst.getOffset()); + cl.setLinkMode(CodeLabel::JumpImmediate); + addCodeLabel(std::move(cl)); + } + + public: + void flushBuffer() {} + + void comment(const char* msg) { spew("; %s", msg); } + + static uint32_t NopSize() { return 4; } + + static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm); + + static uint8_t* NextInstruction(uint8_t* instruction, + uint32_t* count = nullptr); + + static void ToggleToJmp(CodeLocationLabel inst_); + static void ToggleToCmp(CodeLocationLabel inst_); + + void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, + const Disassembler::HeapAccess& heapAccess) { + // Implement this if we implement a disassembler. + } +}; // AssemblerLOONG64 + +// andi r0, r0, 0 +const uint32_t NopInst = 0x03400000; + +// An Instruction is a structure for both encoding and decoding any and all +// LoongArch instructions. +class Instruction { + public: + uint32_t data; + + protected: + // Standard constructor + Instruction(uint32_t data_) : data(data_) {} + // You should never create an instruction directly. You should create a + // more specific instruction which will eventually call one of these + // constructors for you. + + public: + uint32_t encode() const { return data; } + + void makeNop() { data = NopInst; } + + void setData(uint32_t data) { this->data = data; } + + const Instruction& operator=(const Instruction& src) { + data = src.data; + return *this; + } + + // Extract the one particular bit. + uint32_t extractBit(uint32_t bit) { return (encode() >> bit) & 1; } + // Extract a bit field out of the instruction + uint32_t extractBitField(uint32_t hi, uint32_t lo) { + return (encode() >> lo) & ((2 << (hi - lo)) - 1); + } + + // Get the next instruction in the instruction stream. + // This does neat things like ignoreconstant pools and their guards. + Instruction* next(); + + // Sometimes, an api wants a uint32_t (or a pointer to it) rather than + // an instruction. raw() just coerces this into a pointer to a uint32_t + const uint32_t* raw() const { return &data; } + uint32_t size() const { return 4; } +}; // Instruction + +// make sure that it is the right size +static_assert(sizeof(Instruction) == 4, + "Size of Instruction class has to be 4 bytes."); + +class InstNOP : public Instruction { + public: + InstNOP() : Instruction(NopInst) {} +}; + +// Class for register type instructions. +class InstReg : public Instruction { + public: + InstReg(OpcodeField op, Register rj, Register rd) + : Instruction(op | RJ(rj) | RD(rd)) {} + InstReg(OpcodeField op, Register rk, Register rj, Register rd) + : Instruction(op | RK(rk) | RJ(rj) | RD(rd)) {} + InstReg(OpcodeField op, uint32_t sa, Register rk, Register rj, Register rd, + uint32_t sa_bit) + : Instruction(sa_bit == 2 ? op | SA2(sa) | RK(rk) | RJ(rj) | RD(rd) + : op | SA3(sa) | RK(rk) | RJ(rj) | RD(rd)) { + MOZ_ASSERT(sa_bit == 2 || sa_bit == 3); + } + InstReg(OpcodeField op, Register rj, Register rd, bool HasRd) + : Instruction(HasRd ? op | RJ(rj) | RD(rd) : op | RK(rj) | RJ(rd)) {} + + // For floating-point + InstReg(OpcodeField op, Register rj, FloatRegister fd) + : Instruction(op | RJ(rj) | RD(fd)) {} + InstReg(OpcodeField op, FloatRegister fj, FloatRegister fd) + : Instruction(op | RJ(fj) | RD(fd)) {} + InstReg(OpcodeField op, FloatRegister fk, FloatRegister fj, FloatRegister fd) + : Instruction(op | RK(fk) | RJ(fj) | RD(fd)) {} + InstReg(OpcodeField op, Register rk, Register rj, FloatRegister fd) + : Instruction(op | RK(rk) | RJ(rj) | RD(fd)) {} + InstReg(OpcodeField op, FloatRegister fa, FloatRegister fk, FloatRegister fj, + FloatRegister fd) + : Instruction(op | FA(fa) | RK(fk) | RJ(fj) | RD(fd)) {} + InstReg(OpcodeField op, AssemblerLOONG64::FPConditionBit ca, FloatRegister fk, + FloatRegister fj, FloatRegister fd) + : Instruction(op | ca << CAShift | RK(fk) | RJ(fj) | RD(fd)) { + MOZ_ASSERT(op == op_fsel); + } + InstReg(OpcodeField op, FloatRegister fj, Register rd) + : Instruction(op | RJ(fj) | RD(rd)) { + MOZ_ASSERT((op == op_movfr2gr_s) || (op == op_movfr2gr_d) || + (op == op_movfrh2gr_s)); + } + InstReg(OpcodeField op, Register rj, uint32_t fd) + : Instruction(op | RJ(rj) | fd) { + MOZ_ASSERT(op == op_movgr2fcsr); + } + InstReg(OpcodeField op, uint32_t fj, Register rd) + : Instruction(op | (fj << FJShift) | RD(rd)) { + MOZ_ASSERT(op == op_movfcsr2gr); + } + InstReg(OpcodeField op, FloatRegister fj, AssemblerLOONG64::FPConditionBit cd) + : Instruction(op | RJ(fj) | cd) { + MOZ_ASSERT(op == op_movfr2cf); + } + InstReg(OpcodeField op, AssemblerLOONG64::FPConditionBit cj, FloatRegister fd) + : Instruction(op | (cj << CJShift) | RD(fd)) { + MOZ_ASSERT(op == op_movcf2fr); + } + InstReg(OpcodeField op, Register rj, AssemblerLOONG64::FPConditionBit cd) + : Instruction(op | RJ(rj) | cd) { + MOZ_ASSERT(op == op_movgr2cf); + } + InstReg(OpcodeField op, AssemblerLOONG64::FPConditionBit cj, Register rd) + : Instruction(op | (cj << CJShift) | RD(rd)) { + MOZ_ASSERT(op == op_movcf2gr); + } + InstReg(OpcodeField op, int32_t cond, FloatRegister fk, FloatRegister fj, + AssemblerLOONG64::FPConditionBit cd) + : Instruction(op | (cond & CONDMask) << CONDShift | RK(fk) | RJ(fj) | + (cd & RDMask)) { + MOZ_ASSERT(is_uintN(cond, 5)); + } + + uint32_t extractRK() { + return extractBitField(RKShift + RKBits - 1, RKShift); + } + uint32_t extractRJ() { + return extractBitField(RJShift + RJBits - 1, RJShift); + } + uint32_t extractRD() { + return extractBitField(RDShift + RDBits - 1, RDShift); + } + uint32_t extractSA2() { + return extractBitField(SAShift + SA2Bits - 1, SAShift); + } + uint32_t extractSA3() { + return extractBitField(SAShift + SA3Bits - 1, SAShift); + } +}; + +// Class for branch, load and store instructions with immediate offset. +class InstImm : public Instruction { + public: + void extractImm16(BOffImm16* dest); + uint32_t genImm(int32_t value, uint32_t value_bits) { + uint32_t imm = value & Imm5Mask; + if (value_bits == 6) { + imm = value & Imm6Mask; + } else if (value_bits == 12) { + imm = value & Imm12Mask; + } else if (value_bits == 14) { + imm = value & Imm14Mask; + } + + return imm; + } + + InstImm(OpcodeField op, int32_t value, Register rj, Register rd, + uint32_t value_bits) + : Instruction(op | genImm(value, value_bits) << RKShift | RJ(rj) | + RD(rd)) { + MOZ_ASSERT(value_bits == 5 || value_bits == 6 || value_bits == 12 || + value_bits == 14); + } + InstImm(OpcodeField op, BOffImm16 off, Register rj, Register rd) + : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift | RJ(rj) | + RD(rd)) {} + InstImm(OpcodeField op, int32_t si21, Register rj, bool NotHasRd) + : Instruction(NotHasRd ? op | (si21 & Imm16Mask) << RKShift | RJ(rj) | + (si21 & Imm21Mask) >> 16 + : op | (si21 & Imm20Mask) << Imm20Shift | RD(rj)) { + if (NotHasRd) { + MOZ_ASSERT(op == op_beqz || op == op_bnez); + MOZ_ASSERT(is_intN(si21, 21)); + } else { + MOZ_ASSERT(op == op_lu12i_w || op == op_lu32i_d || op == op_pcaddi || + op == op_pcaddu12i || op == op_pcaddu18i || + op == op_pcalau12i); + // si20 + MOZ_ASSERT(is_intN(si21, 20) || is_uintN(si21, 20)); + } + } + InstImm(OpcodeField op, int32_t si21, AssemblerLOONG64::FPConditionBit cj, + bool isNotEqual) + : Instruction(isNotEqual + ? op | (si21 & Imm16Mask) << RKShift | + (cj + 8) << CJShift | (si21 & Imm21Mask) >> 16 + : op | (si21 & Imm16Mask) << RKShift | cj << CJShift | + (si21 & Imm21Mask) >> 16) { + MOZ_ASSERT(is_intN(si21, 21)); + MOZ_ASSERT(op == op_bcz); + MOZ_ASSERT(cj >= 0 && cj <= 7); + } + InstImm(OpcodeField op, Imm16 off, Register rj, Register rd) + : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift | RJ(rj) | + RD(rd)) {} + InstImm(OpcodeField op, int32_t bit15) + : Instruction(op | (bit15 & Imm15Mask)) { + MOZ_ASSERT(is_uintN(bit15, 15)); + } + + InstImm(OpcodeField op, int32_t bit26, bool jump) + : Instruction(op | (bit26 & Imm16Mask) << Imm16Shift | + (bit26 & Imm26Mask) >> 16) { + MOZ_ASSERT(is_intN(bit26, 26)); + } + InstImm(OpcodeField op, int32_t si12, Register rj, int32_t hint) + : Instruction(op | (si12 & Imm12Mask) << Imm12Shift | RJ(rj) | + (hint & RDMask)) { + MOZ_ASSERT(op == op_preld); + } + InstImm(OpcodeField op, int32_t msb, int32_t lsb, Register rj, Register rd, + uint32_t sb_bits) + : Instruction((sb_bits == 5) + ? op | (msb & MSBWMask) << MSBWShift | + (lsb & LSBMask) << LSBWShift | RJ(rj) | RD(rd) + : op | (msb & MSBDMask) << MSBDShift | + (lsb & LSBMask) << LSBDShift | RJ(rj) | RD(rd)) { + MOZ_ASSERT(sb_bits == 5 || sb_bits == 6); + MOZ_ASSERT(op == op_bstr_w || op == op_bstrins_d || op == op_bstrpick_d); + } + InstImm(OpcodeField op, int32_t msb, int32_t lsb, Register rj, Register rd) + : Instruction(op | (msb & MSBWMask) << MSBWShift | + ((lsb + 0x20) & LSBMask) << LSBWShift | RJ(rj) | RD(rd)) { + MOZ_ASSERT(op == op_bstr_w); + } + + // For floating-point loads and stores. + InstImm(OpcodeField op, int32_t si12, Register rj, FloatRegister fd) + : Instruction(op | (si12 & Imm12Mask) << Imm12Shift | RJ(rj) | RD(fd)) { + MOZ_ASSERT(is_intN(si12, 12)); + } + + void setOpcode(OpcodeField op, uint32_t opBits) { + // opBits not greater than 24. + MOZ_ASSERT(opBits < 25); + uint32_t OpcodeShift = 32 - opBits; + uint32_t OpcodeMask = ((1 << opBits) - 1) << OpcodeShift; + data = (data & ~OpcodeMask) | op; + } + uint32_t extractRK() { + return extractBitField(RKShift + RKBits - 1, RKShift); + } + uint32_t extractRJ() { + return extractBitField(RJShift + RJBits - 1, RJShift); + } + void setRJ(uint32_t rj) { data = (data & ~RJMask) | (rj << RJShift); } + uint32_t extractRD() { + return extractBitField(RDShift + RDBits - 1, RDShift); + } + uint32_t extractImm16Value() { + return extractBitField(Imm16Shift + Imm16Bits - 1, Imm16Shift); + } + void setBOffImm16(BOffImm16 off) { + // Reset immediate field and replace it + data = (data & ~BOffImm16Mask) | (off.encode() << Imm16Shift); + } + void setImm21(int32_t off) { + // Reset immediate field and replace it + uint32_t low16 = (off >> 2) & Imm16Mask; + int32_t high5 = (off >> 18) & Imm5Mask; + uint32_t fcc_info = (data >> 5) & 0x1F; + data = (data & ~((1 << 26) - 1)) | (low16 << Imm16Shift) | high5 | + (fcc_info << 5); + } +}; + +// Class for Jump type instructions. +class InstJump : public Instruction { + public: + InstJump(OpcodeField op, JOffImm26 off) + : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift | + (off.encode() & Imm26Mask) >> 16) { + MOZ_ASSERT(op == op_b || op == op_bl); + } + + void setJOffImm26(JOffImm26 off) { + // Reset immediate field and replace it + data = (data & ~BOffImm26Mask) | + ((off.encode() & Imm16Mask) << Imm16Shift) | + ((off.encode() >> 16) & 0x3ff); + } + uint32_t extractImm26Value() { + return extractBitField(Imm26Shift + Imm26Bits - 1, Imm26Shift); + } +}; + +inline bool IsUnaligned(const wasm::MemoryAccessDesc& access) { + if (!access.align()) { + return false; + } + + return access.align() < access.byteSize(); +} + +static constexpr Register CallTempReg4 = a4; +static constexpr Register CallTempReg5 = a5; + +static constexpr Register CallTempNonArgRegs[] = {t0, t1, t2, t3}; +static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs); + +class ABIArgGenerator { + public: + ABIArgGenerator() + : intRegIndex_(0), floatRegIndex_(0), stackOffset_(0), current_() {} + + ABIArg next(MIRType argType); + ABIArg& current() { return current_; } + uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; } + + protected: + unsigned intRegIndex_; + unsigned floatRegIndex_; + uint32_t stackOffset_; + ABIArg current_; +}; + +// These registers may be volatile or nonvolatile. +static constexpr Register ABINonArgReg0 = t0; +static constexpr Register ABINonArgReg1 = t1; +static constexpr Register ABINonArgReg2 = t2; +static constexpr Register ABINonArgReg3 = t3; + +// This register may be volatile or nonvolatile. Avoid f31 which is the +// ScratchDoubleReg. +static constexpr FloatRegister ABINonArgDoubleReg{FloatRegisters::f21, + FloatRegisters::Double}; + +// These registers may be volatile or nonvolatile. +// Note: these three registers are all guaranteed to be different +static constexpr Register ABINonArgReturnReg0 = t0; +static constexpr Register ABINonArgReturnReg1 = t1; +static constexpr Register ABINonVolatileReg = s0; + +// This register is guaranteed to be clobberable during the prologue and +// epilogue of an ABI call which must preserve both ABI argument, return +// and non-volatile registers. +static constexpr Register ABINonArgReturnVolatileReg = ra; + +// TLS pointer argument register for WebAssembly functions. This must not alias +// any other register used for passing function arguments or return values. +// Preserved by WebAssembly functions. Must be nonvolatile. +static constexpr Register WasmTlsReg = s4; + +// Registers used for wasm table calls. These registers must be disjoint +// from the ABI argument registers, WasmTlsReg and each other. +static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0; +static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1; +static constexpr Register WasmTableCallSigReg = ABINonArgReg2; +static constexpr Register WasmTableCallIndexReg = ABINonArgReg3; + +// Register used as a scratch along the return path in the fast js -> wasm stub +// code. This must not overlap ReturnReg, JSReturnOperand, or WasmTlsReg. It +// must be a volatile register. +static constexpr Register WasmJitEntryReturnScratch = t1; + +// Register used to store a reference to an exception thrown by Wasm to an +// exception handling block. Should not overlap with WasmTlsReg. +static constexpr Register WasmExceptionReg = ABINonArgReg0; + +static constexpr Register InterpreterPCReg = t5; + +static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::f0, + FloatRegisters::Single}; +static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::f0, + FloatRegisters::Double}; +static constexpr FloatRegister ScratchFloat32Reg = {FloatRegisters::f31, + FloatRegisters::Single}; +static constexpr FloatRegister ScratchDoubleReg = {FloatRegisters::f31, + FloatRegisters::Double}; + +struct ScratchFloat32Scope : public AutoFloatRegisterScope { + explicit ScratchFloat32Scope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchFloat32Reg) {} +}; + +struct ScratchDoubleScope : public AutoFloatRegisterScope { + explicit ScratchDoubleScope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchDoubleReg) {} +}; + +static constexpr FloatRegister f0 = {FloatRegisters::f0, + FloatRegisters::Double}; +static constexpr FloatRegister f1 = {FloatRegisters::f1, + FloatRegisters::Double}; +static constexpr FloatRegister f2 = {FloatRegisters::f2, + FloatRegisters::Double}; +static constexpr FloatRegister f3 = {FloatRegisters::f3, + FloatRegisters::Double}; +static constexpr FloatRegister f4 = {FloatRegisters::f4, + FloatRegisters::Double}; +static constexpr FloatRegister f5 = {FloatRegisters::f5, + FloatRegisters::Double}; +static constexpr FloatRegister f6 = {FloatRegisters::f6, + FloatRegisters::Double}; +static constexpr FloatRegister f7 = {FloatRegisters::f7, + FloatRegisters::Double}; +static constexpr FloatRegister f8 = {FloatRegisters::f8, + FloatRegisters::Double}; +static constexpr FloatRegister f9 = {FloatRegisters::f9, + FloatRegisters::Double}; +static constexpr FloatRegister f10 = {FloatRegisters::f10, + FloatRegisters::Double}; +static constexpr FloatRegister f11 = {FloatRegisters::f11, + FloatRegisters::Double}; +static constexpr FloatRegister f12 = {FloatRegisters::f12, + FloatRegisters::Double}; +static constexpr FloatRegister f13 = {FloatRegisters::f13, + FloatRegisters::Double}; +static constexpr FloatRegister f14 = {FloatRegisters::f14, + FloatRegisters::Double}; +static constexpr FloatRegister f15 = {FloatRegisters::f15, + FloatRegisters::Double}; +static constexpr FloatRegister f16 = {FloatRegisters::f16, + FloatRegisters::Double}; +static constexpr FloatRegister f17 = {FloatRegisters::f17, + FloatRegisters::Double}; +static constexpr FloatRegister f18 = {FloatRegisters::f18, + FloatRegisters::Double}; +static constexpr FloatRegister f19 = {FloatRegisters::f19, + FloatRegisters::Double}; +static constexpr FloatRegister f20 = {FloatRegisters::f20, + FloatRegisters::Double}; +static constexpr FloatRegister f21 = {FloatRegisters::f21, + FloatRegisters::Double}; +static constexpr FloatRegister f22 = {FloatRegisters::f22, + FloatRegisters::Double}; +static constexpr FloatRegister f23 = {FloatRegisters::f23, + FloatRegisters::Double}; +static constexpr FloatRegister f24 = {FloatRegisters::f24, + FloatRegisters::Double}; +static constexpr FloatRegister f25 = {FloatRegisters::f25, + FloatRegisters::Double}; +static constexpr FloatRegister f26 = {FloatRegisters::f26, + FloatRegisters::Double}; +static constexpr FloatRegister f27 = {FloatRegisters::f27, + FloatRegisters::Double}; +static constexpr FloatRegister f28 = {FloatRegisters::f28, + FloatRegisters::Double}; +static constexpr FloatRegister f29 = {FloatRegisters::f29, + FloatRegisters::Double}; +static constexpr FloatRegister f30 = {FloatRegisters::f30, + FloatRegisters::Double}; +static constexpr FloatRegister f31 = {FloatRegisters::f31, + FloatRegisters::Double}; + +// LOONG64 CPUs can only load multibyte data that is "naturally" +// eight-byte-aligned, sp register should be sixteen-byte-aligned. +static constexpr uint32_t ABIStackAlignment = 16; +static constexpr uint32_t CodeAlignment = 16; +static constexpr uint32_t JitStackAlignment = 16; + +static constexpr uint32_t JitStackValueAlignment = + JitStackAlignment / sizeof(Value); +static_assert(JitStackAlignment % sizeof(Value) == 0 && + JitStackValueAlignment >= 1, + "Stack alignment should be a non-zero multiple of sizeof(Value)"); + +// TODO this is just a filler to prevent a build failure. The LA SIMD +// alignment requirements still need to be explored. +static constexpr uint32_t SimdMemoryAlignment = 16; + +static_assert(CodeAlignment % SimdMemoryAlignment == 0, + "Code alignment should be larger than any of the alignments " + "which are used for " + "the constant sections of the code buffer. Thus it should be " + "larger than the " + "alignment for SIMD constants."); + +static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment; +static const uint32_t WasmTrapInstructionLength = 4; + +// See comments in wasm::GenerateFunctionPrologue. The difference between these +// is the size of the largest callable prologue on the platform. +static constexpr uint32_t WasmCheckedCallEntryOffset = 0u; +static constexpr uint32_t WasmCheckedTailEntryOffset = 16u; + +static constexpr Scale ScalePointer = TimesEight; + +class Assembler : public AssemblerLOONG64 { + public: + Assembler() : AssemblerLOONG64() {} + + static uintptr_t GetPointer(uint8_t*); + + using AssemblerLOONG64::bind; + + static void Bind(uint8_t* rawCode, const CodeLabel& label); + + void processCodeLabels(uint8_t* rawCode); + + static void TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader); + static void TraceDataRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader); + + void bind(InstImm* inst, uintptr_t branch, uintptr_t target); + + // Copy the assembly code to the given buffer, and perform any pending + // relocations relying on the target address. + void executableCopy(uint8_t* buffer); + + static uint32_t PatchWrite_NearCallSize(); + + static uint64_t ExtractLoad64Value(Instruction* inst0); + static void UpdateLoad64Value(Instruction* inst0, uint64_t value); + static void WriteLoad64Instructions(Instruction* inst0, Register reg, + uint64_t value); + + static void PatchWrite_NearCall(CodeLocationLabel start, + CodeLocationLabel toCall); + static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue, + ImmPtr expectedValue); + static void PatchDataWithValueCheck(CodeLocationLabel label, + PatchedImmPtr newValue, + PatchedImmPtr expectedValue); + + static uint64_t ExtractInstructionImmediate(uint8_t* code); + + static void ToggleCall(CodeLocationLabel inst_, bool enabled); +}; // Assembler + +static const uint32_t NumIntArgRegs = 8; +static const uint32_t NumFloatArgRegs = 8; + +static inline bool GetIntArgReg(uint32_t usedIntArgs, Register* out) { + if (usedIntArgs < NumIntArgRegs) { + *out = Register::FromCode(a0.code() + usedIntArgs); + return true; + } + return false; +} + +static inline bool GetFloatArgReg(uint32_t usedFloatArgs, FloatRegister* out) { + if (usedFloatArgs < NumFloatArgRegs) { + *out = FloatRegister::FromCode(f0.code() + usedFloatArgs); + return true; + } + return false; +} + +// Get a register in which we plan to put a quantity that will be used as an +// integer argument. This differs from GetIntArgReg in that if we have no more +// actual argument registers to use we will fall back on using whatever +// CallTempReg* don't overlap the argument registers, and only fail once those +// run out too. +static inline bool GetTempRegForIntArg(uint32_t usedIntArgs, + uint32_t usedFloatArgs, Register* out) { + // NOTE: We can't properly determine which regs are used if there are + // float arguments. If this is needed, we will have to guess. + MOZ_ASSERT(usedFloatArgs == 0); + + if (GetIntArgReg(usedIntArgs, out)) { + return true; + } + // Unfortunately, we have to assume things about the point at which + // GetIntArgReg returns false, because we need to know how many registers it + // can allocate. + usedIntArgs -= NumIntArgRegs; + if (usedIntArgs >= NumCallTempNonArgRegs) { + return false; + } + *out = CallTempNonArgRegs[usedIntArgs]; + return true; +} + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_Assembler_loong64_h */ diff --git a/js/src/jit/loong64/AtomicOperations-loong64.h b/js/src/jit/loong64/AtomicOperations-loong64.h new file mode 100644 index 0000000000..b5e09614b6 --- /dev/null +++ b/js/src/jit/loong64/AtomicOperations-loong64.h @@ -0,0 +1,196 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* For documentation, see jit/AtomicOperations.h */ + +// Access to a aspecific 64-bit variable in memory is protected by an +// AddressLock whose instance is shared between jit and AtomicOperations. + +#ifndef jit_loong64_AtomicOperations_loong64_h +#define jit_loong64_AtomicOperations_loong64_h + +#include "mozilla/Assertions.h" +#include "mozilla/Types.h" + +#include "builtin/AtomicsObject.h" +#include "vm/Uint8Clamped.h" + +#if !defined(__clang__) && !defined(__GNUC__) +# error "This file only for gcc-compatible compilers" +#endif + +inline bool js::jit::AtomicOperations::Initialize() { + // Nothing + return true; +} + +inline void js::jit::AtomicOperations::ShutDown() { + // Nothing +} + +inline bool js::jit::AtomicOperations::hasAtomic8() { return true; } + +inline bool js::jit::AtomicOperations::isLockfree8() { + MOZ_ASSERT(__atomic_always_lock_free(sizeof(int8_t), 0)); + MOZ_ASSERT(__atomic_always_lock_free(sizeof(int16_t), 0)); + MOZ_ASSERT(__atomic_always_lock_free(sizeof(int32_t), 0)); + MOZ_ASSERT(__atomic_always_lock_free(sizeof(int64_t), 0)); + return true; +} + +inline void js::jit::AtomicOperations::fenceSeqCst() { + __atomic_thread_fence(__ATOMIC_SEQ_CST); +} + +template +inline T js::jit::AtomicOperations::loadSeqCst(T* addr) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + T v; + __atomic_load(addr, &v, __ATOMIC_SEQ_CST); + return v; +} + +template +inline void js::jit::AtomicOperations::storeSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + __atomic_store(addr, &val, __ATOMIC_SEQ_CST); +} + +template +inline T js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval, + T newval) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST); + return oldval; +} + +template +inline T js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST); +} + +template +inline T js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST); +} + +template +inline T js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST); +} + +template +inline T js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST); +} + +template +inline T js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST); +} + +template +inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + T v; + __atomic_load(addr, &v, __ATOMIC_RELAXED); + return v; +} + +namespace js { +namespace jit { + +template <> +inline uint8_clamped js::jit::AtomicOperations::loadSafeWhenRacy( + uint8_clamped* addr) { + uint8_t v; + __atomic_load(&addr->val, &v, __ATOMIC_RELAXED); + return uint8_clamped(v); +} + +template <> +inline float js::jit::AtomicOperations::loadSafeWhenRacy(float* addr) { + return *addr; +} + +template <> +inline double js::jit::AtomicOperations::loadSafeWhenRacy(double* addr) { + return *addr; +} + +} // namespace jit +} // namespace js + +template +inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + __atomic_store(addr, &val, __ATOMIC_RELAXED); +} + +namespace js { +namespace jit { + +template <> +inline void js::jit::AtomicOperations::storeSafeWhenRacy(uint8_clamped* addr, + uint8_clamped val) { + __atomic_store(&addr->val, &val.val, __ATOMIC_RELAXED); +} + +template <> +inline void js::jit::AtomicOperations::storeSafeWhenRacy(float* addr, + float val) { + *addr = val; +} + +template <> +inline void js::jit::AtomicOperations::storeSafeWhenRacy(double* addr, + double val) { + *addr = val; +} + +} // namespace jit +} // namespace js + +inline void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest, + const void* src, + size_t nbytes) { + MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes)); + MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes)); + ::memcpy(dest, src, nbytes); +} + +inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest, + const void* src, + size_t nbytes) { + ::memmove(dest, src, nbytes); +} + +template +inline T js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= sizeof(void*), + "atomics supported up to pointer size only"); + T v; + __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST); + return v; +} + +#endif // jit_loong64_AtomicOperations_loong64_h diff --git a/js/src/jit/loong64/Bailouts-loong64.cpp b/js/src/jit/loong64/Bailouts-loong64.cpp new file mode 100644 index 0000000000..3f2ec2cf09 --- /dev/null +++ b/js/src/jit/loong64/Bailouts-loong64.cpp @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Bailouts.h" +#include "jit/JitFrames.h" +#include "jit/SafepointIndex.h" + +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::jit; + +namespace js { +namespace jit { + +class BailoutStack { + RegisterDump::FPUArray fpregs_; + RegisterDump::GPRArray regs_; + uintptr_t frameSize_; + uintptr_t snapshotOffset_; + + public: + MachineState machineState() { + return MachineState::FromBailout(regs_, fpregs_); + } + uint32_t snapshotOffset() const { return snapshotOffset_; } + uint32_t frameSize() const { return frameSize_; } + uint8_t* parentStackPointer() { + return (uint8_t*)this + sizeof(BailoutStack); + } + static size_t offsetOfFrameSize() { + return offsetof(BailoutStack, frameSize_); + } +}; + +} // namespace jit +} // namespace js + +BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations, + BailoutStack* bailout) + : machine_(bailout->machineState()) { + uint8_t* sp = bailout->parentStackPointer(); + framePointer_ = sp + bailout->frameSize(); + topFrameSize_ = framePointer_ - sp; + + JSScript* script = + ScriptFromCalleeToken(((JitFrameLayout*)framePointer_)->calleeToken()); + topIonScript_ = script->ionScript(); + + attachOnJitActivation(activations); + snapshotOffset_ = bailout->snapshotOffset(); +} + +BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations, + InvalidationBailoutStack* bailout) + : machine_(bailout->machine()) { + framePointer_ = (uint8_t*)bailout->fp(); + topFrameSize_ = framePointer_ - bailout->sp(); + topIonScript_ = bailout->ionScript(); + attachOnJitActivation(activations); + + uint8_t* returnAddressToFp_ = bailout->osiPointReturnAddress(); + const OsiIndex* osiIndex = topIonScript_->getOsiIndex(returnAddressToFp_); + snapshotOffset_ = osiIndex->snapshotOffset(); +} diff --git a/js/src/jit/loong64/CodeGenerator-loong64.cpp b/js/src/jit/loong64/CodeGenerator-loong64.cpp new file mode 100644 index 0000000000..e5090bdce9 --- /dev/null +++ b/js/src/jit/loong64/CodeGenerator-loong64.cpp @@ -0,0 +1,2839 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loong64/CodeGenerator-loong64.h" + +#include "mozilla/MathAlgorithms.h" + +#include "jsnum.h" + +#include "jit/CodeGenerator.h" +#include "jit/InlineScriptTree.h" +#include "jit/JitRuntime.h" +#include "jit/MIR.h" +#include "jit/MIRGraph.h" +#include "vm/JSContext.h" +#include "vm/Realm.h" +#include "vm/Shape.h" +#include "vm/TraceLogging.h" + +#include "jit/shared/CodeGenerator-shared-inl.h" +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::jit; + +using JS::GenericNaN; +using mozilla::FloorLog2; +using mozilla::NegativeInfinity; + +// shared +CodeGeneratorLOONG64::CodeGeneratorLOONG64(MIRGenerator* gen, LIRGraph* graph, + MacroAssembler* masm) + : CodeGeneratorShared(gen, graph, masm) {} + +Operand CodeGeneratorLOONG64::ToOperand(const LAllocation& a) { + if (a.isGeneralReg()) { + return Operand(a.toGeneralReg()->reg()); + } + if (a.isFloatReg()) { + return Operand(a.toFloatReg()->reg()); + } + return Operand(ToAddress(a)); +} + +Operand CodeGeneratorLOONG64::ToOperand(const LAllocation* a) { + return ToOperand(*a); +} + +Operand CodeGeneratorLOONG64::ToOperand(const LDefinition* def) { + return ToOperand(def->output()); +} + +#ifdef JS_PUNBOX64 +Operand CodeGeneratorLOONG64::ToOperandOrRegister64( + const LInt64Allocation input) { + return ToOperand(input.value()); +} +#else +Register64 CodeGeneratorLOONG64::ToOperandOrRegister64( + const LInt64Allocation input) { + return ToRegister64(input); +} +#endif + +void CodeGeneratorLOONG64::branchToBlock(Assembler::FloatFormat fmt, + FloatRegister lhs, FloatRegister rhs, + MBasicBlock* mir, + Assembler::DoubleCondition cond) { + // Skip past trivial blocks. + Label* label = skipTrivialBlocks(mir)->lir()->label(); + if (fmt == Assembler::DoubleFloat) { + masm.branchDouble(cond, lhs, rhs, label); + } else { + masm.branchFloat(cond, lhs, rhs, label); + } +} + +FrameSizeClass FrameSizeClass::FromDepth(uint32_t frameDepth) { + return FrameSizeClass::None(); +} + +FrameSizeClass FrameSizeClass::ClassLimit() { return FrameSizeClass(0); } + +uint32_t FrameSizeClass::frameSize() const { + MOZ_CRASH("LA does not use frame size classes"); +} + +void OutOfLineBailout::accept(CodeGeneratorLOONG64* codegen) { + codegen->visitOutOfLineBailout(this); +} + +MoveOperand CodeGeneratorLOONG64::toMoveOperand(LAllocation a) const { + if (a.isGeneralReg()) { + return MoveOperand(ToRegister(a)); + } + if (a.isFloatReg()) { + return MoveOperand(ToFloatRegister(a)); + } + MoveOperand::Kind kind = + a.isStackArea() ? MoveOperand::EFFECTIVE_ADDRESS : MoveOperand::MEMORY; + Address address = ToAddress(a); + MOZ_ASSERT((address.offset & 3) == 0); + + return MoveOperand(address, kind); +} + +void CodeGeneratorLOONG64::bailoutFrom(Label* label, LSnapshot* snapshot) { + MOZ_ASSERT_IF(!masm.oom(), label->used()); + MOZ_ASSERT_IF(!masm.oom(), !label->bound()); + + encode(snapshot); + + // Though the assembler doesn't track all frame pushes, at least make sure + // the known value makes sense. We can't use bailout tables if the stack + // isn't properly aligned to the static frame size. + MOZ_ASSERT_IF(frameClass_ != FrameSizeClass::None(), + frameClass_.frameSize() == masm.framePushed()); + + // We don't use table bailouts because retargeting is easier this way. + InlineScriptTree* tree = snapshot->mir()->block()->trackedTree(); + OutOfLineBailout* ool = new (alloc()) OutOfLineBailout(snapshot); + addOutOfLineCode(ool, + new (alloc()) BytecodeSite(tree, tree->script()->code())); + + masm.retarget(label, ool->entry()); +} + +void CodeGeneratorLOONG64::bailout(LSnapshot* snapshot) { + Label label; + masm.jump(&label); + bailoutFrom(&label, snapshot); +} + +bool CodeGeneratorLOONG64::generateOutOfLineCode() { + if (!CodeGeneratorShared::generateOutOfLineCode()) { + return false; + } + + if (deoptLabel_.used()) { + // All non-table-based bailouts will go here. + masm.bind(&deoptLabel_); + + // Push the frame size, so the handler can recover the IonScript. + // Frame size is stored in 'ra' and pushed by GenerateBailoutThunk + // We have to use 'ra' because generateBailoutTable will implicitly do + // the same. + masm.move32(Imm32(frameSize()), ra); + + TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler(); + masm.jump(handler); + } + + return !masm.oom(); +} + +class js::jit::OutOfLineTableSwitch + : public OutOfLineCodeBase { + MTableSwitch* mir_; + CodeLabel jumpLabel_; + + void accept(CodeGeneratorLOONG64* codegen) { + codegen->visitOutOfLineTableSwitch(this); + } + + public: + OutOfLineTableSwitch(MTableSwitch* mir) : mir_(mir) {} + + MTableSwitch* mir() const { return mir_; } + + CodeLabel* jumpLabel() { return &jumpLabel_; } +}; + +void CodeGeneratorLOONG64::emitTableSwitchDispatch(MTableSwitch* mir, + Register index, + Register base) { + Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label(); + + // Lower value with low value + if (mir->low() != 0) { + masm.subPtr(Imm32(mir->low()), index); + } + + // Jump to default case if input is out of range + int32_t cases = mir->numCases(); + masm.branchPtr(Assembler::AboveOrEqual, index, ImmWord(cases), defaultcase); + + // To fill in the CodeLabels for the case entries, we need to first + // generate the case entries (we don't yet know their offsets in the + // instruction stream). + OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(mir); + addOutOfLineCode(ool, mir); + + // Compute the position where a pointer to the right case stands. + masm.ma_li(base, ool->jumpLabel()); + + BaseIndex pointer(base, index, ScalePointer); + + // Jump to the right case + masm.branchToComputedAddress(pointer); +} + +void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) { + MOZ_ASSERT(ins->tlsPtr()->isBogus()); + masm.movePtr(HeapReg, ToRegister(ins->output())); +} + +template +void CodeGeneratorLOONG64::emitWasmLoad(T* lir) { + const MWasmLoad* mir = lir->mir(); + SecondScratchRegisterScope scratch2(masm); + + Register ptr = ToRegister(lir->ptr()); + Register ptrScratch = InvalidReg; + if (!lir->ptrCopy()->isBogusTemp()) { + ptrScratch = ToRegister(lir->ptrCopy()); + } + + if (mir->base()->type() == MIRType::Int32) { + masm.move32To64ZeroExtend(ptr, Register64(scratch2)); + ptr = scratch2; + ptrScratch = ptrScratch != InvalidReg ? scratch2 : InvalidReg; + } + + masm.wasmLoad(mir->access(), HeapReg, ptr, ptrScratch, + ToAnyRegister(lir->output())); +} + +template +void CodeGeneratorLOONG64::emitWasmStore(T* lir) { + const MWasmStore* mir = lir->mir(); + SecondScratchRegisterScope scratch2(masm); + + Register ptr = ToRegister(lir->ptr()); + Register ptrScratch = InvalidReg; + if (!lir->ptrCopy()->isBogusTemp()) { + ptrScratch = ToRegister(lir->ptrCopy()); + } + + if (mir->base()->type() == MIRType::Int32) { + masm.move32To64ZeroExtend(ptr, Register64(scratch2)); + ptr = scratch2; + ptrScratch = ptrScratch != InvalidReg ? scratch2 : InvalidReg; + } + + masm.wasmStore(mir->access(), ToAnyRegister(lir->value()), HeapReg, ptr, + ptrScratch); +} + +void CodeGeneratorLOONG64::generateInvalidateEpilogue() { + // Ensure that there is enough space in the buffer for the OsiPoint + // patching to occur. Otherwise, we could overwrite the invalidation + // epilogue + for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) { + masm.nop(); + } + + masm.bind(&invalidate_); + + // Push the return address of the point that we bailed out at to the stack + masm.Push(ra); + + // Push the Ion script onto the stack (when we determine what that + // pointer is). + invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1))); + + // Jump to the invalidator which will replace the current frame. + TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk(); + + masm.jump(thunk); +} + +void CodeGeneratorLOONG64::visitOutOfLineBailout(OutOfLineBailout* ool) { + // Push snapshotOffset and make sure stack is aligned. + masm.subPtr(Imm32(sizeof(Value)), StackPointer); + masm.storePtr(ImmWord(ool->snapshot()->snapshotOffset()), + Address(StackPointer, 0)); + + masm.jump(&deoptLabel_); +} + +void CodeGeneratorLOONG64::visitOutOfLineTableSwitch( + OutOfLineTableSwitch* ool) { + MTableSwitch* mir = ool->mir(); + + masm.haltingAlign(sizeof(void*)); + masm.bind(ool->jumpLabel()); + masm.addCodeLabel(*ool->jumpLabel()); + + for (size_t i = 0; i < mir->numCases(); i++) { + LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir(); + Label* caseheader = caseblock->label(); + uint32_t caseoffset = caseheader->offset(); + + // The entries of the jump table need to be absolute addresses and thus + // must be patched after codegen is finished. + CodeLabel cl; + masm.writeCodePointer(&cl); + cl.target()->bind(caseoffset); + masm.addCodeLabel(cl); + } +} + +void CodeGeneratorLOONG64::visitOutOfLineWasmTruncateCheck( + OutOfLineWasmTruncateCheck* ool) { + if (ool->toType() == MIRType::Int32) { + masm.outOfLineWasmTruncateToInt32Check( + ool->input(), ool->output(), ool->fromType(), ool->flags(), + ool->rejoin(), ool->bytecodeOffset()); + } else { + MOZ_ASSERT(ool->toType() == MIRType::Int64); + masm.outOfLineWasmTruncateToInt64Check( + ool->input(), ool->output64(), ool->fromType(), ool->flags(), + ool->rejoin(), ool->bytecodeOffset()); + } +} + +ValueOperand CodeGeneratorLOONG64::ToValue(LInstruction* ins, size_t pos) { + return ValueOperand(ToRegister(ins->getOperand(pos))); +} + +ValueOperand CodeGeneratorLOONG64::ToTempValue(LInstruction* ins, size_t pos) { + return ValueOperand(ToRegister(ins->getTemp(pos))); +} + +void CodeGenerator::visitBox(LBox* box) { + const LAllocation* in = box->getOperand(0); + ValueOperand result = ToOutValue(box); + + masm.moveValue(TypedOrValueRegister(box->type(), ToAnyRegister(in)), result); +} + +void CodeGenerator::visitUnbox(LUnbox* unbox) { + MUnbox* mir = unbox->mir(); + + Register result = ToRegister(unbox->output()); + + if (mir->fallible()) { + const ValueOperand value = ToValue(unbox, LUnbox::Input); + Label bail; + switch (mir->type()) { + case MIRType::Int32: + masm.fallibleUnboxInt32(value, result, &bail); + break; + case MIRType::Boolean: + masm.fallibleUnboxBoolean(value, result, &bail); + break; + case MIRType::Object: + masm.fallibleUnboxObject(value, result, &bail); + break; + case MIRType::String: + masm.fallibleUnboxString(value, result, &bail); + break; + case MIRType::Symbol: + masm.fallibleUnboxSymbol(value, result, &bail); + break; + case MIRType::BigInt: + masm.fallibleUnboxBigInt(value, result, &bail); + break; + default: + MOZ_CRASH("Given MIRType cannot be unboxed."); + } + bailoutFrom(&bail, unbox->snapshot()); + return; + } + + LAllocation* input = unbox->getOperand(LUnbox::Input); + if (input->isRegister()) { + Register inputReg = ToRegister(input); + switch (mir->type()) { + case MIRType::Int32: + masm.unboxInt32(inputReg, result); + break; + case MIRType::Boolean: + masm.unboxBoolean(inputReg, result); + break; + case MIRType::Object: + masm.unboxObject(inputReg, result); + break; + case MIRType::String: + masm.unboxString(inputReg, result); + break; + case MIRType::Symbol: + masm.unboxSymbol(inputReg, result); + break; + case MIRType::BigInt: + masm.unboxBigInt(inputReg, result); + break; + default: + MOZ_CRASH("Given MIRType cannot be unboxed."); + } + return; + } + + Address inputAddr = ToAddress(input); + switch (mir->type()) { + case MIRType::Int32: + masm.unboxInt32(inputAddr, result); + break; + case MIRType::Boolean: + masm.unboxBoolean(inputAddr, result); + break; + case MIRType::Object: + masm.unboxObject(inputAddr, result); + break; + case MIRType::String: + masm.unboxString(inputAddr, result); + break; + case MIRType::Symbol: + masm.unboxSymbol(inputAddr, result); + break; + case MIRType::BigInt: + masm.unboxBigInt(inputAddr, result); + break; + default: + MOZ_CRASH("Given MIRType cannot be unboxed."); + } +} + +void CodeGeneratorLOONG64::splitTagForTest(const ValueOperand& value, + ScratchTagScope& tag) { + masm.splitTag(value.valueReg(), tag); +} + +void CodeGenerator::visitCompareI64(LCompareI64* lir) { + MCompare* mir = lir->mir(); + const mozilla::DebugOnly type = mir->compareType(); + MOZ_ASSERT(type == MCompare::Compare_Int64 || + type == MCompare::Compare_UInt64); + + const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs); + const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs); + Register lhsReg = ToRegister64(lhs).reg; + Register output = ToRegister(lir->output()); + bool isSigned = mir->compareType() == MCompare::Compare_Int64; + Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned); + + if (IsConstant(rhs)) { + masm.cmpPtrSet(cond, lhsReg, ImmWord(ToInt64(rhs)), output); + } else if (rhs.value().isGeneralReg()) { + masm.cmpPtrSet(cond, lhsReg, ToRegister64(rhs).reg, output); + } else { + masm.cmpPtrSet(cond, lhsReg, ToAddress(rhs.value()), output); + } +} + +void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) { + MCompare* mir = lir->cmpMir(); + const mozilla::DebugOnly type = mir->compareType(); + MOZ_ASSERT(type == MCompare::Compare_Int64 || + type == MCompare::Compare_UInt64); + + const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs); + const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs); + Register lhsReg = ToRegister64(lhs).reg; + bool isSigned = mir->compareType() == MCompare::Compare_Int64; + Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned); + + if (IsConstant(rhs)) { + emitBranch(lhsReg, ImmWord(ToInt64(rhs)), cond, lir->ifTrue(), + lir->ifFalse()); + } else if (rhs.value().isGeneralReg()) { + emitBranch(lhsReg, ToRegister64(rhs).reg, cond, lir->ifTrue(), + lir->ifFalse()); + } else { + emitBranch(lhsReg, ToAddress(rhs.value()), cond, lir->ifTrue(), + lir->ifFalse()); + } +} + +void CodeGenerator::visitCompare(LCompare* comp) { + MCompare* mir = comp->mir(); + Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop()); + const LAllocation* left = comp->getOperand(0); + const LAllocation* right = comp->getOperand(1); + const LDefinition* def = comp->getDef(0); + + if (mir->compareType() == MCompare::Compare_Object || + mir->compareType() == MCompare::Compare_Symbol || + mir->compareType() == MCompare::Compare_UIntPtr) { + if (right->isConstant()) { + MOZ_ASSERT(mir->compareType() == MCompare::Compare_UIntPtr); + masm.cmpPtrSet(cond, ToRegister(left), Imm32(ToInt32(right)), + ToRegister(def)); + } else if (right->isGeneralReg()) { + masm.cmpPtrSet(cond, ToRegister(left), ToRegister(right), + ToRegister(def)); + } else { + masm.cmpPtrSet(cond, ToRegister(left), ToAddress(right), ToRegister(def)); + } + return; + } + + if (right->isConstant()) { + masm.cmp32Set(cond, ToRegister(left), Imm32(ToInt32(right)), + ToRegister(def)); + } else if (right->isGeneralReg()) { + masm.cmp32Set(cond, ToRegister(left), ToRegister(right), ToRegister(def)); + } else { + masm.cmp32Set(cond, ToRegister(left), ToAddress(right), ToRegister(def)); + } +} + +void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) { + const MCompare* mir = comp->cmpMir(); + const MCompare::CompareType type = mir->compareType(); + const LAllocation* lhs = comp->left(); + const LAllocation* rhs = comp->right(); + MBasicBlock* ifTrue = comp->ifTrue(); + MBasicBlock* ifFalse = comp->ifFalse(); + Register lhsReg = ToRegister(lhs); + const Assembler::Condition cond = JSOpToCondition(type, comp->jsop()); + + if (type == MCompare::Compare_Object || type == MCompare::Compare_Symbol || + type == MCompare::Compare_UIntPtr) { + if (rhs->isConstant()) { + emitBranch(ToRegister(lhs), Imm32(ToInt32(rhs)), cond, ifTrue, ifFalse); + } else if (rhs->isGeneralReg()) { + emitBranch(lhsReg, ToRegister(rhs), cond, ifTrue, ifFalse); + } else { + } + return; + } + + if (rhs->isConstant()) { + emitBranch(lhsReg, Imm32(ToInt32(comp->right())), cond, ifTrue, ifFalse); + } else if (comp->right()->isGeneralReg()) { + emitBranch(lhsReg, ToRegister(rhs), cond, ifTrue, ifFalse); + } else { + ScratchRegisterScope scratch(masm); + // TODO:LOONG64 emitBranch with 32-bit comparision + masm.load32(ToAddress(rhs), scratch); + emitBranch(lhsReg, Register(scratch), cond, ifTrue, ifFalse); + } +} + +void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) { + Register lhs = ToRegister(lir->lhs()); + Register rhs = ToRegister(lir->rhs()); + Register output = ToRegister(lir->output()); + + Label done; + + // Handle divide by zero. + if (lir->canBeDivideByZero()) { + Label nonZero; + masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset()); + masm.bind(&nonZero); + } + + // Handle an integer overflow exception from INT64_MIN / -1. + if (lir->canBeNegativeOverflow()) { + Label notOverflow; + masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), ¬Overflow); + masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), ¬Overflow); + if (lir->mir()->isMod()) { + masm.as_xor(output, output, output); + } else { + masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset()); + } + masm.jump(&done); + masm.bind(¬Overflow); + } + + if (lir->mir()->isMod()) { + masm.as_mod_d(output, lhs, rhs); + } else { + masm.as_div_d(output, lhs, rhs); + } + + masm.bind(&done); +} + +void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) { + Register lhs = ToRegister(lir->lhs()); + Register rhs = ToRegister(lir->rhs()); + Register output = ToRegister(lir->output()); + + Label done; + + // Prevent divide by zero. + if (lir->canBeDivideByZero()) { + Label nonZero; + masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset()); + masm.bind(&nonZero); + } + + if (lir->mir()->isMod()) { + masm.as_mod_du(output, lhs, rhs); + } else { + masm.as_div_du(output, lhs, rhs); + } + + masm.bind(&done); +} + +void CodeGeneratorLOONG64::emitBigIntDiv(LBigIntDiv* ins, Register dividend, + Register divisor, Register output, + Label* fail) { + // Callers handle division by zero and integer overflow. + masm.as_div_d(/* result= */ dividend, dividend, divisor); + + // Create and return the result. + masm.newGCBigInt(output, divisor, fail, bigIntsCanBeInNursery()); + masm.initializeBigInt(output, dividend); +} + +void CodeGeneratorLOONG64::emitBigIntMod(LBigIntMod* ins, Register dividend, + Register divisor, Register output, + Label* fail) { + // Callers handle division by zero and integer overflow. + masm.as_mod_d(/* result= */ dividend, dividend, divisor); + + // Create and return the result. + masm.newGCBigInt(output, divisor, fail, bigIntsCanBeInNursery()); + masm.initializeBigInt(output, dividend); +} + +void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir) { + const MWasmLoad* mir = lir->mir(); + + Register ptrScratch = InvalidReg; + if (!lir->ptrCopy()->isBogusTemp()) { + ptrScratch = ToRegister(lir->ptrCopy()); + } + + masm.wasmLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()), ptrScratch, + ToOutRegister64(lir)); +} + +void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir) { + const MWasmStore* mir = lir->mir(); + + Register ptrScratch = InvalidReg; + if (!lir->ptrCopy()->isBogusTemp()) { + ptrScratch = ToRegister(lir->ptrCopy()); + } + + masm.wasmStoreI64(mir->access(), ToRegister64(lir->value()), HeapReg, + ToRegister(lir->ptr()), ptrScratch); +} + +void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) { + MOZ_ASSERT(lir->mir()->type() == MIRType::Int64); + + Register cond = ToRegister(lir->condExpr()); + const LInt64Allocation falseExpr = lir->falseExpr(); + + Register64 out = ToOutRegister64(lir); + MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out, + "true expr is reused for input"); + + if (falseExpr.value().isRegister()) { + masm.moveIfZero(out.reg, ToRegister(falseExpr.value()), cond); + } else { + Label done; + masm.ma_b(cond, cond, &done, Assembler::NonZero, ShortJump); + masm.loadPtr(ToAddress(falseExpr.value()), out.reg); + masm.bind(&done); + } +} + +void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) { + MOZ_ASSERT(lir->mir()->type() == MIRType::Double); + MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64); + masm.as_movgr2fr_d(ToFloatRegister(lir->output()), ToRegister(lir->input())); +} + +void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) { + MOZ_ASSERT(lir->mir()->type() == MIRType::Int64); + MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double); + masm.as_movfr2gr_d(ToRegister(lir->output()), ToFloatRegister(lir->input())); +} + +void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) { + const LAllocation* input = lir->getOperand(0); + Register output = ToRegister(lir->output()); + + if (lir->mir()->isUnsigned()) { + masm.as_bstrpick_d(output, ToRegister(input), 31, 0); + } else { + masm.as_slli_w(output, ToRegister(input), 0); + } +} + +void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) { + const LAllocation* input = lir->getOperand(0); + Register output = ToRegister(lir->output()); + + if (lir->mir()->bottomHalf()) { + if (input->isMemory()) { + masm.load32(ToAddress(input), output); + } else { + masm.as_slli_w(output, ToRegister(input), 0); + } + } else { + MOZ_CRASH("Not implemented."); + } +} + +void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + Register64 output = ToOutRegister64(lir); + switch (lir->mode()) { + case MSignExtendInt64::Byte: + masm.move32To64SignExtend(input.reg, output); + masm.move8SignExtend(output.reg, output.reg); + break; + case MSignExtendInt64::Half: + masm.move32To64SignExtend(input.reg, output); + masm.move16SignExtend(output.reg, output.reg); + break; + case MSignExtendInt64::Word: + masm.move32To64SignExtend(input.reg, output); + break; + } +} + +void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) { + Register input = ToRegister(lir->input()); + Register output = ToRegister(lir->output()); + MOZ_ASSERT(input == output); + masm.move32To64ZeroExtend(input, Register64(output)); +} + +void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) { + Register input = ToRegister(lir->input()); + Register output = ToRegister(lir->output()); + MOZ_ASSERT(input == output); + masm.move64To32(Register64(input), output); +} + +void CodeGenerator::visitClzI64(LClzI64* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + Register64 output = ToOutRegister64(lir); + masm.clz64(input, output.reg); +} + +void CodeGenerator::visitCtzI64(LCtzI64* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + Register64 output = ToOutRegister64(lir); + masm.ctz64(input, output.reg); +} + +void CodeGenerator::visitNotI64(LNotI64* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + Register output = ToRegister(lir->output()); + + masm.cmp64Set(Assembler::Equal, input.reg, Imm32(0), output); +} + +void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) { + FloatRegister input = ToFloatRegister(lir->input()); + Register64 output = ToOutRegister64(lir); + + MWasmTruncateToInt64* mir = lir->mir(); + MIRType fromType = mir->input()->type(); + + MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32); + + auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output); + addOutOfLineCode(ool, mir); + + Label* oolEntry = ool->entry(); + Label* oolRejoin = ool->rejoin(); + bool isSaturating = mir->isSaturating(); + + if (fromType == MIRType::Double) { + if (mir->isUnsigned()) { + masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry, + oolRejoin, InvalidFloatReg); + } else { + masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry, + oolRejoin, InvalidFloatReg); + } + } else { + if (mir->isUnsigned()) { + masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry, + oolRejoin, InvalidFloatReg); + } else { + masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry, + oolRejoin, InvalidFloatReg); + } + } +} + +void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + FloatRegister output = ToFloatRegister(lir->output()); + + MIRType outputType = lir->mir()->type(); + MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32); + + if (outputType == MIRType::Double) { + if (lir->mir()->isUnsigned()) { + masm.convertUInt64ToDouble(input, output, Register::Invalid()); + } else { + masm.convertInt64ToDouble(input, output); + } + } else { + if (lir->mir()->isUnsigned()) { + masm.convertUInt64ToFloat32(input, output, Register::Invalid()); + } else { + masm.convertInt64ToFloat32(input, output); + } + } +} + +void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + MBasicBlock* ifTrue = lir->ifTrue(); + MBasicBlock* ifFalse = lir->ifFalse(); + + emitBranch(input.reg, Imm32(0), Assembler::NonZero, ifTrue, ifFalse); +} + +void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) { + const LAllocation* opd = test->getOperand(0); + MBasicBlock* ifTrue = test->ifTrue(); + MBasicBlock* ifFalse = test->ifFalse(); + + emitBranch(ToRegister(opd), Imm32(0), Assembler::NonZero, ifTrue, ifFalse); +} + +void CodeGenerator::visitMinMaxD(LMinMaxD* ins) { + FloatRegister first = ToFloatRegister(ins->first()); + FloatRegister second = ToFloatRegister(ins->second()); + + MOZ_ASSERT(first == ToFloatRegister(ins->output())); + + if (ins->mir()->isMax()) { + masm.maxDouble(second, first, true); + } else { + masm.minDouble(second, first, true); + } +} + +void CodeGenerator::visitMinMaxF(LMinMaxF* ins) { + FloatRegister first = ToFloatRegister(ins->first()); + FloatRegister second = ToFloatRegister(ins->second()); + + MOZ_ASSERT(first == ToFloatRegister(ins->output())); + + if (ins->mir()->isMax()) { + masm.maxFloat32(second, first, true); + } else { + masm.minFloat32(second, first, true); + } +} + +void CodeGenerator::visitAddI(LAddI* ins) { + const LAllocation* lhs = ins->getOperand(0); + const LAllocation* rhs = ins->getOperand(1); + const LDefinition* dest = ins->getDef(0); + + MOZ_ASSERT(rhs->isConstant() || rhs->isGeneralReg()); + + // If there is no snapshot, we don't need to check for overflow + if (!ins->snapshot()) { + if (rhs->isConstant()) { + masm.ma_add_w(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs))); + } else { + masm.as_add_w(ToRegister(dest), ToRegister(lhs), ToRegister(rhs)); + } + return; + } + + Label overflow; + if (rhs->isConstant()) { + masm.ma_add32TestOverflow(ToRegister(dest), ToRegister(lhs), + Imm32(ToInt32(rhs)), &overflow); + } else { + masm.ma_add32TestOverflow(ToRegister(dest), ToRegister(lhs), + ToRegister(rhs), &overflow); + } + + bailoutFrom(&overflow, ins->snapshot()); +} + +void CodeGenerator::visitAddI64(LAddI64* lir) { + const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs); + const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs); + + MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs)); + + if (IsConstant(rhs)) { + masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs)); + return; + } + + masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs)); +} + +void CodeGenerator::visitSubI(LSubI* ins) { + const LAllocation* lhs = ins->getOperand(0); + const LAllocation* rhs = ins->getOperand(1); + const LDefinition* dest = ins->getDef(0); + + MOZ_ASSERT(rhs->isConstant() || rhs->isGeneralReg()); + + // If there is no snapshot, we don't need to check for overflow + + if (!ins->snapshot()) { + if (rhs->isConstant()) { + masm.ma_sub_w(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs))); + } else { + masm.as_sub_w(ToRegister(dest), ToRegister(lhs), ToRegister(rhs)); + } + return; + } + + Label overflow; + if (rhs->isConstant()) { + masm.ma_sub32TestOverflow(ToRegister(dest), ToRegister(lhs), + Imm32(ToInt32(rhs)), &overflow); + } else { + masm.ma_sub32TestOverflow(ToRegister(dest), ToRegister(lhs), + ToRegister(rhs), &overflow); + } + + bailoutFrom(&overflow, ins->snapshot()); +} + +void CodeGenerator::visitSubI64(LSubI64* lir) { + const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs); + const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs); + + MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs)); + + if (IsConstant(rhs)) { + masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs)); + return; + } + + masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs)); +} + +void CodeGenerator::visitMulI(LMulI* ins) { + const LAllocation* lhs = ins->lhs(); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + MMul* mul = ins->mir(); + + MOZ_ASSERT_IF(mul->mode() == MMul::Integer, + !mul->canBeNegativeZero() && !mul->canOverflow()); + + if (rhs->isConstant()) { + int32_t constant = ToInt32(rhs); + Register src = ToRegister(lhs); + + // Bailout on -0.0 + if (mul->canBeNegativeZero() && constant <= 0) { + Assembler::Condition cond = + (constant == 0) ? Assembler::LessThan : Assembler::Equal; + bailoutCmp32(cond, src, Imm32(0), ins->snapshot()); + } + + switch (constant) { + case -1: + if (mul->canOverflow()) { + bailoutCmp32(Assembler::Equal, src, Imm32(INT32_MIN), + ins->snapshot()); + } + + masm.as_sub_w(dest, zero, src); + break; + case 0: + masm.move32(zero, dest); + break; + case 1: + masm.move32(src, dest); + break; + case 2: + if (mul->canOverflow()) { + Label mulTwoOverflow; + masm.ma_add32TestOverflow(dest, src, src, &mulTwoOverflow); + + bailoutFrom(&mulTwoOverflow, ins->snapshot()); + } else { + masm.as_add_w(dest, src, src); + } + break; + default: + uint32_t shift = FloorLog2(constant); + + if (!mul->canOverflow() && (constant > 0)) { + // If it cannot overflow, we can do lots of optimizations. + uint32_t rest = constant - (1 << shift); + + // See if the constant has one bit set, meaning it can be + // encoded as a bitshift. + if ((1 << shift) == constant) { + masm.as_slli_w(dest, src, shift % 32); + return; + } + + // If the constant cannot be encoded as (1<canOverflow() && (constant > 0) && (src != dest)) { + // To stay on the safe side, only optimize things that are a + // power of 2. + + if ((1 << shift) == constant) { + ScratchRegisterScope scratch(masm); + // dest = lhs * pow(2, shift) + masm.as_slli_w(dest, src, shift % 32); + // At runtime, check (lhs == dest >> shift), if this does + // not hold, some bits were lost due to overflow, and the + // computation should be resumed as a double. + masm.as_srai_w(scratch, dest, shift % 32); + bailoutCmp32(Assembler::NotEqual, src, Register(scratch), + ins->snapshot()); + return; + } + } + + if (mul->canOverflow()) { + Label mulConstOverflow; + masm.ma_mul32TestOverflow(dest, ToRegister(lhs), Imm32(ToInt32(rhs)), + &mulConstOverflow); + + bailoutFrom(&mulConstOverflow, ins->snapshot()); + } else { + masm.ma_mul(dest, src, Imm32(ToInt32(rhs))); + } + break; + } + } else { + Label multRegOverflow; + + if (mul->canOverflow()) { + masm.ma_mul32TestOverflow(dest, ToRegister(lhs), ToRegister(rhs), + &multRegOverflow); + bailoutFrom(&multRegOverflow, ins->snapshot()); + } else { + masm.as_mul_w(dest, ToRegister(lhs), ToRegister(rhs)); + } + + if (mul->canBeNegativeZero()) { + Label done; + masm.ma_b(dest, dest, &done, Assembler::NonZero, ShortJump); + + // Result is -0 if lhs or rhs is negative. + // In that case result must be double value so bailout + Register scratch = SecondScratchReg; + masm.as_or(scratch, ToRegister(lhs), ToRegister(rhs)); + bailoutCmp32(Assembler::Signed, scratch, scratch, ins->snapshot()); + + masm.bind(&done); + } + } +} + +void CodeGenerator::visitMulI64(LMulI64* lir) { + const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs); + const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs); + const Register64 output = ToOutRegister64(lir); + + if (IsConstant(rhs)) { + int64_t constant = ToInt64(rhs); + switch (constant) { + case -1: + masm.neg64(ToRegister64(lhs)); + return; + case 0: + masm.xor64(ToRegister64(lhs), ToRegister64(lhs)); + return; + case 1: + // nop + return; + default: + if (constant > 0) { + if (mozilla::IsPowerOfTwo(static_cast(constant + 1))) { + masm.move64(ToRegister64(lhs), output); + masm.lshift64(Imm32(FloorLog2(constant + 1)), output); + masm.sub64(ToRegister64(lhs), output); + return; + } else if (mozilla::IsPowerOfTwo( + static_cast(constant - 1))) { + masm.move64(ToRegister64(lhs), output); + masm.lshift64(Imm32(FloorLog2(constant - 1u)), output); + masm.add64(ToRegister64(lhs), output); + return; + } + // Use shift if constant is power of 2. + int32_t shift = mozilla::FloorLog2(constant); + if (int64_t(1) << shift == constant) { + masm.lshift64(Imm32(shift), ToRegister64(lhs)); + return; + } + } + Register temp = ToTempRegisterOrInvalid(lir->temp()); + masm.mul64(Imm64(constant), ToRegister64(lhs), temp); + } + } else { + Register temp = ToTempRegisterOrInvalid(lir->temp()); + masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp); + } +} + +void CodeGenerator::visitDivI(LDivI* ins) { + // Extract the registers from this instruction + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register dest = ToRegister(ins->output()); + Register temp = ToRegister(ins->getTemp(0)); + MDiv* mir = ins->mir(); + + Label done; + + // Handle divide by zero. + if (mir->canBeDivideByZero()) { + if (mir->trapOnError()) { + Label nonZero; + masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset()); + masm.bind(&nonZero); + } else if (mir->canTruncateInfinities()) { + // Truncated division by zero is zero (Infinity|0 == 0) + Label notzero; + masm.ma_b(rhs, rhs, ¬zero, Assembler::NonZero, ShortJump); + masm.move32(Imm32(0), dest); + masm.ma_b(&done, ShortJump); + masm.bind(¬zero); + } else { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Zero, rhs, rhs, ins->snapshot()); + } + } + + // Handle an integer overflow exception from -2147483648 / -1. + if (mir->canBeNegativeOverflow()) { + Label notMinInt; + masm.move32(Imm32(INT32_MIN), temp); + masm.ma_b(lhs, temp, ¬MinInt, Assembler::NotEqual, ShortJump); + + masm.move32(Imm32(-1), temp); + if (mir->trapOnError()) { + Label ok; + masm.ma_b(rhs, temp, &ok, Assembler::NotEqual); + masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset()); + masm.bind(&ok); + } else if (mir->canTruncateOverflow()) { + // (-INT32_MIN)|0 == INT32_MIN + Label skip; + masm.ma_b(rhs, temp, &skip, Assembler::NotEqual, ShortJump); + masm.move32(Imm32(INT32_MIN), dest); + masm.ma_b(&done, ShortJump); + masm.bind(&skip); + } else { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, rhs, temp, ins->snapshot()); + } + masm.bind(¬MinInt); + } + + // Handle negative 0. (0/-Y) + if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) { + Label nonzero; + masm.ma_b(lhs, lhs, &nonzero, Assembler::NonZero, ShortJump); + bailoutCmp32(Assembler::LessThan, rhs, Imm32(0), ins->snapshot()); + masm.bind(&nonzero); + } + // Note: above safety checks could not be verified as Ion seems to be + // smarter and requires double arithmetic in such cases. + + // All regular. Lets call div. + if (mir->canTruncateRemainder()) { + masm.as_div_w(dest, lhs, rhs); + } else { + MOZ_ASSERT(mir->fallible()); + + Label remainderNonZero; + masm.ma_div_branch_overflow(dest, lhs, rhs, &remainderNonZero); + bailoutFrom(&remainderNonZero, ins->snapshot()); + } + + masm.bind(&done); +} + +void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) { + Register lhs = ToRegister(ins->numerator()); + Register dest = ToRegister(ins->output()); + Register tmp = ToRegister(ins->getTemp(0)); + int32_t shift = ins->shift(); + + if (shift != 0) { + MDiv* mir = ins->mir(); + if (!mir->isTruncated()) { + // If the remainder is going to be != 0, bailout since this must + // be a double. + masm.as_slli_w(tmp, lhs, (32 - shift) % 32); + bailoutCmp32(Assembler::NonZero, tmp, tmp, ins->snapshot()); + } + + if (!mir->canBeNegativeDividend()) { + // Numerator is unsigned, so needs no adjusting. Do the shift. + masm.as_srai_w(dest, lhs, shift % 32); + return; + } + + // Adjust the value so that shifting produces a correctly rounded result + // when the numerator is negative. See 10-1 "Signed Division by a Known + // Power of 2" in Henry S. Warren, Jr.'s Hacker's Delight. + if (shift > 1) { + masm.as_srai_w(tmp, lhs, 31); + masm.as_srli_w(tmp, tmp, (32 - shift) % 32); + masm.add32(lhs, tmp); + } else { + masm.as_srli_w(tmp, lhs, (32 - shift) % 32); + masm.add32(lhs, tmp); + } + + // Do the shift. + masm.as_srai_w(dest, tmp, shift % 32); + } else { + masm.move32(lhs, dest); + } +} + +void CodeGenerator::visitModI(LModI* ins) { + // Extract the registers from this instruction + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register dest = ToRegister(ins->output()); + Register callTemp = ToRegister(ins->callTemp()); + MMod* mir = ins->mir(); + Label done, prevent; + + masm.move32(lhs, callTemp); + + // Prevent INT_MIN % -1; + // The integer division will give INT_MIN, but we want -(double)INT_MIN. + if (mir->canBeNegativeDividend()) { + masm.ma_b(lhs, Imm32(INT_MIN), &prevent, Assembler::NotEqual, ShortJump); + if (mir->isTruncated()) { + // (INT_MIN % -1)|0 == 0 + Label skip; + masm.ma_b(rhs, Imm32(-1), &skip, Assembler::NotEqual, ShortJump); + masm.move32(Imm32(0), dest); + masm.ma_b(&done, ShortJump); + masm.bind(&skip); + } else { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, rhs, Imm32(-1), ins->snapshot()); + } + masm.bind(&prevent); + } + + // 0/X (with X < 0) is bad because both of these values *should* be + // doubles, and the result should be -0.0, which cannot be represented in + // integers. X/0 is bad because it will give garbage (or abort), when it + // should give either \infty, -\infty or NAN. + + // Prevent 0 / X (with X < 0) and X / 0 + // testing X / Y. Compare Y with 0. + // There are three cases: (Y < 0), (Y == 0) and (Y > 0) + // If (Y < 0), then we compare X with 0, and bail if X == 0 + // If (Y == 0), then we simply want to bail. + // if (Y > 0), we don't bail. + + if (mir->canBeDivideByZero()) { + if (mir->isTruncated()) { + if (mir->trapOnError()) { + Label nonZero; + masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset()); + masm.bind(&nonZero); + } else { + Label skip; + masm.ma_b(rhs, Imm32(0), &skip, Assembler::NotEqual, ShortJump); + masm.move32(Imm32(0), dest); + masm.ma_b(&done, ShortJump); + masm.bind(&skip); + } + } else { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot()); + } + } + + if (mir->canBeNegativeDividend()) { + Label notNegative; + masm.ma_b(rhs, Imm32(0), ¬Negative, Assembler::GreaterThan, ShortJump); + if (mir->isTruncated()) { + // NaN|0 == 0 and (0 % -X)|0 == 0 + Label skip; + masm.ma_b(lhs, Imm32(0), &skip, Assembler::NotEqual, ShortJump); + masm.move32(Imm32(0), dest); + masm.ma_b(&done, ShortJump); + masm.bind(&skip); + } else { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, lhs, Imm32(0), ins->snapshot()); + } + masm.bind(¬Negative); + } + + masm.as_mod_w(dest, lhs, rhs); + + // If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0 + if (mir->canBeNegativeDividend()) { + if (mir->isTruncated()) { + // -0.0|0 == 0 + } else { + MOZ_ASSERT(mir->fallible()); + // See if X < 0 + masm.ma_b(dest, Imm32(0), &done, Assembler::NotEqual, ShortJump); + bailoutCmp32(Assembler::Signed, callTemp, Imm32(0), ins->snapshot()); + } + } + masm.bind(&done); +} + +void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) { + Register in = ToRegister(ins->getOperand(0)); + Register out = ToRegister(ins->getDef(0)); + MMod* mir = ins->mir(); + Label negative, done; + + masm.move32(in, out); + masm.ma_b(in, in, &done, Assembler::Zero, ShortJump); + // Switch based on sign of the lhs. + // Positive numbers are just a bitmask + masm.ma_b(in, in, &negative, Assembler::Signed, ShortJump); + { + masm.and32(Imm32((1 << ins->shift()) - 1), out); + masm.ma_b(&done, ShortJump); + } + + // Negative numbers need a negate, bitmask, negate + { + masm.bind(&negative); + masm.neg32(out); + masm.and32(Imm32((1 << ins->shift()) - 1), out); + masm.neg32(out); + } + if (mir->canBeNegativeDividend()) { + if (!mir->isTruncated()) { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, out, zero, ins->snapshot()); + } else { + // -0|0 == 0 + } + } + masm.bind(&done); +} + +void CodeGenerator::visitModMaskI(LModMaskI* ins) { + Register src = ToRegister(ins->getOperand(0)); + Register dest = ToRegister(ins->getDef(0)); + Register tmp0 = ToRegister(ins->getTemp(0)); + Register tmp1 = ToRegister(ins->getTemp(1)); + MMod* mir = ins->mir(); + + if (!mir->isTruncated() && mir->canBeNegativeDividend()) { + MOZ_ASSERT(mir->fallible()); + + Label bail; + masm.ma_mod_mask(src, dest, tmp0, tmp1, ins->shift(), &bail); + bailoutFrom(&bail, ins->snapshot()); + } else { + masm.ma_mod_mask(src, dest, tmp0, tmp1, ins->shift(), nullptr); + } +} + +void CodeGenerator::visitBitNotI(LBitNotI* ins) { + const LAllocation* input = ins->getOperand(0); + const LDefinition* dest = ins->getDef(0); + MOZ_ASSERT(!input->isConstant()); + + masm.as_nor(ToRegister(dest), ToRegister(input), zero); +} + +void CodeGenerator::visitBitOpI(LBitOpI* ins) { + const LAllocation* lhs = ins->getOperand(0); + const LAllocation* rhs = ins->getOperand(1); + const LDefinition* dest = ins->getDef(0); + // all of these bitops should be either imm32's, or integer registers. + switch (ins->bitop()) { + case JSOp::BitOr: + if (rhs->isConstant()) { + masm.ma_or(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)), + true); + } else { + masm.as_or(ToRegister(dest), ToRegister(lhs), ToRegister(rhs)); + masm.as_slli_w(ToRegister(dest), ToRegister(dest), 0); + } + break; + case JSOp::BitXor: + if (rhs->isConstant()) { + masm.ma_xor(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)), + true); + } else { + masm.as_xor(ToRegister(dest), ToRegister(lhs), ToRegister(rhs)); + masm.as_slli_w(ToRegister(dest), ToRegister(dest), 0); + } + break; + case JSOp::BitAnd: + if (rhs->isConstant()) { + masm.ma_and(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)), + true); + } else { + masm.as_and(ToRegister(dest), ToRegister(lhs), ToRegister(rhs)); + masm.as_slli_w(ToRegister(dest), ToRegister(dest), 0); + } + break; + default: + MOZ_CRASH("unexpected binary opcode"); + } +} + +void CodeGenerator::visitBitOpI64(LBitOpI64* lir) { + const LInt64Allocation lhs = lir->getInt64Operand(LBitOpI64::Lhs); + const LInt64Allocation rhs = lir->getInt64Operand(LBitOpI64::Rhs); + + MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs)); + + switch (lir->bitop()) { + case JSOp::BitOr: + if (IsConstant(rhs)) { + masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs)); + } else { + masm.or64(ToOperandOrRegister64(rhs), ToRegister64(lhs)); + } + break; + case JSOp::BitXor: + if (IsConstant(rhs)) { + masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs)); + } else { + masm.xor64(ToOperandOrRegister64(rhs), ToRegister64(lhs)); + } + break; + case JSOp::BitAnd: + if (IsConstant(rhs)) { + masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs)); + } else { + masm.and64(ToOperandOrRegister64(rhs), ToRegister64(lhs)); + } + break; + default: + MOZ_CRASH("unexpected binary opcode"); + } +} + +void CodeGenerator::visitShiftI(LShiftI* ins) { + Register lhs = ToRegister(ins->lhs()); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + + if (rhs->isConstant()) { + int32_t shift = ToInt32(rhs) & 0x1F; + switch (ins->bitop()) { + case JSOp::Lsh: + if (shift) { + masm.as_slli_w(dest, lhs, shift % 32); + } else { + masm.move32(lhs, dest); + } + break; + case JSOp::Rsh: + if (shift) { + masm.as_srai_w(dest, lhs, shift % 32); + } else { + masm.move32(lhs, dest); + } + break; + case JSOp::Ursh: + if (shift) { + masm.as_srli_w(dest, lhs, shift % 32); + } else { + // x >>> 0 can overflow. + if (ins->mir()->toUrsh()->fallible()) { + bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot()); + } + masm.move32(lhs, dest); + } + break; + default: + MOZ_CRASH("Unexpected shift op"); + } + } else { + // The shift amounts should be AND'ed into the 0-31 range + masm.ma_and(dest, ToRegister(rhs), Imm32(0x1F)); + + switch (ins->bitop()) { + case JSOp::Lsh: + masm.as_sll_w(dest, lhs, dest); + break; + case JSOp::Rsh: + masm.as_sra_w(dest, lhs, dest); + break; + case JSOp::Ursh: + masm.as_srl_w(dest, lhs, dest); + if (ins->mir()->toUrsh()->fallible()) { + // x >>> 0 can overflow. + bailoutCmp32(Assembler::LessThan, dest, Imm32(0), ins->snapshot()); + } + break; + default: + MOZ_CRASH("Unexpected shift op"); + } + } +} + +void CodeGenerator::visitShiftI64(LShiftI64* lir) { + const LInt64Allocation lhs = lir->getInt64Operand(LShiftI64::Lhs); + LAllocation* rhs = lir->getOperand(LShiftI64::Rhs); + + MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs)); + + if (rhs->isConstant()) { + int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F); + switch (lir->bitop()) { + case JSOp::Lsh: + if (shift) { + masm.lshift64(Imm32(shift), ToRegister64(lhs)); + } + break; + case JSOp::Rsh: + if (shift) { + masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs)); + } + break; + case JSOp::Ursh: + if (shift) { + masm.rshift64(Imm32(shift), ToRegister64(lhs)); + } + break; + default: + MOZ_CRASH("Unexpected shift op"); + } + return; + } + + switch (lir->bitop()) { + case JSOp::Lsh: + masm.lshift64(ToRegister(rhs), ToRegister64(lhs)); + break; + case JSOp::Rsh: + masm.rshift64Arithmetic(ToRegister(rhs), ToRegister64(lhs)); + break; + case JSOp::Ursh: + masm.rshift64(ToRegister(rhs), ToRegister64(lhs)); + break; + default: + MOZ_CRASH("Unexpected shift op"); + } +} + +void CodeGenerator::visitRotateI64(LRotateI64* lir) { + MRotate* mir = lir->mir(); + LAllocation* count = lir->count(); + + Register64 input = ToRegister64(lir->input()); + Register64 output = ToOutRegister64(lir); + Register temp = ToTempRegisterOrInvalid(lir->temp()); + + MOZ_ASSERT(input == output); + + if (count->isConstant()) { + int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F); + if (!c) { + return; + } + if (mir->isLeftRotate()) { + masm.rotateLeft64(Imm32(c), input, output, temp); + } else { + masm.rotateRight64(Imm32(c), input, output, temp); + } + } else { + if (mir->isLeftRotate()) { + masm.rotateLeft64(ToRegister(count), input, output, temp); + } else { + masm.rotateRight64(ToRegister(count), input, output, temp); + } + } +} + +void CodeGenerator::visitUrshD(LUrshD* ins) { + Register lhs = ToRegister(ins->lhs()); + Register temp = ToRegister(ins->temp()); + + const LAllocation* rhs = ins->rhs(); + FloatRegister out = ToFloatRegister(ins->output()); + + if (rhs->isConstant()) { + masm.as_srli_w(temp, lhs, ToInt32(rhs) % 32); + } else { + masm.as_srl_w(temp, lhs, ToRegister(rhs)); + } + + masm.convertUInt32ToDouble(temp, out); +} + +void CodeGenerator::visitClzI(LClzI* ins) { + Register input = ToRegister(ins->input()); + Register output = ToRegister(ins->output()); + + masm.as_clz_w(output, input); +} + +void CodeGenerator::visitCtzI(LCtzI* ins) { + Register input = ToRegister(ins->input()); + Register output = ToRegister(ins->output()); + + masm.as_ctz_w(output, input); +} + +void CodeGenerator::visitPopcntI(LPopcntI* ins) { + Register input = ToRegister(ins->input()); + Register output = ToRegister(ins->output()); + Register tmp = ToRegister(ins->temp()); + + masm.popcnt32(input, output, tmp); +} + +void CodeGenerator::visitPopcntI64(LPopcntI64* ins) { + Register64 input = ToRegister64(ins->getInt64Operand(0)); + Register64 output = ToOutRegister64(ins); + Register tmp = ToRegister(ins->getTemp(0)); + + masm.popcnt64(input, output, tmp); +} + +void CodeGenerator::visitPowHalfD(LPowHalfD* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + FloatRegister output = ToFloatRegister(ins->output()); + + Label done, skip; + + // Masm.pow(-Infinity, 0.5) == Infinity. + masm.loadConstantDouble(NegativeInfinity(), ScratchDoubleReg); + masm.ma_bc1d(input, ScratchDoubleReg, &skip, + Assembler::DoubleNotEqualOrUnordered, ShortJump); + masm.as_fneg_d(output, ScratchDoubleReg); + masm.ma_b(&done, ShortJump); + + masm.bind(&skip); + // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5). + // Adding 0 converts any -0 to 0. + masm.loadConstantDouble(0.0, ScratchDoubleReg); + masm.as_fadd_d(output, input, ScratchDoubleReg); + masm.as_fsqrt_d(output, output); + + masm.bind(&done); +} + +void CodeGenerator::visitMathD(LMathD* math) { + FloatRegister src1 = ToFloatRegister(math->getOperand(0)); + FloatRegister src2 = ToFloatRegister(math->getOperand(1)); + FloatRegister output = ToFloatRegister(math->getDef(0)); + + switch (math->jsop()) { + case JSOp::Add: + masm.as_fadd_d(output, src1, src2); + break; + case JSOp::Sub: + masm.as_fsub_d(output, src1, src2); + break; + case JSOp::Mul: + masm.as_fmul_d(output, src1, src2); + break; + case JSOp::Div: + masm.as_fdiv_d(output, src1, src2); + break; + default: + MOZ_CRASH("unexpected opcode"); + } +} + +void CodeGenerator::visitMathF(LMathF* math) { + FloatRegister src1 = ToFloatRegister(math->getOperand(0)); + FloatRegister src2 = ToFloatRegister(math->getOperand(1)); + FloatRegister output = ToFloatRegister(math->getDef(0)); + + switch (math->jsop()) { + case JSOp::Add: + masm.as_fadd_s(output, src1, src2); + break; + case JSOp::Sub: + masm.as_fsub_s(output, src1, src2); + break; + case JSOp::Mul: + masm.as_fmul_s(output, src1, src2); + break; + case JSOp::Div: + masm.as_fdiv_s(output, src1, src2); + break; + default: + MOZ_CRASH("unexpected opcode"); + } +} + +void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) { + emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()), + ins->mir()); +} + +void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) { + emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()), + ins->mir()); +} + +void CodeGenerator::visitWasmBuiltinTruncateDToInt32( + LWasmBuiltinTruncateDToInt32* lir) { + emitTruncateDouble(ToFloatRegister(lir->getOperand(0)), + ToRegister(lir->getDef(0)), lir->mir()); +} + +void CodeGenerator::visitWasmBuiltinTruncateFToInt32( + LWasmBuiltinTruncateFToInt32* lir) { + emitTruncateFloat32(ToFloatRegister(lir->getOperand(0)), + ToRegister(lir->getDef(0)), lir->mir()); +} + +void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) { + auto input = ToFloatRegister(lir->input()); + auto output = ToRegister(lir->output()); + + MWasmTruncateToInt32* mir = lir->mir(); + MIRType fromType = mir->input()->type(); + + MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32); + + auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output); + addOutOfLineCode(ool, mir); + + Label* oolEntry = ool->entry(); + if (mir->isUnsigned()) { + if (fromType == MIRType::Double) { + masm.wasmTruncateDoubleToUInt32(input, output, mir->isSaturating(), + oolEntry); + } else if (fromType == MIRType::Float32) { + masm.wasmTruncateFloat32ToUInt32(input, output, mir->isSaturating(), + oolEntry); + } else { + MOZ_CRASH("unexpected type"); + } + + masm.bind(ool->rejoin()); + return; + } + + if (fromType == MIRType::Double) { + masm.wasmTruncateDoubleToInt32(input, output, mir->isSaturating(), + oolEntry); + } else if (fromType == MIRType::Float32) { + masm.wasmTruncateFloat32ToInt32(input, output, mir->isSaturating(), + oolEntry); + } else { + MOZ_CRASH("unexpected type"); + } + + masm.bind(ool->rejoin()); +} + +void CodeGenerator::visitCopySignF(LCopySignF* ins) { + FloatRegister lhs = ToFloatRegister(ins->getOperand(0)); + FloatRegister rhs = ToFloatRegister(ins->getOperand(1)); + FloatRegister output = ToFloatRegister(ins->getDef(0)); + + Register lhsi = ToRegister(ins->getTemp(0)); + Register rhsi = ToRegister(ins->getTemp(1)); + + masm.moveFromFloat32(lhs, lhsi); + masm.moveFromFloat32(rhs, rhsi); + + // Combine. + masm.as_bstrins_w(rhsi, lhsi, 30, 0); + + masm.moveToFloat32(rhsi, output); +} + +void CodeGenerator::visitCopySignD(LCopySignD* ins) { + FloatRegister lhs = ToFloatRegister(ins->getOperand(0)); + FloatRegister rhs = ToFloatRegister(ins->getOperand(1)); + FloatRegister output = ToFloatRegister(ins->getDef(0)); + + Register lhsi = ToRegister(ins->getTemp(0)); + Register rhsi = ToRegister(ins->getTemp(1)); + + // Manipulate high words of double inputs. + masm.moveFromDoubleHi(lhs, lhsi); + masm.moveFromDoubleHi(rhs, rhsi); + + // Combine. + masm.as_bstrins_w(rhsi, lhsi, 30, 0); + + masm.moveToDoubleHi(rhsi, output); +} + +void CodeGenerator::visitValue(LValue* value) { + const ValueOperand out = ToOutValue(value); + + masm.moveValue(value->value(), out); +} + +void CodeGenerator::visitDouble(LDouble* ins) { + const LDefinition* out = ins->getDef(0); + + masm.loadConstantDouble(ins->getDouble(), ToFloatRegister(out)); +} + +void CodeGenerator::visitFloat32(LFloat32* ins) { + const LDefinition* out = ins->getDef(0); + masm.loadConstantFloat32(ins->getFloat(), ToFloatRegister(out)); +} + +void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) { + FloatRegister input = ToFloatRegister(test->input()); + + MBasicBlock* ifTrue = test->ifTrue(); + MBasicBlock* ifFalse = test->ifFalse(); + + masm.loadConstantDouble(0.0, ScratchDoubleReg); + // If 0, or NaN, the result is false. + if (isNextBlock(ifFalse->lir())) { + branchToBlock(Assembler::DoubleFloat, input, ScratchDoubleReg, ifTrue, + Assembler::DoubleNotEqual); + } else { + branchToBlock(Assembler::DoubleFloat, input, ScratchDoubleReg, ifFalse, + Assembler::DoubleEqualOrUnordered); + jumpToBlock(ifTrue); + } +} + +void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) { + FloatRegister input = ToFloatRegister(test->input()); + + MBasicBlock* ifTrue = test->ifTrue(); + MBasicBlock* ifFalse = test->ifFalse(); + + masm.loadConstantFloat32(0.0f, ScratchFloat32Reg); + // If 0, or NaN, the result is false. + + if (isNextBlock(ifFalse->lir())) { + branchToBlock(Assembler::SingleFloat, input, ScratchFloat32Reg, ifTrue, + Assembler::DoubleNotEqual); + } else { + branchToBlock(Assembler::SingleFloat, input, ScratchFloat32Reg, ifFalse, + Assembler::DoubleEqualOrUnordered); + jumpToBlock(ifTrue); + } +} + +void CodeGenerator::visitCompareD(LCompareD* comp) { + FloatRegister lhs = ToFloatRegister(comp->left()); + FloatRegister rhs = ToFloatRegister(comp->right()); + Register dest = ToRegister(comp->output()); + + Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop()); + masm.ma_cmp_set_double(dest, lhs, rhs, cond); +} + +void CodeGenerator::visitCompareF(LCompareF* comp) { + FloatRegister lhs = ToFloatRegister(comp->left()); + FloatRegister rhs = ToFloatRegister(comp->right()); + Register dest = ToRegister(comp->output()); + + Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop()); + masm.ma_cmp_set_float32(dest, lhs, rhs, cond); +} + +void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) { + FloatRegister lhs = ToFloatRegister(comp->left()); + FloatRegister rhs = ToFloatRegister(comp->right()); + + Assembler::DoubleCondition cond = + JSOpToDoubleCondition(comp->cmpMir()->jsop()); + MBasicBlock* ifTrue = comp->ifTrue(); + MBasicBlock* ifFalse = comp->ifFalse(); + + if (isNextBlock(ifFalse->lir())) { + branchToBlock(Assembler::DoubleFloat, lhs, rhs, ifTrue, cond); + } else { + branchToBlock(Assembler::DoubleFloat, lhs, rhs, ifFalse, + Assembler::InvertCondition(cond)); + jumpToBlock(ifTrue); + } +} + +void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) { + FloatRegister lhs = ToFloatRegister(comp->left()); + FloatRegister rhs = ToFloatRegister(comp->right()); + + Assembler::DoubleCondition cond = + JSOpToDoubleCondition(comp->cmpMir()->jsop()); + MBasicBlock* ifTrue = comp->ifTrue(); + MBasicBlock* ifFalse = comp->ifFalse(); + + if (isNextBlock(ifFalse->lir())) { + branchToBlock(Assembler::SingleFloat, lhs, rhs, ifTrue, cond); + } else { + branchToBlock(Assembler::SingleFloat, lhs, rhs, ifFalse, + Assembler::InvertCondition(cond)); + jumpToBlock(ifTrue); + } +} + +void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* lir) { + ScratchRegisterScope scratch(masm); + if (lir->right()->isConstant()) { + masm.ma_and(scratch, ToRegister(lir->left()), Imm32(ToInt32(lir->right()))); + } else { + masm.as_and(scratch, ToRegister(lir->left()), ToRegister(lir->right())); + } + emitBranch(scratch, Register(scratch), lir->cond(), lir->ifTrue(), + lir->ifFalse()); +} + +// See ../CodeGenerator.cpp for more information. +void CodeGenerator::visitWasmRegisterResult(LWasmRegisterResult* lir) {} + +void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) { + masm.convertUInt32ToDouble(ToRegister(lir->input()), + ToFloatRegister(lir->output())); +} + +void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) { + masm.convertUInt32ToFloat32(ToRegister(lir->input()), + ToFloatRegister(lir->output())); +} + +void CodeGenerator::visitNotI(LNotI* ins) { + masm.cmp32Set(Assembler::Equal, ToRegister(ins->input()), Imm32(0), + ToRegister(ins->output())); +} + +void CodeGenerator::visitNotD(LNotD* ins) { + // Since this operation is not, we want to set a bit if + // the double is falsey, which means 0.0, -0.0 or NaN. + FloatRegister in = ToFloatRegister(ins->input()); + Register dest = ToRegister(ins->output()); + + masm.loadConstantDouble(0.0, ScratchDoubleReg); + masm.ma_cmp_set_double(dest, in, ScratchDoubleReg, + Assembler::DoubleEqualOrUnordered); +} + +void CodeGenerator::visitNotF(LNotF* ins) { + // Since this operation is not, we want to set a bit if + // the float32 is falsey, which means 0.0, -0.0 or NaN. + FloatRegister in = ToFloatRegister(ins->input()); + Register dest = ToRegister(ins->output()); + + masm.loadConstantFloat32(0.0f, ScratchFloat32Reg); + masm.ma_cmp_set_float32(dest, in, ScratchFloat32Reg, + Assembler::DoubleEqualOrUnordered); +} + +void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) { + masm.memoryBarrier(ins->type()); +} + +void CodeGenerator::visitWasmLoad(LWasmLoad* lir) { emitWasmLoad(lir); } + +void CodeGenerator::visitWasmStore(LWasmStore* lir) { emitWasmStore(lir); } + +void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) { + const MAsmJSLoadHeap* mir = ins->mir(); + const LAllocation* ptr = ins->ptr(); + const LDefinition* out = ins->output(); + const LAllocation* boundsCheckLimit = ins->boundsCheckLimit(); + + bool isSigned; + int size; + bool isFloat = false; + switch (mir->access().type()) { + case Scalar::Int8: + isSigned = true; + size = 8; + break; + case Scalar::Uint8: + isSigned = false; + size = 8; + break; + case Scalar::Int16: + isSigned = true; + size = 16; + break; + case Scalar::Uint16: + isSigned = false; + size = 16; + break; + case Scalar::Int32: + isSigned = true; + size = 32; + break; + case Scalar::Uint32: + isSigned = false; + size = 32; + break; + case Scalar::Float64: + isFloat = true; + size = 64; + break; + case Scalar::Float32: + isFloat = true; + size = 32; + break; + default: + MOZ_CRASH("unexpected array type"); + } + + if (ptr->isConstant()) { + MOZ_ASSERT(!mir->needsBoundsCheck()); + int32_t ptrImm = ptr->toConstant()->toInt32(); + MOZ_ASSERT(ptrImm >= 0); + if (isFloat) { + if (size == 32) { + masm.loadFloat32(Address(HeapReg, ptrImm), ToFloatRegister(out)); + } else { + masm.loadDouble(Address(HeapReg, ptrImm), ToFloatRegister(out)); + } + } else { + masm.ma_load(ToRegister(out), Address(HeapReg, ptrImm), + static_cast(size), + isSigned ? SignExtend : ZeroExtend); + } + return; + } + + Register ptrReg = ToRegister(ptr); + + if (!mir->needsBoundsCheck()) { + if (isFloat) { + if (size == 32) { + masm.loadFloat32(BaseIndex(HeapReg, ptrReg, TimesOne), + ToFloatRegister(out)); + } else { + masm.loadDouble(BaseIndex(HeapReg, ptrReg, TimesOne), + ToFloatRegister(out)); + } + } else { + masm.ma_load(ToRegister(out), BaseIndex(HeapReg, ptrReg, TimesOne), + static_cast(size), + isSigned ? SignExtend : ZeroExtend); + } + return; + } + + Label done, outOfRange; + masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg, + ToRegister(boundsCheckLimit), &outOfRange); + // Offset is ok, let's load value. + if (isFloat) { + if (size == 32) { + masm.loadFloat32(BaseIndex(HeapReg, ptrReg, TimesOne), + ToFloatRegister(out)); + } else { + masm.loadDouble(BaseIndex(HeapReg, ptrReg, TimesOne), + ToFloatRegister(out)); + } + } else { + masm.ma_load(ToRegister(out), BaseIndex(HeapReg, ptrReg, TimesOne), + static_cast(size), + isSigned ? SignExtend : ZeroExtend); + } + masm.ma_b(&done, ShortJump); + masm.bind(&outOfRange); + // Offset is out of range. Load default values. + if (isFloat) { + if (size == 32) { + masm.loadConstantFloat32(float(GenericNaN()), ToFloatRegister(out)); + } else { + masm.loadConstantDouble(GenericNaN(), ToFloatRegister(out)); + } + } else { + masm.move32(Imm32(0), ToRegister(out)); + } + masm.bind(&done); +} + +void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) { + const MAsmJSStoreHeap* mir = ins->mir(); + const LAllocation* value = ins->value(); + const LAllocation* ptr = ins->ptr(); + const LAllocation* boundsCheckLimit = ins->boundsCheckLimit(); + + bool isSigned; + int size; + bool isFloat = false; + switch (mir->access().type()) { + case Scalar::Int8: + isSigned = true; + size = 8; + break; + case Scalar::Uint8: + isSigned = false; + size = 8; + break; + case Scalar::Int16: + isSigned = true; + size = 16; + break; + case Scalar::Uint16: + isSigned = false; + size = 16; + break; + case Scalar::Int32: + isSigned = true; + size = 32; + break; + case Scalar::Uint32: + isSigned = false; + size = 32; + break; + case Scalar::Float64: + isFloat = true; + size = 64; + break; + case Scalar::Float32: + isFloat = true; + size = 32; + break; + default: + MOZ_CRASH("unexpected array type"); + } + + if (ptr->isConstant()) { + MOZ_ASSERT(!mir->needsBoundsCheck()); + int32_t ptrImm = ptr->toConstant()->toInt32(); + MOZ_ASSERT(ptrImm >= 0); + + if (isFloat) { + FloatRegister freg = ToFloatRegister(value); + Address addr(HeapReg, ptrImm); + if (size == 32) { + masm.storeFloat32(freg, addr); + } else { + masm.storeDouble(freg, addr); + } + } else { + masm.ma_store(ToRegister(value), Address(HeapReg, ptrImm), + static_cast(size), + isSigned ? SignExtend : ZeroExtend); + } + return; + } + + Register ptrReg = ToRegister(ptr); + Address dstAddr(ptrReg, 0); + + if (!mir->needsBoundsCheck()) { + if (isFloat) { + FloatRegister freg = ToFloatRegister(value); + BaseIndex bi(HeapReg, ptrReg, TimesOne); + if (size == 32) { + masm.storeFloat32(freg, bi); + } else { + masm.storeDouble(freg, bi); + } + } else { + masm.ma_store(ToRegister(value), BaseIndex(HeapReg, ptrReg, TimesOne), + static_cast(size), + isSigned ? SignExtend : ZeroExtend); + } + return; + } + + Label outOfRange; + masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg, + ToRegister(boundsCheckLimit), &outOfRange); + + // Offset is ok, let's store value. + if (isFloat) { + if (size == 32) { + masm.storeFloat32(ToFloatRegister(value), + BaseIndex(HeapReg, ptrReg, TimesOne)); + } else + masm.storeDouble(ToFloatRegister(value), + BaseIndex(HeapReg, ptrReg, TimesOne)); + } else { + masm.ma_store(ToRegister(value), BaseIndex(HeapReg, ptrReg, TimesOne), + static_cast(size), + isSigned ? SignExtend : ZeroExtend); + } + + masm.bind(&outOfRange); +} + +void CodeGenerator::visitWasmCompareExchangeHeap( + LWasmCompareExchangeHeap* ins) { + MWasmCompareExchangeHeap* mir = ins->mir(); + Register ptrReg = ToRegister(ins->ptr()); + BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset()); + MOZ_ASSERT(ins->addrTemp()->isBogusTemp()); + + Register oldval = ToRegister(ins->oldValue()); + Register newval = ToRegister(ins->newValue()); + Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp()); + Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp()); + Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp()); + + masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval, valueTemp, + offsetTemp, maskTemp, ToRegister(ins->output())); +} + +void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) { + MWasmAtomicExchangeHeap* mir = ins->mir(); + Register ptrReg = ToRegister(ins->ptr()); + Register value = ToRegister(ins->value()); + BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset()); + MOZ_ASSERT(ins->addrTemp()->isBogusTemp()); + + Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp()); + Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp()); + Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp()); + + masm.wasmAtomicExchange(mir->access(), srcAddr, value, valueTemp, offsetTemp, + maskTemp, ToRegister(ins->output())); +} + +void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) { + MOZ_ASSERT(ins->mir()->hasUses()); + MOZ_ASSERT(ins->addrTemp()->isBogusTemp()); + + MWasmAtomicBinopHeap* mir = ins->mir(); + Register ptrReg = ToRegister(ins->ptr()); + Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp()); + Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp()); + Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp()); + + BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset()); + + masm.wasmAtomicFetchOp(mir->access(), mir->operation(), + ToRegister(ins->value()), srcAddr, valueTemp, + offsetTemp, maskTemp, ToRegister(ins->output())); +} + +void CodeGenerator::visitWasmAtomicBinopHeapForEffect( + LWasmAtomicBinopHeapForEffect* ins) { + MOZ_ASSERT(!ins->mir()->hasUses()); + MOZ_ASSERT(ins->addrTemp()->isBogusTemp()); + + MWasmAtomicBinopHeap* mir = ins->mir(); + Register ptrReg = ToRegister(ins->ptr()); + Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp()); + Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp()); + Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp()); + + BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset()); + masm.wasmAtomicEffectOp(mir->access(), mir->operation(), + ToRegister(ins->value()), srcAddr, valueTemp, + offsetTemp, maskTemp); +} + +void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) { + const MWasmStackArg* mir = ins->mir(); + if (ins->arg()->isConstant()) { + masm.storePtr(ImmWord(ToInt32(ins->arg())), + Address(StackPointer, mir->spOffset())); + } else { + if (ins->arg()->isGeneralReg()) { + masm.storePtr(ToRegister(ins->arg()), + Address(StackPointer, mir->spOffset())); + } else if (mir->input()->type() == MIRType::Double) { + masm.storeDouble(ToFloatRegister(ins->arg()), + Address(StackPointer, mir->spOffset())); + } else { + masm.storeFloat32(ToFloatRegister(ins->arg()), + Address(StackPointer, mir->spOffset())); + } + } +} + +void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) { + const MWasmStackArg* mir = ins->mir(); + Address dst(StackPointer, mir->spOffset()); + if (IsConstant(ins->arg())) { + masm.store64(Imm64(ToInt64(ins->arg())), dst); + } else { + masm.store64(ToRegister64(ins->arg()), dst); + } +} + +void CodeGenerator::visitWasmSelect(LWasmSelect* ins) { + MIRType mirType = ins->mir()->type(); + + Register cond = ToRegister(ins->condExpr()); + const LAllocation* falseExpr = ins->falseExpr(); + + if (mirType == MIRType::Int32 || mirType == MIRType::RefOrNull) { + Register out = ToRegister(ins->output()); + MOZ_ASSERT(ToRegister(ins->trueExpr()) == out, + "true expr input is reused for output"); + if (falseExpr->isRegister()) { + masm.moveIfZero(out, ToRegister(falseExpr), cond); + } else { + masm.cmp32Load32(Assembler::Zero, cond, cond, ToAddress(falseExpr), out); + } + return; + } + + FloatRegister out = ToFloatRegister(ins->output()); + MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out, + "true expr input is reused for output"); + + if (falseExpr->isFloatReg()) { + if (mirType == MIRType::Float32) { + masm.ma_fmovz(Assembler::SingleFloat, out, ToFloatRegister(falseExpr), + cond); + } else if (mirType == MIRType::Double) { + masm.ma_fmovz(Assembler::DoubleFloat, out, ToFloatRegister(falseExpr), + cond); + } else { + MOZ_CRASH("unhandled type in visitWasmSelect!"); + } + } else { + Label done; + masm.ma_b(cond, cond, &done, Assembler::NonZero, ShortJump); + + if (mirType == MIRType::Float32) { + masm.loadFloat32(ToAddress(falseExpr), out); + } else if (mirType == MIRType::Double) { + masm.loadDouble(ToAddress(falseExpr), out); + } else { + MOZ_CRASH("unhandled type in visitWasmSelect!"); + } + + masm.bind(&done); + } +} + +void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) { + emitWasmCompareAndSelect(ins); +} + +void CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir) { + MOZ_ASSERT(gen->compilingWasm()); + MWasmReinterpret* ins = lir->mir(); + + MIRType to = ins->type(); + DebugOnly from = ins->input()->type(); + + switch (to) { + case MIRType::Int32: + MOZ_ASSERT(from == MIRType::Float32); + masm.as_movfr2gr_s(ToRegister(lir->output()), + ToFloatRegister(lir->input())); + break; + case MIRType::Float32: + MOZ_ASSERT(from == MIRType::Int32); + masm.as_movgr2fr_w(ToFloatRegister(lir->output()), + ToRegister(lir->input())); + break; + case MIRType::Double: + case MIRType::Int64: + MOZ_CRASH("not handled by this LIR opcode"); + default: + MOZ_CRASH("unexpected WasmReinterpret"); + } +} + +void CodeGenerator::visitUDivOrMod(LUDivOrMod* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register output = ToRegister(ins->output()); + Label done; + + // Prevent divide by zero. + if (ins->canBeDivideByZero()) { + if (ins->mir()->isTruncated()) { + if (ins->trapOnError()) { + Label nonZero; + masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->bytecodeOffset()); + masm.bind(&nonZero); + } else { + // Infinity|0 == 0 + Label notzero; + masm.ma_b(rhs, rhs, ¬zero, Assembler::NonZero, ShortJump); + masm.move32(Imm32(0), output); + masm.ma_b(&done, ShortJump); + masm.bind(¬zero); + } + } else { + bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot()); + } + } + + masm.as_mod_wu(output, lhs, rhs); + + // If the remainder is > 0, bailout since this must be a double. + if (ins->mir()->isDiv()) { + if (!ins->mir()->toDiv()->canTruncateRemainder()) { + bailoutCmp32(Assembler::NonZero, output, output, ins->snapshot()); + } + // Get quotient + masm.as_div_wu(output, lhs, rhs); + } + + if (!ins->mir()->isTruncated()) { + bailoutCmp32(Assembler::LessThan, output, Imm32(0), ins->snapshot()); + } + + masm.bind(&done); +} + +void CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins) { + const MEffectiveAddress* mir = ins->mir(); + Register base = ToRegister(ins->base()); + Register index = ToRegister(ins->index()); + Register output = ToRegister(ins->output()); + + BaseIndex address(base, index, mir->scale(), mir->displacement()); + masm.computeEffectiveAddress(address, output); +} + +void CodeGenerator::visitNegI(LNegI* ins) { + Register input = ToRegister(ins->input()); + Register output = ToRegister(ins->output()); + + masm.as_sub_w(output, zero, input); +} + +void CodeGenerator::visitNegI64(LNegI64* ins) { + Register64 input = ToRegister64(ins->getInt64Operand(0)); + MOZ_ASSERT(input == ToOutRegister64(ins)); + masm.neg64(input); +} + +void CodeGenerator::visitNegD(LNegD* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + FloatRegister output = ToFloatRegister(ins->output()); + + masm.as_fneg_d(output, input); +} + +void CodeGenerator::visitNegF(LNegF* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + FloatRegister output = ToFloatRegister(ins->output()); + + masm.as_fneg_s(output, input); +} + +void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) { + MWasmAddOffset* mir = lir->mir(); + Register base = ToRegister(lir->base()); + Register out = ToRegister(lir->output()); + + Label ok; + masm.ma_add32TestCarry(Assembler::CarryClear, out, base, Imm32(mir->offset()), + &ok); + masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset()); + masm.bind(&ok); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinop( + LAtomicTypedArrayElementBinop* lir) { + MOZ_ASSERT(!lir->mir()->isForEffect()); + + AnyRegister output = ToAnyRegister(lir->output()); + Register elements = ToRegister(lir->elements()); + Register outTemp = ToTempRegisterOrInvalid(lir->temp2()); + Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp()); + Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp()); + Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp()); + Register value = ToRegister(lir->value()); + Scalar::Type arrayType = lir->mir()->arrayType(); + + if (lir->index()->isConstant()) { + Address mem = ToAddress(elements, lir->index(), arrayType); + masm.atomicFetchOpJS(arrayType, Synchronization::Full(), + lir->mir()->operation(), value, mem, valueTemp, + offsetTemp, maskTemp, outTemp, output); + } else { + BaseIndex mem(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicFetchOpJS(arrayType, Synchronization::Full(), + lir->mir()->operation(), value, mem, valueTemp, + offsetTemp, maskTemp, outTemp, output); + } +} + +void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect( + LAtomicTypedArrayElementBinopForEffect* lir) { + MOZ_ASSERT(lir->mir()->isForEffect()); + + Register elements = ToRegister(lir->elements()); + Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp()); + Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp()); + Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp()); + Register value = ToRegister(lir->value()); + Scalar::Type arrayType = lir->mir()->arrayType(); + + if (lir->index()->isConstant()) { + Address mem = ToAddress(elements, lir->index(), arrayType); + masm.atomicEffectOpJS(arrayType, Synchronization::Full(), + lir->mir()->operation(), value, mem, valueTemp, + offsetTemp, maskTemp); + } else { + BaseIndex mem(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicEffectOpJS(arrayType, Synchronization::Full(), + lir->mir()->operation(), value, mem, valueTemp, + offsetTemp, maskTemp); + } +} + +void CodeGenerator::visitCompareExchangeTypedArrayElement( + LCompareExchangeTypedArrayElement* lir) { + Register elements = ToRegister(lir->elements()); + AnyRegister output = ToAnyRegister(lir->output()); + Register outTemp = ToTempRegisterOrInvalid(lir->temp()); + + Register oldval = ToRegister(lir->oldval()); + Register newval = ToRegister(lir->newval()); + Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp()); + Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp()); + Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp()); + Scalar::Type arrayType = lir->mir()->arrayType(); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, + newval, valueTemp, offsetTemp, maskTemp, outTemp, + output); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, + newval, valueTemp, offsetTemp, maskTemp, outTemp, + output); + } +} + +void CodeGenerator::visitAtomicExchangeTypedArrayElement( + LAtomicExchangeTypedArrayElement* lir) { + Register elements = ToRegister(lir->elements()); + AnyRegister output = ToAnyRegister(lir->output()); + Register outTemp = ToTempRegisterOrInvalid(lir->temp()); + + Register value = ToRegister(lir->value()); + Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp()); + Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp()); + Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp()); + Scalar::Type arrayType = lir->mir()->arrayType(); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, + valueTemp, offsetTemp, maskTemp, outTemp, output); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, + valueTemp, offsetTemp, maskTemp, outTemp, output); + } +} + +void CodeGenerator::visitCompareExchangeTypedArrayElement64( + LCompareExchangeTypedArrayElement64* lir) { + Register elements = ToRegister(lir->elements()); + Register oldval = ToRegister(lir->oldval()); + Register newval = ToRegister(lir->newval()); + Register64 temp1 = ToRegister64(lir->temp1()); + Register64 temp2 = ToRegister64(lir->temp2()); + Register out = ToRegister(lir->output()); + Register64 tempOut(out); + Scalar::Type arrayType = lir->mir()->arrayType(); + + masm.loadBigInt64(oldval, temp1); + masm.loadBigInt64(newval, tempOut); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut, + temp2); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut, + temp2); + } + + emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg()); +} + +void CodeGenerator::visitAtomicExchangeTypedArrayElement64( + LAtomicExchangeTypedArrayElement64* lir) { + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + Register64 temp1 = ToRegister64(lir->temp1()); + Register64 temp2 = Register64(ToRegister(lir->temp2())); + Register out = ToRegister(lir->output()); + Scalar::Type arrayType = lir->mir()->arrayType(); + + masm.loadBigInt64(value, temp1); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2); + } + + emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg()); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinop64( + LAtomicTypedArrayElementBinop64* lir) { + MOZ_ASSERT(lir->mir()->hasUses()); + + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + Register64 temp1 = ToRegister64(lir->temp1()); + Register64 temp2 = ToRegister64(lir->temp2()); + Register out = ToRegister(lir->output()); + Register64 tempOut = Register64(out); + + Scalar::Type arrayType = lir->mir()->arrayType(); + AtomicOp atomicOp = lir->mir()->operation(); + + masm.loadBigInt64(value, temp1); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest, + tempOut, temp2); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest, + tempOut, temp2); + } + + emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg()); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64( + LAtomicTypedArrayElementBinopForEffect64* lir) { + MOZ_ASSERT(!lir->mir()->hasUses()); + + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + Register64 temp1 = ToRegister64(lir->temp1()); + Register64 temp2 = ToRegister64(lir->temp2()); + + Scalar::Type arrayType = lir->mir()->arrayType(); + AtomicOp atomicOp = lir->mir()->operation(); + + masm.loadBigInt64(value, temp1); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest, + temp2); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest, + temp2); + } +} + +void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) { + Register elements = ToRegister(lir->elements()); + Register temp = ToRegister(lir->temp()); + Register64 temp64 = ToRegister64(lir->temp64()); + Register out = ToRegister(lir->output()); + const MLoadUnboxedScalar* mir = lir->mir(); + + Scalar::Type storageType = mir->storageType(); + + auto sync = Synchronization::Load(); + masm.memoryBarrierBefore(sync); + if (lir->index()->isConstant()) { + Address source = + ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment()); + masm.load64(source, temp64); + } else { + BaseIndex source(elements, ToRegister(lir->index()), + ScaleFromScalarType(storageType), mir->offsetAdjustment()); + masm.load64(source, temp64); + } + masm.memoryBarrierAfter(sync); + emitCreateBigInt(lir, storageType, temp64, out, temp); +} + +void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) { + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + Register64 temp1 = ToRegister64(lir->temp1()); + + Scalar::Type writeType = lir->mir()->writeType(); + + masm.loadBigInt64(value, temp1); + auto sync = Synchronization::Store(); + masm.memoryBarrierBefore(sync); + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), writeType); + masm.store64(temp1, dest); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(writeType)); + masm.store64(temp1, dest); + } + masm.memoryBarrierAfter(sync); +} + +void CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* lir) { + Register ptr = ToRegister(lir->ptr()); + Register64 oldValue = ToRegister64(lir->oldValue()); + Register64 newValue = ToRegister64(lir->newValue()); + Register64 output = ToOutRegister64(lir); + uint32_t offset = lir->mir()->access().offset(); + + BaseIndex addr(HeapReg, ptr, TimesOne, offset); + masm.wasmCompareExchange64(lir->mir()->access(), addr, oldValue, newValue, + output); +} + +void CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* lir) { + Register ptr = ToRegister(lir->ptr()); + Register64 value = ToRegister64(lir->value()); + Register64 output = ToOutRegister64(lir); + uint32_t offset = lir->mir()->access().offset(); + + BaseIndex addr(HeapReg, ptr, TimesOne, offset); + masm.wasmAtomicExchange64(lir->mir()->access(), addr, value, output); +} + +void CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* lir) { + Register ptr = ToRegister(lir->ptr()); + Register64 value = ToRegister64(lir->value()); + Register64 output = ToOutRegister64(lir); + Register64 temp(ToRegister(lir->getTemp(0))); + uint32_t offset = lir->mir()->access().offset(); + + BaseIndex addr(HeapReg, ptr, TimesOne, offset); + + masm.wasmAtomicFetchOp64(lir->mir()->access(), lir->mir()->operation(), value, + addr, temp, output); +} + +void CodeGenerator::visitNearbyInt(LNearbyInt*) { MOZ_CRASH("NYI"); } + +void CodeGenerator::visitNearbyIntF(LNearbyIntF*) { MOZ_CRASH("NYI"); } + +void CodeGenerator::visitSimd128(LSimd128* ins) { MOZ_CRASH("No SIMD"); } + +void CodeGenerator::visitWasmBitselectSimd128(LWasmBitselectSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmBinarySimd128WithConstant( + LWasmBinarySimd128WithConstant* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmVariableShiftSimd128( + LWasmVariableShiftSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmConstantShiftSimd128( + LWasmConstantShiftSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmSignReplicationSimd128( + LWasmSignReplicationSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReplaceInt64LaneSimd128( + LWasmReplaceInt64LaneSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReduceAndBranchSimd128( + LWasmReduceAndBranchSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReduceSimd128ToInt64( + LWasmReduceSimd128ToInt64* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) { + MOZ_CRASH("No SIMD"); +} diff --git a/js/src/jit/loong64/CodeGenerator-loong64.h b/js/src/jit/loong64/CodeGenerator-loong64.h new file mode 100644 index 0000000000..194d214f01 --- /dev/null +++ b/js/src/jit/loong64/CodeGenerator-loong64.h @@ -0,0 +1,198 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_CodeGenerator_loong64_h +#define jit_loong64_CodeGenerator_loong64_h + +#include "jit/loong64/Assembler-loong64.h" +#include "jit/shared/CodeGenerator-shared.h" + +namespace js { +namespace jit { + +class CodeGeneratorLOONG64; +class OutOfLineBailout; +class OutOfLineTableSwitch; + +using OutOfLineWasmTruncateCheck = + OutOfLineWasmTruncateCheckBase; + +class CodeGeneratorLOONG64 : public CodeGeneratorShared { + friend class MoveResolverLA; + + protected: + CodeGeneratorLOONG64(MIRGenerator* gen, LIRGraph* graph, + MacroAssembler* masm); + + NonAssertingLabel deoptLabel_; + + Operand ToOperand(const LAllocation& a); + Operand ToOperand(const LAllocation* a); + Operand ToOperand(const LDefinition* def); + +#ifdef JS_PUNBOX64 + Operand ToOperandOrRegister64(const LInt64Allocation input); +#else + Register64 ToOperandOrRegister64(const LInt64Allocation input); +#endif + + MoveOperand toMoveOperand(LAllocation a) const; + + template + void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branch32(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } + template + void bailoutTest32(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branchTest32(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } + template + void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branchPtr(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } + void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs, + LSnapshot* snapshot) { + Label bail; + masm.branchTestPtr(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } + void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) { + Label bail; + masm.branchTest32(Assembler::Zero, reg, Imm32(0xFF), &bail); + bailoutFrom(&bail, snapshot); + } + + void bailoutFrom(Label* label, LSnapshot* snapshot); + void bailout(LSnapshot* snapshot); + + bool generateOutOfLineCode(); + + template + void branchToBlock(Register lhs, T rhs, MBasicBlock* mir, + Assembler::Condition cond) { + masm.ma_b(lhs, rhs, skipTrivialBlocks(mir)->lir()->label(), cond); + } + void branchToBlock(Assembler::FloatFormat fmt, FloatRegister lhs, + FloatRegister rhs, MBasicBlock* mir, + Assembler::DoubleCondition cond); + + // Emits a branch that directs control flow to the true block if |cond| is + // true, and the false block if |cond| is false. + template + void emitBranch(Register lhs, T rhs, Assembler::Condition cond, + MBasicBlock* mirTrue, MBasicBlock* mirFalse) { + if (isNextBlock(mirFalse->lir())) { + branchToBlock(lhs, rhs, mirTrue, cond); + } else { + branchToBlock(lhs, rhs, mirFalse, Assembler::InvertCondition(cond)); + jumpToBlock(mirTrue); + } + } + void testZeroEmitBranch(Assembler::Condition cond, Register reg, + MBasicBlock* ifTrue, MBasicBlock* ifFalse) { + emitBranch(reg, Imm32(0), cond, ifTrue, ifFalse); + } + + void emitTableSwitchDispatch(MTableSwitch* mir, Register index, + Register base); + + template + void emitWasmLoad(T* ins); + template + void emitWasmStore(T* ins); + + void generateInvalidateEpilogue(); + + // Generating a result. + template + void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, + const S& value, const T& mem, + Register flagTemp, Register outTemp, + Register valueTemp, Register offsetTemp, + Register maskTemp, AnyRegister output); + + // Generating no result. + template + void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, + const S& value, const T& mem, + Register flagTemp, Register valueTemp, + Register offsetTemp, Register maskTemp); + + public: + // Out of line visitors. + void visitOutOfLineBailout(OutOfLineBailout* ool); + void visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool); + void visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool); + + protected: + void testNullEmitBranch(Assembler::Condition cond, const ValueOperand& value, + MBasicBlock* ifTrue, MBasicBlock* ifFalse) { + MOZ_ASSERT(value.valueReg() != SecondScratchReg); + masm.splitTag(value.valueReg(), SecondScratchReg); + emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_NULL), cond, ifTrue, ifFalse); + } + void testUndefinedEmitBranch(Assembler::Condition cond, + const ValueOperand& value, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) { + MOZ_ASSERT(value.valueReg() != SecondScratchReg); + masm.splitTag(value.valueReg(), SecondScratchReg); + emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_UNDEFINED), cond, ifTrue, + ifFalse); + } + void testObjectEmitBranch(Assembler::Condition cond, + const ValueOperand& value, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) { + MOZ_ASSERT(value.valueReg() != SecondScratchReg); + masm.splitTag(value.valueReg(), SecondScratchReg); + emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_OBJECT), cond, ifTrue, + ifFalse); + } + + void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor, + Register output, Label* fail); + void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor, + Register output, Label* fail); + + template + void emitWasmLoadI64(T* ins); + template + void emitWasmStoreI64(T* ins); + + ValueOperand ToValue(LInstruction* ins, size_t pos); + ValueOperand ToTempValue(LInstruction* ins, size_t pos); + + // Functions for LTestVAndBranch. + void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag); +}; + +typedef CodeGeneratorLOONG64 CodeGeneratorSpecific; + +// An out-of-line bailout thunk. +class OutOfLineBailout : public OutOfLineCodeBase { + protected: + LSnapshot* snapshot_; + + public: + OutOfLineBailout(LSnapshot* snapshot) : snapshot_(snapshot) {} + + void accept(CodeGeneratorLOONG64* codegen) override; + + LSnapshot* snapshot() const { return snapshot_; } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_CodeGenerator_loong64_h */ diff --git a/js/src/jit/loong64/LIR-loong64.h b/js/src/jit/loong64/LIR-loong64.h new file mode 100644 index 0000000000..20fde694a1 --- /dev/null +++ b/js/src/jit/loong64/LIR-loong64.h @@ -0,0 +1,399 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_LIR_loong64_h +#define jit_loong64_LIR_loong64_h + +namespace js { +namespace jit { + +class LUnbox : public LInstructionHelper<1, 1, 0> { + protected: + LUnbox(LNode::Opcode opcode, const LAllocation& input) + : LInstructionHelper(opcode) { + setOperand(0, input); + } + + public: + LIR_HEADER(Unbox); + + explicit LUnbox(const LAllocation& input) : LInstructionHelper(classOpcode) { + setOperand(0, input); + } + + static const size_t Input = 0; + + MUnbox* mir() const { return mir_->toUnbox(); } + const char* extraName() const { return StringFromMIRType(mir()->type()); } +}; + +class LUnboxFloatingPoint : public LUnbox { + MIRType type_; + + public: + LIR_HEADER(UnboxFloatingPoint); + + LUnboxFloatingPoint(const LAllocation& input, MIRType type) + : LUnbox(classOpcode, input), type_(type) {} + + MIRType type() const { return type_; } +}; + +// Convert a 32-bit unsigned integer to a double. +class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(WasmUint32ToDouble) + + explicit LWasmUint32ToDouble(const LAllocation& input) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + } +}; + +// Convert a 32-bit unsigned integer to a float32. +class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(WasmUint32ToFloat32) + + explicit LWasmUint32ToFloat32(const LAllocation& input) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + } +}; + +class LDivI : public LBinaryMath<1> { + public: + LIR_HEADER(DivI); + + LDivI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp); + } + + MDiv* mir() const { return mir_->toDiv(); } +}; + +class LDivPowTwoI : public LInstructionHelper<1, 1, 1> { + const int32_t shift_; + + public: + LIR_HEADER(DivPowTwoI) + + LDivPowTwoI(const LAllocation& lhs, int32_t shift, const LDefinition& temp) + : LInstructionHelper(classOpcode), shift_(shift) { + setOperand(0, lhs); + setTemp(0, temp); + } + + const LAllocation* numerator() { return getOperand(0); } + int32_t shift() const { return shift_; } + MDiv* mir() const { return mir_->toDiv(); } +}; + +class LModI : public LBinaryMath<1> { + public: + LIR_HEADER(ModI); + + LModI(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& callTemp) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, callTemp); + } + + const LDefinition* callTemp() { return getTemp(0); } + MMod* mir() const { return mir_->toMod(); } +}; + +class LModPowTwoI : public LInstructionHelper<1, 1, 0> { + const int32_t shift_; + + public: + LIR_HEADER(ModPowTwoI); + + LModPowTwoI(const LAllocation& lhs, int32_t shift) + : LInstructionHelper(classOpcode), shift_(shift) { + setOperand(0, lhs); + } + + int32_t shift() const { return shift_; } + MMod* mir() const { return mir_->toMod(); } +}; + +class LModMaskI : public LInstructionHelper<1, 1, 2> { + const int32_t shift_; + + public: + LIR_HEADER(ModMaskI); + + LModMaskI(const LAllocation& lhs, const LDefinition& temp0, + const LDefinition& temp1, int32_t shift) + : LInstructionHelper(classOpcode), shift_(shift) { + setOperand(0, lhs); + setTemp(0, temp0); + setTemp(1, temp1); + } + + int32_t shift() const { return shift_; } + MMod* mir() const { return mir_->toMod(); } +}; + +// Takes a tableswitch with an integer to decide +class LTableSwitch : public LInstructionHelper<0, 1, 2> { + public: + LIR_HEADER(TableSwitch); + + LTableSwitch(const LAllocation& in, const LDefinition& inputCopy, + const LDefinition& jumpTablePointer, MTableSwitch* ins) + : LInstructionHelper(classOpcode) { + setOperand(0, in); + setTemp(0, inputCopy); + setTemp(1, jumpTablePointer); + setMir(ins); + } + + MTableSwitch* mir() const { return mir_->toTableSwitch(); } + const LAllocation* index() { return getOperand(0); } + const LDefinition* tempInt() { return getTemp(0); } + // This is added to share the same CodeGenerator prefixes. + const LDefinition* tempPointer() { return getTemp(1); } +}; + +// Takes a tableswitch with an integer to decide +class LTableSwitchV : public LInstructionHelper<0, BOX_PIECES, 3> { + public: + LIR_HEADER(TableSwitchV); + + LTableSwitchV(const LBoxAllocation& input, const LDefinition& inputCopy, + const LDefinition& floatCopy, + const LDefinition& jumpTablePointer, MTableSwitch* ins) + : LInstructionHelper(classOpcode) { + setBoxOperand(InputValue, input); + setTemp(0, inputCopy); + setTemp(1, floatCopy); + setTemp(2, jumpTablePointer); + setMir(ins); + } + + MTableSwitch* mir() const { return mir_->toTableSwitch(); } + + static const size_t InputValue = 0; + + const LDefinition* tempInt() { return getTemp(0); } + const LDefinition* tempFloat() { return getTemp(1); } + const LDefinition* tempPointer() { return getTemp(2); } +}; + +class LMulI : public LBinaryMath<0> { + public: + LIR_HEADER(MulI); + + LMulI() : LBinaryMath(classOpcode) {} + + MMul* mir() { return mir_->toMul(); } +}; + +class LUDivOrMod : public LBinaryMath<0> { + public: + LIR_HEADER(UDivOrMod); + + LUDivOrMod() : LBinaryMath(classOpcode) {} + + MBinaryArithInstruction* mir() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + return static_cast(mir_); + } + + bool canBeDivideByZero() const { + if (mir_->isMod()) { + return mir_->toMod()->canBeDivideByZero(); + } + return mir_->toDiv()->canBeDivideByZero(); + } + + bool trapOnError() const { + if (mir_->isMod()) { + return mir_->toMod()->trapOnError(); + } + return mir_->toDiv()->trapOnError(); + } + + wasm::BytecodeOffset bytecodeOffset() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + if (mir_->isMod()) { + return mir_->toMod()->bytecodeOffset(); + } + return mir_->toDiv()->bytecodeOffset(); + } +}; + +class LWasmCompareExchangeI64 + : public LInstructionHelper { + public: + LIR_HEADER(WasmCompareExchangeI64); + + LWasmCompareExchangeI64(const LAllocation& ptr, + const LInt64Allocation& oldValue, + const LInt64Allocation& newValue) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setInt64Operand(1, oldValue); + setInt64Operand(1 + INT64_PIECES, newValue); + } + + const LAllocation* ptr() { return getOperand(0); } + const LInt64Allocation oldValue() { return getInt64Operand(1); } + const LInt64Allocation newValue() { + return getInt64Operand(1 + INT64_PIECES); + } + const MWasmCompareExchangeHeap* mir() const { + return mir_->toWasmCompareExchangeHeap(); + } +}; + +class LWasmAtomicExchangeI64 + : public LInstructionHelper { + public: + LIR_HEADER(WasmAtomicExchangeI64); + + LWasmAtomicExchangeI64(const LAllocation& ptr, const LInt64Allocation& value) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setInt64Operand(1, value); + } + + const LAllocation* ptr() { return getOperand(0); } + const LInt64Allocation value() { return getInt64Operand(1); } + const MWasmAtomicExchangeHeap* mir() const { + return mir_->toWasmAtomicExchangeHeap(); + } +}; + +class LWasmAtomicBinopI64 + : public LInstructionHelper { + public: + LIR_HEADER(WasmAtomicBinopI64); + + LWasmAtomicBinopI64(const LAllocation& ptr, const LInt64Allocation& value) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setInt64Operand(1, value); + } + + const LAllocation* ptr() { return getOperand(0); } + const LInt64Allocation value() { return getInt64Operand(1); } + const MWasmAtomicBinopHeap* mir() const { + return mir_->toWasmAtomicBinopHeap(); + } +}; + +class LDivOrModI64 : public LBinaryMath<1> { + public: + LIR_HEADER(DivOrModI64) + + LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp); + } + + const LDefinition* remainder() { return getTemp(0); } + MBinaryArithInstruction* mir() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + return static_cast(mir_); + } + + bool canBeDivideByZero() const { + if (mir_->isMod()) { + return mir_->toMod()->canBeDivideByZero(); + } + return mir_->toDiv()->canBeDivideByZero(); + } + bool canBeNegativeOverflow() const { + if (mir_->isMod()) { + return mir_->toMod()->canBeNegativeDividend(); + } + return mir_->toDiv()->canBeNegativeOverflow(); + } + wasm::BytecodeOffset bytecodeOffset() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + if (mir_->isMod()) { + return mir_->toMod()->bytecodeOffset(); + } + return mir_->toDiv()->bytecodeOffset(); + } +}; + +class LUDivOrModI64 : public LBinaryMath<1> { + public: + LIR_HEADER(UDivOrModI64); + + LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp); + } + + const LDefinition* remainder() { return getTemp(0); } + const char* extraName() const { + return mir()->isTruncated() ? "Truncated" : nullptr; + } + + MBinaryArithInstruction* mir() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + return static_cast(mir_); + } + bool canBeDivideByZero() const { + if (mir_->isMod()) { + return mir_->toMod()->canBeDivideByZero(); + } + return mir_->toDiv()->canBeDivideByZero(); + } + wasm::BytecodeOffset bytecodeOffset() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + if (mir_->isMod()) { + return mir_->toMod()->bytecodeOffset(); + } + return mir_->toDiv()->bytecodeOffset(); + } +}; + +class LWasmTruncateToInt64 : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(WasmTruncateToInt64); + + explicit LWasmTruncateToInt64(const LAllocation& in) + : LInstructionHelper(classOpcode) { + setOperand(0, in); + } + + MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); } +}; + +class LInt64ToFloatingPoint : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(Int64ToFloatingPoint); + + explicit LInt64ToFloatingPoint(const LInt64Allocation& in) + : LInstructionHelper(classOpcode) { + setInt64Operand(0, in); + } + + MInt64ToFloatingPoint* mir() const { return mir_->toInt64ToFloatingPoint(); } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_LIR_loong64_h */ diff --git a/js/src/jit/loong64/Lowering-loong64.cpp b/js/src/jit/loong64/Lowering-loong64.cpp new file mode 100644 index 0000000000..cb404b9468 --- /dev/null +++ b/js/src/jit/loong64/Lowering-loong64.cpp @@ -0,0 +1,1056 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loong64/Lowering-loong64.h" + +#include "mozilla/MathAlgorithms.h" + +#include "jit/loong64/Assembler-loong64.h" +#include "jit/Lowering.h" +#include "jit/MIR.h" +#include "jit/shared/Lowering-shared-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::FloorLog2; + +LAllocation LIRGeneratorLOONG64::useByteOpRegister(MDefinition* mir) { + return useRegister(mir); +} + +LAllocation LIRGeneratorLOONG64::useByteOpRegisterAtStart(MDefinition* mir) { + return useRegisterAtStart(mir); +} + +LAllocation LIRGeneratorLOONG64::useByteOpRegisterOrNonDoubleConstant( + MDefinition* mir) { + return useRegisterOrNonDoubleConstant(mir); +} + +LDefinition LIRGeneratorLOONG64::tempByteOpRegister() { return temp(); } + +void LIRGeneratorLOONG64::lowerWasmBuiltinTruncateToInt32( + MWasmBuiltinTruncateToInt32* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32); + + if (opd->type() == MIRType::Double) { + define(new (alloc()) LWasmBuiltinTruncateDToInt32( + useRegister(opd), useFixed(ins->tls(), WasmTlsReg), + LDefinition::BogusTemp()), + ins); + return; + } + + define(new (alloc()) LWasmBuiltinTruncateFToInt32( + useRegister(opd), useFixed(ins->tls(), WasmTlsReg), + LDefinition::BogusTemp()), + ins); +} + +void LIRGeneratorLOONG64::lowerForShift(LInstructionHelper<1, 2, 0>* ins, + MDefinition* mir, MDefinition* lhs, + MDefinition* rhs) { + ins->setOperand(0, useRegister(lhs)); + ins->setOperand(1, useRegisterOrConstant(rhs)); + define(ins, mir); +} + +void LIRGeneratorLOONG64::lowerUrshD(MUrsh* mir) { + MDefinition* lhs = mir->lhs(); + MDefinition* rhs = mir->rhs(); + + MOZ_ASSERT(lhs->type() == MIRType::Int32); + MOZ_ASSERT(rhs->type() == MIRType::Int32); + + LUrshD* lir = new (alloc()) + LUrshD(useRegister(lhs), useRegisterOrConstant(rhs), temp()); + define(lir, mir); +} + +// x = !y +void LIRGeneratorLOONG64::lowerForALU(LInstructionHelper<1, 1, 0>* ins, + MDefinition* mir, MDefinition* input) { + ins->setOperand(0, useRegister(input)); + define( + ins, mir, + LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER)); +} + +// z = x + y +void LIRGeneratorLOONG64::lowerForALU(LInstructionHelper<1, 2, 0>* ins, + MDefinition* mir, MDefinition* lhs, + MDefinition* rhs) { + ins->setOperand(0, useRegister(lhs)); + ins->setOperand(1, useRegisterOrConstant(rhs)); + define( + ins, mir, + LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER)); +} + +void LIRGeneratorLOONG64::lowerForALUInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs) { + ins->setInt64Operand(0, useInt64RegisterAtStart(lhs)); + ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs) + ? useInt64OrConstant(rhs) + : useInt64OrConstantAtStart(rhs)); + defineInt64ReuseInput(ins, mir, 0); +} + +void LIRGeneratorLOONG64::lowerForMulInt64(LMulI64* ins, MMul* mir, + MDefinition* lhs, MDefinition* rhs) { + bool needsTemp = false; + bool cannotAliasRhs = false; + bool reuseInput = true; + + ins->setInt64Operand(0, useInt64RegisterAtStart(lhs)); + ins->setInt64Operand(INT64_PIECES, + (willHaveDifferentLIRNodes(lhs, rhs) || cannotAliasRhs) + ? useInt64OrConstant(rhs) + : useInt64OrConstantAtStart(rhs)); + + if (needsTemp) { + ins->setTemp(0, temp()); + } + if (reuseInput) { + defineInt64ReuseInput(ins, mir, 0); + } else { + defineInt64(ins, mir); + } +} + +template +void LIRGeneratorLOONG64::lowerForShiftInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs) { + ins->setInt64Operand(0, useInt64RegisterAtStart(lhs)); + + static_assert(LShiftI64::Rhs == INT64_PIECES, + "Assume Rhs is located at INT64_PIECES."); + static_assert(LRotateI64::Count == INT64_PIECES, + "Assume Count is located at INT64_PIECES."); + + ins->setOperand(INT64_PIECES, useRegisterOrConstant(rhs)); + + defineInt64ReuseInput(ins, mir, 0); +} + +template void LIRGeneratorLOONG64::lowerForShiftInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs); +template void LIRGeneratorLOONG64::lowerForShiftInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs); + +void LIRGeneratorLOONG64::lowerForCompareI64AndBranch( + MTest* mir, MCompare* comp, JSOp op, MDefinition* left, MDefinition* right, + MBasicBlock* ifTrue, MBasicBlock* ifFalse) { + LCompareI64AndBranch* lir = new (alloc()) + LCompareI64AndBranch(comp, op, useInt64Register(left), + useInt64OrConstant(right), ifTrue, ifFalse); + add(lir, mir); +} + +void LIRGeneratorLOONG64::lowerForFPU(LInstructionHelper<1, 1, 0>* ins, + MDefinition* mir, MDefinition* input) { + ins->setOperand(0, useRegister(input)); + define( + ins, mir, + LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER)); +} + +template +void LIRGeneratorLOONG64::lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, + MDefinition* mir, MDefinition* lhs, + MDefinition* rhs) { + ins->setOperand(0, useRegister(lhs)); + ins->setOperand(1, useRegister(rhs)); + define( + ins, mir, + LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER)); +} + +template void LIRGeneratorLOONG64::lowerForFPU(LInstructionHelper<1, 2, 0>* ins, + MDefinition* mir, + MDefinition* lhs, + MDefinition* rhs); +template void LIRGeneratorLOONG64::lowerForFPU(LInstructionHelper<1, 2, 1>* ins, + MDefinition* mir, + MDefinition* lhs, + MDefinition* rhs); + +void LIRGeneratorLOONG64::lowerForBitAndAndBranch(LBitAndAndBranch* baab, + MInstruction* mir, + MDefinition* lhs, + MDefinition* rhs) { + baab->setOperand(0, useRegisterAtStart(lhs)); + baab->setOperand(1, useRegisterOrConstantAtStart(rhs)); + add(baab, mir); +} + +void LIRGeneratorLOONG64::lowerDivI(MDiv* div) { + if (div->isUnsigned()) { + lowerUDiv(div); + return; + } + + // Division instructions are slow. Division by constant denominators can be + // rewritten to use other instructions. + if (div->rhs()->isConstant()) { + int32_t rhs = div->rhs()->toConstant()->toInt32(); + // Check for division by a positive power of two, which is an easy and + // important case to optimize. Note that other optimizations are also + // possible; division by negative powers of two can be optimized in a + // similar manner as positive powers of two, and division by other + // constants can be optimized by a reciprocal multiplication technique. + int32_t shift = FloorLog2(rhs); + if (rhs > 0 && 1 << shift == rhs) { + LDivPowTwoI* lir = + new (alloc()) LDivPowTwoI(useRegister(div->lhs()), shift, temp()); + if (div->fallible()) { + assignSnapshot(lir, div->bailoutKind()); + } + define(lir, div); + return; + } + } + + LDivI* lir = new (alloc()) + LDivI(useRegister(div->lhs()), useRegister(div->rhs()), temp()); + if (div->fallible()) { + assignSnapshot(lir, div->bailoutKind()); + } + define(lir, div); +} + +void LIRGeneratorLOONG64::lowerNegI(MInstruction* ins, MDefinition* input) { + define(new (alloc()) LNegI(useRegisterAtStart(input)), ins); +} + +void LIRGeneratorLOONG64::lowerNegI64(MInstruction* ins, MDefinition* input) { + defineInt64ReuseInput(new (alloc()) LNegI64(useInt64RegisterAtStart(input)), + ins, 0); +} + +void LIRGenerator::visitAbs(MAbs* ins) { + define(allocateAbs(ins, useRegisterAtStart(ins->input())), ins); +} + +void LIRGeneratorLOONG64::lowerModI(MMod* mod) { + if (mod->isUnsigned()) { + lowerUMod(mod); + return; + } + + if (mod->rhs()->isConstant()) { + int32_t rhs = mod->rhs()->toConstant()->toInt32(); + int32_t shift = FloorLog2(rhs); + if (rhs > 0 && 1 << shift == rhs) { + LModPowTwoI* lir = + new (alloc()) LModPowTwoI(useRegister(mod->lhs()), shift); + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + define(lir, mod); + return; + } else if (shift < 31 && (1 << (shift + 1)) - 1 == rhs) { + LModMaskI* lir = new (alloc()) + LModMaskI(useRegister(mod->lhs()), temp(LDefinition::GENERAL), + temp(LDefinition::GENERAL), shift + 1); + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + define(lir, mod); + return; + } + } + LModI* lir = + new (alloc()) LModI(useRegister(mod->lhs()), useRegister(mod->rhs()), + temp(LDefinition::GENERAL)); + + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + define(lir, mod); +} + +void LIRGeneratorLOONG64::lowerMulI(MMul* mul, MDefinition* lhs, + MDefinition* rhs) { + LMulI* lir = new (alloc()) LMulI; + if (mul->fallible()) { + assignSnapshot(lir, mul->bailoutKind()); + } + + lowerForALU(lir, mul, lhs, rhs); +} + +void LIRGeneratorLOONG64::lowerUDiv(MDiv* div) { + MDefinition* lhs = div->getOperand(0); + MDefinition* rhs = div->getOperand(1); + + LUDivOrMod* lir = new (alloc()) LUDivOrMod; + lir->setOperand(0, useRegister(lhs)); + lir->setOperand(1, useRegister(rhs)); + if (div->fallible()) { + assignSnapshot(lir, div->bailoutKind()); + } + + define(lir, div); +} + +void LIRGeneratorLOONG64::lowerUMod(MMod* mod) { + MDefinition* lhs = mod->getOperand(0); + MDefinition* rhs = mod->getOperand(1); + + LUDivOrMod* lir = new (alloc()) LUDivOrMod; + lir->setOperand(0, useRegister(lhs)); + lir->setOperand(1, useRegister(rhs)); + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + + define(lir, mod); +} + +void LIRGeneratorLOONG64::lowerWasmSelectI(MWasmSelect* select) { + auto* lir = new (alloc()) + LWasmSelect(useRegisterAtStart(select->trueExpr()), + useAny(select->falseExpr()), useRegister(select->condExpr())); + defineReuseInput(lir, select, LWasmSelect::TrueExprIndex); +} + +void LIRGeneratorLOONG64::lowerWasmSelectI64(MWasmSelect* select) { + auto* lir = new (alloc()) LWasmSelectI64( + useInt64RegisterAtStart(select->trueExpr()), + useInt64(select->falseExpr()), useRegister(select->condExpr())); + defineInt64ReuseInput(lir, select, LWasmSelectI64::TrueExprIndex); +} + +LTableSwitch* LIRGeneratorLOONG64::newLTableSwitch(const LAllocation& in, + const LDefinition& inputCopy, + MTableSwitch* tableswitch) { + return new (alloc()) LTableSwitch(in, inputCopy, temp(), tableswitch); +} + +LTableSwitchV* LIRGeneratorLOONG64::newLTableSwitchV( + MTableSwitch* tableswitch) { + return new (alloc()) LTableSwitchV(useBox(tableswitch->getOperand(0)), temp(), + tempDouble(), temp(), tableswitch); +} + +void LIRGeneratorLOONG64::defineInt64Phi(MPhi* phi, size_t lirIndex) { + defineTypedPhi(phi, lirIndex); +} + +void LIRGeneratorLOONG64::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, + LBlock* block, size_t lirIndex) { + lowerTypedPhiInput(phi, inputPosition, block, lirIndex); +} + +LBoxAllocation LIRGeneratorLOONG64::useBoxFixed(MDefinition* mir, Register reg1, + Register reg2, + bool useAtStart) { + MOZ_ASSERT(mir->type() == MIRType::Value); + + ensureDefined(mir); + return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart)); +} + +void LIRGeneratorLOONG64::lowerDivI64(MDiv* div) { + if (div->isUnsigned()) { + lowerUDivI64(div); + return; + } + + LDivOrModI64* lir = new (alloc()) + LDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()), temp()); + defineInt64(lir, div); +} + +void LIRGeneratorLOONG64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) { + MOZ_CRASH("We don't use runtime div for this architecture"); +} + +void LIRGeneratorLOONG64::lowerModI64(MMod* mod) { + if (mod->isUnsigned()) { + lowerUModI64(mod); + return; + } + + LDivOrModI64* lir = new (alloc()) + LDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()), temp()); + defineInt64(lir, mod); +} + +void LIRGeneratorLOONG64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) { + MOZ_CRASH("We don't use runtime mod for this architecture"); +} + +void LIRGeneratorLOONG64::lowerUDivI64(MDiv* div) { + LUDivOrModI64* lir = new (alloc()) + LUDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()), temp()); + defineInt64(lir, div); +} + +void LIRGeneratorLOONG64::lowerUModI64(MMod* mod) { + LUDivOrModI64* lir = new (alloc()) + LUDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()), temp()); + defineInt64(lir, mod); +} + +void LIRGenerator::visitPowHalf(MPowHalf* ins) { + MDefinition* input = ins->input(); + MOZ_ASSERT(input->type() == MIRType::Double); + LPowHalfD* lir = new (alloc()) LPowHalfD(useRegisterAtStart(input)); + defineReuseInput(lir, ins, 0); +} + +void LIRGeneratorLOONG64::lowerPowOfTwoI(MPow* mir) { + int32_t base = mir->input()->toConstant()->toInt32(); + MDefinition* power = mir->power(); + + auto* lir = new (alloc()) LPowOfTwoI(base, useRegister(power)); + assignSnapshot(lir, mir->bailoutKind()); + define(lir, mir); +} + +void LIRGeneratorLOONG64::lowerBigIntLsh(MBigIntLsh* ins) { + auto* lir = new (alloc()) LBigIntLsh( + useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp()); + define(lir, ins); + assignSafepoint(lir, ins); +} + +void LIRGeneratorLOONG64::lowerBigIntRsh(MBigIntRsh* ins) { + auto* lir = new (alloc()) LBigIntRsh( + useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp()); + define(lir, ins); + assignSafepoint(lir, ins); +} + +void LIRGeneratorLOONG64::lowerBigIntDiv(MBigIntDiv* ins) { + auto* lir = new (alloc()) LBigIntDiv(useRegister(ins->lhs()), + useRegister(ins->rhs()), temp(), temp()); + define(lir, ins); + assignSafepoint(lir, ins); +} + +void LIRGeneratorLOONG64::lowerBigIntMod(MBigIntMod* ins) { + auto* lir = new (alloc()) LBigIntMod(useRegister(ins->lhs()), + useRegister(ins->rhs()), temp(), temp()); + define(lir, ins); + assignSafepoint(lir, ins); +} + +void LIRGenerator::visitWasmNeg(MWasmNeg* ins) { + if (ins->type() == MIRType::Int32) { + define(new (alloc()) LNegI(useRegisterAtStart(ins->input())), ins); + } else if (ins->type() == MIRType::Float32) { + define(new (alloc()) LNegF(useRegisterAtStart(ins->input())), ins); + } else { + MOZ_ASSERT(ins->type() == MIRType::Double); + define(new (alloc()) LNegD(useRegisterAtStart(ins->input())), ins); + } +} + +void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) { + auto* lir = new (alloc()) LWasmHeapBase(LAllocation()); + define(lir, ins); +} + +void LIRGenerator::visitWasmLoad(MWasmLoad* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + + LAllocation ptr; + ptr = useRegisterAtStart(base); + + if (ins->type() == MIRType::Int64) { + auto* lir = new (alloc()) LWasmLoadI64(ptr); + if (ins->access().offset()) { + lir->setTemp(0, tempCopy(base, 0)); + } + + defineInt64(lir, ins); + return; + } + + auto* lir = new (alloc()) LWasmLoad(ptr); + if (ins->access().offset()) { + lir->setTemp(0, tempCopy(base, 0)); + } + + define(lir, ins); +} + +void LIRGenerator::visitWasmStore(MWasmStore* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + + MDefinition* value = ins->value(); + + if (ins->access().type() == Scalar::Int64) { + LAllocation baseAlloc = useRegisterAtStart(base); + LInt64Allocation valueAlloc = useInt64RegisterAtStart(value); + auto* lir = new (alloc()) LWasmStoreI64(baseAlloc, valueAlloc); + if (ins->access().offset()) { + lir->setTemp(0, tempCopy(base, 0)); + } + + add(lir, ins); + return; + } + + LAllocation baseAlloc = useRegisterAtStart(base); + LAllocation valueAlloc = useRegisterAtStart(value); + auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc); + if (ins->access().offset()) { + lir->setTemp(0, tempCopy(base, 0)); + } + + add(lir, ins); +} + +void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) { + MOZ_ASSERT(ins->input()->type() == MIRType::Int32); + LWasmUint32ToDouble* lir = + new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input())); + define(lir, ins); +} + +void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) { + MOZ_ASSERT(ins->input()->type() == MIRType::Int32); + LWasmUint32ToFloat32* lir = + new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input())); + define(lir, ins); +} + +void LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins) { + MOZ_ASSERT(ins->access().offset() == 0); + + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + LAllocation baseAlloc; + LAllocation limitAlloc; + // For LA it is best to keep the 'base' in a register if a bounds check + // is needed + if (base->isConstant() && !ins->needsBoundsCheck()) { + // A bounds check is only skipped for a positive index. + MOZ_ASSERT(base->toConstant()->toInt32() >= 0); + baseAlloc = LAllocation(base->toConstant()); + } else { + baseAlloc = useRegisterAtStart(base); + if (ins->needsBoundsCheck()) { + MDefinition* boundsCheckLimit = ins->boundsCheckLimit(); + MOZ_ASSERT(boundsCheckLimit->type() == MIRType::Int32); + limitAlloc = useRegisterAtStart(boundsCheckLimit); + } + } + + define(new (alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc), ins); +} + +void LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins) { + MOZ_ASSERT(ins->access().offset() == 0); + + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + LAllocation baseAlloc; + LAllocation limitAlloc; + if (base->isConstant() && !ins->needsBoundsCheck()) { + MOZ_ASSERT(base->toConstant()->toInt32() >= 0); + baseAlloc = LAllocation(base->toConstant()); + } else { + baseAlloc = useRegisterAtStart(base); + if (ins->needsBoundsCheck()) { + MDefinition* boundsCheckLimit = ins->boundsCheckLimit(); + MOZ_ASSERT(boundsCheckLimit->type() == MIRType::Int32); + limitAlloc = useRegisterAtStart(boundsCheckLimit); + } + } + + add(new (alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()), + limitAlloc), + ins); +} + +void LIRGenerator::visitSubstr(MSubstr* ins) { + LSubstr* lir = new (alloc()) + LSubstr(useRegister(ins->string()), useRegister(ins->begin()), + useRegister(ins->length()), temp(), temp(), tempByteOpRegister()); + define(lir, ins); + assignSafepoint(lir, ins); +} + +void LIRGenerator::visitCompareExchangeTypedArrayElement( + MCompareExchangeTypedArrayElement* ins) { + MOZ_ASSERT(ins->arrayType() != Scalar::Float32); + MOZ_ASSERT(ins->arrayType() != Scalar::Float64); + + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + + const LAllocation newval = useRegister(ins->newval()); + const LAllocation oldval = useRegister(ins->oldval()); + + if (Scalar::isBigIntType(ins->arrayType())) { + LInt64Definition temp1 = tempInt64(); + LInt64Definition temp2 = tempInt64(); + + auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64( + elements, index, oldval, newval, temp1, temp2); + define(lir, ins); + assignSafepoint(lir, ins); + return; + } + + // If the target is a floating register then we need a temp at the + // CodeGenerator level for creating the result. + + LDefinition outTemp = LDefinition::BogusTemp(); + LDefinition valueTemp = LDefinition::BogusTemp(); + LDefinition offsetTemp = LDefinition::BogusTemp(); + LDefinition maskTemp = LDefinition::BogusTemp(); + + if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) { + outTemp = temp(); + } + + if (Scalar::byteSize(ins->arrayType()) < 4) { + valueTemp = temp(); + offsetTemp = temp(); + maskTemp = temp(); + } + + LCompareExchangeTypedArrayElement* lir = new (alloc()) + LCompareExchangeTypedArrayElement(elements, index, oldval, newval, + outTemp, valueTemp, offsetTemp, + maskTemp); + + define(lir, ins); +} + +void LIRGenerator::visitAtomicExchangeTypedArrayElement( + MAtomicExchangeTypedArrayElement* ins) { + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + + const LAllocation value = useRegister(ins->value()); + + if (Scalar::isBigIntType(ins->arrayType())) { + LInt64Definition temp1 = tempInt64(); + LDefinition temp2 = temp(); + + auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64( + elements, index, value, temp1, temp2); + define(lir, ins); + assignSafepoint(lir, ins); + return; + } + + // If the target is a floating register then we need a temp at the + // CodeGenerator level for creating the result. + + MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32); + + LDefinition outTemp = LDefinition::BogusTemp(); + LDefinition valueTemp = LDefinition::BogusTemp(); + LDefinition offsetTemp = LDefinition::BogusTemp(); + LDefinition maskTemp = LDefinition::BogusTemp(); + + if (ins->arrayType() == Scalar::Uint32) { + MOZ_ASSERT(ins->type() == MIRType::Double); + outTemp = temp(); + } + + if (Scalar::byteSize(ins->arrayType()) < 4) { + valueTemp = temp(); + offsetTemp = temp(); + maskTemp = temp(); + } + + LAtomicExchangeTypedArrayElement* lir = + new (alloc()) LAtomicExchangeTypedArrayElement( + elements, index, value, outTemp, valueTemp, offsetTemp, maskTemp); + + define(lir, ins); +} + +void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) { + MOZ_ASSERT(ins->base()->type() == MIRType::Int32); + + if (ins->access().type() == Scalar::Int64) { + auto* lir = new (alloc()) LWasmCompareExchangeI64( + useRegister(ins->base()), useInt64Register(ins->oldValue()), + useInt64Register(ins->newValue())); + defineInt64(lir, ins); + return; + } + + LDefinition valueTemp = LDefinition::BogusTemp(); + LDefinition offsetTemp = LDefinition::BogusTemp(); + LDefinition maskTemp = LDefinition::BogusTemp(); + + if (ins->access().byteSize() < 4) { + valueTemp = temp(); + offsetTemp = temp(); + maskTemp = temp(); + } + + LWasmCompareExchangeHeap* lir = new (alloc()) LWasmCompareExchangeHeap( + useRegister(ins->base()), useRegister(ins->oldValue()), + useRegister(ins->newValue()), valueTemp, offsetTemp, maskTemp); + + define(lir, ins); +} + +void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) { + MOZ_ASSERT(ins->base()->type() == MIRType::Int32); + + if (ins->access().type() == Scalar::Int64) { + auto* lir = new (alloc()) LWasmAtomicExchangeI64( + useRegister(ins->base()), useInt64Register(ins->value())); + defineInt64(lir, ins); + return; + } + + LDefinition valueTemp = LDefinition::BogusTemp(); + LDefinition offsetTemp = LDefinition::BogusTemp(); + LDefinition maskTemp = LDefinition::BogusTemp(); + + if (ins->access().byteSize() < 4) { + valueTemp = temp(); + offsetTemp = temp(); + maskTemp = temp(); + } + + LWasmAtomicExchangeHeap* lir = new (alloc()) LWasmAtomicExchangeHeap( + useRegister(ins->base()), useRegister(ins->value()), valueTemp, + offsetTemp, maskTemp); + define(lir, ins); +} + +void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) { + MOZ_ASSERT(ins->base()->type() == MIRType::Int32); + + if (ins->access().type() == Scalar::Int64) { + auto* lir = new (alloc()) LWasmAtomicBinopI64( + useRegister(ins->base()), useInt64Register(ins->value())); + lir->setTemp(0, temp()); + defineInt64(lir, ins); + return; + } + + LDefinition valueTemp = LDefinition::BogusTemp(); + LDefinition offsetTemp = LDefinition::BogusTemp(); + LDefinition maskTemp = LDefinition::BogusTemp(); + + if (ins->access().byteSize() < 4) { + valueTemp = temp(); + offsetTemp = temp(); + maskTemp = temp(); + } + + if (!ins->hasUses()) { + LWasmAtomicBinopHeapForEffect* lir = new (alloc()) + LWasmAtomicBinopHeapForEffect(useRegister(ins->base()), + useRegister(ins->value()), valueTemp, + offsetTemp, maskTemp); + add(lir, ins); + return; + } + + LWasmAtomicBinopHeap* lir = new (alloc()) + LWasmAtomicBinopHeap(useRegister(ins->base()), useRegister(ins->value()), + valueTemp, offsetTemp, maskTemp); + + define(lir, ins); +} + +void LIRGenerator::visitAtomicTypedArrayElementBinop( + MAtomicTypedArrayElementBinop* ins) { + MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped); + MOZ_ASSERT(ins->arrayType() != Scalar::Float32); + MOZ_ASSERT(ins->arrayType() != Scalar::Float64); + + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + const LAllocation value = useRegister(ins->value()); + + if (Scalar::isBigIntType(ins->arrayType())) { + LInt64Definition temp1 = tempInt64(); + LInt64Definition temp2 = tempInt64(); + + // Case 1: the result of the operation is not used. + // + // We can omit allocating the result BigInt. + + if (ins->isForEffect()) { + auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64( + elements, index, value, temp1, temp2); + add(lir, ins); + return; + } + + // Case 2: the result of the operation is used. + + auto* lir = new (alloc()) + LAtomicTypedArrayElementBinop64(elements, index, value, temp1, temp2); + define(lir, ins); + assignSafepoint(lir, ins); + return; + } + + LDefinition valueTemp = LDefinition::BogusTemp(); + LDefinition offsetTemp = LDefinition::BogusTemp(); + LDefinition maskTemp = LDefinition::BogusTemp(); + + if (Scalar::byteSize(ins->arrayType()) < 4) { + valueTemp = temp(); + offsetTemp = temp(); + maskTemp = temp(); + } + + if (ins->isForEffect()) { + LAtomicTypedArrayElementBinopForEffect* lir = + new (alloc()) LAtomicTypedArrayElementBinopForEffect( + elements, index, value, valueTemp, offsetTemp, maskTemp); + add(lir, ins); + return; + } + + // For a Uint32Array with a known double result we need a temp for + // the intermediate output. + + LDefinition outTemp = LDefinition::BogusTemp(); + + if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) { + outTemp = temp(); + } + + LAtomicTypedArrayElementBinop* lir = + new (alloc()) LAtomicTypedArrayElementBinop( + elements, index, value, outTemp, valueTemp, offsetTemp, maskTemp); + define(lir, ins); +} + +void LIRGenerator::visitCopySign(MCopySign* ins) { + MDefinition* lhs = ins->lhs(); + MDefinition* rhs = ins->rhs(); + + MOZ_ASSERT(IsFloatingPointType(lhs->type())); + MOZ_ASSERT(lhs->type() == rhs->type()); + MOZ_ASSERT(lhs->type() == ins->type()); + + LInstructionHelper<1, 2, 2>* lir; + if (lhs->type() == MIRType::Double) { + lir = new (alloc()) LCopySignD(); + } else { + lir = new (alloc()) LCopySignF(); + } + + lir->setTemp(0, temp()); + lir->setTemp(1, temp()); + + lir->setOperand(0, useRegisterAtStart(lhs)); + lir->setOperand(1, useRegister(rhs)); + defineReuseInput(lir, ins, 0); +} + +void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) { + defineInt64( + new (alloc()) LExtendInt32ToInt64(useRegisterAtStart(ins->input())), ins); +} + +void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) { + defineInt64(new (alloc()) + LSignExtendInt64(useInt64RegisterAtStart(ins->input())), + ins); +} + +void LIRGenerator::visitWasmBitselectSimd128(MWasmBitselectSimd128* ins) { + MOZ_CRASH("bitselect NYI"); +} + +void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) { + MOZ_CRASH("binary SIMD NYI"); +} + +#ifdef ENABLE_WASM_SIMD +bool MWasmBitselectSimd128::specializeConstantMaskAsShuffle( + int8_t shuffle[16]) { + return false; +} +#endif + +bool MWasmBinarySimd128::specializeForConstantRhs() { + // Probably many we want to do here + return false; +} + +void LIRGenerator::visitWasmBinarySimd128WithConstant( + MWasmBinarySimd128WithConstant* ins) { + MOZ_CRASH("binary SIMD with constant NYI"); +} + +void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) { + MOZ_CRASH("shift SIMD NYI"); +} + +void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) { + MOZ_CRASH("shuffle SIMD NYI"); +} + +void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) { + MOZ_CRASH("replace-lane SIMD NYI"); +} + +void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) { + MOZ_CRASH("scalar-to-SIMD NYI"); +} + +void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) { + MOZ_CRASH("unary SIMD NYI"); +} + +void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) { + MOZ_CRASH("reduce-SIMD NYI"); +} + +void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) { + MOZ_CRASH("load-lane SIMD NYI"); +} + +void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) { + MOZ_CRASH("store-lane SIMD NYI"); +} + +void LIRGeneratorLOONG64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) { + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->storageType()); + + auto* lir = new (alloc()) LAtomicLoad64(elements, index, temp(), tempInt64()); + define(lir, ins); + assignSafepoint(lir, ins); +} + +void LIRGeneratorLOONG64::lowerAtomicStore64(MStoreUnboxedScalar* ins) { + LUse elements = useRegister(ins->elements()); + LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->writeType()); + LAllocation value = useRegister(ins->value()); + + add(new (alloc()) LAtomicStore64(elements, index, value, tempInt64()), ins); +} + +void LIRGenerator::visitBox(MBox* box) { + MDefinition* opd = box->getOperand(0); + + // If the operand is a constant, emit near its uses. + if (opd->isConstant() && box->canEmitAtUses()) { + emitAtUses(box); + return; + } + + if (opd->isConstant()) { + define(new (alloc()) LValue(opd->toConstant()->toJSValue()), box, + LDefinition(LDefinition::BOX)); + } else { + LBox* ins = new (alloc()) LBox(useRegister(opd), opd->type()); + define(ins, box, LDefinition(LDefinition::BOX)); + } +} + +void LIRGenerator::visitUnbox(MUnbox* unbox) { + MDefinition* box = unbox->getOperand(0); + MOZ_ASSERT(box->type() == MIRType::Value); + + LUnbox* lir; + if (IsFloatingPointType(unbox->type())) { + lir = new (alloc()) + LUnboxFloatingPoint(useRegisterAtStart(box), unbox->type()); + } else if (unbox->fallible()) { + // If the unbox is fallible, load the Value in a register first to + // avoid multiple loads. + lir = new (alloc()) LUnbox(useRegisterAtStart(box)); + } else { + lir = new (alloc()) LUnbox(useAtStart(box)); + } + + if (unbox->fallible()) { + assignSnapshot(lir, unbox->bailoutKind()); + } + + define(lir, unbox); +} + +void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) { + MOZ_ASSERT(opd->type() == MIRType::Value); + + LReturn* ins = new (alloc()) LReturn(isGenerator); + ins->setOperand(0, useFixed(opd, JSReturnReg)); + add(ins); +} + +void LIRGeneratorLOONG64::lowerUntypedPhiInput(MPhi* phi, + uint32_t inputPosition, + LBlock* block, size_t lirIndex) { + lowerTypedPhiInput(phi, inputPosition, block, lirIndex); +} + +void LIRGeneratorLOONG64::lowerTruncateDToInt32(MTruncateToInt32* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Double); + + define(new (alloc()) LTruncateDToInt32(useRegister(opd), tempDouble()), ins); +} + +void LIRGeneratorLOONG64::lowerTruncateFToInt32(MTruncateToInt32* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Float32); + + define(new (alloc()) LTruncateFToInt32(useRegister(opd), tempFloat32()), ins); +} + +void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32); + + defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd)), ins); +} + +void LIRGeneratorLOONG64::lowerWasmBuiltinTruncateToInt64( + MWasmBuiltinTruncateToInt64* ins) { + MOZ_CRASH("We don't use it for this architecture"); +} + +void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Int64); + MOZ_ASSERT(IsFloatingPointType(ins->type())); + + define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd)), ins); +} + +void LIRGeneratorLOONG64::lowerBuiltinInt64ToFloatingPoint( + MBuiltinInt64ToFloatingPoint* ins) { + MOZ_CRASH("We don't use it for this architecture"); +} diff --git a/js/src/jit/loong64/Lowering-loong64.h b/js/src/jit/loong64/Lowering-loong64.h new file mode 100644 index 0000000000..2483295dcc --- /dev/null +++ b/js/src/jit/loong64/Lowering-loong64.h @@ -0,0 +1,120 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_Lowering_loong64_h +#define jit_loong64_Lowering_loong64_h + +#include "jit/shared/Lowering-shared.h" + +namespace js { +namespace jit { + +class LIRGeneratorLOONG64 : public LIRGeneratorShared { + protected: + LIRGeneratorLOONG64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph) + : LIRGeneratorShared(gen, graph, lirGraph) {} + + void lowerInt64PhiInput(MPhi*, uint32_t, LBlock*, size_t); + void defineInt64Phi(MPhi*, size_t); + + // Returns a box allocation. reg2 is ignored on 64-bit platforms. + LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2, + bool useAtStart = false); + + inline LDefinition tempToUnbox() { return temp(); } + + void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block, + size_t lirIndex); + + void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins); + void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins); + void lowerTruncateDToInt32(MTruncateToInt32* ins); + void lowerTruncateFToInt32(MTruncateToInt32* ins); + + void lowerDivI64(MDiv* div); + void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div); + void lowerModI64(MMod* mod); + void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod); + void lowerUDivI64(MDiv* div); + void lowerUModI64(MMod* mod); + + void lowerBigIntDiv(MBigIntDiv* ins); + void lowerBigIntMod(MBigIntMod* ins); + + void lowerAtomicLoad64(MLoadUnboxedScalar* ins); + void lowerAtomicStore64(MStoreUnboxedScalar* ins); + + // x86 has constraints on what registers can be formatted for 1-byte + // stores and loads; on LA all registers are okay. + + LAllocation useByteOpRegister(MDefinition* mir); + LAllocation useByteOpRegisterAtStart(MDefinition* mir); + LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir); + LDefinition tempByteOpRegister(); + + bool needTempForPostBarrier() { return true; } + + void lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir, + MDefinition* lhs, MDefinition* rhs); + void lowerUrshD(MUrsh* mir); + + void lowerPowOfTwoI(MPow* mir); + + void lowerForALU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir, + MDefinition* input); + void lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir, + MDefinition* lhs, MDefinition* rhs); + + void lowerForALUInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs); + void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs, + MDefinition* rhs); + template + void lowerForShiftInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs); + + void lowerForCompareI64AndBranch(MTest* mir, MCompare* comp, JSOp op, + MDefinition* left, MDefinition* right, + MBasicBlock* ifTrue, MBasicBlock* ifFalse); + + void lowerForFPU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir, + MDefinition* src); + template + void lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, MDefinition* mir, + MDefinition* lhs, MDefinition* rhs); + + void lowerForBitAndAndBranch(LBitAndAndBranch* baab, MInstruction* mir, + MDefinition* lhs, MDefinition* rhs); + void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins); + void lowerDivI(MDiv* div); + void lowerModI(MMod* mod); + void lowerNegI(MInstruction* ins, MDefinition* input); + void lowerNegI64(MInstruction* ins, MDefinition* input); + void lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs); + void lowerUDiv(MDiv* div); + void lowerUMod(MMod* mod); + void lowerWasmSelectI(MWasmSelect* select); + void lowerWasmSelectI64(MWasmSelect* select); + + void lowerBigIntLsh(MBigIntLsh* ins); + void lowerBigIntRsh(MBigIntRsh* ins); + + LTableSwitch* newLTableSwitch(const LAllocation& in, + const LDefinition& inputCopy, + MTableSwitch* ins); + LTableSwitchV* newLTableSwitchV(MTableSwitch* ins); + + void lowerPhi(MPhi* phi); +}; + +typedef LIRGeneratorLOONG64 LIRGeneratorSpecific; + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_Lowering_loong64_h */ diff --git a/js/src/jit/loong64/MacroAssembler-loong64-inl.h b/js/src/jit/loong64/MacroAssembler-loong64-inl.h new file mode 100644 index 0000000000..f89b720120 --- /dev/null +++ b/js/src/jit/loong64/MacroAssembler-loong64-inl.h @@ -0,0 +1,1904 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_MacroAssembler_loong64_inl_h +#define jit_loong64_MacroAssembler_loong64_inl_h + +#include "jit/loong64/MacroAssembler-loong64.h" + +namespace js { +namespace jit { + +//{{{ check_macroassembler_style + +void MacroAssembler::move64(Register64 src, Register64 dest) { + movePtr(src.reg, dest.reg); +} + +void MacroAssembler::move64(Imm64 imm, Register64 dest) { + movePtr(ImmWord(imm.value), dest.reg); +} + +void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) { + moveFromDouble(src, dest.reg); +} + +void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) { + moveToDouble(src.reg, dest); +} + +void MacroAssembler::move64To32(Register64 src, Register dest) { + as_slli_w(dest, src.reg, 0); +} + +void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) { + as_bstrpick_d(dest.reg, src, 31, 0); +} + +void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) { + move32To64SignExtend(src, dest); + move8SignExtend(dest.reg, dest.reg); +} + +void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) { + move32To64SignExtend(src, dest); + move16SignExtend(dest.reg, dest.reg); +} + +void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) { + as_slli_w(dest.reg, src, 0); +} + +void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) { + moveFromFloat32(src, dest); +} + +void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) { + moveToFloat32(src, dest); +} + +void MacroAssembler::move8SignExtend(Register src, Register dest) { + as_ext_w_b(dest, src); +} + +void MacroAssembler::move16SignExtend(Register src, Register dest) { + as_ext_w_h(dest, src); +} + +void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) { + as_slli_w(dest, src, 0); +} + +void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) { + as_bstrpick_d(dest, src, 31, 0); +} + +// =============================================================== +// Load instructions + +void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) { + load32(src, dest); +} + +void MacroAssembler::loadAbiReturnAddress(Register dest) { movePtr(ra, dest); } + +// =============================================================== +// Logical instructions + +void MacroAssembler::not32(Register reg) { as_nor(reg, reg, zero); } + +void MacroAssembler::notPtr(Register reg) { as_nor(reg, reg, zero); } + +void MacroAssembler::andPtr(Register src, Register dest) { + as_and(dest, dest, src); +} + +void MacroAssembler::andPtr(Imm32 imm, Register dest) { + ma_and(dest, dest, imm); +} + +void MacroAssembler::and64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, ImmWord(imm.value)); + as_and(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::and64(Register64 src, Register64 dest) { + as_and(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::and64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(*this); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + and64(scratch64, dest); + } else { + and64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::and32(Register src, Register dest) { + as_and(dest, dest, src); +} + +void MacroAssembler::and32(Imm32 imm, Register dest) { + ma_and(dest, dest, imm); +} + +void MacroAssembler::and32(Imm32 imm, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(dest, scratch2); + and32(imm, scratch2); + store32(scratch2, dest); +} + +void MacroAssembler::and32(const Address& src, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(src, scratch2); + as_and(dest, dest, scratch2); +} + +void MacroAssembler::or64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, ImmWord(imm.value)); + as_or(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::or32(Register src, Register dest) { + as_or(dest, dest, src); +} + +void MacroAssembler::or32(Imm32 imm, Register dest) { ma_or(dest, dest, imm); } + +void MacroAssembler::or32(Imm32 imm, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(dest, scratch2); + or32(imm, scratch2); + store32(scratch2, dest); +} + +void MacroAssembler::xor64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, ImmWord(imm.value)); + as_xor(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::orPtr(Register src, Register dest) { + as_or(dest, dest, src); +} + +void MacroAssembler::orPtr(Imm32 imm, Register dest) { ma_or(dest, dest, imm); } + +void MacroAssembler::or64(Register64 src, Register64 dest) { + as_or(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::or64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + or64(scratch64, dest); + } else { + or64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::xor64(Register64 src, Register64 dest) { + as_xor(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::xor64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + xor64(scratch64, dest); + } else { + xor64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::xorPtr(Register src, Register dest) { + as_xor(dest, dest, src); +} + +void MacroAssembler::xorPtr(Imm32 imm, Register dest) { + ma_xor(dest, dest, imm); +} + +void MacroAssembler::xor32(Register src, Register dest) { + as_xor(dest, dest, src); +} + +void MacroAssembler::xor32(Imm32 imm, Register dest) { + ma_xor(dest, dest, imm); +} + +void MacroAssembler::xor32(Imm32 imm, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(dest, scratch2); + xor32(imm, scratch2); + store32(scratch2, dest); +} + +void MacroAssembler::xor32(const Address& src, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(src, scratch2); + xor32(scratch2, dest); +} + +// =============================================================== +// Swap instructions + +void MacroAssembler::byteSwap16SignExtend(Register reg) { + as_revb_2h(reg, reg); + as_ext_w_h(reg, reg); +} + +void MacroAssembler::byteSwap16ZeroExtend(Register reg) { + as_revb_2h(reg, reg); + as_bstrpick_d(reg, reg, 15, 0); +} + +void MacroAssembler::byteSwap32(Register reg) { + as_revb_2w(reg, reg); + as_slli_w(reg, reg, 0); +} + +void MacroAssembler::byteSwap64(Register64 reg64) { + as_revb_d(reg64.reg, reg64.reg); +} + +// =============================================================== +// Arithmetic functions + +void MacroAssembler::addPtr(Register src, Register dest) { + as_add_d(dest, dest, src); +} + +void MacroAssembler::addPtr(Imm32 imm, Register dest) { + ma_add_d(dest, dest, imm); +} + +void MacroAssembler::addPtr(ImmWord imm, Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(imm, scratch); + addPtr(scratch, dest); +} + +void MacroAssembler::add64(Register64 src, Register64 dest) { + addPtr(src.reg, dest.reg); +} + +void MacroAssembler::add64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + add64(scratch64, dest); + } else { + add64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::add64(Imm32 imm, Register64 dest) { + ma_add_d(dest.reg, dest.reg, imm); +} + +void MacroAssembler::add64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dest.reg != scratch); + mov(ImmWord(imm.value), scratch); + as_add_d(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::add32(Register src, Register dest) { + as_add_w(dest, dest, src); +} + +void MacroAssembler::add32(Imm32 imm, Register dest) { + ma_add_w(dest, dest, imm); +} + +void MacroAssembler::add32(Imm32 imm, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(dest, scratch2); + ma_add_w(scratch2, scratch2, imm); + store32(scratch2, dest); +} + +void MacroAssembler::addPtr(Imm32 imm, const Address& dest) { + ScratchRegisterScope scratch(asMasm()); + loadPtr(dest, scratch); + addPtr(imm, scratch); + storePtr(scratch, dest); +} + +void MacroAssembler::addPtr(const Address& src, Register dest) { + ScratchRegisterScope scratch(asMasm()); + loadPtr(src, scratch); + addPtr(scratch, dest); +} + +void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) { + as_fadd_d(dest, dest, src); +} + +void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) { + as_fadd_s(dest, dest, src); +} + +CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) { + CodeOffset offset = CodeOffset(currentOffset()); + MacroAssemblerLOONG64::ma_liPatchable(dest, Imm32(0)); + as_sub_d(dest, StackPointer, dest); + return offset; +} + +void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) { + // TODO: by wangqing + Instruction* inst0 = + (Instruction*)m_buffer.getInst(BufferOffset(offset.offset())); + + InstImm* i0 = (InstImm*)inst0; + InstImm* i1 = (InstImm*)i0->next(); + + MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25)); + MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22)); + + *i0 = InstImm(op_lu12i_w, (int32_t)((imm.value >> 12) & 0xfffff), + Register::FromCode(i0->extractRD()), false); + *i1 = InstImm(op_ori, (int32_t)(imm.value & 0xfff), + Register::FromCode(i1->extractRJ()), + Register::FromCode(i1->extractRD()), 12); +} + +void MacroAssembler::subPtr(Register src, Register dest) { + as_sub_d(dest, dest, src); +} + +void MacroAssembler::subPtr(Imm32 imm, Register dest) { + ma_sub_d(dest, dest, imm); +} + +void MacroAssembler::sub64(Register64 src, Register64 dest) { + as_sub_d(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::sub64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + sub64(scratch64, dest); + } else { + sub64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::sub64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dest.reg != scratch); + mov(ImmWord(imm.value), scratch); + as_sub_d(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::sub32(Register src, Register dest) { + as_sub_w(dest, dest, src); +} + +void MacroAssembler::sub32(Imm32 imm, Register dest) { + ma_sub_w(dest, dest, imm); +} + +void MacroAssembler::sub32(const Address& src, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(src, scratch2); + as_sub_w(dest, dest, scratch2); +} + +void MacroAssembler::subPtr(Register src, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(dest, scratch2); + subPtr(src, scratch2); + storePtr(scratch2, dest); +} + +void MacroAssembler::subPtr(const Address& addr, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(addr, scratch2); + subPtr(scratch2, dest); +} + +void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) { + as_fsub_d(dest, dest, src); +} + +void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) { + as_fsub_s(dest, dest, src); +} + +void MacroAssembler::mul64(Imm64 imm, const Register64& dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dest.reg != scratch); + mov(ImmWord(imm.value), scratch); + as_mul_d(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::mul64(Imm64 imm, const Register64& dest, + const Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + mul64(imm, dest); +} + +void MacroAssembler::mul64(const Register64& src, const Register64& dest, + const Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + as_mul_d(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::mul64(const Operand& src, const Register64& dest, + const Register temp) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + mul64(scratch64, dest, temp); + } else { + mul64(Register64(src.toReg()), dest, temp); + } +} + +void MacroAssembler::mulPtr(Register rhs, Register srcDest) { + as_mul_d(srcDest, srcDest, rhs); +} + +void MacroAssembler::mulBy3(Register src, Register dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + as_add_d(scratch, src, src); + as_add_d(dest, scratch, src); +} + +void MacroAssembler::mul32(Register rhs, Register srcDest) { + as_mul_w(srcDest, srcDest, rhs); +} + +void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) { + as_fmul_s(dest, dest, src); +} + +void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) { + as_fmul_d(dest, dest, src); +} + +void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp, + FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(imm, scratch); + loadDouble(Address(scratch, 0), ScratchDoubleReg); + mulDouble(ScratchDoubleReg, dest); +} + +void MacroAssembler::inc64(AbsoluteAddress dest) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch, ImmWord(uintptr_t(dest.addr))); + as_ld_d(scratch2, scratch, 0); + as_addi_d(scratch2, scratch2, 1); + as_st_d(scratch2, scratch, 0); +} + +void MacroAssembler::quotient32(Register rhs, Register srcDest, + bool isUnsigned) { + if (isUnsigned) { + as_div_wu(srcDest, srcDest, rhs); + } else { + as_div_w(srcDest, srcDest, rhs); + } +} + +void MacroAssembler::remainder32(Register rhs, Register srcDest, + bool isUnsigned) { + if (isUnsigned) { + as_mod_wu(srcDest, srcDest, rhs); + } else { + as_mod_w(srcDest, srcDest, rhs); + } +} + +void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) { + as_fdiv_s(dest, dest, src); +} + +void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) { + as_fdiv_d(dest, dest, src); +} + +void MacroAssembler::neg64(Register64 reg) { as_sub_d(reg.reg, zero, reg.reg); } + +void MacroAssembler::negPtr(Register reg) { as_sub_d(reg, zero, reg); } + +void MacroAssembler::neg32(Register reg) { as_sub_w(reg, zero, reg); } + +void MacroAssembler::negateDouble(FloatRegister reg) { as_fneg_d(reg, reg); } + +void MacroAssembler::negateFloat(FloatRegister reg) { as_fneg_s(reg, reg); } + +void MacroAssembler::abs32(Register src, Register dest) { + // TODO:LOONG64 There's probably a better way to do this. + if (src != dest) { + move32(src, dest); + } + Label positive; + branchTest32(Assembler::NotSigned, dest, dest, &positive); + neg32(dest); + bind(&positive); +} + +void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) { + as_fabs_s(dest, src); +} + +void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) { + as_fabs_d(dest, src); +} + +void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) { + as_fsqrt_s(dest, src); +} + +void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) { + as_fsqrt_d(dest, src); +} + +void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + minMaxFloat32(srcDest, other, handleNaN, false); +} + +void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + minMaxDouble(srcDest, other, handleNaN, false); +} + +void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + minMaxFloat32(srcDest, other, handleNaN, true); +} + +void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + minMaxDouble(srcDest, other, handleNaN, true); +} + +// =============================================================== +// Shift functions + +void MacroAssembler::lshift32(Register src, Register dest) { + as_sll_w(dest, dest, src); +} + +void MacroAssembler::lshift32(Imm32 imm, Register dest) { + as_slli_w(dest, dest, imm.value % 32); +} + +void MacroAssembler::flexibleLshift32(Register src, Register dest) { + lshift32(src, dest); +} + +void MacroAssembler::lshift64(Register shift, Register64 dest) { + as_sll_d(dest.reg, dest.reg, shift); +} + +void MacroAssembler::lshift64(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_slli_d(dest.reg, dest.reg, imm.value); +} + +void MacroAssembler::lshiftPtr(Register shift, Register dest) { + as_sll_d(dest, dest, shift); +} + +void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_slli_d(dest, dest, imm.value); +} + +void MacroAssembler::rshift32(Register src, Register dest) { + as_srl_w(dest, dest, src); +} + +void MacroAssembler::rshift32(Imm32 imm, Register dest) { + as_srli_w(dest, dest, imm.value % 32); +} + +void MacroAssembler::flexibleRshift32(Register src, Register dest) { + rshift32(src, dest); +} + +void MacroAssembler::rshift32Arithmetic(Register src, Register dest) { + as_sra_w(dest, dest, src); +} + +void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) { + as_srai_w(dest, dest, imm.value % 32); +} + +void MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest) { + rshift32Arithmetic(src, dest); +} + +void MacroAssembler::rshift64(Register shift, Register64 dest) { + as_srl_d(dest.reg, dest.reg, shift); +} + +void MacroAssembler::rshift64(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_srli_d(dest.reg, dest.reg, imm.value); +} + +void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_srai_d(dest.reg, dest.reg, imm.value); +} + +void MacroAssembler::rshift64Arithmetic(Register shift, Register64 dest) { + as_sra_d(dest.reg, dest.reg, shift); +} + +void MacroAssembler::rshiftPtr(Register shift, Register dest) { + as_srl_d(dest, dest, shift); +} + +void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_srli_d(dest, dest, imm.value); +} + +void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_srai_d(dest, dest, imm.value); +} + +// =============================================================== +// Rotation functions + +void MacroAssembler::rotateLeft(Register count, Register input, Register dest) { + ScratchRegisterScope scratch(asMasm()); + as_sub_w(scratch, zero, count); + as_rotr_w(dest, input, scratch); +} + +void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) { + as_rotri_w(dest, input, (32 - count.value) & 31); +} + +void MacroAssembler::rotateLeft64(Register count, Register64 src, + Register64 dest, Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + ScratchRegisterScope scratch(asMasm()); + as_sub_d(scratch, zero, count); + as_rotr_d(dest.reg, src.reg, scratch); +} + +void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest, + Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + as_rotri_d(dest.reg, src.reg, (64 - count.value) & 63); +} + +void MacroAssembler::rotateRight(Register count, Register input, + Register dest) { + as_rotr_w(dest, input, count); +} + +void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) { + as_rotri_w(dest, input, count.value & 31); +} + +void MacroAssembler::rotateRight64(Register count, Register64 src, + Register64 dest, Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + as_rotr_d(dest.reg, src.reg, count); +} + +void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest, + Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + as_rotri_d(dest.reg, src.reg, count.value & 63); +} + +// Bit counting functions + +void MacroAssembler::clz64(Register64 src, Register dest) { + as_clz_d(dest, src.reg); +} + +void MacroAssembler::ctz64(Register64 src, Register dest) { + as_ctz_d(dest, src.reg); +} + +void MacroAssembler::popcnt64(Register64 input, Register64 output, + Register tmp) { + ScratchRegisterScope scratch(asMasm()); + as_or(output.reg, input.reg, zero); + as_srai_d(tmp, input.reg, 1); + ma_li(scratch, Imm32(0x55555555)); + as_bstrins_d(scratch, scratch, 63, 32); + as_and(tmp, tmp, scratch); + as_sub_d(output.reg, output.reg, tmp); + as_srai_d(tmp, output.reg, 2); + ma_li(scratch, Imm32(0x33333333)); + as_bstrins_d(scratch, scratch, 63, 32); + as_and(output.reg, output.reg, scratch); + as_and(tmp, tmp, scratch); + as_add_d(output.reg, output.reg, tmp); + as_srli_d(tmp, output.reg, 4); + as_add_d(output.reg, output.reg, tmp); + ma_li(scratch, Imm32(0xF0F0F0F)); + as_bstrins_d(scratch, scratch, 63, 32); + as_and(output.reg, output.reg, scratch); + ma_li(tmp, Imm32(0x1010101)); + as_bstrins_d(tmp, tmp, 63, 32); + as_mul_d(output.reg, output.reg, tmp); + as_srai_d(output.reg, output.reg, 56); +} + +void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) { + as_clz_w(dest, src); +} + +void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) { + as_ctz_w(dest, src); +} + +void MacroAssembler::popcnt32(Register input, Register output, Register tmp) { + // Equivalent to GCC output of mozilla::CountPopulation32() + as_or(output, input, zero); + as_srai_w(tmp, input, 1); + ma_and(tmp, tmp, Imm32(0x55555555)); + as_sub_w(output, output, tmp); + as_srai_w(tmp, output, 2); + ma_and(output, output, Imm32(0x33333333)); + ma_and(tmp, tmp, Imm32(0x33333333)); + as_add_w(output, output, tmp); + as_srli_w(tmp, output, 4); + as_add_w(output, output, tmp); + ma_and(output, output, Imm32(0xF0F0F0F)); + as_slli_w(tmp, output, 8); + as_add_w(output, output, tmp); + as_slli_w(tmp, output, 16); + as_add_w(output, output, tmp); + as_srai_w(output, output, 24); +} + +// =============================================================== +// Condition functions + +// Also see below for specializations of cmpPtrSet. +template +void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) { + ma_cmp_set(dest, lhs, rhs, cond); +} + +template +void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) { + ma_cmp_set(dest, lhs, rhs, cond); +} + +// =============================================================== +// =============================================================== +// Branch functions + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val, + Label* success, Label* fail) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal || + cond == Assembler::LessThan || + cond == Assembler::LessThanOrEqual || + cond == Assembler::GreaterThan || + cond == Assembler::GreaterThanOrEqual || + cond == Assembler::Below || cond == Assembler::BelowOrEqual || + cond == Assembler::Above || cond == Assembler::AboveOrEqual, + "other condition codes not supported"); + + branchPtr(cond, lhs.reg, ImmWord(val.value), success); + if (fail) { + jump(fail); + } +} + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs, + Label* success, Label* fail) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal || + cond == Assembler::LessThan || + cond == Assembler::LessThanOrEqual || + cond == Assembler::GreaterThan || + cond == Assembler::GreaterThanOrEqual || + cond == Assembler::Below || cond == Assembler::BelowOrEqual || + cond == Assembler::Above || cond == Assembler::AboveOrEqual, + "other condition codes not supported"); + + branchPtr(cond, lhs.reg, rhs.reg, success); + if (fail) { + jump(fail); + } +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val, + Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual, + "other condition codes not supported"); + + branchPtr(cond, lhs, ImmWord(val.value), label); +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, + const Address& rhs, Register scratch, + Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual, + "other condition codes not supported"); + MOZ_ASSERT(lhs.base != scratch); + MOZ_ASSERT(rhs.base != scratch); + + loadPtr(rhs, scratch); + branchPtr(cond, lhs, scratch, label); +} + +template +void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs, + L label) { + ma_b(lhs, rhs, label, cond); +} + +template +void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm, + L label) { + ma_b(lhs, imm, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress addr, + Imm32 imm, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(addr, scratch2); + ma_b(scratch2, imm, label, cond); +} + +template +void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs, + L label) { + ma_b(lhs, rhs, label, cond); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + ma_b(lhs, rhs, label, cond); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs, + Label* label) { + ma_b(lhs, rhs, label, cond); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs, + Label* label) { + ma_b(lhs, rhs, label, cond); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs, + Label* label) { + ma_b(lhs, rhs, label, cond); +} + +template +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs, + L label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + ImmWord rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs, + Register rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs, + Register rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs, + ImmWord rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs, + Register rhs, Label* label) { + branchPtr(cond, lhs, rhs, label); +} + +void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs, + FloatRegister rhs, Label* label) { + ma_bc1s(lhs, rhs, label, cond); +} + +void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + ScratchRegisterScope scratch(asMasm()); + as_ftintrz_l_s(ScratchDoubleReg, src); + as_movfcsr2gr(scratch); + moveFromDouble(ScratchDoubleReg, dest); + MOZ_ASSERT(Assembler::CauseV < 32); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, Imm32(0), fail, Assembler::NotEqual); + + as_slli_w(dest, dest, 0); +} + +void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src, + Register dest, Label* fail) { + convertFloat32ToInt32(src, dest, fail, false); +} + +void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs, + FloatRegister rhs, Label* label) { + ma_bc1d(lhs, rhs, label, cond); +} + +void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + ScratchRegisterScope scratch(asMasm()); + as_ftintrz_l_d(ScratchDoubleReg, src); + as_movfcsr2gr(scratch); + moveFromDouble(ScratchDoubleReg, dest); + MOZ_ASSERT(Assembler::CauseV < 32); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, Imm32(0), fail, Assembler::NotEqual); + + as_slli_w(dest, dest, 0); +} + +void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src, + Register dest, Label* fail) { + convertDoubleToInt32(src, dest, fail, false); +} + +template +void MacroAssembler::branchAdd32(Condition cond, T src, Register dest, + Label* overflow) { + switch (cond) { + case Overflow: + ma_add32TestOverflow(dest, dest, src, overflow); + break; + case CarryClear: + case CarrySet: + ma_add32TestCarry(cond, dest, dest, src, overflow); + break; + default: + MOZ_CRASH("NYI"); + } +} + +// the type of 'T src' maybe a Register, maybe a Imm32,depends on who call it. +template +void MacroAssembler::branchSub32(Condition cond, T src, Register dest, + Label* overflow) { + switch (cond) { + case Overflow: + ma_sub32TestOverflow(dest, dest, src, overflow); + break; + case NonZero: + case Zero: + case Signed: + case NotSigned: + ma_sub_w(dest, dest, src); + ma_b(dest, dest, overflow, cond); + break; + default: + MOZ_CRASH("NYI"); + } +} + +template +void MacroAssembler::branchMul32(Condition cond, T src, Register dest, + Label* overflow) { + MOZ_ASSERT(cond == Assembler::Overflow); + ma_mul32TestOverflow(dest, dest, src, overflow); +} + +template +void MacroAssembler::branchRshift32(Condition cond, T src, Register dest, + Label* label) { + MOZ_ASSERT(cond == Zero || cond == NonZero); + rshift32(src, dest); + branch32(cond == Zero ? Equal : NotEqual, dest, Imm32(0), label); +} + +void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) { + MOZ_ASSERT(cond == Overflow); + neg32(reg); + branch32(Assembler::Equal, reg, Imm32(INT32_MIN), label); +} + +template +void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest, + Label* label) { + switch (cond) { + case Overflow: + ma_addPtrTestOverflow(dest, dest, src, label); + break; + case CarryClear: + case CarrySet: + ma_addPtrTestCarry(cond, dest, dest, src, label); + break; + default: + MOZ_CRASH("NYI"); + } +} + +template +void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest, + Label* label) { + switch (cond) { + case Overflow: + ma_subPtrTestOverflow(dest, dest, src, label); + break; + case NonZero: + case Zero: + case Signed: + case NotSigned: + subPtr(src, dest); + ma_b(dest, dest, label, cond); + break; + default: + MOZ_CRASH("NYI"); + } +} + +void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest, + Label* label) { + MOZ_ASSERT(cond == Assembler::Overflow); + ma_mulPtrTestOverflow(dest, dest, src, label); +} + +void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + subPtr(rhs, lhs); + branchPtr(cond, lhs, Imm32(0), label); +} + +template +void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs, + L label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + if (lhs == rhs) { + ma_b(lhs, rhs, label, cond); + } else { + ScratchRegisterScope scratch(asMasm()); + as_and(scratch, lhs, rhs); + ma_b(scratch, scratch, label, cond); + } +} + +template +void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs, + L label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + SecondScratchRegisterScope scratch2(asMasm()); + ma_and(scratch2, lhs, rhs); + ma_b(scratch2, scratch2, label, cond); +} + +void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs, + Label* label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + and32(rhs, scratch2); + ma_b(scratch2, scratch2, label, cond); +} + +void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + and32(rhs, scratch2); + ma_b(scratch2, scratch2, label, cond); +} + +template +void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs, + L label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + if (lhs == rhs) { + ma_b(lhs, rhs, label, cond); + } else { + ScratchRegisterScope scratch(asMasm()); + as_and(scratch, lhs, rhs); + ma_b(scratch, scratch, label, cond); + } +} + +void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + ScratchRegisterScope scratch(asMasm()); + ma_and(scratch, lhs, rhs); + ma_b(scratch, scratch, label, cond); +} + +void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs, + Imm32 rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchTestPtr(cond, scratch2, rhs, label); +} + +template +void MacroAssembler::branchTest64(Condition cond, Register64 lhs, + Register64 rhs, Register temp, L label) { + branchTestPtr(cond, lhs.reg, rhs.reg, label); +} + +void MacroAssembler::branchTestUndefined(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_UNDEFINED), label, cond); +} + +void MacroAssembler::branchTestUndefined(Condition cond, + const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestUndefined(cond, scratch2, label); +} + +void MacroAssembler::branchTestUndefined(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestUndefined(cond, tag, label); +} + +void MacroAssembler::branchTestUndefined(Condition cond, + const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestUndefined(cond, tag, label); +} + +void MacroAssembler::branchTestInt32(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_INT32), label, cond); +} + +void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestInt32(cond, scratch2, label); +} + +void MacroAssembler::branchTestInt32(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestInt32(cond, tag, label); +} + +void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestInt32(cond, tag, label); +} + +void MacroAssembler::branchTestInt32Truthy(bool b, const ValueOperand& value, + Label* label) { + ScratchRegisterScope scratch(*this); + as_bstrpick_d(scratch, value.valueReg(), 31, 0); + ma_b(scratch, scratch, label, b ? NonZero : Zero); +} + +void MacroAssembler::branchTestDouble(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + Condition actual = (cond == Equal) ? BelowOrEqual : Above; + ma_b(tag, ImmTag(JSVAL_TAG_MAX_DOUBLE), label, actual); +} + +void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestDouble(cond, scratch2, label); +} + +void MacroAssembler::branchTestDouble(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestDouble(cond, tag, label); +} + +void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestDouble(cond, tag, label); +} + +void MacroAssembler::branchTestDoubleTruthy(bool b, FloatRegister value, + Label* label) { + ma_lid(ScratchDoubleReg, 0.0); + DoubleCondition cond = b ? DoubleNotEqual : DoubleEqualOrUnordered; + ma_bc1d(value, ScratchDoubleReg, label, cond); +} + +void MacroAssembler::branchTestNumber(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + Condition actual = cond == Equal ? BelowOrEqual : Above; + ma_b(tag, ImmTag(JS::detail::ValueUpperInclNumberTag), label, actual); +} + +void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestNumber(cond, scratch2, label); +} + +void MacroAssembler::branchTestBoolean(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_BOOLEAN), label, cond); +} + +void MacroAssembler::branchTestBoolean(Condition cond, + const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestBoolean(cond, scratch2, label); +} + +void MacroAssembler::branchTestBoolean(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestBoolean(cond, tag, label); +} + +void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestBoolean(cond, tag, label); +} + +void MacroAssembler::branchTestBooleanTruthy(bool b, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + unboxBoolean(value, scratch2); + ma_b(scratch2, scratch2, label, b ? NonZero : Zero); +} + +void MacroAssembler::branchTestString(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_STRING), label, cond); +} + +void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestString(cond, scratch2, label); +} + +void MacroAssembler::branchTestString(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestString(cond, tag, label); +} + +void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestString(cond, tag, label); +} + +void MacroAssembler::branchTestStringTruthy(bool b, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + unboxString(value, scratch2); + load32(Address(scratch2, JSString::offsetOfLength()), scratch2); + ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal); +} + +void MacroAssembler::branchTestSymbol(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_SYMBOL), label, cond); +} + +void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestSymbol(cond, scratch2, label); +} + +void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestSymbol(cond, tag, label); +} + +void MacroAssembler::branchTestSymbol(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestSymbol(cond, tag, label); +} + +void MacroAssembler::branchTestBigInt(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_BIGINT), label, cond); +} + +void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestBigInt(cond, scratch2, label); +} + +void MacroAssembler::branchTestBigInt(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestBigInt(cond, tag, label); +} + +void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + // TODO:LOONG64 branchTestBigInt + breakpoint(); + // computeEffectiveAddress(address, scratch2); + splitTag(scratch2, scratch2); + branchTestBigInt(cond, scratch2, label); +} + +void MacroAssembler::branchTestBigIntTruthy(bool b, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + unboxBigInt(value, scratch2); + load32(Address(scratch2, BigInt::offsetOfDigitLength()), scratch2); + ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal); +} + +void MacroAssembler::branchTestNull(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_NULL), label, cond); +} + +void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestNull(cond, scratch2, label); +} + +void MacroAssembler::branchTestNull(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestNull(cond, tag, label); +} + +void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestNull(cond, tag, label); +} + +void MacroAssembler::branchTestObject(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_OBJECT), label, cond); +} + +void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestObject(cond, scratch2, label); +} + +void MacroAssembler::branchTestObject(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestObject(cond, tag, label); +} + +void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestObject(cond, tag, label); +} + +void MacroAssembler::branchTestPrimitive(Condition cond, + const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestPrimitive(cond, scratch2, label); +} + +void MacroAssembler::branchTestGCThing(Condition cond, const Address& address, + Label* label) { + branchTestGCThingImpl(cond, address, label); +} + +void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address, + Label* label) { + branchTestGCThingImpl(cond, address, label); +} + +void MacroAssembler::branchTestGCThing(Condition cond, + const ValueOperand& address, + Label* label) { + branchTestGCThingImpl(cond, address, label); +} + +template +void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& address, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + ma_b(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag), label, + (cond == Equal) ? AboveOrEqual : Below); +} + +void MacroAssembler::branchTestPrimitive(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag), label, + (cond == Equal) ? Below : AboveOrEqual); +} + +void MacroAssembler::branchTestMagic(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_MAGIC), label, cond); +} + +void MacroAssembler::branchTestMagic(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestMagic(cond, tag, label); +} + +void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestMagic(cond, tag, label); +} + +template +void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value, + L label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + ma_b(scratch2, ImmTag(JSVAL_TAG_MAGIC), label, cond); +} + +void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, + JSWhyMagic why, Label* label) { + uint64_t magic = MagicValue(why).asRawBits(); + SecondScratchRegisterScope scratch(*this); + loadPtr(valaddr, scratch); + ma_b(scratch, ImmWord(magic), label, cond); +} + +void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs, + const ValueOperand& rhs, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + branchPtr(cond, lhs, rhs.valueReg(), label); +} + +void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(addr, scratch2); + branch(scratch2); +} + +void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs, + Register src, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + cmp32Set(cond, lhs, rhs, scratch2); + moveIfNotZero(dest, src, scratch2); +} + +void MacroAssembler::cmp32Move32(Condition cond, Register lhs, + const Address& rhs, Register src, + Register dest) { + SecondScratchRegisterScope scratch2(*this); + MOZ_ASSERT(lhs != scratch2 && src != scratch2 && dest != scratch2); + load32(rhs, scratch2); + cmp32Move32(cond, lhs, scratch2, src, dest); +} + +void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs, + Register src, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + cmp32Set(cond, lhs, rhs, scratch2); + moveIfNotZero(dest, src, scratch2); +} + +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs, + Register src, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + cmpPtrSet(cond, lhs, rhs, scratch2); + moveIfNotZero(dest, src, scratch2); +} + +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, + const Address& rhs, Register src, + Register dest) { + MOZ_CRASH("NYI"); +} + +void MacroAssembler::cmp32Load32(Condition cond, Register lhs, + const Address& rhs, const Address& src, + Register dest) { + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(lhs != scratch && dest != scratch); + load32(rhs, scratch); + cmp32Load32(cond, lhs, scratch, src, dest); +} + +void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs, + const Address& src, Register dest) { + Label skip; + branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip); + load32(src, dest); + bind(&skip); +} + +void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs, + const Address& src, Register dest) { + Label skip; + branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip); + loadPtr(src, dest); + bind(&skip); +} + +void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr, + Imm32 mask, const Address& src, + Register dest) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreStringMitigations); + Label skip; + branchTest32(Assembler::InvertCondition(cond), addr, mask, &skip); + loadPtr(src, dest); + bind(&skip); +} + +void MacroAssembler::test32MovePtr(Condition cond, const Address& addr, + Imm32 mask, Register src, Register dest) { + MOZ_CRASH(); +} + +void MacroAssembler::spectreBoundsCheck32(Register index, Register length, + Register maybeScratch, + Label* failure) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking); + branch32(Assembler::BelowOrEqual, length, index, failure); +} + +void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length, + Register maybeScratch, + Label* failure) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking); + branch32(Assembler::BelowOrEqual, length, index, failure); +} + +void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length, + Register maybeScratch, + Label* failure) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking); + branchPtr(Assembler::BelowOrEqual, length, index, failure); +} + +void MacroAssembler::spectreBoundsCheckPtr(Register index, + const Address& length, + Register maybeScratch, + Label* failure) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking); + branchPtr(Assembler::BelowOrEqual, length, index, failure); +} + +void MacroAssembler::spectreMovePtr(Condition cond, Register src, + Register dest) { + MOZ_CRASH(); +} + +void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch, + Register dest) { + MOZ_CRASH(); +} + +// ======================================================================== +// Memory access primitives. + +void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src, + const Address& addr) { + ma_fst_s(src, addr); +} +void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src, + const BaseIndex& addr) { + ma_fst_s(src, addr); +} + +void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src, + const Address& addr) { + ma_fst_d(src, addr); +} +void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src, + const BaseIndex& addr) { + ma_fst_d(src, addr); +} + +void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) { + if (barrier) { + as_dbar(0); + } +} + +// =============================================================== +// Clamping functions. + +void MacroAssembler::clampIntToUint8(Register reg) { + ScratchRegisterScope scratch(*this); + // If reg is < 0, then we want to clamp to 0. + as_slti(scratch, reg, 0); + as_masknez(reg, reg, scratch); + + // If reg is >= 255, then we want to clamp to 255. + as_addi_d(reg, reg, -255); + as_slt(scratch, reg, zero); + as_maskeqz(reg, reg, scratch); + as_addi_d(reg, reg, 255); +} + +void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest, + JSValueType type, Label* fail) { + MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING || + type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT); + // dest := src XOR mask + // scratch := dest >> JSVAL_TAG_SHIFT + // fail if scratch != 0 + // + // Note: src and dest can be the same register + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src.valueReg() != scratch); + mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch); + as_xor(dest, src.valueReg(), scratch); + as_srli_d(scratch, dest, JSVAL_TAG_SHIFT); + ma_b(scratch, Imm32(0), fail, Assembler::NotEqual); +} + +void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest, + JSValueType type, Label* fail) { + loadValue(src, ValueOperand(dest)); + fallibleUnboxPtr(ValueOperand(dest), dest, type, fail); +} + +void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest, + JSValueType type, Label* fail) { + loadValue(src, ValueOperand(dest)); + fallibleUnboxPtr(ValueOperand(dest), dest, type, fail); +} + +//}}} check_macroassembler_style +// =============================================================== + +// The specializations for cmpPtrSet are outside the braces because +// check_macroassembler_style can't yet deal with specializations. + +template <> +inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs, + ImmPtr rhs, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + cmpPtrSet(cond, Register(scratch2), rhs, dest); +} + +template <> +inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Register lhs, + Address rhs, Register dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + loadPtr(rhs, scratch); + cmpPtrSet(cond, lhs, Register(scratch), dest); +} + +template <> +inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs, + Register rhs, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rhs != scratch2); + loadPtr(lhs, scratch2); + cmpPtrSet(cond, Register(scratch2), rhs, dest); +} + +template <> +inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Register lhs, + Address rhs, Register dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + load32(rhs, scratch); + cmp32Set(cond, lhs, Register(scratch), dest); +} + +template <> +inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Address lhs, + Register rhs, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rhs != scratch2); + load32(lhs, scratch2); + cmp32Set(cond, Register(scratch2), rhs, dest); +} + +void MacroAssemblerLOONG64Compat::incrementInt32Value(const Address& addr) { + asMasm().add32(Imm32(1), addr); +} + +void MacroAssemblerLOONG64Compat::retn(Imm32 n) { + // pc <- [sp]; sp += n + loadPtr(Address(StackPointer, 0), ra); + asMasm().addPtr(n, StackPointer); + as_jirl(zero, ra, BOffImm16(0)); +} + +// If source is a double, load into dest. +// If source is int32, convert to double and store in dest. +// Else, branch to failure. +void MacroAssemblerLOONG64Compat::ensureDouble(const ValueOperand& source, + FloatRegister dest, + Label* failure) { + Label isDouble, done; + { + ScratchTagScope tag(asMasm(), source); + splitTagForTest(source, tag); + asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble); + asMasm().branchTestInt32(Assembler::NotEqual, tag, failure); + } + + convertInt32ToDouble(source.valueReg(), dest); + jump(&done); + + bind(&isDouble); + unboxDouble(source, dest); + + bind(&done); +} + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_MacroAssembler_loong64_inl_h */ diff --git a/js/src/jit/loong64/MacroAssembler-loong64.cpp b/js/src/jit/loong64/MacroAssembler-loong64.cpp new file mode 100644 index 0000000000..37452b1276 --- /dev/null +++ b/js/src/jit/loong64/MacroAssembler-loong64.cpp @@ -0,0 +1,5335 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loong64/MacroAssembler-loong64.h" + +#include "jsmath.h" + +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/loong64/SharedICRegisters-loong64.h" +#include "jit/MacroAssembler.h" +#include "jit/MoveEmitter.h" +#include "util/Memory.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/JSContext.h" + +#include "jit/MacroAssembler-inl.h" + +namespace js { +namespace jit { + +void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) { + ScratchRegisterScope scratch(asMasm()); + as_ftintrne_l_d(ScratchDoubleReg, input); + as_movfr2gr_d(output, ScratchDoubleReg); + // if (res < 0); res = 0; + as_slt(scratch, output, zero); + as_masknez(output, output, scratch); + // if res > 255; res = 255; + as_sltui(scratch, output, 255); + as_addi_d(output, output, -255); + as_maskeqz(output, output, scratch); + as_addi_d(output, output, 255); +} + +bool MacroAssemblerLOONG64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr) { + uint32_t descriptor = MakeFrameDescriptor( + asMasm().framePushed(), FrameType::IonJS, ExitFrameLayout::Size()); + + asMasm().Push(Imm32(descriptor)); // descriptor_ + asMasm().Push(ImmPtr(fakeReturnAddr)); + + return true; +} + +void MacroAssemblerLOONG64Compat::convertUInt32ToDouble(Register src, + FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + as_bstrpick_d(scratch, src, 31, 0); + asMasm().convertInt64ToDouble(Register64(scratch), dest); +} + +void MacroAssemblerLOONG64Compat::convertUInt64ToDouble(Register src, + FloatRegister dest) { + Label positive, done; + ma_b(src, src, &positive, NotSigned, ShortJump); + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(src != scratch2); + + ma_and(scratch, src, Imm32(1)); + as_srli_d(scratch2, src, 1); + as_or(scratch, scratch, scratch2); + as_movgr2fr_d(dest, scratch); + as_ffint_d_l(dest, dest); + asMasm().addDouble(dest, dest); + ma_b(&done, ShortJump); + + bind(&positive); + as_movgr2fr_d(dest, src); + as_ffint_d_l(dest, dest); + + bind(&done); +} + +void MacroAssemblerLOONG64Compat::convertUInt32ToFloat32(Register src, + FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + as_bstrpick_d(scratch, src, 31, 0); + asMasm().convertInt64ToFloat32(Register64(scratch), dest); +} + +void MacroAssemblerLOONG64Compat::convertDoubleToFloat32(FloatRegister src, + FloatRegister dest) { + as_fcvt_s_d(dest, src); +} + +const int CauseBitPos = int(Assembler::CauseI); +const int CauseBitCount = 1 + int(Assembler::CauseV) - int(Assembler::CauseI); +const int CauseIOrVMask = ((1 << int(Assembler::CauseI)) | + (1 << int(Assembler::CauseV))) >> + int(Assembler::CauseI); + +// Checks whether a double is representable as a 32-bit integer. If so, the +// integer is written to the output register. Otherwise, a bailout is taken to +// the given snapshot. This function overwrites the scratch float register. +void MacroAssemblerLOONG64Compat::convertDoubleToInt32(FloatRegister src, + Register dest, + Label* fail, + bool negativeZeroCheck) { + if (negativeZeroCheck) { + moveFromDouble(src, dest); + as_rotri_d(dest, dest, 63); + ma_b(dest, Imm32(1), fail, Assembler::Equal); + } + + ScratchRegisterScope scratch(asMasm()); + // Truncate double to int ; if result is inexact or invalid fail. + as_ftintrz_w_d(ScratchFloat32Reg, src); + as_movfcsr2gr(scratch); + moveFromFloat32(ScratchFloat32Reg, dest); + as_bstrpick_d(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); + as_andi(scratch, scratch, + CauseIOrVMask); // masking for Inexact and Invalid flag. + ma_b(scratch, zero, fail, Assembler::NotEqual); +} + +void MacroAssemblerLOONG64Compat::convertDoubleToPtr(FloatRegister src, + Register dest, Label* fail, + bool negativeZeroCheck) { + if (negativeZeroCheck) { + moveFromDouble(src, dest); + as_rotri_d(dest, dest, 63); + ma_b(dest, Imm32(1), fail, Assembler::Equal); + } + + ScratchRegisterScope scratch(asMasm()); + // Truncate double to int64 ; if result is inexact or invalid fail. + as_ftintrz_l_d(ScratchDoubleReg, src); + as_movfcsr2gr(scratch); + moveFromDouble(ScratchDoubleReg, dest); + as_bstrpick_d(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); + as_andi(scratch, scratch, + CauseIOrVMask); // masking for Inexact and Invalid flag. + ma_b(scratch, zero, fail, Assembler::NotEqual); +} + +// Checks whether a float32 is representable as a 32-bit integer. If so, the +// integer is written to the output register. Otherwise, a bailout is taken to +// the given snapshot. This function overwrites the scratch float register. +void MacroAssemblerLOONG64Compat::convertFloat32ToInt32( + FloatRegister src, Register dest, Label* fail, bool negativeZeroCheck) { + if (negativeZeroCheck) { + moveFromFloat32(src, dest); + ma_b(dest, Imm32(INT32_MIN), fail, Assembler::Equal); + } + + ScratchRegisterScope scratch(asMasm()); + as_ftintrz_w_s(ScratchFloat32Reg, src); + as_movfcsr2gr(scratch); + moveFromFloat32(ScratchFloat32Reg, dest); + MOZ_ASSERT(CauseBitPos + CauseBitCount < 33); + MOZ_ASSERT(CauseBitPos < 32); + as_bstrpick_w(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); + as_andi(scratch, scratch, CauseIOrVMask); + ma_b(scratch, zero, fail, Assembler::NotEqual); +} + +void MacroAssemblerLOONG64Compat::convertFloat32ToDouble(FloatRegister src, + FloatRegister dest) { + as_fcvt_d_s(dest, src); +} + +void MacroAssemblerLOONG64Compat::convertInt32ToFloat32(Register src, + FloatRegister dest) { + as_movgr2fr_w(dest, src); + as_ffint_s_w(dest, dest); +} + +void MacroAssemblerLOONG64Compat::convertInt32ToFloat32(const Address& src, + FloatRegister dest) { + ma_fld_s(dest, src); + as_ffint_s_w(dest, dest); +} + +void MacroAssemblerLOONG64Compat::movq(Register rj, Register rd) { + as_or(rd, rj, zero); +} + +void MacroAssemblerLOONG64::ma_li(Register dest, CodeLabel* label) { + BufferOffset bo = m_buffer.nextOffset(); + ma_liPatchable(dest, ImmWord(/* placeholder */ 0)); + label->patchAt()->bind(bo.getOffset()); + label->setLinkMode(CodeLabel::MoveImmediate); +} + +void MacroAssemblerLOONG64::ma_li(Register dest, ImmWord imm) { + int64_t value = imm.value; + + if (-1 == (value >> 11) || 0 == (value >> 11)) { + as_addi_w(dest, zero, value); + return; + } + + if (0 == (value >> 12)) { + as_ori(dest, zero, value); + return; + } + + if (-1 == (value >> 31) || 0 == (value >> 31)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + } else if (0 == (value >> 32)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + as_bstrins_d(dest, zero, 63, 32); + } else if (-1 == (value >> 51) || 0 == (value >> 51)) { + if (is_uintN((value >> 12) & 0xfffff, 20)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + } + as_lu32i_d(dest, (value >> 32) & 0xfffff); + } else if (0 == (value >> 52)) { + if (is_uintN((value >> 12) & 0xfffff, 20)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + } + as_lu32i_d(dest, (value >> 32) & 0xfffff); + as_bstrins_d(dest, zero, 63, 52); + } else { + if (is_uintN((value >> 12) & 0xfffff, 20)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + } + if (is_uintN((value >> 32) & 0xfffff, 20)) { + as_lu32i_d(dest, (value >> 32) & 0xfffff); + } + as_lu52i_d(dest, dest, (value >> 52) & 0xfff); + } + + if (is_uintN(value & 0xfff, 12)) { + as_ori(dest, dest, value & 0xfff); + } +} + +// This method generates lu32i_d, lu12i_w and ori instruction block that can be +// modified by UpdateLoad64Value, either during compilation (eg. +// Assembler::bind), or during execution (eg. jit::PatchJump). +void MacroAssemblerLOONG64::ma_liPatchable(Register dest, ImmPtr imm) { + return ma_liPatchable(dest, ImmWord(uintptr_t(imm.value))); +} + +void MacroAssemblerLOONG64::ma_liPatchable(Register dest, ImmWord imm, + LiFlags flags) { + // hi12, hi20, low20, low12 + if (Li64 == flags) { // Li64: Imm data + m_buffer.ensureSpace(4 * sizeof(uint32_t)); + as_lu12i_w(dest, imm.value >> 12 & 0xfffff); // low20 + as_ori(dest, dest, imm.value & 0xfff); // low12 + as_lu32i_d(dest, imm.value >> 32 & 0xfffff); // hi20 + as_lu52i_d(dest, dest, imm.value >> 52 & 0xfff); // hi12 + } else { // Li48 address + m_buffer.ensureSpace(3 * sizeof(uint32_t)); + as_lu12i_w(dest, imm.value >> 12 & 0xfffff); // low20 + as_ori(dest, dest, imm.value & 0xfff); // low12 + as_lu32i_d(dest, imm.value >> 32 & 0xfffff); // hi20 + } +} + +// Memory access ops. + +void MacroAssemblerLOONG64::ma_ld_b(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_b(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_b(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_b(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_ld_bu(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_bu(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_bu(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_bu(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_ld_h(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_h(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_h(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_h(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_ld_hu(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_hu(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_hu(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_hu(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_ld_w(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_w(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_w(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_w(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_ld_wu(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_wu(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_wu(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_wu(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_ld_d(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_d(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_d(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_d(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_st_b(Register src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_st_b(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_stx_b(src, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_st_h(Register src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_st_h(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_stx_h(src, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_st_w(Register src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_st_w(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_stx_w(src, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_st_d(Register src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_st_d(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_stx_d(src, base, scratch); + } +} + +// Arithmetic-based ops. + +// Add. +void MacroAssemblerLOONG64::ma_add_d(Register rd, Register rj, Imm32 imm) { + if (is_intN(imm.value, 12)) { + as_addi_d(rd, rj, imm.value); + } else if (rd != rj) { + ma_li(rd, imm); + as_add_d(rd, rj, rd); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_add_d(rd, rj, scratch); + } +} + +void MacroAssemblerLOONG64::ma_add32TestOverflow(Register rd, Register rj, + Register rk, Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + as_add_d(scratch, rj, rk); + as_add_w(rd, rj, rk); + ma_b(rd, Register(scratch), overflow, Assembler::NotEqual); +} + +void MacroAssemblerLOONG64::ma_add32TestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + // Check for signed range because of as_addi_d + if (is_intN(imm.value, 12)) { + ScratchRegisterScope scratch(asMasm()); + as_addi_d(scratch, rj, imm.value); + as_addi_w(rd, rj, imm.value); + ma_b(rd, scratch, overflow, Assembler::NotEqual); + } else { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_add32TestOverflow(rd, rj, scratch2, overflow); + } +} + +void MacroAssemblerLOONG64::ma_addPtrTestOverflow(Register rd, Register rj, + Register rk, + Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rd != scratch); + + if (rj == rk) { + if (rj == rd) { + as_or(scratch, rj, zero); + rj = scratch; + } + + as_add_d(rd, rj, rj); + as_xor(scratch, rj, rd); + ma_b(scratch, zero, overflow, Assembler::LessThan); + } else { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rj != scratch); + MOZ_ASSERT(rd != scratch2); + + if (rj == rd) { + as_or(scratch2, rj, zero); + rj = scratch2; + } + + as_add_d(rd, rj, rk); + as_slti(scratch, rj, 0); + as_slt(scratch2, rd, rj); + ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); + } +} + +void MacroAssemblerLOONG64::ma_addPtrTestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + + if (imm.value == 0) { + as_ori(rd, rj, 0); + return; + } + + if (rj == rd) { + as_ori(scratch2, rj, 0); + rj = scratch2; + } + + ma_add_d(rd, rj, imm); + + if (imm.value > 0) { + ma_b(rd, rj, overflow, Assembler::LessThan); + } else { + MOZ_ASSERT(imm.value < 0); + ma_b(rd, rj, overflow, Assembler::GreaterThan); + } +} + +void MacroAssemblerLOONG64::ma_addPtrTestCarry(Condition cond, Register rd, + Register rj, Register rk, + Label* label) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rd != rk); + MOZ_ASSERT(rd != scratch); + as_add_d(rd, rj, rk); + as_sltu(scratch, rd, rk); + ma_b(scratch, Register(scratch), label, + cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); +} + +void MacroAssemblerLOONG64::ma_addPtrTestCarry(Condition cond, Register rd, + Register rj, Imm32 imm, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + + // Check for signed range because of as_addi_d + if (is_intN(imm.value, 12)) { + as_addi_d(rd, rj, imm.value); + as_sltui(scratch2, rd, imm.value); + ma_b(scratch2, scratch2, label, + cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); + } else { + ma_li(scratch2, imm); + ma_addPtrTestCarry(cond, rd, rj, scratch2, label); + } +} + +// Subtract. +void MacroAssemblerLOONG64::ma_sub_d(Register rd, Register rj, Imm32 imm) { + if (is_intN(-imm.value, 12)) { + as_addi_d(rd, rj, -imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + as_sub_d(rd, rj, scratch); + } +} + +void MacroAssemblerLOONG64::ma_sub32TestOverflow(Register rd, Register rj, + Register rk, Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + as_sub_d(scratch, rj, rk); + as_sub_w(rd, rj, rk); + ma_b(rd, Register(scratch), overflow, Assembler::NotEqual); +} + +void MacroAssemblerLOONG64::ma_subPtrTestOverflow(Register rd, Register rj, + Register rk, + Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT_IF(rj == rd, rj != rk); + MOZ_ASSERT(rj != scratch2); + MOZ_ASSERT(rk != scratch2); + MOZ_ASSERT(rd != scratch2); + + Register rj_copy = rj; + + if (rj == rd) { + as_or(scratch2, rj, zero); + rj_copy = scratch2; + } + + { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rd != scratch); + + as_sub_d(rd, rj, rk); + // If the sign of rj and rk are the same, no overflow + as_xor(scratch, rj_copy, rk); + // Check if the sign of rd and rs are the same + as_xor(scratch2, rd, rj_copy); + as_and(scratch2, scratch2, scratch); + } + + ma_b(scratch2, zero, overflow, Assembler::LessThan); +} + +void MacroAssemblerLOONG64::ma_subPtrTestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + // TODO:LOONG64 subPtrTestOverflow + MOZ_ASSERT(imm.value != INT32_MIN); + ma_addPtrTestOverflow(rd, rj, Imm32(-imm.value), overflow); +} + +void MacroAssemblerLOONG64::ma_mul_d(Register rd, Register rj, Imm32 imm) { + // li handles the relocation. + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_mul_d(rd, rj, scratch); +} + +void MacroAssemblerLOONG64::ma_mulh_d(Register rd, Register rj, Imm32 imm) { + // li handles the relocation. + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_mulh_d(rd, rj, scratch); +} + +void MacroAssemblerLOONG64::ma_mulPtrTestOverflow(Register rd, Register rj, + Register rk, + Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rd != scratch); + + if (rd == rj) { + as_or(scratch, rj, zero); + rj = scratch; + rk = (rd == rk) ? rj : rk; + } else if (rd == rk) { + as_or(scratch, rk, zero); + rk = scratch; + } + + as_mul_d(rd, rj, rk); + as_mulh_d(scratch, rj, rk); + as_srai_d(scratch2, rd, 63); + ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); +} + +// Memory. + +void MacroAssemblerLOONG64::ma_load(Register dest, Address address, + LoadStoreSize size, + LoadStoreExtension extension) { + int32_t encodedOffset; + Register base; + + // TODO: use as_ldx_b/h/w/d, could decrease as_add_d instr. + switch (size) { + case SizeByte: + case SizeHalfWord: + if (!is_intN(address.offset, 12)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeByte) { + if (ZeroExtend == extension) { + as_ld_bu(dest, base, encodedOffset); + } else { + as_ld_b(dest, base, encodedOffset); + } + } else { + if (ZeroExtend == extension) { + as_ld_hu(dest, base, encodedOffset); + } else { + as_ld_h(dest, base, encodedOffset); + } + } + break; + case SizeWord: + case SizeDouble: + if ((address.offset & 0x3) == 0 && + (size == SizeDouble || + (size == SizeWord && SignExtend == extension))) { + if (!Imm16::IsInSignedRange(address.offset)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeWord) { + as_ldptr_w(dest, base, encodedOffset); + } else { + as_ldptr_d(dest, base, encodedOffset); + } + } else { + if (!is_intN(address.offset, 12)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeWord) { + if (ZeroExtend == extension) { + as_ld_wu(dest, base, encodedOffset); + } else { + as_ld_w(dest, base, encodedOffset); + } + } else { + as_ld_d(dest, base, encodedOffset); + } + } + break; + default: + MOZ_CRASH("Invalid argument for ma_load"); + } +} + +void MacroAssemblerLOONG64::ma_store(Register data, Address address, + LoadStoreSize size, + LoadStoreExtension extension) { + int32_t encodedOffset; + Register base; + + // TODO: use as_stx_b/h/w/d, could decrease as_add_d instr. + switch (size) { + case SizeByte: + case SizeHalfWord: + if (!is_intN(address.offset, 12)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeByte) { + as_st_b(data, base, encodedOffset); + } else { + as_st_h(data, base, encodedOffset); + } + break; + case SizeWord: + case SizeDouble: + if ((address.offset & 0x3) == 0) { + if (!Imm16::IsInSignedRange(address.offset)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeWord) { + as_stptr_w(data, base, encodedOffset); + } else { + as_stptr_d(data, base, encodedOffset); + } + } else { + if (!is_intN(address.offset, 12)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeWord) { + as_st_w(data, base, encodedOffset); + } else { + as_st_d(data, base, encodedOffset); + } + } + break; + default: + MOZ_CRASH("Invalid argument for ma_store"); + } +} + +void MacroAssemblerLOONG64Compat::computeScaledAddress(const BaseIndex& address, + Register dest) { + Register base = address.base; + Register index = address.index; + int32_t shift = Imm32::ShiftOf(address.scale).value; + + if (shift) { + MOZ_ASSERT(shift <= 4); + as_alsl_d(dest, index, base, shift - 1); + } else { + as_add_d(dest, base, index); + } +} + +void MacroAssemblerLOONG64::ma_pop(Register r) { + MOZ_ASSERT(r != StackPointer); + as_ld_d(r, StackPointer, 0); + as_addi_d(StackPointer, StackPointer, sizeof(intptr_t)); +} + +void MacroAssemblerLOONG64::ma_push(Register r) { + if (r == StackPointer) { + ScratchRegisterScope scratch(asMasm()); + as_or(scratch, r, zero); + as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(intptr_t)); + as_st_d(scratch, StackPointer, 0); + } else { + as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(intptr_t)); + as_st_d(r, StackPointer, 0); + } +} + +// Branches when done from within loongarch-specific code. +void MacroAssemblerLOONG64::ma_b(Register lhs, ImmWord imm, Label* label, + Condition c, JumpKind jumpKind) { + if (imm.value <= INT32_MAX) { + ma_b(lhs, Imm32(uint32_t(imm.value)), label, c, jumpKind); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + ma_li(scratch, imm); + ma_b(lhs, Register(scratch), label, c, jumpKind); + } +} + +void MacroAssemblerLOONG64::ma_b(Register lhs, Address addr, Label* label, + Condition c, JumpKind jumpKind) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + ma_ld_d(scratch, addr); + ma_b(lhs, Register(scratch), label, c, jumpKind); +} + +void MacroAssemblerLOONG64::ma_b(Address addr, Imm32 imm, Label* label, + Condition c, JumpKind jumpKind) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_ld_d(scratch2, addr); + ma_b(Register(scratch2), imm, label, c, jumpKind); +} + +void MacroAssemblerLOONG64::ma_b(Address addr, ImmGCPtr imm, Label* label, + Condition c, JumpKind jumpKind) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_load(scratch2, addr); + ma_b(Register(scratch2), imm, label, c, jumpKind); +} + +void MacroAssemblerLOONG64::ma_bl(Label* label) { + spew("branch .Llabel %p\n", label); + if (label->bound()) { + // Generate the long jump for calls because return address has to be + // the address after the reserved block. + addLongJump(nextOffset(), BufferOffset(label->offset())); + ScratchRegisterScope scratch(asMasm()); + ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); + as_jirl(ra, scratch, BOffImm16(0)); + return; + } + + // Second word holds a pointer to the next branch in label's chain. + uint32_t nextInChain = + label->used() ? label->offset() : LabelBase::INVALID_OFFSET; + + // Make the whole branch continous in the buffer. The '5' + // instructions are writing at below. + m_buffer.ensureSpace(5 * sizeof(uint32_t)); + + spew("bal .Llabel %p\n", label); + BufferOffset bo = writeInst(getBranchCode(BranchIsCall).encode()); + writeInst(nextInChain); + if (!oom()) { + label->use(bo.getOffset()); + } + // Leave space for long jump. + as_nop(); + as_nop(); + as_nop(); +} + +void MacroAssemblerLOONG64::branchWithCode(InstImm code, Label* label, + JumpKind jumpKind) { + // simply output the pointer of one label as its id, + // notice that after one label destructor, the pointer will be reused. + spew("branch .Llabel %p", label); + MOZ_ASSERT(code.encode() != + InstImm(op_jirl, BOffImm16(0), zero, ra).encode()); + InstImm inst_beq = InstImm(op_beq, BOffImm16(0), zero, zero); + + if (label->bound()) { + int32_t offset = label->offset() - m_buffer.nextOffset().getOffset(); + + if (BOffImm16::IsInRange(offset)) { + jumpKind = ShortJump; + } + + // ShortJump + if (jumpKind == ShortJump) { + MOZ_ASSERT(BOffImm16::IsInRange(offset)); + + if (code.extractBitField(31, 26) == ((uint32_t)op_bcz >> 26)) { + code.setImm21(offset); + } else { + code.setBOffImm16(BOffImm16(offset)); + } +#ifdef JS_JITSPEW + decodeBranchInstAndSpew(code); +#endif + writeInst(code.encode()); + return; + } + + // LongJump + if (code.encode() == inst_beq.encode()) { + // Handle long jump + addLongJump(nextOffset(), BufferOffset(label->offset())); + ScratchRegisterScope scratch(asMasm()); + ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); + as_jirl(zero, scratch, BOffImm16(0)); // jr scratch + as_nop(); + return; + } + + // OpenLongJump + // Handle long conditional branch, the target offset is based on self, + // point to next instruction of nop at below. + spew("invert branch .Llabel %p", label); + InstImm code_r = invertBranch(code, BOffImm16(5 * sizeof(uint32_t))); +#ifdef JS_JITSPEW + decodeBranchInstAndSpew(code_r); +#endif + writeInst(code_r.encode()); + addLongJump(nextOffset(), BufferOffset(label->offset())); + ScratchRegisterScope scratch(asMasm()); + ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); + as_jirl(zero, scratch, BOffImm16(0)); + as_nop(); + return; + } + + // Generate open jump and link it to a label. + + // Second word holds a pointer to the next branch in label's chain. + uint32_t nextInChain = + label->used() ? label->offset() : LabelBase::INVALID_OFFSET; + + if (jumpKind == ShortJump) { + // Make the whole branch continous in the buffer. + m_buffer.ensureSpace(2 * sizeof(uint32_t)); + + // Indicate that this is short jump with offset 4. + code.setBOffImm16(BOffImm16(4)); +#ifdef JS_JITSPEW + decodeBranchInstAndSpew(code); +#endif + BufferOffset bo = writeInst(code.encode()); + writeInst(nextInChain); + if (!oom()) { + label->use(bo.getOffset()); + } + return; + } + + bool conditional = code.encode() != inst_beq.encode(); + + // Make the whole branch continous in the buffer. The '5' + // instructions are writing at below (contain conditional nop). + m_buffer.ensureSpace(5 * sizeof(uint32_t)); + +#ifdef JS_JITSPEW + decodeBranchInstAndSpew(code); +#endif + BufferOffset bo = writeInst(code.encode()); // invert + writeInst(nextInChain); + if (!oom()) { + label->use(bo.getOffset()); + } + // Leave space for potential long jump. + as_nop(); + as_nop(); + if (conditional) { + as_nop(); + } +} + +void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, ImmWord imm, + Condition c) { + if (imm.value <= INT32_MAX) { + ma_cmp_set(rd, rj, Imm32(uint32_t(imm.value)), c); + } else { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_cmp_set(rd, rj, scratch, c); + } +} + +void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, ImmPtr imm, + Condition c) { + ma_cmp_set(rd, rj, ImmWord(uintptr_t(imm.value)), c); +} + +void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Address address, Imm32 imm, + Condition c) { + // TODO:LOONG64 Recheck ma_cmp_set + SecondScratchRegisterScope scratch2(asMasm()); + ma_ld_w(scratch2, address); + ma_cmp_set(rd, Register(scratch2), imm, c); +} + +// fp instructions +void MacroAssemblerLOONG64::ma_lid(FloatRegister dest, double value) { + ImmWord imm(mozilla::BitwiseCast(value)); + + if (imm.value != 0) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + moveToDouble(scratch, dest); + } else { + moveToDouble(zero, dest); + } +} + +void MacroAssemblerLOONG64::ma_mv(FloatRegister src, ValueOperand dest) { + as_movfr2gr_d(dest.valueReg(), src); +} + +void MacroAssemblerLOONG64::ma_mv(ValueOperand src, FloatRegister dest) { + as_movgr2fr_d(dest, src.valueReg()); +} + +void MacroAssemblerLOONG64::ma_fld_s(FloatRegister dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_fld_s(dest, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_fldx_s(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_fld_d(FloatRegister dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_fld_d(dest, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_fldx_d(dest, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_fst_s(FloatRegister src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_fst_s(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_fstx_s(src, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_fst_d(FloatRegister src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_fst_d(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_fstx_d(src, base, scratch); + } +} + +void MacroAssemblerLOONG64::ma_pop(FloatRegister f) { + as_fld_d(f, StackPointer, 0); + as_addi_d(StackPointer, StackPointer, sizeof(double)); +} + +void MacroAssemblerLOONG64::ma_push(FloatRegister f) { + as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(double)); + as_fst_d(f, StackPointer, 0); +} + +void MacroAssemblerLOONG64::ma_li(Register dest, ImmGCPtr ptr) { + writeDataRelocation(ptr); + asMasm().ma_liPatchable(dest, ImmPtr(ptr.value)); +} + +void MacroAssemblerLOONG64::ma_li(Register dest, Imm32 imm) { + if (is_intN(imm.value, 12)) { + as_addi_w(dest, zero, imm.value); + } else if (is_uintN(imm.value, 12)) { + as_ori(dest, zero, imm.value & 0xfff); + } else { + as_lu12i_w(dest, imm.value >> 12 & 0xfffff); + if (imm.value & 0xfff) { + as_ori(dest, dest, imm.value & 0xfff); + } + } +} + +// This method generates lu12i_w and ori instruction pair that can be modified +// by UpdateLuiOriValue, either during compilation (eg. Assembler::bind), or +// during execution (eg. jit::PatchJump). +void MacroAssemblerLOONG64::ma_liPatchable(Register dest, Imm32 imm) { + m_buffer.ensureSpace(2 * sizeof(uint32_t)); + as_lu12i_w(dest, imm.value >> 12 & 0xfffff); + as_ori(dest, dest, imm.value & 0xfff); +} + +void MacroAssemblerLOONG64::ma_fmovz(FloatFormat fmt, FloatRegister fd, + FloatRegister fj, Register rk) { + Label done; + ma_b(rk, zero, &done, Assembler::NotEqual); + if (fmt == SingleFloat) { + as_fmov_s(fd, fj); + } else { + as_fmov_d(fd, fj); + } + bind(&done); +} + +void MacroAssemblerLOONG64::ma_fmovn(FloatFormat fmt, FloatRegister fd, + FloatRegister fj, Register rk) { + Label done; + ma_b(rk, zero, &done, Assembler::Equal); + if (fmt == SingleFloat) { + as_fmov_s(fd, fj); + } else { + as_fmov_d(fd, fj); + } + bind(&done); +} + +void MacroAssemblerLOONG64::ma_and(Register rd, Register rj, Imm32 imm, + bool bit32) { + if (is_uintN(imm.value, 12)) { + as_andi(rd, rj, imm.value); + } else if (rd != rj) { + ma_li(rd, imm); + as_and(rd, rj, rd); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_and(rd, rj, scratch); + } +} + +void MacroAssemblerLOONG64::ma_or(Register rd, Register rj, Imm32 imm, + bool bit32) { + if (is_uintN(imm.value, 12)) { + as_ori(rd, rj, imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_or(rd, rj, scratch); + } +} + +void MacroAssemblerLOONG64::ma_xor(Register rd, Register rj, Imm32 imm, + bool bit32) { + if (is_uintN(imm.value, 12)) { + as_xori(rd, rj, imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_xor(rd, rj, scratch); + } +} + +// Arithmetic-based ops. + +// Add. +void MacroAssemblerLOONG64::ma_add_w(Register rd, Register rj, Imm32 imm) { + if (is_intN(imm.value, 12)) { + as_addi_w(rd, rj, imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_add_w(rd, rj, scratch); + } +} + +void MacroAssemblerLOONG64::ma_add32TestCarry(Condition cond, Register rd, + Register rj, Register rk, + Label* overflow) { + MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear); + MOZ_ASSERT_IF(rd == rj, rk != rd); + ScratchRegisterScope scratch(asMasm()); + as_add_w(rd, rj, rk); + as_sltu(scratch, rd, rd == rj ? rk : rj); + ma_b(Register(scratch), Register(scratch), overflow, + cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); +} + +void MacroAssemblerLOONG64::ma_add32TestCarry(Condition cond, Register rd, + Register rj, Imm32 imm, + Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rj != scratch2); + ma_li(scratch2, imm); + ma_add32TestCarry(cond, rd, rj, scratch2, overflow); +} + +// Subtract. +void MacroAssemblerLOONG64::ma_sub_w(Register rd, Register rj, Imm32 imm) { + if (is_intN(-imm.value, 12)) { + as_addi_w(rd, rj, -imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_sub_w(rd, rj, scratch); + } +} + +void MacroAssemblerLOONG64::ma_sub_w(Register rd, Register rj, Register rk) { + as_sub_w(rd, rj, rk); +} + +void MacroAssemblerLOONG64::ma_sub32TestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + if (imm.value != INT32_MIN) { + asMasm().ma_add32TestOverflow(rd, rj, Imm32(-imm.value), overflow); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, Imm32(imm.value)); + asMasm().ma_sub32TestOverflow(rd, rj, scratch, overflow); + } +} + +void MacroAssemblerLOONG64::ma_mul(Register rd, Register rj, Imm32 imm) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_mul_w(rd, rj, scratch); +} + +void MacroAssemblerLOONG64::ma_mul32TestOverflow(Register rd, Register rj, + Register rk, Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + as_mulh_w(scratch, rj, rk); + as_mul_w(rd, rj, rk); + as_srai_w(scratch2, rd, 31); + ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); +} + +void MacroAssemblerLOONG64::ma_mul32TestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch, imm); + as_mulh_w(scratch2, rj, scratch); + as_mul_w(rd, rj, scratch); + as_srai_w(scratch, rd, 31); + ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); +} + +void MacroAssemblerLOONG64::ma_div_branch_overflow(Register rd, Register rj, + Register rk, + Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + as_mod_w(scratch, rj, rk); + ma_b(scratch, scratch, overflow, Assembler::NonZero); + as_div_w(rd, rj, rk); +} + +void MacroAssemblerLOONG64::ma_div_branch_overflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_div_branch_overflow(rd, rj, scratch2, overflow); +} + +void MacroAssemblerLOONG64::ma_mod_mask(Register src, Register dest, + Register hold, Register remain, + int32_t shift, Label* negZero) { + // MATH: + // We wish to compute x % (1< 0, store sum - C back into sum, thus performing a + // modulus. + ma_b(scratch2, Register(scratch2), &sumSigned, Signed, ShortJump); + as_or(dest, scratch2, zero); + bind(&sumSigned); + // Get rid of the bits that we extracted before. + as_srli_w(remain, remain, shift); + // If the shift produced zero, finish, otherwise, continue in the loop. + ma_b(remain, remain, &head, NonZero, ShortJump); + // Check the hold to see if we need to negate the result. + ma_b(hold, hold, &done, NotSigned, ShortJump); + + // If the hold was non-zero, negate the result to be in line with + // what JS wants + if (negZero != nullptr) { + // Jump out in case of negative zero. + ma_b(hold, hold, negZero, Zero); + as_sub_w(dest, zero, dest); + } else { + as_sub_w(dest, zero, dest); + } + + bind(&done); +} + +// Memory. + +void MacroAssemblerLOONG64::ma_load(Register dest, const BaseIndex& src, + LoadStoreSize size, + LoadStoreExtension extension) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(src, scratch2); + asMasm().ma_load(dest, Address(scratch2, src.offset), size, extension); +} + +void MacroAssemblerLOONG64::ma_store(Register data, const BaseIndex& dest, + LoadStoreSize size, + LoadStoreExtension extension) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(dest, scratch2); + asMasm().ma_store(data, Address(scratch2, dest.offset), size, extension); +} + +void MacroAssemblerLOONG64::ma_store(Imm32 imm, const BaseIndex& dest, + LoadStoreSize size, + LoadStoreExtension extension) { + SecondScratchRegisterScope scratch2(asMasm()); + // Make sure that scratch2 contains absolute address so that offset is 0. + asMasm().computeEffectiveAddress(dest, scratch2); + + ScratchRegisterScope scratch(asMasm()); + // Scrach register is free now, use it for loading imm value + ma_li(scratch, imm); + + // with offset=0 ScratchRegister will not be used in ma_store() + // so we can use it as a parameter here + asMasm().ma_store(scratch, Address(scratch2, 0), size, extension); +} + +// Branches when done from within loongarch-specific code. +// TODO:LOONG64 Optimize ma_b +void MacroAssemblerLOONG64::ma_b(Register lhs, Register rhs, Label* label, + Condition c, JumpKind jumpKind) { + switch (c) { + case Equal: + case NotEqual: + asMasm().branchWithCode(getBranchCode(lhs, rhs, c), label, jumpKind); + break; + case Always: + ma_b(label, jumpKind); + break; + case Zero: + case NonZero: + case Signed: + case NotSigned: + MOZ_ASSERT(lhs == rhs); + asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind); + break; + default: { + Condition cond = ma_cmp(ScratchRegister, lhs, rhs, c); + asMasm().branchWithCode(getBranchCode(ScratchRegister, cond), label, + jumpKind); + break; + } + } +} + +void MacroAssemblerLOONG64::ma_b(Register lhs, Imm32 imm, Label* label, + Condition c, JumpKind jumpKind) { + MOZ_ASSERT(c != Overflow); + if (imm.value == 0) { + if (c == Always || c == AboveOrEqual) { + ma_b(label, jumpKind); + } else if (c == Below) { + ; // This condition is always false. No branch required. + } else { + asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind); + } + } else { + switch (c) { + case Equal: + case NotEqual: + MOZ_ASSERT(lhs != ScratchRegister); + ma_li(ScratchRegister, imm); + ma_b(lhs, ScratchRegister, label, c, jumpKind); + break; + default: + Condition cond = ma_cmp(ScratchRegister, lhs, imm, c); + asMasm().branchWithCode(getBranchCode(ScratchRegister, cond), label, + jumpKind); + } + } +} + +void MacroAssemblerLOONG64::ma_b(Register lhs, ImmPtr imm, Label* l, + Condition c, JumpKind jumpKind) { + asMasm().ma_b(lhs, ImmWord(uintptr_t(imm.value)), l, c, jumpKind); +} + +void MacroAssemblerLOONG64::ma_b(Label* label, JumpKind jumpKind) { + asMasm().branchWithCode(getBranchCode(BranchIsJump), label, jumpKind); +} + +Assembler::Condition MacroAssemblerLOONG64::ma_cmp(Register dest, Register lhs, + Register rhs, Condition c) { + switch (c) { + case Above: + // bgtu s,t,label => + // sltu at,t,s + // bne at,$zero,offs + as_sltu(dest, rhs, lhs); + return NotEqual; + case AboveOrEqual: + // bgeu s,t,label => + // sltu at,s,t + // beq at,$zero,offs + as_sltu(dest, lhs, rhs); + return Equal; + case Below: + // bltu s,t,label => + // sltu at,s,t + // bne at,$zero,offs + as_sltu(dest, lhs, rhs); + return NotEqual; + case BelowOrEqual: + // bleu s,t,label => + // sltu at,t,s + // beq at,$zero,offs + as_sltu(dest, rhs, lhs); + return Equal; + case GreaterThan: + // bgt s,t,label => + // slt at,t,s + // bne at,$zero,offs + as_slt(dest, rhs, lhs); + return NotEqual; + case GreaterThanOrEqual: + // bge s,t,label => + // slt at,s,t + // beq at,$zero,offs + as_slt(dest, lhs, rhs); + return Equal; + case LessThan: + // blt s,t,label => + // slt at,s,t + // bne at,$zero,offs + as_slt(dest, lhs, rhs); + return NotEqual; + case LessThanOrEqual: + // ble s,t,label => + // slt at,t,s + // beq at,$zero,offs + as_slt(dest, rhs, lhs); + return Equal; + default: + MOZ_CRASH("Invalid condition."); + } + return Always; +} + +Assembler::Condition MacroAssemblerLOONG64::ma_cmp(Register dest, Register lhs, + Imm32 imm, Condition c) { + ScratchRegisterScope scratch(asMasm()); + MOZ_RELEASE_ASSERT(lhs != scratch); + + switch (c) { + case Above: + case BelowOrEqual: + if (imm.value != 0x7fffffff && is_intN(imm.value + 1, 12) && + imm.value != -1) { + // lhs <= rhs via lhs < rhs + 1 if rhs + 1 does not overflow + as_sltui(dest, lhs, imm.value + 1); + + return (c == BelowOrEqual ? NotEqual : Equal); + } else { + ma_li(scratch, imm); + as_sltu(dest, scratch, lhs); + return (c == BelowOrEqual ? Equal : NotEqual); + } + case AboveOrEqual: + case Below: + if (is_intN(imm.value, 12)) { + as_sltui(dest, lhs, imm.value); + } else { + ma_li(scratch, imm); + as_sltu(dest, lhs, scratch); + } + return (c == AboveOrEqual ? Equal : NotEqual); + case GreaterThan: + case LessThanOrEqual: + if (imm.value != 0x7fffffff && is_intN(imm.value + 1, 12)) { + // lhs <= rhs via lhs < rhs + 1. + as_slti(dest, lhs, imm.value + 1); + return (c == LessThanOrEqual ? NotEqual : Equal); + } else { + ma_li(scratch, imm); + as_slt(dest, scratch, lhs); + return (c == LessThanOrEqual ? Equal : NotEqual); + } + case GreaterThanOrEqual: + case LessThan: + if (is_intN(imm.value, 12)) { + as_slti(dest, lhs, imm.value); + } else { + ma_li(scratch, imm); + as_slt(dest, lhs, scratch); + } + return (c == GreaterThanOrEqual ? Equal : NotEqual); + default: + MOZ_CRASH("Invalid condition."); + } + return Always; +} + +// fp instructions +void MacroAssemblerLOONG64::ma_lis(FloatRegister dest, float value) { + Imm32 imm(mozilla::BitwiseCast(value)); + + if (imm.value != 0) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + moveToFloat32(scratch, dest); + } else { + moveToFloat32(zero, dest); + } +} + +void MacroAssemblerLOONG64::ma_fst_d(FloatRegister ft, BaseIndex address) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(address, scratch2); + asMasm().ma_fst_d(ft, Address(scratch2, address.offset)); +} + +void MacroAssemblerLOONG64::ma_fst_s(FloatRegister ft, BaseIndex address) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(address, scratch2); + asMasm().ma_fst_s(ft, Address(scratch2, address.offset)); +} + +void MacroAssemblerLOONG64::ma_fld_d(FloatRegister ft, const BaseIndex& src) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(src, scratch2); + asMasm().ma_fld_d(ft, Address(scratch2, src.offset)); +} + +void MacroAssemblerLOONG64::ma_fld_s(FloatRegister ft, const BaseIndex& src) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(src, scratch2); + asMasm().ma_fld_s(ft, Address(scratch2, src.offset)); +} + +void MacroAssemblerLOONG64::ma_bc1s(FloatRegister lhs, FloatRegister rhs, + Label* label, DoubleCondition c, + JumpKind jumpKind, FPConditionBit fcc) { + compareFloatingPoint(SingleFloat, lhs, rhs, c, fcc); + asMasm().branchWithCode(getBranchCode(fcc), label, jumpKind); +} + +void MacroAssemblerLOONG64::ma_bc1d(FloatRegister lhs, FloatRegister rhs, + Label* label, DoubleCondition c, + JumpKind jumpKind, FPConditionBit fcc) { + compareFloatingPoint(DoubleFloat, lhs, rhs, c, fcc); + asMasm().branchWithCode(getBranchCode(fcc), label, jumpKind); +} + +void MacroAssemblerLOONG64::ma_call(ImmPtr dest) { + asMasm().ma_liPatchable(CallReg, dest); + as_jirl(ra, CallReg, BOffImm16(0)); +} + +void MacroAssemblerLOONG64::ma_jump(ImmPtr dest) { + ScratchRegisterScope scratch(asMasm()); + asMasm().ma_liPatchable(scratch, dest); + as_jirl(zero, scratch, BOffImm16(0)); +} + +void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, Register rk, + Condition c) { + switch (c) { + case Equal: + // seq d,s,t => + // xor d,s,t + // sltiu d,d,1 + as_xor(rd, rj, rk); + as_sltui(rd, rd, 1); + break; + case NotEqual: + // sne d,s,t => + // xor d,s,t + // sltu d,$zero,d + as_xor(rd, rj, rk); + as_sltu(rd, zero, rd); + break; + case Above: + // sgtu d,s,t => + // sltu d,t,s + as_sltu(rd, rk, rj); + break; + case AboveOrEqual: + // sgeu d,s,t => + // sltu d,s,t + // xori d,d,1 + as_sltu(rd, rj, rk); + as_xori(rd, rd, 1); + break; + case Below: + // sltu d,s,t + as_sltu(rd, rj, rk); + break; + case BelowOrEqual: + // sleu d,s,t => + // sltu d,t,s + // xori d,d,1 + as_sltu(rd, rk, rj); + as_xori(rd, rd, 1); + break; + case GreaterThan: + // sgt d,s,t => + // slt d,t,s + as_slt(rd, rk, rj); + break; + case GreaterThanOrEqual: + // sge d,s,t => + // slt d,s,t + // xori d,d,1 + as_slt(rd, rj, rk); + as_xori(rd, rd, 1); + break; + case LessThan: + // slt d,s,t + as_slt(rd, rj, rk); + break; + case LessThanOrEqual: + // sle d,s,t => + // slt d,t,s + // xori d,d,1 + as_slt(rd, rk, rj); + as_xori(rd, rd, 1); + break; + case Zero: + MOZ_ASSERT(rj == rk); + // seq d,s,$zero => + // sltiu d,s,1 + as_sltui(rd, rj, 1); + break; + case NonZero: + MOZ_ASSERT(rj == rk); + // sne d,s,$zero => + // sltu d,$zero,s + as_sltu(rd, zero, rj); + break; + case Signed: + MOZ_ASSERT(rj == rk); + as_slt(rd, rj, zero); + break; + case NotSigned: + MOZ_ASSERT(rj == rk); + // sge d,s,$zero => + // slt d,s,$zero + // xori d,d,1 + as_slt(rd, rj, zero); + as_xori(rd, rd, 1); + break; + default: + MOZ_CRASH("Invalid condition."); + } +} + +void MacroAssemblerLOONG64::ma_cmp_set_double(Register dest, FloatRegister lhs, + FloatRegister rhs, + DoubleCondition c) { + compareFloatingPoint(DoubleFloat, lhs, rhs, c); + as_movcf2gr(dest, FCC0); +} + +void MacroAssemblerLOONG64::ma_cmp_set_float32(Register dest, FloatRegister lhs, + FloatRegister rhs, + DoubleCondition c) { + compareFloatingPoint(SingleFloat, lhs, rhs, c); + as_movcf2gr(dest, FCC0); +} + +void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, Imm32 imm, + Condition c) { + if (imm.value == 0) { + switch (c) { + case Equal: + case BelowOrEqual: + as_sltui(rd, rj, 1); + break; + case NotEqual: + case Above: + as_sltu(rd, zero, rj); + break; + case AboveOrEqual: + case Below: + as_ori(rd, zero, c == AboveOrEqual ? 1 : 0); + break; + case GreaterThan: + case LessThanOrEqual: + as_slt(rd, zero, rj); + if (c == LessThanOrEqual) { + as_xori(rd, rd, 1); + } + break; + case LessThan: + case GreaterThanOrEqual: + as_slt(rd, rj, zero); + if (c == GreaterThanOrEqual) { + as_xori(rd, rd, 1); + } + break; + case Zero: + as_sltui(rd, rj, 1); + break; + case NonZero: + as_sltu(rd, zero, rj); + break; + case Signed: + as_slt(rd, rj, zero); + break; + case NotSigned: + as_slt(rd, rj, zero); + as_xori(rd, rd, 1); + break; + default: + MOZ_CRASH("Invalid condition."); + } + return; + } + + switch (c) { + case Equal: + case NotEqual: + ma_xor(rd, rj, imm); + if (c == Equal) { + as_sltui(rd, rd, 1); + } else { + as_sltu(rd, zero, rd); + } + break; + case Zero: + case NonZero: + case Signed: + case NotSigned: + MOZ_CRASH("Invalid condition."); + default: + Condition cond = ma_cmp(rd, rj, imm, c); + MOZ_ASSERT(cond == Equal || cond == NotEqual); + + if (cond == Equal) as_xori(rd, rd, 1); + } +} + +void MacroAssemblerLOONG64::compareFloatingPoint(FloatFormat fmt, + FloatRegister lhs, + FloatRegister rhs, + DoubleCondition c, + FPConditionBit fcc) { + switch (c) { + case DoubleOrdered: + as_fcmp_cor(fmt, lhs, rhs, fcc); + break; + case DoubleEqual: + as_fcmp_ceq(fmt, lhs, rhs, fcc); + break; + case DoubleNotEqual: + as_fcmp_cne(fmt, lhs, rhs, fcc); + break; + case DoubleGreaterThan: + as_fcmp_clt(fmt, rhs, lhs, fcc); + break; + case DoubleGreaterThanOrEqual: + as_fcmp_cle(fmt, rhs, lhs, fcc); + break; + case DoubleLessThan: + as_fcmp_clt(fmt, lhs, rhs, fcc); + break; + case DoubleLessThanOrEqual: + as_fcmp_cle(fmt, lhs, rhs, fcc); + break; + case DoubleUnordered: + as_fcmp_cun(fmt, lhs, rhs, fcc); + break; + case DoubleEqualOrUnordered: + as_fcmp_cueq(fmt, lhs, rhs, fcc); + break; + case DoubleNotEqualOrUnordered: + as_fcmp_cune(fmt, lhs, rhs, fcc); + break; + case DoubleGreaterThanOrUnordered: + as_fcmp_cult(fmt, rhs, lhs, fcc); + break; + case DoubleGreaterThanOrEqualOrUnordered: + as_fcmp_cule(fmt, rhs, lhs, fcc); + break; + case DoubleLessThanOrUnordered: + as_fcmp_cult(fmt, lhs, rhs, fcc); + break; + case DoubleLessThanOrEqualOrUnordered: + as_fcmp_cule(fmt, lhs, rhs, fcc); + break; + default: + MOZ_CRASH("Invalid DoubleCondition."); + } +} + +void MacroAssemblerLOONG64::minMaxDouble(FloatRegister srcDest, + FloatRegister second, bool handleNaN, + bool isMax) { + if (srcDest == second) return; + + Label nan, done; + + // First or second is NaN, result is NaN. + ma_bc1d(srcDest, second, &nan, Assembler::DoubleUnordered, ShortJump); + if (isMax) { + as_fmax_d(srcDest, srcDest, second); + } else { + as_fmin_d(srcDest, srcDest, second); + } + ma_b(&done, ShortJump); + + bind(&nan); + as_fadd_d(srcDest, srcDest, second); + + bind(&done); +} + +void MacroAssemblerLOONG64::minMaxFloat32(FloatRegister srcDest, + FloatRegister second, bool handleNaN, + bool isMax) { + if (srcDest == second) return; + + Label nan, done; + + // First or second is NaN, result is NaN. + ma_bc1s(srcDest, second, &nan, Assembler::DoubleUnordered, ShortJump); + if (isMax) { + as_fmax_s(srcDest, srcDest, second); + } else { + as_fmin_s(srcDest, srcDest, second); + } + ma_b(&done, ShortJump); + + bind(&nan); + as_fadd_s(srcDest, srcDest, second); + + bind(&done); +} + +void MacroAssemblerLOONG64::loadDouble(const Address& address, + FloatRegister dest) { + asMasm().ma_fld_d(dest, address); +} + +void MacroAssemblerLOONG64::loadDouble(const BaseIndex& src, + FloatRegister dest) { + asMasm().ma_fld_d(dest, src); +} + +void MacroAssemblerLOONG64::loadFloatAsDouble(const Address& address, + FloatRegister dest) { + asMasm().ma_fld_s(dest, address); + as_fcvt_d_s(dest, dest); +} + +void MacroAssemblerLOONG64::loadFloatAsDouble(const BaseIndex& src, + FloatRegister dest) { + asMasm().loadFloat32(src, dest); + as_fcvt_d_s(dest, dest); +} + +void MacroAssemblerLOONG64::loadFloat32(const Address& address, + FloatRegister dest) { + asMasm().ma_fld_s(dest, address); +} + +void MacroAssemblerLOONG64::loadFloat32(const BaseIndex& src, + FloatRegister dest) { + asMasm().ma_fld_s(dest, src); +} + +void MacroAssemblerLOONG64::wasmLoadImpl(const wasm::MemoryAccessDesc& access, + Register memoryBase, Register ptr, + Register ptrScratch, + AnyRegister output, Register tmp) { + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit()); + MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); + + MOZ_ASSERT(!access.isZeroExtendSimd128Load()); + MOZ_ASSERT(!access.isSplatSimd128Load()); + MOZ_ASSERT(!access.isWidenSimd128Load()); + + // Maybe add the offset. + if (offset) { + asMasm().addPtr(ImmWord(offset), ptrScratch); + ptr = ptrScratch; + } + + asMasm().memoryBarrierBefore(access.sync()); + + switch (access.type()) { + case Scalar::Int8: + as_ldx_b(output.gpr(), memoryBase, ptr); + break; + case Scalar::Uint8: + as_ldx_bu(output.gpr(), memoryBase, ptr); + break; + case Scalar::Int16: + as_ldx_h(output.gpr(), memoryBase, ptr); + break; + case Scalar::Uint16: + as_ldx_hu(output.gpr(), memoryBase, ptr); + break; + case Scalar::Int32: + case Scalar::Uint32: + as_ldx_w(output.gpr(), memoryBase, ptr); + break; + case Scalar::Float64: + as_fldx_d(output.fpu(), memoryBase, ptr); + break; + case Scalar::Float32: + as_fldx_s(output.fpu(), memoryBase, ptr); + break; + default: + MOZ_CRASH("unexpected array type"); + } + + asMasm().append(access, asMasm().size() - 4); + asMasm().memoryBarrierAfter(access.sync()); +} + +void MacroAssemblerLOONG64::wasmStoreImpl(const wasm::MemoryAccessDesc& access, + AnyRegister value, + Register memoryBase, Register ptr, + Register ptrScratch, Register tmp) { + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit()); + MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); + + // Maybe add the offset. + if (offset) { + asMasm().addPtr(ImmWord(offset), ptrScratch); + ptr = ptrScratch; + } + + asMasm().memoryBarrierBefore(access.sync()); + + switch (access.type()) { + case Scalar::Int8: + case Scalar::Uint8: + as_stx_b(value.gpr(), memoryBase, ptr); + break; + case Scalar::Int16: + case Scalar::Uint16: + as_stx_h(value.gpr(), memoryBase, ptr); + break; + case Scalar::Int32: + case Scalar::Uint32: + as_stx_w(value.gpr(), memoryBase, ptr); + break; + case Scalar::Int64: + as_stx_d(value.gpr(), memoryBase, ptr); + break; + case Scalar::Float64: + as_fstx_d(value.fpu(), memoryBase, ptr); + break; + case Scalar::Float32: + as_fstx_s(value.fpu(), memoryBase, ptr); + break; + default: + MOZ_CRASH("unexpected array type"); + } + + // Only the last emitted instruction is a memory access. + asMasm().append(access, asMasm().size() - 4); + asMasm().memoryBarrierAfter(access.sync()); +} + +// TODO:LOONG64 wasmLoadI64Impl +void MacroAssemblerLOONG64Compat::wasmLoadI64Impl( + const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr, + Register ptrScratch, Register64 output, Register tmp) { + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit()); + MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); + + // Maybe add the offset. + if (offset) { + asMasm().addPtr(Imm32(offset), ptrScratch); + ptr = ptrScratch; + } + + unsigned byteSize = access.byteSize(); + bool isSigned; + + switch (access.type()) { + case Scalar::Int8: + isSigned = true; + break; + case Scalar::Uint8: + isSigned = false; + break; + case Scalar::Int16: + isSigned = true; + break; + case Scalar::Uint16: + isSigned = false; + break; + case Scalar::Int32: + isSigned = true; + break; + case Scalar::Uint32: + isSigned = false; + break; + case Scalar::Int64: + isSigned = true; + break; + default: + MOZ_CRASH("unexpected array type"); + } + + BaseIndex address(memoryBase, ptr, TimesOne); + + asMasm().memoryBarrierBefore(access.sync()); + asMasm().ma_load(output.reg, address, + static_cast(8 * byteSize), + isSigned ? SignExtend : ZeroExtend); + asMasm().append(access, asMasm().size() - 4); + asMasm().memoryBarrierAfter(access.sync()); +} + +void MacroAssemblerLOONG64Compat::wasmStoreI64Impl( + const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase, + Register ptr, Register ptrScratch, Register tmp) { + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit()); + MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); + + // Maybe add the offset. + if (offset) { + asMasm().addPtr(Imm32(offset), ptrScratch); + ptr = ptrScratch; + } + + unsigned byteSize = access.byteSize(); + bool isSigned; + switch (access.type()) { + case Scalar::Int8: + isSigned = true; + break; + case Scalar::Uint8: + isSigned = false; + break; + case Scalar::Int16: + isSigned = true; + break; + case Scalar::Uint16: + isSigned = false; + break; + case Scalar::Int32: + isSigned = true; + break; + case Scalar::Uint32: + isSigned = false; + break; + case Scalar::Int64: + isSigned = true; + break; + default: + MOZ_CRASH("unexpected array type"); + } + + BaseIndex address(memoryBase, ptr, TimesOne); + + asMasm().memoryBarrierBefore(access.sync()); + asMasm().ma_store(value.reg, address, + static_cast(8 * byteSize), + isSigned ? SignExtend : ZeroExtend); + asMasm().append(access, asMasm().size() - 4); + asMasm().memoryBarrierAfter(access.sync()); +} + +void MacroAssemblerLOONG64::outOfLineWasmTruncateToInt32Check( + FloatRegister input, Register output, MIRType fromType, TruncFlags flags, + Label* rejoin, wasm::BytecodeOffset trapOffset) { + bool isUnsigned = flags & TRUNC_UNSIGNED; + bool isSaturating = flags & TRUNC_SATURATING; + + if (isSaturating) { + if (fromType == MIRType::Double) { + asMasm().loadConstantDouble(0.0, ScratchDoubleReg); + } else { + asMasm().loadConstantFloat32(0.0f, ScratchFloat32Reg); + } + + if (isUnsigned) { + ma_li(output, Imm32(UINT32_MAX)); + + compareFloatingPoint( + fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, + fromType == MIRType::Double ? ScratchDoubleReg : ScratchFloat32Reg, + Assembler::DoubleLessThanOrUnordered); + + ScratchRegisterScope scratch(asMasm()); + as_movcf2gr(scratch, FCC0); + // FCC0 = 1, output = zero; else not change. + as_masknez(output, output, scratch); + } else { + // Positive overflow is already saturated to INT32_MAX, so we only have + // to handle NaN and negative overflow here. + + compareFloatingPoint( + fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, input, + Assembler::DoubleLessThanOrUnordered); + + ScratchRegisterScope scratch(asMasm()); + as_movcf2gr(scratch, FCC0); + // FCC0 = 1, output = zero; else not change. + as_masknez(output, output, scratch); + + compareFloatingPoint( + fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, + fromType == MIRType::Double ? ScratchDoubleReg : ScratchFloat32Reg, + Assembler::DoubleLessThan); + + as_movcf2gr(scratch, FCC0); + // FCC0 == 1, move INT32_MIN to output; else not change. + as_slli_w(scratch, scratch, 31); + as_or(output, output, scratch); + } + + MOZ_ASSERT(rejoin->bound()); + asMasm().jump(rejoin); + return; + } + + Label inputIsNaN; + + if (fromType == MIRType::Double) { + asMasm().branchDouble(Assembler::DoubleUnordered, input, input, + &inputIsNaN); + } else if (fromType == MIRType::Float32) { + asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN); + } + + asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset); + asMasm().bind(&inputIsNaN); + asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset); +} + +void MacroAssemblerLOONG64::outOfLineWasmTruncateToInt64Check( + FloatRegister input, Register64 output_, MIRType fromType, TruncFlags flags, + Label* rejoin, wasm::BytecodeOffset trapOffset) { + bool isUnsigned = flags & TRUNC_UNSIGNED; + bool isSaturating = flags & TRUNC_SATURATING; + + if (isSaturating) { + Register output = output_.reg; + + if (fromType == MIRType::Double) { + asMasm().loadConstantDouble(0.0, ScratchDoubleReg); + } else { + asMasm().loadConstantFloat32(0.0f, ScratchFloat32Reg); + } + + if (isUnsigned) { + asMasm().ma_li(output, ImmWord(UINT64_MAX)); + + compareFloatingPoint( + fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, + fromType == MIRType::Double ? ScratchDoubleReg : ScratchFloat32Reg, + Assembler::DoubleLessThanOrUnordered); + + ScratchRegisterScope scratch(asMasm()); + as_movcf2gr(scratch, FCC0); + // FCC0 = 1, output = zero; else not change. + as_masknez(output, output, scratch); + } else { + // Positive overflow is already saturated to INT64_MAX, so we only have + // to handle NaN and negative overflow here. + + compareFloatingPoint( + fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, input, + Assembler::DoubleLessThanOrUnordered); + + ScratchRegisterScope scratch(asMasm()); + as_movcf2gr(scratch, FCC0); + // FCC0 = 1, output = zero; else not change. + as_masknez(output, output, scratch); + + compareFloatingPoint( + fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, + fromType == MIRType::Double ? ScratchDoubleReg : ScratchFloat32Reg, + Assembler::DoubleLessThan); + + as_movcf2gr(scratch, FCC0); + // FCC0 == 1, move INT64_MIN to output; else not change. + as_slli_d(scratch, scratch, 63); + as_or(output, output, scratch); + } + + MOZ_ASSERT(rejoin->bound()); + asMasm().jump(rejoin); + return; + } + + Label inputIsNaN; + + if (fromType == MIRType::Double) { + asMasm().branchDouble(Assembler::DoubleUnordered, input, input, + &inputIsNaN); + } else if (fromType == MIRType::Float32) { + asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN); + } + + asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset); + asMasm().bind(&inputIsNaN); + asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset); +} + +void MacroAssemblerLOONG64Compat::profilerEnterFrame(Register framePtr, + Register scratch) { + asMasm().loadJSContext(scratch); + loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch); + storePtr(framePtr, + Address(scratch, JitActivation::offsetOfLastProfilingFrame())); + storePtr(ImmPtr(nullptr), + Address(scratch, JitActivation::offsetOfLastProfilingCallSite())); +} + +void MacroAssemblerLOONG64Compat::profilerExitFrame() { + jump(GetJitContext()->runtime->jitRuntime()->getProfilerExitFrameTail()); +} + +MacroAssembler& MacroAssemblerLOONG64::asMasm() { + return *static_cast(this); +} + +const MacroAssembler& MacroAssemblerLOONG64::asMasm() const { + return *static_cast(this); +} + +void MacroAssembler::subFromStackPtr(Imm32 imm32) { + if (imm32.value) { + asMasm().subPtr(imm32, StackPointer); + } +} + +//{{{ check_macroassembler_style +// =============================================================== +// MacroAssembler high-level usage. + +void MacroAssembler::flush() {} + +// =============================================================== +// Stack manipulation functions. + +size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) { + return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); +} + +void MacroAssembler::PushRegsInMask(LiveRegisterSet set) { + int32_t diff = + set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); + const int32_t reserved = diff; + + reserveStack(reserved); + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diff -= sizeof(intptr_t); + storePtr(*iter, Address(StackPointer, diff)); + } + +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); + iter.more(); ++iter) { + diff -= sizeof(double); + storeDouble(*iter, Address(StackPointer, diff)); + } + MOZ_ASSERT(diff == 0); +} + +void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set, + LiveRegisterSet ignore) { + int32_t diff = + set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); + const int32_t reserved = diff; + + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diff -= sizeof(intptr_t); + if (!ignore.has(*iter)) { + loadPtr(Address(StackPointer, diff), *iter); + } + } + +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); + iter.more(); ++iter) { + diff -= sizeof(double); + if (!ignore.has(*iter)) { + loadDouble(Address(StackPointer, diff), *iter); + } + } + MOZ_ASSERT(diff == 0); + freeStack(reserved); +} + +void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest, + Register) { + FloatRegisterSet fpuSet(set.fpus().reduceSetForPush()); + unsigned numFpu = fpuSet.size(); + int32_t diffF = fpuSet.getPushSizeInBytes(); + int32_t diffG = set.gprs().size() * sizeof(intptr_t); + + MOZ_ASSERT(dest.offset >= diffG + diffF); + + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diffG -= sizeof(intptr_t); + dest.offset -= sizeof(intptr_t); + storePtr(*iter, dest); + } + MOZ_ASSERT(diffG == 0); + +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) { + FloatRegister reg = *iter; + diffF -= reg.size(); + numFpu -= 1; + dest.offset -= reg.size(); + if (reg.isDouble()) { + storeDouble(reg, dest); + } else if (reg.isSingle()) { + storeFloat32(reg, dest); + } else { + MOZ_CRASH("Unknown register type."); + } + } + MOZ_ASSERT(numFpu == 0); + diffF -= diffF % sizeof(uintptr_t); + MOZ_ASSERT(diffF == 0); +} + +void MacroAssembler::Push(Register reg) { + ma_push(reg); + adjustFrame(int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Push(const Imm32 imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + adjustFrame(int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Push(const ImmWord imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + adjustFrame(int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Push(const ImmPtr imm) { + Push(ImmWord(uintptr_t(imm.value))); +} + +void MacroAssembler::Push(const ImmGCPtr ptr) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, ptr); + ma_push(scratch); + adjustFrame(int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Push(FloatRegister f) { + ma_push(f); + adjustFrame(int32_t(sizeof(double))); +} + +void MacroAssembler::PushBoxed(FloatRegister reg) { + subFromStackPtr(Imm32(sizeof(double))); + boxDouble(reg, Address(getStackPointer(), 0)); + adjustFrame(sizeof(double)); +} + +void MacroAssembler::Pop(Register reg) { + ma_pop(reg); + adjustFrame(-int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Pop(FloatRegister f) { + ma_pop(f); + adjustFrame(-int32_t(sizeof(double))); +} + +void MacroAssembler::Pop(const ValueOperand& val) { + popValue(val); + adjustFrame(-int32_t(sizeof(Value))); +} + +void MacroAssembler::PopStackPtr() { + loadPtr(Address(StackPointer, 0), StackPointer); + adjustFrame(-int32_t(sizeof(intptr_t))); +} + +// =============================================================== +// Simple call functions. + +CodeOffset MacroAssembler::call(Register reg) { + as_jirl(ra, reg, BOffImm16(0)); + return CodeOffset(currentOffset()); +} + +CodeOffset MacroAssembler::call(Label* label) { + ma_bl(label); + return CodeOffset(currentOffset()); +} + +CodeOffset MacroAssembler::callWithPatch() { + as_bl(JOffImm26(1 * sizeof(uint32_t))); + return CodeOffset(currentOffset()); +} + +void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) { + BufferOffset call(callerOffset - 1 * sizeof(uint32_t)); + + JOffImm26 offset = BufferOffset(calleeOffset).diffB(call); + if (!offset.isInvalid()) { + InstJump* bal = (InstJump*)editSrc(call); + bal->setJOffImm26(offset); + } else { + uint32_t u32Offset = callerOffset - 4 * sizeof(uint32_t); + uint32_t* u32 = + reinterpret_cast(editSrc(BufferOffset(u32Offset))); + *u32 = calleeOffset - callerOffset; + } +} + +CodeOffset MacroAssembler::farJumpWithPatch() { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + as_pcaddi(scratch, 4); + as_ld_w(scratch2, scratch, 0); + as_add_d(scratch, scratch, scratch2); + as_jirl(zero, scratch, BOffImm16(0)); + // Allocate space which will be patched by patchFarJump(). + CodeOffset farJump(currentOffset()); + spew(".space 32bit initValue 0xffff ffff"); + writeInst(UINT32_MAX); + return farJump; +} + +void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) { + uint32_t* u32 = + reinterpret_cast(editSrc(BufferOffset(farJump.offset()))); + MOZ_ASSERT(*u32 == UINT32_MAX); + *u32 = targetOffset - farJump.offset(); +} + +CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) { + movePtr(target, CallReg); + return call(CallReg); +} + +void MacroAssembler::call(const Address& addr) { + loadPtr(addr, CallReg); + call(CallReg); +} + +void MacroAssembler::call(ImmWord target) { call(ImmPtr((void*)target.value)); } + +void MacroAssembler::call(ImmPtr target) { + BufferOffset bo = m_buffer.nextOffset(); + addPendingJump(bo, target, RelocationKind::HARDCODED); + ma_call(target); +} + +void MacroAssembler::call(JitCode* c) { + ScratchRegisterScope scratch(asMasm()); + BufferOffset bo = m_buffer.nextOffset(); + addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE); + ma_liPatchable(scratch, ImmPtr(c->raw())); + callJitNoProfiler(scratch); +} + +CodeOffset MacroAssembler::nopPatchableToCall() { + // LOONG64 + as_nop(); // lu12i_w + as_nop(); // ori + as_nop(); // lu32i_d + as_nop(); // jirl + return CodeOffset(currentOffset()); +} + +void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) { + Instruction* inst = (Instruction*)call - 4 /* four nops */; + Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)target); + inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); +} + +void MacroAssembler::patchCallToNop(uint8_t* call) { + Instruction* inst = (Instruction*)call - 4 /* four nops */; + inst[0].makeNop(); // lu12i_w + inst[1].makeNop(); // ori + inst[2].makeNop(); // lu32i_d + inst[3].makeNop(); // jirl +} + +void MacroAssembler::pushReturnAddress() { push(ra); } + +void MacroAssembler::popReturnAddress() { pop(ra); } + +// =============================================================== +// ABI function calls. + +void MacroAssembler::setupUnalignedABICall(Register scratch) { + MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls"); + setupNativeABICall(); + dynamicAlignment_ = true; + + as_or(scratch, StackPointer, zero); + + // Force sp to be aligned + asMasm().subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1))); + storePtr(scratch, Address(StackPointer, 0)); +} + +void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { + MOZ_ASSERT(inCall_); + uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar(); + + // Reserve place for $ra. + stackForCall += sizeof(intptr_t); + + if (dynamicAlignment_) { + stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment); + } else { + uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0; + stackForCall += ComputeByteAlignment( + stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment); + } + + *stackAdjust = stackForCall; + reserveStack(stackForCall); + + // Save $ra because call is going to clobber it. Restore it in + // callWithABIPost. NOTE: This is needed for calls from SharedIC. + // Maybe we can do this differently. + storePtr(ra, Address(StackPointer, stackForCall - sizeof(intptr_t))); + + // Position all arguments. + { + enoughMemory_ &= moveResolver_.resolve(); + if (!enoughMemory_) { + return; + } + + MoveEmitter emitter(*this); + emitter.emit(moveResolver_); + emitter.finish(); + } + + assertStackAlignment(ABIStackAlignment); +} + +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result, + bool callFromWasm) { + // Restore ra value (as stored in callWithABIPre()). + loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra); + + if (dynamicAlignment_) { + // Restore sp value from stack (as stored in setupUnalignedABICall()). + loadPtr(Address(StackPointer, stackAdjust), StackPointer); + // Use adjustFrame instead of freeStack because we already restored sp. + adjustFrame(-stackAdjust); + } else { + freeStack(stackAdjust); + } + +#ifdef DEBUG + MOZ_ASSERT(inCall_); + inCall_ = false; +#endif +} + +void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) { + SecondScratchRegisterScope scratch2(asMasm()); + // Load the callee in scratch2, no instruction between the movePtr and + // call should clobber it. Note that we can't use fun because it may be + // one of the IntArg registers clobbered before the call. + movePtr(fun, scratch2); + + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + call(scratch2); + callWithABIPost(stackAdjust, result); +} + +void MacroAssembler::callWithABINoProfiler(const Address& fun, + MoveOp::Type result) { + SecondScratchRegisterScope scratch2(asMasm()); + // Load the callee in scratch2, as above. + loadPtr(fun, scratch2); + + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + call(scratch2); + callWithABIPost(stackAdjust, result); +} + +// =============================================================== +// Jit Frames. + +uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) { + CodeLabel cl; + + ma_li(scratch, &cl); + Push(scratch); + bind(&cl); + uint32_t retAddr = currentOffset(); + + addCodeLabel(cl); + return retAddr; +} + +// =============================================================== +// Move instructions + +void MacroAssembler::moveValue(const TypedOrValueRegister& src, + const ValueOperand& dest) { + if (src.hasValue()) { + moveValue(src.valueReg(), dest); + return; + } + + MIRType type = src.type(); + AnyRegister reg = src.typedReg(); + + if (!IsFloatingPointType(type)) { + boxNonDouble(ValueTypeFromMIRType(type), reg.gpr(), dest); + return; + } + + FloatRegister scratch = ScratchDoubleReg; + FloatRegister freg = reg.fpu(); + if (type == MIRType::Float32) { + convertFloat32ToDouble(freg, scratch); + freg = scratch; + } + boxDouble(freg, dest, scratch); +} + +void MacroAssembler::moveValue(const ValueOperand& src, + const ValueOperand& dest) { + if (src == dest) { + return; + } + movePtr(src.valueReg(), dest.valueReg()); +} + +void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) { + if (!src.isGCThing()) { + ma_li(dest.valueReg(), ImmWord(src.asRawBits())); + return; + } + + writeDataRelocation(src); + movWithPatch(ImmWord(src.asRawBits()), dest.valueReg()); +} + +// =============================================================== +// Branch functions + +void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) { + if (ptr != buffer) { + movePtr(ptr, buffer); + } + orPtr(Imm32(gc::ChunkMask), buffer); + loadPtr(Address(buffer, gc::ChunkStoreBufferOffsetFromLastByte), buffer); +} + +void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, + Register temp, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + MOZ_ASSERT(ptr != temp); + MOZ_ASSERT(ptr != ScratchRegister && + ptr != SecondScratchReg); // Both may be used internally. + MOZ_ASSERT(temp != ScratchRegister && temp != SecondScratchReg); + MOZ_ASSERT(temp != InvalidReg); + + movePtr(ptr, temp); + orPtr(Imm32(gc::ChunkMask), temp); + branchPtr(InvertCondition(cond), + Address(temp, gc::ChunkStoreBufferOffsetFromLastByte), zero, label); +} + +void MacroAssembler::branchValueIsNurseryCell(Condition cond, + const Address& address, + Register temp, Label* label) { + branchValueIsNurseryCellImpl(cond, address, temp, label); +} + +void MacroAssembler::branchValueIsNurseryCell(Condition cond, + ValueOperand value, Register temp, + Label* label) { + branchValueIsNurseryCellImpl(cond, value, temp, label); +} + +template +void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, + const T& value, Register temp, + Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + MOZ_ASSERT(temp != InvalidReg); + Label done; + branchTestGCThing(Assembler::NotEqual, value, + cond == Assembler::Equal ? &done : label); + + unboxGCThingForGCBarrier(value, temp); + orPtr(Imm32(gc::ChunkMask), temp); + loadPtr(Address(temp, gc::ChunkStoreBufferOffsetFromLastByte), temp); + branchPtr(InvertCondition(cond), temp, zero, label); + + bind(&done); +} + +void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs, + const Value& rhs, Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs.valueReg() != scratch); + moveValue(rhs, ValueOperand(scratch)); + ma_b(lhs.valueReg(), scratch, label, cond); +} + +// ======================================================================== +// Memory access primitives. + +template +void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, const T& dest, + MIRType slotType) { + if (valueType == MIRType::Double) { + boxDouble(value.reg().typedReg().fpu(), dest); + return; + } + + // For known integers and booleans, we can just store the unboxed value if + // the slot has the same type. + if ((valueType == MIRType::Int32 || valueType == MIRType::Boolean) && + slotType == valueType) { + if (value.constant()) { + Value val = value.value(); + if (valueType == MIRType::Int32) { + store32(Imm32(val.toInt32()), dest); + } else { + store32(Imm32(val.toBoolean() ? 1 : 0), dest); + } + } else { + store32(value.reg().typedReg().gpr(), dest); + } + return; + } + + if (value.constant()) { + storeValue(value.value(), dest); + } else { + storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(), + dest); + } +} + +template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, + const Address& dest, + MIRType slotType); +template void MacroAssembler::storeUnboxedValue( + const ConstantOrRegister& value, MIRType valueType, + const BaseObjectElementIndex& dest, MIRType slotType); + +void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); } + +// =============================================================== +// WebAssembly + +CodeOffset MacroAssembler::wasmTrapInstruction() { + CodeOffset offset(currentOffset()); + as_break(WASM_TRAP); // TODO: as_teq(zero, zero, WASM_TRAP) + return offset; +} + +void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, + Register boundsCheckLimit, + Label* label) { + ma_b(index, boundsCheckLimit, label, cond); +} + +void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, + Address boundsCheckLimit, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(boundsCheckLimit, scratch2); + ma_b(index, Register(scratch2), label, cond); +} + +void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, + Register64 boundsCheckLimit, + Label* label) { + ma_b(index.reg, boundsCheckLimit.reg, label, cond); +} + +void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, + Address boundsCheckLimit, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(boundsCheckLimit, scratch2); + ma_b(index.reg, scratch2, label, cond); +} + +// FTINTRZ behaves as follows: +// +// on NaN it produces zero +// on too large it produces INT_MAX (for appropriate type) +// on too small it produces INT_MIN (ditto) + +void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + ScratchRegisterScope scratch(asMasm()); + if (!isSaturating) { + ma_bc1d(input, input, oolEntry, Assembler::DoubleUnordered); + } + as_ftintrz_l_d(ScratchDoubleReg, input); + moveFromDouble(ScratchDoubleReg, output); + as_srli_d(scratch, output, 32); + as_slli_w(output, output, 0); + ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); +} + +void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + ScratchRegisterScope scratch(asMasm()); + if (!isSaturating) { + ma_bc1s(input, input, oolEntry, Assembler::DoubleUnordered); + } + as_ftintrz_l_s(ScratchDoubleReg, input); + moveFromDouble(ScratchDoubleReg, output); + as_srli_d(scratch, output, 32); + as_slli_w(output, output, 0); + ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); +} + +// Assembler::CauseV is a enum,called FCSRBit. Assembler::CauseV == 16 +void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + ScratchRegisterScope scratch(asMasm()); + as_ftintrz_w_d(ScratchFloat32Reg, input); + as_movfcsr2gr(scratch); + moveFromFloat32(ScratchFloat32Reg, output); + MOZ_ASSERT(Assembler::CauseV < 32); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); +} + +void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + ScratchRegisterScope scratch(asMasm()); + as_ftintrz_w_s(ScratchFloat32Reg, input); + as_movfcsr2gr(scratch); + moveFromFloat32(ScratchFloat32Reg, output); + MOZ_ASSERT(Assembler::CauseV < 32); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); +} + +void MacroAssembler::wasmTruncateDoubleToUInt64( + FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempDouble) { + MOZ_ASSERT(tempDouble.isInvalid()); + Register output = output_.reg; + + Label done; + + if (!isSaturating) { + ma_bc1d(input, input, oolEntry, Assembler::DoubleUnordered); + } + as_ftintrz_l_d(ScratchDoubleReg, input); + moveFromDouble(ScratchDoubleReg, output); + loadConstantDouble(double(INT64_MAX + 1ULL), ScratchDoubleReg); + + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, ImmWord(INT64_MAX)); + // For numbers in -1.[ : ]INT64_MAX range do nothing more + ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump); + + ma_li(scratch2, ImmWord(INT64_MIN)); + as_fsub_d(ScratchDoubleReg, input, ScratchDoubleReg); + as_ftintrz_l_d(ScratchDoubleReg, ScratchDoubleReg); + as_movfcsr2gr(scratch); + moveFromDouble(ScratchDoubleReg, output); + as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + as_add_d(output, output, scratch2); + + // Guard against negative values that result in 0 due the precision loss. + as_sltui(scratch2, output, 1); + as_or(scratch, scratch, scratch2); + + ma_b(scratch, zero, oolEntry, Assembler::NotEqual); + + bind(&done); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::wasmTruncateFloat32ToUInt64( + FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempFloat) { + MOZ_ASSERT(tempFloat.isInvalid()); + Register output = output_.reg; + + Label done; + + if (!isSaturating) { + ma_bc1s(input, input, oolEntry, Assembler::DoubleUnordered); + } + as_ftintrz_l_s(ScratchDoubleReg, input); + moveFromDouble(ScratchDoubleReg, output); + loadConstantFloat32(float(INT64_MAX + 1ULL), ScratchFloat32Reg); + + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, ImmWord(INT64_MAX)); + // For numbers in -1.[ : ]INT64_MAX range do nothing more + ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump); + + ma_li(scratch2, ImmWord(INT64_MIN)); + as_fsub_s(ScratchFloat32Reg, input, ScratchFloat32Reg); + as_ftintrz_l_s(ScratchDoubleReg, ScratchFloat32Reg); + as_movfcsr2gr(scratch); + moveFromDouble(ScratchDoubleReg, output); + as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + as_add_d(output, output, scratch2); + + // Guard against negative values that result in 0 due the precision loss. + as_sltui(scratch2, output, 1); + as_or(scratch, scratch, scratch2); + + ma_b(scratch, zero, oolEntry, Assembler::NotEqual); + + bind(&done); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::wasmTruncateDoubleToInt64( + FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempDouble) { + MOZ_ASSERT(tempDouble.isInvalid()); + ScratchRegisterScope scratch(asMasm()); + + as_ftintrz_l_d(ScratchDoubleReg, input); + as_movfcsr2gr(scratch); + moveFromDouble(ScratchDoubleReg, output.reg); + as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, zero, oolEntry, Assembler::NotEqual); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::wasmTruncateFloat32ToInt64( + FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempFloat) { + MOZ_ASSERT(tempFloat.isInvalid()); + ScratchRegisterScope scratch(asMasm()); + + as_ftintrz_l_s(ScratchDoubleReg, input); + as_movfcsr2gr(scratch); + moveFromDouble(ScratchDoubleReg, output.reg); + as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, zero, oolEntry, Assembler::NotEqual); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input, + Register output, + TruncFlags flags, + wasm::BytecodeOffset off, + Label* rejoin) { + outOfLineWasmTruncateToInt32Check(input, output, MIRType::Float32, flags, + rejoin, off); +} + +void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input, + Register output, + TruncFlags flags, + wasm::BytecodeOffset off, + Label* rejoin) { + outOfLineWasmTruncateToInt32Check(input, output, MIRType::Double, flags, + rejoin, off); +} + +void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input, + Register64 output, + TruncFlags flags, + wasm::BytecodeOffset off, + Label* rejoin) { + outOfLineWasmTruncateToInt64Check(input, output, MIRType::Float32, flags, + rejoin, off); +} + +void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input, + Register64 output, + TruncFlags flags, + wasm::BytecodeOffset off, + Label* rejoin) { + outOfLineWasmTruncateToInt64Check(input, output, MIRType::Double, flags, + rejoin, off); +} + +void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, + Register memoryBase, Register ptr, + Register ptrScratch, AnyRegister output) { + wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg); +} + +void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, + Register memoryBase, Register ptr, + Register ptrScratch, Register64 output) { + wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg); +} + +void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, + AnyRegister value, Register memoryBase, + Register ptr, Register ptrScratch) { + wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg); +} + +void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, + Register64 value, Register memoryBase, + Register ptr, Register ptrScratch) { + wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg); +} + +void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch, + ExitFrameType type) { + enterFakeExitFrame(cxreg, scratch, type); +} + +// ======================================================================== +// Convert floating point. + +void MacroAssembler::convertUInt64ToFloat32(Register64 src_, FloatRegister dest, + Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + + Register src = src_.reg; + Label positive, done; + ma_b(src, src, &positive, NotSigned, ShortJump); + + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(src != scratch2); + + ma_and(scratch, src, Imm32(1)); + as_srli_d(scratch2, src, 1); + as_or(scratch, scratch, scratch2); + as_movgr2fr_d(dest, scratch); + as_ffint_s_l(dest, dest); + addFloat32(dest, dest); + ma_b(&done, ShortJump); + + bind(&positive); + as_movgr2fr_d(dest, src); + as_ffint_s_l(dest, dest); + + bind(&done); +} + +void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) { + as_movgr2fr_d(dest, src.reg); + as_ffint_s_l(dest, dest); +} + +bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; } + +void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, + Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + MacroAssemblerSpecific::convertUInt64ToDouble(src.reg, dest); +} + +void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) { + as_movgr2fr_d(dest, src.reg); + as_ffint_d_l(dest, dest); +} + +void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) { + convertInt64ToDouble(Register64(src), dest); +} + +// ======================================================================== +// Primitive atomic operations. + +template +static void CompareExchange(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + Scalar::Type type, const Synchronization& sync, + const T& mem, Register oldval, Register newval, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + switch (nbytes) { + case 1: + case 2: + break; + case 4: + MOZ_ASSERT(valueTemp == InvalidReg); + MOZ_ASSERT(offsetTemp == InvalidReg); + MOZ_ASSERT(maskTemp == InvalidReg); + break; + default: + MOZ_CRASH(); + } + + Label again, end; + + masm.computeEffectiveAddress(mem, scratch); + + if (nbytes == 4) { + masm.memoryBarrierBefore(sync); + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(output, scratch, 0); + masm.ma_b(output, oldval, &end, Assembler::NotEqual, ShortJump); + masm.as_or(scratch2, newval, zero); + masm.as_sc_w(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + masm.bind(&end); + + return; + } + + masm.as_andi(offsetTemp, scratch, 3); + masm.subPtr(offsetTemp, scratch); + masm.as_slli_w(offsetTemp, offsetTemp, 3); + masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); + masm.as_sll_w(maskTemp, maskTemp, offsetTemp); + masm.as_nor(maskTemp, zero, maskTemp); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(scratch2, scratch, 0); + + masm.as_srl_w(output, scratch2, offsetTemp); + + switch (nbytes) { + case 1: + if (signExtend) { + masm.as_ext_w_b(valueTemp, oldval); + masm.as_ext_w_b(output, output); + } else { + masm.as_andi(valueTemp, oldval, 0xff); + masm.as_andi(output, output, 0xff); + } + break; + case 2: + if (signExtend) { + masm.as_ext_w_h(valueTemp, oldval); + masm.as_ext_w_h(output, output); + } else { + masm.as_bstrpick_d(valueTemp, oldval, 15, 0); + masm.as_bstrpick_d(output, output, 15, 0); + } + break; + } + + masm.ma_b(output, valueTemp, &end, Assembler::NotEqual, ShortJump); + + masm.as_sll_w(valueTemp, newval, offsetTemp); + masm.as_and(scratch2, scratch2, maskTemp); + masm.as_or(scratch2, scratch2, valueTemp); + + masm.as_sc_w(scratch2, scratch, 0); + + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + + masm.bind(&end); +} + +template +static void CompareExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + const Synchronization& sync, const T& mem, + Register64 expect, Register64 replace, + Register64 output) { + MOZ_ASSERT(expect != output && replace != output); + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + masm.computeEffectiveAddress(mem, scratch); + + Label tryAgain; + Label exit; + + masm.memoryBarrierBefore(sync); + + masm.bind(&tryAgain); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_d(output.reg, scratch, 0); + + masm.ma_b(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump); + masm.movePtr(replace.reg, scratch2); + masm.as_sc_d(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero, + ShortJump); + + masm.memoryBarrierAfter(sync); + + masm.bind(&exit); +} + +template +static void AtomicExchange(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + Scalar::Type type, const Synchronization& sync, + const T& mem, Register value, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + switch (nbytes) { + case 1: + case 2: + break; + case 4: + MOZ_ASSERT(valueTemp == InvalidReg); + MOZ_ASSERT(offsetTemp == InvalidReg); + MOZ_ASSERT(maskTemp == InvalidReg); + break; + default: + MOZ_CRASH(); + } + + Label again; + + masm.computeEffectiveAddress(mem, scratch); + + if (nbytes == 4) { + masm.memoryBarrierBefore(sync); + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(output, scratch, 0); + masm.as_or(scratch2, value, zero); + masm.as_sc_w(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + + return; + } + + masm.as_andi(offsetTemp, scratch, 3); + masm.subPtr(offsetTemp, scratch); + masm.as_slli_w(offsetTemp, offsetTemp, 3); + masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); + masm.as_sll_w(maskTemp, maskTemp, offsetTemp); + masm.as_nor(maskTemp, zero, maskTemp); + switch (nbytes) { + case 1: + masm.as_andi(valueTemp, value, 0xff); + break; + case 2: + masm.as_bstrpick_d(valueTemp, value, 15, 0); + break; + } + masm.as_sll_w(valueTemp, valueTemp, offsetTemp); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(output, scratch, 0); + masm.as_and(scratch2, output, maskTemp); + masm.as_or(scratch2, scratch2, valueTemp); + + masm.as_sc_w(scratch2, scratch, 0); + + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.as_srl_w(output, output, offsetTemp); + + switch (nbytes) { + case 1: + if (signExtend) { + masm.as_ext_w_b(output, output); + } else { + masm.as_andi(output, output, 0xff); + } + break; + case 2: + if (signExtend) { + masm.as_ext_w_h(output, output); + } else { + masm.as_bstrpick_d(output, output, 15, 0); + } + break; + } + + masm.memoryBarrierAfter(sync); +} + +template +static void AtomicExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + const Synchronization& sync, const T& mem, + Register64 value, Register64 output) { + MOZ_ASSERT(value != output); + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + masm.computeEffectiveAddress(mem, scratch); + + Label tryAgain; + + masm.memoryBarrierBefore(sync); + + masm.bind(&tryAgain); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_d(output.reg, scratch, 0); + + masm.movePtr(value.reg, scratch2); + masm.as_sc_d(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero, + ShortJump); + + masm.memoryBarrierAfter(sync); +} + +template +static void AtomicFetchOp(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + Scalar::Type type, const Synchronization& sync, + AtomicOp op, const T& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + switch (nbytes) { + case 1: + case 2: + break; + case 4: + MOZ_ASSERT(valueTemp == InvalidReg); + MOZ_ASSERT(offsetTemp == InvalidReg); + MOZ_ASSERT(maskTemp == InvalidReg); + break; + default: + MOZ_CRASH(); + } + + Label again; + + masm.computeEffectiveAddress(mem, scratch); + + if (nbytes == 4) { + masm.memoryBarrierBefore(sync); + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(output, scratch, 0); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_w(scratch2, output, value); + break; + case AtomicFetchSubOp: + masm.as_sub_w(scratch2, output, value); + break; + case AtomicFetchAndOp: + masm.as_and(scratch2, output, value); + break; + case AtomicFetchOrOp: + masm.as_or(scratch2, output, value); + break; + case AtomicFetchXorOp: + masm.as_xor(scratch2, output, value); + break; + default: + MOZ_CRASH(); + } + + masm.as_sc_w(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + + return; + } + + masm.as_andi(offsetTemp, scratch, 3); + masm.subPtr(offsetTemp, scratch); + masm.as_slli_w(offsetTemp, offsetTemp, 3); + masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); + masm.as_sll_w(maskTemp, maskTemp, offsetTemp); + masm.as_nor(maskTemp, zero, maskTemp); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(scratch2, scratch, 0); + masm.as_srl_w(output, scratch2, offsetTemp); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_w(valueTemp, output, value); + break; + case AtomicFetchSubOp: + masm.as_sub_w(valueTemp, output, value); + break; + case AtomicFetchAndOp: + masm.as_and(valueTemp, output, value); + break; + case AtomicFetchOrOp: + masm.as_or(valueTemp, output, value); + break; + case AtomicFetchXorOp: + masm.as_xor(valueTemp, output, value); + break; + default: + MOZ_CRASH(); + } + + switch (nbytes) { + case 1: + masm.as_andi(valueTemp, valueTemp, 0xff); + break; + case 2: + masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0); + break; + } + + masm.as_sll_w(valueTemp, valueTemp, offsetTemp); + + masm.as_and(scratch2, scratch2, maskTemp); + masm.as_or(scratch2, scratch2, valueTemp); + + masm.as_sc_w(scratch2, scratch, 0); + + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + switch (nbytes) { + case 1: + if (signExtend) { + masm.as_ext_w_b(output, output); + } else { + masm.as_andi(output, output, 0xff); + } + break; + case 2: + if (signExtend) { + masm.as_ext_w_h(output, output); + } else { + masm.as_bstrpick_d(output, output, 15, 0); + } + break; + } + + masm.memoryBarrierAfter(sync); +} + +template +static void AtomicFetchOp64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + const Synchronization& sync, AtomicOp op, + Register64 value, const T& mem, Register64 temp, + Register64 output) { + MOZ_ASSERT(value != output); + MOZ_ASSERT(value != temp); + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + masm.computeEffectiveAddress(mem, scratch); + + Label tryAgain; + + masm.memoryBarrierBefore(sync); + + masm.bind(&tryAgain); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_d(output.reg, scratch, 0); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_d(temp.reg, output.reg, value.reg); + break; + case AtomicFetchSubOp: + masm.as_sub_d(temp.reg, output.reg, value.reg); + break; + case AtomicFetchAndOp: + masm.as_and(temp.reg, output.reg, value.reg); + break; + case AtomicFetchOrOp: + masm.as_or(temp.reg, output.reg, value.reg); + break; + case AtomicFetchXorOp: + masm.as_xor(temp.reg, output.reg, value.reg); + break; + default: + MOZ_CRASH(); + } + + masm.as_sc_d(temp.reg, scratch, 0); + masm.ma_b(temp.reg, temp.reg, &tryAgain, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); +} + +void MacroAssembler::compareExchange(Scalar::Type type, + const Synchronization& sync, + const Address& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, output); +} + +void MacroAssembler::compareExchange(Scalar::Type type, + const Synchronization& sync, + const BaseIndex& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, output); +} + +void MacroAssembler::compareExchange64(const Synchronization& sync, + const Address& mem, Register64 expect, + Register64 replace, Register64 output) { + CompareExchange64(*this, nullptr, sync, mem, expect, replace, output); +} + +void MacroAssembler::compareExchange64(const Synchronization& sync, + const BaseIndex& mem, Register64 expect, + Register64 replace, Register64 output) { + CompareExchange64(*this, nullptr, sync, mem, expect, replace, output); +} + +void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, + const Address& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, + newval, valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, + newval, valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, + const Address& mem, + Register64 expect, + Register64 replace, + Register64 output) { + CompareExchange64(*this, &access, access.sync(), mem, expect, replace, + output); +} + +void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, + Register64 expect, + Register64 replace, + Register64 output) { + CompareExchange64(*this, &access, access.sync(), mem, expect, replace, + output); +} + +void MacroAssembler::atomicExchange(Scalar::Type type, + const Synchronization& sync, + const Address& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, + maskTemp, output); +} + +void MacroAssembler::atomicExchange(Scalar::Type type, + const Synchronization& sync, + const BaseIndex& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, + maskTemp, output); +} + +void MacroAssembler::atomicExchange64(const Synchronization& sync, + const Address& mem, Register64 value, + Register64 output) { + AtomicExchange64(*this, nullptr, sync, mem, value, output); +} + +void MacroAssembler::atomicExchange64(const Synchronization& sync, + const BaseIndex& mem, Register64 value, + Register64 output) { + AtomicExchange64(*this, nullptr, sync, mem, value, output); +} + +void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, + const Address& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, + valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, + valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::atomicFetchOp(Scalar::Type type, + const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, + offsetTemp, maskTemp, output); +} + +void MacroAssembler::atomicFetchOp(Scalar::Type type, + const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, + offsetTemp, maskTemp, output); +} + +void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, + Register64 value, const Address& mem, + Register64 temp, Register64 output) { + AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output); +} + +void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, + Register64 value, const BaseIndex& mem, + Register64 temp, Register64 output) { + AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output); +} + +void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op, + Register64 value, const Address& mem, + Register64 temp) { + AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp); +} + +void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op, + Register64 value, const BaseIndex& mem, + Register64 temp) { + AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp); +} + +void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register value, + const Address& mem, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, + valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register value, + const BaseIndex& mem, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, + valueTemp, offsetTemp, maskTemp, output); +} + +template +static void AtomicEffectOp(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + Scalar::Type type, const Synchronization& sync, + AtomicOp op, const T& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp) { + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + unsigned nbytes = Scalar::byteSize(type); + + switch (nbytes) { + case 1: + case 2: + break; + case 4: + MOZ_ASSERT(valueTemp == InvalidReg); + MOZ_ASSERT(offsetTemp == InvalidReg); + MOZ_ASSERT(maskTemp == InvalidReg); + break; + default: + MOZ_CRASH(); + } + + Label again; + + masm.computeEffectiveAddress(mem, scratch); + + if (nbytes == 4) { + masm.memoryBarrierBefore(sync); + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(scratch2, scratch, 0); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_w(scratch2, scratch2, value); + break; + case AtomicFetchSubOp: + masm.as_sub_w(scratch2, scratch2, value); + break; + case AtomicFetchAndOp: + masm.as_and(scratch2, scratch2, value); + break; + case AtomicFetchOrOp: + masm.as_or(scratch2, scratch2, value); + break; + case AtomicFetchXorOp: + masm.as_xor(scratch2, scratch2, value); + break; + default: + MOZ_CRASH(); + } + + masm.as_sc_w(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + + return; + } + + masm.as_andi(offsetTemp, scratch, 3); + masm.subPtr(offsetTemp, scratch); + masm.as_slli_w(offsetTemp, offsetTemp, 3); + masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); + masm.as_sll_w(maskTemp, maskTemp, offsetTemp); + masm.as_nor(maskTemp, zero, maskTemp); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(scratch2, scratch, 0); + masm.as_srl_w(valueTemp, scratch2, offsetTemp); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_w(valueTemp, valueTemp, value); + break; + case AtomicFetchSubOp: + masm.as_sub_w(valueTemp, valueTemp, value); + break; + case AtomicFetchAndOp: + masm.as_and(valueTemp, valueTemp, value); + break; + case AtomicFetchOrOp: + masm.as_or(valueTemp, valueTemp, value); + break; + case AtomicFetchXorOp: + masm.as_xor(valueTemp, valueTemp, value); + break; + default: + MOZ_CRASH(); + } + + switch (nbytes) { + case 1: + masm.as_andi(valueTemp, valueTemp, 0xff); + break; + case 2: + masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0); + break; + } + + masm.as_sll_w(valueTemp, valueTemp, offsetTemp); + + masm.as_and(scratch2, scratch2, maskTemp); + masm.as_or(scratch2, scratch2, valueTemp); + + masm.as_sc_w(scratch2, scratch, 0); + + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); +} + +void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register value, + const Address& mem, Register valueTemp, + Register offsetTemp, + Register maskTemp) { + AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value, + valueTemp, offsetTemp, maskTemp); +} + +void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register value, + const BaseIndex& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp) { + AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value, + valueTemp, offsetTemp, maskTemp); +} + +template +static void WasmAtomicExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc& access, + const T& mem, Register64 value, + Register64 output) { + AtomicExchange64(masm, &access, access.sync(), mem, value, output); +} + +void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, + const Address& mem, Register64 src, + Register64 output) { + WasmAtomicExchange64(*this, access, mem, src, output); +} + +void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, Register64 src, + Register64 output) { + WasmAtomicExchange64(*this, access, mem, src, output); +} + +void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register64 value, + const Address& mem, Register64 temp, + Register64 output) { + AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output); +} + +void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register64 value, + const BaseIndex& mem, Register64 temp, + Register64 output) { + AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output); +} + +// ======================================================================== +// JS atomic operations. + +template +static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, + const Synchronization& sync, const T& mem, + Register oldval, Register newval, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + if (arrayType == Scalar::Uint32) { + masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, temp); + masm.convertUInt32ToDouble(temp, output.fpu()); + } else { + masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, output.gpr()); + } +} + +template +static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, + const Synchronization& sync, const T& mem, + Register value, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register temp, AnyRegister output) { + if (arrayType == Scalar::Uint32) { + masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, + maskTemp, temp); + masm.convertUInt32ToDouble(temp, output.fpu()); + } else { + masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, + maskTemp, output.gpr()); + } +} + +template +static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const T& mem, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register temp, AnyRegister output) { + if (arrayType == Scalar::Uint32) { + masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, + maskTemp, temp); + masm.convertUInt32ToDouble(temp, output.fpu()); + } else { + masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, + maskTemp, output.gpr()); + } +} + +void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, + const Synchronization& sync, + const Address& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register temp, AnyRegister output) { + CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, temp, output); +} + +void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, + const Synchronization& sync, + const BaseIndex& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register temp, AnyRegister output) { + CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, temp, output); +} + +void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, + const Synchronization& sync, + const Address& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, + maskTemp, temp, output); +} + +void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, + const Synchronization& sync, + const BaseIndex& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, + maskTemp, temp, output); +} + +void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, + maskTemp, temp, output); +} + +void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, + maskTemp, temp, output); +} + +void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp) { + AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp, + offsetTemp, maskTemp); +} + +void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp) { + AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp, + offsetTemp, maskTemp); +} + +void MacroAssembler::flexibleQuotient32(Register rhs, Register srcDest, + bool isUnsigned, + const LiveRegisterSet&) { + quotient32(rhs, srcDest, isUnsigned); +} + +void MacroAssembler::flexibleRemainder32(Register rhs, Register srcDest, + bool isUnsigned, + const LiveRegisterSet&) { + remainder32(rhs, srcDest, isUnsigned); +} + +void MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest, + Register remOutput, bool isUnsigned, + const LiveRegisterSet&) { + if (isUnsigned) { + as_mod_wu(remOutput, srcDest, rhs); + as_div_wu(srcDest, srcDest, rhs); + } else { + as_mod_w(remOutput, srcDest, rhs); + as_div_w(srcDest, srcDest, rhs); + } +} + +CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) { + return movWithPatch(ImmPtr(nullptr), dest); +} + +void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc, + CodeLocationLabel target) { + PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr)); +} + +// ======================================================================== +// Spectre Mitigations. + +void MacroAssembler::speculationBarrier() { MOZ_CRASH(); } + +void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + FloatRegister scratch = ScratchFloat32Reg; + Label skipCheck, done; + + // If Nan, 0 or -0 check for bailout + loadConstantFloat32(0.0f, scratch); + ma_bc1s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); + + // If high part is not zero, it is NaN or -0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromDoubleLo(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&done, ShortJump); + + bind(&skipCheck); + as_ftintrm_w_s(scratch, src); + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + bind(&done); +} + +void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + FloatRegister scratch = ScratchDoubleReg; + Label skipCheck, done; + + // If Nan, 0 or -0 check for bailout + loadConstantDouble(0.0, scratch); + ma_bc1d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); + + // If high part is not zero, it is NaN or -0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromDoubleHi(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&done, ShortJump); + + bind(&skipCheck); + as_ftintrm_w_d(scratch, src); + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + bind(&done); +} + +void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + FloatRegister scratch = ScratchFloat32Reg; + Label performCeil, done; + + // If x < -1 or x > 0 then perform ceil. + loadConstantFloat32(0.0f, scratch); + branchFloat(Assembler::DoubleGreaterThan, src, scratch, &performCeil); + loadConstantFloat32(-1.0f, scratch); + branchFloat(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil); + + // If binary value is not zero, the input was not 0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromFloat32(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&done, ShortJump); + + bind(&performCeil); + as_ftintrp_w_s(scratch, src); + moveFromFloat32(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + bind(&done); +} + +void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + FloatRegister scratch = ScratchDoubleReg; + Label performCeil, done; + + // If x < -1 or x > 0 then perform ceil. + loadConstantDouble(0, scratch); + branchDouble(Assembler::DoubleGreaterThan, src, scratch, &performCeil); + loadConstantDouble(-1.0, scratch); + branchDouble(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil); + + // If binary value is not zero, the input was not 0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromDoubleHi(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&done, ShortJump); + + bind(&performCeil); + as_ftintrp_w_d(scratch, src); + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + bind(&done); +} + +void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest, + FloatRegister temp, Label* fail) { + ScratchFloat32Scope scratch(*this); + + Label negative, end, skipCheck; + + // Load biggest number less than 0.5 in the temp register. + loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp); + + // Branch to a slow path for negative inputs. Doesn't catch NaN or -0. + loadConstantFloat32(0.0f, scratch); + ma_bc1s(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump); + + // If Nan, 0 or -0 check for bailout + ma_bc1s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); + + // If binary value is not zero, it is NaN or -0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromFloat32(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&end, ShortJump); + + bind(&skipCheck); + as_fadd_s(scratch, src, temp); + as_ftintrm_w_s(scratch, scratch); + + moveFromFloat32(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + jump(&end); + + // Input is negative, but isn't -0. + bind(&negative); + + // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to + // be added the biggest double less than 0.5. + Label loadJoin; + loadConstantFloat32(-0.5f, scratch); + branchFloat(Assembler::DoubleLessThan, src, scratch, &loadJoin); + loadConstantFloat32(0.5f, temp); + bind(&loadJoin); + + as_fadd_s(temp, src, temp); + + // If input + 0.5 >= 0, input is a negative number >= -0.5 and the + // result is -0. + branchFloat(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail); + + // Truncate and round toward zero. + // This is off-by-one for everything but integer-valued inputs. + as_ftintrm_w_s(scratch, temp); + moveFromFloat32(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + + bind(&end); +} + +void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest, + FloatRegister temp, Label* fail) { + ScratchDoubleScope scratch(*this); + + Label negative, end, skipCheck; + + // Load biggest number less than 0.5 in the temp register. + loadConstantDouble(GetBiggestNumberLessThan(0.5), temp); + + // Branch to a slow path for negative inputs. Doesn't catch NaN or -0. + loadConstantDouble(0.0, scratch); + ma_bc1d(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump); + + // If Nan, 0 or -0 check for bailout + ma_bc1d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); + + // If high part is not zero, it is NaN or -0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromDoubleHi(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&end, ShortJump); + + bind(&skipCheck); + as_fadd_d(scratch, src, temp); + as_ftintrm_w_d(scratch, scratch); + + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + jump(&end); + + // Input is negative, but isn't -0. + bind(&negative); + + // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to + // be added the biggest double less than 0.5. + Label loadJoin; + loadConstantDouble(-0.5, scratch); + branchDouble(Assembler::DoubleLessThan, src, scratch, &loadJoin); + loadConstantDouble(0.5, temp); + bind(&loadJoin); + + addDouble(src, temp); + + // If input + 0.5 >= 0, input is a negative number >= -0.5 and the + // result is -0. + branchDouble(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail); + + // Truncate and round toward zero. + // This is off-by-one for everything but integer-valued inputs. + as_ftintrm_w_d(scratch, temp); + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + + bind(&end); +} + +void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchRegisterScope scratch(asMasm()); + + Label notZero; + as_ftintrz_w_s(ScratchFloat32Reg, src); + as_movfcsr2gr(scratch); + moveFromFloat32(ScratchFloat32Reg, dest); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); + + { + // dest == zero + SecondScratchRegisterScope scratch2(asMasm()); + moveFromFloat32(src, scratch2); + // Check if input is in ]-1; -0] range by checking the sign bit. + as_slt(scratch2, scratch2, zero); + as_add_d(scratch, scratch, scratch2); + } + + bind(¬Zero); + branch32(Assembler::NotEqual, Register(scratch), zero, fail); +} + +void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchRegisterScope scratch(asMasm()); + + Label notZero; + as_ftintrz_w_d(ScratchFloat32Reg, src); + as_movfcsr2gr(scratch); + moveFromFloat32(ScratchFloat32Reg, dest); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); + + { + // dest == zero + SecondScratchRegisterScope scratch2(asMasm()); + moveFromDoubleHi(src, scratch2); + // Check if input is in ]-1; -0] range by checking the sign bit. + as_slt(scratch2, scratch2, zero); + as_add_d(scratch, scratch, scratch2); + } + + bind(¬Zero); + branch32(Assembler::NotEqual, Register(scratch), zero, fail); +} + +void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src, + FloatRegister dest) { + MOZ_CRASH("not supported on this platform"); +} + +void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src, + FloatRegister dest) { + MOZ_CRASH("not supported on this platform"); +} + +void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs, + FloatRegister output) { + MOZ_CRASH("not supported on this platform"); +} + +void MacroAssemblerLOONG64Compat::move32(Imm32 imm, Register dest) { + ma_li(dest, imm); +} + +void MacroAssemblerLOONG64Compat::move32(Register src, Register dest) { + as_slli_w(dest, src, 0); +} + +void MacroAssemblerLOONG64Compat::movePtr(Register src, Register dest) { + as_or(dest, src, zero); +} +void MacroAssemblerLOONG64Compat::movePtr(ImmWord imm, Register dest) { + ma_li(dest, imm); +} + +void MacroAssemblerLOONG64Compat::movePtr(ImmGCPtr imm, Register dest) { + ma_li(dest, imm); +} + +void MacroAssemblerLOONG64Compat::movePtr(ImmPtr imm, Register dest) { + movePtr(ImmWord(uintptr_t(imm.value)), dest); +} + +void MacroAssemblerLOONG64Compat::movePtr(wasm::SymbolicAddress imm, + Register dest) { + append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm)); + ma_liPatchable(dest, ImmWord(-1)); +} + +void MacroAssemblerLOONG64Compat::load8ZeroExtend(const Address& address, + Register dest) { + ma_load(dest, address, SizeByte, ZeroExtend); +} + +void MacroAssemblerLOONG64Compat::load8ZeroExtend(const BaseIndex& src, + Register dest) { + ma_load(dest, src, SizeByte, ZeroExtend); +} + +void MacroAssemblerLOONG64Compat::load8SignExtend(const Address& address, + Register dest) { + ma_load(dest, address, SizeByte, SignExtend); +} + +void MacroAssemblerLOONG64Compat::load8SignExtend(const BaseIndex& src, + Register dest) { + ma_load(dest, src, SizeByte, SignExtend); +} + +void MacroAssemblerLOONG64Compat::load16ZeroExtend(const Address& address, + Register dest) { + ma_load(dest, address, SizeHalfWord, ZeroExtend); +} + +void MacroAssemblerLOONG64Compat::load16ZeroExtend(const BaseIndex& src, + Register dest) { + ma_load(dest, src, SizeHalfWord, ZeroExtend); +} + +void MacroAssemblerLOONG64Compat::load16SignExtend(const Address& address, + Register dest) { + ma_load(dest, address, SizeHalfWord, SignExtend); +} + +void MacroAssemblerLOONG64Compat::load16SignExtend(const BaseIndex& src, + Register dest) { + ma_load(dest, src, SizeHalfWord, SignExtend); +} + +void MacroAssemblerLOONG64Compat::load32(const Address& address, + Register dest) { + ma_ld_w(dest, address); +} + +void MacroAssemblerLOONG64Compat::load32(const BaseIndex& address, + Register dest) { + Register base = address.base; + Register index = address.index; + int32_t offset = address.offset; + uint32_t shift = Imm32::ShiftOf(address.scale).value; + + if (offset != 0) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, Imm32(offset)); + if (shift != 0) { + MOZ_ASSERT(shift <= 4); + as_alsl_d(scratch, index, scratch, shift - 1); + } else { + as_add_d(scratch, index, scratch); + } + as_ldx_w(dest, base, scratch); + } else if (shift != 0) { + ScratchRegisterScope scratch(asMasm()); + as_slli_d(scratch, index, shift); + as_ldx_w(dest, base, scratch); + } else { + as_ldx_w(dest, base, index); + } +} + +void MacroAssemblerLOONG64Compat::load32(AbsoluteAddress address, + Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(ImmPtr(address.addr), scratch); + load32(Address(scratch, 0), dest); +} + +void MacroAssemblerLOONG64Compat::load32(wasm::SymbolicAddress address, + Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(address, scratch); + load32(Address(scratch, 0), dest); +} + +void MacroAssemblerLOONG64Compat::loadPtr(const Address& address, + Register dest) { + ma_ld_d(dest, address); +} + +void MacroAssemblerLOONG64Compat::loadPtr(const BaseIndex& src, Register dest) { + Register base = src.base; + Register index = src.index; + int32_t offset = src.offset; + uint32_t shift = Imm32::ShiftOf(src.scale).value; + + if (offset != 0) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, Imm32(offset)); + if (shift != 0) { + MOZ_ASSERT(shift <= 4); + as_alsl_d(scratch, index, scratch, shift - 1); + } else { + as_add_d(scratch, index, scratch); + } + as_ldx_d(dest, base, scratch); + } else if (shift != 0) { + ScratchRegisterScope scratch(asMasm()); + as_slli_d(scratch, index, shift); + as_ldx_d(dest, base, scratch); + } else { + as_ldx_d(dest, base, index); + } +} + +void MacroAssemblerLOONG64Compat::loadPtr(AbsoluteAddress address, + Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(ImmPtr(address.addr), scratch); + loadPtr(Address(scratch, 0), dest); +} + +void MacroAssemblerLOONG64Compat::loadPtr(wasm::SymbolicAddress address, + Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(address, scratch); + loadPtr(Address(scratch, 0), dest); +} + +void MacroAssemblerLOONG64Compat::loadPrivate(const Address& address, + Register dest) { + loadPtr(address, dest); +} + +void MacroAssemblerLOONG64Compat::store8(Imm32 imm, const Address& address) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_store(scratch2, address, SizeByte); +} + +void MacroAssemblerLOONG64Compat::store8(Register src, const Address& address) { + ma_store(src, address, SizeByte); +} + +void MacroAssemblerLOONG64Compat::store8(Imm32 imm, const BaseIndex& dest) { + ma_store(imm, dest, SizeByte); +} + +void MacroAssemblerLOONG64Compat::store8(Register src, const BaseIndex& dest) { + ma_store(src, dest, SizeByte); +} + +void MacroAssemblerLOONG64Compat::store16(Imm32 imm, const Address& address) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_store(scratch2, address, SizeHalfWord); +} + +void MacroAssemblerLOONG64Compat::store16(Register src, + const Address& address) { + ma_store(src, address, SizeHalfWord); +} + +void MacroAssemblerLOONG64Compat::store16(Imm32 imm, const BaseIndex& dest) { + ma_store(imm, dest, SizeHalfWord); +} + +void MacroAssemblerLOONG64Compat::store16(Register src, + const BaseIndex& address) { + ma_store(src, address, SizeHalfWord); +} + +void MacroAssemblerLOONG64Compat::store32(Register src, + AbsoluteAddress address) { + ScratchRegisterScope scratch(asMasm()); + movePtr(ImmPtr(address.addr), scratch); + store32(src, Address(scratch, 0)); +} + +void MacroAssemblerLOONG64Compat::store32(Register src, + const Address& address) { + ma_store(src, address, SizeWord); +} + +void MacroAssemblerLOONG64Compat::store32(Imm32 src, const Address& address) { + SecondScratchRegisterScope scratch2(asMasm()); + move32(src, scratch2); + ma_store(scratch2, address, SizeWord); +} + +void MacroAssemblerLOONG64Compat::store32(Imm32 imm, const BaseIndex& dest) { + ma_store(imm, dest, SizeWord); +} + +void MacroAssemblerLOONG64Compat::store32(Register src, const BaseIndex& dest) { + ma_store(src, dest, SizeWord); +} + +template +void MacroAssemblerLOONG64Compat::storePtr(ImmWord imm, T address) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_store(scratch2, address, SizeDouble); +} + +template void MacroAssemblerLOONG64Compat::storePtr
(ImmWord imm, + Address address); +template void MacroAssemblerLOONG64Compat::storePtr( + ImmWord imm, BaseIndex address); + +template +void MacroAssemblerLOONG64Compat::storePtr(ImmPtr imm, T address) { + storePtr(ImmWord(uintptr_t(imm.value)), address); +} + +template void MacroAssemblerLOONG64Compat::storePtr
(ImmPtr imm, + Address address); +template void MacroAssemblerLOONG64Compat::storePtr( + ImmPtr imm, BaseIndex address); + +template +void MacroAssemblerLOONG64Compat::storePtr(ImmGCPtr imm, T address) { + SecondScratchRegisterScope scratch2(asMasm()); + movePtr(imm, scratch2); + storePtr(scratch2, address); +} + +template void MacroAssemblerLOONG64Compat::storePtr
(ImmGCPtr imm, + Address address); +template void MacroAssemblerLOONG64Compat::storePtr( + ImmGCPtr imm, BaseIndex address); + +void MacroAssemblerLOONG64Compat::storePtr(Register src, + const Address& address) { + ma_st_d(src, address); +} + +void MacroAssemblerLOONG64Compat::storePtr(Register src, + const BaseIndex& address) { + Register base = address.base; + Register index = address.index; + int32_t offset = address.offset; + int32_t shift = Imm32::ShiftOf(address.scale).value; + + if ((offset == 0) && (shift == 0)) { + as_stx_d(src, base, index); + } else if (is_intN(offset, 12)) { + ScratchRegisterScope scratch(asMasm()); + if (shift == 0) { + as_add_d(scratch, base, index); + } else { + as_alsl_d(scratch, index, base, shift - 1); + } + as_st_d(src, scratch, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, Imm32(offset)); + if (shift == 0) { + as_add_d(scratch, scratch, index); + } else { + as_alsl_d(scratch, index, scratch, shift - 1); + } + as_stx_d(src, base, scratch); + } +} + +void MacroAssemblerLOONG64Compat::storePtr(Register src, AbsoluteAddress dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(ImmPtr(dest.addr), scratch); + storePtr(src, Address(scratch, 0)); +} + +void MacroAssemblerLOONG64Compat::testNullSet(Condition cond, + const ValueOperand& value, + Register dest) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(asMasm()); + splitTag(value, scratch2); + ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_NULL), cond); +} + +void MacroAssemblerLOONG64Compat::testObjectSet(Condition cond, + const ValueOperand& value, + Register dest) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(asMasm()); + splitTag(value, scratch2); + ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_OBJECT), cond); +} + +void MacroAssemblerLOONG64Compat::testUndefinedSet(Condition cond, + const ValueOperand& value, + Register dest) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(asMasm()); + splitTag(value, scratch2); + ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_UNDEFINED), cond); +} + +void MacroAssemblerLOONG64Compat::unboxInt32(const ValueOperand& operand, + Register dest) { + as_slli_w(dest, operand.valueReg(), 0); +} + +void MacroAssemblerLOONG64Compat::unboxInt32(Register src, Register dest) { + as_slli_w(dest, src, 0); +} + +void MacroAssemblerLOONG64Compat::unboxInt32(const Address& src, + Register dest) { + load32(Address(src.base, src.offset), dest); +} + +void MacroAssemblerLOONG64Compat::unboxInt32(const BaseIndex& src, + Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + computeScaledAddress(src, scratch2); + load32(Address(scratch2, src.offset), dest); +} + +void MacroAssemblerLOONG64Compat::unboxBoolean(const ValueOperand& operand, + Register dest) { + as_slli_w(dest, operand.valueReg(), 0); +} + +void MacroAssemblerLOONG64Compat::unboxBoolean(Register src, Register dest) { + as_slli_w(dest, src, 0); +} + +void MacroAssemblerLOONG64Compat::unboxBoolean(const Address& src, + Register dest) { + ma_load(dest, Address(src.base, src.offset), SizeWord, ZeroExtend); +} + +void MacroAssemblerLOONG64Compat::unboxBoolean(const BaseIndex& src, + Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + computeScaledAddress(src, scratch2); + ma_load(dest, Address(scratch2, src.offset), SizeWord, ZeroExtend); +} + +void MacroAssemblerLOONG64Compat::unboxDouble(const ValueOperand& operand, + FloatRegister dest) { + as_movgr2fr_d(dest, operand.valueReg()); +} + +void MacroAssemblerLOONG64Compat::unboxDouble(const Address& src, + FloatRegister dest) { + ma_fld_d(dest, Address(src.base, src.offset)); +} + +void MacroAssemblerLOONG64Compat::unboxDouble(const BaseIndex& src, + FloatRegister dest) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(src, scratch2); + unboxDouble(ValueOperand(scratch2), dest); +} + +void MacroAssemblerLOONG64Compat::unboxString(const ValueOperand& operand, + Register dest) { + unboxNonDouble(operand, dest, JSVAL_TYPE_STRING); +} + +void MacroAssemblerLOONG64Compat::unboxString(Register src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_STRING); +} + +void MacroAssemblerLOONG64Compat::unboxString(const Address& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_STRING); +} + +void MacroAssemblerLOONG64Compat::unboxSymbol(const ValueOperand& operand, + Register dest) { + unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL); +} + +void MacroAssemblerLOONG64Compat::unboxSymbol(Register src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); +} + +void MacroAssemblerLOONG64Compat::unboxSymbol(const Address& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); +} + +void MacroAssemblerLOONG64Compat::unboxBigInt(const ValueOperand& operand, + Register dest) { + unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT); +} + +void MacroAssemblerLOONG64Compat::unboxBigInt(Register src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); +} + +void MacroAssemblerLOONG64Compat::unboxBigInt(const Address& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); +} + +void MacroAssemblerLOONG64Compat::unboxObject(const ValueOperand& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); +} + +void MacroAssemblerLOONG64Compat::unboxObject(Register src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); +} + +void MacroAssemblerLOONG64Compat::unboxObject(const Address& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); +} + +void MacroAssemblerLOONG64Compat::unboxValue(const ValueOperand& src, + AnyRegister dest, + JSValueType type) { + if (dest.isFloat()) { + Label notInt32, end; + asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); + convertInt32ToDouble(src.valueReg(), dest.fpu()); + ma_b(&end, ShortJump); + bind(¬Int32); + unboxDouble(src, dest.fpu()); + bind(&end); + } else { + unboxNonDouble(src, dest.gpr(), type); + } +} + +void MacroAssemblerLOONG64Compat::boxDouble(FloatRegister src, + const ValueOperand& dest, + FloatRegister) { + as_movfr2gr_d(dest.valueReg(), src); +} + +void MacroAssemblerLOONG64Compat::boxNonDouble(JSValueType type, Register src, + const ValueOperand& dest) { + boxValue(type, src, dest.valueReg()); +} + +void MacroAssemblerLOONG64Compat::boolValueToDouble(const ValueOperand& operand, + FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + convertBoolToInt32(operand.valueReg(), scratch); + convertInt32ToDouble(scratch, dest); +} + +void MacroAssemblerLOONG64Compat::int32ValueToDouble( + const ValueOperand& operand, FloatRegister dest) { + convertInt32ToDouble(operand.valueReg(), dest); +} + +void MacroAssemblerLOONG64Compat::boolValueToFloat32( + const ValueOperand& operand, FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + convertBoolToInt32(operand.valueReg(), scratch); + convertInt32ToFloat32(scratch, dest); +} + +void MacroAssemblerLOONG64Compat::int32ValueToFloat32( + const ValueOperand& operand, FloatRegister dest) { + convertInt32ToFloat32(operand.valueReg(), dest); +} + +void MacroAssemblerLOONG64Compat::loadConstantFloat32(float f, + FloatRegister dest) { + ma_lis(dest, f); +} + +void MacroAssemblerLOONG64Compat::loadInt32OrDouble(const Address& src, + FloatRegister dest) { + SecondScratchRegisterScope scratch2(asMasm()); + Label end; + + // If it's an int, convert it to double. + loadPtr(Address(src.base, src.offset), scratch2); + as_movgr2fr_d(dest, scratch2); + as_srli_d(scratch2, scratch2, JSVAL_TAG_SHIFT); + asMasm().branchTestInt32(Assembler::NotEqual, scratch2, &end); + as_ffint_d_w(dest, dest); + + bind(&end); +} + +void MacroAssemblerLOONG64Compat::loadInt32OrDouble(const BaseIndex& addr, + FloatRegister dest) { + SecondScratchRegisterScope scratch2(asMasm()); + Label end; + + // If it's an int, convert it to double. + computeScaledAddress(addr, scratch2); + // Since we only have one scratch, we need to stomp over it with the tag. + loadPtr(Address(scratch2, 0), scratch2); + as_movgr2fr_d(dest, scratch2); + as_srli_d(scratch2, scratch2, JSVAL_TAG_SHIFT); + asMasm().branchTestInt32(Assembler::NotEqual, scratch2, &end); + as_ffint_d_w(dest, dest); + + bind(&end); +} + +void MacroAssemblerLOONG64Compat::loadConstantDouble(double dp, + FloatRegister dest) { + ma_lid(dest, dp); +} + +Register MacroAssemblerLOONG64Compat::extractObject(const Address& address, + Register scratch) { + loadPtr(Address(address.base, address.offset), scratch); + as_bstrpick_d(scratch, scratch, JSVAL_TAG_SHIFT - 1, 0); + return scratch; +} + +Register MacroAssemblerLOONG64Compat::extractTag(const Address& address, + Register scratch) { + loadPtr(Address(address.base, address.offset), scratch); + as_bstrpick_d(scratch, scratch, 63, JSVAL_TAG_SHIFT); + return scratch; +} + +Register MacroAssemblerLOONG64Compat::extractTag(const BaseIndex& address, + Register scratch) { + computeScaledAddress(address, scratch); + return extractTag(Address(scratch, address.offset), scratch); +} + +///////////////////////////////////////////////////////////////// +// X86/X64-common/ARM/LoongArch interface. +///////////////////////////////////////////////////////////////// +void MacroAssemblerLOONG64Compat::storeValue(ValueOperand val, + const Address& dest) { + storePtr(val.valueReg(), Address(dest.base, dest.offset)); +} + +void MacroAssemblerLOONG64Compat::storeValue(ValueOperand val, + const BaseIndex& dest) { + storePtr(val.valueReg(), dest); +} + +void MacroAssemblerLOONG64Compat::storeValue(JSValueType type, Register reg, + Address dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(dest.base != scratch2); + + tagValue(type, reg, ValueOperand(scratch2)); + storePtr(scratch2, dest); +} + +void MacroAssemblerLOONG64Compat::storeValue(JSValueType type, Register reg, + BaseIndex dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(dest.base != scratch2); + + tagValue(type, reg, ValueOperand(scratch2)); + storePtr(scratch2, dest); +} + +void MacroAssemblerLOONG64Compat::storeValue(const Value& val, Address dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(dest.base != scratch2); + + if (val.isGCThing()) { + writeDataRelocation(val); + movWithPatch(ImmWord(val.asRawBits()), scratch2); + } else { + ma_li(scratch2, ImmWord(val.asRawBits())); + } + storePtr(scratch2, dest); +} + +void MacroAssemblerLOONG64Compat::storeValue(const Value& val, BaseIndex dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(dest.base != scratch2); + + if (val.isGCThing()) { + writeDataRelocation(val); + movWithPatch(ImmWord(val.asRawBits()), scratch2); + } else { + ma_li(scratch2, ImmWord(val.asRawBits())); + } + storePtr(scratch2, dest); +} + +void MacroAssemblerLOONG64Compat::loadValue(Address src, ValueOperand val) { + loadPtr(src, val.valueReg()); +} + +void MacroAssemblerLOONG64Compat::loadValue(const BaseIndex& src, + ValueOperand val) { + loadPtr(src, val.valueReg()); +} + +void MacroAssemblerLOONG64Compat::tagValue(JSValueType type, Register payload, + ValueOperand dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dest.valueReg() != scratch); + + if (payload == dest.valueReg()) { + as_or(scratch, payload, zero); + payload = scratch; + } + ma_li(dest.valueReg(), ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type))); + as_bstrins_d(dest.valueReg(), payload, JSVAL_TAG_SHIFT - 1, 0); +} + +void MacroAssemblerLOONG64Compat::pushValue(ValueOperand val) { + push(val.valueReg()); +} + +void MacroAssemblerLOONG64Compat::pushValue(const Address& addr) { push(addr); } + +void MacroAssemblerLOONG64Compat::popValue(ValueOperand val) { + pop(val.valueReg()); +} + +void MacroAssemblerLOONG64Compat::breakpoint(uint32_t value) { + as_break(value); +} + +void MacroAssemblerLOONG64Compat::handleFailureWithHandlerTail( + Label* profilerExitTail) { + // Reserve space for exception information. + int size = (sizeof(ResumeFromException) + ABIStackAlignment) & + ~(ABIStackAlignment - 1); + asMasm().subPtr(Imm32(size), StackPointer); + mov(StackPointer, a0); // Use a0 since it is a first function argument + + // Call the handler. + using Fn = void (*)(ResumeFromException * rfe); + asMasm().setupUnalignedABICall(a1); + asMasm().passABIArg(a0); + asMasm().callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + Label entryFrame; + Label catch_; + Label finally; + Label return_; + Label bailout; + Label wasm; + Label wasmCatch; + + // Already clobbered a0, so use it... + load32(Address(StackPointer, offsetof(ResumeFromException, kind)), a0); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_ENTRY_FRAME), + &entryFrame); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_CATCH), &catch_); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_FINALLY), &finally); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_FORCED_RETURN), &return_); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_BAILOUT), &bailout); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_WASM), &wasm); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_WASM_CATCH), &wasmCatch); + + breakpoint(); // Invalid kind. + + // No exception handler. Load the error value, load the new stack pointer + // and return from the entry frame. + bind(&entryFrame); + asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + + // We're going to be returning by the ion calling convention + ma_pop(ra); + as_jirl(zero, ra, BOffImm16(0)); + + // If we found a catch handler, this must be a baseline frame. Restore + // state and jump to the catch block. + bind(&catch_); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, target)), a0); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), + BaselineFrameReg); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + jump(a0); + + // If we found a finally block, this must be a baseline frame. Push + // two values expected by JSOp::Retsub: BooleanValue(true) and the + // exception. + bind(&finally); + ValueOperand exception = ValueOperand(a1); + loadValue(Address(sp, offsetof(ResumeFromException, exception)), exception); + + loadPtr(Address(sp, offsetof(ResumeFromException, target)), a0); + loadPtr(Address(sp, offsetof(ResumeFromException, framePointer)), + BaselineFrameReg); + loadPtr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp); + + pushValue(BooleanValue(true)); + pushValue(exception); + jump(a0); + + // Only used in debug mode. Return BaselineFrame->returnValue() to the + // caller. + bind(&return_); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), + BaselineFrameReg); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + loadValue( + Address(BaselineFrameReg, BaselineFrame::reverseOffsetOfReturnValue()), + JSReturnOperand); + as_or(StackPointer, BaselineFrameReg, zero); + pop(BaselineFrameReg); + + // If profiling is enabled, then update the lastProfilingFrame to refer to + // caller frame before returning. + { + Label skipProfilingInstrumentation; + // Test if profiler enabled. + AbsoluteAddress addressOfEnabled( + GetJitContext()->runtime->geckoProfiler().addressOfEnabled()); + asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + jump(profilerExitTail); + bind(&skipProfilingInstrumentation); + } + + ret(); + + // If we are bailing out to baseline to handle an exception, jump to + // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success. + bind(&bailout); + loadPtr(Address(sp, offsetof(ResumeFromException, bailoutInfo)), a2); + ma_li(ReturnReg, Imm32(1)); + loadPtr(Address(sp, offsetof(ResumeFromException, target)), a1); + jump(a1); + + // If we are throwing and the innermost frame was a wasm frame, reset SP and + // FP; SP is pointing to the unwound return address to the wasm entry, so + // we can just ret(). + bind(&wasm); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), + FramePointer); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + ret(); + + // Found a wasm catch handler, restore state and jump to it. + bind(&wasmCatch); + loadPtr(Address(sp, offsetof(ResumeFromException, target)), a1); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), + FramePointer); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + jump(a1); +} + +CodeOffset MacroAssemblerLOONG64Compat::toggledJump(Label* label) { + CodeOffset ret(nextOffset().getOffset()); + ma_b(label); + return ret; +} + +CodeOffset MacroAssemblerLOONG64Compat::toggledCall(JitCode* target, + bool enabled) { + ScratchRegisterScope scratch(asMasm()); + BufferOffset bo = nextOffset(); + CodeOffset offset(bo.getOffset()); // first instruction location,not changed. + addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE); + ma_liPatchable(scratch, ImmPtr(target->raw())); + if (enabled) { + as_jirl(ra, scratch, BOffImm16(0)); + } else { + as_nop(); + } + MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() == + ToggledCallSize(nullptr)); + return offset; // location of first instruction of call instr sequence. +} + +// Stack manipulation functions. + +//}}} check_macroassembler_style + +} // namespace jit +} // namespace js diff --git a/js/src/jit/loong64/MacroAssembler-loong64.h b/js/src/jit/loong64/MacroAssembler-loong64.h new file mode 100644 index 0000000000..c9d35fdf61 --- /dev/null +++ b/js/src/jit/loong64/MacroAssembler-loong64.h @@ -0,0 +1,1025 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_MacroAssembler_loong64_h +#define jit_loong64_MacroAssembler_loong64_h + +#include "jit/loong64/Assembler-loong64.h" +#include "jit/MoveResolver.h" +#include "wasm/WasmTypes.h" + +namespace js { +namespace jit { + +enum LoadStoreSize { + SizeByte = 8, + SizeHalfWord = 16, + SizeWord = 32, + SizeDouble = 64 +}; + +enum LoadStoreExtension { ZeroExtend = 0, SignExtend = 1 }; + +enum JumpKind { LongJump = 0, ShortJump = 1 }; + +static Register CallReg = t8; + +enum LiFlags { + Li64 = 0, + Li48 = 1, +}; + +struct ImmShiftedTag : public ImmWord { + explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {} + + explicit ImmShiftedTag(JSValueType type) + : ImmWord(uintptr_t(JSValueShiftedTag(JSVAL_TYPE_TO_SHIFTED_TAG(type)))) { + } +}; + +struct ImmTag : public Imm32 { + ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {} +}; + +static const int defaultShift = 3; +static_assert(1 << defaultShift == sizeof(JS::Value), + "The defaultShift is wrong"); + +// See documentation for ScratchTagScope and ScratchTagScopeRelease in +// MacroAssembler-x64.h. + +class ScratchTagScope : public SecondScratchRegisterScope { + public: + ScratchTagScope(MacroAssembler& masm, const ValueOperand&) + : SecondScratchRegisterScope(masm) {} +}; + +class ScratchTagScopeRelease { + ScratchTagScope* ts_; + + public: + explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) { + ts_->release(); + } + ~ScratchTagScopeRelease() { ts_->reacquire(); } +}; + +class MacroAssemblerLOONG64 : public Assembler { + protected: + // Perform a downcast. Should be removed by Bug 996602. + MacroAssembler& asMasm(); + const MacroAssembler& asMasm() const; + + Condition ma_cmp(Register rd, Register lhs, Register rhs, Condition c); + Condition ma_cmp(Register rd, Register lhs, Imm32 imm, Condition c); + + void compareFloatingPoint(FloatFormat fmt, FloatRegister lhs, + FloatRegister rhs, DoubleCondition c, + FPConditionBit fcc = FCC0); + + public: + void ma_li(Register dest, CodeLabel* label); + void ma_li(Register dest, ImmWord imm); + void ma_liPatchable(Register dest, ImmPtr imm); + void ma_liPatchable(Register dest, ImmWord imm, LiFlags flags = Li48); + + // load + void ma_ld_b(Register dest, Address address); + void ma_ld_h(Register dest, Address address); + void ma_ld_w(Register dest, Address address); + void ma_ld_d(Register dest, Address address); + void ma_ld_bu(Register dest, Address address); + void ma_ld_hu(Register dest, Address address); + void ma_ld_wu(Register dest, Address address); + void ma_load(Register dest, Address address, LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + + // store + void ma_st_b(Register src, Address address); + void ma_st_h(Register src, Address address); + void ma_st_w(Register src, Address address); + void ma_st_d(Register src, Address address); + void ma_store(Register data, Address address, LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + + // arithmetic based ops + // add + void ma_add_d(Register rd, Register rj, Imm32 imm); + void ma_add32TestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_add32TestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + void ma_addPtrTestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_addPtrTestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + void ma_addPtrTestCarry(Condition cond, Register rd, Register rj, Register rk, + Label* overflow); + void ma_addPtrTestCarry(Condition cond, Register rd, Register rj, Imm32 imm, + Label* overflow); + + // subtract + void ma_sub_d(Register rd, Register rj, Imm32 imm); + void ma_sub32TestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_subPtrTestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_subPtrTestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + + // multiplies. For now, there are only few that we care about. + void ma_mul_d(Register rd, Register rj, Imm32 imm); + void ma_mulh_d(Register rd, Register rj, Imm32 imm); + void ma_mulPtrTestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + + // stack + void ma_pop(Register r); + void ma_push(Register r); + + void branchWithCode(InstImm code, Label* label, JumpKind jumpKind); + // branches when done from within la-specific code + void ma_b(Register lhs, ImmWord imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Register lhs, Address addr, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Address addr, Imm32 imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Address addr, ImmGCPtr imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Address addr, Register rhs, Label* l, Condition c, + JumpKind jumpKind = LongJump) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rhs != scratch); + ma_ld_d(scratch, addr); + ma_b(scratch, rhs, l, c, jumpKind); + } + + void ma_bl(Label* l); + + // fp instructions + void ma_lid(FloatRegister dest, double value); + + void ma_mv(FloatRegister src, ValueOperand dest); + void ma_mv(ValueOperand src, FloatRegister dest); + + void ma_fld_s(FloatRegister ft, Address address); + void ma_fld_d(FloatRegister ft, Address address); + void ma_fst_d(FloatRegister ft, Address address); + void ma_fst_s(FloatRegister ft, Address address); + + void ma_pop(FloatRegister f); + void ma_push(FloatRegister f); + + void ma_cmp_set(Register dst, Register lhs, ImmWord imm, Condition c); + void ma_cmp_set(Register dst, Register lhs, ImmPtr imm, Condition c); + void ma_cmp_set(Register dst, Address address, Imm32 imm, Condition c); + + void moveIfZero(Register dst, Register src, Register cond) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dst != scratch && cond != scratch); + as_masknez(scratch, src, cond); + as_maskeqz(dst, dst, cond); + as_or(dst, dst, scratch); + } + void moveIfNotZero(Register dst, Register src, Register cond) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dst != scratch && cond != scratch); + as_maskeqz(scratch, src, cond); + as_masknez(dst, dst, cond); + as_or(dst, dst, scratch); + } + + // These functions abstract the access to high part of the double precision + // float register. They are intended to work on both 32 bit and 64 bit + // floating point coprocessor. + void moveToDoubleHi(Register src, FloatRegister dest) { + as_movgr2frh_w(dest, src); + } + void moveFromDoubleHi(FloatRegister src, Register dest) { + as_movfrh2gr_s(dest, src); + } + + void moveToDouble(Register src, FloatRegister dest) { + as_movgr2fr_d(dest, src); + } + void moveFromDouble(FloatRegister src, Register dest) { + as_movfr2gr_d(dest, src); + } + + public: + void ma_li(Register dest, ImmGCPtr ptr); + + void ma_li(Register dest, Imm32 imm); + void ma_liPatchable(Register dest, Imm32 imm); + + void ma_rotr_w(Register rd, Register rj, Imm32 shift); + + void ma_fmovz(FloatFormat fmt, FloatRegister fd, FloatRegister fj, + Register rk); + void ma_fmovn(FloatFormat fmt, FloatRegister fd, FloatRegister fj, + Register rk); + + void ma_and(Register rd, Register rj, Imm32 imm, bool bit32 = false); + + void ma_or(Register rd, Register rj, Imm32 imm, bool bit32 = false); + + void ma_xor(Register rd, Register rj, Imm32 imm, bool bit32 = false); + + // load + void ma_load(Register dest, const BaseIndex& src, + LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + + // store + void ma_store(Register data, const BaseIndex& dest, + LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + void ma_store(Imm32 imm, const BaseIndex& dest, LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + + // arithmetic based ops + // add + void ma_add_w(Register rd, Register rj, Imm32 imm); + void ma_add32TestCarry(Condition cond, Register rd, Register rj, Register rk, + Label* overflow); + void ma_add32TestCarry(Condition cond, Register rd, Register rj, Imm32 imm, + Label* overflow); + + // subtract + void ma_sub_w(Register rd, Register rj, Imm32 imm); + void ma_sub_w(Register rd, Register rj, Register rk); + void ma_sub32TestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + + // multiplies. For now, there are only few that we care about. + void ma_mul(Register rd, Register rj, Imm32 imm); + void ma_mul32TestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_mul32TestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + + // divisions + void ma_div_branch_overflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_div_branch_overflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + + // fast mod, uses scratch registers, and thus needs to be in the assembler + // implicitly assumes that we can overwrite dest at the beginning of the + // sequence + void ma_mod_mask(Register src, Register dest, Register hold, Register remain, + int32_t shift, Label* negZero = nullptr); + + // branches when done from within la-specific code + void ma_b(Register lhs, Register rhs, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Register lhs, Imm32 imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Register lhs, ImmPtr imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Register lhs, ImmGCPtr imm, Label* l, Condition c, + JumpKind jumpKind = LongJump) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + ma_li(scratch, imm); + ma_b(lhs, scratch, l, c, jumpKind); + } + + void ma_b(Label* l, JumpKind jumpKind = LongJump); + + // fp instructions + void ma_lis(FloatRegister dest, float value); + + void ma_fst_d(FloatRegister src, BaseIndex address); + void ma_fst_s(FloatRegister src, BaseIndex address); + + void ma_fld_d(FloatRegister dest, const BaseIndex& src); + void ma_fld_s(FloatRegister dest, const BaseIndex& src); + + // FP branches + void ma_bc1s(FloatRegister lhs, FloatRegister rhs, Label* label, + DoubleCondition c, JumpKind jumpKind = LongJump, + FPConditionBit fcc = FCC0); + void ma_bc1d(FloatRegister lhs, FloatRegister rhs, Label* label, + DoubleCondition c, JumpKind jumpKind = LongJump, + FPConditionBit fcc = FCC0); + + void ma_call(ImmPtr dest); + + void ma_jump(ImmPtr dest); + + void ma_cmp_set(Register dst, Register lhs, Register rhs, Condition c); + void ma_cmp_set(Register dst, Register lhs, Imm32 imm, Condition c); + void ma_cmp_set_double(Register dst, FloatRegister lhs, FloatRegister rhs, + DoubleCondition c); + void ma_cmp_set_float32(Register dst, FloatRegister lhs, FloatRegister rhs, + DoubleCondition c); + + void moveToDoubleLo(Register src, FloatRegister dest) { + as_movgr2fr_w(dest, src); + } + void moveFromDoubleLo(FloatRegister src, Register dest) { + as_movfr2gr_s(dest, src); + } + + void moveToFloat32(Register src, FloatRegister dest) { + as_movgr2fr_w(dest, src); + } + void moveFromFloat32(FloatRegister src, Register dest) { + as_movfr2gr_s(dest, src); + } + + // Evaluate srcDest = minmax{Float32,Double}(srcDest, other). + // Handle NaN specially if handleNaN is true. + void minMaxDouble(FloatRegister srcDest, FloatRegister other, bool handleNaN, + bool isMax); + void minMaxFloat32(FloatRegister srcDest, FloatRegister other, bool handleNaN, + bool isMax); + + void loadDouble(const Address& addr, FloatRegister dest); + void loadDouble(const BaseIndex& src, FloatRegister dest); + + // Load a float value into a register, then expand it to a double. + void loadFloatAsDouble(const Address& addr, FloatRegister dest); + void loadFloatAsDouble(const BaseIndex& src, FloatRegister dest); + + void loadFloat32(const Address& addr, FloatRegister dest); + void loadFloat32(const BaseIndex& src, FloatRegister dest); + + void outOfLineWasmTruncateToInt32Check(FloatRegister input, Register output, + MIRType fromType, TruncFlags flags, + Label* rejoin, + wasm::BytecodeOffset trapOffset); + void outOfLineWasmTruncateToInt64Check(FloatRegister input, Register64 output, + MIRType fromType, TruncFlags flags, + Label* rejoin, + wasm::BytecodeOffset trapOffset); + + protected: + void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register memoryBase, + Register ptr, Register ptrScratch, AnyRegister output, + Register tmp); + void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister value, + Register memoryBase, Register ptr, Register ptrScratch, + Register tmp); +}; + +class MacroAssembler; + +class MacroAssemblerLOONG64Compat : public MacroAssemblerLOONG64 { + public: + using MacroAssemblerLOONG64::call; + + MacroAssemblerLOONG64Compat() {} + + void convertBoolToInt32(Register src, Register dest) { + ma_and(dest, src, Imm32(0xff)); + }; + void convertInt32ToDouble(Register src, FloatRegister dest) { + as_movgr2fr_w(dest, src); + as_ffint_d_w(dest, dest); + }; + void convertInt32ToDouble(const Address& src, FloatRegister dest) { + ma_fld_s(dest, src); + as_ffint_d_w(dest, dest); + }; + void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(scratch != src.base); + MOZ_ASSERT(scratch != src.index); + computeScaledAddress(src, scratch); + convertInt32ToDouble(Address(scratch, src.offset), dest); + }; + void convertUInt32ToDouble(Register src, FloatRegister dest); + void convertUInt32ToFloat32(Register src, FloatRegister dest); + void convertDoubleToFloat32(FloatRegister src, FloatRegister dest); + void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true); + void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true); + void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true); + + void convertFloat32ToDouble(FloatRegister src, FloatRegister dest); + void convertInt32ToFloat32(Register src, FloatRegister dest); + void convertInt32ToFloat32(const Address& src, FloatRegister dest); + + void movq(Register rj, Register rd); + + void computeScaledAddress(const BaseIndex& address, Register dest); + + void computeEffectiveAddress(const Address& address, Register dest) { + ma_add_d(dest, address.base, Imm32(address.offset)); + } + + void computeEffectiveAddress(const BaseIndex& address, Register dest) { + computeScaledAddress(address, dest); + if (address.offset) { + ma_add_d(dest, dest, Imm32(address.offset)); + } + } + + void j(Label* dest) { ma_b(dest); } + + void mov(Register src, Register dest) { as_ori(dest, src, 0); } + void mov(ImmWord imm, Register dest) { ma_li(dest, imm); } + void mov(ImmPtr imm, Register dest) { + mov(ImmWord(uintptr_t(imm.value)), dest); + } + void mov(CodeLabel* label, Register dest) { ma_li(dest, label); } + void mov(Register src, Address dest) { MOZ_CRASH("NYI-IC"); } + void mov(Address src, Register dest) { MOZ_CRASH("NYI-IC"); } + + void writeDataRelocation(const Value& val) { + // Raw GC pointer relocations and Value relocations both end up in + // TraceOneDataRelocation. + if (val.isGCThing()) { + gc::Cell* cell = val.toGCThing(); + if (cell && gc::IsInsideNursery(cell)) { + embedsNurseryPointers_ = true; + } + dataRelocations_.writeUnsigned(currentOffset()); + } + } + + void branch(JitCode* c) { + ScratchRegisterScope scratch(asMasm()); + BufferOffset bo = m_buffer.nextOffset(); + addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE); + ma_liPatchable(scratch, ImmPtr(c->raw())); + as_jirl(zero, scratch, BOffImm16(0)); + } + void branch(const Register reg) { as_jirl(zero, reg, BOffImm16(0)); } + void nop() { as_nop(); } + void ret() { + ma_pop(ra); + as_jirl(zero, ra, BOffImm16(0)); + } + inline void retn(Imm32 n); + void push(Imm32 imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + } + void push(ImmWord imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + } + void push(ImmGCPtr imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + } + void push(const Address& address) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(address, scratch2); + ma_push(scratch2); + } + void push(Register reg) { ma_push(reg); } + void push(FloatRegister reg) { ma_push(reg); } + void pop(Register reg) { ma_pop(reg); } + void pop(FloatRegister reg) { ma_pop(reg); } + + // Emit a branch that can be toggled to a non-operation. On LOONG64 we use + // "andi" instruction to toggle the branch. + // See ToggleToJmp(), ToggleToCmp(). + CodeOffset toggledJump(Label* label); + + // Emit a "jalr" or "nop" instruction. ToggleCall can be used to patch + // this instruction. + CodeOffset toggledCall(JitCode* target, bool enabled); + + static size_t ToggledCallSize(uint8_t* code) { + // Four instructions used in: MacroAssemblerLOONG64Compat::toggledCall + return 4 * sizeof(uint32_t); + } + + CodeOffset pushWithPatch(ImmWord imm) { + ScratchRegisterScope scratch(asMasm()); + CodeOffset offset = movWithPatch(imm, scratch); + ma_push(scratch); + return offset; + } + + CodeOffset movWithPatch(ImmWord imm, Register dest) { + CodeOffset offset = CodeOffset(currentOffset()); + ma_liPatchable(dest, imm, Li64); + return offset; + } + CodeOffset movWithPatch(ImmPtr imm, Register dest) { + CodeOffset offset = CodeOffset(currentOffset()); + ma_liPatchable(dest, imm); + return offset; + } + + void writeCodePointer(CodeLabel* label) { + label->patchAt()->bind(currentOffset()); + label->setLinkMode(CodeLabel::RawPointer); + m_buffer.ensureSpace(sizeof(void*)); + writeInst(-1); + writeInst(-1); + } + + void jump(Label* label) { ma_b(label); } + void jump(Register reg) { as_jirl(zero, reg, BOffImm16(0)); } + void jump(const Address& address) { + ScratchRegisterScope scratch(asMasm()); + loadPtr(address, scratch); + as_jirl(zero, scratch, BOffImm16(0)); + } + + void jump(JitCode* code) { branch(code); } + + void jump(ImmPtr ptr) { + BufferOffset bo = m_buffer.nextOffset(); + addPendingJump(bo, ptr, RelocationKind::HARDCODED); + ma_jump(ptr); + } + + void jump(TrampolinePtr code) { jump(ImmPtr(code.value)); } + + void splitTag(Register src, Register dest) { + as_srli_d(dest, src, JSVAL_TAG_SHIFT); + } + + void splitTag(const ValueOperand& operand, Register dest) { + splitTag(operand.valueReg(), dest); + } + + void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) { + splitTag(value, tag); + } + + // unboxing code + void unboxNonDouble(const ValueOperand& operand, Register dest, + JSValueType type) { + unboxNonDouble(operand.valueReg(), dest, type); + } + + template + void unboxNonDouble(T src, Register dest, JSValueType type) { + MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + load32(src, dest); + return; + } + loadPtr(src, dest); + unboxNonDouble(dest, dest, type); + } + + void unboxNonDouble(Register src, Register dest, JSValueType type) { + MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + as_slli_w(dest, src, 0); + return; + } + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(scratch != src); + mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch); + as_xor(dest, src, scratch); + } + + template + void unboxObjectOrNull(const T& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); + static_assert(JS::detail::ValueObjectOrNullBit == + (uint64_t(0x8) << JSVAL_TAG_SHIFT)); + as_bstrins_d(dest, zero, JSVAL_TAG_SHIFT + 3, JSVAL_TAG_SHIFT + 3); + } + + void unboxGCThingForGCBarrier(const Address& src, Register dest) { + loadPtr(src, dest); + as_bstrpick_d(dest, dest, JSVAL_TAG_SHIFT - 1, 0); + } + void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) { + as_bstrpick_d(dest, src.valueReg(), JSVAL_TAG_SHIFT - 1, 0); + } + + void unboxInt32(const ValueOperand& operand, Register dest); + void unboxInt32(Register src, Register dest); + void unboxInt32(const Address& src, Register dest); + void unboxInt32(const BaseIndex& src, Register dest); + void unboxBoolean(const ValueOperand& operand, Register dest); + void unboxBoolean(Register src, Register dest); + void unboxBoolean(const Address& src, Register dest); + void unboxBoolean(const BaseIndex& src, Register dest); + void unboxDouble(const ValueOperand& operand, FloatRegister dest); + void unboxDouble(Register src, Register dest); + void unboxDouble(const Address& src, FloatRegister dest); + void unboxDouble(const BaseIndex& src, FloatRegister dest); + void unboxString(const ValueOperand& operand, Register dest); + void unboxString(Register src, Register dest); + void unboxString(const Address& src, Register dest); + void unboxSymbol(const ValueOperand& src, Register dest); + void unboxSymbol(Register src, Register dest); + void unboxSymbol(const Address& src, Register dest); + void unboxBigInt(const ValueOperand& operand, Register dest); + void unboxBigInt(Register src, Register dest); + void unboxBigInt(const Address& src, Register dest); + void unboxObject(const ValueOperand& src, Register dest); + void unboxObject(Register src, Register dest); + void unboxObject(const Address& src, Register dest); + void unboxObject(const BaseIndex& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); + } + void unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType type); + + void notBoolean(const ValueOperand& val) { + as_xori(val.valueReg(), val.valueReg(), 1); + } + + // boxing code + void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister); + void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest); + + // Extended unboxing API. If the payload is already in a register, returns + // that register. Otherwise, provides a move to the given scratch register, + // and returns that. + [[nodiscard]] Register extractObject(const Address& address, + Register scratch); + [[nodiscard]] Register extractObject(const ValueOperand& value, + Register scratch) { + unboxObject(value, scratch); + return scratch; + } + [[nodiscard]] Register extractString(const ValueOperand& value, + Register scratch) { + unboxString(value, scratch); + return scratch; + } + [[nodiscard]] Register extractSymbol(const ValueOperand& value, + Register scratch) { + unboxSymbol(value, scratch); + return scratch; + } + [[nodiscard]] Register extractInt32(const ValueOperand& value, + Register scratch) { + unboxInt32(value, scratch); + return scratch; + } + [[nodiscard]] Register extractBoolean(const ValueOperand& value, + Register scratch) { + unboxBoolean(value, scratch); + return scratch; + } + [[nodiscard]] Register extractTag(const Address& address, Register scratch); + [[nodiscard]] Register extractTag(const BaseIndex& address, Register scratch); + [[nodiscard]] Register extractTag(const ValueOperand& value, + Register scratch) { + splitTag(value, scratch); + return scratch; + } + + inline void ensureDouble(const ValueOperand& source, FloatRegister dest, + Label* failure); + + void boolValueToDouble(const ValueOperand& operand, FloatRegister dest); + void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest); + void loadInt32OrDouble(const Address& src, FloatRegister dest); + void loadInt32OrDouble(const BaseIndex& addr, FloatRegister dest); + void loadConstantDouble(double dp, FloatRegister dest); + + void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest); + void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest); + void loadConstantFloat32(float f, FloatRegister dest); + + void testNullSet(Condition cond, const ValueOperand& value, Register dest); + + void testObjectSet(Condition cond, const ValueOperand& value, Register dest); + + void testUndefinedSet(Condition cond, const ValueOperand& value, + Register dest); + + // higher level tag testing code + Address ToPayload(Address value) { return value; } + + template + void loadUnboxedValue(const T& address, MIRType type, AnyRegister dest) { + if (dest.isFloat()) { + loadInt32OrDouble(address, dest.fpu()); + } else { + unboxNonDouble(address, dest.gpr(), ValueTypeFromMIRType(type)); + } + } + + void storeUnboxedPayload(ValueOperand value, BaseIndex address, size_t nbytes, + JSValueType type) { + switch (nbytes) { + case 8: { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + if (type == JSVAL_TYPE_OBJECT) { + unboxObjectOrNull(value, scratch2); + } else { + unboxNonDouble(value, scratch2, type); + } + computeEffectiveAddress(address, scratch); + as_st_d(scratch2, scratch, 0); + return; + } + case 4: + store32(value.valueReg(), address); + return; + case 1: + store8(value.valueReg(), address); + return; + default: + MOZ_CRASH("Bad payload width"); + } + } + + void storeUnboxedPayload(ValueOperand value, Address address, size_t nbytes, + JSValueType type) { + switch (nbytes) { + case 8: { + SecondScratchRegisterScope scratch2(asMasm()); + if (type == JSVAL_TYPE_OBJECT) { + unboxObjectOrNull(value, scratch2); + } else { + unboxNonDouble(value, scratch2, type); + } + storePtr(scratch2, address); + return; + } + case 4: + store32(value.valueReg(), address); + return; + case 1: + store8(value.valueReg(), address); + return; + default: + MOZ_CRASH("Bad payload width"); + } + } + + void boxValue(JSValueType type, Register src, Register dest) { + ScratchRegisterScope scratch(asMasm()); + if (src == dest) { + as_ori(scratch, src, 0); + src = scratch; + } +#ifdef DEBUG + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + Label upper32BitsSignExtended; + as_slli_w(dest, src, 0); + ma_b(src, dest, &upper32BitsSignExtended, Equal, ShortJump); + breakpoint(); + bind(&upper32BitsSignExtended); + } +#endif + ma_li(dest, ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type))); + as_bstrins_d(dest, src, JSVAL_TAG_SHIFT - 1, 0); + } + + void storeValue(ValueOperand val, const Address& dest); + void storeValue(ValueOperand val, const BaseIndex& dest); + void storeValue(JSValueType type, Register reg, Address dest); + void storeValue(JSValueType type, Register reg, BaseIndex dest); + void storeValue(const Value& val, Address dest); + void storeValue(const Value& val, BaseIndex dest); + void storeValue(const Address& src, const Address& dest, Register temp) { + loadPtr(src, temp); + storePtr(temp, dest); + } + + void loadValue(Address src, ValueOperand val); + void loadValue(const BaseIndex& src, ValueOperand val); + + void loadUnalignedValue(const Address& src, ValueOperand dest) { + loadValue(src, dest); + } + + void tagValue(JSValueType type, Register payload, ValueOperand dest); + + void pushValue(ValueOperand val); + void popValue(ValueOperand val); + void pushValue(const Value& val) { + if (val.isGCThing()) { + ScratchRegisterScope scratch(asMasm()); + writeDataRelocation(val); + movWithPatch(ImmWord(val.asRawBits()), scratch); + push(scratch); + } else { + push(ImmWord(val.asRawBits())); + } + } + void pushValue(JSValueType type, Register reg) { + SecondScratchRegisterScope scratch2(asMasm()); + boxValue(type, reg, scratch2); + push(scratch2); + } + void pushValue(const Address& addr); + + void handleFailureWithHandlerTail(Label* profilerExitTail); + + ///////////////////////////////////////////////////////////////// + // Common interface. + ///////////////////////////////////////////////////////////////// + public: + // The following functions are exposed for use in platform-shared code. + + inline void incrementInt32Value(const Address& addr); + + void move32(Imm32 imm, Register dest); + void move32(Register src, Register dest); + + void movePtr(Register src, Register dest); + void movePtr(ImmWord imm, Register dest); + void movePtr(ImmPtr imm, Register dest); + void movePtr(wasm::SymbolicAddress imm, Register dest); + void movePtr(ImmGCPtr imm, Register dest); + + void load8SignExtend(const Address& address, Register dest); + void load8SignExtend(const BaseIndex& src, Register dest); + + void load8ZeroExtend(const Address& address, Register dest); + void load8ZeroExtend(const BaseIndex& src, Register dest); + + void load16SignExtend(const Address& address, Register dest); + void load16SignExtend(const BaseIndex& src, Register dest); + + template + void load16UnalignedSignExtend(const S& src, Register dest) { + load16SignExtend(src, dest); + } + + void load16ZeroExtend(const Address& address, Register dest); + void load16ZeroExtend(const BaseIndex& src, Register dest); + + template + void load16UnalignedZeroExtend(const S& src, Register dest) { + load16ZeroExtend(src, dest); + } + + void load32(const Address& address, Register dest); + void load32(const BaseIndex& address, Register dest); + void load32(AbsoluteAddress address, Register dest); + void load32(wasm::SymbolicAddress address, Register dest); + + template + void load32Unaligned(const S& src, Register dest) { + load32(src, dest); + } + + void load64(const Address& address, Register64 dest) { + loadPtr(address, dest.reg); + } + void load64(const BaseIndex& address, Register64 dest) { + loadPtr(address, dest.reg); + } + + template + void load64Unaligned(const S& src, Register64 dest) { + load64(src, dest); + } + + void loadPtr(const Address& address, Register dest); + void loadPtr(const BaseIndex& src, Register dest); + void loadPtr(AbsoluteAddress address, Register dest); + void loadPtr(wasm::SymbolicAddress address, Register dest); + + void loadPrivate(const Address& address, Register dest); + + void store8(Register src, const Address& address); + void store8(Imm32 imm, const Address& address); + void store8(Register src, const BaseIndex& address); + void store8(Imm32 imm, const BaseIndex& address); + + void store16(Register src, const Address& address); + void store16(Imm32 imm, const Address& address); + void store16(Register src, const BaseIndex& address); + void store16(Imm32 imm, const BaseIndex& address); + + template + void store16Unaligned(Register src, const T& dest) { + store16(src, dest); + } + + void store32(Register src, AbsoluteAddress address); + void store32(Register src, const Address& address); + void store32(Register src, const BaseIndex& address); + void store32(Imm32 src, const Address& address); + void store32(Imm32 src, const BaseIndex& address); + + // NOTE: This will use second scratch on LOONG64. Only ARM needs the + // implementation without second scratch. + void store32_NoSecondScratch(Imm32 src, const Address& address) { + store32(src, address); + } + + template + void store32Unaligned(Register src, const T& dest) { + store32(src, dest); + } + + void store64(Imm64 imm, Address address) { + storePtr(ImmWord(imm.value), address); + } + void store64(Imm64 imm, const BaseIndex& address) { + storePtr(ImmWord(imm.value), address); + } + + void store64(Register64 src, Address address) { storePtr(src.reg, address); } + void store64(Register64 src, const BaseIndex& address) { + storePtr(src.reg, address); + } + + template + void store64Unaligned(Register64 src, const T& dest) { + store64(src, dest); + } + + template + void storePtr(ImmWord imm, T address); + template + void storePtr(ImmPtr imm, T address); + template + void storePtr(ImmGCPtr imm, T address); + void storePtr(Register src, const Address& address); + void storePtr(Register src, const BaseIndex& address); + void storePtr(Register src, AbsoluteAddress dest); + + void moveDouble(FloatRegister src, FloatRegister dest) { + as_fmov_d(dest, src); + } + + void zeroDouble(FloatRegister reg) { moveToDouble(zero, reg); } + + void convertUInt64ToDouble(Register src, FloatRegister dest); + + void breakpoint(uint32_t value = 0); + + void checkStackAlignment() { +#ifdef DEBUG + Label aligned; + ScratchRegisterScope scratch(asMasm()); + as_andi(scratch, sp, ABIStackAlignment - 1); + ma_b(scratch, zero, &aligned, Equal, ShortJump); + breakpoint(); + bind(&aligned); +#endif + }; + + static void calculateAlignedStackPointer(void** stackPointer); + + void cmpPtrSet(Assembler::Condition cond, Address lhs, ImmPtr rhs, + Register dest); + void cmpPtrSet(Assembler::Condition cond, Register lhs, Address rhs, + Register dest); + void cmpPtrSet(Assembler::Condition cond, Address lhs, Register rhs, + Register dest); + + void cmp32Set(Assembler::Condition cond, Register lhs, Address rhs, + Register dest); + + void cmp64Set(Assembler::Condition cond, Register lhs, Imm32 rhs, + Register dest) { + ma_cmp_set(dest, lhs, rhs, cond); + } + + protected: + bool buildOOLFakeExitFrame(void* fakeReturnAddr); + + void wasmLoadI64Impl(const wasm::MemoryAccessDesc& access, + Register memoryBase, Register ptr, Register ptrScratch, + Register64 output, Register tmp); + void wasmStoreI64Impl(const wasm::MemoryAccessDesc& access, Register64 value, + Register memoryBase, Register ptr, Register ptrScratch, + Register tmp); + + public: + void lea(Operand addr, Register dest) { + ma_add_d(dest, addr.baseReg(), Imm32(addr.disp())); + } + + void abiret() { as_jirl(zero, ra, BOffImm16(0)); } + + void moveFloat32(FloatRegister src, FloatRegister dest) { + as_fmov_s(dest, src); + } + + void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest) { + loadPtr(Address(WasmTlsReg, + offsetof(wasm::TlsData, globalArea) + globalDataOffset), + dest); + } + void loadWasmPinnedRegsFromTls() { + loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, memoryBase)), HeapReg); + } + + // Instrumentation for entering and leaving the profiler. + void profilerEnterFrame(Register framePtr, Register scratch); + void profilerExitFrame(); +}; + +typedef MacroAssemblerLOONG64Compat MacroAssemblerSpecific; + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_MacroAssembler_loong64_h */ diff --git a/js/src/jit/loong64/MoveEmitter-loong64.cpp b/js/src/jit/loong64/MoveEmitter-loong64.cpp new file mode 100644 index 0000000000..603722f037 --- /dev/null +++ b/js/src/jit/loong64/MoveEmitter-loong64.cpp @@ -0,0 +1,327 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loong64/MoveEmitter-loong64.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +void MoveEmitterLOONG64::breakCycle(const MoveOperand& from, + const MoveOperand& to, MoveOp::Type type, + uint32_t slotId) { + // There is some pattern: + // (A -> B) + // (B -> A) + // + // This case handles (A -> B), which we reach first. We save B, then allow + // the original move to continue. + switch (type) { + case MoveOp::FLOAT32: + if (to.isMemory()) { + FloatRegister temp = ScratchFloat32Reg; + masm.loadFloat32(getAdjustedAddress(to), temp); + masm.storeFloat32(temp, cycleSlot(slotId)); + } else { + masm.storeFloat32(to.floatReg(), cycleSlot(slotId)); + } + break; + case MoveOp::DOUBLE: + if (to.isMemory()) { + FloatRegister temp = ScratchDoubleReg; + masm.loadDouble(getAdjustedAddress(to), temp); + masm.storeDouble(temp, cycleSlot(slotId)); + } else { + masm.storeDouble(to.floatReg(), cycleSlot(slotId)); + } + break; + case MoveOp::INT32: + if (to.isMemory()) { + SecondScratchRegisterScope scratch2(masm); + masm.load32(getAdjustedAddress(to), scratch2); + masm.store32(scratch2, cycleSlot(0)); + } else { + masm.store32(to.reg(), cycleSlot(0)); + } + break; + case MoveOp::GENERAL: + if (to.isMemory()) { + SecondScratchRegisterScope scratch2(masm); + masm.loadPtr(getAdjustedAddress(to), scratch2); + masm.storePtr(scratch2, cycleSlot(0)); + } else { + masm.storePtr(to.reg(), cycleSlot(0)); + } + break; + default: + MOZ_CRASH("Unexpected move type"); + } +} + +void MoveEmitterLOONG64::completeCycle(const MoveOperand& from, + const MoveOperand& to, MoveOp::Type type, + uint32_t slotId) { + // There is some pattern: + // (A -> B) + // (B -> A) + // + // This case handles (B -> A), which we reach last. We emit a move from the + // saved value of B, to A. + switch (type) { + case MoveOp::FLOAT32: + if (to.isMemory()) { + FloatRegister temp = ScratchFloat32Reg; + masm.loadFloat32(cycleSlot(slotId), temp); + masm.storeFloat32(temp, getAdjustedAddress(to)); + } else { + masm.loadFloat32(cycleSlot(slotId), to.floatReg()); + } + break; + case MoveOp::DOUBLE: + if (to.isMemory()) { + FloatRegister temp = ScratchDoubleReg; + masm.loadDouble(cycleSlot(slotId), temp); + masm.storeDouble(temp, getAdjustedAddress(to)); + } else { + masm.loadDouble(cycleSlot(slotId), to.floatReg()); + } + break; + case MoveOp::INT32: + MOZ_ASSERT(slotId == 0); + if (to.isMemory()) { + SecondScratchRegisterScope scratch2(masm); + masm.load32(cycleSlot(0), scratch2); + masm.store32(scratch2, getAdjustedAddress(to)); + } else { + masm.load32(cycleSlot(0), to.reg()); + } + break; + case MoveOp::GENERAL: + MOZ_ASSERT(slotId == 0); + if (to.isMemory()) { + SecondScratchRegisterScope scratch2(masm); + masm.loadPtr(cycleSlot(0), scratch2); + masm.storePtr(scratch2, getAdjustedAddress(to)); + } else { + masm.loadPtr(cycleSlot(0), to.reg()); + } + break; + default: + MOZ_CRASH("Unexpected move type"); + } +} + +void MoveEmitterLOONG64::emitDoubleMove(const MoveOperand& from, + const MoveOperand& to) { + if (from.isFloatReg()) { + if (to.isFloatReg()) { + masm.moveDouble(from.floatReg(), to.floatReg()); + } else if (to.isGeneralReg()) { + masm.moveFromDouble(from.floatReg(), to.reg()); + } else { + MOZ_ASSERT(to.isMemory()); + masm.storeDouble(from.floatReg(), getAdjustedAddress(to)); + } + } else if (to.isFloatReg()) { + if (from.isMemory()) { + masm.loadDouble(getAdjustedAddress(from), to.floatReg()); + } else { + masm.moveToDouble(from.reg(), to.floatReg()); + } + } else { + MOZ_ASSERT(from.isMemory()); + MOZ_ASSERT(to.isMemory()); + masm.loadDouble(getAdjustedAddress(from), ScratchDoubleReg); + masm.storeDouble(ScratchDoubleReg, getAdjustedAddress(to)); + } +} + +void MoveEmitterLOONG64::emit(const MoveResolver& moves) { + if (moves.numCycles()) { + // Reserve stack for cycle resolution + masm.reserveStack(moves.numCycles() * sizeof(double)); + pushedAtCycle_ = masm.framePushed(); + } + + for (size_t i = 0; i < moves.numMoves(); i++) { + emit(moves.getMove(i)); + } +} + +Address MoveEmitterLOONG64::cycleSlot(uint32_t slot, uint32_t subslot) const { + int32_t offset = masm.framePushed() - pushedAtCycle_; + MOZ_ASSERT(Imm16::IsInSignedRange(offset)); + return Address(StackPointer, offset + slot * sizeof(double) + subslot); +} + +int32_t MoveEmitterLOONG64::getAdjustedOffset(const MoveOperand& operand) { + MOZ_ASSERT(operand.isMemoryOrEffectiveAddress()); + if (operand.base() != StackPointer) { + return operand.disp(); + } + + // Adjust offset if stack pointer has been moved. + return operand.disp() + masm.framePushed() - pushedAtStart_; +} + +Address MoveEmitterLOONG64::getAdjustedAddress(const MoveOperand& operand) { + return Address(operand.base(), getAdjustedOffset(operand)); +} + +void MoveEmitterLOONG64::emitMove(const MoveOperand& from, + const MoveOperand& to) { + if (from.isGeneralReg()) { + if (to.isGeneralReg()) { + masm.movePtr(from.reg(), to.reg()); + } else if (to.isMemory()) { + masm.storePtr(from.reg(), getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitMove arguments."); + } + } else if (from.isMemory()) { + if (to.isGeneralReg()) { + masm.loadPtr(getAdjustedAddress(from), to.reg()); + } else if (to.isMemory()) { + SecondScratchRegisterScope scratch2(masm); + masm.loadPtr(getAdjustedAddress(from), scratch2); + masm.storePtr(scratch2, getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitMove arguments."); + } + } else if (from.isEffectiveAddress()) { + if (to.isGeneralReg()) { + masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg()); + } else if (to.isMemory()) { + SecondScratchRegisterScope scratch2(masm); + masm.computeEffectiveAddress(getAdjustedAddress(from), scratch2); + masm.storePtr(scratch2, getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitMove arguments."); + } + } else { + MOZ_CRASH("Invalid emitMove arguments."); + } +} + +void MoveEmitterLOONG64::emitInt32Move(const MoveOperand& from, + const MoveOperand& to) { + if (from.isGeneralReg()) { + if (to.isGeneralReg()) { + masm.move32(from.reg(), to.reg()); + } else if (to.isMemory()) { + masm.store32(from.reg(), getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitInt32Move arguments."); + } + } else if (from.isMemory()) { + if (to.isGeneralReg()) { + masm.load32(getAdjustedAddress(from), to.reg()); + } else if (to.isMemory()) { + SecondScratchRegisterScope scratch2(masm); + masm.load32(getAdjustedAddress(from), scratch2); + masm.store32(scratch2, getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitInt32Move arguments."); + } + } else if (from.isEffectiveAddress()) { + if (to.isGeneralReg()) { + masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg()); + } else if (to.isMemory()) { + SecondScratchRegisterScope scratch2(masm); + masm.computeEffectiveAddress(getAdjustedAddress(from), scratch2); + masm.store32(scratch2, getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitInt32Move arguments."); + } + } else { + MOZ_CRASH("Invalid emitInt32Move arguments."); + } +} + +void MoveEmitterLOONG64::emitFloat32Move(const MoveOperand& from, + const MoveOperand& to) { + // Ensure that we can use ScratchFloat32Reg in memory move. + MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg() != ScratchFloat32Reg); + MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg() != ScratchFloat32Reg); + + if (from.isFloatReg()) { + if (to.isFloatReg()) { + masm.moveFloat32(from.floatReg(), to.floatReg()); + } else if (to.isGeneralReg()) { + // This should only be used when passing float parameter in a1,a2,a3 + MOZ_ASSERT(to.reg() == a1 || to.reg() == a2 || to.reg() == a3); + masm.moveFromFloat32(from.floatReg(), to.reg()); + } else { + MOZ_ASSERT(to.isMemory()); + masm.storeFloat32(from.floatReg(), getAdjustedAddress(to)); + } + } else if (to.isFloatReg()) { + MOZ_ASSERT(from.isMemory()); + masm.loadFloat32(getAdjustedAddress(from), to.floatReg()); + } else if (to.isGeneralReg()) { + MOZ_ASSERT(from.isMemory()); + // This should only be used when passing float parameter in a1,a2,a3 + MOZ_ASSERT(to.reg() == a1 || to.reg() == a2 || to.reg() == a3); + masm.loadPtr(getAdjustedAddress(from), to.reg()); + } else { + MOZ_ASSERT(from.isMemory()); + MOZ_ASSERT(to.isMemory()); + masm.loadFloat32(getAdjustedAddress(from), ScratchFloat32Reg); + masm.storeFloat32(ScratchFloat32Reg, getAdjustedAddress(to)); + } +} + +void MoveEmitterLOONG64::emit(const MoveOp& move) { + const MoveOperand& from = move.from(); + const MoveOperand& to = move.to(); + + if (move.isCycleEnd() && move.isCycleBegin()) { + // A fun consequence of aliased registers is you can have multiple + // cycles at once, and one can end exactly where another begins. + breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot()); + completeCycle(from, to, move.type(), move.cycleEndSlot()); + return; + } + + if (move.isCycleEnd()) { + MOZ_ASSERT(inCycle_); + completeCycle(from, to, move.type(), move.cycleEndSlot()); + MOZ_ASSERT(inCycle_ > 0); + inCycle_--; + return; + } + + if (move.isCycleBegin()) { + breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot()); + inCycle_++; + } + + switch (move.type()) { + case MoveOp::FLOAT32: + emitFloat32Move(from, to); + break; + case MoveOp::DOUBLE: + emitDoubleMove(from, to); + break; + case MoveOp::INT32: + emitInt32Move(from, to); + break; + case MoveOp::GENERAL: + emitMove(from, to); + break; + default: + MOZ_CRASH("Unexpected move type"); + } +} + +void MoveEmitterLOONG64::assertDone() { MOZ_ASSERT(inCycle_ == 0); } + +void MoveEmitterLOONG64::finish() { + assertDone(); + + masm.freeStack(masm.framePushed() - pushedAtStart_); +} diff --git a/js/src/jit/loong64/MoveEmitter-loong64.h b/js/src/jit/loong64/MoveEmitter-loong64.h new file mode 100644 index 0000000000..1481c8f973 --- /dev/null +++ b/js/src/jit/loong64/MoveEmitter-loong64.h @@ -0,0 +1,76 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_MoveEmitter_loong64_h +#define jit_loong64_MoveEmitter_loong64_h + +#include "jit/MacroAssembler.h" +#include "jit/MoveResolver.h" + +namespace js { +namespace jit { + +class MoveEmitterLOONG64 { + void emitDoubleMove(const MoveOperand& from, const MoveOperand& to); + void breakCycle(const MoveOperand& from, const MoveOperand& to, + MoveOp::Type type, uint32_t slot); + void completeCycle(const MoveOperand& from, const MoveOperand& to, + MoveOp::Type type, uint32_t slot); + + protected: + uint32_t inCycle_; + MacroAssembler& masm; + + // Original stack push value. + uint32_t pushedAtStart_; + + // These store stack offsets to spill locations, snapshotting + // codegen->framePushed_ at the time they were allocated. They are -1 if no + // stack space has been allocated for that particular spill. + int32_t pushedAtCycle_; + int32_t pushedAtSpill_; + + // These are registers that are available for temporary use. They may be + // assigned InvalidReg. If no corresponding spill space has been assigned, + // then these registers do not need to be spilled. + Register spilledReg_; + FloatRegister spilledFloatReg_; + + void assertDone(); + Register tempReg(); + FloatRegister tempFloatReg(); + Address cycleSlot(uint32_t slot, uint32_t subslot = 0) const; + int32_t getAdjustedOffset(const MoveOperand& operand); + Address getAdjustedAddress(const MoveOperand& operand); + + void emitMove(const MoveOperand& from, const MoveOperand& to); + void emitInt32Move(const MoveOperand& from, const MoveOperand& to); + void emitFloat32Move(const MoveOperand& from, const MoveOperand& to); + void emit(const MoveOp& move); + + public: + MoveEmitterLOONG64(MacroAssembler& masm) + : inCycle_(0), + masm(masm), + pushedAtStart_(masm.framePushed()), + pushedAtCycle_(-1), + pushedAtSpill_(-1), + spilledReg_(InvalidReg), + spilledFloatReg_(InvalidFloatReg) {} + + ~MoveEmitterLOONG64() { assertDone(); } + + void emit(const MoveResolver& moves); + void finish(); + void setScratchRegister(Register reg) {} +}; + +typedef MoveEmitterLOONG64 MoveEmitter; + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_MoveEmitter_loong64_h */ diff --git a/js/src/jit/loong64/SharedICHelpers-loong64-inl.h b/js/src/jit/loong64/SharedICHelpers-loong64-inl.h new file mode 100644 index 0000000000..5d945213ba --- /dev/null +++ b/js/src/jit/loong64/SharedICHelpers-loong64-inl.h @@ -0,0 +1,111 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_SharedICHelpers_loong64_inl_h +#define jit_loong64_SharedICHelpers_loong64_inl_h + +#include "jit/BaselineFrame.h" +#include "jit/SharedICHelpers.h" + +#include "jit/MacroAssembler-inl.h" + +namespace js { +namespace jit { + +inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm, + uint32_t argSize) { + Register scratch = R2.scratchReg(); + + // Compute frame size. + masm.movePtr(BaselineFrameReg, scratch); + masm.addPtr(Imm32(BaselineFrame::FramePointerOffset), scratch); + masm.subPtr(BaselineStackReg, scratch); + +#ifdef DEBUG + // Store frame size without VMFunction arguments for debug assertions. + masm.subPtr(Imm32(argSize), scratch); + Address frameSizeAddr(BaselineFrameReg, + BaselineFrame::reverseOffsetOfDebugFrameSize()); + masm.store32(scratch, frameSizeAddr); + masm.addPtr(Imm32(argSize), scratch); +#endif + + // Push frame descriptor and perform the tail call. + // ICTailCallReg (ra) already contains the return address (as we + // keep it there through the stub calls), but the VMWrapper code being + // called expects the return address to also be pushed on the stack. + MOZ_ASSERT(ICTailCallReg == ra); + masm.makeFrameDescriptor(scratch, FrameType::BaselineJS, + ExitFrameLayout::Size()); + masm.subPtr(Imm32(sizeof(CommonFrameLayout)), StackPointer); + masm.storePtr(scratch, + Address(StackPointer, CommonFrameLayout::offsetOfDescriptor())); + masm.storePtr( + ra, Address(StackPointer, CommonFrameLayout::offsetOfReturnAddress())); + + masm.jump(target); +} + +inline void EmitBaselineCreateStubFrameDescriptor(MacroAssembler& masm, + Register reg, + uint32_t headerSize) { + // Compute stub frame size. We have to add two pointers: the stub reg and + // previous frame pointer pushed by EmitEnterStubFrame. + masm.movePtr(BaselineFrameReg, reg); + masm.addPtr(Imm32(sizeof(intptr_t) * 2), reg); + masm.subPtr(BaselineStackReg, reg); + + masm.makeFrameDescriptor(reg, FrameType::BaselineStub, headerSize); +} + +inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) { + Register scratch = R2.scratchReg(); + EmitBaselineCreateStubFrameDescriptor(masm, scratch, ExitFrameLayout::Size()); + masm.push(scratch); + masm.call(target); +} + +inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) { + MOZ_ASSERT(scratch != ICTailCallReg); + + // Compute frame size. + masm.movePtr(BaselineFrameReg, scratch); + masm.addPtr(Imm32(BaselineFrame::FramePointerOffset), scratch); + masm.subPtr(BaselineStackReg, scratch); + +#ifdef DEBUG + Address frameSizeAddr(BaselineFrameReg, + BaselineFrame::reverseOffsetOfDebugFrameSize()); + masm.store32(scratch, frameSizeAddr); +#endif + + // Note: when making changes here, don't forget to update + // BaselineStubFrame if needed. + + // Push frame descriptor and return address. + masm.makeFrameDescriptor(scratch, FrameType::BaselineJS, + BaselineStubFrameLayout::Size()); + masm.subPtr(Imm32(STUB_FRAME_SIZE), StackPointer); + masm.storePtr(scratch, + Address(StackPointer, offsetof(BaselineStubFrame, descriptor))); + masm.storePtr(ICTailCallReg, Address(StackPointer, offsetof(BaselineStubFrame, + returnAddress))); + + // Save old frame pointer, stack pointer and stub reg. + masm.storePtr(ICStubReg, + Address(StackPointer, offsetof(BaselineStubFrame, savedStub))); + masm.storePtr(BaselineFrameReg, + Address(StackPointer, offsetof(BaselineStubFrame, savedFrame))); + masm.movePtr(BaselineStackReg, BaselineFrameReg); + + // Stack should remain aligned. + masm.assertStackAlignment(sizeof(Value), 0); +} + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_SharedICHelpers_loong64_inl_h */ diff --git a/js/src/jit/loong64/SharedICHelpers-loong64.h b/js/src/jit/loong64/SharedICHelpers-loong64.h new file mode 100644 index 0000000000..a32558bd1f --- /dev/null +++ b/js/src/jit/loong64/SharedICHelpers-loong64.h @@ -0,0 +1,104 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_SharedICHelpers_loong64_h +#define jit_loong64_SharedICHelpers_loong64_h + +#include "jit/BaselineIC.h" +#include "jit/JitFrames.h" +#include "jit/MacroAssembler.h" +#include "jit/SharedICRegisters.h" + +namespace js { +namespace jit { + +// Distance from sp to the top Value inside an IC stub (no return address on +// the stack on LoongArch). +static const size_t ICStackValueOffset = 0; + +struct BaselineStubFrame { + uintptr_t savedFrame; + uintptr_t savedStub; + uintptr_t returnAddress; + uintptr_t descriptor; +}; + +// Size of values pushed by EmitBaselineEnterStubFrame. +static const uint32_t STUB_FRAME_SIZE = sizeof(BaselineStubFrame); +static const uint32_t STUB_FRAME_SAVED_STUB_OFFSET = + offsetof(BaselineStubFrame, savedStub); + +inline void EmitRestoreTailCallReg(MacroAssembler& masm) { + // No-op on LA because ra register is always holding the return address. +} + +inline void EmitRepushTailCallReg(MacroAssembler& masm) { + // No-op on LA because ra register is always holding the return address. +} + +inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) { + // The stub pointer must already be in ICStubReg. + // Load stubcode pointer from the ICStub. + // R2 won't be active when we call ICs, so we can use it as scratch. + masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg()); + + // Call the stubcode via a direct jump-and-link + masm.call(R2.scratchReg()); + *callOffset = CodeOffset(masm.currentOffset()); +} + +inline void EmitReturnFromIC(MacroAssembler& masm) { masm.branch(ra); } + +inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm, + bool calledIntoIon = false) { + // Ion frames do not save and restore the frame pointer. If we called + // into Ion, we have to restore the stack pointer from the frame descriptor. + // If we performed a VM call, the descriptor has been popped already so + // in that case we use the frame pointer. + if (calledIntoIon) { + ScratchRegisterScope scratch(masm); + masm.pop(scratch); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch); + masm.addPtr(scratch, BaselineStackReg); + } else { + masm.movePtr(BaselineFrameReg, BaselineStackReg); + } + + // Load savedFrame, savedStub and return address, discard frame descriptor. + masm.loadPtr(Address(StackPointer, offsetof(BaselineStubFrame, savedFrame)), + BaselineFrameReg); + masm.loadPtr(Address(StackPointer, offsetof(BaselineStubFrame, savedStub)), + ICStubReg); + masm.loadPtr( + Address(StackPointer, offsetof(BaselineStubFrame, returnAddress)), + ICTailCallReg); + masm.addPtr(Imm32(STUB_FRAME_SIZE), StackPointer); + + masm.checkStackAlignment(); +} + +template +inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr, + MIRType type) { + // On LoongArch, $ra is clobbered by guardedCallPreBarrier. Save it first. + masm.push(ra); + masm.guardedCallPreBarrier(addr, type); + masm.pop(ra); +} + +inline void EmitStubGuardFailure(MacroAssembler& masm) { + // Load next stub into ICStubReg + masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg); + + // Return address is already loaded, just jump to the next stubcode. + MOZ_ASSERT(ICTailCallReg == ra); + masm.jump(Address(ICStubReg, ICStub::offsetOfStubCode())); +} + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_SharedICHelpers_loong64_h */ diff --git a/js/src/jit/loong64/SharedICRegisters-loong64.h b/js/src/jit/loong64/SharedICRegisters-loong64.h new file mode 100644 index 0000000000..826c824152 --- /dev/null +++ b/js/src/jit/loong64/SharedICRegisters-loong64.h @@ -0,0 +1,48 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loong64_SharedICRegisters_loong64_h +#define jit_loong64_SharedICRegisters_loong64_h + +#include "jit/loong64/Assembler-loong64.h" +#include "jit/Registers.h" +#include "jit/RegisterSets.h" + +namespace js { +namespace jit { + +static constexpr Register BaselineFrameReg = s5; +static constexpr Register BaselineStackReg = sp; + +// ValueOperands R0, R1, and R2. +// R0 == JSReturnReg, and R2 uses registers not preserved across calls. R1 value +// should be preserved across calls. +static constexpr ValueOperand R0(a2); +static constexpr ValueOperand R1(s0); +static constexpr ValueOperand R2(a0); + +// ICTailCallReg and ICStubReg +// These use registers that are not preserved across calls. +static constexpr Register ICTailCallReg = ra; +static constexpr Register ICStubReg = a5; + +static constexpr Register ExtractTemp0 = s6; +static constexpr Register ExtractTemp1 = s7; + +// Note that ICTailCallReg is actually just the link register. +// In MIPS code emission, we do not clobber ICTailCallReg since we keep +// the return address for calls there. + +// FloatReg0 must be equal to ReturnFloatReg. +static constexpr FloatRegister FloatReg0 = f0; +static constexpr FloatRegister FloatReg1 = f1; +static constexpr FloatRegister FloatReg2 = f2; +static constexpr FloatRegister FloatReg3 = f3; + +} // namespace jit +} // namespace js + +#endif /* jit_loong64_SharedICRegisters_loong64_h */ diff --git a/js/src/jit/loong64/Simulator-loong64.cpp b/js/src/jit/loong64/Simulator-loong64.cpp new file mode 100644 index 0000000000..0da738c5c2 --- /dev/null +++ b/js/src/jit/loong64/Simulator-loong64.cpp @@ -0,0 +1,6045 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: */ +// Copyright 2020 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "jit/loong64/Simulator-loong64.h" + +#include +#include + +#include "jit/AtomicOperations.h" +#include "jit/loong64/Assembler-loong64.h" +#include "threading/LockGuard.h" +#include "vm/Runtime.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmSignalHandlers.h" + +#define I8(v) static_cast(v) +#define I16(v) static_cast(v) +#define U16(v) static_cast(v) +#define I32(v) static_cast(v) +#define U32(v) static_cast(v) +#define I64(v) static_cast(v) +#define U64(v) static_cast(v) +#define I128(v) static_cast<__int128_t>(v) +#define U128(v) static_cast<__uint128_t>(v) + +#define I32_CHECK(v) \ + ({ \ + MOZ_ASSERT(I64(I32(v)) == I64(v)); \ + I32((v)); \ + }) + +namespace js { +namespace jit { + +/* print the instr + * 1 - need + * 0 - not need + */ +#define PRINT_SIM_LOG 0 + +#if PRINT_SIM_LOG +inline void printf_instr(const char* _Format, ...) { + va_list varList; + va_start(varList, _Format); + vprintf(_Format, varList); + va_end(varList); +} +#else +# define printf_instr(...) +#endif + +static int64_t MultiplyHighSigned(int64_t u, int64_t v) { + uint64_t u0, v0, w0; + int64_t u1, v1, w1, w2, t; + + u0 = u & 0xFFFFFFFFL; + u1 = u >> 32; + v0 = v & 0xFFFFFFFFL; + v1 = v >> 32; + + w0 = u0 * v0; + t = u1 * v0 + (w0 >> 32); + w1 = t & 0xFFFFFFFFL; + w2 = t >> 32; + w1 = u0 * v1 + w1; + + return u1 * v1 + w2 + (w1 >> 32); +} + +static uint64_t MultiplyHighUnsigned(uint64_t u, uint64_t v) { + uint64_t u0, v0, w0; + uint64_t u1, v1, w1, w2, t; + + u0 = u & 0xFFFFFFFFL; + u1 = u >> 32; + v0 = v & 0xFFFFFFFFL; + v1 = v >> 32; + + w0 = u0 * v0; + t = u1 * v0 + (w0 >> 32); + w1 = t & 0xFFFFFFFFL; + w2 = t >> 32; + w1 = u0 * v1 + w1; + + return u1 * v1 + w2 + (w1 >> 32); +} + +// Precondition: 0 <= shift < 32 +inline constexpr uint32_t RotateRight32(uint32_t value, uint32_t shift) { + return (value >> shift) | (value << ((32 - shift) & 31)); +} + +// Precondition: 0 <= shift < 32 +inline constexpr uint32_t RotateLeft32(uint32_t value, uint32_t shift) { + return (value << shift) | (value >> ((32 - shift) & 31)); +} + +// Precondition: 0 <= shift < 64 +inline constexpr uint64_t RotateRight64(uint64_t value, uint64_t shift) { + return (value >> shift) | (value << ((64 - shift) & 63)); +} + +// Precondition: 0 <= shift < 64 +inline constexpr uint64_t RotateLeft64(uint64_t value, uint64_t shift) { + return (value << shift) | (value >> ((64 - shift) & 63)); +} + +// break instr with MAX_BREAK_CODE. +static const Instr kCallRedirInstr = op_break | Imm15Mask; + +// ----------------------------------------------------------------------------- +// LOONG64 assembly various constants. + +class SimInstruction { + public: + enum { + kInstrSize = 4, + // On Loongisa PC cannot actually be directly accessed. We behave as if PC + // was always the value of the current instruction being executed. + kPCReadOffset = 0 + }; + + // Get the raw instruction bits. + inline Instr instructionBits() const { + return *reinterpret_cast(this); + } + + // Set the raw instruction bits to value. + inline void setInstructionBits(Instr value) { + *reinterpret_cast(this) = value; + } + + // Read one particular bit out of the instruction bits. + inline int bit(int nr) const { return (instructionBits() >> nr) & 1; } + + // Read a bit field out of the instruction bits. + inline int bits(int hi, int lo) const { + return (instructionBits() >> lo) & ((2 << (hi - lo)) - 1); + } + + // Instruction type. + enum Type { + kUnsupported = -1, + kOp6Type, + kOp7Type, + kOp8Type, + kOp10Type, + kOp11Type, + kOp12Type, + kOp14Type, + kOp15Type, + kOp16Type, + kOp17Type, + kOp22Type, + kOp24Type + }; + + // Get the encoding type of the instruction. + Type instructionType() const; + + inline int rjValue() const { return bits(RJShift + RJBits - 1, RJShift); } + + inline int rkValue() const { return bits(RKShift + RKBits - 1, RKShift); } + + inline int rdValue() const { return bits(RDShift + RDBits - 1, RDShift); } + + inline int sa2Value() const { return bits(SAShift + SA2Bits - 1, SAShift); } + + inline int sa3Value() const { return bits(SAShift + SA3Bits - 1, SAShift); } + + inline int lsbwValue() const { + return bits(LSBWShift + LSBWBits - 1, LSBWShift); + } + + inline int msbwValue() const { + return bits(MSBWShift + MSBWBits - 1, MSBWShift); + } + + inline int lsbdValue() const { + return bits(LSBDShift + LSBDBits - 1, LSBDShift); + } + + inline int msbdValue() const { + return bits(MSBDShift + MSBDBits - 1, MSBDShift); + } + + inline int fdValue() const { return bits(FDShift + FDBits - 1, FDShift); } + + inline int fjValue() const { return bits(FJShift + FJBits - 1, FJShift); } + + inline int fkValue() const { return bits(FKShift + FKBits - 1, FKShift); } + + inline int faValue() const { return bits(FAShift + FABits - 1, FAShift); } + + inline int cdValue() const { return bits(CDShift + CDBits - 1, CDShift); } + + inline int cjValue() const { return bits(CJShift + CJBits - 1, CJShift); } + + inline int caValue() const { return bits(CAShift + CABits - 1, CAShift); } + + inline int condValue() const { + return bits(CONDShift + CONDBits - 1, CONDShift); + } + + inline int imm5Value() const { + return bits(Imm5Shift + Imm5Bits - 1, Imm5Shift); + } + + inline int imm6Value() const { + return bits(Imm6Shift + Imm6Bits - 1, Imm6Shift); + } + + inline int imm12Value() const { + return bits(Imm12Shift + Imm12Bits - 1, Imm12Shift); + } + + inline int imm14Value() const { + return bits(Imm14Shift + Imm14Bits - 1, Imm14Shift); + } + + inline int imm16Value() const { + return bits(Imm16Shift + Imm16Bits - 1, Imm16Shift); + } + + inline int imm20Value() const { + return bits(Imm20Shift + Imm20Bits - 1, Imm20Shift); + } + + inline int32_t imm26Value() const { + return bits(Imm26Shift + Imm26Bits - 1, Imm26Shift); + } + + // Say if the instruction is a debugger break/trap. + bool isTrap() const; + + private: + SimInstruction() = delete; + SimInstruction(const SimInstruction& other) = delete; + void operator=(const SimInstruction& other) = delete; +}; + +bool SimInstruction::isTrap() const { + // is break?? + switch (bits(31, 15) << 15) { + case op_break: + return (instructionBits() != kCallRedirInstr) && (bits(15, 0) != 6); + default: + return false; + }; +} + +SimInstruction::Type SimInstruction::instructionType() const { + SimInstruction::Type kType = kUnsupported; + + // Check for kOp6Type + switch (bits(31, 26) << 26) { + case op_beqz: + case op_bnez: + case op_bcz: + case op_jirl: + case op_b: + case op_bl: + case op_beq: + case op_bne: + case op_blt: + case op_bge: + case op_bltu: + case op_bgeu: + case op_addu16i_d: + kType = kOp6Type; + break; + default: + kType = kUnsupported; + } + + if (kType == kUnsupported) { + // Check for kOp7Type + switch (bits(31, 25) << 25) { + case op_lu12i_w: + case op_lu32i_d: + case op_pcaddi: + case op_pcalau12i: + case op_pcaddu12i: + case op_pcaddu18i: + kType = kOp7Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp8Type + switch (bits(31, 24) << 24) { + case op_ll_w: + case op_sc_w: + case op_ll_d: + case op_sc_d: + case op_ldptr_w: + case op_stptr_w: + case op_ldptr_d: + case op_stptr_d: + kType = kOp8Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp10Type + switch (bits(31, 22) << 22) { + case op_bstrins_d: + case op_bstrpick_d: + case op_slti: + case op_sltui: + case op_addi_w: + case op_addi_d: + case op_lu52i_d: + case op_andi: + case op_ori: + case op_xori: + case op_ld_b: + case op_ld_h: + case op_ld_w: + case op_ld_d: + case op_st_b: + case op_st_h: + case op_st_w: + case op_st_d: + case op_ld_bu: + case op_ld_hu: + case op_ld_wu: + case op_preld: + case op_fld_s: + case op_fst_s: + case op_fld_d: + case op_fst_d: + case op_bstr_w: // BSTRINS_W & BSTRPICK_W + kType = kOp10Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp11Type + switch (bits(31, 21) << 21) { + case op_bstr_w: + kType = kOp11Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp12Type + switch (bits(31, 20) << 20) { + case op_fmadd_s: + case op_fmadd_d: + case op_fmsub_s: + case op_fmsub_d: + case op_fnmadd_s: + case op_fnmadd_d: + case op_fnmsub_s: + case op_fnmsub_d: + case op_fcmp_cond_s: + case op_fcmp_cond_d: + kType = kOp12Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp14Type + switch (bits(31, 18) << 18) { + case op_bytepick_d: + case op_fsel: + kType = kOp14Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp15Type + switch (bits(31, 17) << 17) { + case op_bytepick_w: + case op_alsl_w: + case op_alsl_wu: + case op_alsl_d: + kType = kOp15Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp16Type + switch (bits(31, 16) << 16) { + case op_slli_d: + case op_srli_d: + case op_srai_d: + case op_rotri_d: + kType = kOp16Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp17Type + switch (bits(31, 15) << 15) { + case op_slli_w: + case op_srli_w: + case op_srai_w: + case op_rotri_w: + case op_add_w: + case op_add_d: + case op_sub_w: + case op_sub_d: + case op_slt: + case op_sltu: + case op_maskeqz: + case op_masknez: + case op_nor: + case op_and: + case op_or: + case op_xor: + case op_orn: + case op_andn: + case op_sll_w: + case op_srl_w: + case op_sra_w: + case op_sll_d: + case op_srl_d: + case op_sra_d: + case op_rotr_w: + case op_rotr_d: + case op_mul_w: + case op_mul_d: + case op_mulh_d: + case op_mulh_du: + case op_mulh_w: + case op_mulh_wu: + case op_mulw_d_w: + case op_mulw_d_wu: + case op_div_w: + case op_mod_w: + case op_div_wu: + case op_mod_wu: + case op_div_d: + case op_mod_d: + case op_div_du: + case op_mod_du: + case op_break: + case op_fadd_s: + case op_fadd_d: + case op_fsub_s: + case op_fsub_d: + case op_fmul_s: + case op_fmul_d: + case op_fdiv_s: + case op_fdiv_d: + case op_fmax_s: + case op_fmax_d: + case op_fmin_s: + case op_fmin_d: + case op_fmaxa_s: + case op_fmaxa_d: + case op_fmina_s: + case op_fmina_d: + case op_fcopysign_s: + case op_fcopysign_d: + case op_ldx_b: + case op_ldx_h: + case op_ldx_w: + case op_ldx_d: + case op_stx_b: + case op_stx_h: + case op_stx_w: + case op_stx_d: + case op_ldx_bu: + case op_ldx_hu: + case op_ldx_wu: + case op_fldx_s: + case op_fldx_d: + case op_fstx_s: + case op_fstx_d: + case op_amswap_w: + case op_amswap_d: + case op_amadd_w: + case op_amadd_d: + case op_amand_w: + case op_amand_d: + case op_amor_w: + case op_amor_d: + case op_amxor_w: + case op_amxor_d: + case op_ammax_w: + case op_ammax_d: + case op_ammin_w: + case op_ammin_d: + case op_ammax_wu: + case op_ammax_du: + case op_ammin_wu: + case op_ammin_du: + case op_amswap_db_w: + case op_amswap_db_d: + case op_amadd_db_w: + case op_amadd_db_d: + case op_amand_db_w: + case op_amand_db_d: + case op_amor_db_w: + case op_amor_db_d: + case op_amxor_db_w: + case op_amxor_db_d: + case op_ammax_db_w: + case op_ammax_db_d: + case op_ammin_db_w: + case op_ammin_db_d: + case op_ammax_db_wu: + case op_ammax_db_du: + case op_ammin_db_wu: + case op_ammin_db_du: + case op_dbar: + case op_ibar: + kType = kOp17Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp22Type + switch (bits(31, 10) << 10) { + case op_clo_w: + case op_clz_w: + case op_cto_w: + case op_ctz_w: + case op_clo_d: + case op_clz_d: + case op_cto_d: + case op_ctz_d: + case op_revb_2h: + case op_revb_4h: + case op_revb_2w: + case op_revb_d: + case op_revh_2w: + case op_revh_d: + case op_bitrev_4b: + case op_bitrev_8b: + case op_bitrev_w: + case op_bitrev_d: + case op_ext_w_h: + case op_ext_w_b: + case op_fabs_s: + case op_fabs_d: + case op_fneg_s: + case op_fneg_d: + case op_fsqrt_s: + case op_fsqrt_d: + case op_fmov_s: + case op_fmov_d: + case op_movgr2fr_w: + case op_movgr2fr_d: + case op_movgr2frh_w: + case op_movfr2gr_s: + case op_movfr2gr_d: + case op_movfrh2gr_s: + case op_movfcsr2gr: + case op_movfr2cf: + case op_movgr2cf: + case op_fcvt_s_d: + case op_fcvt_d_s: + case op_ftintrm_w_s: + case op_ftintrm_w_d: + case op_ftintrm_l_s: + case op_ftintrm_l_d: + case op_ftintrp_w_s: + case op_ftintrp_w_d: + case op_ftintrp_l_s: + case op_ftintrp_l_d: + case op_ftintrz_w_s: + case op_ftintrz_w_d: + case op_ftintrz_l_s: + case op_ftintrz_l_d: + case op_ftintrne_w_s: + case op_ftintrne_w_d: + case op_ftintrne_l_s: + case op_ftintrne_l_d: + case op_ftint_w_s: + case op_ftint_w_d: + case op_ftint_l_s: + case op_ftint_l_d: + case op_ffint_s_w: + case op_ffint_s_l: + case op_ffint_d_w: + case op_ffint_d_l: + case op_frint_s: + case op_frint_d: + kType = kOp22Type; + break; + default: + kType = kUnsupported; + } + } + + if (kType == kUnsupported) { + // Check for kOp24Type + switch (bits(31, 8) << 8) { + case op_movcf2fr: + case op_movcf2gr: + kType = kOp24Type; + break; + default: + kType = kUnsupported; + } + } + + return kType; +} + +// C/C++ argument slots size. +const int kCArgSlotCount = 0; +const int kCArgsSlotsSize = kCArgSlotCount * sizeof(uintptr_t); + +class CachePage { + public: + static const int LINE_VALID = 0; + static const int LINE_INVALID = 1; + + static const int kPageShift = 12; + static const int kPageSize = 1 << kPageShift; + static const int kPageMask = kPageSize - 1; + static const int kLineShift = 2; // The cache line is only 4 bytes right now. + static const int kLineLength = 1 << kLineShift; + static const int kLineMask = kLineLength - 1; + + CachePage() { memset(&validity_map_, LINE_INVALID, sizeof(validity_map_)); } + + char* validityByte(int offset) { + return &validity_map_[offset >> kLineShift]; + } + + char* cachedData(int offset) { return &data_[offset]; } + + private: + char data_[kPageSize]; // The cached data. + static const int kValidityMapSize = kPageSize >> kLineShift; + char validity_map_[kValidityMapSize]; // One byte per line. +}; + +// Protects the icache() and redirection() properties of the +// Simulator. +class AutoLockSimulatorCache : public LockGuard { + using Base = LockGuard; + + public: + explicit AutoLockSimulatorCache() + : Base(SimulatorProcess::singleton_->cacheLock_) {} +}; + +mozilla::Atomic + SimulatorProcess::ICacheCheckingDisableCount( + 1); // Checking is disabled by default. +SimulatorProcess* SimulatorProcess::singleton_ = nullptr; + +int64_t Simulator::StopSimAt = -1; + +Simulator* Simulator::Create() { + auto sim = MakeUnique(); + if (!sim) { + return nullptr; + } + + if (!sim->init()) { + return nullptr; + } + + int64_t stopAt; + char* stopAtStr = getenv("MIPS_SIM_STOP_AT"); + if (stopAtStr && sscanf(stopAtStr, "%" PRIi64, &stopAt) == 1) { + fprintf(stderr, "\nStopping simulation at icount %" PRIi64 "\n", stopAt); + Simulator::StopSimAt = stopAt; + } + + return sim.release(); +} + +void Simulator::Destroy(Simulator* sim) { js_delete(sim); } + +// The loong64Debugger class is used by the simulator while debugging simulated +// code. +class loong64Debugger { + public: + explicit loong64Debugger(Simulator* sim) : sim_(sim) {} + + void stop(SimInstruction* instr); + void debug(); + // Print all registers with a nice formatting. + void printAllRegs(); + void printAllRegsIncludingFPU(); + + private: + // We set the breakpoint code to 0x7fff to easily recognize it. + static const Instr kBreakpointInstr = op_break | (0x7fff & Imm15Mask); + static const Instr kNopInstr = 0x0; + + Simulator* sim_; + + int64_t getRegisterValue(int regnum); + int64_t getFPURegisterValueLong(int regnum); + float getFPURegisterValueFloat(int regnum); + double getFPURegisterValueDouble(int regnum); + bool getValue(const char* desc, int64_t* value); + + // Set or delete a breakpoint. Returns true if successful. + bool setBreakpoint(SimInstruction* breakpc); + bool deleteBreakpoint(SimInstruction* breakpc); + + // Undo and redo all breakpoints. This is needed to bracket disassembly and + // execution to skip past breakpoints when run from the debugger. + void undoBreakpoints(); + void redoBreakpoints(); +}; + +static void UNIMPLEMENTED() { + printf("UNIMPLEMENTED instruction.\n"); + MOZ_CRASH(); +} +static void UNREACHABLE() { + printf("UNREACHABLE instruction.\n"); + MOZ_CRASH(); +} +static void UNSUPPORTED() { + printf("Unsupported instruction.\n"); + MOZ_CRASH(); +} + +void loong64Debugger::stop(SimInstruction* instr) { + // Get the stop code. + uint32_t code = instr->bits(25, 6); + // Retrieve the encoded address, which comes just after this stop. + char* msg = + *reinterpret_cast(sim_->get_pc() + SimInstruction::kInstrSize); + // Update this stop description. + if (!sim_->watchedStops_[code].desc_) { + sim_->watchedStops_[code].desc_ = msg; + } + // Print the stop message and code if it is not the default code. + if (code != kMaxStopCode) { + printf("Simulator hit stop %u: %s\n", code, msg); + } else { + printf("Simulator hit %s\n", msg); + } + sim_->set_pc(sim_->get_pc() + 2 * SimInstruction::kInstrSize); + // sim_->set_pc(sim_->get_pc() + 1 * SimInstruction::kInstrSize); + debug(); +} + +int64_t loong64Debugger::getRegisterValue(int regnum) { + if (regnum == kPCRegister) { + // if (regnum == kNumSimuRegisters) { //same as V8. + return sim_->get_pc(); + } + return sim_->getRegister(regnum); +} + +int64_t loong64Debugger::getFPURegisterValueLong(int regnum) { + return sim_->getFpuRegister(regnum); +} + +float loong64Debugger::getFPURegisterValueFloat(int regnum) { + return sim_->getFpuRegisterFloat(regnum); +} + +double loong64Debugger::getFPURegisterValueDouble(int regnum) { + return sim_->getFpuRegisterDouble(regnum); +} + +bool loong64Debugger::getValue(const char* desc, int64_t* value) { + Register reg = Register::FromName(desc); + if (reg != InvalidReg) { + *value = getRegisterValue(reg.code()); + return true; + } + + if (strncmp(desc, "0x", 2) == 0) { + return sscanf(desc + 2, "%lx", reinterpret_cast(value)) == 1; + } + return sscanf(desc, "%lu", reinterpret_cast(value)) == 1; +} + +bool loong64Debugger::setBreakpoint(SimInstruction* breakpc) { + // Check if a breakpoint can be set. If not return without any side-effects. + if (sim_->break_pc_ != nullptr) { + return false; + } + + // Set the breakpoint. + sim_->break_pc_ = breakpc; + sim_->break_instr_ = breakpc->instructionBits(); + // Not setting the breakpoint instruction in the code itself. It will be set + // when the debugger shell continues. + return true; +} + +bool loong64Debugger::deleteBreakpoint(SimInstruction* breakpc) { + if (sim_->break_pc_ != nullptr) { + sim_->break_pc_->setInstructionBits(sim_->break_instr_); + } + + sim_->break_pc_ = nullptr; + sim_->break_instr_ = 0; + return true; +} + +void loong64Debugger::undoBreakpoints() { + if (sim_->break_pc_) { + sim_->break_pc_->setInstructionBits(sim_->break_instr_); + } +} + +void loong64Debugger::redoBreakpoints() { + if (sim_->break_pc_) { + sim_->break_pc_->setInstructionBits(kBreakpointInstr); + } +} + +void loong64Debugger::printAllRegs() { + int64_t value; + for (uint32_t i = 0; i < Registers::Total; i++) { + value = getRegisterValue(i); + printf("%3s: 0x%016" PRIx64 " %20" PRIi64 " ", Registers::GetName(i), + value, value); + + if (i % 2) { + printf("\n"); + } + } + printf("\n"); + + value = getRegisterValue(Simulator::pc); + printf(" pc: 0x%016" PRIx64 "\n", value); +} + +void loong64Debugger::printAllRegsIncludingFPU() { + printAllRegs(); + + printf("\n\n"); + // f0, f1, f2, ... f31. + for (uint32_t i = 0; i < FloatRegisters::TotalPhys; i++) { + printf("%3s: 0x%016" PRIi64 "\tflt: %-8.4g\tdbl: %-16.4g\n", + FloatRegisters::GetName(i), getFPURegisterValueLong(i), + getFPURegisterValueFloat(i), getFPURegisterValueDouble(i)); + } +} + +static char* ReadLine(const char* prompt) { + UniqueChars result; + char lineBuf[256]; + int offset = 0; + bool keepGoing = true; + fprintf(stdout, "%s", prompt); + fflush(stdout); + while (keepGoing) { + if (fgets(lineBuf, sizeof(lineBuf), stdin) == nullptr) { + // fgets got an error. Just give up. + return nullptr; + } + int len = strlen(lineBuf); + if (len > 0 && lineBuf[len - 1] == '\n') { + // Since we read a new line we are done reading the line. This + // will exit the loop after copying this buffer into the result. + keepGoing = false; + } + if (!result) { + // Allocate the initial result and make room for the terminating '\0' + result.reset(js_pod_malloc(len + 1)); + if (!result) { + return nullptr; + } + } else { + // Allocate a new result with enough room for the new addition. + int new_len = offset + len + 1; + char* new_result = js_pod_malloc(new_len); + if (!new_result) { + return nullptr; + } + // Copy the existing input into the new array and set the new + // array as the result. + memcpy(new_result, result.get(), offset * sizeof(char)); + result.reset(new_result); + } + // Copy the newly read line into the result. + memcpy(result.get() + offset, lineBuf, len * sizeof(char)); + offset += len; + } + + MOZ_ASSERT(result); + result[offset] = '\0'; + return result.release(); +} + +static void DisassembleInstruction(uint64_t pc) { + // uint8_t* bytes = reinterpret_cast(pc); + // char hexbytes[256]; + // sprintf(hexbytes, "0x%x 0x%x 0x%x 0x%x", bytes[0], bytes[1], bytes[2], + // bytes[3]); + // char llvmcmd[1024]; + // sprintf(llvmcmd, + // "bash -c \"echo -n '%p'; echo '%s' | " + // "llvm-mc -disassemble -arch=mips64el -mcpu=mips64r2 | " + // "grep -v pure_instructions | grep -v .text\"", + // static_cast(bytes), hexbytes); + // if (system(llvmcmd)) { + // printf("Cannot disassemble instruction.\n"); + // } +} + +void loong64Debugger::debug() { + intptr_t lastPC = -1; + bool done = false; + +#define COMMAND_SIZE 63 +#define ARG_SIZE 255 + +#define STR(a) #a +#define XSTR(a) STR(a) + + char cmd[COMMAND_SIZE + 1]; + char arg1[ARG_SIZE + 1]; + char arg2[ARG_SIZE + 1]; + char* argv[3] = {cmd, arg1, arg2}; + + // Make sure to have a proper terminating character if reaching the limit. + cmd[COMMAND_SIZE] = 0; + arg1[ARG_SIZE] = 0; + arg2[ARG_SIZE] = 0; + + // Undo all set breakpoints while running in the debugger shell. This will + // make them invisible to all commands. + undoBreakpoints(); + + while (!done && (sim_->get_pc() != Simulator::end_sim_pc)) { + if (lastPC != sim_->get_pc()) { + DisassembleInstruction(sim_->get_pc()); + printf(" 0x%016" PRIi64 " \n", sim_->get_pc()); + lastPC = sim_->get_pc(); + } + char* line = ReadLine("sim> "); + if (line == nullptr) { + break; + } else { + char* last_input = sim_->lastDebuggerInput(); + if (strcmp(line, "\n") == 0 && last_input != nullptr) { + line = last_input; + } else { + // Ownership is transferred to sim_; + sim_->setLastDebuggerInput(line); + } + // Use sscanf to parse the individual parts of the command line. At the + // moment no command expects more than two parameters. + int argc = sscanf(line, + "%" XSTR(COMMAND_SIZE) "s " + "%" XSTR(ARG_SIZE) "s " + "%" XSTR(ARG_SIZE) "s", + cmd, arg1, arg2); + if ((strcmp(cmd, "si") == 0) || (strcmp(cmd, "stepi") == 0)) { + SimInstruction* instr = + reinterpret_cast(sim_->get_pc()); + if (!instr->isTrap()) { + sim_->instructionDecode( + reinterpret_cast(sim_->get_pc())); + } else { + // Allow si to jump over generated breakpoints. + printf("/!\\ Jumping over generated breakpoint.\n"); + sim_->set_pc(sim_->get_pc() + SimInstruction::kInstrSize); + } + sim_->icount_++; + } else if ((strcmp(cmd, "c") == 0) || (strcmp(cmd, "cont") == 0)) { + // Execute the one instruction we broke at with breakpoints disabled. + sim_->instructionDecode( + reinterpret_cast(sim_->get_pc())); + sim_->icount_++; + // Leave the debugger shell. + done = true; + } else if ((strcmp(cmd, "p") == 0) || (strcmp(cmd, "print") == 0)) { + if (argc == 2) { + int64_t value; + if (strcmp(arg1, "all") == 0) { + printAllRegs(); + } else if (strcmp(arg1, "allf") == 0) { + printAllRegsIncludingFPU(); + } else { + Register reg = Register::FromName(arg1); + FloatRegisters::Code fReg = FloatRegisters::FromName(arg1); + if (reg != InvalidReg) { + value = getRegisterValue(reg.code()); + printf("%s: 0x%016" PRIi64 " %20" PRIi64 " \n", arg1, value, + value); + } else if (fReg != FloatRegisters::Invalid) { + printf("%3s: 0x%016" PRIi64 "\tflt: %-8.4g\tdbl: %-16.4g\n", + FloatRegisters::GetName(fReg), + getFPURegisterValueLong(fReg), + getFPURegisterValueFloat(fReg), + getFPURegisterValueDouble(fReg)); + } else { + printf("%s unrecognized\n", arg1); + } + } + } else { + printf("print or print single\n"); + } + } else if (strcmp(cmd, "stack") == 0 || strcmp(cmd, "mem") == 0) { + int64_t* cur = nullptr; + int64_t* end = nullptr; + int next_arg = 1; + + if (strcmp(cmd, "stack") == 0) { + cur = reinterpret_cast(sim_->getRegister(Simulator::sp)); + } else { // Command "mem". + int64_t value; + if (!getValue(arg1, &value)) { + printf("%s unrecognized\n", arg1); + continue; + } + cur = reinterpret_cast(value); + next_arg++; + } + + int64_t words; + if (argc == next_arg) { + words = 10; + } else { + if (!getValue(argv[next_arg], &words)) { + words = 10; + } + } + end = cur + words; + + while (cur < end) { + printf(" %p: 0x%016" PRIx64 " %20" PRIi64, cur, *cur, *cur); + printf("\n"); + cur++; + } + + } else if ((strcmp(cmd, "disasm") == 0) || (strcmp(cmd, "dpc") == 0) || + (strcmp(cmd, "di") == 0)) { + uint8_t* cur = nullptr; + uint8_t* end = nullptr; + + if (argc == 1) { + cur = reinterpret_cast(sim_->get_pc()); + end = cur + (10 * SimInstruction::kInstrSize); + } else if (argc == 2) { + Register reg = Register::FromName(arg1); + if (reg != InvalidReg || strncmp(arg1, "0x", 2) == 0) { + // The argument is an address or a register name. + int64_t value; + if (getValue(arg1, &value)) { + cur = reinterpret_cast(value); + // Disassemble 10 instructions at . + end = cur + (10 * SimInstruction::kInstrSize); + } + } else { + // The argument is the number of instructions. + int64_t value; + if (getValue(arg1, &value)) { + cur = reinterpret_cast(sim_->get_pc()); + // Disassemble instructions. + end = cur + (value * SimInstruction::kInstrSize); + } + } + } else { + int64_t value1; + int64_t value2; + if (getValue(arg1, &value1) && getValue(arg2, &value2)) { + cur = reinterpret_cast(value1); + end = cur + (value2 * SimInstruction::kInstrSize); + } + } + + while (cur < end) { + DisassembleInstruction(uint64_t(cur)); + cur += SimInstruction::kInstrSize; + } + } else if (strcmp(cmd, "gdb") == 0) { + printf("relinquishing control to gdb\n"); + asm("int $3"); + printf("regaining control from gdb\n"); + } else if (strcmp(cmd, "break") == 0) { + if (argc == 2) { + int64_t value; + if (getValue(arg1, &value)) { + if (!setBreakpoint(reinterpret_cast(value))) { + printf("setting breakpoint failed\n"); + } + } else { + printf("%s unrecognized\n", arg1); + } + } else { + printf("break
\n"); + } + } else if (strcmp(cmd, "del") == 0) { + if (!deleteBreakpoint(nullptr)) { + printf("deleting breakpoint failed\n"); + } + } else if (strcmp(cmd, "flags") == 0) { + printf("No flags on LOONG64 !\n"); + } else if (strcmp(cmd, "stop") == 0) { + int64_t value; + intptr_t stop_pc = sim_->get_pc() - 2 * SimInstruction::kInstrSize; + SimInstruction* stop_instr = reinterpret_cast(stop_pc); + SimInstruction* msg_address = reinterpret_cast( + stop_pc + SimInstruction::kInstrSize); + if ((argc == 2) && (strcmp(arg1, "unstop") == 0)) { + // Remove the current stop. + if (sim_->isStopInstruction(stop_instr)) { + stop_instr->setInstructionBits(kNopInstr); + msg_address->setInstructionBits(kNopInstr); + } else { + printf("Not at debugger stop.\n"); + } + } else if (argc == 3) { + // Print information about all/the specified breakpoint(s). + if (strcmp(arg1, "info") == 0) { + if (strcmp(arg2, "all") == 0) { + printf("Stop information:\n"); + for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode; + i++) { + sim_->printStopInfo(i); + } + } else if (getValue(arg2, &value)) { + sim_->printStopInfo(value); + } else { + printf("Unrecognized argument.\n"); + } + } else if (strcmp(arg1, "enable") == 0) { + // Enable all/the specified breakpoint(s). + if (strcmp(arg2, "all") == 0) { + for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode; + i++) { + sim_->enableStop(i); + } + } else if (getValue(arg2, &value)) { + sim_->enableStop(value); + } else { + printf("Unrecognized argument.\n"); + } + } else if (strcmp(arg1, "disable") == 0) { + // Disable all/the specified breakpoint(s). + if (strcmp(arg2, "all") == 0) { + for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode; + i++) { + sim_->disableStop(i); + } + } else if (getValue(arg2, &value)) { + sim_->disableStop(value); + } else { + printf("Unrecognized argument.\n"); + } + } + } else { + printf("Wrong usage. Use help command for more information.\n"); + } + } else if ((strcmp(cmd, "h") == 0) || (strcmp(cmd, "help") == 0)) { + printf("cont\n"); + printf(" continue execution (alias 'c')\n"); + printf("stepi\n"); + printf(" step one instruction (alias 'si')\n"); + printf("print \n"); + printf(" print register content (alias 'p')\n"); + printf(" use register name 'all' to print all registers\n"); + printf("printobject \n"); + printf(" print an object from a register (alias 'po')\n"); + printf("stack []\n"); + printf(" dump stack content, default dump 10 words)\n"); + printf("mem
[]\n"); + printf(" dump memory content, default dump 10 words)\n"); + printf("flags\n"); + printf(" print flags\n"); + printf("disasm []\n"); + printf("disasm [
]\n"); + printf("disasm [[
] ]\n"); + printf(" disassemble code, default is 10 instructions\n"); + printf(" from pc (alias 'di')\n"); + printf("gdb\n"); + printf(" enter gdb\n"); + printf("break
\n"); + printf(" set a break point on the address\n"); + printf("del\n"); + printf(" delete the breakpoint\n"); + printf("stop feature:\n"); + printf(" Description:\n"); + printf(" Stops are debug instructions inserted by\n"); + printf(" the Assembler::stop() function.\n"); + printf(" When hitting a stop, the Simulator will\n"); + printf(" stop and and give control to the Debugger.\n"); + printf(" All stop codes are watched:\n"); + printf(" - They can be enabled / disabled: the Simulator\n"); + printf(" will / won't stop when hitting them.\n"); + printf(" - The Simulator keeps track of how many times they \n"); + printf(" are met. (See the info command.) Going over a\n"); + printf(" disabled stop still increases its counter. \n"); + printf(" Commands:\n"); + printf(" stop info all/ : print infos about number \n"); + printf(" or all stop(s).\n"); + printf(" stop enable/disable all/ : enables / disables\n"); + printf(" all or number stop(s)\n"); + printf(" stop unstop\n"); + printf(" ignore the stop instruction at the current location\n"); + printf(" from now on\n"); + } else { + printf("Unknown command: %s\n", cmd); + } + } + } + + // Add all the breakpoints back to stop execution and enter the debugger + // shell when hit. + redoBreakpoints(); + +#undef COMMAND_SIZE +#undef ARG_SIZE + +#undef STR +#undef XSTR +} + +static bool AllOnOnePage(uintptr_t start, int size) { + intptr_t start_page = (start & ~CachePage::kPageMask); + intptr_t end_page = ((start + size) & ~CachePage::kPageMask); + return start_page == end_page; +} + +void Simulator::setLastDebuggerInput(char* input) { + js_free(lastDebuggerInput_); + lastDebuggerInput_ = input; +} + +static CachePage* GetCachePageLocked(SimulatorProcess::ICacheMap& i_cache, + void* page) { + SimulatorProcess::ICacheMap::AddPtr p = i_cache.lookupForAdd(page); + if (p) { + return p->value(); + } + AutoEnterOOMUnsafeRegion oomUnsafe; + CachePage* new_page = js_new(); + if (!new_page || !i_cache.add(p, page, new_page)) { + oomUnsafe.crash("Simulator CachePage"); + } + return new_page; +} + +// Flush from start up to and not including start + size. +static void FlushOnePageLocked(SimulatorProcess::ICacheMap& i_cache, + intptr_t start, int size) { + MOZ_ASSERT(size <= CachePage::kPageSize); + MOZ_ASSERT(AllOnOnePage(start, size - 1)); + MOZ_ASSERT((start & CachePage::kLineMask) == 0); + MOZ_ASSERT((size & CachePage::kLineMask) == 0); + void* page = reinterpret_cast(start & (~CachePage::kPageMask)); + int offset = (start & CachePage::kPageMask); + CachePage* cache_page = GetCachePageLocked(i_cache, page); + char* valid_bytemap = cache_page->validityByte(offset); + memset(valid_bytemap, CachePage::LINE_INVALID, size >> CachePage::kLineShift); +} + +static void FlushICacheLocked(SimulatorProcess::ICacheMap& i_cache, + void* start_addr, size_t size) { + intptr_t start = reinterpret_cast(start_addr); + int intra_line = (start & CachePage::kLineMask); + start -= intra_line; + size += intra_line; + size = ((size - 1) | CachePage::kLineMask) + 1; + int offset = (start & CachePage::kPageMask); + while (!AllOnOnePage(start, size - 1)) { + int bytes_to_flush = CachePage::kPageSize - offset; + FlushOnePageLocked(i_cache, start, bytes_to_flush); + start += bytes_to_flush; + size -= bytes_to_flush; + MOZ_ASSERT((start & CachePage::kPageMask) == 0); + offset = 0; + } + if (size != 0) { + FlushOnePageLocked(i_cache, start, size); + } +} + +/* static */ +void SimulatorProcess::checkICacheLocked(SimInstruction* instr) { + intptr_t address = reinterpret_cast(instr); + void* page = reinterpret_cast(address & (~CachePage::kPageMask)); + void* line = reinterpret_cast(address & (~CachePage::kLineMask)); + int offset = (address & CachePage::kPageMask); + CachePage* cache_page = GetCachePageLocked(icache(), page); + char* cache_valid_byte = cache_page->validityByte(offset); + bool cache_hit = (*cache_valid_byte == CachePage::LINE_VALID); + char* cached_line = cache_page->cachedData(offset & ~CachePage::kLineMask); + + if (cache_hit) { + // Check that the data in memory matches the contents of the I-cache. + int cmpret = + memcmp(reinterpret_cast(instr), cache_page->cachedData(offset), + SimInstruction::kInstrSize); + MOZ_ASSERT(cmpret == 0); + } else { + // Cache miss. Load memory into the cache. + memcpy(cached_line, line, CachePage::kLineLength); + *cache_valid_byte = CachePage::LINE_VALID; + } +} + +HashNumber SimulatorProcess::ICacheHasher::hash(const Lookup& l) { + return U32(reinterpret_cast(l)) >> 2; +} + +bool SimulatorProcess::ICacheHasher::match(const Key& k, const Lookup& l) { + MOZ_ASSERT((reinterpret_cast(k) & CachePage::kPageMask) == 0); + MOZ_ASSERT((reinterpret_cast(l) & CachePage::kPageMask) == 0); + return k == l; +} + +/* static */ +void SimulatorProcess::FlushICache(void* start_addr, size_t size) { + if (!ICacheCheckingDisableCount) { + AutoLockSimulatorCache als; + js::jit::FlushICacheLocked(icache(), start_addr, size); + } +} + +Simulator::Simulator() { + // Set up simulator support first. Some of this information is needed to + // setup the architecture state. + + // Note, allocation and anything that depends on allocated memory is + // deferred until init(), in order to handle OOM properly. + + stack_ = nullptr; + stackLimit_ = 0; + pc_modified_ = false; + icount_ = 0; + break_count_ = 0; + break_pc_ = nullptr; + break_instr_ = 0; + single_stepping_ = false; + single_step_callback_ = nullptr; + single_step_callback_arg_ = nullptr; + + // Set up architecture state. + // All registers are initialized to zero to start with. + for (int i = 0; i < Register::kNumSimuRegisters; i++) { + registers_[i] = 0; + } + for (int i = 0; i < Simulator::FPURegister::kNumFPURegisters; i++) { + FPUregisters_[i] = 0; + } + + for (int i = 0; i < kNumCFRegisters; i++) { + CFregisters_[i] = 0; + } + + FCSR_ = 0; + LLBit_ = false; + LLAddr_ = 0; + lastLLValue_ = 0; + + // The ra and pc are initialized to a known bad value that will cause an + // access violation if the simulator ever tries to execute it. + registers_[pc] = bad_ra; + registers_[ra] = bad_ra; + + for (int i = 0; i < kNumExceptions; i++) { + exceptions[i] = 0; + } + + lastDebuggerInput_ = nullptr; +} + +bool Simulator::init() { + // Allocate 2MB for the stack. Note that we will only use 1MB, see below. + static const size_t stackSize = 2 * 1024 * 1024; + stack_ = js_pod_malloc(stackSize); + if (!stack_) { + return false; + } + + // Leave a safety margin of 1MB to prevent overrunning the stack when + // pushing values (total stack size is 2MB). + stackLimit_ = reinterpret_cast(stack_) + 1024 * 1024; + + // The sp is initialized to point to the bottom (high address) of the + // allocated stack area. To be safe in potential stack underflows we leave + // some buffer below. + registers_[sp] = reinterpret_cast(stack_) + stackSize - 64; + + return true; +} + +// When the generated code calls an external reference we need to catch that in +// the simulator. The external reference will be a function compiled for the +// host architecture. We need to call that function instead of trying to +// execute it with the simulator. We do that by redirecting the external +// reference to a swi (software-interrupt) instruction that is handled by +// the simulator. We write the original destination of the jump just at a known +// offset from the swi instruction so the simulator knows what to call. +class Redirection { + friend class SimulatorProcess; + + // sim's lock must already be held. + Redirection(void* nativeFunction, ABIFunctionType type) + : nativeFunction_(nativeFunction), + swiInstruction_(kCallRedirInstr), + type_(type), + next_(nullptr) { + next_ = SimulatorProcess::redirection(); + if (!SimulatorProcess::ICacheCheckingDisableCount) { + FlushICacheLocked(SimulatorProcess::icache(), addressOfSwiInstruction(), + SimInstruction::kInstrSize); + } + SimulatorProcess::setRedirection(this); + } + + public: + void* addressOfSwiInstruction() { return &swiInstruction_; } + void* nativeFunction() const { return nativeFunction_; } + ABIFunctionType type() const { return type_; } + + static Redirection* Get(void* nativeFunction, ABIFunctionType type) { + AutoLockSimulatorCache als; + + Redirection* current = SimulatorProcess::redirection(); + for (; current != nullptr; current = current->next_) { + if (current->nativeFunction_ == nativeFunction) { + MOZ_ASSERT(current->type() == type); + return current; + } + } + + // Note: we can't use js_new here because the constructor is private. + AutoEnterOOMUnsafeRegion oomUnsafe; + Redirection* redir = js_pod_malloc(1); + if (!redir) { + oomUnsafe.crash("Simulator redirection"); + } + new (redir) Redirection(nativeFunction, type); + return redir; + } + + static Redirection* FromSwiInstruction(SimInstruction* swiInstruction) { + uint8_t* addrOfSwi = reinterpret_cast(swiInstruction); + uint8_t* addrOfRedirection = + addrOfSwi - offsetof(Redirection, swiInstruction_); + return reinterpret_cast(addrOfRedirection); + } + + private: + void* nativeFunction_; + uint32_t swiInstruction_; + ABIFunctionType type_; + Redirection* next_; +}; + +Simulator::~Simulator() { js_free(stack_); } + +SimulatorProcess::SimulatorProcess() + : cacheLock_(mutexid::SimulatorCacheLock), redirection_(nullptr) { + if (getenv("MIPS_SIM_ICACHE_CHECKS")) { + ICacheCheckingDisableCount = 0; + } +} + +SimulatorProcess::~SimulatorProcess() { + Redirection* r = redirection_; + while (r) { + Redirection* next = r->next_; + js_delete(r); + r = next; + } +} + +/* static */ +void* Simulator::RedirectNativeFunction(void* nativeFunction, + ABIFunctionType type) { + Redirection* redirection = Redirection::Get(nativeFunction, type); + return redirection->addressOfSwiInstruction(); +} + +// Get the active Simulator for the current thread. +Simulator* Simulator::Current() { + JSContext* cx = TlsContext.get(); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(cx->runtime())); + return cx->simulator(); +} + +// Sets the register in the architecture state. It will also deal with updating +// Simulator internal state for special registers such as PC. +void Simulator::setRegister(int reg, int64_t value) { + MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters)); + if (reg == pc) { + pc_modified_ = true; + } + + // Zero register always holds 0. + registers_[reg] = (reg == 0) ? 0 : value; +} + +void Simulator::setFpuRegister(int fpureg, int64_t value) { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + FPUregisters_[fpureg] = value; +} + +void Simulator::setFpuRegisterHiWord(int fpureg, int32_t value) { + // Set ONLY upper 32-bits, leaving lower bits untouched. + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + int32_t* phiword; + phiword = (reinterpret_cast(&FPUregisters_[fpureg])) + 1; + + *phiword = value; +} + +void Simulator::setFpuRegisterWord(int fpureg, int32_t value) { + // Set ONLY lower 32-bits, leaving upper bits untouched. + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + int32_t* pword; + pword = reinterpret_cast(&FPUregisters_[fpureg]); + + *pword = value; +} + +// 1: +void Simulator::setFpuRegisterWordInvalidResult(float original, float rounded, + int fpureg) { + double max_int32 = static_cast(INT32_MAX); + double min_int32 = static_cast(INT32_MIN); + + if (std::isnan(original)) { + setFpuRegisterWord(fpureg, 0); + } else if (rounded > max_int32) { + setFpuRegister(fpureg, kFPUInvalidResult); + } else if (rounded < min_int32) { + setFpuRegister(fpureg, kFPUInvalidResultNegative); + } else { + UNREACHABLE(); + } +} +// 2: +void Simulator::setFpuRegisterWordInvalidResult(double original, double rounded, + int fpureg) { + double max_int32 = static_cast(INT32_MAX); + double min_int32 = static_cast(INT32_MIN); + + if (std::isnan(original)) { + setFpuRegisterWord(fpureg, 0); + } else if (rounded > max_int32) { + setFpuRegisterWord(fpureg, kFPUInvalidResult); + } else if (rounded < min_int32) { + setFpuRegisterWord(fpureg, kFPUInvalidResultNegative); + } else { + UNREACHABLE(); + } +} + +// 3: +void Simulator::setFpuRegisterInvalidResult(float original, float rounded, + int fpureg) { + double max_int32 = static_cast(INT32_MAX); + double min_int32 = static_cast(INT32_MIN); + + if (std::isnan(original)) { + setFpuRegister(fpureg, 0); + } else if (rounded > max_int32) { + setFpuRegister(fpureg, kFPUInvalidResult); + } else if (rounded < min_int32) { + setFpuRegister(fpureg, kFPUInvalidResultNegative); + } else { + UNREACHABLE(); + } +} + +// 4: +void Simulator::setFpuRegisterInvalidResult(double original, double rounded, + int fpureg) { + double max_int32 = static_cast(INT32_MAX); + double min_int32 = static_cast(INT32_MIN); + + if (std::isnan(original)) { + setFpuRegister(fpureg, 0); + } else if (rounded > max_int32) { + setFpuRegister(fpureg, kFPUInvalidResult); + } else if (rounded < min_int32) { + setFpuRegister(fpureg, kFPUInvalidResultNegative); + } else { + UNREACHABLE(); + } +} + +// 5: +void Simulator::setFpuRegisterInvalidResult64(float original, float rounded, + int fpureg) { + // The value of INT64_MAX (2^63-1) can't be represented as double exactly, + // loading the most accurate representation into max_int64, which is 2^63. + double max_int64 = static_cast(INT64_MAX); + double min_int64 = static_cast(INT64_MIN); + + if (std::isnan(original)) { + setFpuRegister(fpureg, 0); + } else if (rounded >= max_int64) { + setFpuRegister(fpureg, kFPU64InvalidResult); + } else if (rounded < min_int64) { + setFpuRegister(fpureg, kFPU64InvalidResultNegative); + } else { + UNREACHABLE(); + } +} + +// 6: +void Simulator::setFpuRegisterInvalidResult64(double original, double rounded, + int fpureg) { + // The value of INT64_MAX (2^63-1) can't be represented as double exactly, + // loading the most accurate representation into max_int64, which is 2^63. + double max_int64 = static_cast(INT64_MAX); + double min_int64 = static_cast(INT64_MIN); + + if (std::isnan(original)) { + setFpuRegister(fpureg, 0); + } else if (rounded >= max_int64) { + setFpuRegister(fpureg, kFPU64InvalidResult); + } else if (rounded < min_int64) { + setFpuRegister(fpureg, kFPU64InvalidResultNegative); + } else { + UNREACHABLE(); + } +} + +void Simulator::setFpuRegisterFloat(int fpureg, float value) { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + *mozilla::BitwiseCast(&FPUregisters_[fpureg]) = value; +} + +void Simulator::setFpuRegisterDouble(int fpureg, double value) { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + *mozilla::BitwiseCast(&FPUregisters_[fpureg]) = value; +} + +void Simulator::setCFRegister(int cfreg, bool value) { + MOZ_ASSERT((cfreg >= 0) && (cfreg < kNumCFRegisters)); + CFregisters_[cfreg] = value; +} + +bool Simulator::getCFRegister(int cfreg) const { + MOZ_ASSERT((cfreg >= 0) && (cfreg < kNumCFRegisters)); + return CFregisters_[cfreg]; +} + +// Get the register from the architecture state. This function does handle +// the special case of accessing the PC register. +int64_t Simulator::getRegister(int reg) const { + MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters)); + if (reg == 0) { + return 0; + } + return registers_[reg] + ((reg == pc) ? SimInstruction::kPCReadOffset : 0); +} + +int64_t Simulator::getFpuRegister(int fpureg) const { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + return FPUregisters_[fpureg]; +} + +int32_t Simulator::getFpuRegisterWord(int fpureg) const { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + return *mozilla::BitwiseCast(&FPUregisters_[fpureg]); +} + +int32_t Simulator::getFpuRegisterSignedWord(int fpureg) const { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + return *mozilla::BitwiseCast(&FPUregisters_[fpureg]); +} + +int32_t Simulator::getFpuRegisterHiWord(int fpureg) const { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + return *((mozilla::BitwiseCast(&FPUregisters_[fpureg])) + 1); +} + +float Simulator::getFpuRegisterFloat(int fpureg) const { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + return *mozilla::BitwiseCast(&FPUregisters_[fpureg]); +} + +double Simulator::getFpuRegisterDouble(int fpureg) const { + MOZ_ASSERT((fpureg >= 0) && + (fpureg < Simulator::FPURegister::kNumFPURegisters)); + return *mozilla::BitwiseCast(&FPUregisters_[fpureg]); +} + +void Simulator::setCallResultDouble(double result) { + setFpuRegisterDouble(f0, result); +} + +void Simulator::setCallResultFloat(float result) { + setFpuRegisterFloat(f0, result); +} + +void Simulator::setCallResult(int64_t res) { setRegister(v0, res); } + +void Simulator::setCallResult(__int128_t res) { + setRegister(v0, I64(res)); + setRegister(v1, I64(res >> 64)); +} + +// Helper functions for setting and testing the FCSR register's bits. +void Simulator::setFCSRBit(uint32_t cc, bool value) { + if (value) { + FCSR_ |= (1 << cc); + } else { + FCSR_ &= ~(1 << cc); + } +} + +bool Simulator::testFCSRBit(uint32_t cc) { return FCSR_ & (1 << cc); } + +unsigned int Simulator::getFCSRRoundingMode() { + return FCSR_ & kFPURoundingModeMask; +} + +// Sets the rounding error codes in FCSR based on the result of the rounding. +// Returns true if the operation was invalid. +template +bool Simulator::setFCSRRoundError(double original, double rounded) { + bool ret = false; + + setFCSRBit(kFCSRInexactCauseBit, false); + setFCSRBit(kFCSRUnderflowCauseBit, false); + setFCSRBit(kFCSROverflowCauseBit, false); + setFCSRBit(kFCSRInvalidOpCauseBit, false); + + if (!std::isfinite(original) || !std::isfinite(rounded)) { + setFCSRBit(kFCSRInvalidOpFlagBit, true); + setFCSRBit(kFCSRInvalidOpCauseBit, true); + ret = true; + } + + if (original != rounded) { + setFCSRBit(kFCSRInexactFlagBit, true); + setFCSRBit(kFCSRInexactCauseBit, true); + } + + if (rounded < DBL_MIN && rounded > -DBL_MIN && rounded != 0) { + setFCSRBit(kFCSRUnderflowFlagBit, true); + setFCSRBit(kFCSRUnderflowCauseBit, true); + ret = true; + } + + if ((long double)rounded > (long double)std::numeric_limits::max() || + (long double)rounded < (long double)std::numeric_limits::min()) { + setFCSRBit(kFCSROverflowFlagBit, true); + setFCSRBit(kFCSROverflowCauseBit, true); + // The reference is not really clear but it seems this is required: + setFCSRBit(kFCSRInvalidOpFlagBit, true); + setFCSRBit(kFCSRInvalidOpCauseBit, true); + ret = true; + } + + return ret; +} + +// For cvt instructions only +template +void Simulator::roundAccordingToFCSR(T toRound, T* rounded, + int32_t* rounded_int) { + // v8: roundAccordingToFCSR have double and float arguments + // 0 RN (round to nearest): Round a result to the nearest + // representable value; if the result is exactly halfway between + // two representable values, round to zero. Behave like round_w_d. + + // 1 RZ (round toward zero): Round a result to the closest + // representable value whose absolute value is less than or + // equal to the infinitely accurate result. Behave like trunc_w_d. + + // 2 RP (round up, or toward +infinity): Round a result to the + // next representable value up. Behave like ceil_w_d. + + // 3 RN (round down, or toward −infinity): Round a result to + // the next representable value down. Behave like floor_w_d. + switch ((FCSR_ >> 8) & 3) { + case kRoundToNearest: + *rounded = std::floor(toRound + 0.5); + *rounded_int = static_cast(*rounded); + if ((*rounded_int & 1) != 0 && *rounded_int - toRound == 0.5) { + // If the number is halfway between two integers, + // round to the even one. + *rounded_int -= 1; + *rounded -= 1.; + } + break; + case kRoundToZero: + *rounded = trunc(toRound); + *rounded_int = static_cast(*rounded); + break; + case kRoundToPlusInf: + *rounded = std::ceil(toRound); + *rounded_int = static_cast(*rounded); + break; + case kRoundToMinusInf: + *rounded = std::floor(toRound); + *rounded_int = static_cast(*rounded); + break; + } +} + +template +void Simulator::round64AccordingToFCSR(T toRound, T* rounded, + int64_t* rounded_int) { + // 0 RN (round to nearest): Round a result to the nearest + // representable value; if the result is exactly halfway between + // two representable values, round to zero. Behave like round_w_d. + + // 1 RZ (round toward zero): Round a result to the closest + // representable value whose absolute value is less than or. + // equal to the infinitely accurate result. Behave like trunc_w_d. + + // 2 RP (round up, or toward +infinity): Round a result to the + // next representable value up. Behave like ceil_w_d. + + // 3 RN (round down, or toward −infinity): Round a result to + // the next representable value down. Behave like floor_w_d. + switch ((FCSR_ >> 8) & 3) { + case kRoundToNearest: + *rounded = std::floor(toRound + 0.5); + *rounded_int = static_cast(*rounded); + if ((*rounded_int & 1) != 0 && *rounded_int - toRound == 0.5) { + // If the number is halfway between two integers, + // round to the even one. + *rounded_int -= 1; + *rounded -= 1.; + } + break; + case kRoundToZero: + *rounded = trunc(toRound); + *rounded_int = static_cast(*rounded); + break; + case kRoundToPlusInf: + *rounded = std::ceil(toRound); + *rounded_int = static_cast(*rounded); + break; + case kRoundToMinusInf: + *rounded = std::floor(toRound); + *rounded_int = static_cast(*rounded); + break; + } +} + +// Raw access to the PC register. +void Simulator::set_pc(int64_t value) { + pc_modified_ = true; + registers_[pc] = value; +} + +bool Simulator::has_bad_pc() const { + return ((registers_[pc] == bad_ra) || (registers_[pc] == end_sim_pc)); +} + +// Raw access to the PC register without the special adjustment when reading. +int64_t Simulator::get_pc() const { return registers_[pc]; } + +JS::ProfilingFrameIterator::RegisterState Simulator::registerState() { + wasm::RegisterState state; + state.pc = (void*)get_pc(); + state.fp = (void*)getRegister(fp); + state.sp = (void*)getRegister(sp); + state.lr = (void*)getRegister(ra); + return state; +} + +// MIPS memory instructions (except lw(d)l/r , sw(d)l/r) trap on unaligned +// memory access enabling the OS to handle them via trap-and-emulate. Note that +// simulator runs have the runtime system running directly on the host system +// and only generated code is executed in the simulator. Since the host is +// typically IA32 it will not trap on unaligned memory access. We assume that +// that executing correct generated code will not produce unaligned memory +// access, so we explicitly check for address alignment and trap. Note that +// trapping does not occur when executing wasm code, which requires that +// unaligned memory access provides correct result. + +uint8_t Simulator::readBU(uint64_t addr) { + if (handleWasmSegFault(addr, 1)) { + return 0xff; + } + + uint8_t* ptr = reinterpret_cast(addr); + return *ptr; +} + +int8_t Simulator::readB(uint64_t addr) { + if (handleWasmSegFault(addr, 1)) { + return -1; + } + + int8_t* ptr = reinterpret_cast(addr); + return *ptr; +} + +void Simulator::writeB(uint64_t addr, uint8_t value) { + if (handleWasmSegFault(addr, 1)) { + return; + } + + uint8_t* ptr = reinterpret_cast(addr); + *ptr = value; +} + +void Simulator::writeB(uint64_t addr, int8_t value) { + if (handleWasmSegFault(addr, 1)) { + return; + } + + int8_t* ptr = reinterpret_cast(addr); + *ptr = value; +} + +uint16_t Simulator::readHU(uint64_t addr, SimInstruction* instr) { + if (handleWasmSegFault(addr, 2)) { + return 0xffff; + } + + uint16_t* ptr = reinterpret_cast(addr); + return *ptr; +} + +int16_t Simulator::readH(uint64_t addr, SimInstruction* instr) { + if (handleWasmSegFault(addr, 2)) { + return -1; + } + + int16_t* ptr = reinterpret_cast(addr); + return *ptr; +} + +void Simulator::writeH(uint64_t addr, uint16_t value, SimInstruction* instr) { + if (handleWasmSegFault(addr, 2)) { + return; + } + + uint16_t* ptr = reinterpret_cast(addr); + LLBit_ = false; + *ptr = value; + return; +} + +void Simulator::writeH(uint64_t addr, int16_t value, SimInstruction* instr) { + if (handleWasmSegFault(addr, 2)) { + return; + } + + int16_t* ptr = reinterpret_cast(addr); + LLBit_ = false; + *ptr = value; + return; +} + +uint32_t Simulator::readWU(uint64_t addr, SimInstruction* instr) { + if (handleWasmSegFault(addr, 4)) { + return -1; + } + + uint32_t* ptr = reinterpret_cast(addr); + return *ptr; +} + +int32_t Simulator::readW(uint64_t addr, SimInstruction* instr) { + if (handleWasmSegFault(addr, 4)) { + return -1; + } + + int32_t* ptr = reinterpret_cast(addr); + return *ptr; +} + +void Simulator::writeW(uint64_t addr, uint32_t value, SimInstruction* instr) { + if (handleWasmSegFault(addr, 4)) { + return; + } + + uint32_t* ptr = reinterpret_cast(addr); + LLBit_ = false; + *ptr = value; + return; +} + +void Simulator::writeW(uint64_t addr, int32_t value, SimInstruction* instr) { + if (handleWasmSegFault(addr, 4)) { + return; + } + + int32_t* ptr = reinterpret_cast(addr); + LLBit_ = false; + *ptr = value; + return; +} + +int64_t Simulator::readDW(uint64_t addr, SimInstruction* instr) { + if (handleWasmSegFault(addr, 8)) { + return -1; + } + + intptr_t* ptr = reinterpret_cast(addr); + return *ptr; +} + +void Simulator::writeDW(uint64_t addr, int64_t value, SimInstruction* instr) { + if (handleWasmSegFault(addr, 8)) { + return; + } + + int64_t* ptr = reinterpret_cast(addr); + LLBit_ = false; + *ptr = value; + return; +} + +double Simulator::readD(uint64_t addr, SimInstruction* instr) { + if (handleWasmSegFault(addr, 8)) { + return NAN; + } + + double* ptr = reinterpret_cast(addr); + return *ptr; +} + +void Simulator::writeD(uint64_t addr, double value, SimInstruction* instr) { + if (handleWasmSegFault(addr, 8)) { + return; + } + + double* ptr = reinterpret_cast(addr); + LLBit_ = false; + *ptr = value; + return; +} + +int Simulator::loadLinkedW(uint64_t addr, SimInstruction* instr) { + if ((addr & 3) == 0) { + if (handleWasmSegFault(addr, 4)) { + return -1; + } + + volatile int32_t* ptr = reinterpret_cast(addr); + int32_t value = *ptr; + lastLLValue_ = value; + LLAddr_ = addr; + // Note that any memory write or "external" interrupt should reset this + // value to false. + LLBit_ = true; + return value; + } + printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr, + reinterpret_cast(instr)); + MOZ_CRASH(); + return 0; +} + +int Simulator::storeConditionalW(uint64_t addr, int value, + SimInstruction* instr) { + // Correct behavior in this case, as defined by architecture, is to just + // return 0, but there is no point at allowing that. It is certainly an + // indicator of a bug. + if (addr != LLAddr_) { + printf("SC to bad address: 0x%016" PRIx64 ", pc=0x%016" PRIx64 + ", expected: 0x%016" PRIx64 "\n", + addr, reinterpret_cast(instr), LLAddr_); + MOZ_CRASH(); + } + + if ((addr & 3) == 0) { + SharedMem ptr = + SharedMem::shared(reinterpret_cast(addr)); + + if (!LLBit_) { + return 0; + } + + LLBit_ = false; + LLAddr_ = 0; + int32_t expected = int32_t(lastLLValue_); + int32_t old = + AtomicOperations::compareExchangeSeqCst(ptr, expected, int32_t(value)); + return (old == expected) ? 1 : 0; + } + printf("Unaligned SC at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr, + reinterpret_cast(instr)); + MOZ_CRASH(); + return 0; +} + +int64_t Simulator::loadLinkedD(uint64_t addr, SimInstruction* instr) { + if ((addr & kPointerAlignmentMask) == 0) { + if (handleWasmSegFault(addr, 8)) { + return -1; + } + + volatile int64_t* ptr = reinterpret_cast(addr); + int64_t value = *ptr; + lastLLValue_ = value; + LLAddr_ = addr; + // Note that any memory write or "external" interrupt should reset this + // value to false. + LLBit_ = true; + return value; + } + printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr, + reinterpret_cast(instr)); + MOZ_CRASH(); + return 0; +} + +int Simulator::storeConditionalD(uint64_t addr, int64_t value, + SimInstruction* instr) { + // Correct behavior in this case, as defined by architecture, is to just + // return 0, but there is no point at allowing that. It is certainly an + // indicator of a bug. + if (addr != LLAddr_) { + printf("SC to bad address: 0x%016" PRIx64 ", pc=0x%016" PRIx64 + ", expected: 0x%016" PRIx64 "\n", + addr, reinterpret_cast(instr), LLAddr_); + MOZ_CRASH(); + } + + if ((addr & kPointerAlignmentMask) == 0) { + SharedMem ptr = + SharedMem::shared(reinterpret_cast(addr)); + + if (!LLBit_) { + return 0; + } + + LLBit_ = false; + LLAddr_ = 0; + int64_t expected = lastLLValue_; + int64_t old = + AtomicOperations::compareExchangeSeqCst(ptr, expected, int64_t(value)); + return (old == expected) ? 1 : 0; + } + printf("Unaligned SC at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr, + reinterpret_cast(instr)); + MOZ_CRASH(); + return 0; +} + +uintptr_t Simulator::stackLimit() const { return stackLimit_; } + +uintptr_t* Simulator::addressOfStackLimit() { return &stackLimit_; } + +bool Simulator::overRecursed(uintptr_t newsp) const { + if (newsp == 0) { + newsp = getRegister(sp); + } + return newsp <= stackLimit(); +} + +bool Simulator::overRecursedWithExtra(uint32_t extra) const { + uintptr_t newsp = getRegister(sp) - extra; + return newsp <= stackLimit(); +} + +// Unsupported instructions use format to print an error and stop execution. +void Simulator::format(SimInstruction* instr, const char* format) { + printf("Simulator found unsupported instruction:\n 0x%016lx: %s\n", + reinterpret_cast(instr), format); + MOZ_CRASH(); +} + +// Note: With the code below we assume that all runtime calls return a 64 bits +// result. If they don't, the v1 result register contains a bogus value, which +// is fine because it is caller-saved. +typedef int64_t (*Prototype_General0)(); +typedef int64_t (*Prototype_General1)(int64_t arg0); +typedef int64_t (*Prototype_General2)(int64_t arg0, int64_t arg1); +typedef int64_t (*Prototype_General3)(int64_t arg0, int64_t arg1, int64_t arg2); +typedef int64_t (*Prototype_General4)(int64_t arg0, int64_t arg1, int64_t arg2, + int64_t arg3); +typedef int64_t (*Prototype_General5)(int64_t arg0, int64_t arg1, int64_t arg2, + int64_t arg3, int64_t arg4); +typedef int64_t (*Prototype_General6)(int64_t arg0, int64_t arg1, int64_t arg2, + int64_t arg3, int64_t arg4, int64_t arg5); +typedef int64_t (*Prototype_General7)(int64_t arg0, int64_t arg1, int64_t arg2, + int64_t arg3, int64_t arg4, int64_t arg5, + int64_t arg6); +typedef int64_t (*Prototype_General8)(int64_t arg0, int64_t arg1, int64_t arg2, + int64_t arg3, int64_t arg4, int64_t arg5, + int64_t arg6, int64_t arg7); +typedef int64_t (*Prototype_GeneralGeneralGeneralInt64)(int64_t arg0, + int64_t arg1, + int64_t arg2, + int64_t arg3); +typedef int64_t (*Prototype_GeneralGeneralInt64Int64)(int64_t arg0, + int64_t arg1, + int64_t arg2, + int64_t arg3); +typedef int64_t (*Prototype_Int_Double)(double arg0); +typedef int64_t (*Prototype_Int_IntDouble)(int64_t arg0, double arg1); +typedef int64_t (*Prototype_Int_DoubleInt)(double arg0, int64_t arg1); +typedef int64_t (*Prototype_Int_DoubleIntInt)(double arg0, int64_t arg1, + int64_t arg2); +typedef int64_t (*Prototype_Int_IntDoubleIntInt)(int64_t arg0, double arg1, + int64_t arg2, int64_t arg3); + +typedef float (*Prototype_Float32_Float32)(float arg0); +typedef float (*Prototype_Float32_Float32Float32)(float arg0, float arg1); + +typedef double (*Prototype_Double_None)(); +typedef double (*Prototype_Double_Double)(double arg0); +typedef double (*Prototype_Double_Int)(int64_t arg0); +typedef double (*Prototype_Double_DoubleInt)(double arg0, int64_t arg1); +typedef double (*Prototype_Double_IntDouble)(int64_t arg0, double arg1); +typedef double (*Prototype_Double_DoubleDouble)(double arg0, double arg1); +typedef double (*Prototype_Double_DoubleDoubleDouble)(double arg0, double arg1, + double arg2); +typedef double (*Prototype_Double_DoubleDoubleDoubleDouble)(double arg0, + double arg1, + double arg2, + double arg3); + +typedef int32_t (*Prototype_Int32_General)(int64_t); +typedef int32_t (*Prototype_Int32_GeneralInt32)(int64_t, int32_t); +typedef int32_t (*Prototype_Int32_GeneralInt32Int32)(int64_t, int32_t, int32_t); +typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32)(int64_t, int32_t, + int32_t, int32_t, + int32_t); +typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32Int32)( + int64_t, int32_t, int32_t, int32_t, int32_t, int32_t); +typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32General)( + int64_t, int32_t, int32_t, int32_t, int64_t); +typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int64)(int64_t, int32_t, + int32_t, int64_t); +typedef int32_t (*Prototype_Int32_GeneralInt32Int32General)(int64_t, int32_t, + int32_t, int64_t); +typedef int32_t (*Prototype_Int32_GeneralInt32Int64Int64)(int64_t, int32_t, + int64_t, int64_t); +typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32)(int64_t, int32_t, + int64_t, int32_t); +typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32Int32)( + int64_t, int32_t, int64_t, int32_t, int32_t); +typedef int32_t (*Prototype_Int32_GeneralGeneral)(int64_t, int64_t); +typedef int32_t (*Prototype_Int32_GeneralGeneralInt32Int32)(int64_t, int64_t, + int32_t, int32_t); +typedef int64_t (*Prototype_General_GeneralInt32)(int64_t, int32_t); +typedef int64_t (*Prototype_General_GeneralInt32Int32)(int64_t, int32_t, + int32_t); + +inline int32_t Simulator::rj_reg(SimInstruction* instr) const { + return instr->rjValue(); +} + +inline int64_t Simulator::rj(SimInstruction* instr) const { + return getRegister(rj_reg(instr)); +} + +inline uint64_t Simulator::rj_u(SimInstruction* instr) const { + return static_cast(getRegister(rj_reg(instr))); +} + +inline int32_t Simulator::rk_reg(SimInstruction* instr) const { + return instr->rkValue(); +} + +inline int64_t Simulator::rk(SimInstruction* instr) const { + return getRegister(rk_reg(instr)); +} + +inline uint64_t Simulator::rk_u(SimInstruction* instr) const { + return static_cast(getRegister(rk_reg(instr))); +} + +inline int32_t Simulator::rd_reg(SimInstruction* instr) const { + return instr->rdValue(); +} + +inline int64_t Simulator::rd(SimInstruction* instr) const { + return getRegister(rd_reg(instr)); +} + +inline uint64_t Simulator::rd_u(SimInstruction* instr) const { + return static_cast(getRegister(rd_reg(instr))); +} + +inline int32_t Simulator::fa_reg(SimInstruction* instr) const { + return instr->faValue(); +} + +inline float Simulator::fa_float(SimInstruction* instr) const { + return getFpuRegisterFloat(fa_reg(instr)); +} + +inline double Simulator::fa_double(SimInstruction* instr) const { + return getFpuRegisterDouble(fa_reg(instr)); +} + +inline int32_t Simulator::fj_reg(SimInstruction* instr) const { + return instr->fjValue(); +} + +inline float Simulator::fj_float(SimInstruction* instr) const { + return getFpuRegisterFloat(fj_reg(instr)); +} + +inline double Simulator::fj_double(SimInstruction* instr) const { + return getFpuRegisterDouble(fj_reg(instr)); +} + +inline int32_t Simulator::fk_reg(SimInstruction* instr) const { + return instr->fkValue(); +} + +inline float Simulator::fk_float(SimInstruction* instr) const { + return getFpuRegisterFloat(fk_reg(instr)); +} + +inline double Simulator::fk_double(SimInstruction* instr) const { + return getFpuRegisterDouble(fk_reg(instr)); +} + +inline int32_t Simulator::fd_reg(SimInstruction* instr) const { + return instr->fdValue(); +} + +inline float Simulator::fd_float(SimInstruction* instr) const { + return getFpuRegisterFloat(fd_reg(instr)); +} + +inline double Simulator::fd_double(SimInstruction* instr) const { + return getFpuRegisterDouble(fd_reg(instr)); +} + +inline int32_t Simulator::cj_reg(SimInstruction* instr) const { + return instr->cjValue(); +} + +inline bool Simulator::cj(SimInstruction* instr) const { + return getCFRegister(cj_reg(instr)); +} + +inline int32_t Simulator::cd_reg(SimInstruction* instr) const { + return instr->cdValue(); +} + +inline bool Simulator::cd(SimInstruction* instr) const { + return getCFRegister(cd_reg(instr)); +} + +inline int32_t Simulator::ca_reg(SimInstruction* instr) const { + return instr->caValue(); +} + +inline bool Simulator::ca(SimInstruction* instr) const { + return getCFRegister(ca_reg(instr)); +} + +inline uint32_t Simulator::sa2(SimInstruction* instr) const { + return instr->sa2Value(); +} + +inline uint32_t Simulator::sa3(SimInstruction* instr) const { + return instr->sa3Value(); +} + +inline uint32_t Simulator::ui5(SimInstruction* instr) const { + return instr->imm5Value(); +} + +inline uint32_t Simulator::ui6(SimInstruction* instr) const { + return instr->imm6Value(); +} + +inline uint32_t Simulator::lsbw(SimInstruction* instr) const { + return instr->lsbwValue(); +} + +inline uint32_t Simulator::msbw(SimInstruction* instr) const { + return instr->msbwValue(); +} + +inline uint32_t Simulator::lsbd(SimInstruction* instr) const { + return instr->lsbdValue(); +} + +inline uint32_t Simulator::msbd(SimInstruction* instr) const { + return instr->msbdValue(); +} + +inline uint32_t Simulator::cond(SimInstruction* instr) const { + return instr->condValue(); +} + +inline int32_t Simulator::si12(SimInstruction* instr) const { + return (instr->imm12Value() << 20) >> 20; +} + +inline uint32_t Simulator::ui12(SimInstruction* instr) const { + return instr->imm12Value(); +} + +inline int32_t Simulator::si14(SimInstruction* instr) const { + return (instr->imm14Value() << 18) >> 18; +} + +inline int32_t Simulator::si16(SimInstruction* instr) const { + return (instr->imm16Value() << 16) >> 16; +} + +inline int32_t Simulator::si20(SimInstruction* instr) const { + return (instr->imm20Value() << 12) >> 12; +} + +// Software interrupt instructions are used by the simulator to call into C++. +void Simulator::softwareInterrupt(SimInstruction* instr) { + // the break_ instruction could get us here. + int32_t opcode_hi15 = instr->bits(31, 17); + MOZ_ASSERT(opcode_hi15 == 0x15); + uint32_t code = instr->bits(14, 0); + // We first check if we met a call_rt_redirected. + if (instr->instructionBits() == kCallRedirInstr) { + printf_instr("Simulator::softwareInterrupt: kCallRedirInstr 0x7FFFF\n"); + Redirection* redirection = Redirection::FromSwiInstruction(instr); + uintptr_t nativeFn = + reinterpret_cast(redirection->nativeFunction()); + + int64_t arg0 = getRegister(a0); + int64_t arg1 = getRegister(a1); + int64_t arg2 = getRegister(a2); + int64_t arg3 = getRegister(a3); + int64_t arg4 = getRegister(a4); + int64_t arg5 = getRegister(a5); + + // This is dodgy but it works because the C entry stubs are never moved. + // See comment in codegen-arm.cc and bug 1242173. + int64_t saved_ra = getRegister(ra); + + intptr_t external = + reinterpret_cast(redirection->nativeFunction()); + + bool stack_aligned = (getRegister(sp) & (ABIStackAlignment - 1)) == 0; + if (!stack_aligned) { + fprintf(stderr, "Runtime call with unaligned stack!\n"); + MOZ_CRASH(); + } + + if (single_stepping_) { + single_step_callback_(single_step_callback_arg_, this, nullptr); + } + + switch (redirection->type()) { + case Args_General0: { + Prototype_General0 target = + reinterpret_cast(external); + int64_t result = target(); + setCallResult(result); + break; + } + case Args_General1: { + Prototype_General1 target = + reinterpret_cast(external); + int64_t result = target(arg0); + setCallResult(result); + break; + } + case Args_General2: { + Prototype_General2 target = + reinterpret_cast(external); + int64_t result = target(arg0, arg1); + setCallResult(result); + break; + } + case Args_General3: { + Prototype_General3 target = + reinterpret_cast(external); + int64_t result = target(arg0, arg1, arg2); + if (external == intptr_t(&js::wasm::Instance::wake)) { + result = int32_t(result); + } + setCallResult(result); + break; + } + case Args_General4: { + Prototype_General4 target = + reinterpret_cast(external); + int64_t result = target(arg0, arg1, arg2, arg3); + setCallResult(result); + break; + } + case Args_General5: { + Prototype_General5 target = + reinterpret_cast(external); + int64_t result = target(arg0, arg1, arg2, arg3, arg4); + setCallResult(result); + break; + } + case Args_General6: { + Prototype_General6 target = + reinterpret_cast(external); + int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5); + setCallResult(result); + break; + } + case Args_General7: { + Prototype_General7 target = + reinterpret_cast(external); + int64_t arg6 = getRegister(a6); + int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6); + setCallResult(result); + break; + } + case Args_General8: { + Prototype_General8 target = + reinterpret_cast(external); + int64_t arg6 = getRegister(a6); + int64_t arg7 = getRegister(a7); + int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7); + setCallResult(result); + break; + } + case Args_Double_None: { + Prototype_Double_None target = + reinterpret_cast(external); + double dresult = target(); + setCallResultDouble(dresult); + break; + } + case Args_Int_Double: { + double dval0 = getFpuRegisterDouble(0); + Prototype_Int_Double target = + reinterpret_cast(external); + int64_t result = target(dval0); + if (external == intptr_t((int32_t(*)(double))JS::ToInt32)) { + result = int32_t(result); + } + setRegister(v0, result); + break; + } + case Args_Int_GeneralGeneralGeneralInt64: { + Prototype_GeneralGeneralGeneralInt64 target = + reinterpret_cast(external); + int64_t result = target(arg0, arg1, arg2, arg3); + if (external == intptr_t(&js::wasm::Instance::wait_i32)) { + result = int32_t(result); + } + setRegister(v0, result); + break; + } + case Args_Int_GeneralGeneralInt64Int64: { + Prototype_GeneralGeneralInt64Int64 target = + reinterpret_cast(external); + int64_t result = target(arg0, arg1, arg2, arg3); + if (external == intptr_t(&js::wasm::Instance::wait_i64)) { + result = int32_t(result); + } + setRegister(v0, result); + break; + } + case Args_Int_DoubleInt: { + printf("Args_Int_DoubleInt\n"); + double dval = getFpuRegisterDouble(0); + Prototype_Int_DoubleInt target = + reinterpret_cast(external); + int64_t result = target(dval, arg0); + setRegister(v0, result); + break; + } + case Args_Int_DoubleIntInt: { + double dval = getFpuRegisterDouble(0); + Prototype_Int_DoubleIntInt target = + reinterpret_cast(external); + int64_t result = target(dval, arg0, arg1); + setRegister(v0, result); + break; + } + case Args_Int_IntDoubleIntInt: { + double dval = getFpuRegisterDouble(0); + Prototype_Int_IntDoubleIntInt target = + reinterpret_cast(external); + int64_t result = target(arg0, dval, arg1, arg2); + setRegister(a0, result); + break; + } + case Args_Double_Double: { + double dval0 = getFpuRegisterDouble(0); + Prototype_Double_Double target = + reinterpret_cast(external); + double dresult = target(dval0); + setCallResultDouble(dresult); + break; + } + case Args_Float32_Float32: { + float fval0; + fval0 = getFpuRegisterFloat(0); + Prototype_Float32_Float32 target = + reinterpret_cast(external); + float fresult = target(fval0); + setCallResultFloat(fresult); + break; + } + case Args_Float32_Float32Float32: { + float fval0; + float fval1; + fval0 = getFpuRegisterFloat(0); + fval1 = getFpuRegisterFloat(1); + Prototype_Float32_Float32Float32 target = + reinterpret_cast(external); + float fresult = target(fval0, fval1); + setCallResultFloat(fresult); + break; + } + case Args_Double_Int: { + Prototype_Double_Int target = + reinterpret_cast(external); + double dresult = target(arg0); + setCallResultDouble(dresult); + break; + } + case Args_Double_DoubleInt: { + double dval0 = getFpuRegisterDouble(0); + Prototype_Double_DoubleInt target = + reinterpret_cast(external); + double dresult = target(dval0, arg0); + setCallResultDouble(dresult); + break; + } + case Args_Double_DoubleDouble: { + double dval0 = getFpuRegisterDouble(0); + double dval1 = getFpuRegisterDouble(1); + Prototype_Double_DoubleDouble target = + reinterpret_cast(external); + double dresult = target(dval0, dval1); + setCallResultDouble(dresult); + break; + } + case Args_Double_IntDouble: { + double dval0 = getFpuRegisterDouble(0); + Prototype_Double_IntDouble target = + reinterpret_cast(external); + double dresult = target(arg0, dval0); + setCallResultDouble(dresult); + break; + } + case Args_Int_IntDouble: { + double dval0 = getFpuRegisterDouble(0); + Prototype_Int_IntDouble target = + reinterpret_cast(external); + int64_t result = target(arg0, dval0); + setRegister(a0, result); + break; + } + case Args_Double_DoubleDoubleDouble: { + double dval0 = getFpuRegisterDouble(0); + double dval1 = getFpuRegisterDouble(1); + double dval2 = getFpuRegisterDouble(2); + Prototype_Double_DoubleDoubleDouble target = + reinterpret_cast(external); + double dresult = target(dval0, dval1, dval2); + setCallResultDouble(dresult); + break; + } + case Args_Double_DoubleDoubleDoubleDouble: { + double dval0 = getFpuRegisterDouble(0); + double dval1 = getFpuRegisterDouble(1); + double dval2 = getFpuRegisterDouble(2); + double dval3 = getFpuRegisterDouble(3); + Prototype_Double_DoubleDoubleDoubleDouble target = + reinterpret_cast( + external); + double dresult = target(dval0, dval1, dval2, dval3); + setCallResultDouble(dresult); + break; + } + case Args_Int32_General: { + int32_t ret = reinterpret_cast(nativeFn)(arg0); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32: { + int32_t ret = reinterpret_cast(nativeFn)( + arg0, I32(arg1)); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32Int32: { + int32_t ret = reinterpret_cast( + nativeFn)(arg0, I32(arg1), I32(arg2)); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32Int32Int32Int32: { + int32_t ret = + reinterpret_cast( + nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), I32(arg4)); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32Int32Int32Int32Int32: { + int32_t ret = + reinterpret_cast( + nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), I32(arg4), + I32(arg5)); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32Int32Int32General: { + int32_t ret = + reinterpret_cast( + nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), arg4); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32Int32Int64: { + int32_t ret = reinterpret_cast( + nativeFn)(arg0, I32(arg1), I32(arg2), arg3); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32Int32General: { + int32_t ret = + reinterpret_cast( + nativeFn)(arg0, I32(arg1), I32(arg2), arg3); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32Int64Int64: { + int32_t ret = reinterpret_cast( + nativeFn)(arg0, I32(arg1), arg2, arg3); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32GeneralInt32: { + int32_t ret = + reinterpret_cast( + nativeFn)(arg0, I32(arg1), arg2, I32(arg3)); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralInt32GeneralInt32Int32: { + int32_t ret = + reinterpret_cast( + nativeFn)(arg0, I32(arg1), arg2, I32(arg3), I32(arg4)); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralGeneral: { + int32_t ret = reinterpret_cast( + nativeFn)(arg0, arg1); + setRegister(a0, I64(ret)); + break; + } + case Args_Int32_GeneralGeneralInt32Int32: { + int32_t ret = + reinterpret_cast( + nativeFn)(arg0, arg1, I32(arg2), I32(arg3)); + setRegister(a0, I64(ret)); + break; + } + case Args_General_GeneralInt32: { + int64_t ret = reinterpret_cast( + nativeFn)(arg0, I32(arg1)); + setRegister(a0, ret); + break; + } + case Args_General_GeneralInt32Int32: { + int64_t ret = reinterpret_cast( + nativeFn)(arg0, I32(arg1), I32(arg2)); + setRegister(a0, ret); + break; + } + default: + MOZ_CRASH("Unknown function type."); + } + + if (single_stepping_) { + single_step_callback_(single_step_callback_arg_, this, nullptr); + } + + setRegister(ra, saved_ra); + set_pc(getRegister(ra)); + } else if ((instr->bits(31, 15) << 15 == op_break) && code == kWasmTrapCode) { + uint8_t* newPC; + if (wasm::HandleIllegalInstruction(registerState(), &newPC)) { + set_pc(int64_t(newPC)); + return; + } + } else if ((instr->bits(31, 15) << 15 == op_break) && code <= kMaxStopCode && + code != 6) { + printf_instr("Simulator::softwareInterrupt: BREAK 0x%lx\n", + instr->bits(14, 0)); + if (isWatchpoint(code)) { + printf_instr("Simulator::softwareInterrupt: isWatchpoint \n"); + // printWatchpoint(code); + } else { + printf_instr("Simulator::softwareInterrupt: increaseStopCounter \n"); + increaseStopCounter(code); + handleStop(code, instr); + } + } else { + printf_instr("Simulator::softwareInterrupt: else \n"); + + // All remaining break_ codes, and all traps are handled here. + loong64Debugger dbg(this); + dbg.debug(); + } +} + +// Stop helper functions. +bool Simulator::isWatchpoint(uint32_t code) { + return (code <= kMaxWatchpointCode); +} + +void Simulator::printWatchpoint(uint32_t code) { + loong64Debugger dbg(this); + ++break_count_; + printf("\n---- break %d marker: %20" PRIi64 " (instr count: %20" PRIi64 + ") ----\n", + code, break_count_, icount_); + dbg.printAllRegs(); // Print registers and continue running. +} + +void Simulator::handleStop(uint32_t code, SimInstruction* instr) { + // Stop if it is enabled, otherwise go on jumping over the stop + // and the message address. + if (isEnabledStop(code)) { + loong64Debugger dbg(this); + dbg.stop(instr); + } else { + // set_pc(get_pc() + 2 * SimInstruction::kInstrSize); + set_pc(get_pc() + 1 * SimInstruction::kInstrSize); + } +} + +bool Simulator::isStopInstruction(SimInstruction* instr) { + int32_t opcode_hi15 = instr->bits(31, 17); + uint32_t code = static_cast(instr->bits(14, 0)); + return (opcode_hi15 == 0x15) && code > kMaxWatchpointCode && + code <= kMaxStopCode; +} + +bool Simulator::isEnabledStop(uint32_t code) { + MOZ_ASSERT(code <= kMaxStopCode); + MOZ_ASSERT(code > kMaxWatchpointCode); + return !(watchedStops_[code].count_ & kStopDisabledBit); +} + +void Simulator::enableStop(uint32_t code) { + if (!isEnabledStop(code)) { + watchedStops_[code].count_ &= ~kStopDisabledBit; + } +} + +void Simulator::disableStop(uint32_t code) { + if (isEnabledStop(code)) { + watchedStops_[code].count_ |= kStopDisabledBit; + } +} + +void Simulator::increaseStopCounter(uint32_t code) { + MOZ_ASSERT(code <= kMaxStopCode); + if ((watchedStops_[code].count_ & ~(1 << 31)) == 0x7fffffff) { + printf( + "Stop counter for code %i has overflowed.\n" + "Enabling this code and reseting the counter to 0.\n", + code); + watchedStops_[code].count_ = 0; + enableStop(code); + } else { + watchedStops_[code].count_++; + } +} + +// Print a stop status. +void Simulator::printStopInfo(uint32_t code) { + if (code <= kMaxWatchpointCode) { + printf("That is a watchpoint, not a stop.\n"); + return; + } else if (code > kMaxStopCode) { + printf("Code too large, only %u stops can be used\n", kMaxStopCode + 1); + return; + } + const char* state = isEnabledStop(code) ? "Enabled" : "Disabled"; + int32_t count = watchedStops_[code].count_ & ~kStopDisabledBit; + // Don't print the state of unused breakpoints. + if (count != 0) { + if (watchedStops_[code].desc_) { + printf("stop %i - 0x%x: \t%s, \tcounter = %i, \t%s\n", code, code, state, + count, watchedStops_[code].desc_); + } else { + printf("stop %i - 0x%x: \t%s, \tcounter = %i\n", code, code, state, + count); + } + } +} + +void Simulator::signalExceptions() { + for (int i = 1; i < kNumExceptions; i++) { + if (exceptions[i] != 0) { + MOZ_CRASH("Error: Exception raised."); + } + } +} + +// ReverseBits(value) returns |value| in reverse bit order. +template +T ReverseBits(T value) { + MOZ_ASSERT((sizeof(value) == 1) || (sizeof(value) == 2) || + (sizeof(value) == 4) || (sizeof(value) == 8)); + T result = 0; + for (unsigned i = 0; i < (sizeof(value) * 8); i++) { + result = (result << 1) | (value & 1); + value >>= 1; + } + return result; +} + +// Min/Max template functions for Double and Single arguments. + +template +static T FPAbs(T a); + +template <> +double FPAbs(double a) { + return fabs(a); +} + +template <> +float FPAbs(float a) { + return fabsf(a); +} + +enum class MaxMinKind : int { kMin = 0, kMax = 1 }; + +template +static bool FPUProcessNaNsAndZeros(T a, T b, MaxMinKind kind, T* result) { + if (std::isnan(a) && std::isnan(b)) { + *result = a; + } else if (std::isnan(a)) { + *result = b; + } else if (std::isnan(b)) { + *result = a; + } else if (b == a) { + // Handle -0.0 == 0.0 case. + // std::signbit() returns int 0 or 1 so subtracting MaxMinKind::kMax + // negates the result. + *result = std::signbit(b) - static_cast(kind) ? b : a; + } else { + return false; + } + return true; +} + +template +static T FPUMin(T a, T b) { + T result; + if (FPUProcessNaNsAndZeros(a, b, MaxMinKind::kMin, &result)) { + return result; + } else { + return b < a ? b : a; + } +} + +template +static T FPUMax(T a, T b) { + T result; + if (FPUProcessNaNsAndZeros(a, b, MaxMinKind::kMax, &result)) { + return result; + } else { + return b > a ? b : a; + } +} + +template +static T FPUMinA(T a, T b) { + T result; + if (!FPUProcessNaNsAndZeros(a, b, MaxMinKind::kMin, &result)) { + if (FPAbs(a) < FPAbs(b)) { + result = a; + } else if (FPAbs(b) < FPAbs(a)) { + result = b; + } else { + result = a < b ? a : b; + } + } + return result; +} + +template +static T FPUMaxA(T a, T b) { + T result; + if (!FPUProcessNaNsAndZeros(a, b, MaxMinKind::kMin, &result)) { + if (FPAbs(a) > FPAbs(b)) { + result = a; + } else if (FPAbs(b) > FPAbs(a)) { + result = b; + } else { + result = a > b ? a : b; + } + } + return result; +} + +enum class KeepSign : bool { no = false, yes }; + +// Handle execution based on instruction types. +// decodeTypeImmediate +void Simulator::decodeTypeOp6(SimInstruction* instr) { + // Next pc. + int64_t next_pc = bad_ra; + + // Used for memory instructions. + int64_t alu_out = 0; + + // Branch instructions common part. + auto BranchAndLinkHelper = [this, &next_pc](SimInstruction* instr) { + int64_t current_pc = get_pc(); + setRegister(ra, current_pc + SimInstruction::kInstrSize); + int32_t offs26_low16 = + static_cast(instr->bits(25, 10) << 16) >> 16; + int32_t offs26_high10 = static_cast(instr->bits(9, 0) << 22) >> 6; + int32_t offs26 = offs26_low16 | offs26_high10; + next_pc = current_pc + (offs26 << 2); + printf_instr("Offs26: %08x\n", offs26); + set_pc(next_pc); + }; + + auto BranchOff16Helper = [this, &next_pc](SimInstruction* instr, + bool do_branch) { + int64_t current_pc = get_pc(); + int32_t offs16 = static_cast(instr->bits(25, 10) << 16) >> 16; + printf_instr("Offs16: %08x\n", offs16); + int32_t offs = do_branch ? (offs16 << 2) : SimInstruction::kInstrSize; + next_pc = current_pc + offs; + set_pc(next_pc); + }; + + auto BranchOff21Helper = [this, &next_pc](SimInstruction* instr, + bool do_branch) { + int64_t current_pc = get_pc(); + int32_t offs21_low16 = + static_cast(instr->bits(25, 10) << 16) >> 16; + int32_t offs21_high5 = static_cast(instr->bits(4, 0) << 27) >> 11; + int32_t offs = offs21_low16 | offs21_high5; + printf_instr("Offs21: %08x\n", offs); + offs = do_branch ? (offs << 2) : SimInstruction::kInstrSize; + next_pc = current_pc + offs; + set_pc(next_pc); + }; + + auto BranchOff26Helper = [this, &next_pc](SimInstruction* instr) { + int64_t current_pc = get_pc(); + int32_t offs26_low16 = + static_cast(instr->bits(25, 10) << 16) >> 16; + int32_t offs26_high10 = static_cast(instr->bits(9, 0) << 22) >> 6; + int32_t offs26 = offs26_low16 | offs26_high10; + next_pc = current_pc + (offs26 << 2); + printf_instr("Offs26: %08x\n", offs26); + set_pc(next_pc); + }; + + auto JumpOff16Helper = [this, &next_pc](SimInstruction* instr) { + int32_t offs16 = static_cast(instr->bits(25, 10) << 16) >> 16; + printf_instr("JIRL\t %s: %016lx, %s: %016lx, offs16: %x\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), offs16); + setRegister(rd_reg(instr), get_pc() + SimInstruction::kInstrSize); + next_pc = rj(instr) + (offs16 << 2); + set_pc(next_pc); + }; + + switch (instr->bits(31, 26) << 26) { + case op_addu16i_d: { + printf_instr("ADDU16I_D\t %s: %016lx, %s: %016lx, si16: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si16(instr)); + int32_t si16_upper = static_cast(si16(instr)) << 16; + alu_out = static_cast(si16_upper) + rj(instr); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_beqz: { + printf_instr("BEQZ\t %s: %016lx, ", Registers::GetName(rj_reg(instr)), + rj(instr)); + BranchOff21Helper(instr, rj(instr) == 0); + break; + } + case op_bnez: { + printf_instr("BNEZ\t %s: %016lx, ", Registers::GetName(rj_reg(instr)), + rj(instr)); + BranchOff21Helper(instr, rj(instr) != 0); + break; + } + case op_bcz: { + if (instr->bits(9, 8) == 0b00) { + // BCEQZ + printf_instr("BCEQZ\t fcc%d: %s, ", cj_reg(instr), + cj(instr) ? "True" : "False"); + BranchOff21Helper(instr, cj(instr) == false); + } else if (instr->bits(9, 8) == 0b01) { + // BCNEZ + printf_instr("BCNEZ\t fcc%d: %s, ", cj_reg(instr), + cj(instr) ? "True" : "False"); + BranchOff21Helper(instr, cj(instr) == true); + } else { + UNREACHABLE(); + } + break; + } + case op_jirl: { + printf_instr("JIRL\t "); + JumpOff16Helper(instr); + break; + } + case op_b: { + printf_instr("B\t "); + BranchOff26Helper(instr); + break; + } + case op_bl: { + printf_instr("BL\t "); + BranchAndLinkHelper(instr); + break; + } + case op_beq: { + printf_instr("BEQ\t %s: %016lx, %s, %016lx, ", + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rd_reg(instr)), rd(instr)); + BranchOff16Helper(instr, rj(instr) == rd(instr)); + break; + } + case op_bne: { + printf_instr("BNE\t %s: %016lx, %s, %016lx, ", + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rd_reg(instr)), rd(instr)); + BranchOff16Helper(instr, rj(instr) != rd(instr)); + break; + } + case op_blt: { + printf_instr("BLT\t %s: %016lx, %s, %016lx, ", + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rd_reg(instr)), rd(instr)); + BranchOff16Helper(instr, rj(instr) < rd(instr)); + break; + } + case op_bge: { + printf_instr("BGE\t %s: %016lx, %s, %016lx, ", + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rd_reg(instr)), rd(instr)); + BranchOff16Helper(instr, rj(instr) >= rd(instr)); + break; + } + case op_bltu: { + printf_instr("BLTU\t %s: %016lx, %s, %016lx, ", + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rd_reg(instr)), rd(instr)); + BranchOff16Helper(instr, rj_u(instr) < rd_u(instr)); + break; + } + case op_bgeu: { + printf_instr("BGEU\t %s: %016lx, %s, %016lx, ", + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rd_reg(instr)), rd(instr)); + BranchOff16Helper(instr, rj_u(instr) >= rd_u(instr)); + break; + } + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp7(SimInstruction* instr) { + int64_t alu_out; + + switch (instr->bits(31, 25) << 25) { + case op_lu12i_w: { + printf_instr("LU12I_W\t %s: %016lx, si20: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), si20(instr)); + int32_t si20_upper = static_cast(si20(instr) << 12); + setRegister(rd_reg(instr), static_cast(si20_upper)); + break; + } + case op_lu32i_d: { + printf_instr("LU32I_D\t %s: %016lx, si20: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), si20(instr)); + int32_t si20_signExtend = static_cast(si20(instr) << 12) >> 12; + int64_t lower_32bit_mask = 0xFFFFFFFF; + alu_out = (static_cast(si20_signExtend) << 32) | + (rd(instr) & lower_32bit_mask); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_pcaddi: { + printf_instr("PCADDI\t %s: %016lx, si20: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), si20(instr)); + int32_t si20_signExtend = static_cast(si20(instr) << 12) >> 10; + int64_t current_pc = get_pc(); + alu_out = static_cast(si20_signExtend) + current_pc; + setRegister(rd_reg(instr), alu_out); + break; + } + case op_pcalau12i: { + printf_instr("PCALAU12I\t %s: %016lx, si20: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), si20(instr)); + int32_t si20_signExtend = static_cast(si20(instr) << 12); + int64_t current_pc = get_pc(); + int64_t clear_lower12bit_mask = 0xFFFFFFFFFFFFF000; + alu_out = static_cast(si20_signExtend) + current_pc; + setRegister(rd_reg(instr), alu_out & clear_lower12bit_mask); + break; + } + case op_pcaddu12i: { + printf_instr("PCADDU12I\t %s: %016lx, si20: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), si20(instr)); + int32_t si20_signExtend = static_cast(si20(instr) << 12); + int64_t current_pc = get_pc(); + alu_out = static_cast(si20_signExtend) + current_pc; + setRegister(rd_reg(instr), alu_out); + break; + } + case op_pcaddu18i: { + printf_instr("PCADDU18I\t %s: %016lx, si20: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), si20(instr)); + int64_t si20_signExtend = (static_cast(si20(instr)) << 44) >> 26; + int64_t current_pc = get_pc(); + alu_out = si20_signExtend + current_pc; + setRegister(rd_reg(instr), alu_out); + break; + } + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp8(SimInstruction* instr) { + int64_t addr = 0x0; + int64_t si14_se = (static_cast(si14(instr)) << 50) >> 48; + + switch (instr->bits(31, 24) << 24) { + case op_ldptr_w: { + printf_instr("LDPTR_W\t %s: %016lx, %s: %016lx, si14: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si14_se); + setRegister(rd_reg(instr), readW(rj(instr) + si14_se, instr)); + break; + } + case op_stptr_w: { + printf_instr("STPTR_W\t %s: %016lx, %s: %016lx, si14: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si14_se); + writeW(rj(instr) + si14_se, static_cast(rd(instr)), instr); + break; + } + case op_ldptr_d: { + printf_instr("LDPTR_D\t %s: %016lx, %s: %016lx, si14: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si14_se); + setRegister(rd_reg(instr), readDW(rj(instr) + si14_se, instr)); + break; + } + case op_stptr_d: { + printf_instr("STPTR_D\t %s: %016lx, %s: %016lx, si14: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si14_se); + writeDW(rj(instr) + si14_se, rd(instr), instr); + break; + } + case op_ll_w: { + printf_instr("LL_W\t %s: %016lx, %s: %016lx, si14: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si14_se); + addr = si14_se + rj(instr); + setRegister(rd_reg(instr), loadLinkedW(addr, instr)); + break; + } + case op_sc_w: { + printf_instr("SC_W\t %s: %016lx, %s: %016lx, si14: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si14_se); + addr = si14_se + rj(instr); + setRegister( + rd_reg(instr), + storeConditionalW(addr, static_cast(rd(instr)), instr)); + break; + } + case op_ll_d: { + printf_instr("LL_D\t %s: %016lx, %s: %016lx, si14: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si14_se); + addr = si14_se + rj(instr); + setRegister(rd_reg(instr), loadLinkedD(addr, instr)); + break; + } + case op_sc_d: { + printf_instr("SC_D\t %s: %016lx, %s: %016lx, si14: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si14_se); + addr = si14_se + rj(instr); + setRegister(rd_reg(instr), storeConditionalD(addr, rd(instr), instr)); + break; + } + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp10(SimInstruction* instr) { + int64_t alu_out = 0x0; + int64_t si12_se = (static_cast(si12(instr)) << 52) >> 52; + uint64_t si12_ze = (static_cast(ui12(instr)) << 52) >> 52; + + switch (instr->bits(31, 22) << 22) { + case op_bstrins_d: { + uint8_t lsbd_ = lsbd(instr); + uint8_t msbd_ = msbd(instr); + MOZ_ASSERT(lsbd_ <= msbd_); + printf_instr( + "BSTRINS_D\t %s: %016lx, %s: %016lx, msbd: %02x, lsbd: %02x\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), msbd_, lsbd_); + uint8_t size = msbd_ - lsbd_ + 1; + if (size < 64) { + uint64_t mask = (1ULL << size) - 1; + alu_out = + (rd_u(instr) & ~(mask << lsbd_)) | ((rj_u(instr) & mask) << lsbd_); + setRegister(rd_reg(instr), alu_out); + } else if (size == 64) { + setRegister(rd_reg(instr), rj(instr)); + } + break; + } + case op_bstrpick_d: { + uint8_t lsbd_ = lsbd(instr); + uint8_t msbd_ = msbd(instr); + MOZ_ASSERT(lsbd_ <= msbd_); + printf_instr( + "BSTRPICK_D\t %s: %016lx, %s: %016lx, msbd: %02x, lsbd: %02x\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), msbd_, lsbd_); + uint8_t size = msbd_ - lsbd_ + 1; + if (size < 64) { + uint64_t mask = (1ULL << size) - 1; + alu_out = (rj_u(instr) & (mask << lsbd_)) >> lsbd_; + setRegister(rd_reg(instr), alu_out); + } else if (size == 64) { + setRegister(rd_reg(instr), rj(instr)); + } + break; + } + case op_slti: { + printf_instr("SLTI\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_se); + setRegister(rd_reg(instr), rj(instr) < si12_se ? 1 : 0); + break; + } + case op_sltui: { + printf_instr("SLTUI\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_se); + setRegister(rd_reg(instr), + rj_u(instr) < static_cast(si12_se) ? 1 : 0); + break; + } + case op_addi_w: { + printf_instr("ADDI_W\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_se); + int32_t alu32_out = + static_cast(rj(instr)) + static_cast(si12_se); + setRegister(rd_reg(instr), alu32_out); + break; + } + case op_addi_d: { + printf_instr("ADDI_D\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_se); + setRegister(rd_reg(instr), rj(instr) + si12_se); + break; + } + case op_lu52i_d: { + printf_instr("LU52I_D\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_se); + int64_t si12_se = static_cast(si12(instr)) << 52; + uint64_t mask = (1ULL << 52) - 1; + alu_out = si12_se + (rj(instr) & mask); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_andi: { + printf_instr("ANDI\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), rj(instr) & si12_ze); + break; + } + case op_ori: { + printf_instr("ORI\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), rj_u(instr) | si12_ze); + break; + } + case op_xori: { + printf_instr("XORI\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), rj_u(instr) ^ si12_ze); + break; + } + case op_ld_b: { + printf_instr("LD_B\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), readB(rj(instr) + si12_se)); + break; + } + case op_ld_h: { + printf_instr("LD_H\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), readH(rj(instr) + si12_se, instr)); + break; + } + case op_ld_w: { + printf_instr("LD_W\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), readW(rj(instr) + si12_se, instr)); + break; + } + case op_ld_d: { + printf_instr("LD_D\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), readDW(rj(instr) + si12_se, instr)); + break; + } + case op_st_b: { + printf_instr("ST_B\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + writeB(rj(instr) + si12_se, static_cast(rd(instr))); + break; + } + case op_st_h: { + printf_instr("ST_H\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + writeH(rj(instr) + si12_se, static_cast(rd(instr)), instr); + break; + } + case op_st_w: { + printf_instr("ST_W\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + writeW(rj(instr) + si12_se, static_cast(rd(instr)), instr); + break; + } + case op_st_d: { + printf_instr("ST_D\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + writeDW(rj(instr) + si12_se, rd(instr), instr); + break; + } + case op_ld_bu: { + printf_instr("LD_BU\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), readBU(rj(instr) + si12_se)); + break; + } + case op_ld_hu: { + printf_instr("LD_HU\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), readHU(rj(instr) + si12_se, instr)); + break; + } + case op_ld_wu: { + printf_instr("LD_WU\t %s: %016lx, %s: %016lx, si12: %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setRegister(rd_reg(instr), readWU(rj(instr) + si12_se, instr)); + break; + } + case op_fld_s: { + printf_instr("FLD_S\t %s: %016f, %s: %016lx, si12: %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setFpuRegister(fd_reg(instr), kFPUInvalidResult); // Trash upper 32 bits. + setFpuRegisterWord(fd_reg(instr), readW(rj(instr) + si12_se, instr)); + break; + } + case op_fst_s: { + printf_instr("FST_S\t %s: %016f, %s: %016lx, si12: %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + int32_t alu_out_32 = static_cast(getFpuRegister(fd_reg(instr))); + writeW(rj(instr) + si12_se, alu_out_32, instr); + break; + } + case op_fld_d: { + printf_instr("FLD_D\t %s: %016f, %s: %016lx, si12: %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + setFpuRegisterDouble(fd_reg(instr), readD(rj(instr) + si12_se, instr)); + break; + } + case op_fst_d: { + printf_instr("FSD_D\t %s: %016f, %s: %016lx, si12: %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + Registers::GetName(rj_reg(instr)), rj(instr), si12_ze); + writeD(rj(instr) + si12_se, getFpuRegisterDouble(fd_reg(instr)), instr); + break; + } + case op_preld: + UNIMPLEMENTED(); + break; + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp11(SimInstruction* instr) { + int64_t alu_out = 0x0; + + switch (instr->bits(31, 21) << 21) { + case op_bstr_w: { + MOZ_ASSERT(instr->bit(21) == 1); + uint8_t lsbw_ = lsbw(instr); + uint8_t msbw_ = msbw(instr); + MOZ_ASSERT(lsbw_ <= msbw_); + uint8_t size = msbw_ - lsbw_ + 1; + uint64_t mask = (1ULL << size) - 1; + if (instr->bit(15) == 0) { + // BSTRINS_W + printf_instr( + "BSTRINS_W\t %s: %016lx, %s: %016lx, msbw: %02x, lsbw: %02x\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), msbw_, lsbw_); + alu_out = static_cast((rd_u(instr) & ~(mask << lsbw_)) | + ((rj_u(instr) & mask) << lsbw_)); + } else { + // BSTRPICK_W + printf_instr( + "BSTRPICK_W\t %s: %016lx, %s: %016lx, msbw: %02x, lsbw: %02x\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), msbw_, lsbw_); + alu_out = + static_cast((rj_u(instr) & (mask << lsbw_)) >> lsbw_); + } + setRegister(rd_reg(instr), alu_out); + break; + } + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp12(SimInstruction* instr) { + switch (instr->bits(31, 20) << 20) { + case op_fmadd_s: { + printf_instr("FMADD_S\t %s: %016f, %s: %016f, %s: %016f %s: %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr), + FloatRegisters::GetName(fa_reg(instr)), fa_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterFloat( + fd_reg(instr), + std::fma(fj_float(instr), fk_float(instr), fa_float(instr))); + break; + } + case op_fmadd_d: { + printf_instr("FMADD_D\t %s: %016f, %s: %016f, %s: %016f %s: %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr), + FloatRegisters::GetName(fa_reg(instr)), fa_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterDouble( + fd_reg(instr), + std::fma(fj_double(instr), fk_double(instr), fa_double(instr))); + break; + } + case op_fmsub_s: { + printf_instr("FMSUB_S\t %s: %016f, %s: %016f, %s: %016f %s: %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr), + FloatRegisters::GetName(fa_reg(instr)), fa_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterFloat( + fd_reg(instr), + std::fma(-fj_float(instr), fk_float(instr), fa_float(instr))); + break; + } + case op_fmsub_d: { + printf_instr("FMSUB_D\t %s: %016f, %s: %016f, %s: %016f %s: %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr), + FloatRegisters::GetName(fa_reg(instr)), fa_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterDouble( + fd_reg(instr), + std::fma(-fj_double(instr), fk_double(instr), fa_double(instr))); + break; + } + case op_fnmadd_s: { + printf_instr("FNMADD_S\t %s: %016f, %s: %016f, %s: %016f %s: %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr), + FloatRegisters::GetName(fa_reg(instr)), fa_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterFloat( + fd_reg(instr), + std::fma(-fj_float(instr), fk_float(instr), -fa_float(instr))); + break; + } + case op_fnmadd_d: { + printf_instr("FNMADD_D\t %s: %016f, %s: %016f, %s: %016f %s: %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr), + FloatRegisters::GetName(fa_reg(instr)), fa_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterDouble( + fd_reg(instr), + std::fma(-fj_double(instr), fk_double(instr), -fa_double(instr))); + break; + } + case op_fnmsub_s: { + printf_instr("FNMSUB_S\t %s: %016f, %s: %016f, %s: %016f %s: %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr), + FloatRegisters::GetName(fa_reg(instr)), fa_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterFloat( + fd_reg(instr), + std::fma(fj_float(instr), fk_float(instr), -fa_float(instr))); + break; + } + case op_fnmsub_d: { + printf_instr("FNMSUB_D\t %s: %016f, %s: %016f, %s: %016f %s: %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr), + FloatRegisters::GetName(fa_reg(instr)), fa_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterDouble( + fd_reg(instr), + std::fma(fj_double(instr), fk_double(instr), -fa_double(instr))); + break; + } + case op_fcmp_cond_s: { + MOZ_ASSERT(instr->bits(4, 3) == 0); + float fj = fj_float(instr); + float fk = fk_float(instr); + switch (cond(instr)) { + case AssemblerLOONG64::CAF: { + printf_instr("FCMP_CAF_S fcc%d\n", cd_reg(instr)); + setCFRegister(cd_reg(instr), false); + break; + } + case AssemblerLOONG64::CUN: { + printf_instr("FCMP_CUN_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::CEQ: { + printf_instr("FCMP_CEQ_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), fj == fk); + break; + } + case AssemblerLOONG64::CUEQ: { + printf_instr("FCMP_CUEQ_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), + (fj == fk) || std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::CLT: { + printf_instr("FCMP_CLT_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), fj < fk); + break; + } + case AssemblerLOONG64::CULT: { + printf_instr("FCMP_CULT_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), + (fj < fk) || std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::CLE: { + printf_instr("FCMP_CLE_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), fj <= fk); + break; + } + case AssemblerLOONG64::CULE: { + printf_instr("FCMP_CULE_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), + (fj <= fk) || std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::CNE: { + printf_instr("FCMP_CNE_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), (fj < fk) || (fj > fk)); + break; + } + case AssemblerLOONG64::COR: { + printf_instr("FCMP_COR_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), !std::isnan(fj) && !std::isnan(fk)); + break; + } + case AssemblerLOONG64::CUNE: { + printf_instr("FCMP_CUNE_S fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), (fj < fk) || (fj > fk) || + std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::SAF: + printf_instr("FCMP_SAF_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SUN: + printf_instr("FCMP_SUN_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SEQ: + printf_instr("FCMP_SEQ_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SUEQ: + printf_instr("FCMP_SUEQ_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SLT: + printf_instr("FCMP_SLS_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SULT: + printf_instr("FCMP_SULT_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SLE: + printf_instr("FCMP_SLE_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SULE: + printf_instr("FCMP_SULE_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SNE: + printf_instr("FCMP_SNE_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SOR: + printf_instr("FCMP_SOR_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SUNE: + printf_instr("FCMP_COND_S: UNIMPLEMENTED \t\n"); + UNIMPLEMENTED(); + break; + default: + UNREACHABLE(); + } + break; + } + case op_fcmp_cond_d: { + MOZ_ASSERT(instr->bits(4, 3) == 0); + double fj = fj_double(instr); + double fk = fk_double(instr); + switch (cond(instr)) { + case AssemblerLOONG64::CAF: { + printf_instr("FCMP_CAF_D fcc%d\n", cd_reg(instr)); + setCFRegister(cd_reg(instr), false); + break; + } + case AssemblerLOONG64::CUN: { + printf_instr("FCMP_CUN_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::CEQ: { + printf_instr("FCMP_CEQ_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), fj == fk); + break; + } + case AssemblerLOONG64::CUEQ: { + printf_instr("FCMP_CUEQ_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), + (fj == fk) || std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::CLT: { + printf_instr("FCMP_CLT_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), fj < fk); + break; + } + case AssemblerLOONG64::CULT: { + printf_instr("FCMP_CULT_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), + (fj < fk) || std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::CLE: { + printf_instr("FCMP_CLE_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), fj <= fk); + break; + } + case AssemblerLOONG64::CULE: { + printf_instr("FCMP_CULE_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), + (fj <= fk) || std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::CNE: { + printf_instr("FCMP_CNE_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), (fj < fk) || (fj > fk)); + break; + } + case AssemblerLOONG64::COR: { + printf_instr("FCMP_COR_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), !std::isnan(fj) && !std::isnan(fk)); + break; + } + case AssemblerLOONG64::CUNE: { + printf_instr("FCMP_CUNE_D fcc%d, %s: %016f, %s: %016f\n", + cd_reg(instr), FloatRegisters::GetName(fj_reg(instr)), + fj, FloatRegisters::GetName(fk_reg(instr)), fk); + setCFRegister(cd_reg(instr), + (fj != fk) || std::isnan(fj) || std::isnan(fk)); + break; + } + case AssemblerLOONG64::SAF: + printf_instr("FCMP_SAF_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SUN: + printf_instr("FCMP_SUN_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SEQ: + printf_instr("FCMP_SEQ_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SUEQ: + printf_instr("FCMP_SUEQ_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SLT: + printf_instr("FCMP_SLT_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SULT: + printf_instr("FCMP_SULT_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SLE: + printf_instr("FCMP_SLE_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SULE: + printf_instr("FCMP_SULE_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SNE: + printf_instr("FCMP_SNE_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SOR: + printf_instr("FCMP_SOR_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + case AssemblerLOONG64::SUNE: + printf_instr("FCMP_SUNE_D: UNIMPLEMENTED \t \n"); + UNIMPLEMENTED(); + break; + default: + UNREACHABLE(); + } + break; + } + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp14(SimInstruction* instr) { + int64_t alu_out = 0x0; + + switch (instr->bits(31, 18) << 18) { + case op_bytepick_d: { + printf_instr("BYTEPICK_D\t %s: %016lx, %s: %016lx, %s: %016lx, sa3: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr), sa3(instr)); + uint8_t sa = sa3(instr) * 8; + if (sa == 0) { + alu_out = rk(instr); + } else { + int64_t mask = (1ULL << 63) >> (sa - 1); + int64_t rk_hi = (rk(instr) & (~mask)) << sa; + int64_t rj_lo = (rj(instr) & mask) >> (64 - sa); + alu_out = rk_hi | rj_lo; + } + setRegister(rd_reg(instr), alu_out); + break; + } + case op_fsel: { + MOZ_ASSERT(instr->bits(19, 18) == 0); + printf_instr("FSEL fcc%d, %s: %016f, %s: %016f, %s: %016f\n", + ca_reg(instr), FloatRegisters::GetName(fd_reg(instr)), + fd_double(instr), FloatRegisters::GetName(fj_reg(instr)), + fj_double(instr), FloatRegisters::GetName(fk_reg(instr)), + fk_double(instr)); + if (ca(instr) == 0) { + setFpuRegisterDouble(fd_reg(instr), fj_double(instr)); + } else { + setFpuRegisterDouble(fd_reg(instr), fk_double(instr)); + } + break; + } + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp15(SimInstruction* instr) { + int64_t alu_out = 0x0; + int32_t alu32_out = 0x0; + + switch (instr->bits(31, 17) << 17) { + case op_bytepick_w: { + MOZ_ASSERT(instr->bit(17) == 0); + printf_instr("BYTEPICK_W\t %s: %016lx, %s: %016lx, %s: %016lx, sa2: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr), sa2(instr)); + uint8_t sa = sa2(instr) * 8; + if (sa == 0) { + alu32_out = static_cast(rk(instr)); + } else { + int32_t mask = (1 << 31) >> (sa - 1); + int32_t rk_hi = (static_cast(rk(instr)) & (~mask)) << sa; + int32_t rj_lo = (static_cast(rj(instr)) & mask) >> (32 - sa); + alu32_out = rk_hi | rj_lo; + } + setRegister(rd_reg(instr), static_cast(alu32_out)); + break; + } + case op_alsl_w: { + uint8_t sa = sa2(instr) + 1; + alu32_out = (static_cast(rj(instr)) << sa) + + static_cast(rk(instr)); + printf_instr("ALSL_W\t %s: %016lx, %s: %016lx, %s: %016lx, sa2: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr), sa2(instr)); + setRegister(rd_reg(instr), alu32_out); + break; + } + case op_alsl_wu: { + uint8_t sa = sa2(instr) + 1; + alu32_out = (static_cast(rj(instr)) << sa) + + static_cast(rk(instr)); + printf_instr("ALSL_WU\t %s: %016lx, %s: %016lx, %s: %016lx, sa2: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr), sa2(instr)); + setRegister(rd_reg(instr), static_cast(alu32_out)); + break; + } + case op_alsl_d: { + printf_instr("ALSL_D\t %s: %016lx, %s: %016lx, %s: %016lx, sa2: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr), sa2(instr)); + MOZ_ASSERT(instr->bit(17) == 0); + uint8_t sa = sa2(instr) + 1; + alu_out = (rj(instr) << sa) + rk(instr); + setRegister(rd_reg(instr), alu_out); + break; + } + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp16(SimInstruction* instr) { + int64_t alu_out; + switch (instr->bits(31, 16) << 16) { + case op_slli_d: { + // SLLI_D + MOZ_ASSERT(instr->bit(17) == 0); + MOZ_ASSERT(instr->bits(17, 16) == 0b01); + printf_instr("SLLI_D\t %s: %016lx, %s: %016lx, ui6: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), ui6(instr)); + setRegister(rd_reg(instr), rj(instr) << ui6(instr)); + break; + } + case op_srli_d: { + MOZ_ASSERT(instr->bit(17) == 0); + // SRLI_D + printf_instr("SRLI_D\t %s: %016lx, %s: %016lx, ui6: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), ui6(instr)); + setRegister(rd_reg(instr), rj_u(instr) >> ui6(instr)); + break; + } + case op_srai_d: { + MOZ_ASSERT(instr->bit(17) == 0); + // SRAI_D + printf_instr("SRAI_D\t %s: %016lx, %s: %016lx, ui6: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), ui6(instr)); + setRegister(rd_reg(instr), rj(instr) >> ui6(instr)); + break; + } + case op_rotri_d: { + MOZ_ASSERT(instr->bit(17) == 0); + MOZ_ASSERT(instr->bits(17, 16) == 0b01); + // ROTRI_D + printf_instr("ROTRI_D\t %s: %016lx, %s: %016lx, ui6: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), ui6(instr)); + alu_out = static_cast(RotateRight64(rj_u(instr), ui6(instr))); + setRegister(rd_reg(instr), alu_out); + break; + } + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp17(SimInstruction* instr) { + int64_t alu_out; + int32_t alu32_out; + + switch (instr->bits(31, 15) << 15) { + case op_slli_w: { + MOZ_ASSERT(instr->bit(17) == 0); + MOZ_ASSERT(instr->bits(17, 15) == 0b001); + printf_instr("SLLI_W\t %s: %016lx, %s: %016lx, ui5: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), ui5(instr)); + alu32_out = static_cast(rj(instr)) << ui5(instr); + setRegister(rd_reg(instr), static_cast(alu32_out)); + break; + } + case op_srai_w: { + MOZ_ASSERT(instr->bit(17) == 0); + MOZ_ASSERT(instr->bits(17, 15) == 0b001); + // SRAI_W + printf_instr("SRAI_W\t %s: %016lx, %s: %016lx, ui5: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), ui5(instr)); + alu32_out = static_cast(rj(instr)) >> ui5(instr); + setRegister(rd_reg(instr), static_cast(alu32_out)); + break; + } + case op_rotri_w: { + MOZ_ASSERT(instr->bit(17) == 0); + MOZ_ASSERT(instr->bits(17, 15) == 0b001); + // ROTRI_W + printf_instr("ROTRI_W\t %s: %016lx, %s: %016lx, ui5: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), ui5(instr)); + alu32_out = static_cast( + RotateRight32(static_cast(rj_u(instr)), + static_cast(ui5(instr)))); + setRegister(rd_reg(instr), static_cast(alu32_out)); + break; + } + case op_srli_w: { + MOZ_ASSERT(instr->bit(17) == 0); + MOZ_ASSERT(instr->bits(17, 15) == 0b001); + // if (instr->bits(17, 15) == 0b001) { + // SRLI_W + printf_instr("SRLI_W\t %s: %016lx, %s: %016lx, ui5: %d\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), ui5(instr)); + alu32_out = static_cast(rj(instr)) >> ui5(instr); + setRegister(rd_reg(instr), static_cast(alu32_out)); + break; + } + case op_add_w: { + printf_instr("ADD_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + int32_t alu32_out = static_cast(rj(instr) + rk(instr)); + // Sign-extend result of 32bit operation into 64bit register. + setRegister(rd_reg(instr), static_cast(alu32_out)); + break; + } + case op_add_d: + printf_instr("ADD_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) + rk(instr)); + break; + case op_sub_w: { + printf_instr("SUB_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + int32_t alu32_out = static_cast(rj(instr) - rk(instr)); + // Sign-extend result of 32bit operation into 64bit register. + setRegister(rd_reg(instr), static_cast(alu32_out)); + break; + } + case op_sub_d: + printf_instr("SUB_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) - rk(instr)); + break; + case op_slt: + printf_instr("SLT\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) < rk(instr) ? 1 : 0); + break; + case op_sltu: + printf_instr("SLTU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj_u(instr) < rk_u(instr) ? 1 : 0); + break; + case op_maskeqz: + printf_instr("MASKEQZ\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rk(instr) == 0 ? 0 : rj(instr)); + break; + case op_masknez: + printf_instr("MASKNEZ\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rk(instr) != 0 ? 0 : rj(instr)); + break; + case op_nor: + printf_instr("NOR\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), ~(rj(instr) | rk(instr))); + break; + case op_and: + printf_instr("AND\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) & rk(instr)); + break; + case op_or: + printf_instr("OR\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) | rk(instr)); + break; + case op_xor: + printf_instr("XOR\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) ^ rk(instr)); + break; + case op_orn: + printf_instr("ORN\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) | (~rk(instr))); + break; + case op_andn: + printf_instr("ANDN\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) & (~rk(instr))); + break; + case op_sll_w: + printf_instr("SLL_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), (int32_t)rj(instr) << (rk_u(instr) % 32)); + break; + case op_srl_w: { + printf_instr("SRL_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + alu_out = + static_cast((uint32_t)rj_u(instr) >> (rk_u(instr) % 32)); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_sra_w: + printf_instr("SRA_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), (int32_t)rj(instr) >> (rk_u(instr) % 32)); + break; + case op_sll_d: + printf_instr("SLL_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) << (rk_u(instr) % 64)); + break; + case op_srl_d: { + printf_instr("SRL_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + alu_out = static_cast(rj_u(instr) >> (rk_u(instr) % 64)); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_sra_d: + printf_instr("SRA_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) >> (rk_u(instr) % 64)); + break; + case op_rotr_w: { + printf_instr("ROTR_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + alu_out = static_cast( + RotateRight32(static_cast(rj_u(instr)), + static_cast(rk_u(instr) % 32))); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_rotr_d: { + printf_instr("ROTR_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + alu_out = static_cast( + RotateRight64((rj_u(instr)), (rk_u(instr) % 64))); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_mul_w: { + printf_instr("MUL_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + alu_out = + static_cast(rj(instr)) * static_cast(rk(instr)); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_mulh_w: { + printf_instr("MULH_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + int32_t rj_lo = static_cast(rj(instr)); + int32_t rk_lo = static_cast(rk(instr)); + alu_out = static_cast(rj_lo) * static_cast(rk_lo); + setRegister(rd_reg(instr), alu_out >> 32); + break; + } + case op_mulh_wu: { + printf_instr("MULH_WU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + uint32_t rj_lo = static_cast(rj_u(instr)); + uint32_t rk_lo = static_cast(rk_u(instr)); + alu_out = static_cast(rj_lo) * static_cast(rk_lo); + setRegister(rd_reg(instr), alu_out >> 32); + break; + } + case op_mul_d: + printf_instr("MUL_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), rj(instr) * rk(instr)); + break; + case op_mulh_d: + printf_instr("MULH_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), MultiplyHighSigned(rj(instr), rk(instr))); + break; + case op_mulh_du: + printf_instr("MULH_DU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), + MultiplyHighUnsigned(rj_u(instr), rk_u(instr))); + break; + case op_mulw_d_w: { + printf_instr("MULW_D_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + int64_t rj_i32 = static_cast(rj(instr)); + int64_t rk_i32 = static_cast(rk(instr)); + setRegister(rd_reg(instr), rj_i32 * rk_i32); + break; + } + case op_mulw_d_wu: { + printf_instr("MULW_D_WU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + uint64_t rj_u32 = static_cast(rj_u(instr)); + uint64_t rk_u32 = static_cast(rk_u(instr)); + setRegister(rd_reg(instr), rj_u32 * rk_u32); + break; + } + case op_div_w: { + printf_instr("DIV_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + int32_t rj_i32 = static_cast(rj(instr)); + int32_t rk_i32 = static_cast(rk(instr)); + if (rj_i32 == INT_MIN && rk_i32 == -1) { + setRegister(rd_reg(instr), INT_MIN); + } else if (rk_i32 != 0) { + setRegister(rd_reg(instr), rj_i32 / rk_i32); + } + break; + } + case op_mod_w: { + printf_instr("MOD_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + int32_t rj_i32 = static_cast(rj(instr)); + int32_t rk_i32 = static_cast(rk(instr)); + if (rj_i32 == INT_MIN && rk_i32 == -1) { + setRegister(rd_reg(instr), 0); + } else if (rk_i32 != 0) { + setRegister(rd_reg(instr), rj_i32 % rk_i32); + } + break; + } + case op_div_wu: { + printf_instr("DIV_WU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + uint32_t rj_u32 = static_cast(rj(instr)); + uint32_t rk_u32 = static_cast(rk(instr)); + if (rk_u32 != 0) { + setRegister(rd_reg(instr), static_cast(rj_u32 / rk_u32)); + } + break; + } + case op_mod_wu: { + printf_instr("MOD_WU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + uint32_t rj_u32 = static_cast(rj(instr)); + uint32_t rk_u32 = static_cast(rk(instr)); + if (rk_u32 != 0) { + setRegister(rd_reg(instr), static_cast(rj_u32 % rk_u32)); + } + break; + } + case op_div_d: { + printf_instr("DIV_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + if (rj(instr) == INT64_MIN && rk(instr) == -1) { + setRegister(rd_reg(instr), INT64_MIN); + } else if (rk(instr) != 0) { + setRegister(rd_reg(instr), rj(instr) / rk(instr)); + } + break; + } + case op_mod_d: { + printf_instr("MOD_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + if (rj(instr) == LONG_MIN && rk(instr) == -1) { + setRegister(rd_reg(instr), 0); + } else if (rk(instr) != 0) { + setRegister(rd_reg(instr), rj(instr) % rk(instr)); + } + break; + } + case op_div_du: { + printf_instr("DIV_DU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + if (rk_u(instr) != 0) { + setRegister(rd_reg(instr), + static_cast(rj_u(instr) / rk_u(instr))); + } + break; + } + case op_mod_du: { + printf_instr("MOD_DU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + if (rk_u(instr) != 0) { + setRegister(rd_reg(instr), + static_cast(rj_u(instr) % rk_u(instr))); + } + break; + } + case op_break: + printf_instr("BREAK\t code: %x\n", instr->bits(14, 0)); + softwareInterrupt(instr); + break; + case op_fadd_s: { + printf_instr("FADD_S\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr)); + setFpuRegisterFloat(fd_reg(instr), fj_float(instr) + fk_float(instr)); + break; + } + case op_fadd_d: { + printf_instr("FADD_D\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_double(instr)); + setFpuRegisterDouble(fd_reg(instr), fj_double(instr) + fk_double(instr)); + break; + } + case op_fsub_s: { + printf_instr("FSUB_S\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr)); + setFpuRegisterFloat(fd_reg(instr), fj_float(instr) - fk_float(instr)); + break; + } + case op_fsub_d: { + printf_instr("FSUB_D\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_double(instr)); + setFpuRegisterDouble(fd_reg(instr), fj_double(instr) - fk_double(instr)); + break; + } + case op_fmul_s: { + printf_instr("FMUL_S\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr)); + setFpuRegisterFloat(fd_reg(instr), fj_float(instr) * fk_float(instr)); + break; + } + case op_fmul_d: { + printf_instr("FMUL_D\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_double(instr)); + setFpuRegisterDouble(fd_reg(instr), fj_double(instr) * fk_double(instr)); + break; + } + case op_fdiv_s: { + printf_instr("FDIV_S\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr)); + setFpuRegisterFloat(fd_reg(instr), fj_float(instr) / fk_float(instr)); + break; + } + + case op_fdiv_d: { + printf_instr("FDIV_D\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_double(instr)); + setFpuRegisterDouble(fd_reg(instr), fj_double(instr) / fk_double(instr)); + break; + } + case op_fmax_s: { + printf_instr("FMAX_S\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr)); + setFpuRegisterFloat(fd_reg(instr), + FPUMax(fk_float(instr), fj_float(instr))); + break; + } + case op_fmax_d: { + printf_instr("FMAX_D\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_double(instr)); + setFpuRegisterDouble(fd_reg(instr), + FPUMax(fk_double(instr), fj_double(instr))); + break; + } + case op_fmin_s: { + printf_instr("FMIN_S\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr)); + setFpuRegisterFloat(fd_reg(instr), + FPUMin(fk_float(instr), fj_float(instr))); + break; + } + case op_fmin_d: { + printf_instr("FMIN_D\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_double(instr)); + setFpuRegisterDouble(fd_reg(instr), + FPUMin(fk_double(instr), fj_double(instr))); + break; + } + case op_fmaxa_s: { + printf_instr("FMAXA_S\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr)); + setFpuRegisterFloat(fd_reg(instr), + FPUMaxA(fk_float(instr), fj_float(instr))); + break; + } + case op_fmaxa_d: { + printf_instr("FMAXA_D\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_double(instr)); + setFpuRegisterDouble(fd_reg(instr), + FPUMaxA(fk_double(instr), fj_double(instr))); + break; + } + case op_fmina_s: { + printf_instr("FMINA_S\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_float(instr)); + setFpuRegisterFloat(fd_reg(instr), + FPUMinA(fk_float(instr), fj_float(instr))); + break; + } + case op_fmina_d: { + printf_instr("FMINA_D\t %s: %016f, %s, %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr), + FloatRegisters::GetName(fk_reg(instr)), fk_double(instr)); + setFpuRegisterDouble(fd_reg(instr), + FPUMinA(fk_double(instr), fj_double(instr))); + break; + } + case op_ldx_b: + printf_instr("LDX_B\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), readB(rj(instr) + rk(instr))); + break; + case op_ldx_h: + printf_instr("LDX_H\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), readH(rj(instr) + rk(instr), instr)); + break; + case op_ldx_w: + printf_instr("LDX_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), readW(rj(instr) + rk(instr), instr)); + break; + case op_ldx_d: + printf_instr("LDX_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), readDW(rj(instr) + rk(instr), instr)); + break; + case op_stx_b: + printf_instr("STX_B\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + writeB(rj(instr) + rk(instr), static_cast(rd(instr))); + break; + case op_stx_h: + printf_instr("STX_H\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + writeH(rj(instr) + rk(instr), static_cast(rd(instr)), instr); + break; + case op_stx_w: + printf_instr("STX_W\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + writeW(rj(instr) + rk(instr), static_cast(rd(instr)), instr); + break; + case op_stx_d: + printf_instr("STX_D\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + writeDW(rj(instr) + rk(instr), rd(instr), instr); + break; + case op_ldx_bu: + printf_instr("LDX_BU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), readBU(rj(instr) + rk(instr))); + break; + case op_ldx_hu: + printf_instr("LDX_HU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), readHU(rj(instr) + rk(instr), instr)); + break; + case op_ldx_wu: + printf_instr("LDX_WU\t %s: %016lx, %s, %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setRegister(rd_reg(instr), readWU(rj(instr) + rk(instr), instr)); + break; + case op_fldx_s: + printf_instr("FLDX_S\t %s: %016f, %s: %016lx, %s: %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setFpuRegister(fd_reg(instr), kFPUInvalidResult); // Trash upper 32 bits. + setFpuRegisterWord(fd_reg(instr), readW(rj(instr) + rk(instr), instr)); + break; + case op_fldx_d: + printf_instr("FLDX_D\t %s: %016f, %s: %016lx, %s: %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + setFpuRegister(fd_reg(instr), kFPUInvalidResult); // Trash upper 32 bits. + setFpuRegisterDouble(fd_reg(instr), readD(rj(instr) + rk(instr), instr)); + break; + case op_fstx_s: { + printf_instr("FSTX_S\t %s: %016f, %s: %016lx, %s: %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + int32_t alu_out_32 = static_cast(getFpuRegister(fd_reg(instr))); + writeW(rj(instr) + rk(instr), alu_out_32, instr); + break; + } + case op_fstx_d: { + printf_instr("FSTX_D\t %s: %016f, %s: %016lx, %s: %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + Registers::GetName(rj_reg(instr)), rj(instr), + Registers::GetName(rk_reg(instr)), rk(instr)); + writeD(rj(instr) + rk(instr), getFpuRegisterDouble(fd_reg(instr)), instr); + break; + } + case op_amswap_w: + printf_instr("Sim UNIMPLEMENTED: AMSWAP_W\n"); + UNIMPLEMENTED(); + break; + case op_amswap_d: + printf_instr("Sim UNIMPLEMENTED: AMSWAP_D\n"); + UNIMPLEMENTED(); + break; + case op_amadd_w: + printf_instr("Sim UNIMPLEMENTED: AMADD_W\n"); + UNIMPLEMENTED(); + break; + case op_amadd_d: + printf_instr("Sim UNIMPLEMENTED: AMADD_D\n"); + UNIMPLEMENTED(); + break; + case op_amand_w: + printf_instr("Sim UNIMPLEMENTED: AMAND_W\n"); + UNIMPLEMENTED(); + break; + case op_amand_d: + printf_instr("Sim UNIMPLEMENTED: AMAND_D\n"); + UNIMPLEMENTED(); + break; + case op_amor_w: + printf_instr("Sim UNIMPLEMENTED: AMOR_W\n"); + UNIMPLEMENTED(); + break; + case op_amor_d: + printf_instr("Sim UNIMPLEMENTED: AMOR_D\n"); + UNIMPLEMENTED(); + break; + case op_amxor_w: + printf_instr("Sim UNIMPLEMENTED: AMXOR_W\n"); + UNIMPLEMENTED(); + break; + case op_amxor_d: + printf_instr("Sim UNIMPLEMENTED: AMXOR_D\n"); + UNIMPLEMENTED(); + break; + case op_ammax_w: + printf_instr("Sim UNIMPLEMENTED: AMMAX_W\n"); + UNIMPLEMENTED(); + break; + case op_ammax_d: + printf_instr("Sim UNIMPLEMENTED: AMMAX_D\n"); + UNIMPLEMENTED(); + break; + case op_ammin_w: + printf_instr("Sim UNIMPLEMENTED: AMMIN_W\n"); + UNIMPLEMENTED(); + break; + case op_ammin_d: + printf_instr("Sim UNIMPLEMENTED: AMMIN_D\n"); + UNIMPLEMENTED(); + break; + case op_ammax_wu: + printf_instr("Sim UNIMPLEMENTED: AMMAX_WU\n"); + UNIMPLEMENTED(); + break; + case op_ammax_du: + printf_instr("Sim UNIMPLEMENTED: AMMAX_DU\n"); + UNIMPLEMENTED(); + break; + case op_ammin_wu: + printf_instr("Sim UNIMPLEMENTED: AMMAX_WU\n"); + UNIMPLEMENTED(); + break; + case op_ammin_du: + printf_instr("Sim UNIMPLEMENTED: AMMIN_DU\n"); + UNIMPLEMENTED(); + break; + case op_amswap_db_w: + printf_instr("Sim UNIMPLEMENTED: AMSWAP_DB_W\n"); + UNIMPLEMENTED(); + break; + case op_amswap_db_d: + printf_instr("Sim UNIMPLEMENTED: AMSWAP_DB_D\n"); + UNIMPLEMENTED(); + break; + case op_amadd_db_w: + printf_instr("Sim UNIMPLEMENTED: AMADD_DB_W\n"); + UNIMPLEMENTED(); + break; + case op_amadd_db_d: + printf_instr("Sim UNIMPLEMENTED: AMADD_DB_D\n"); + UNIMPLEMENTED(); + break; + case op_amand_db_w: + printf_instr("Sim UNIMPLEMENTED: AMAND_DB_W\n"); + UNIMPLEMENTED(); + break; + case op_amand_db_d: + printf_instr("Sim UNIMPLEMENTED: AMAND_DB_D\n"); + UNIMPLEMENTED(); + break; + case op_amor_db_w: + printf_instr("Sim UNIMPLEMENTED: AMOR_DB_W\n"); + UNIMPLEMENTED(); + break; + case op_amor_db_d: + printf_instr("Sim UNIMPLEMENTED: AMOR_DB_D\n"); + UNIMPLEMENTED(); + break; + case op_amxor_db_w: + printf_instr("Sim UNIMPLEMENTED: AMXOR_DB_W\n"); + UNIMPLEMENTED(); + break; + case op_amxor_db_d: + printf_instr("Sim UNIMPLEMENTED: AMXOR_DB_D\n"); + UNIMPLEMENTED(); + break; + case op_ammax_db_w: + printf_instr("Sim UNIMPLEMENTED: AMMAX_DB_W\n"); + UNIMPLEMENTED(); + break; + case op_ammax_db_d: + printf_instr("Sim UNIMPLEMENTED: AMMAX_DB_D\n"); + UNIMPLEMENTED(); + break; + case op_ammin_db_w: + printf_instr("Sim UNIMPLEMENTED: AMMIN_DB_W\n"); + UNIMPLEMENTED(); + break; + case op_ammin_db_d: + printf_instr("Sim UNIMPLEMENTED: AMMIN_DB_D\n"); + UNIMPLEMENTED(); + break; + case op_ammax_db_wu: + printf_instr("Sim UNIMPLEMENTED: AMMAX_DB_WU\n"); + UNIMPLEMENTED(); + break; + case op_ammax_db_du: + printf_instr("Sim UNIMPLEMENTED: AMMAX_DB_DU\n"); + UNIMPLEMENTED(); + break; + case op_ammin_db_wu: + printf_instr("Sim UNIMPLEMENTED: AMMIN_DB_WU\n"); + UNIMPLEMENTED(); + break; + case op_ammin_db_du: + printf_instr("Sim UNIMPLEMENTED: AMMIN_DB_DU\n"); + UNIMPLEMENTED(); + break; + case op_dbar: + // TODO:LOONG64 Recheck dbar simulation + break; + case op_ibar: + printf_instr("Sim UNIMPLEMENTED: IBAR\n"); + UNIMPLEMENTED(); + break; + case op_fcopysign_s: + printf_instr("Sim UNIMPLEMENTED: FCOPYSIGN_S\n"); + UNIMPLEMENTED(); + break; + case op_fcopysign_d: + printf_instr("Sim UNIMPLEMENTED: FCOPYSIGN_D\n"); + UNIMPLEMENTED(); + break; + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp22(SimInstruction* instr) { + int64_t alu_out; + + switch (instr->bits(31, 10) << 10) { + case op_clz_w: { + printf_instr("CLZ_W\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + alu_out = U32(I32_CHECK(rj_u(instr))) + ? __builtin_clz(U32(I32_CHECK(rj_u(instr)))) + : 32; + // alu_out = + // base::bits::CountLeadingZeros32(static_cast(rj_u(instr))); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_ctz_w: { + printf_instr("CTZ_W\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + alu_out = U32(I32_CHECK(rj_u(instr))) + ? __builtin_ctz(U32(I32_CHECK(rj_u(instr)))) + : 32; + // alu_out = + // base::bits::CountTrailingZeros32(static_cast(rj_u(instr))); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_clz_d: { + printf_instr("CLZ_D\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + // alu_out = + // base::bits::CountLeadingZeros64(static_cast(rj_u(instr))); + alu_out = U64(rj_u(instr)) ? __builtin_clzll(U64(rj_u(instr))) : 64; + setRegister(rd_reg(instr), alu_out); + break; + } + case op_ctz_d: { + printf_instr("CTZ_D\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + alu_out = U64(rj_u(instr)) ? __builtin_ctzll(U64(rj_u(instr))) : 64; + // alu_out = U64(rj_u(instr)) ? __builtin_ctzll(U64(rj_u(instr))) : + // 64; alu_out = + // base::bits::CountTrailingZeros64(static_cast(rj_u(instr))); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_revb_2h: { + printf_instr("REVB_2H\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint32_t input = static_cast(rj(instr)); + uint64_t output = 0; + + uint32_t mask = 0xFF000000; + for (int i = 0; i < 4; i++) { + uint32_t tmp = mask & input; + if (i % 2 == 0) { + tmp = tmp >> 8; + } else { + tmp = tmp << 8; + } + output = output | tmp; + mask = mask >> 8; + } + + alu_out = static_cast(static_cast(output)); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_revb_4h: { + printf_instr("REVB_4H\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint64_t input = rj_u(instr); + uint64_t output = 0; + + uint64_t mask = 0xFF00000000000000; + for (int i = 0; i < 8; i++) { + uint64_t tmp = mask & input; + if (i % 2 == 0) { + tmp = tmp >> 8; + } else { + tmp = tmp << 8; + } + output = output | tmp; + mask = mask >> 8; + } + + alu_out = static_cast(output); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_revb_2w: { + printf_instr("REVB_2W\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint64_t input = rj_u(instr); + uint64_t output = 0; + + uint64_t mask = 0xFF000000FF000000; + for (int i = 0; i < 4; i++) { + uint64_t tmp = mask & input; + if (i <= 1) { + tmp = tmp >> (24 - i * 16); + } else { + tmp = tmp << (i * 16 - 24); + } + output = output | tmp; + mask = mask >> 8; + } + + alu_out = static_cast(output); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_revb_d: { + printf_instr("REVB_D\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint64_t input = rj_u(instr); + uint64_t output = 0; + + uint64_t mask = 0xFF00000000000000; + for (int i = 0; i < 8; i++) { + uint64_t tmp = mask & input; + if (i <= 3) { + tmp = tmp >> (56 - i * 16); + } else { + tmp = tmp << (i * 16 - 56); + } + output = output | tmp; + mask = mask >> 8; + } + + alu_out = static_cast(output); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_revh_2w: { + printf_instr("REVH_2W\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint64_t input = rj_u(instr); + uint64_t output = 0; + + uint64_t mask = 0xFFFF000000000000; + for (int i = 0; i < 4; i++) { + uint64_t tmp = mask & input; + if (i % 2 == 0) { + tmp = tmp >> 16; + } else { + tmp = tmp << 16; + } + output = output | tmp; + mask = mask >> 16; + } + + alu_out = static_cast(output); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_revh_d: { + printf_instr("REVH_D\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint64_t input = rj_u(instr); + uint64_t output = 0; + + uint64_t mask = 0xFFFF000000000000; + for (int i = 0; i < 4; i++) { + uint64_t tmp = mask & input; + if (i <= 1) { + tmp = tmp >> (48 - i * 32); + } else { + tmp = tmp << (i * 32 - 48); + } + output = output | tmp; + mask = mask >> 16; + } + + alu_out = static_cast(output); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_bitrev_4b: { + printf_instr("BITREV_4B\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint32_t input = static_cast(rj(instr)); + uint32_t output = 0; + uint8_t i_byte, o_byte; + + // Reverse the bit in byte for each individual byte + for (int i = 0; i < 4; i++) { + output = output >> 8; + i_byte = input & 0xFF; + + // Fast way to reverse bits in byte + // Devised by Sean Anderson, July 13, 2001 + o_byte = static_cast(((i_byte * 0x0802LU & 0x22110LU) | + (i_byte * 0x8020LU & 0x88440LU)) * + 0x10101LU >> + 16); + + output = output | (static_cast(o_byte << 24)); + input = input >> 8; + } + + alu_out = static_cast(static_cast(output)); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_bitrev_8b: { + printf_instr("BITREV_8B\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint64_t input = rj_u(instr); + uint64_t output = 0; + uint8_t i_byte, o_byte; + + // Reverse the bit in byte for each individual byte + for (int i = 0; i < 8; i++) { + output = output >> 8; + i_byte = input & 0xFF; + + // Fast way to reverse bits in byte + // Devised by Sean Anderson, July 13, 2001 + o_byte = static_cast(((i_byte * 0x0802LU & 0x22110LU) | + (i_byte * 0x8020LU & 0x88440LU)) * + 0x10101LU >> + 16); + + output = output | (static_cast(o_byte) << 56); + input = input >> 8; + } + + alu_out = static_cast(output); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_bitrev_w: { + printf_instr("BITREV_W\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint32_t input = static_cast(rj(instr)); + uint32_t output = 0; + output = ReverseBits(input); + alu_out = static_cast(static_cast(output)); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_bitrev_d: { + printf_instr("BITREV_D\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + alu_out = static_cast(ReverseBits(rj_u(instr))); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_ext_w_b: { + printf_instr("EXT_W_B\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint8_t input = static_cast(rj(instr)); + alu_out = static_cast(static_cast(input)); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_ext_w_h: { + printf_instr("EXT_W_H\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj_u(instr)); + uint16_t input = static_cast(rj(instr)); + alu_out = static_cast(static_cast(input)); + setRegister(rd_reg(instr), alu_out); + break; + } + case op_fabs_s: { + printf_instr("FABS_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterFloat(fd_reg(instr), std::abs(fj_float(instr))); + break; + } + case op_fabs_d: { + printf_instr("FABS_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterDouble(fd_reg(instr), std::abs(fj_double(instr))); + break; + } + case op_fneg_s: { + printf_instr("FNEG_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterFloat(fd_reg(instr), -fj_float(instr)); + break; + } + case op_fneg_d: { + printf_instr("FNEG_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterDouble(fd_reg(instr), -fj_double(instr)); + break; + } + case op_fsqrt_s: { + printf_instr("FSQRT_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + if (fj_float(instr) >= 0) { + setFpuRegisterFloat(fd_reg(instr), std::sqrt(fj_float(instr))); + } else { + setFpuRegisterFloat(fd_reg(instr), std::sqrt(-1)); // qnan + setFCSRBit(kFCSRInvalidOpFlagBit, true); + } + break; + } + case op_fsqrt_d: { + printf_instr("FSQRT_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + if (fj_double(instr) >= 0) { + setFpuRegisterDouble(fd_reg(instr), std::sqrt(fj_double(instr))); + } else { + setFpuRegisterDouble(fd_reg(instr), std::sqrt(-1)); // qnan + setFCSRBit(kFCSRInvalidOpFlagBit, true); + } + break; + } + case op_fmov_s: { + printf_instr("FMOV_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_float(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + setFpuRegisterFloat(fd_reg(instr), fj_float(instr)); + break; + } + case op_fmov_d: { + printf_instr("FMOV_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + setFpuRegisterDouble(fd_reg(instr), fj_double(instr)); + break; + } + case op_movgr2fr_w: { + printf_instr("MOVGR2FR_W\t %s: %016f, %s, %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + Registers::GetName(rj_reg(instr)), rj(instr)); + setFpuRegisterWord(fd_reg(instr), static_cast(rj(instr))); + break; + } + case op_movgr2fr_d: { + printf_instr("MOVGR2FR_D\t %s: %016f, %s, %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + Registers::GetName(rj_reg(instr)), rj(instr)); + setFpuRegister(fd_reg(instr), rj(instr)); + break; + } + case op_movgr2frh_w: { + printf_instr("MOVGR2FRH_W\t %s: %016f, %s, %016lx\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + Registers::GetName(rj_reg(instr)), rj(instr)); + setFpuRegisterHiWord(fd_reg(instr), static_cast(rj(instr))); + break; + } + case op_movfr2gr_s: { + printf_instr("MOVFR2GR_S\t %s: %016lx, %s, %016f\n", + Registers::GetName(rd_reg(instr)), rd(instr), + FloatRegisters::GetName(rj_reg(instr)), fj_float(instr)); + setRegister(rd_reg(instr), + static_cast(getFpuRegisterWord(fj_reg(instr)))); + break; + } + case op_movfr2gr_d: { + printf_instr("MOVFR2GR_D\t %s: %016lx, %s, %016f\n", + Registers::GetName(rd_reg(instr)), rd(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + setRegister(rd_reg(instr), getFpuRegister(fj_reg(instr))); + break; + } + case op_movfrh2gr_s: { + printf_instr("MOVFRH2GR_S\t %s: %016lx, %s, %016lx\n", + Registers::GetName(rd_reg(instr)), rd(instr), + Registers::GetName(rj_reg(instr)), rj(instr)); + setRegister(rd_reg(instr), getFpuRegisterHiWord(fj_reg(instr))); + break; + } + case op_movgr2fcsr: { + printf_instr("MOVGR2FCSR\t fcsr: %016x, %s, %016lx\n", FCSR_, + Registers::GetName(rj_reg(instr)), rj(instr)); + // fcsr could be 0-3 + MOZ_ASSERT(rd_reg(instr) < 4); + FCSR_ = static_cast(rj(instr)); + break; + } + case op_movfcsr2gr: { + printf_instr("MOVFCSR2GR\t fcsr: %016x, %s, %016lx\n", FCSR_, + Registers::GetName(rd_reg(instr)), rd(instr)); + setRegister(rd_reg(instr), FCSR_); + break; + } + case op_fcvt_s_d: { + printf_instr("FCVT_S_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + setFpuRegisterFloat(fd_reg(instr), static_cast(fj_double(instr))); + break; + } + case op_fcvt_d_s: { + printf_instr("FCVT_D_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + setFpuRegisterDouble(fd_reg(instr), static_cast(fj_float(instr))); + break; + } + case op_ftintrm_w_s: { + printf_instr("FTINTRM_W_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded = std::floor(fj); + int32_t result = static_cast(rounded); + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrm_w_d: { + printf_instr("FTINTRM_W_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded = std::floor(fj); + int32_t result = static_cast(rounded); + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrm_l_s: { + printf_instr("FTINTRM_L_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded = std::floor(fj); + int64_t result = static_cast(rounded); + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrm_l_d: { + printf_instr("FTINTRM_L_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded = std::floor(fj); + int64_t result = static_cast(rounded); + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrp_w_s: { + printf_instr("FTINTRP_W_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded = std::ceil(fj); + int32_t result = static_cast(rounded); + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrp_w_d: { + printf_instr("FTINTRP_W_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded = std::ceil(fj); + int32_t result = static_cast(rounded); + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrp_l_s: { + printf_instr("FTINTRP_L_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded = std::ceil(fj); + int64_t result = static_cast(rounded); + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrp_l_d: { + printf_instr("FTINTRP_L_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded = std::ceil(fj); + int64_t result = static_cast(rounded); + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrz_w_s: { + printf_instr("FTINTRZ_W_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded = std::trunc(fj); + int32_t result = static_cast(rounded); + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrz_w_d: { + printf_instr("FTINTRZ_W_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded = std::trunc(fj); + int32_t result = static_cast(rounded); + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrz_l_s: { + printf_instr("FTINTRZ_L_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded = std::trunc(fj); + int64_t result = static_cast(rounded); + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrz_l_d: { + printf_instr("FTINTRZ_L_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded = std::trunc(fj); + int64_t result = static_cast(rounded); + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrne_w_s: { + printf_instr("FTINTRNE_W_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded = std::floor(fj + 0.5); + int32_t result = static_cast(rounded); + if ((result & 1) != 0 && result - fj == 0.5) { + // If the number is halfway between two integers, + // round to the even one. + result--; + } + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrne_w_d: { + printf_instr("FTINTRNE_W_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded = std::floor(fj + 0.5); + int32_t result = static_cast(rounded); + if ((result & 1) != 0 && result - fj == 0.5) { + // If the number is halfway between two integers, + // round to the even one. + result--; + } + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrne_l_s: { + printf_instr("FTINTRNE_L_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded = std::floor(fj + 0.5); + int64_t result = static_cast(rounded); + if ((result & 1) != 0 && result - fj == 0.5) { + // If the number is halfway between two integers, + // round to the even one. + result--; + } + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftintrne_l_d: { + printf_instr("FTINTRNE_L_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded = std::floor(fj + 0.5); + int64_t result = static_cast(rounded); + if ((result & 1) != 0 && result - fj == 0.5) { + // If the number is halfway between two integers, + // round to the even one. + result--; + } + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftint_w_s: { + printf_instr("FTINT_W_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded; + int32_t result; + roundAccordingToFCSR(fj, &rounded, &result); + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftint_w_d: { + printf_instr("FTINT_W_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded; + int32_t result; + roundAccordingToFCSR(fj, &rounded, &result); + setFpuRegisterWord(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftint_l_s: { + printf_instr("FTINT_L_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float rounded; + int64_t result; + round64AccordingToFCSR(fj, &rounded, &result); + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ftint_l_d: { + printf_instr("FTINT_L_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double rounded; + int64_t result; + round64AccordingToFCSR(fj, &rounded, &result); + setFpuRegister(fd_reg(instr), result); + if (setFCSRRoundError(fj, rounded)) { + setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr)); + } + break; + } + case op_ffint_s_w: { + alu_out = getFpuRegisterSignedWord(fj_reg(instr)); + printf_instr("FFINT_S_W\t %s: %016f, %s, %016x\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), (int)alu_out); + setFpuRegisterFloat(fd_reg(instr), static_cast(alu_out)); + break; + } + case op_ffint_s_l: { + alu_out = getFpuRegister(fj_reg(instr)); + printf_instr("FFINT_S_L\t %s: %016f, %s, %016x\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), (int)alu_out); + setFpuRegisterFloat(fd_reg(instr), static_cast(alu_out)); + break; + } + case op_ffint_d_w: { + alu_out = getFpuRegisterSignedWord(fj_reg(instr)); + printf_instr("FFINT_D_W\t %s: %016f, %s, %016x\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), (int)alu_out); + setFpuRegisterDouble(fd_reg(instr), static_cast(alu_out)); + break; + } + case op_ffint_d_l: { + alu_out = getFpuRegister(fj_reg(instr)); + printf_instr("FFINT_D_L\t %s: %016f, %s, %016x\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), (int)alu_out); + setFpuRegisterDouble(fd_reg(instr), static_cast(alu_out)); + break; + } + case op_frint_s: { + printf_instr("FRINT_S\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_float(instr)); + float fj = fj_float(instr); + float result, temp_result; + double temp; + float upper = std::ceil(fj); + float lower = std::floor(fj); + switch (getFCSRRoundingMode()) { + case kRoundToNearest: + printf_instr(" kRoundToNearest\n"); + if (upper - fj < fj - lower) { + result = upper; + } else if (upper - fj > fj - lower) { + result = lower; + } else { + temp_result = upper / 2; + float reminder = std::modf(temp_result, &temp); + if (reminder == 0) { + result = upper; + } else { + result = lower; + } + } + break; + case kRoundToZero: + printf_instr(" kRoundToZero\n"); + result = (fj > 0 ? lower : upper); + break; + case kRoundToPlusInf: + printf_instr(" kRoundToPlusInf\n"); + result = upper; + break; + case kRoundToMinusInf: + printf_instr(" kRoundToMinusInf\n"); + result = lower; + break; + } + setFpuRegisterFloat(fd_reg(instr), result); + if (result != fj) { + setFCSRBit(kFCSRInexactFlagBit, true); + } + break; + } + case op_frint_d: { + printf_instr("FRINT_D\t %s: %016f, %s, %016f\n", + FloatRegisters::GetName(fd_reg(instr)), fd_double(instr), + FloatRegisters::GetName(fj_reg(instr)), fj_double(instr)); + double fj = fj_double(instr); + double result, temp, temp_result; + double upper = std::ceil(fj); + double lower = std::floor(fj); + switch (getFCSRRoundingMode()) { + case kRoundToNearest: + printf_instr(" kRoundToNearest\n"); + if (upper - fj < fj - lower) { + result = upper; + } else if (upper - fj > fj - lower) { + result = lower; + } else { + temp_result = upper / 2; + double reminder = std::modf(temp_result, &temp); + if (reminder == 0) { + result = upper; + } else { + result = lower; + } + } + break; + case kRoundToZero: + printf_instr(" kRoundToZero\n"); + result = (fj > 0 ? lower : upper); + break; + case kRoundToPlusInf: + printf_instr(" kRoundToPlusInf\n"); + result = upper; + break; + case kRoundToMinusInf: + printf_instr(" kRoundToMinusInf\n"); + result = lower; + break; + } + setFpuRegisterDouble(fd_reg(instr), result); + if (result != fj) { + setFCSRBit(kFCSRInexactFlagBit, true); + } + break; + } + case op_movfr2cf: + printf("Sim UNIMPLEMENTED: MOVFR2CF\n"); + UNIMPLEMENTED(); + break; + case op_movgr2cf: + printf("Sim UNIMPLEMENTED: MOVGR2CF\n"); + UNIMPLEMENTED(); + break; + case op_clo_w: + printf("Sim UNIMPLEMENTED: FCO_W\n"); + UNIMPLEMENTED(); + break; + case op_cto_w: + printf("Sim UNIMPLEMENTED: FTO_W\n"); + UNIMPLEMENTED(); + break; + case op_clo_d: + printf("Sim UNIMPLEMENTED: FLO_D\n"); + UNIMPLEMENTED(); + break; + case op_cto_d: + printf("Sim UNIMPLEMENTED: FTO_D\n"); + UNIMPLEMENTED(); + break; + // Unimplemented opcodes raised an error in the configuration step before, + // so we can use the default here to set the destination register in common + // cases. + default: + UNREACHABLE(); + } +} + +void Simulator::decodeTypeOp24(SimInstruction* instr) { + switch (instr->bits(31, 8) << 8) { + case op_movcf2fr: + printf("Sim UNIMPLEMENTED: MOVCF2FR\n"); + UNIMPLEMENTED(); + break; + case op_movcf2gr: + printf_instr("MOVCF2GR\t %s, %016lx, FCC%d\n", + Registers::GetName(rd_reg(instr)), rd(instr), cj_reg(instr)); + setRegister(rd_reg(instr), getCFRegister(cj_reg(instr))); + break; + UNIMPLEMENTED(); + break; + default: + UNREACHABLE(); + } +} + +// Executes the current instruction. +void Simulator::instructionDecode(SimInstruction* instr) { + if (!SimulatorProcess::ICacheCheckingDisableCount) { + AutoLockSimulatorCache als; + SimulatorProcess::checkICacheLocked(instr); + } + pc_modified_ = false; + + static int instr_count = 0; + printf_instr("\nInstr%3d: %08x, PC: %016lx\t", instr_count++, + instr->bits(31, 0), get_pc()); + + switch (instr->instructionType()) { + case SimInstruction::kOp6Type: + decodeTypeOp6(instr); + break; + case SimInstruction::kOp7Type: + decodeTypeOp7(instr); + break; + case SimInstruction::kOp8Type: + decodeTypeOp8(instr); + break; + case SimInstruction::kOp10Type: + decodeTypeOp10(instr); + break; + case SimInstruction::kOp11Type: + decodeTypeOp11(instr); + break; + case SimInstruction::kOp12Type: + decodeTypeOp12(instr); + break; + case SimInstruction::kOp14Type: + decodeTypeOp14(instr); + break; + case SimInstruction::kOp15Type: + decodeTypeOp15(instr); + break; + case SimInstruction::kOp16Type: + decodeTypeOp16(instr); + break; + case SimInstruction::kOp17Type: + decodeTypeOp17(instr); + break; + case SimInstruction::kOp22Type: + decodeTypeOp22(instr); + break; + case SimInstruction::kOp24Type: + decodeTypeOp24(instr); + break; + default: + printf("instr: %08x\n", instr->bits(31, 0)); + UNSUPPORTED(); + } + if (!pc_modified_) { + setRegister(pc, reinterpret_cast(instr) + + SimInstruction::kInstrSize); // todo + } +} + +void Simulator::enable_single_stepping(SingleStepCallback cb, void* arg) { + single_stepping_ = true; + single_step_callback_ = cb; + single_step_callback_arg_ = arg; + single_step_callback_(single_step_callback_arg_, this, (void*)get_pc()); +} + +void Simulator::disable_single_stepping() { + if (!single_stepping_) { + return; + } + single_step_callback_(single_step_callback_arg_, this, (void*)get_pc()); + single_stepping_ = false; + single_step_callback_ = nullptr; + single_step_callback_arg_ = nullptr; +} + +template +void Simulator::execute() { + if (single_stepping_) { + single_step_callback_(single_step_callback_arg_, this, nullptr); + } + + // Get the PC to simulate. Cannot use the accessor here as we need the + // raw PC value and not the one used as input to arithmetic instructions. + int64_t program_counter = get_pc(); + + while (program_counter != end_sim_pc) { + if (enableStopSimAt && (icount_ == Simulator::StopSimAt)) { + loong64Debugger dbg(this); + dbg.debug(); + } else { + if (single_stepping_) { + single_step_callback_(single_step_callback_arg_, this, + (void*)program_counter); + } + SimInstruction* instr = + reinterpret_cast(program_counter); + instructionDecode(instr); + icount_++; + } + program_counter = get_pc(); + } + + if (single_stepping_) { + single_step_callback_(single_step_callback_arg_, this, nullptr); + } +} + +void Simulator::callInternal(uint8_t* entry) { + // Prepare to execute the code at entry. + setRegister(pc, reinterpret_cast(entry)); + // Put down marker for end of simulation. The simulator will stop simulation + // when the PC reaches this value. By saving the "end simulation" value into + // the LR the simulation stops when returning to this call point. + setRegister(ra, end_sim_pc); + + // Remember the values of callee-saved registers. + // The code below assumes that r9 is not used as sb (static base) in + // simulator code and therefore is regarded as a callee-saved register. + int64_t s0_val = getRegister(s0); + int64_t s1_val = getRegister(s1); + int64_t s2_val = getRegister(s2); + int64_t s3_val = getRegister(s3); + int64_t s4_val = getRegister(s4); + int64_t s5_val = getRegister(s5); + int64_t s6_val = getRegister(s6); + int64_t s7_val = getRegister(s7); + int64_t s8_val = getRegister(s8); + int64_t gp_val = getRegister(gp); + int64_t sp_val = getRegister(sp); + int64_t tp_val = getRegister(tp); + int64_t fp_val = getRegister(fp); + + // Set up the callee-saved registers with a known value. To be able to check + // that they are preserved properly across JS execution. + int64_t callee_saved_value = icount_; + setRegister(s0, callee_saved_value); + setRegister(s1, callee_saved_value); + setRegister(s2, callee_saved_value); + setRegister(s3, callee_saved_value); + setRegister(s4, callee_saved_value); + setRegister(s5, callee_saved_value); + setRegister(s6, callee_saved_value); + setRegister(s7, callee_saved_value); + setRegister(s8, callee_saved_value); + setRegister(gp, callee_saved_value); + setRegister(tp, callee_saved_value); + setRegister(fp, callee_saved_value); + + // Start the simulation. + if (Simulator::StopSimAt != -1) { + execute(); + } else { + execute(); + } + + // Check that the callee-saved registers have been preserved. + MOZ_ASSERT(callee_saved_value == getRegister(s0)); + MOZ_ASSERT(callee_saved_value == getRegister(s1)); + MOZ_ASSERT(callee_saved_value == getRegister(s2)); + MOZ_ASSERT(callee_saved_value == getRegister(s3)); + MOZ_ASSERT(callee_saved_value == getRegister(s4)); + MOZ_ASSERT(callee_saved_value == getRegister(s5)); + MOZ_ASSERT(callee_saved_value == getRegister(s6)); + MOZ_ASSERT(callee_saved_value == getRegister(s7)); + MOZ_ASSERT(callee_saved_value == getRegister(s8)); + MOZ_ASSERT(callee_saved_value == getRegister(gp)); + MOZ_ASSERT(callee_saved_value == getRegister(tp)); + MOZ_ASSERT(callee_saved_value == getRegister(fp)); + + // Restore callee-saved registers with the original value. + setRegister(s0, s0_val); + setRegister(s1, s1_val); + setRegister(s2, s2_val); + setRegister(s3, s3_val); + setRegister(s4, s4_val); + setRegister(s5, s5_val); + setRegister(s6, s6_val); + setRegister(s7, s7_val); + setRegister(s8, s8_val); + setRegister(gp, gp_val); + setRegister(sp, sp_val); + setRegister(tp, tp_val); + setRegister(fp, fp_val); +} + +int64_t Simulator::call(uint8_t* entry, int argument_count, ...) { + va_list parameters; + va_start(parameters, argument_count); + + int64_t original_stack = getRegister(sp); + // Compute position of stack on entry to generated code. + int64_t entry_stack = original_stack; + if (argument_count > kCArgSlotCount) { + entry_stack = entry_stack - argument_count * sizeof(int64_t); + } else { + entry_stack = entry_stack - kCArgsSlotsSize; + } + + entry_stack &= ~U64(ABIStackAlignment - 1); + + intptr_t* stack_argument = reinterpret_cast(entry_stack); + + // Setup the arguments. + for (int i = 0; i < argument_count; i++) { + js::jit::Register argReg; + if (GetIntArgReg(i, &argReg)) { + setRegister(argReg.code(), va_arg(parameters, int64_t)); + } else { + stack_argument[i] = va_arg(parameters, int64_t); + } + } + + va_end(parameters); + setRegister(sp, entry_stack); + + callInternal(entry); + + // Pop stack passed arguments. + MOZ_ASSERT(entry_stack == getRegister(sp)); + setRegister(sp, original_stack); + + int64_t result = getRegister(v0); + return result; +} + +uintptr_t Simulator::pushAddress(uintptr_t address) { + int new_sp = getRegister(sp) - sizeof(uintptr_t); + uintptr_t* stack_slot = reinterpret_cast(new_sp); + *stack_slot = address; + setRegister(sp, new_sp); + return new_sp; +} + +uintptr_t Simulator::popAddress() { + int current_sp = getRegister(sp); + uintptr_t* stack_slot = reinterpret_cast(current_sp); + uintptr_t address = *stack_slot; + setRegister(sp, current_sp + sizeof(uintptr_t)); + return address; +} + +} // namespace jit +} // namespace js + +js::jit::Simulator* JSContext::simulator() const { return simulator_; } diff --git a/js/src/jit/loong64/Simulator-loong64.h b/js/src/jit/loong64/Simulator-loong64.h new file mode 100644 index 0000000000..409b0fb707 --- /dev/null +++ b/js/src/jit/loong64/Simulator-loong64.h @@ -0,0 +1,651 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: */ +// Copyright 2020 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef jit_loong64_Simulator_loong64_h +#define jit_loong64_Simulator_loong64_h + +#ifdef JS_SIMULATOR_LOONG64 + +# include "mozilla/Atomics.h" + +# include "jit/IonTypes.h" +# include "js/ProfilingFrameIterator.h" +# include "threading/Thread.h" +# include "vm/MutexIDs.h" +# include "wasm/WasmSignalHandlers.h" + +namespace js { + +namespace jit { + +class JitActivation; + +class Simulator; +class Redirection; +class CachePage; +class AutoLockSimulator; + +// When the SingleStepCallback is called, the simulator is about to execute +// sim->get_pc() and the current machine state represents the completed +// execution of the previous pc. +typedef void (*SingleStepCallback)(void* arg, Simulator* sim, void* pc); + +const intptr_t kPointerAlignment = 8; +const intptr_t kPointerAlignmentMask = kPointerAlignment - 1; + +const intptr_t kDoubleAlignment = 8; +const intptr_t kDoubleAlignmentMask = kDoubleAlignment - 1; + +// Number of general purpose registers. +const int kNumRegisters = 32; + +// In the simulator, the PC register is simulated as the 34th register. +const int kPCRegister = 32; + +// Number coprocessor registers. +const int kNumFPURegisters = 32; + +// FPU (coprocessor 1) control registers. Currently only FCSR is implemented. +// TODO fcsr0 fcsr1 fcsr2 fcsr3 +const int kFCSRRegister = 0; +const int kInvalidFPUControlRegister = -1; +const uint32_t kFPUInvalidResult = static_cast(1 << 31) - 1; +const int32_t kFPUInvalidResultNegative = static_cast(1u << 31); +const uint64_t kFPU64InvalidResult = + static_cast(static_cast(1) << 63) - 1; +const int64_t kFPU64InvalidResultNegative = + static_cast(static_cast(1) << 63); + +const uint32_t kFPURoundingModeShift = 8; +const uint32_t kFPURoundingModeMask = 0b11 << kFPURoundingModeShift; + +// FPU rounding modes. +enum FPURoundingMode { + RN = 0b00 << kFPURoundingModeShift, // Round to Nearest. + RZ = 0b01 << kFPURoundingModeShift, // Round towards zero. + RP = 0b10 << kFPURoundingModeShift, // Round towards Plus Infinity. + RM = 0b11 << kFPURoundingModeShift, // Round towards Minus Infinity. + + // Aliases. + kRoundToNearest = RN, + kRoundToZero = RZ, + kRoundToPlusInf = RP, + kRoundToMinusInf = RM, + + mode_round = RN, + mode_ceil = RP, + mode_floor = RM, + mode_trunc = RZ +}; + +// FCSR constants. +// TODO +const uint32_t kFCSRInexactFlagBit = 16; +const uint32_t kFCSRUnderflowFlagBit = 17; +const uint32_t kFCSROverflowFlagBit = 18; +const uint32_t kFCSRDivideByZeroFlagBit = 19; +const uint32_t kFCSRInvalidOpFlagBit = 20; + +const uint32_t kFCSRInexactCauseBit = 24; +const uint32_t kFCSRUnderflowCauseBit = 25; +const uint32_t kFCSROverflowCauseBit = 26; +const uint32_t kFCSRDivideByZeroCauseBit = 27; +const uint32_t kFCSRInvalidOpCauseBit = 28; + +const uint32_t kFCSRInexactFlagMask = 1 << kFCSRInexactFlagBit; +const uint32_t kFCSRUnderflowFlagMask = 1 << kFCSRUnderflowFlagBit; +const uint32_t kFCSROverflowFlagMask = 1 << kFCSROverflowFlagBit; +const uint32_t kFCSRDivideByZeroFlagMask = 1 << kFCSRDivideByZeroFlagBit; +const uint32_t kFCSRInvalidOpFlagMask = 1 << kFCSRInvalidOpFlagBit; + +const uint32_t kFCSRFlagMask = + kFCSRInexactFlagMask | kFCSRUnderflowFlagMask | kFCSROverflowFlagMask | + kFCSRDivideByZeroFlagMask | kFCSRInvalidOpFlagMask; + +const uint32_t kFCSRExceptionFlagMask = kFCSRFlagMask ^ kFCSRInexactFlagMask; + +// On LOONG64 Simulator breakpoints can have different codes: +// - Breaks between 0 and kMaxWatchpointCode are treated as simple watchpoints, +// the simulator will run through them and print the registers. +// - Breaks between kMaxWatchpointCode and kMaxStopCode are treated as stop() +// instructions (see Assembler::stop()). +// - Breaks larger than kMaxStopCode are simple breaks, dropping you into the +// debugger. +const uint32_t kMaxWatchpointCode = 31; +const uint32_t kMaxStopCode = 127; +const uint32_t kWasmTrapCode = 6; + +// ----------------------------------------------------------------------------- +// Utility functions + +typedef uint32_t Instr; +class SimInstruction; + +// Per thread simulator state. +class Simulator { + friend class loong64Debugger; + + public: + // Registers are declared in order. + enum Register { + no_reg = -1, + zero_reg = 0, + ra, + gp, + sp, + a0, + a1, + a2, + a3, + a4, + a5, + a6, + a7, + t0, + t1, + t2, + t3, + t4, + t5, + t6, + t7, + t8, + tp, + fp, + s0, + s1, + s2, + s3, + s4, + s5, + s6, + s7, + s8, + pc, // pc must be the last register. + kNumSimuRegisters, + // aliases + v0 = a0, + v1 = a1, + }; + + // Condition flag registers. + enum CFRegister { + fcc0, + fcc1, + fcc2, + fcc3, + fcc4, + fcc5, + fcc6, + fcc7, + kNumCFRegisters + }; + + // Floating point registers. + enum FPURegister { + f0, + f1, + f2, + f3, + f4, + f5, + f6, + f7, + f8, + f9, + f10, + f11, + f12, + f13, + f14, + f15, + f16, + f17, + f18, + f19, + f20, + f21, + f22, + f23, + f24, + f25, + f26, + f27, + f28, + f29, + f30, + f31, + kNumFPURegisters + }; + + // Returns nullptr on OOM. + static Simulator* Create(); + + static void Destroy(Simulator* simulator); + + // Constructor/destructor are for internal use only; use the static methods + // above. + Simulator(); + ~Simulator(); + + // The currently executing Simulator instance. Potentially there can be one + // for each native thread. + static Simulator* Current(); + + static inline uintptr_t StackLimit() { + return Simulator::Current()->stackLimit(); + } + + uintptr_t* addressOfStackLimit(); + + // Accessors for register state. Reading the pc value adheres to the LOONG64 + // architecture specification and is off by a 8 from the currently executing + // instruction. + void setRegister(int reg, int64_t value); + int64_t getRegister(int reg) const; + // Same for FPURegisters. + void setFpuRegister(int fpureg, int64_t value); + void setFpuRegisterWord(int fpureg, int32_t value); + void setFpuRegisterHiWord(int fpureg, int32_t value); + void setFpuRegisterFloat(int fpureg, float value); + void setFpuRegisterDouble(int fpureg, double value); + + void setFpuRegisterWordInvalidResult(float original, float rounded, + int fpureg); + void setFpuRegisterWordInvalidResult(double original, double rounded, + int fpureg); + void setFpuRegisterInvalidResult(float original, float rounded, int fpureg); + void setFpuRegisterInvalidResult(double original, double rounded, int fpureg); + void setFpuRegisterInvalidResult64(float original, float rounded, int fpureg); + void setFpuRegisterInvalidResult64(double original, double rounded, + int fpureg); + + int64_t getFpuRegister(int fpureg) const; + // int32_t getFpuRegisterLo(int fpureg) const; + // int32_t getFpuRegisterHi(int fpureg) const; + int32_t getFpuRegisterWord(int fpureg) const; + int32_t getFpuRegisterSignedWord(int fpureg) const; + int32_t getFpuRegisterHiWord(int fpureg) const; + float getFpuRegisterFloat(int fpureg) const; + double getFpuRegisterDouble(int fpureg) const; + + void setCFRegister(int cfreg, bool value); + bool getCFRegister(int cfreg) const; + + void set_fcsr_rounding_mode(FPURoundingMode mode); + + void setFCSRBit(uint32_t cc, bool value); + bool testFCSRBit(uint32_t cc); + unsigned int getFCSRRoundingMode(); + template + bool setFCSRRoundError(double original, double rounded); + bool setFCSRRound64Error(float original, float rounded); + + template + void roundAccordingToFCSR(T toRound, T* rounded, int32_t* rounded_int); + + template + void round64AccordingToFCSR(T toRound, T* rounded, int64_t* rounded_int); + + // Special case of set_register and get_register to access the raw PC value. + void set_pc(int64_t value); + int64_t get_pc() const; + + template + T get_pc_as() const { + return reinterpret_cast(get_pc()); + } + + void enable_single_stepping(SingleStepCallback cb, void* arg); + void disable_single_stepping(); + + // Accessor to the internal simulator stack area. + uintptr_t stackLimit() const; + bool overRecursed(uintptr_t newsp = 0) const; + bool overRecursedWithExtra(uint32_t extra) const; + + // Executes LOONG64 instructions until the PC reaches end_sim_pc. + template + void execute(); + + // Sets up the simulator state and grabs the result on return. + int64_t call(uint8_t* entry, int argument_count, ...); + + // Push an address onto the JS stack. + uintptr_t pushAddress(uintptr_t address); + + // Pop an address from the JS stack. + uintptr_t popAddress(); + + // Debugger input. + void setLastDebuggerInput(char* input); + char* lastDebuggerInput() { return lastDebuggerInput_; } + + // Returns true if pc register contains one of the 'SpecialValues' defined + // below (bad_ra, end_sim_pc). + bool has_bad_pc() const; + + private: + enum SpecialValues { + // Known bad pc value to ensure that the simulator does not execute + // without being properly setup. + bad_ra = -1, + // A pc value used to signal the simulator to stop execution. Generally + // the ra is set to this value on transition from native C code to + // simulated execution, so that the simulator can "return" to the native + // C code. + end_sim_pc = -2, + // Unpredictable value. + Unpredictable = 0xbadbeaf + }; + + bool init(); + + // Unsupported instructions use Format to print an error and stop execution. + void format(SimInstruction* instr, const char* format); + + // Read and write memory. + inline uint8_t readBU(uint64_t addr); + inline int8_t readB(uint64_t addr); + inline void writeB(uint64_t addr, uint8_t value); + inline void writeB(uint64_t addr, int8_t value); + + inline uint16_t readHU(uint64_t addr, SimInstruction* instr); + inline int16_t readH(uint64_t addr, SimInstruction* instr); + inline void writeH(uint64_t addr, uint16_t value, SimInstruction* instr); + inline void writeH(uint64_t addr, int16_t value, SimInstruction* instr); + + inline uint32_t readWU(uint64_t addr, SimInstruction* instr); + inline int32_t readW(uint64_t addr, SimInstruction* instr); + inline void writeW(uint64_t addr, uint32_t value, SimInstruction* instr); + inline void writeW(uint64_t addr, int32_t value, SimInstruction* instr); + + inline int64_t readDW(uint64_t addr, SimInstruction* instr); + inline void writeDW(uint64_t addr, int64_t value, SimInstruction* instr); + + inline double readD(uint64_t addr, SimInstruction* instr); + inline void writeD(uint64_t addr, double value, SimInstruction* instr); + + inline int32_t loadLinkedW(uint64_t addr, SimInstruction* instr); + inline int storeConditionalW(uint64_t addr, int32_t value, + SimInstruction* instr); + + inline int64_t loadLinkedD(uint64_t addr, SimInstruction* instr); + inline int storeConditionalD(uint64_t addr, int64_t value, + SimInstruction* instr); + + // Executing is handled based on the instruction type. + void decodeTypeOp6(SimInstruction* instr); + void decodeTypeOp7(SimInstruction* instr); + void decodeTypeOp8(SimInstruction* instr); + void decodeTypeOp10(SimInstruction* instr); + void decodeTypeOp11(SimInstruction* instr); + void decodeTypeOp12(SimInstruction* instr); + void decodeTypeOp14(SimInstruction* instr); + void decodeTypeOp15(SimInstruction* instr); + void decodeTypeOp16(SimInstruction* instr); + void decodeTypeOp17(SimInstruction* instr); + void decodeTypeOp22(SimInstruction* instr); + void decodeTypeOp24(SimInstruction* instr); + + inline int32_t rj_reg(SimInstruction* instr) const; + inline int64_t rj(SimInstruction* instr) const; + inline uint64_t rj_u(SimInstruction* instr) const; + inline int32_t rk_reg(SimInstruction* instr) const; + inline int64_t rk(SimInstruction* instr) const; + inline uint64_t rk_u(SimInstruction* instr) const; + inline int32_t rd_reg(SimInstruction* instr) const; + inline int64_t rd(SimInstruction* instr) const; + inline uint64_t rd_u(SimInstruction* instr) const; + inline int32_t fa_reg(SimInstruction* instr) const; + inline float fa_float(SimInstruction* instr) const; + inline double fa_double(SimInstruction* instr) const; + + inline int32_t fj_reg(SimInstruction* instr) const; + inline float fj_float(SimInstruction* instr) const; + inline double fj_double(SimInstruction* instr) const; + + inline int32_t fk_reg(SimInstruction* instr) const; + inline float fk_float(SimInstruction* instr) const; + inline double fk_double(SimInstruction* instr) const; + inline int32_t fd_reg(SimInstruction* instr) const; + inline float fd_float(SimInstruction* instr) const; + inline double fd_double(SimInstruction* instr) const; + + inline int32_t cj_reg(SimInstruction* instr) const; + inline bool cj(SimInstruction* instr) const; + + inline int32_t cd_reg(SimInstruction* instr) const; + inline bool cd(SimInstruction* instr) const; + + inline int32_t ca_reg(SimInstruction* instr) const; + inline bool ca(SimInstruction* instr) const; + inline uint32_t sa2(SimInstruction* instr) const; + inline uint32_t sa3(SimInstruction* instr) const; + inline uint32_t ui5(SimInstruction* instr) const; + inline uint32_t ui6(SimInstruction* instr) const; + inline uint32_t lsbw(SimInstruction* instr) const; + inline uint32_t msbw(SimInstruction* instr) const; + inline uint32_t lsbd(SimInstruction* instr) const; + inline uint32_t msbd(SimInstruction* instr) const; + inline uint32_t cond(SimInstruction* instr) const; + inline int32_t si12(SimInstruction* instr) const; + inline uint32_t ui12(SimInstruction* instr) const; + inline int32_t si14(SimInstruction* instr) const; + inline int32_t si16(SimInstruction* instr) const; + inline int32_t si20(SimInstruction* instr) const; + + // Used for breakpoints. + void softwareInterrupt(SimInstruction* instr); + + // Stop helper functions. + bool isWatchpoint(uint32_t code); + void printWatchpoint(uint32_t code); + void handleStop(uint32_t code, SimInstruction* instr); + bool isStopInstruction(SimInstruction* instr); + bool isEnabledStop(uint32_t code); + void enableStop(uint32_t code); + void disableStop(uint32_t code); + void increaseStopCounter(uint32_t code); + void printStopInfo(uint32_t code); + + JS::ProfilingFrameIterator::RegisterState registerState(); + + // Handle any wasm faults, returning true if the fault was handled. + // This method is rather hot so inline the normal (no-wasm) case. + bool MOZ_ALWAYS_INLINE handleWasmSegFault(uint64_t addr, unsigned numBytes) { + if (MOZ_LIKELY(!js::wasm::CodeExists)) { + return false; + } + + uint8_t* newPC; + if (!js::wasm::MemoryAccessTraps(registerState(), (uint8_t*)addr, numBytes, + &newPC)) { + return false; + } + + LLBit_ = false; + set_pc(int64_t(newPC)); + return true; + } + + // Executes one instruction. + void instructionDecode(SimInstruction* instr); + + public: + static int64_t StopSimAt; + + // Runtime call support. + static void* RedirectNativeFunction(void* nativeFunction, + ABIFunctionType type); + + private: + enum Exception { + kNone, + kIntegerOverflow, + kIntegerUnderflow, + kDivideByZero, + kNumExceptions + }; + int16_t exceptions[kNumExceptions]; + + // Exceptions. + void signalExceptions(); + + // Handle return value for runtime FP functions. + void setCallResultDouble(double result); + void setCallResultFloat(float result); + void setCallResult(int64_t res); + void setCallResult(__int128 res); + + void callInternal(uint8_t* entry); + + // Architecture state. + // Registers. + int64_t registers_[kNumSimuRegisters]; + // Floating point Registers. + int64_t FPUregisters_[kNumFPURegisters]; + // Condition flags Registers. + bool CFregisters_[kNumCFRegisters]; + // FPU control register. + uint32_t FCSR_; + + bool LLBit_; + uintptr_t LLAddr_; + int64_t lastLLValue_; + + // Simulator support. + char* stack_; + uintptr_t stackLimit_; + bool pc_modified_; + int64_t icount_; + int64_t break_count_; + + // Debugger input. + char* lastDebuggerInput_; + + // Registered breakpoints. + SimInstruction* break_pc_; + Instr break_instr_; + + // Single-stepping support + bool single_stepping_; + SingleStepCallback single_step_callback_; + void* single_step_callback_arg_; + + // A stop is watched if its code is less than kNumOfWatchedStops. + // Only watched stops support enabling/disabling and the counter feature. + static const uint32_t kNumOfWatchedStops = 256; + + // Stop is disabled if bit 31 is set. + static const uint32_t kStopDisabledBit = 1U << 31; + + // A stop is enabled, meaning the simulator will stop when meeting the + // instruction, if bit 31 of watchedStops_[code].count is unset. + // The value watchedStops_[code].count & ~(1 << 31) indicates how many times + // the breakpoint was hit or gone through. + struct StopCountAndDesc { + uint32_t count_; + char* desc_; + }; + StopCountAndDesc watchedStops_[kNumOfWatchedStops]; +}; + +// Process wide simulator state. +class SimulatorProcess { + friend class Redirection; + friend class AutoLockSimulatorCache; + + private: + // ICache checking. + struct ICacheHasher { + typedef void* Key; + typedef void* Lookup; + static HashNumber hash(const Lookup& l); + static bool match(const Key& k, const Lookup& l); + }; + + public: + typedef HashMap ICacheMap; + + static mozilla::Atomic + ICacheCheckingDisableCount; + static void FlushICache(void* start, size_t size); + + static void checkICacheLocked(SimInstruction* instr); + + static bool initialize() { + singleton_ = js_new(); + return singleton_; + } + static void destroy() { + js_delete(singleton_); + singleton_ = nullptr; + } + + SimulatorProcess(); + ~SimulatorProcess(); + + private: + static SimulatorProcess* singleton_; + + // This lock creates a critical section around 'redirection_' and + // 'icache_', which are referenced both by the execution engine + // and by the off-thread compiler (see Redirection::Get in the cpp file). + Mutex cacheLock_; + + Redirection* redirection_; + ICacheMap icache_; + + public: + static ICacheMap& icache() { + // Technically we need the lock to access the innards of the + // icache, not to take its address, but the latter condition + // serves as a useful complement to the former. + singleton_->cacheLock_.assertOwnedByCurrentThread(); + return singleton_->icache_; + } + + static Redirection* redirection() { + singleton_->cacheLock_.assertOwnedByCurrentThread(); + return singleton_->redirection_; + } + + static void setRedirection(js::jit::Redirection* redirection) { + singleton_->cacheLock_.assertOwnedByCurrentThread(); + singleton_->redirection_ = redirection; + } +}; + +} // namespace jit +} // namespace js + +#endif /* JS_SIMULATOR_LOONG64 */ + +#endif /* jit_loong64_Simulator_loong64_h */ diff --git a/js/src/jit/loong64/Trampoline-loong64.cpp b/js/src/jit/loong64/Trampoline-loong64.cpp new file mode 100644 index 0000000000..02409d47a9 --- /dev/null +++ b/js/src/jit/loong64/Trampoline-loong64.cpp @@ -0,0 +1,1274 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/CalleeToken.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#ifdef JS_ION_PERF +# include "jit/PerfSpewer.h" +#endif +#include "jit/loong64/SharedICHelpers-loong64.h" +#include "jit/VMFunctions.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/JSContext.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +// All registers to save and restore. This includes the stack pointer, since we +// use the ability to reference register values on the stack by index. +static const LiveRegisterSet AllRegs = + LiveRegisterSet(GeneralRegisterSet(Registers::AllMask), + FloatRegisterSet(FloatRegisters::AllMask)); + +static_assert(sizeof(uintptr_t) == sizeof(uint64_t), "Not 32-bit clean."); + +struct EnterJITRegs { + double f31; + double f30; + double f29; + double f28; + double f27; + double f26; + double f25; + double f24; + + // uintptr_t align; + + // non-volatile registers. + uint64_t ra; + uint64_t fp; + uint64_t s8; + uint64_t s7; + uint64_t s6; + uint64_t s5; + uint64_t s4; + uint64_t s3; + uint64_t s2; + uint64_t s1; + uint64_t s0; + // Save reg_vp(a7) on stack, use it after call jit code. + uint64_t a7; +}; + +static void GenerateReturn(MacroAssembler& masm, int returnCode) { + MOZ_ASSERT(masm.framePushed() == sizeof(EnterJITRegs)); + + // Restore non-volatile registers + masm.as_ld_d(s0, StackPointer, offsetof(EnterJITRegs, s0)); + masm.as_ld_d(s1, StackPointer, offsetof(EnterJITRegs, s1)); + masm.as_ld_d(s2, StackPointer, offsetof(EnterJITRegs, s2)); + masm.as_ld_d(s3, StackPointer, offsetof(EnterJITRegs, s3)); + masm.as_ld_d(s4, StackPointer, offsetof(EnterJITRegs, s4)); + masm.as_ld_d(s5, StackPointer, offsetof(EnterJITRegs, s5)); + masm.as_ld_d(s6, StackPointer, offsetof(EnterJITRegs, s6)); + masm.as_ld_d(s7, StackPointer, offsetof(EnterJITRegs, s7)); + masm.as_ld_d(s8, StackPointer, offsetof(EnterJITRegs, s8)); + masm.as_ld_d(fp, StackPointer, offsetof(EnterJITRegs, fp)); + masm.as_ld_d(ra, StackPointer, offsetof(EnterJITRegs, ra)); + + // Restore non-volatile floating point registers + masm.as_fld_d(f24, StackPointer, offsetof(EnterJITRegs, f24)); + masm.as_fld_d(f25, StackPointer, offsetof(EnterJITRegs, f25)); + masm.as_fld_d(f26, StackPointer, offsetof(EnterJITRegs, f26)); + masm.as_fld_d(f27, StackPointer, offsetof(EnterJITRegs, f27)); + masm.as_fld_d(f28, StackPointer, offsetof(EnterJITRegs, f28)); + masm.as_fld_d(f29, StackPointer, offsetof(EnterJITRegs, f29)); + masm.as_fld_d(f30, StackPointer, offsetof(EnterJITRegs, f30)); + masm.as_fld_d(f31, StackPointer, offsetof(EnterJITRegs, f31)); + + masm.freeStack(sizeof(EnterJITRegs)); + + masm.branch(ra); +} + +static void GeneratePrologue(MacroAssembler& masm) { + masm.reserveStack(sizeof(EnterJITRegs)); + + masm.as_st_d(s0, StackPointer, offsetof(EnterJITRegs, s0)); + masm.as_st_d(s1, StackPointer, offsetof(EnterJITRegs, s1)); + masm.as_st_d(s2, StackPointer, offsetof(EnterJITRegs, s2)); + masm.as_st_d(s3, StackPointer, offsetof(EnterJITRegs, s3)); + masm.as_st_d(s4, StackPointer, offsetof(EnterJITRegs, s4)); + masm.as_st_d(s5, StackPointer, offsetof(EnterJITRegs, s5)); + masm.as_st_d(s6, StackPointer, offsetof(EnterJITRegs, s6)); + masm.as_st_d(s7, StackPointer, offsetof(EnterJITRegs, s7)); + masm.as_st_d(s8, StackPointer, offsetof(EnterJITRegs, s8)); + masm.as_st_d(fp, StackPointer, offsetof(EnterJITRegs, fp)); + masm.as_st_d(ra, StackPointer, offsetof(EnterJITRegs, ra)); + masm.as_st_d(a7, StackPointer, offsetof(EnterJITRegs, a7)); + + masm.as_fst_d(f24, StackPointer, offsetof(EnterJITRegs, f24)); + masm.as_fst_d(f25, StackPointer, offsetof(EnterJITRegs, f25)); + masm.as_fst_d(f26, StackPointer, offsetof(EnterJITRegs, f26)); + masm.as_fst_d(f27, StackPointer, offsetof(EnterJITRegs, f27)); + masm.as_fst_d(f28, StackPointer, offsetof(EnterJITRegs, f28)); + masm.as_fst_d(f29, StackPointer, offsetof(EnterJITRegs, f29)); + masm.as_fst_d(f30, StackPointer, offsetof(EnterJITRegs, f30)); + masm.as_fst_d(f31, StackPointer, offsetof(EnterJITRegs, f31)); +} + +// Generates a trampoline for calling Jit compiled code from a C++ function. +// The trampoline use the EnterJitCode signature, with the standard x64 fastcall +// calling convention. +void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) { + enterJITOffset_ = startTrampolineCode(masm); + + const Register reg_code = IntArgReg0; + const Register reg_argc = IntArgReg1; + const Register reg_argv = IntArgReg2; + const mozilla::DebugOnly reg_frame = IntArgReg3; + const Register reg_token = IntArgReg4; + const Register reg_chain = IntArgReg5; + const Register reg_values = IntArgReg6; + const Register reg_vp = IntArgReg7; + + MOZ_ASSERT(OsrFrameReg == reg_frame); + + GeneratePrologue(masm); + + // Save stack pointer into s4 + masm.movePtr(StackPointer, s4); + + // Save stack pointer as baseline frame. + masm.movePtr(StackPointer, BaselineFrameReg); + + // Load the number of actual arguments into s3. + masm.unboxInt32(Address(reg_vp, 0), s3); + + /*************************************************************** + Loop over argv vector, push arguments onto stack in reverse order + ***************************************************************/ + + // if we are constructing, that also needs to include newTarget + { + Label noNewTarget; + masm.branchTest32(Assembler::Zero, reg_token, + Imm32(CalleeToken_FunctionConstructing), &noNewTarget); + + masm.add32(Imm32(1), reg_argc); + + masm.bind(&noNewTarget); + } + + // Make stack algined + masm.ma_and(s0, reg_argc, Imm32(1)); + masm.ma_sub_d(s1, zero, Imm32(sizeof(Value))); + masm.as_maskeqz(s1, s1, s0); + masm.as_add_d(StackPointer, StackPointer, s1); + + masm.as_slli_d(s0, reg_argc, 3); // Value* argv + masm.addPtr(reg_argv, s0); // s0 = &argv[argc] + + // Loop over arguments, copying them from an unknown buffer onto the Ion + // stack so they can be accessed from JIT'ed code. + Label header, footer; + // If there aren't any arguments, don't do anything + masm.ma_b(s0, reg_argv, &footer, Assembler::BelowOrEqual, ShortJump); + { + masm.bind(&header); + + masm.subPtr(Imm32(sizeof(Value)), s0); + masm.subPtr(Imm32(sizeof(Value)), StackPointer); + + ValueOperand value = ValueOperand(s6); + masm.loadValue(Address(s0, 0), value); + masm.storeValue(value, Address(StackPointer, 0)); + + masm.ma_b(s0, reg_argv, &header, Assembler::Above, ShortJump); + } + masm.bind(&footer); + + // Create the frame descriptor. + masm.subPtr(StackPointer, s4); + masm.makeFrameDescriptor(s4, FrameType::CppToJSJit, JitFrameLayout::Size()); + + masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + masm.storePtr(s3, + Address(StackPointer, sizeof(uintptr_t))); // actual arguments + masm.storePtr(reg_token, Address(StackPointer, 0)); // callee token + + masm.push(s4); // descriptor + + CodeLabel returnLabel; + CodeLabel oomReturnLabel; + { + // Handle Interpreter -> Baseline OSR. + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + regs.take(OsrFrameReg); + regs.take(BaselineFrameReg); + regs.take(reg_code); + // regs.take(ReturnReg); + regs.take(JSReturnOperand); + + Label notOsr; + masm.ma_b(OsrFrameReg, OsrFrameReg, ¬Osr, Assembler::Zero, ShortJump); + + Register numStackValues = reg_values; + regs.take(numStackValues); + Register scratch = regs.takeAny(); + + // Push return address. + masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + masm.ma_li(scratch, &returnLabel); + masm.storePtr(scratch, Address(StackPointer, 0)); + + // Push previous frame pointer. + masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + masm.storePtr(BaselineFrameReg, Address(StackPointer, 0)); + + // Reserve frame. + Register framePtr = BaselineFrameReg; + masm.subPtr(Imm32(BaselineFrame::Size()), StackPointer); + masm.movePtr(StackPointer, framePtr); + + // Reserve space for locals and stack values. + masm.as_slli_d(scratch, numStackValues, 3); + masm.subPtr(scratch, StackPointer); + + // Enter exit frame. + masm.addPtr( + Imm32(BaselineFrame::Size() + BaselineFrame::FramePointerOffset), + scratch); + masm.makeFrameDescriptor(scratch, FrameType::BaselineJS, + ExitFrameLayout::Size()); + + // Push frame descriptor and fake return address. + masm.reserveStack(2 * sizeof(uintptr_t)); + masm.storePtr( + scratch, Address(StackPointer, sizeof(uintptr_t))); // Frame descriptor + masm.storePtr(zero, Address(StackPointer, 0)); // fake return address + + // No GC things to mark, push a bare token. + masm.loadJSContext(scratch); + masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare); + + masm.reserveStack(2 * sizeof(uintptr_t)); + masm.storePtr(framePtr, + Address(StackPointer, sizeof(uintptr_t))); // BaselineFrame + masm.storePtr(reg_code, Address(StackPointer, 0)); // jitcode + + using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame, + uint32_t numStackValues); + masm.setupUnalignedABICall(scratch); + masm.passABIArg(BaselineFrameReg); // BaselineFrame + masm.passABIArg(OsrFrameReg); // InterpreterFrame + masm.passABIArg(numStackValues); + masm.callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + regs.add(OsrFrameReg); + Register jitcode = regs.takeAny(); + masm.loadPtr(Address(StackPointer, 0), jitcode); + masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), framePtr); + masm.freeStack(2 * sizeof(uintptr_t)); + + Label error; + masm.freeStack(ExitFrameLayout::SizeWithFooter()); + masm.addPtr(Imm32(BaselineFrame::Size()), framePtr); + masm.branchIfFalseBool(ReturnReg, &error); + + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + Register realFramePtr = numStackValues; + AbsoluteAddress addressOfEnabled( + cx->runtime()->geckoProfiler().addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.ma_add_d(realFramePtr, framePtr, Imm32(sizeof(void*))); + masm.profilerEnterFrame(realFramePtr, scratch); + masm.bind(&skipProfilingInstrumentation); + } + + masm.jump(jitcode); + + // OOM: load error value, discard return address and previous frame + // pointer and return. + masm.bind(&error); + masm.movePtr(framePtr, StackPointer); + masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + masm.ma_li(scratch, &oomReturnLabel); + masm.jump(scratch); + + masm.bind(¬Osr); + // Load the scope chain in R1. + MOZ_ASSERT(R1.scratchReg() != reg_code); + masm.as_or(R1.scratchReg(), reg_chain, zero); + } + + // The call will push the return address on the stack, thus we check that + // the stack would be aligned once the call is complete. + masm.assertStackAlignment(JitStackAlignment, sizeof(uintptr_t)); + + // Call the function with pushing return address to stack. + masm.callJitNoProfiler(reg_code); + + { + // Interpreter -> Baseline OSR will return here. + masm.bind(&returnLabel); + masm.addCodeLabel(returnLabel); + masm.bind(&oomReturnLabel); + masm.addCodeLabel(oomReturnLabel); + } + + // s0 <- 8*argc (size of all arguments we pushed on the stack) + masm.pop(s0); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), s0); + + // Discard calleeToken, numActualArgs. + masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + + // Pop arguments off the stack. + masm.addPtr(s0, StackPointer); + + // Store the returned value into the vp + masm.as_ld_d(reg_vp, StackPointer, offsetof(EnterJITRegs, a7)); + masm.storeValue(JSReturnOperand, Address(reg_vp, 0)); + + // Restore non-volatile registers and return. + GenerateReturn(masm, ShortJump); +} + +// static +mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState> +JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) { + // Not supported, or not implemented yet. + // TODO: Implement along with the corresponding stack-walker changes, in + // coordination with the Gecko Profiler, see bug 1635987 and follow-ups. + return mozilla::Nothing{}; +} + +void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) { + invalidatorOffset_ = startTrampolineCode(masm); + + // Stack has to be alligned here. If not, we will have to fix it. + masm.checkStackAlignment(); + + // Push registers such that we can access them from [base + code]. + masm.PushRegsInMask(AllRegs); + + // Pass pointer to InvalidationBailoutStack structure. + masm.movePtr(StackPointer, a0); + + // Reserve place for return value and BailoutInfo pointer + masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + // Pass pointer to return value. + masm.ma_add_d(a1, StackPointer, Imm32(sizeof(uintptr_t))); + // Pass pointer to BailoutInfo + masm.movePtr(StackPointer, a2); + + using Fn = bool (*)(InvalidationBailoutStack * sp, size_t * frameSizeOut, + BaselineBailoutInfo * *info); + masm.setupAlignedABICall(); + masm.passABIArg(a0); + masm.passABIArg(a1); + masm.passABIArg(a2); + masm.callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther); + + masm.loadPtr(Address(StackPointer, 0), a2); + masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), a1); + // Remove the return address, the IonScript, the register state + // (InvaliationBailoutStack) and the space that was allocated for the + // return value. + masm.addPtr(Imm32(sizeof(InvalidationBailoutStack) + 2 * sizeof(uintptr_t)), + StackPointer); + // remove the space that this frame was using before the bailout + // (computed by InvalidationBailout) + masm.addPtr(a1, StackPointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2. + masm.jump(bailoutTail); +} + +void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm, + ArgumentsRectifierKind kind) { + // Do not erase the frame pointer in this function. + + switch (kind) { + case ArgumentsRectifierKind::Normal: + argumentsRectifierOffset_ = startTrampolineCode(masm); + break; + case ArgumentsRectifierKind::TrialInlining: + trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm); + break; + } + masm.pushReturnAddress(); + // Caller: + // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- sp + + // Add |this|, in the counter of known arguments. + masm.loadPtr( + Address(StackPointer, RectifierFrameLayout::offsetOfNumActualArgs()), s3); + masm.addPtr(Imm32(1), s3); + + Register numActArgsReg = a6; + Register calleeTokenReg = a7; + Register numArgsReg = a5; + + // Load |nformals| into numArgsReg. + masm.loadPtr( + Address(StackPointer, RectifierFrameLayout::offsetOfCalleeToken()), + calleeTokenReg); + masm.mov(calleeTokenReg, numArgsReg); + masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), numArgsReg); + masm.load16ZeroExtend(Address(numArgsReg, JSFunction::offsetOfNargs()), + numArgsReg); + + // Stash another copy in t3, since we are going to do destructive operations + // on numArgsReg + masm.mov(numArgsReg, t3); + + static_assert( + CalleeToken_FunctionConstructing == 1, + "Ensure that we can use the constructing bit to count the value"); + masm.mov(calleeTokenReg, t2); + masm.ma_and(t2, t2, Imm32(uint32_t(CalleeToken_FunctionConstructing))); + + // Including |this|, and |new.target|, there are (|nformals| + 1 + + // isConstructing) arguments to push to the stack. Then we push a + // JitFrameLayout. We compute the padding expressed in the number of extra + // |undefined| values to push on the stack. + static_assert( + sizeof(JitFrameLayout) % JitStackAlignment == 0, + "No need to consider the JitFrameLayout for aligning the stack"); + static_assert( + JitStackAlignment % sizeof(Value) == 0, + "Ensure that we can pad the stack by pushing extra UndefinedValue"); + + MOZ_ASSERT(mozilla::IsPowerOfTwo(JitStackValueAlignment)); + masm.add32( + Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */), + numArgsReg); + masm.add32(t2, numArgsReg); + masm.and32(Imm32(~(JitStackValueAlignment - 1)), numArgsReg); + + // Load the number of |undefined|s to push into t1. + masm.as_sub_d(t1, numArgsReg, s3); + + // Caller: + // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- sp <- t2 + // '------ s3 -------' + // + // Rectifier frame: + // [undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee] + // [descr] [raddr] ] + // '-------- t1 ---------' '------- s3 -------' + + // Copy number of actual arguments into numActArgsReg + masm.loadPtr( + Address(StackPointer, RectifierFrameLayout::offsetOfNumActualArgs()), + numActArgsReg); + + masm.moveValue(UndefinedValue(), ValueOperand(t0)); + + masm.movePtr(StackPointer, t2); // Save %sp. + + // Push undefined. (including the padding) + { + Label undefLoopTop; + + masm.bind(&undefLoopTop); + masm.sub32(Imm32(1), t1); + masm.subPtr(Imm32(sizeof(Value)), StackPointer); + masm.storeValue(ValueOperand(t0), Address(StackPointer, 0)); + + masm.ma_b(t1, t1, &undefLoopTop, Assembler::NonZero, ShortJump); + } + + // Get the topmost argument. + static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments"); + + // | - sizeof(Value)| is used to put rcx such that we can read the last + // argument, and not the value which is after. + masm.as_slli_d(t0, s3, 3); // t0 <- nargs * 8 + masm.as_add_d(t1, t2, t0); // t1 <- t2(saved sp) + nargs * 8 + masm.addPtr(Imm32(sizeof(RectifierFrameLayout) - sizeof(Value)), t1); + + // Copy & Push arguments, |nargs| + 1 times (to include |this|). + { + Label copyLoopTop; + + masm.bind(©LoopTop); + masm.sub32(Imm32(1), s3); + masm.subPtr(Imm32(sizeof(Value)), StackPointer); + masm.loadValue(Address(t1, 0), ValueOperand(t0)); + masm.storeValue(ValueOperand(t0), Address(StackPointer, 0)); + masm.subPtr(Imm32(sizeof(Value)), t1); + + masm.ma_b(s3, s3, ©LoopTop, Assembler::NonZero, ShortJump); + } + + // if constructing, copy newTarget + { + Label notConstructing; + + masm.branchTest32(Assembler::Zero, calleeTokenReg, + Imm32(CalleeToken_FunctionConstructing), + ¬Constructing); + + // thisFrame[numFormals] = prevFrame[argc] + ValueOperand newTarget(t0); + + // +1 for |this|. We want vp[argc], so don't subtract 1 + BaseIndex newTargetSrc(t2, numActArgsReg, TimesEight, + sizeof(RectifierFrameLayout) + sizeof(Value)); + masm.loadValue(newTargetSrc, newTarget); + + // Again, 1 for |this| + BaseIndex newTargetDest(StackPointer, t3, TimesEight, sizeof(Value)); + masm.storeValue(newTarget, newTargetDest); + + masm.bind(¬Constructing); + } + + // Caller: + // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- t2 + // + // + // Rectifier frame: + // [undef] [undef] [undef] [arg2] [arg1] [this] <- sp [ [argc] [callee] + // [descr] [raddr] ] + + // Construct sizeDescriptor. + masm.subPtr(StackPointer, t2); + masm.makeFrameDescriptor(t2, FrameType::Rectifier, JitFrameLayout::Size()); + + // Construct JitFrameLayout. + masm.subPtr(Imm32(3 * sizeof(uintptr_t)), StackPointer); + // Push actual arguments. + masm.storePtr(numActArgsReg, Address(StackPointer, 2 * sizeof(uintptr_t))); + // Push callee token. + masm.storePtr(calleeTokenReg, Address(StackPointer, sizeof(uintptr_t))); + // Push frame descriptor. + masm.storePtr(t2, Address(StackPointer, 0)); + + // Call the target function. + masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), calleeTokenReg); + switch (kind) { + case ArgumentsRectifierKind::Normal: + masm.loadJitCodeRaw(calleeTokenReg, t1); + argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(t1); + break; + case ArgumentsRectifierKind::TrialInlining: + Label noBaselineScript, done; + masm.loadBaselineJitCodeRaw(calleeTokenReg, t1, &noBaselineScript); + masm.callJitNoProfiler(t1); + masm.jump(&done); + + // See BaselineCacheIRCompiler::emitCallInlinedFunction. + masm.bind(&noBaselineScript); + masm.loadJitCodeRaw(calleeTokenReg, t1); + masm.callJitNoProfiler(t1); + masm.bind(&done); + break; + } + + // Remove the rectifier frame. + // t2 <- descriptor with FrameType. + masm.loadPtr(Address(StackPointer, 0), t2); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), t2); // t2 <- descriptor. + + // Discard descriptor, calleeToken and number of actual arguments. + masm.addPtr(Imm32(3 * sizeof(uintptr_t)), StackPointer); + + // Discard pushed arguments. + masm.addPtr(t2, StackPointer); + + masm.ret(); +} + +/* - When bailout is done via out of line code (lazy bailout). + * Frame size is stored in $ra (look at + * CodeGeneratorLOONG64::generateOutOfLineCode()) and thunk code should save it + * on stack. Other difference is that members snapshotOffset_ and padding_ are + * pushed to the stack by CodeGeneratorLOONG64::visitOutOfLineBailout(). Field + * frameClassId_ is forced to be NO_FRAME_SIZE_CLASS_ID + * (See: JitRuntime::generateBailoutHandler). + */ +static void PushBailoutFrame(MacroAssembler& masm, Register spArg) { + // Push the frameSize_ stored in ra + // See: CodeGeneratorLOONG64::generateOutOfLineCode() + masm.push(ra); + + // Push registers such that we can access them from [base + code]. + masm.PushRegsInMask(AllRegs); + + // Put pointer to BailoutStack as first argument to the Bailout() + masm.movePtr(StackPointer, spArg); +} + +static void GenerateBailoutThunk(MacroAssembler& masm, uint32_t frameClass, + Label* bailoutTail) { + PushBailoutFrame(masm, a0); + + // Make space for Bailout's bailoutInfo outparam. + masm.reserveStack(sizeof(void*)); + masm.movePtr(StackPointer, a1); + + // Call the bailout function. + using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info); + masm.setupUnalignedABICall(a2); + masm.passABIArg(a0); + masm.passABIArg(a1); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + + // Get the bailoutInfo outparam. + masm.pop(a2); + + // Stack is: + // [frame] + // snapshotOffset + // frameSize + // [bailoutFrame] + // + // We want to remove both the bailout frame and the topmost Ion frame's stack. + + // Remove the bailoutFrame. + static const uint32_t BailoutDataSize = sizeof(RegisterDump); + masm.addPtr(Imm32(BailoutDataSize), StackPointer); + + // Remove the frame, snapshotOffset, and frameSize. + masm.ma_ld_d(a1, Address(StackPointer, 0)); + masm.addPtr(Imm32(2 * sizeof(void*)), a1); + masm.addPtr(a1, StackPointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in a2. + masm.jump(bailoutTail); +} + +JitRuntime::BailoutTable JitRuntime::generateBailoutTable(MacroAssembler& masm, + Label* bailoutTail, + uint32_t frameClass) { + MOZ_CRASH("loong64 does not use bailout tables"); +} + +void JitRuntime::generateBailoutHandler(MacroAssembler& masm, + Label* bailoutTail) { + bailoutHandlerOffset_ = startTrampolineCode(masm); + + GenerateBailoutThunk(masm, NO_FRAME_SIZE_CLASS_ID, bailoutTail); +} + +bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm, + const VMFunctionData& f, DynFn nativeFun, + uint32_t* wrapperOffset) { + *wrapperOffset = startTrampolineCode(masm); + + // Avoid conflicts with argument registers while discarding the result after + // the function call. + AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask); + + static_assert( + (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0, + "Wrapper register set should be a superset of Volatile register set."); + + // The context is the first argument; a0 is the first argument register. + Register cxreg = a0; + regs.take(cxreg); + + // If it isn't a tail call, then the return address needs to be saved + if (f.expectTailCall == NonTailCall) { + masm.pushReturnAddress(); + } + + // We're aligned to an exit frame, so link it up. + masm.loadJSContext(cxreg); + masm.enterExitFrame(cxreg, regs.getAny(), &f); + + // Save the base of the argument set stored on the stack. + Register argsBase = InvalidReg; + if (f.explicitArgs) { + argsBase = t1; // Use temporary register. + regs.take(argsBase); + masm.ma_add_d(argsBase, StackPointer, + Imm32(ExitFrameLayout::SizeWithFooter())); + } + + // Reserve space for the outparameter. + Register outReg = InvalidReg; + switch (f.outParam) { + case Type_Value: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(Value)); + masm.movePtr(StackPointer, outReg); + break; + + case Type_Handle: + outReg = regs.takeAny(); + masm.PushEmptyRooted(f.outParamRootType); + masm.movePtr(StackPointer, outReg); + break; + + case Type_Bool: + case Type_Int32: + outReg = regs.takeAny(); + // Reserve 4-byte space to make stack aligned to 8-byte. + masm.reserveStack(2 * sizeof(int32_t)); + masm.movePtr(StackPointer, outReg); + break; + + case Type_Pointer: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(uintptr_t)); + masm.movePtr(StackPointer, outReg); + break; + + case Type_Double: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(double)); + masm.movePtr(StackPointer, outReg); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + if (!generateTLEnterVM(masm, f)) { + return false; + } + + masm.setupUnalignedABICall(regs.getAny()); + masm.passABIArg(cxreg); + + size_t argDisp = 0; + + // Copy any arguments. + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) { + switch (f.argProperties(explicitArg)) { + case VMFunctionData::WordByValue: + if (f.argPassedInFloatReg(explicitArg)) { + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE); + } else { + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL); + } + argDisp += sizeof(void*); + break; + case VMFunctionData::WordByRef: + masm.passABIArg( + MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS), + MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunctionData::DoubleByValue: + case VMFunctionData::DoubleByRef: + MOZ_CRASH( + "NYI: LOONG64 callVM should not be used with 128bits values."); + break; + } + } + + // Copy the implicit outparam, if any. + if (InvalidReg != outReg) { + masm.passABIArg(outReg); + } + + masm.callWithABI(nativeFun, MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + if (!generateTLExitVM(masm, f)) { + return false; + } + + // Test for failure. + switch (f.failType()) { + case Type_Object: + masm.branchTestPtr(Assembler::Zero, a0, a0, masm.failureLabel()); + break; + case Type_Bool: + // Called functions return bools, which are 0/false and non-zero/true + masm.branchIfFalseBool(a0, masm.failureLabel()); + break; + case Type_Void: + break; + default: + MOZ_CRASH("unknown failure kind"); + } + + // Load the outparam and free any allocated stack. + switch (f.outParam) { + case Type_Handle: + masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand); + break; + + case Type_Value: + masm.loadValue(Address(StackPointer, 0), JSReturnOperand); + masm.freeStack(sizeof(Value)); + break; + + case Type_Int32: + masm.load32(Address(StackPointer, 0), ReturnReg); + masm.freeStack(2 * sizeof(int32_t)); + break; + + case Type_Pointer: + masm.loadPtr(Address(StackPointer, 0), ReturnReg); + masm.freeStack(sizeof(uintptr_t)); + break; + + case Type_Bool: + masm.load8ZeroExtend(Address(StackPointer, 0), ReturnReg); + masm.freeStack(2 * sizeof(int32_t)); + break; + + case Type_Double: + if (JitOptions.supportsFloatingPoint) { + masm.as_fld_d(ReturnDoubleReg, StackPointer, 0); + } else { + masm.assumeUnreachable( + "Unable to load into float reg, with no FP support."); + } + masm.freeStack(sizeof(double)); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + masm.leaveExitFrame(); + masm.retn(Imm32(sizeof(ExitFrameLayout) + + f.explicitStackSlots() * sizeof(void*) + + f.extraValuesToPop * sizeof(Value))); + + return true; +} + +uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm, + MIRType type) { + uint32_t offset = startTrampolineCode(masm); + + MOZ_ASSERT(PreBarrierReg == a1); + Register temp1 = a0; + Register temp2 = a2; + Register temp3 = a3; + masm.push(temp1); + masm.push(temp2); + masm.push(temp3); + + Label noBarrier; + masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3, + &noBarrier); + + // Call into C++ to mark this GC thing. + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + + LiveRegisterSet save; + if (JitOptions.supportsFloatingPoint) { + save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask), + FloatRegisterSet(FloatRegisters::VolatileMask)); + } else { + save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask), + FloatRegisterSet()); + } + // LA not same as mips64. Because ra register number is different. + masm.push(ra); + masm.PushRegsInMask(save); + + masm.movePtr(ImmPtr(cx->runtime()), a0); + + masm.setupUnalignedABICall(a2); + masm.passABIArg(a0); + masm.passABIArg(a1); + masm.callWithABI(JitPreWriteBarrier(type)); + + masm.PopRegsInMask(save); + masm.ret(); + + masm.bind(&noBarrier); + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + masm.abiret(); + + return offset; +} + +void JitRuntime::generateExceptionTailStub(MacroAssembler& masm, + Label* profilerExitTail) { + exceptionTailOffset_ = startTrampolineCode(masm); + + masm.bind(masm.failureLabel()); + masm.handleFailureWithHandlerTail(profilerExitTail); +} + +void JitRuntime::generateBailoutTailStub(MacroAssembler& masm, + Label* bailoutTail) { + bailoutTailOffset_ = startTrampolineCode(masm); + masm.bind(bailoutTail); + + masm.generateBailoutTail(a1, a2); +} + +void JitRuntime::generateProfilerExitFrameTailStub(MacroAssembler& masm, + Label* profilerExitTail) { + profilerExitFrameTailOffset_ = startTrampolineCode(masm); + masm.bind(profilerExitTail); + + Register scratch1 = t0; + Register scratch2 = t1; + Register scratch3 = t2; + Register scratch4 = t3; + + // + // The code generated below expects that the current stack pointer points + // to an Ion or Baseline frame, at the state it would be immediately + // before a ret(). Thus, after this stub's business is done, it executes + // a ret() and returns directly to the caller script, on behalf of the + // callee script that jumped to this code. + // + // Thus the expected stack is: + // + // StackPointer ----+ + // v + // ..., ActualArgc, CalleeToken, Descriptor, ReturnAddr + // MEM-HI MEM-LOW + // + // + // The generated jitcode is responsible for overwriting the + // jitActivation->lastProfilingFrame field with a pointer to the previous + // Ion or Baseline jit-frame that was pushed before this one. It is also + // responsible for overwriting jitActivation->lastProfilingCallSite with + // the return address into that frame. The frame could either be an + // immediate "caller" frame, or it could be a frame in a previous + // JitActivation (if the current frame was entered from C++, and the C++ + // was entered by some caller jit-frame further down the stack). + // + // So this jitcode is responsible for "walking up" the jit stack, finding + // the previous Ion or Baseline JS frame, and storing its address and the + // return address into the appropriate fields on the current jitActivation. + // + // There are a fixed number of different path types that can lead to the + // current frame, which is either a baseline or ion frame: + // + // + // ^ + // | + // ^--- Ion + // | + // ^--- Baseline Stub <---- Baseline + // | + // ^--- Argument Rectifier + // | ^ + // | | + // | ^--- Ion + // | | + // | ^--- Baseline Stub <---- Baseline + // | + // ^--- Entry Frame (From C++) + // + Register actReg = scratch4; + masm.loadJSContext(actReg); + masm.loadPtr(Address(actReg, offsetof(JSContext, profilingActivation_)), + actReg); + + Address lastProfilingFrame(actReg, + JitActivation::offsetOfLastProfilingFrame()); + Address lastProfilingCallSite(actReg, + JitActivation::offsetOfLastProfilingCallSite()); + +#ifdef DEBUG + // Ensure that frame we are exiting is current lastProfilingFrame + { + masm.loadPtr(lastProfilingFrame, scratch1); + Label checkOk; + masm.branchPtr(Assembler::Equal, scratch1, ImmWord(0), &checkOk); + masm.branchPtr(Assembler::Equal, StackPointer, scratch1, &checkOk); + masm.assumeUnreachable( + "Mismatch between stored lastProfilingFrame and current stack " + "pointer."); + masm.bind(&checkOk); + } +#endif + + // Load the frame descriptor into |scratch1|, figure out what to do depending + // on its type. + masm.loadPtr(Address(StackPointer, JitFrameLayout::offsetOfDescriptor()), + scratch1); + + // Going into the conditionals, we will have: + // FrameDescriptor.size in scratch1 + // FrameDescriptor.type in scratch2 + masm.ma_and(scratch2, scratch1, Imm32((1 << FRAMETYPE_BITS) - 1)); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch1); + + // Handling of each case is dependent on FrameDescriptor.type + Label handle_IonJS; + Label handle_BaselineStub; + Label handle_Rectifier; + Label handle_IonICCall; + Label handle_Entry; + Label end; + + masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::IonJS), + &handle_IonJS); + masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::BaselineJS), + &handle_IonJS); + masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::BaselineStub), + &handle_BaselineStub); + masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::Rectifier), + &handle_Rectifier); + masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::IonICCall), + &handle_IonICCall); + masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::CppToJSJit), + &handle_Entry); + + // The WasmToJSJit is just another kind of entry. + masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::WasmToJSJit), + &handle_Entry); + + masm.assumeUnreachable( + "Invalid caller frame type when exiting from Ion frame."); + + // + // FrameType::IonJS + // + // Stack layout: + // ... + // Ion-Descriptor + // Prev-FP ---> Ion-ReturnAddr + // ... previous frame data ... |- Descriptor.Size + // ... arguments ... | + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + masm.bind(&handle_IonJS); + { + // |scratch1| contains Descriptor.size + + // returning directly to an IonJS frame. Store return addr to frame + // in lastProfilingCallSite. + masm.loadPtr(Address(StackPointer, JitFrameLayout::offsetOfReturnAddress()), + scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + // Store return frame in lastProfilingFrame. + // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size(); + masm.as_add_d(scratch2, StackPointer, scratch1); + masm.ma_add_d(scratch2, scratch2, Imm32(JitFrameLayout::Size())); + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + // + // FrameType::BaselineStub + // + // Look past the stub and store the frame pointer to + // the baselineJS frame prior to it. + // + // Stack layout: + // ... + // BL-Descriptor + // Prev-FP ---> BL-ReturnAddr + // +-----> BL-PrevFramePointer + // | ... BL-FrameData ... + // | BLStub-Descriptor + // | BLStub-ReturnAddr + // | BLStub-StubPointer | + // +------ BLStub-SavedFramePointer |- Descriptor.Size + // ... arguments ... | + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + // We take advantage of the fact that the stub frame saves the frame + // pointer pointing to the baseline frame, so a bunch of calculation can + // be avoided. + // + masm.bind(&handle_BaselineStub); + { + masm.as_add_d(scratch3, StackPointer, scratch1); + Address stubFrameReturnAddr( + scratch3, JitFrameLayout::Size() + + BaselineStubFrameLayout::offsetOfReturnAddress()); + masm.loadPtr(stubFrameReturnAddr, scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + Address stubFrameSavedFramePtr( + scratch3, JitFrameLayout::Size() - (2 * sizeof(void*))); + masm.loadPtr(stubFrameSavedFramePtr, scratch2); + masm.addPtr(Imm32(sizeof(void*)), scratch2); // Skip past BL-PrevFramePtr + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + // + // FrameType::Rectifier + // + // The rectifier frame can be preceded by either an IonJS, a BaselineStub, + // or a CppToJSJit/WasmToJSJit frame. + // + // Stack layout if caller of rectifier was Ion or CppToJSJit/WasmToJSJit: + // + // Ion-Descriptor + // Ion-ReturnAddr + // ... ion frame data ... |- Rect-Descriptor.Size + // < COMMON LAYOUT > + // + // Stack layout if caller of rectifier was Baseline: + // + // BL-Descriptor + // Prev-FP ---> BL-ReturnAddr + // +-----> BL-SavedFramePointer + // | ... baseline frame data ... + // | BLStub-Descriptor + // | BLStub-ReturnAddr + // | BLStub-StubPointer | + // +------ BLStub-SavedFramePointer |- Rect-Descriptor.Size + // ... args to rectifier ... | + // < COMMON LAYOUT > + // + // Common stack layout: + // + // ActualArgc | + // CalleeToken |- IonRectitiferFrameLayout::Size() + // Rect-Descriptor | + // Rect-ReturnAddr | + // ... rectifier data & args ... |- Descriptor.Size + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + masm.bind(&handle_Rectifier); + { + // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size(); + masm.as_add_d(scratch2, StackPointer, scratch1); + masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2); + masm.loadPtr(Address(scratch2, RectifierFrameLayout::offsetOfDescriptor()), + scratch3); + masm.as_srli_d(scratch1, scratch3, FRAMESIZE_SHIFT); + masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch3); + + // Now |scratch1| contains Rect-Descriptor.Size + // and |scratch2| points to Rectifier frame + // and |scratch3| contains Rect-Descriptor.Type + + masm.assertRectifierFrameParentType(scratch3); + + // Check for either Ion or BaselineStub frame. + Label notIonFrame; + masm.branch32(Assembler::NotEqual, scratch3, Imm32(FrameType::IonJS), + ¬IonFrame); + + // Handle Rectifier <- IonJS + // scratch3 := RectFrame[ReturnAddr] + masm.loadPtr( + Address(scratch2, RectifierFrameLayout::offsetOfReturnAddress()), + scratch3); + masm.storePtr(scratch3, lastProfilingCallSite); + + // scratch3 := RectFrame + Rect-Descriptor.Size + + // RectifierFrameLayout::Size() + masm.as_add_d(scratch3, scratch2, scratch1); + masm.addPtr(Imm32(RectifierFrameLayout::Size()), scratch3); + masm.storePtr(scratch3, lastProfilingFrame); + masm.ret(); + + masm.bind(¬IonFrame); + + // Check for either BaselineStub or a CppToJSJit/WasmToJSJit entry + // frame. + masm.branch32(Assembler::NotEqual, scratch3, Imm32(FrameType::BaselineStub), + &handle_Entry); + + // Handle Rectifier <- BaselineStub <- BaselineJS + masm.as_add_d(scratch3, scratch2, scratch1); + Address stubFrameReturnAddr( + scratch3, RectifierFrameLayout::Size() + + BaselineStubFrameLayout::offsetOfReturnAddress()); + masm.loadPtr(stubFrameReturnAddr, scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + Address stubFrameSavedFramePtr( + scratch3, RectifierFrameLayout::Size() - (2 * sizeof(void*))); + masm.loadPtr(stubFrameSavedFramePtr, scratch2); + masm.addPtr(Imm32(sizeof(void*)), scratch2); + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + // FrameType::IonICCall + // + // The caller is always an IonJS frame. + // + // Ion-Descriptor + // Ion-ReturnAddr + // ... ion frame data ... |- CallFrame-Descriptor.Size + // StubCode | + // ICCallFrame-Descriptor |- IonICCallFrameLayout::Size() + // ICCallFrame-ReturnAddr | + // ... call frame data & args ... |- Descriptor.Size + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + masm.bind(&handle_IonICCall); + { + // scratch2 := StackPointer + Descriptor.size + JitFrameLayout::Size() + masm.as_add_d(scratch2, StackPointer, scratch1); + masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2); + + // scratch3 := ICCallFrame-Descriptor.Size + masm.loadPtr(Address(scratch2, IonICCallFrameLayout::offsetOfDescriptor()), + scratch3); +#ifdef DEBUG + // Assert previous frame is an IonJS frame. + masm.movePtr(scratch3, scratch1); + masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch1); + { + Label checkOk; + masm.branch32(Assembler::Equal, scratch1, Imm32(FrameType::IonJS), + &checkOk); + masm.assumeUnreachable("IonICCall frame must be preceded by IonJS frame"); + masm.bind(&checkOk); + } +#endif + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch3); + + // lastProfilingCallSite := ICCallFrame-ReturnAddr + masm.loadPtr( + Address(scratch2, IonICCallFrameLayout::offsetOfReturnAddress()), + scratch1); + masm.storePtr(scratch1, lastProfilingCallSite); + + // lastProfilingFrame := ICCallFrame + ICCallFrame-Descriptor.Size + + // IonICCallFrameLayout::Size() + masm.as_add_d(scratch1, scratch2, scratch3); + masm.addPtr(Imm32(IonICCallFrameLayout::Size()), scratch1); + masm.storePtr(scratch1, lastProfilingFrame); + masm.ret(); + } + + // + // FrameType::CppToJSJit / FrameType::WasmToJSJit + // + // If at an entry frame, store null into both fields. + // A fast-path wasm->jit transition frame is an entry frame from the point + // of view of the JIT. + // + masm.bind(&handle_Entry); + { + masm.movePtr(ImmPtr(nullptr), scratch1); + masm.storePtr(scratch1, lastProfilingCallSite); + masm.storePtr(scratch1, lastProfilingFrame); + masm.ret(); + } +} diff --git a/js/src/jit/moz.build b/js/src/jit/moz.build index f50d86fc44..125a77dbce 100644 --- a/js/src/jit/moz.build +++ b/js/src/jit/moz.build @@ -232,6 +232,20 @@ elif CONFIG["JS_CODEGEN_MIPS32"] or CONFIG["JS_CODEGEN_MIPS64"]: ] if CONFIG["JS_SIMULATOR_MIPS64"]: UNIFIED_SOURCES += ["mips64/Simulator-mips64.cpp"] +elif CONFIG["JS_CODEGEN_LOONG64"]: + lir_inputs += ["loong64/LIR-loong64.h"] + UNIFIED_SOURCES += [ + "loong64/Architecture-loong64.cpp", + "loong64/Assembler-loong64.cpp", + "loong64/Bailouts-loong64.cpp", + "loong64/CodeGenerator-loong64.cpp", + "loong64/Lowering-loong64.cpp", + "loong64/MacroAssembler-loong64.cpp", + "loong64/MoveEmitter-loong64.cpp", + "loong64/Trampoline-loong64.cpp", + ] + if CONFIG["JS_SIMULATOR_LOONG64"]: + UNIFIED_SOURCES += ["loong64/Simulator-loong64.cpp"] # Generate jit/MIROpsGenerated.h from jit/MIROps.yaml diff --git a/js/src/jit/shared/Assembler-shared.h b/js/src/jit/shared/Assembler-shared.h index dfb2bcb6b8..0fb29f9d56 100644 --- a/js/src/jit/shared/Assembler-shared.h +++ b/js/src/jit/shared/Assembler-shared.h @@ -24,14 +24,15 @@ #include "vm/NativeObject.h" #include "wasm/WasmTypes.h" -#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) // Push return addresses callee-side. # define JS_USE_LINK_REGISTER #endif #if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ - defined(JS_CODEGEN_ARM64) + defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_LOONG64) // JS_CODELABEL_LINKMODE gives labels additional metadata // describing how Bind() should patch them. # define JS_CODELABEL_LINKMODE diff --git a/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h b/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h index e063d9f971..97ab8c45cd 100644 --- a/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h +++ b/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h @@ -50,7 +50,7 @@ // code in this file. #if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \ - defined(JS_SIMULATOR_MIPS64) + defined(JS_SIMULATOR_MIPS64) || defined(JS_SIMULATOR_LOONG64) // On some x86 (32-bit) systems this will not work because the compiler does not // open-code 64-bit atomics. If so, try linking with -latomic. If that doesn't // work, you're mostly on your own. @@ -69,6 +69,11 @@ # define HAS_64BIT_LOCKFREE #endif +#if defined(__loongarch64) +# define HAS_64BIT_ATOMICS +# define HAS_64BIT_LOCKFREE +#endif + #ifdef __sparc__ # ifdef __LP64__ # define HAS_64BIT_ATOMICS diff --git a/js/src/jit/shared/CodeGenerator-shared.cpp b/js/src/jit/shared/CodeGenerator-shared.cpp index 0d825fbbbd..d888f0fe82 100644 --- a/js/src/jit/shared/CodeGenerator-shared.cpp +++ b/js/src/jit/shared/CodeGenerator-shared.cpp @@ -998,8 +998,10 @@ Label* CodeGeneratorShared::getJumpLabelForBranch(MBasicBlock* block) { return skipTrivialBlocks(block)->lir()->label(); } -// This function is not used for MIPS/MIPS64. MIPS has branchToBlock. -#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) +// This function is not used for MIPS/MIPS64/LOONG64. They have +// branchToBlock. +#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) && \ + !defined(JS_CODEGEN_LOONG64) void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir, Assembler::Condition cond) { // Skip past trivial blocks. diff --git a/js/src/jit/shared/CodeGenerator-shared.h b/js/src/jit/shared/CodeGenerator-shared.h index 41cc7842f8..ef600cc5c4 100644 --- a/js/src/jit/shared/CodeGenerator-shared.h +++ b/js/src/jit/shared/CodeGenerator-shared.h @@ -419,6 +419,7 @@ class CodeGeneratorShared : public LElementVisitor { // This function is not used for MIPS. MIPS has branchToBlock. #if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) + // TODO:LOONG64 Check jumpToBlock. void jumpToBlock(MBasicBlock* mir, Assembler::Condition cond); #endif diff --git a/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h b/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h index d21bb7f35b..24b959b8d6 100644 --- a/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h +++ b/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h @@ -895,8 +895,9 @@ struct AssemblerBufferWithConstantPools return allocEntry(1, 0, (uint8_t*)&value, nullptr, nullptr); } -#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) return this->putU32Aligned(value); #else return this->AssemblerBuffer::putInt(value); diff --git a/js/src/jit/shared/Lowering-shared-inl.h b/js/src/jit/shared/Lowering-shared-inl.h index c30fa73cf7..752679a150 100644 --- a/js/src/jit/shared/Lowering-shared-inl.h +++ b/js/src/jit/shared/Lowering-shared-inl.h @@ -516,7 +516,8 @@ LAllocation LIRGeneratorShared::useRegisterOrNonDoubleConstant( return useRegister(mir); } -#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_LOONG64) LAllocation LIRGeneratorShared::useAnyOrConstant(MDefinition* mir) { return useRegisterOrConstant(mir); } diff --git a/js/src/jsapi-tests/testJitABIcalls.cpp b/js/src/jsapi-tests/testJitABIcalls.cpp index 02b67da3ca..a731dc2288 100644 --- a/js/src/jsapi-tests/testJitABIcalls.cpp +++ b/js/src/jsapi-tests/testJitABIcalls.cpp @@ -658,6 +658,9 @@ class JitABICall final : public JSAPITest, public DefineCheckArgs { #elif defined(JS_CODEGEN_MIPS64) Register base = t1; regs.take(base); +#elif defined(JS_CODEGEN_LOONG64) + Register base = t0; + regs.take(base); #else # error "Unknown architecture!" #endif diff --git a/js/src/jsapi-tests/testsJit.cpp b/js/src/jsapi-tests/testsJit.cpp index 069eef43fe..71591fdb9b 100644 --- a/js/src/jsapi-tests/testsJit.cpp +++ b/js/src/jsapi-tests/testsJit.cpp @@ -23,7 +23,7 @@ void PrepareJit(js::jit::MacroAssembler& masm) { #if defined(JS_CODEGEN_ARM) save.add(js::jit::d15); #endif -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) save.add(js::jit::ra); #elif defined(JS_USE_LINK_REGISTER) save.add(js::jit::lr); @@ -40,7 +40,7 @@ bool ExecuteJit(JSContext* cx, js::jit::MacroAssembler& masm) { #if defined(JS_CODEGEN_ARM) save.add(js::jit::d15); #endif -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) save.add(js::jit::ra); #elif defined(JS_USE_LINK_REGISTER) save.add(js::jit::lr); diff --git a/js/src/old-configure.in b/js/src/old-configure.in index 77652f6752..e5503977f0 100644 --- a/js/src/old-configure.in +++ b/js/src/old-configure.in @@ -333,9 +333,9 @@ if test "$GNU_CC"; then AC_MSG_RESULT([no]) LDFLAGS=$_SAVE_LDFLAGS) - AC_MSG_CHECKING([for -z text option to ld]) + AC_MSG_CHECKING([for -z notext option to ld]) _SAVE_LDFLAGS=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-z,text" + LDFLAGS="$LDFLAGS -Wl,-z,notext" AC_TRY_LINK(,,AC_MSG_RESULT([yes]), AC_MSG_RESULT([no]) LDFLAGS=$_SAVE_LDFLAGS) diff --git a/js/src/shell/js.cpp b/js/src/shell/js.cpp index 24dd27e27f..be02bd47e9 100644 --- a/js/src/shell/js.cpp +++ b/js/src/shell/js.cpp @@ -96,6 +96,10 @@ #ifdef JS_SIMULATOR_MIPS32 # include "jit/mips32/Simulator-mips32.h" #endif +// TODO:LOONG64 Recheck include simulator header +#ifdef JS_SIMULATOR_LOONG64 +# include "jit/loong64/Simulator-loong64.h" +#endif #ifdef JS_SIMULATOR_MIPS64 # include "jit/mips64/Simulator-mips64.h" #endif @@ -7676,6 +7680,10 @@ static void SingleStepCallback(void* arg, jit::Simulator* sim, void* pc) { state.sp = (void*)sim->getRegister(jit::Simulator::sp); state.lr = (void*)sim->getRegister(jit::Simulator::ra); state.fp = (void*)sim->getRegister(jit::Simulator::fp); +# elif defined(JS_SIMULATOR_LOONG64) + state.sp = (void*)sim->getRegister(jit::Simulator::sp); + state.lr = (void*)sim->getRegister(jit::Simulator::ra); + state.fp = (void*)sim->getRegister(jit::Simulator::fp); # else # error "NYI: Single-step profiling support" # endif @@ -11655,6 +11663,15 @@ static bool SetContextOptions(JSContext* cx, const OptionParser& op) { if (stopAt >= 0) { jit::Simulator::StopSimAt = stopAt; } +#elif defined(JS_SIMULATOR_LOONG64) + if (op.getBoolOption("loong64-sim-icache-checks")) { + jit::SimulatorProcess::ICacheCheckingDisableCount = 0; + } + + int32_t stopAt = op.getIntOption("loong64-sim-stop-at"); + if (stopAt >= 0) { + jit::Simulator::StopSimAt = stopAt; + } #endif reportWarnings = op.getBoolOption('w'); @@ -12468,6 +12485,13 @@ int main(int argc, char** argv) { "Stop the MIPS simulator after the given " "NUMBER of instructions.", -1) || + !op.addBoolOption('\0', "loong64-sim-icache-checks", + "Enable icache flush checks in the LoongArch64 " + "simulator.") || + !op.addIntOption('\0', "loong64-sim-stop-at", "NUMBER", + "Stop the LoongArch64 simulator after the given " + "NUMBER of instructions.", + -1) || !op.addIntOption('\0', "nursery-size", "SIZE-MB", "Set the maximum nursery size in MB", JS::DefaultNurseryMaxBytes / 1024 / 1024) || diff --git a/js/src/tests/non262/Array/regress-157652.js b/js/src/tests/non262/Array/regress-157652.js index b21dcc3ddf..5452c78e6a 100644 --- a/js/src/tests/non262/Array/regress-157652.js +++ b/js/src/tests/non262/Array/regress-157652.js @@ -1,4 +1,4 @@ -// |reftest| skip-if(xulRuntime.XPCOMABI.match(/x86_64|aarch64|ppc64|ppc64le|s390x|mips64/)||Android) -- No test results +// |reftest| skip-if(xulRuntime.XPCOMABI.match(/x86_64|aarch64|ppc64|ppc64le|s390x|mips64|loongarch64/)||Android) -- No test results /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/js/src/tests/non262/Array/regress-330812.js b/js/src/tests/non262/Array/regress-330812.js index adc29b52e1..cf50ee9bff 100644 --- a/js/src/tests/non262/Array/regress-330812.js +++ b/js/src/tests/non262/Array/regress-330812.js @@ -1,4 +1,4 @@ -// |reftest| slow-if(xulRuntime.XPCOMABI.match(/x86_64|aarch64|ppc64|ppc64le|s390x|mips64/)||Android) -- No test results +// |reftest| slow-if(xulRuntime.XPCOMABI.match(/x86_64|aarch64|ppc64|ppc64le|s390x|mips64|loongarch64/)||Android) -- No test results /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/js/src/tests/non262/regress/regress-422348.js b/js/src/tests/non262/regress/regress-422348.js index e94c397db2..117e19363e 100644 --- a/js/src/tests/non262/regress/regress-422348.js +++ b/js/src/tests/non262/regress/regress-422348.js @@ -1,4 +1,4 @@ -// |reftest| skip-if(xulRuntime.XPCOMABI.match(/x86_64|aarch64|ppc64|ppc64le|s390x|mips64/)) -- On 64-bit, takes forever rather than throwing +// |reftest| skip-if(xulRuntime.XPCOMABI.match(/x86_64|aarch64|ppc64|ppc64le|s390x|mips64|loongarch64/)) -- On 64-bit, takes forever rather than throwing /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/js/src/util/Poison.h b/js/src/util/Poison.h index 8356ca1f00..cb8e1abc64 100644 --- a/js/src/util/Poison.h +++ b/js/src/util/Poison.h @@ -93,6 +93,8 @@ const uint8_t JS_SCOPE_DATA_TRAILING_NAMES_PATTERN = 0xCC; # define JS_SWEPT_CODE_PATTERN 0xA3 // undefined instruction #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) # define JS_SWEPT_CODE_PATTERN 0x01 // undefined instruction +#elif defined(JS_CODEGEN_LOONG64) +# define JS_SWEPT_CODE_PATTERN 0x01 // undefined instruction #else # error "JS_SWEPT_CODE_PATTERN not defined for this platform" #endif diff --git a/js/src/vm/ArrayBufferObject.h b/js/src/vm/ArrayBufferObject.h index c7c9f84ef9..ab2399664d 100644 --- a/js/src/vm/ArrayBufferObject.h +++ b/js/src/vm/ArrayBufferObject.h @@ -179,7 +179,7 @@ class ArrayBufferObject : public ArrayBufferObjectMaybeShared { // experimental large-buffers flag is used. static size_t maxBufferByteLength() { #ifdef JS_64BIT -# ifdef JS_CODEGEN_MIPS64 +# if defined(JS_CODEGEN_MIPS64) // Fallthrough to the "small" case because there's no evidence that the // platform code can handle buffers > 2GB. # else diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp index 156f3cbbba..063d97ed9c 100644 --- a/js/src/wasm/WasmBaselineCompile.cpp +++ b/js/src/wasm/WasmBaselineCompile.cpp @@ -143,6 +143,9 @@ # include "jit/mips-shared/Assembler-mips-shared.h" # include "jit/mips64/Assembler-mips64.h" #endif +#if defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/Assembler-loong64.h" +#endif #include "js/ScalarType.h" // js::Scalar::Type #include "util/Memory.h" #include "wasm/TypedObject.h" @@ -210,6 +213,9 @@ static constexpr Register RabaldrScratchI32 = Register::Invalid(); static constexpr FloatRegister RabaldrScratchF32 = InvalidFloatReg; static constexpr FloatRegister RabaldrScratchF64 = InvalidFloatReg; #endif +#if defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/Assembler-loong64.h" +#endif #ifdef JS_CODEGEN_ARM64 # define RABALDR_CHUNKY_STACK @@ -282,7 +288,7 @@ static constexpr Register RabaldrScratchI32 = CallTempReg2; # define RABALDR_FLOAT_TO_I64_CALLOUT #endif -#ifdef JS_CODEGEN_MIPS64 +#if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) # define RABALDR_SCRATCH_I32 static constexpr Register RabaldrScratchI32 = CallTempReg2; #endif @@ -553,7 +559,7 @@ struct SpecificRegs { SpecificRegs() : abiReturnRegI64(ReturnReg64) {} }; -#elif defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) struct SpecificRegs {}; #else struct SpecificRegs { @@ -5100,7 +5106,11 @@ class BaseCompiler final : public BaseCompilerInterface { Stk& v = stk_.back(); switch (v.kind()) { case Stk::ConstI32: +#if defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) + fr.storeImmediatePtrToStack(int32_t(v.i32val_), resultHeight, temp); +#else fr.storeImmediatePtrToStack(uint32_t(v.i32val_), resultHeight, temp); +#endif break; case Stk::ConstF32: fr.storeImmediateF32ToStack(v.f32val_, resultHeight, temp); @@ -6253,9 +6263,9 @@ class BaseCompiler final : public BaseCompilerInterface { // Sundry low-level code generators. // The compiler depends on moveImm32() clearing the high bits of a 64-bit - // register on 64-bit systems except MIPS64 where high bits are sign extended - // from lower bits, see doc block "64-bit GPRs carrying 32-bit values" in - // MacroAssembler.h. + // register on 64-bit systems except MIPS64 and LoongArch64 where high bits + // are sign extended from lower bits, see doc block "64-bit GPRs carrying + // 32-bit values" in MacroAssembler.h. void moveImm32(int32_t v, RegI32 dest) { masm.move32(Imm32(v), dest); } @@ -6329,7 +6339,8 @@ class BaseCompiler final : public BaseCompilerInterface { // Jump indirect via table element. masm.ma_ldr(DTRAddr(scratch, DtrRegImmShift(switchValue, LSL, 2)), pc, Offset, Assembler::Always); -#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) ScratchI32 scratch(*this); CodeLabel tableCl; @@ -6483,6 +6494,12 @@ class BaseCompiler final : public BaseCompilerInterface { masm.cqo(); masm.idivq(rhs.reg); } +# elif defined(JS_CODEGEN_LOONG64) + if (isUnsigned) { + masm.as_div_du(srcDest.reg, srcDest.reg, rhs.reg); + } else { + masm.as_div_d(srcDest.reg, srcDest.reg, rhs.reg); + } # elif defined(JS_CODEGEN_MIPS64) if (isUnsigned) { masm.as_ddivu(srcDest.reg, rhs.reg); @@ -6529,6 +6546,12 @@ class BaseCompiler final : public BaseCompilerInterface { masm.idivq(rhs.reg); } masm.movq(rdx, rax); +# elif defined(JS_CODEGEN_LOONG64) + if (isUnsigned) { + masm.as_mod_du(srcDest.reg, srcDest.reg, rhs.reg); + } else { + masm.as_mod_d(srcDest.reg, srcDest.reg, rhs.reg); + } # elif defined(JS_CODEGEN_MIPS64) if (isUnsigned) { masm.as_ddivu(srcDest.reg, rhs.reg); @@ -6561,7 +6584,7 @@ class BaseCompiler final : public BaseCompilerInterface { return needI32(); #elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) return RegI32::Invalid(); #else MOZ_CRASH("BaseCompiler platform hook: needRotate64Temp"); @@ -6809,6 +6832,17 @@ class BaseCompiler final : public BaseCompilerInterface { bceSafe_ &= ~(BCESet(1) << local); } + static inline void ToValidIndex(MacroAssembler& masm, RegI32 ptr) { +#if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + // When ptr is used as an index, it will be added to a 64-bit register. + // So we should explicitly promote ptr to 64-bit. Since now ptr holds a + // unsigned 32-bit value, we zero-extend it to 64-bit here. + masm.move32To64ZeroExtend(ptr, Register64(ptr)); +#endif + } + + static inline void ToValidIndex(MacroAssembler& masm, RegI64 ptr) {} + void prepareMemoryAccess(MemoryAccessDesc* access, AccessCheck* check, RegI32 tls, RegI32 ptr) { uint32_t offsetGuardLimit = @@ -6875,7 +6909,7 @@ class BaseCompiler final : public BaseCompilerInterface { // carrying 32-bit values" in MacroAssembler.h); we already have that. masm.assertCanonicalInt32(ptr); # else - MOZ_CRASH("Platform code needed here"); + masm.move32To64ZeroExtend(ptr, ptr64); # endif // Any Spectre mitigation will appear to update the ptr64 register. @@ -6889,7 +6923,7 @@ class BaseCompiler final : public BaseCompilerInterface { # if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM64) // The canonical value is zero-extended; we already have that. # else - MOZ_CRASH("Platform code needed here"); + masm.move64To32(ptr64, ptr); # endif } else { masm.wasmBoundsCheck32( @@ -6904,11 +6938,13 @@ class BaseCompiler final : public BaseCompilerInterface { masm.wasmTrap(Trap::OutOfBounds, bytecodeOffset()); masm.bind(&ok); } + + ToValidIndex(masm, ptr); } #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) BaseIndex prepareAtomicMemoryAccess(MemoryAccessDesc* access, AccessCheck* check, RegI32 tls, RegI32 ptr) { @@ -7046,6 +7082,12 @@ class BaseCompiler final : public BaseCompilerInterface { } else { masm.wasmLoad(*access, HeapReg, ptr, dest.any()); } +#elif defined(JS_CODEGEN_LOONG64) + if (dest.tag == AnyReg::I64) { + masm.wasmLoadI64(*access, HeapReg, ptr, ptr, dest.i64()); + } else { + masm.wasmLoad(*access, HeapReg, ptr, ptr, dest.any()); + } #else MOZ_CRASH("BaseCompiler platform hook: load"); #endif @@ -7142,6 +7184,13 @@ class BaseCompiler final : public BaseCompilerInterface { } else { masm.wasmStore(*access, src.any(), HeapReg, ptr); } +#elif defined(JS_CODEGEN_LOONG64) + MOZ_ASSERT(temp.isInvalid()); + if (access->type() == Scalar::Int64) { + masm.wasmStoreI64(*access, src.i64(), HeapReg, ptr, ptr); + } else { + masm.wasmStore(*access, src.any(), HeapReg, ptr, ptr); + } #else MOZ_CRASH("BaseCompiler platform hook: store"); #endif @@ -7165,7 +7214,8 @@ class BaseCompiler final : public BaseCompilerInterface { } }; -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) using AtomicRMW32Temps = Atomic32Temps<3>; #else using AtomicRMW32Temps = Atomic32Temps<1>; @@ -7192,7 +7242,8 @@ class BaseCompiler final : public BaseCompilerInterface { case Scalar::Uint16: case Scalar::Int32: case Scalar::Uint32: -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) masm.wasmAtomicFetchOp(access, op, rv, srcAddr, temps[0], temps[1], temps[2], rd); #else @@ -7213,7 +7264,8 @@ class BaseCompiler final : public BaseCompilerInterface { masm.wasmAtomicFetchOp64(access, op, value, srcAddr, temp, rd); } -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) using AtomicCmpXchg32Temps = Atomic32Temps<3>; #else using AtomicCmpXchg32Temps = Atomic32Temps<0>; @@ -7241,7 +7293,8 @@ class BaseCompiler final : public BaseCompilerInterface { case Scalar::Uint16: case Scalar::Int32: case Scalar::Uint32: -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) masm.wasmCompareExchange(access, srcAddr, rexpect, rnew, temps[0], temps[1], temps[2], rd); #else @@ -7253,7 +7306,8 @@ class BaseCompiler final : public BaseCompilerInterface { } } -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) using AtomicXchg32Temps = Atomic32Temps<3>; #else using AtomicXchg32Temps = Atomic32Temps<0>; @@ -7280,7 +7334,8 @@ class BaseCompiler final : public BaseCompilerInterface { case Scalar::Uint16: case Scalar::Int32: case Scalar::Uint32: -#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) masm.wasmAtomicExchange(access, srcAddr, rv, temps[0], temps[1], temps[2], rd); #else @@ -7339,6 +7394,8 @@ class BaseCompiler final : public BaseCompilerInterface { *temp = needI32(); #elif defined(JS_CODEGEN_MIPS64) pop2xI64(r0, r1); +#elif defined(JS_CODEGEN_LOONG64) + pop2xI64(r0, r1); #elif defined(JS_CODEGEN_MIPS32) pop2xI64(r0, r1); *temp = needI32(); @@ -7534,7 +7591,8 @@ class BaseCompiler final : public BaseCompilerInterface { bc->freeI32(rnew); bc->freeI32(rexpect); } -#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) explicit PopAtomicCmpXchg32Regs(BaseCompiler* bc, ValType type, Scalar::Type viewType) : Base(bc) { @@ -7611,7 +7669,7 @@ class BaseCompiler final : public BaseCompilerInterface { bc->freeI64(rnew); } #elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) explicit PopAtomicCmpXchg64Regs(BaseCompiler* bc) : Base(bc) { rnew = bc->popI64(); rexpect = bc->popI64(); @@ -7663,7 +7721,8 @@ class BaseCompiler final : public BaseCompilerInterface { explicit PopAtomicLoad64Regs(BaseCompiler* bc) : Base(bc) { setRd(bc->needI64Pair()); } -# elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +# elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) explicit PopAtomicLoad64Regs(BaseCompiler* bc) : Base(bc) { setRd(bc->needI64()); } @@ -7750,7 +7809,8 @@ class BaseCompiler final : public BaseCompilerInterface { bc->freeI32(rv); temps.maybeFree(bc); } -#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) explicit PopAtomicRMW32Regs(BaseCompiler* bc, ValType type, Scalar::Type viewType, AtomicOp op) : Base(bc) { @@ -7838,7 +7898,7 @@ class BaseCompiler final : public BaseCompilerInterface { bc->freeI64(temp); } #elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) explicit PopAtomicRMW64Regs(BaseCompiler* bc, AtomicOp) : Base(bc) { rv = bc->popI64(); temp = bc->needI64(); @@ -7893,7 +7953,8 @@ class BaseCompiler final : public BaseCompilerInterface { setRd(bc->needI32()); } ~PopAtomicXchg32Regs() { bc->freeI32(rv); } -#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) explicit PopAtomicXchg32Regs(BaseCompiler* bc, ValType type, Scalar::Type viewType) : Base(bc) { @@ -7959,7 +8020,8 @@ class BaseCompiler final : public BaseCompilerInterface { setRd(bc->needI64Pair()); } ~PopAtomicXchg64Regs() { bc->freeI64(rv); } -#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) explicit PopAtomicXchg64Regs(BaseCompiler* bc) : Base(bc) { rv = bc->popI64ToSpecific(bc->needI64()); setRd(bc->needI64()); @@ -8970,8 +9032,9 @@ static void CtzI32(MacroAssembler& masm, RegI32 rsd) { static RegI32 PopcntTemp(BaseCompiler& bc) { #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) return AssemblerX86Shared::HasPOPCNT() ? RegI32::Invalid() : bc.needI32(); -#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) return bc.needI32(); #else MOZ_CRASH("BaseCompiler platform hook: PopcntTemp"); @@ -11987,7 +12050,7 @@ RegI32 BaseCompiler::popMemory32Access(MemoryAccessDesc* access, void BaseCompiler::pushHeapBase() { #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) RegI64 heapBase = needI64(); moveI64(RegI64(Register64(HeapReg)), heapBase); pushI64(heapBase); @@ -17247,9 +17310,10 @@ bool js::wasm::BaselinePlatformSupport() { return false; } #endif -#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \ - defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \ + defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) return true; #else return false; diff --git a/js/src/wasm/WasmCompile.cpp b/js/src/wasm/WasmCompile.cpp index 0f456aaaa5..12bdf844e0 100644 --- a/js/src/wasm/WasmCompile.cpp +++ b/js/src/wasm/WasmCompile.cpp @@ -50,6 +50,7 @@ uint32_t wasm::ObservedCPUFeatures() { MIPS = 0x4, MIPS64 = 0x5, ARM64 = 0x6, + LOONG64 = 0x7, ARCH_BITS = 3 }; @@ -73,6 +74,9 @@ uint32_t wasm::ObservedCPUFeatures() { #elif defined(JS_CODEGEN_MIPS64) MOZ_ASSERT(jit::GetMIPSFlags() <= (UINT32_MAX >> ARCH_BITS)); return MIPS64 | (jit::GetMIPSFlags() << ARCH_BITS); +#elif defined(JS_CODEGEN_LOONG64) + MOZ_ASSERT(jit::GetLOONG64Flags() <= (UINT32_MAX >> ARCH_BITS)); + return LOONG64 | (jit::GetLOONG64Flags() << ARCH_BITS); #elif defined(JS_CODEGEN_NONE) return 0; #else diff --git a/js/src/wasm/WasmFrameIter.cpp b/js/src/wasm/WasmFrameIter.cpp index dffab53940..67aab6f726 100644 --- a/js/src/wasm/WasmFrameIter.cpp +++ b/js/src/wasm/WasmFrameIter.cpp @@ -363,6 +363,11 @@ static const unsigned PushedRetAddr = 8; static const unsigned PushedFP = 12; static const unsigned SetFP = 16; static const unsigned PoppedFP = 4; +#elif defined(JS_CODEGEN_LOONG64) +static const unsigned PushedRetAddr = 8; +static const unsigned PushedFP = 12; +static const unsigned SetFP = 16; +static const unsigned PoppedFP = 4; #elif defined(JS_CODEGEN_NONE) // Synthetic values to satisfy asserts and avoid compiler warnings. static const unsigned PushedRetAddr = 0; @@ -431,6 +436,18 @@ static void GenerateCallablePrologue(MacroAssembler& masm, uint32_t* entry) { masm.moveStackPtrTo(FramePointer); MOZ_ASSERT_IF(!masm.oom(), SetFP == masm.currentOffset() - *entry); } +#elif defined(JS_CODEGEN_LOONG64) + { + *entry = masm.currentOffset(); + + masm.subFromStackPtr(Imm32(sizeof(Frame))); + masm.storePtr(ra, Address(StackPointer, Frame::returnAddressOffset())); + MOZ_ASSERT_IF(!masm.oom(), PushedRetAddr == masm.currentOffset() - *entry); + masm.storePtr(FramePointer, Address(StackPointer, Frame::callerFPOffset())); + MOZ_ASSERT_IF(!masm.oom(), PushedFP == masm.currentOffset() - *entry); + masm.moveStackPtrTo(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), SetFP == masm.currentOffset() - *entry); + } #elif defined(JS_CODEGEN_ARM64) { // We do not use the PseudoStackPointer. However, we may be called in a @@ -504,6 +521,16 @@ static void GenerateCallableEpilogue(MacroAssembler& masm, unsigned framePushed, masm.as_jr(ra); masm.addToStackPtr(Imm32(sizeof(Frame))); +#elif defined(JS_CODEGEN_LOONG64) + + masm.loadPtr(Address(StackPointer, Frame::callerFPOffset()), FramePointer); + poppedFP = masm.currentOffset(); + masm.loadPtr(Address(StackPointer, Frame::returnAddressOffset()), ra); + + *ret = masm.currentOffset(); + masm.addToStackPtr(Imm32(sizeof(Frame))); + masm.as_jirl(zero, ra, BOffImm16(0)); + #elif defined(JS_CODEGEN_ARM64) // See comment at equivalent place in |GenerateCallablePrologue| above. @@ -769,6 +796,9 @@ void wasm::GenerateJitEntryPrologue(MacroAssembler& masm, Offsets* offsets) { #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) offsets->begin = masm.currentOffset(); masm.push(ra); +#elif defined(JS_CODEGEN_LOONG64) + offsets->begin = masm.currentOffset(); + masm.push(ra); #elif defined(JS_CODEGEN_ARM64) AutoForbidPoolsAndNops afp(&masm, /* number of instructions in scope = */ 3); @@ -1066,6 +1096,25 @@ bool js::wasm::StartUnwinding(const RegisterState& registers, fixedFP = fp; AssertMatchesCallSite(fixedPC, fixedFP); } else +#elif defined(JS_CODEGEN_LOONG64) + if (codeRange->isThunk()) { + // The FarJumpIsland sequence temporary scrambles ra. + // Don't unwind to caller. + fixedPC = pc; + fixedFP = fp; + *unwoundCaller = false; + AssertMatchesCallSite( + Frame::fromUntaggedWasmExitFP(fp)->returnAddress(), + Frame::fromUntaggedWasmExitFP(fp)->rawCaller()); + } else if (offsetFromEntry < PushedFP) { + // On LoongArch we rely on register state instead of state saved on + // stack until the wasm::Frame is completely built. + // On entry the return address is in ra (registers.lr) and + // fp holds the caller's fp. + fixedPC = (uint8_t*)registers.lr; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); + } else #elif defined(JS_CODEGEN_ARM64) if (offsetFromEntry < PushedFP || codeRange->isThunk()) { // Constraints above ensure that this covers BeforePushRetAddr and @@ -1113,6 +1162,16 @@ bool js::wasm::StartUnwinding(const RegisterState& registers, fixedPC = Frame::fromUntaggedWasmExitFP(sp)->returnAddress(); fixedFP = fp; AssertMatchesCallSite(fixedPC, fixedFP); +#elif defined(JS_CODEGEN_LOONG64) + } else if (offsetInCode >= codeRange->ret() - PoppedFP && + offsetInCode <= codeRange->ret()) { + // The fixedFP field of the Frame has been loaded into fp. + // The ra might also be loaded, but the Frame structure is still on + // stack, so we can acess the ra from there. + MOZ_ASSERT(*sp == fp); + fixedPC = Frame::fromUntaggedWasmExitFP(sp)->returnAddress(); + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); #elif defined(JS_CODEGEN_ARM64) // The stack pointer does not move until all values have // been restored so several cases can be coalesced here. @@ -1183,8 +1242,9 @@ bool js::wasm::StartUnwinding(const RegisterState& registers, // There's a jit frame above the current one; we don't care about pc // since the Jit entry frame is a jit frame which can be considered as // an exit frame. -#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) if (offsetFromEntry < PushedRetAddr) { // We haven't pushed the jit return address yet, thus the jit // frame is incomplete. During profiling frame iteration, it means diff --git a/js/src/wasm/WasmGC.cpp b/js/src/wasm/WasmGC.cpp index 4eb77a81a2..19c262f269 100644 --- a/js/src/wasm/WasmGC.cpp +++ b/js/src/wasm/WasmGC.cpp @@ -289,6 +289,9 @@ bool IsValidStackMapKey(bool debugEnabled, const uint8_t* nextPC) { // TODO (bug 1699696): Implement this. As for the platforms above, we need to // enumerate all code sequences that can precede the stackmap location. return true; +# elif defined(JS_CODEGEN_LOONG64) + // TODO:LOONG64 IsValidStackMapKey + return true; # else MOZ_CRASH("IsValidStackMapKey: requires implementation on this platform"); # endif diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp index dee5567179..5628013381 100644 --- a/js/src/wasm/WasmIonCompile.cpp +++ b/js/src/wasm/WasmIonCompile.cpp @@ -5748,9 +5748,10 @@ bool wasm::IonCompileFunctions(const ModuleEnvironment& moduleEnv, } bool js::wasm::IonPlatformSupport() { -#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \ - defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_ARM64) +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \ + defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || \ + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_LOONG64) return true; #else return false; diff --git a/js/src/wasm/WasmSignalHandlers.cpp b/js/src/wasm/WasmSignalHandlers.cpp index 4ab2a44192..cf87904475 100644 --- a/js/src/wasm/WasmSignalHandlers.cpp +++ b/js/src/wasm/WasmSignalHandlers.cpp @@ -103,6 +103,10 @@ using mozilla::DebugOnly; # define EPC_sig(p) ((p)->sc_pc) # define RFP_sig(p) ((p)->sc_regs[30]) # endif +# if defined(__loongarch__) +# define EPC_sig(p) ((p)->sc_pc) +# define RFP_sig(p) ((p)->sc_regs[22]) +# endif # if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ defined(__PPC64LE__) # define R01_sig(p) ((p)->sc_frame.fixreg[1]) @@ -144,6 +148,12 @@ using mozilla::DebugOnly; # define RSP_sig(p) ((p)->uc_mcontext.gregs[29]) # define R31_sig(p) ((p)->uc_mcontext.gregs[31]) # endif +# if defined(__linux__) && defined(__loongarch__) +# define EPC_sig(p) ((p)->uc_mcontext.__pc) +# define RRA_sig(p) ((p)->uc_mcontext.__gregs[1]) +# define RSP_sig(p) ((p)->uc_mcontext.__gregs[3]) +# define RFP_sig(p) ((p)->uc_mcontext.__gregs[22]) +# endif # if defined(__linux__) && (defined(__sparc__) && defined(__arch64__)) # define PC_sig(p) ((p)->uc_mcontext.mc_gregs[MC_PC]) # define FP_sig(p) ((p)->uc_mcontext.mc_fp) @@ -291,6 +301,23 @@ typedef struct ucontext { // Other fields are not used so don't define them here. } ucontext_t; +# elif defined(__loongarch64) + +typedef struct { + uint64_t pc; + uint64_t gregs[32]; + uint64_t fpregs[32]; + uint32_t fpc_csr; +} mcontext_t; + +typedef struct ucontext { + uint32_t uc_flags; + struct ucontext* uc_link; + stack_t uc_stack; + mcontext_t uc_mcontext; + // Other fields are not used so don't define them here. +} ucontext_t; + # elif defined(__i386__) // x86 version for Android. typedef struct { @@ -368,6 +395,11 @@ struct macos_aarch64_context { # define FP_sig(p) RFP_sig(p) # define SP_sig(p) RSP_sig(p) # define LR_sig(p) R31_sig(p) +#elif defined(__loongarch__) +# define PC_sig(p) EPC_sig(p) +# define FP_sig(p) RFP_sig(p) +# define SP_sig(p) RSP_sig(p) +# define LR_sig(p) RRA_sig(p) #elif defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ defined(__PPC64LE__) # define PC_sig(p) R32_sig(p) @@ -407,7 +439,8 @@ static uint8_t* ContextToSP(CONTEXT* context) { #endif } -#if defined(__arm__) || defined(__aarch64__) || defined(__mips__) +#if defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ + defined(__loongarch__) static uint8_t* ContextToLR(CONTEXT* context) { # ifdef LR_sig return reinterpret_cast(LR_sig(context)); @@ -423,7 +456,8 @@ static JS::ProfilingFrameIterator::RegisterState ToRegisterState( state.fp = ContextToFP(context); state.pc = ContextToPC(context); state.sp = ContextToSP(context); -#if defined(__arm__) || defined(__aarch64__) || defined(__mips__) +#if defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ + defined(__loongarch__) state.lr = ContextToLR(context); #else state.lr = (void*)UINTPTR_MAX; @@ -692,7 +726,8 @@ static void MachExceptionHandlerThread() { #else // If not Windows or Mac, assume Unix -# ifdef __mips__ +// TODO:LOONG64 kWasmTrapSignal +# if defined(__mips__) || defined(__loongarch__) static const uint32_t kWasmTrapSignal = SIGFPE; # else static const uint32_t kWasmTrapSignal = SIGILL; diff --git a/js/src/wasm/WasmStubs.cpp b/js/src/wasm/WasmStubs.cpp index 59a5cf18bf..9478849144 100644 --- a/js/src/wasm/WasmStubs.cpp +++ b/js/src/wasm/WasmStubs.cpp @@ -724,7 +724,7 @@ static bool GenerateInterpEntry(MacroAssembler& masm, const FuncExport& fe, // Save the return address if it wasn't already saved by the call insn. #ifdef JS_USE_LINK_REGISTER # if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) masm.pushReturnAddress(); # elif defined(JS_CODEGEN_ARM64) // WasmPush updates framePushed() unlike pushReturnAddress(), but that's @@ -2194,7 +2194,7 @@ static bool GenerateImportInterpExit(MacroAssembler& masm, const FuncImport& fi, MOZ_ASSERT(NonVolatileRegs.has(WasmTlsReg)); #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) MOZ_ASSERT(NonVolatileRegs.has(HeapReg)); #endif @@ -2655,6 +2655,17 @@ static const LiveRegisterSet RegsToPreserve( # ifdef ENABLE_WASM_SIMD # error "high lanes of SIMD registers need to be saved too." # endif +#elif defined(JS_CODEGEN_LOONG64) +static const LiveRegisterSet RegsToPreserve( + GeneralRegisterSet(Registers::AllMask & + ~((uint32_t(1) << Registers::tp) | + (uint32_t(1) << Registers::fp) | + (uint32_t(1) << Registers::sp) | + (uint32_t(1) << Registers::zero))), + FloatRegisterSet(FloatRegisters::AllDoubleMask)); +# ifdef ENABLE_WASM_SIMD +# error "high lanes of SIMD registers need to be saved too." +# endif #elif defined(JS_CODEGEN_ARM64) // We assume that traps do not happen while lr is live. This both ensures that // the size of RegsToPreserve is a multiple of 2 (preserving WasmStackAlignment) diff --git a/js/xpconnect/src/XPCJSContext.cpp b/js/xpconnect/src/XPCJSContext.cpp index 3a2ced1ccd..58d9269da1 100644 --- a/js/xpconnect/src/XPCJSContext.cpp +++ b/js/xpconnect/src/XPCJSContext.cpp @@ -886,7 +886,7 @@ static void LoadStartupJSPrefs(XPCJSContext* xpccx) { StaticPrefs::javascript_options_jit_full_debug_checks_DoNotUseDirectly()); #endif -#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) +#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) && !defined(JS_CODEGEN_LOONG64) JS_SetGlobalJitCompilerOption( cx, JSJITCOMPILER_SPECTRE_INDEX_MASKING, StaticPrefs::javascript_options_spectre_index_masking_DoNotUseDirectly()); diff --git a/memory/build/mozjemalloc.cpp b/memory/build/mozjemalloc.cpp index 20a9e9674a..d701bcaa7b 100644 --- a/memory/build/mozjemalloc.cpp +++ b/memory/build/mozjemalloc.cpp @@ -183,7 +183,8 @@ using namespace mozilla; // Debug builds are opted out too, for test coverage. #ifndef MOZ_DEBUG # if !defined(__ia64__) && !defined(__sparc__) && !defined(__mips__) && \ - !defined(__aarch64__) && !defined(__powerpc__) && !defined(XP_MACOSX) + !defined(__aarch64__) && !defined(__powerpc__) && !defined(XP_MACOSX) && \ + !defined(__loongarch__) # define MALLOC_STATIC_PAGESIZE 1 # endif #endif @@ -423,6 +424,8 @@ static const size_t kChunkSizeMask = kChunkSize - 1; // Platform specific page size conditions copied from js/public/HeapAPI.h # if defined(__powerpc64__) static const size_t gPageSize = 64_KiB; +# elif defined(__loongarch__) +static const size_t gPageSize = 16_KiB; # else static const size_t gPageSize = 4_KiB; # endif diff --git a/mfbt/double-conversion/double-conversion/utils.h b/mfbt/double-conversion/double-conversion/utils.h index 6022132e2b..43225a0c90 100644 --- a/mfbt/double-conversion/double-conversion/utils.h +++ b/mfbt/double-conversion/double-conversion/utils.h @@ -107,7 +107,7 @@ int main(int argc, char** argv) { #if defined(_M_X64) || defined(__x86_64__) || \ defined(__ARMEL__) || defined(__avr32__) || defined(_M_ARM) || defined(_M_ARM64) || \ defined(__hppa__) || defined(__ia64__) || \ - defined(__mips__) || \ + defined(__mips__) || defined(__loongarch64) || \ defined(__nios2__) || defined(__ghs) || \ defined(__powerpc__) || defined(__ppc__) || defined(__ppc64__) || \ defined(_POWER) || defined(_ARCH_PPC) || defined(_ARCH_PPC64) || \ diff --git a/mfbt/tests/TestPoisonArea.cpp b/mfbt/tests/TestPoisonArea.cpp index 96af108037..093ebee754 100644 --- a/mfbt/tests/TestPoisonArea.cpp +++ b/mfbt/tests/TestPoisonArea.cpp @@ -156,6 +156,9 @@ # define RETURN_INSTR_TYPE uint64_t # endif +#elif defined __loongarch__ +# define RETURN_INSTR 0x4c000020 /* jirl $r0, $r1, 0 */ + #elif defined __s390__ # define RETURN_INSTR 0x07fe0000 /* br %r14 */ diff --git a/modules/libpref/init/StaticPrefList.yaml b/modules/libpref/init/StaticPrefList.yaml index 058df55a18..832aa92b10 100644 --- a/modules/libpref/init/StaticPrefList.yaml +++ b/modules/libpref/init/StaticPrefList.yaml @@ -6038,10 +6038,10 @@ mirror: always # LoadStartupJSPrefs do_not_use_directly: true -#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) +#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) && !defined(JS_CODEGEN_LOONG64) # Spectre security vulnerability mitigations for the JS JITs. # - # NOTE: The MIPS backends do not support these mitigations (and generally + # NOTE: The MIPS and LoongArch64 backends do not support these mitigations (and generally # do not need them). In that case, leave the pref unlisted with its # default value of false. - name: javascript.options.spectre.index_masking diff --git a/python/mozbuild/mozbuild/configure/constants.py b/python/mozbuild/mozbuild/configure/constants.py index 9f7a977f61..a8ba21d430 100644 --- a/python/mozbuild/mozbuild/configure/constants.py +++ b/python/mozbuild/mozbuild/configure/constants.py @@ -50,6 +50,7 @@ CPU_bitness = { "m68k": 32, "mips32": 32, "mips64": 64, + "loongarch64": 64, "ppc": 32, "ppc64": 64, "riscv64": 64, @@ -93,6 +94,7 @@ CPU_preprocessor_checks = OrderedDict( ("sparc", "__sparc__"), ("m68k", "__m68k__"), ("mips64", "__mips64"), + ("loongarch64", "__loongarch64"), ("mips32", "__mips__"), ("riscv64", "__riscv && __riscv_xlen == 64"), ("sh4", "__sh__"), -- 2.33.0