Bug 1571464: Bump Cranelift to c927680fd42a56efcc9a5ee59e814d9eef6c5b4f; r=jseward
authorBenjamin Bouvier <benj@benj.me>
Wed, 07 Aug 2019 14:20:48 +0000
changeset 486788 d4b4afb9cd960a5c1a6dc6e7e495f33846a67d5f
parent 486787 5ca6110b7cf79c6f213e92d6caa97c023580f997
child 486789 1635cec3e62f6d1c439bdae8c2f9f85ba80c7235
push id36407
push userncsoregi@mozilla.com
push dateThu, 08 Aug 2019 09:33:10 +0000
treeherdermozilla-central@a28a338396c3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjseward
bugs1571464
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1571464: Bump Cranelift to c927680fd42a56efcc9a5ee59e814d9eef6c5b4f; r=jseward Differential Revision: https://phabricator.services.mozilla.com/D40646
.cargo/config.in
Cargo.lock
Cargo.toml
js/src/wasm/cranelift/Cargo.toml
third_party/rust/cranelift-bforest/.cargo-checksum.json
third_party/rust/cranelift-bforest/Cargo.toml
third_party/rust/cranelift-codegen-meta/.cargo-checksum.json
third_party/rust/cranelift-codegen-meta/Cargo.toml
third_party/rust/cranelift-codegen-meta/src/cdsl/ast.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/cpu_modes.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/encodings.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/formats.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/instructions.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/isa.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/mod.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/recipes.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/regs.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/settings.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/type_inference.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/types.rs
third_party/rust/cranelift-codegen-meta/src/cdsl/xform.rs
third_party/rust/cranelift-codegen-meta/src/constant_hash.rs
third_party/rust/cranelift-codegen-meta/src/default_map.rs
third_party/rust/cranelift-codegen-meta/src/gen_binemit.rs
third_party/rust/cranelift-codegen-meta/src/gen_encodings.rs
third_party/rust/cranelift-codegen-meta/src/gen_inst.rs
third_party/rust/cranelift-codegen-meta/src/gen_legalizer.rs
third_party/rust/cranelift-codegen-meta/src/gen_settings.rs
third_party/rust/cranelift-codegen-meta/src/isa/arm32/mod.rs
third_party/rust/cranelift-codegen-meta/src/isa/arm64/mod.rs
third_party/rust/cranelift-codegen-meta/src/isa/riscv/encodings.rs
third_party/rust/cranelift-codegen-meta/src/isa/riscv/mod.rs
third_party/rust/cranelift-codegen-meta/src/isa/riscv/recipes.rs
third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs
third_party/rust/cranelift-codegen-meta/src/isa/x86/instructions.rs
third_party/rust/cranelift-codegen-meta/src/isa/x86/legalize.rs
third_party/rust/cranelift-codegen-meta/src/isa/x86/mod.rs
third_party/rust/cranelift-codegen-meta/src/isa/x86/recipes.rs
third_party/rust/cranelift-codegen-meta/src/isa/x86/settings.rs
third_party/rust/cranelift-codegen-meta/src/lib.rs
third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs
third_party/rust/cranelift-codegen-meta/src/shared/legalize.rs
third_party/rust/cranelift-codegen-meta/src/shared/mod.rs
third_party/rust/cranelift-codegen-meta/src/shared/settings.rs
third_party/rust/cranelift-codegen-meta/src/shared/types.rs
third_party/rust/cranelift-codegen/.cargo-checksum.json
third_party/rust/cranelift-codegen/Cargo.toml
third_party/rust/cranelift-codegen/build.rs
third_party/rust/cranelift-codegen/meta-python/base/__init__.py
third_party/rust/cranelift-codegen/meta-python/base/entities.py
third_party/rust/cranelift-codegen/meta-python/base/formats.py
third_party/rust/cranelift-codegen/meta-python/base/immediates.py
third_party/rust/cranelift-codegen/meta-python/base/instructions.py
third_party/rust/cranelift-codegen/meta-python/base/legalize.py
third_party/rust/cranelift-codegen/meta-python/base/predicates.py
third_party/rust/cranelift-codegen/meta-python/base/semantics.py
third_party/rust/cranelift-codegen/meta-python/base/settings.py
third_party/rust/cranelift-codegen/meta-python/base/types.py
third_party/rust/cranelift-codegen/meta-python/build.py
third_party/rust/cranelift-codegen/meta-python/cdsl/__init__.py
third_party/rust/cranelift-codegen/meta-python/cdsl/ast.py
third_party/rust/cranelift-codegen/meta-python/cdsl/formats.py
third_party/rust/cranelift-codegen/meta-python/cdsl/instructions.py
third_party/rust/cranelift-codegen/meta-python/cdsl/isa.py
third_party/rust/cranelift-codegen/meta-python/cdsl/operands.py
third_party/rust/cranelift-codegen/meta-python/cdsl/predicates.py
third_party/rust/cranelift-codegen/meta-python/cdsl/registers.py
third_party/rust/cranelift-codegen/meta-python/cdsl/settings.py
third_party/rust/cranelift-codegen/meta-python/cdsl/test_ast.py
third_party/rust/cranelift-codegen/meta-python/cdsl/test_package.py
third_party/rust/cranelift-codegen/meta-python/cdsl/test_ti.py
third_party/rust/cranelift-codegen/meta-python/cdsl/test_typevar.py
third_party/rust/cranelift-codegen/meta-python/cdsl/test_xform.py
third_party/rust/cranelift-codegen/meta-python/cdsl/ti.py
third_party/rust/cranelift-codegen/meta-python/cdsl/types.py
third_party/rust/cranelift-codegen/meta-python/cdsl/typevar.py
third_party/rust/cranelift-codegen/meta-python/cdsl/xform.py
third_party/rust/cranelift-codegen/meta-python/check.sh
third_party/rust/cranelift-codegen/meta-python/constant_hash.py
third_party/rust/cranelift-codegen/meta-python/gen_binemit.py
third_party/rust/cranelift-codegen/meta-python/gen_build_deps.py
third_party/rust/cranelift-codegen/meta-python/gen_encoding.py
third_party/rust/cranelift-codegen/meta-python/isa/__init__.py
third_party/rust/cranelift-codegen/meta-python/isa/arm32/__init__.py
third_party/rust/cranelift-codegen/meta-python/isa/arm32/defs.py
third_party/rust/cranelift-codegen/meta-python/isa/arm32/registers.py
third_party/rust/cranelift-codegen/meta-python/isa/arm32/settings.py
third_party/rust/cranelift-codegen/meta-python/isa/arm64/__init__.py
third_party/rust/cranelift-codegen/meta-python/isa/arm64/defs.py
third_party/rust/cranelift-codegen/meta-python/isa/arm64/registers.py
third_party/rust/cranelift-codegen/meta-python/isa/arm64/settings.py
third_party/rust/cranelift-codegen/meta-python/isa/riscv/__init__.py
third_party/rust/cranelift-codegen/meta-python/isa/riscv/defs.py
third_party/rust/cranelift-codegen/meta-python/isa/riscv/encodings.py
third_party/rust/cranelift-codegen/meta-python/isa/riscv/recipes.py
third_party/rust/cranelift-codegen/meta-python/isa/riscv/registers.py
third_party/rust/cranelift-codegen/meta-python/isa/riscv/settings.py
third_party/rust/cranelift-codegen/meta-python/isa/x86/__init__.py
third_party/rust/cranelift-codegen/meta-python/isa/x86/defs.py
third_party/rust/cranelift-codegen/meta-python/isa/x86/encodings.py
third_party/rust/cranelift-codegen/meta-python/isa/x86/instructions.py
third_party/rust/cranelift-codegen/meta-python/isa/x86/legalize.py
third_party/rust/cranelift-codegen/meta-python/isa/x86/recipes.py
third_party/rust/cranelift-codegen/meta-python/isa/x86/registers.py
third_party/rust/cranelift-codegen/meta-python/isa/x86/settings.py
third_party/rust/cranelift-codegen/meta-python/mypy.ini
third_party/rust/cranelift-codegen/meta-python/semantics/__init__.py
third_party/rust/cranelift-codegen/meta-python/semantics/elaborate.py
third_party/rust/cranelift-codegen/meta-python/semantics/macros.py
third_party/rust/cranelift-codegen/meta-python/semantics/primitives.py
third_party/rust/cranelift-codegen/meta-python/semantics/smtlib.py
third_party/rust/cranelift-codegen/meta-python/semantics/test_elaborate.py
third_party/rust/cranelift-codegen/meta-python/srcgen.py
third_party/rust/cranelift-codegen/meta-python/stubs/z3/__init__.pyi
third_party/rust/cranelift-codegen/meta-python/stubs/z3/z3core.pyi
third_party/rust/cranelift-codegen/meta-python/stubs/z3/z3types.pyi
third_party/rust/cranelift-codegen/meta-python/test_constant_hash.py
third_party/rust/cranelift-codegen/meta-python/test_srcgen.py
third_party/rust/cranelift-codegen/meta-python/unique_table.py
third_party/rust/cranelift-codegen/src/binemit/mod.rs
third_party/rust/cranelift-codegen/src/binemit/shrink.rs
third_party/rust/cranelift-codegen/src/cfg_printer.rs
third_party/rust/cranelift-codegen/src/constant_hash.rs
third_party/rust/cranelift-codegen/src/cursor.rs
third_party/rust/cranelift-codegen/src/ir/builder.rs
third_party/rust/cranelift-codegen/src/ir/dfg.rs
third_party/rust/cranelift-codegen/src/ir/entities.rs
third_party/rust/cranelift-codegen/src/ir/function.rs
third_party/rust/cranelift-codegen/src/ir/immediates.rs
third_party/rust/cranelift-codegen/src/ir/instructions.rs
third_party/rust/cranelift-codegen/src/ir/libcall.rs
third_party/rust/cranelift-codegen/src/ir/mod.rs
third_party/rust/cranelift-codegen/src/ir/sourceloc.rs
third_party/rust/cranelift-codegen/src/ir/stackslot.rs
third_party/rust/cranelift-codegen/src/ir/types.rs
third_party/rust/cranelift-codegen/src/ir/valueloc.rs
third_party/rust/cranelift-codegen/src/isa/arm32/enc_tables.rs
third_party/rust/cranelift-codegen/src/isa/arm32/settings.rs
third_party/rust/cranelift-codegen/src/isa/arm64/enc_tables.rs
third_party/rust/cranelift-codegen/src/isa/arm64/settings.rs
third_party/rust/cranelift-codegen/src/isa/enc_tables.rs
third_party/rust/cranelift-codegen/src/isa/registers.rs
third_party/rust/cranelift-codegen/src/isa/riscv/enc_tables.rs
third_party/rust/cranelift-codegen/src/isa/riscv/settings.rs
third_party/rust/cranelift-codegen/src/isa/stack.rs
third_party/rust/cranelift-codegen/src/isa/x86/abi.rs
third_party/rust/cranelift-codegen/src/isa/x86/enc_tables.rs
third_party/rust/cranelift-codegen/src/isa/x86/mod.rs
third_party/rust/cranelift-codegen/src/isa/x86/settings.rs
third_party/rust/cranelift-codegen/src/legalizer/mod.rs
third_party/rust/cranelift-codegen/src/predicates.rs
third_party/rust/cranelift-codegen/src/regalloc/register_set.rs
third_party/rust/cranelift-codegen/src/settings.rs
third_party/rust/cranelift-codegen/src/simple_preopt.rs
third_party/rust/cranelift-codegen/src/value_label.rs
third_party/rust/cranelift-codegen/src/verifier/mod.rs
third_party/rust/cranelift-entity/.cargo-checksum.json
third_party/rust/cranelift-entity/Cargo.toml
third_party/rust/cranelift-entity/src/map.rs
third_party/rust/cranelift-entity/src/primary.rs
third_party/rust/cranelift-frontend/.cargo-checksum.json
third_party/rust/cranelift-frontend/Cargo.toml
third_party/rust/cranelift-frontend/src/frontend.rs
third_party/rust/cranelift-wasm/.cargo-checksum.json
third_party/rust/cranelift-wasm/Cargo.toml
third_party/rust/cranelift-wasm/src/code_translator.rs
third_party/rust/cranelift-wasm/src/environ/spec.rs
third_party/rust/cranelift-wasm/src/func_translator.rs
third_party/rust/cranelift-wasm/src/lib.rs
third_party/rust/cranelift-wasm/src/module_translator.rs
third_party/rust/cranelift-wasm/src/state.rs
third_party/rust/cranelift-wasm/src/translation_utils.rs
--- a/.cargo/config.in
+++ b/.cargo/config.in
@@ -19,17 +19,17 @@ replace-with = "vendored-sources"
 
 [source."https://github.com/rust-lang-nursery/packed_simd"]
 git = "https://github.com/hsivonen/packed_simd"
 branch = "rust_1_32"
 replace-with = "vendored-sources"
 
 [source."https://github.com/CraneStation/Cranelift"]
 git = "https://github.com/CraneStation/Cranelift"
-rev = "312516a69da03dc06eace32f61412389a8dcadf3"
+rev = "c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
 replace-with = "vendored-sources"
 
 [source."https://github.com/ChunMinChang/coreaudio-sys"]
 git = "https://github.com/ChunMinChang/coreaudio-sys"
 branch = "gecko-build"
 replace-with = "vendored-sources"
 
 [source."https://github.com/alexcrichton/mio-named-pipes"]
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -193,18 +193,18 @@ dependencies = [
  "libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "baldrdash"
 version = "0.1.0"
 dependencies = [
  "bindgen 0.51.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "cranelift-codegen 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
- "cranelift-wasm 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
+ "cranelift-codegen 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
+ "cranelift-wasm 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
  "env_logger 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "target-lexicon 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "base64"
 version = "0.9.3"
@@ -623,67 +623,67 @@ name = "cose-c"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "cose 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.32.0"
-source = "git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3#312516a69da03dc06eace32f61412389a8dcadf3"
-dependencies = [
- "cranelift-entity 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
+version = "0.37.0"
+source = "git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f#c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
+dependencies = [
+ "cranelift-entity 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.32.0"
-source = "git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3#312516a69da03dc06eace32f61412389a8dcadf3"
-dependencies = [
- "cranelift-bforest 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
- "cranelift-codegen-meta 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
- "cranelift-entity 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
+version = "0.37.0"
+source = "git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f#c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
+dependencies = [
+ "cranelift-bforest 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
+ "cranelift-codegen-meta 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
+ "cranelift-entity 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
  "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "target-lexicon 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.32.0"
-source = "git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3#312516a69da03dc06eace32f61412389a8dcadf3"
-dependencies = [
- "cranelift-entity 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
+version = "0.37.0"
+source = "git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f#c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
+dependencies = [
+ "cranelift-entity 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.32.0"
-source = "git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3#312516a69da03dc06eace32f61412389a8dcadf3"
+version = "0.37.0"
+source = "git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f#c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.32.0"
-source = "git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3#312516a69da03dc06eace32f61412389a8dcadf3"
-dependencies = [
- "cranelift-codegen 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
+version = "0.37.0"
+source = "git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f#c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
+dependencies = [
+ "cranelift-codegen 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "target-lexicon 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "cranelift-wasm"
-version = "0.32.0"
-source = "git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3#312516a69da03dc06eace32f61412389a8dcadf3"
-dependencies = [
- "cranelift-codegen 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
- "cranelift-entity 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
- "cranelift-frontend 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)",
+version = "0.37.0"
+source = "git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f#c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
+dependencies = [
+ "cranelift-codegen 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
+ "cranelift-entity 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
+ "cranelift-frontend 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)",
  "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "wasmparser 0.32.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "crc"
@@ -3945,22 +3945,22 @@ dependencies = [
 "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e"
 "checksum cookie 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1465f8134efa296b4c19db34d909637cb2bf0f7aaf21299e23e18fa29ac557cf"
 "checksum core-foundation 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4e2640d6d0bf22e82bed1b73c6aef8d5dd31e5abe6666c57e6d45e2649f4f887"
 "checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
 "checksum core-graphics 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)" = "62ceafe1622ffc9a332199096841d0ff9912ec8cf8f9cde01e254a7d5217cd10"
 "checksum core-text 13.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f3f46450d6f2397261af420b4ccce23807add2e45fa206410a03d66fb7f050ae"
 "checksum cose 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "72fa26cb151d3ae4b70f63d67d0fed57ce04220feafafbae7f503bef7aae590d"
 "checksum cose-c 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "49726015ab0ca765144fcca61e4a7a543a16b795a777fa53f554da2fffff9a94"
-"checksum cranelift-bforest 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)" = "<none>"
-"checksum cranelift-codegen 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)" = "<none>"
-"checksum cranelift-codegen-meta 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)" = "<none>"
-"checksum cranelift-entity 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)" = "<none>"
-"checksum cranelift-frontend 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)" = "<none>"
-"checksum cranelift-wasm 0.32.0 (git+https://github.com/CraneStation/Cranelift?rev=312516a69da03dc06eace32f61412389a8dcadf3)" = "<none>"
+"checksum cranelift-bforest 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)" = "<none>"
+"checksum cranelift-codegen 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)" = "<none>"
+"checksum cranelift-codegen-meta 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)" = "<none>"
+"checksum cranelift-entity 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)" = "<none>"
+"checksum cranelift-frontend 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)" = "<none>"
+"checksum cranelift-wasm 0.37.0 (git+https://github.com/CraneStation/Cranelift?rev=c927680fd42a56efcc9a5ee59e814d9eef6c5b4f)" = "<none>"
 "checksum crc 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd5d02c0aac6bd68393ed69e00bbc2457f3e89075c6349db7189618dc4ddc1d7"
 "checksum crossbeam-channel 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8d4f5844607ce8da3fff431e7dba56cda8bfcc570aa50bee36adba8a32b8cad7"
 "checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13"
 "checksum crossbeam-epoch 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "04c9e3102cc2d69cd681412141b390abd55a362afc1540965dad0ad4d34280b4"
 "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
 "checksum crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c"
 "checksum cssparser 0.25.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a921abc45ea75c2c817d951caeda31b94539d09a6b5e8d58a857b3b35c9c3894"
 "checksum cssparser-macros 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b16e382d9b983fdb9ac6a36b37fdeb84ce3ea81f749febfee3463cfa7f24275e"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -58,16 +58,16 @@ codegen-units = 1
 
 [patch.crates-io]
 libudev-sys = { path = "dom/webauthn/libudev-sys" }
 winapi = { git = "https://github.com/froydnj/winapi-rs", branch = "aarch64" }
 packed_simd = { git = "https://github.com/hsivonen/packed_simd", branch = "rust_1_32" }
 
 [patch.crates-io.cranelift-codegen]
 git = "https://github.com/CraneStation/Cranelift"
-rev = "312516a69da03dc06eace32f61412389a8dcadf3"
+rev = "c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
 
 [patch.crates-io.cranelift-wasm]
 git = "https://github.com/CraneStation/Cranelift"
-rev = "312516a69da03dc06eace32f61412389a8dcadf3"
+rev = "c927680fd42a56efcc9a5ee59e814d9eef6c5b4f"
 
 [patch.crates-io.coreaudio-sys]
 path = "third_party/rust/coreaudio-sys"
--- a/js/src/wasm/cranelift/Cargo.toml
+++ b/js/src/wasm/cranelift/Cargo.toml
@@ -11,18 +11,18 @@ name = "baldrdash"
 [dependencies]
 # The build system redirects the versions of cranelift-codegen and
 # cranelift-wasm to pinned commits. If you want to update Cranelift in Gecko,
 # you should update the following files:
 # - $TOP_LEVEL/Cargo.toml, look for the revision (rev) hashes of both cranelift
 # dependencies (codegen and wasm).
 # - $TOP_LEVEL/.cargo/config.in, look for the revision (rev) field of the
 # Cranelift source.
-cranelift-codegen = { version = "0.32", default-features = false }
-cranelift-wasm = "0.32"
+cranelift-codegen = { version = "0.37", default-features = false }
+cranelift-wasm = "0.37"
 target-lexicon = "0.4.0"
 log = { version = "0.4.6", default-features = false, features = ["release_max_level_info"] }
 env_logger = "0.5.6"
 
 [build-dependencies]
 bindgen = {version = "0.51", default-features = false} # disable `logging` to reduce code size
 
 [features]
--- a/third_party/rust/cranelift-bforest/.cargo-checksum.json
+++ b/third_party/rust/cranelift-bforest/.cargo-checksum.json
@@ -1,1 +1,1 @@
-{"files":{"Cargo.toml":"1bd727aa095825902dcacc15d67f21693eea560cf57a13eb3de7d458e4ec047f","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"af367c67340fa7f6fb9a35b0aa637dcf303957f7ae7427a5f4f6356801c8bb04","src/lib.rs":"1b23abbfe5850a4cd77ae6ae5dcfc2f678ef36b4032fd7496f2b333c51e63301","src/map.rs":"5d891d62814941e19dfc88ff36538efa3da5479f3f97de8219a6f610c9a1ee32","src/node.rs":"e620c64e78488035f11723b14892c7986c06ad37dc5b115a35a453ff1ae66ca3","src/path.rs":"a86ee1c882c173e8af96fd53a416a0fb485dd3f045ac590ef313a9d9ecf90f56","src/pool.rs":"6090f8c0e0da16ebee0e31bca66392d0075b3aff529d30d4e716fa20cd0aef99","src/set.rs":"b411158f813a310c7a6c337d4ada3bf0a021088c443875dc25233415dcbe0633"},"package":null}
\ No newline at end of file
+{"files":{"Cargo.toml":"4b2597a03220bf2e539f62dbbdee8758b496c29cc8c509c60e1349b0e933b78d","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"af367c67340fa7f6fb9a35b0aa637dcf303957f7ae7427a5f4f6356801c8bb04","src/lib.rs":"1b23abbfe5850a4cd77ae6ae5dcfc2f678ef36b4032fd7496f2b333c51e63301","src/map.rs":"5d891d62814941e19dfc88ff36538efa3da5479f3f97de8219a6f610c9a1ee32","src/node.rs":"e620c64e78488035f11723b14892c7986c06ad37dc5b115a35a453ff1ae66ca3","src/path.rs":"a86ee1c882c173e8af96fd53a416a0fb485dd3f045ac590ef313a9d9ecf90f56","src/pool.rs":"6090f8c0e0da16ebee0e31bca66392d0075b3aff529d30d4e716fa20cd0aef99","src/set.rs":"b411158f813a310c7a6c337d4ada3bf0a021088c443875dc25233415dcbe0633"},"package":null}
\ No newline at end of file
--- a/third_party/rust/cranelift-bforest/Cargo.toml
+++ b/third_party/rust/cranelift-bforest/Cargo.toml
@@ -1,23 +1,23 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-bforest"
-version = "0.32.0"
+version = "0.37.0"
 description = "A forest of B+-trees"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://cranelift.readthedocs.io/"
 repository = "https://github.com/CraneStation/cranelift"
 categories = ["no-std"]
 readme = "README.md"
 keywords = ["btree", "forest", "set", "map"]
 edition = "2018"
 
 [dependencies]
-cranelift-entity = { path = "../cranelift-entity", version = "0.32.0", default-features = false }
+cranelift-entity = { path = "../cranelift-entity", version = "0.37.0", default-features = false }
 
 [features]
 default = ["std"]
 std = ["cranelift-entity/std"]
 core = []
 
 [badges]
 maintenance = { status = "experimental" }
--- a/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json
+++ b/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json
@@ -1,1 +1,1 @@
-{"files":{"Cargo.toml":"b5dee75ce6c2fc2f678809ee361b2dde077697f0e22b664a937aa1132de9e4c4","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"b123f056d0d458396679c5f7f2a16d2762af0258fcda4ac14b6655a95e5a0022","src/cdsl/ast.rs":"170f0526c73e4ebdd22cf25290607b6610e9e3fe0b1d79ee38962085f878d2e4","src/cdsl/cpu_modes.rs":"57c40621115a58faa7af1c729ecaf8cda01c2b143008bde6d8c70885e7c3fd75","src/cdsl/formats.rs":"8eaee84c9c574e7a2ab73b4bf63b97c1cea3ffce0a477f1701e75a7a523ada57","src/cdsl/instructions.rs":"dd734b1ff0750ec09a06e5746338f479ae97eb122dc64dba37b656ab5ae79ebd","src/cdsl/isa.rs":"569bf078c68c800fbd36fde1b52e0371747f6249252655e2dc4715d4e975be40","src/cdsl/mod.rs":"86149f101e1b00a9623028dcd779ad66b6a67f0f67b423c5ac88cd96dabdd227","src/cdsl/operands.rs":"1cda258798d861c4f467783b5c70c1202a57f554861017eead6477af2ee34063","src/cdsl/regs.rs":"049d299c63a757aad7adb7945b3919bd62c9204348a4f8b9727ca84fecbf3115","src/cdsl/settings.rs":"b6b1653b486fb4b86aaab59328a959cf784cb4b61b3216fe7acebe0490849642","src/cdsl/type_inference.rs":"2771631701c150e077c5dcf705c8ae8705944d86ab945ae9e7adc82f3ca5447a","src/cdsl/types.rs":"4cc1f20eb8383fdee6a9e7ca0f7758e563a4fb715056b5edbd4db72f8dfd471b","src/cdsl/typevar.rs":"7249fcd7c3cd3645e8489c73595bd5327bedc499bd7bc87f2a68e241fba329c3","src/cdsl/xform.rs":"005bd2fca7f8b737c605a75a0f44f2d70f0138e67f3013343ced81f639dce8bb","src/constant_hash.rs":"b8acd3f8712a4999819d9d9beced2938d9940a5748ba016c182f1132d97eefab","src/error.rs":"5110a4e3c1e97396ba02d9f5abbb8af4b586f0cc4d33a5c2473f1718cc4bef05","src/gen_inst.rs":"94e71181683f022fdcc55abf487a20be5d4dd51fe3092345b6bc4e2bde04441b","src/gen_legalizer.rs":"368ea72fe2d253f7d4d7548ae449fde875781e43ce0ecb9a604765595e387cc2","src/gen_registers.rs":"a544a2b91fafe08639e39e50bea0892fda89fe2f6eaf111b2d5f3e98e4d07b86","src/gen_settings.rs":"4469bf496f9539835fce3cd9743ac0fbc270b3029b0e6c949f406e790685199c","src/gen_types.rs":"3935da6c6a53f9332e06f74bc3a46270656b4d4231ad28ed2648d7b1d2774e90","src/isa/arm32/mod.rs":"54e88d89644c5cee61cdc8315f1d77b01c232dc322c52f6e244cef9e7688a3ad","src/isa/arm64/mod.rs":"c5169b92c1bcb562649a48b7880033f542c9affa73274d23b3817e4ac83942fe","src/isa/mod.rs":"fce60d19dd3c099ebee3ac5ae64a2bee363f13da9ff5a4960d3c1a0bee71d29a","src/isa/riscv/mod.rs":"2da05e3974ef0b72431c9fdda14314890f72500b94def1d74cdf5e74bd950bc0","src/isa/x86/instructions.rs":"ad88f311fd51f341f7dfe395f98d8805ea55e1c86d1a89272ed999f5ae8dc3f0","src/isa/x86/legalize.rs":"251af731b2514fead08e27e812ca44dc674a41bd4a9637d9d919259497961961","src/isa/x86/mod.rs":"14715e552eedfeae1693a416bb82bda2156b31cd4b306e07530a3c1acdc17125","src/isa/x86/registers.rs":"c0bc60336a8c8b7b4db92dc623e9419a60af14dd6252f0b19e227e46f7166178","src/isa/x86/settings.rs":"1a74b3d1ef5e99e0b7c75528b9fd5afbb3394979615b005b43b72108757a9886","src/lib.rs":"8c9364c6fce73c158abfb7c88ecff01dc608a05c250e89df9bec3082773cde6d","src/shared/entities.rs":"80b8ff57a09c7b2f9dab755abbcc2738317de474776fe1de0d2a581310aa9af8","src/shared/formats.rs":"20908b1048c5e71a185de6b6ded79cdff2c26ddb38ba7b134b7a27f37e8324f3","src/shared/immediates.rs":"804c4c9ffa2fe55d90279ee158aaa6bd6c7f0c604d63d7457a98e82269cec9a7","src/shared/instructions.rs":"85f74df59fb0564613055df2173699e61d9e711fb481015635de47c5286690fd","src/shared/legalize.rs":"e3ed83c004afd088bedad0db6664983bfc15149fb99d7d28dea535b2ff48d761","src/shared/mod.rs":"6e30631fe2ba19d819330ea14d29cce4910b435f030e706be2fc94af23b88d71","src/shared/settings.rs":"42c2c5b6f5fdef090c35760c5920d5013afe4ddc472ee9cc37eb6d8ddaae2906","src/shared/types.rs":"158d73840185e6aa8385463bbf6568efdda0c8de8284cf6b4e565f425ec5d921","src/srcgen.rs":"79fee2f603b33f76f7c9c8b9452c745a363d732c40c0814d84001ff3ef805677","src/unique_table.rs":"90b7203b29241a1ede70f0a3e50d96799e0b41d8f7455170d6ffb127f87f3cc3"},"package":null}
\ No newline at end of file
+{"files":{"Cargo.toml":"918c996a040b20c30d82c085c666949a1fb3429093fb762af932b07c33030b5d","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"b123f056d0d458396679c5f7f2a16d2762af0258fcda4ac14b6655a95e5a0022","src/cdsl/ast.rs":"1c9bf45110e47710ea160f30b3f3b00b531a8d91ff365bae856dd9d44b928253","src/cdsl/cpu_modes.rs":"7c59ae347d6f9c769d6356fe49c8406934153eefa59d9bf37c182474fcbb9715","src/cdsl/encodings.rs":"da6fa16c95fe4a586c2d00ef4ccf79b4b0f64cd8923b3ca5a5a527da1e799a4f","src/cdsl/formats.rs":"811dcf12f6e9c5be72f18543675ff52c3996edd029064bd0643f878d2ceca3bd","src/cdsl/instructions.rs":"f4c778bf55c0e273a694ecbb995bc733799d6c5930a64210ff619f0a4a3fa039","src/cdsl/isa.rs":"9b6030a935f69b07726d239c23a78d654566785f1fed61ccefdaf7d4f0a97d0e","src/cdsl/mod.rs":"c85f62a7d8d6bedc81c673b8d02be01181f79f272dbc0325a76d52e7eec732e8","src/cdsl/operands.rs":"1cda258798d861c4f467783b5c70c1202a57f554861017eead6477af2ee34063","src/cdsl/recipes.rs":"9f50f29f243f2ed8dbca3ef8b2722e9671bc164b2956254a95ed62641315eab7","src/cdsl/regs.rs":"8a92798a92b2c34c5e572e03602454c72a896d31ac301f15f43d17c101d4da6e","src/cdsl/settings.rs":"7da3c5a9df8e47ed6ca7f1d820e28eae45e783f3759f6c55730d2f17d88f1013","src/cdsl/type_inference.rs":"695d4fd2720f43569a52f29998bd92562951f0b4e7844cc0a239640e0c7d0b0f","src/cdsl/types.rs":"0db5637fa86052384950d4725b2ae27082e7a6aefa1ce2d450ab4beb7d7c1405","src/cdsl/typevar.rs":"7249fcd7c3cd3645e8489c73595bd5327bedc499bd7bc87f2a68e241fba329c3","src/cdsl/xform.rs":"64e9b70ef1265c0331ee9d71c1a1f33dba3f6975b1639385c34d68456cda0e0e","src/constant_hash.rs":"66d6f42c1e98cd4dbc95009997dc00256b57c11d2c2d9eac63b33e458c68a56f","src/default_map.rs":"8bbd6f4d9f93ef2126538cda633e6d9c80a5c7dc79b8d5a759c8be1fe2dbc605","src/error.rs":"5110a4e3c1e97396ba02d9f5abbb8af4b586f0cc4d33a5c2473f1718cc4bef05","src/gen_binemit.rs":"9024619858124b9ad9ecd232683f768beefffd34248da38c2f2e0210ae187267","src/gen_encodings.rs":"024b6ea8086fa1559333245753e909f306adeab7c8dfdc915610877413cba585","src/gen_inst.rs":"9e8c8e62761f4186fdb274c0c99ea03123d454d79371f9f1b20524c3a41631c0","src/gen_legalizer.rs":"094c1ce664637e54a07d6d928e68458cadbc6e8d83ca5f72e207866f61bbe08f","src/gen_registers.rs":"a544a2b91fafe08639e39e50bea0892fda89fe2f6eaf111b2d5f3e98e4d07b86","src/gen_settings.rs":"765ca86fa735342c1875021c34a92cbc39836ba3ad1c12aa615204372a1f7b61","src/gen_types.rs":"3935da6c6a53f9332e06f74bc3a46270656b4d4231ad28ed2648d7b1d2774e90","src/isa/arm32/mod.rs":"a2500871fb270e5624f5561f24fe510f08708cdca2ab1c3163848b67694f7a7a","src/isa/arm64/mod.rs":"dc210e8dc9f179d87d2c5a72d7795a9e34bb30afb91c8053c362d43635629d19","src/isa/mod.rs":"fce60d19dd3c099ebee3ac5ae64a2bee363f13da9ff5a4960d3c1a0bee71d29a","src/isa/riscv/encodings.rs":"8246543bedcc00f407baf140dcffe8632588fe64f87cf098b4445666b31067ec","src/isa/riscv/mod.rs":"895492f0f379cfb02aba1cbb9759dc489bbb4d51ec40759af9f4d71c45c17a86","src/isa/riscv/recipes.rs":"b1fcada01bac0c46376f50c835c1a33b8d96d2cbb77971b9caa73ffb2ad38b89","src/isa/x86/encodings.rs":"c0f28d4d2eea5e81d9f005d4bf9dac30537153dcc51173ab913ae478523ef642","src/isa/x86/instructions.rs":"eea5fa7dd804d435dfdf289fc47f4bcc77db8dfce068d78d0a428155604b9626","src/isa/x86/legalize.rs":"2617153608816928a298e1ef18aa34cb62ccc1aa717d74b89e574881299d27d4","src/isa/x86/mod.rs":"6ed9f102238c1cb9029ec953e37629767438580cf9df8b4f2d1eace75ecd86fc","src/isa/x86/recipes.rs":"7a25de41b3affc092fa4aaaf195e65c77329072a80a6c9aa7c978732bb53f9b7","src/isa/x86/registers.rs":"c0bc60336a8c8b7b4db92dc623e9419a60af14dd6252f0b19e227e46f7166178","src/isa/x86/settings.rs":"4291933922323544f72a56cf5c89427b6d550ac236fb74d617268242396ad3d3","src/lib.rs":"bc458358fd81d092f368b243d07682259dbed7a336b1eed5bcdf5228368289e9","src/shared/entities.rs":"80b8ff57a09c7b2f9dab755abbcc2738317de474776fe1de0d2a581310aa9af8","src/shared/formats.rs":"20908b1048c5e71a185de6b6ded79cdff2c26ddb38ba7b134b7a27f37e8324f3","src/shared/immediates.rs":"804c4c9ffa2fe55d90279ee158aaa6bd6c7f0c604d63d7457a98e82269cec9a7","src/shared/instructions.rs":"178eb2e3817f7f5a1c2ee4498c55c54d2279843c20f6df7480ef055095503e50","src/shared/legalize.rs":"1c32c28f603b11f89e1ba9c4d301b0b8036fd81254d227836202b84573a9a446","src/shared/mod.rs":"f5bb50d8292e46380afdd83a320cb5d6021e1483741e67b1e277053c28f9b943","src/shared/settings.rs":"fd50687cfe558e3d156a22e7445052264ae2327338130e19266970a392275811","src/shared/types.rs":"482c73b5a4cd021a1b1ef6ed609fd0bb56ad70640d65fbaff46db1dd51607de4","src/srcgen.rs":"79fee2f603b33f76f7c9c8b9452c745a363d732c40c0814d84001ff3ef805677","src/unique_table.rs":"90b7203b29241a1ede70f0a3e50d96799e0b41d8f7455170d6ffb127f87f3cc3"},"package":null}
\ No newline at end of file
--- a/third_party/rust/cranelift-codegen-meta/Cargo.toml
+++ b/third_party/rust/cranelift-codegen-meta/Cargo.toml
@@ -1,20 +1,20 @@
 [package]
 name = "cranelift-codegen-meta"
 authors = ["The Cranelift Project Developers"]
-version = "0.32.0"
+version = "0.37.0"
 description = "Metaprogram for cranelift-codegen code generator library"
 license = "Apache-2.0 WITH LLVM-exception"
 repository = "https://github.com/CraneStation/cranelift"
 readme = "README.md"
 edition = "2018"
 
 [dependencies]
-cranelift-entity = { path = "../../cranelift-entity", version = "0.32.0", default-features = false }
+cranelift-entity = { path = "../../cranelift-entity", version = "0.37.0", default-features = false }
 
 [badges]
 maintenance = { status = "experimental" }
 travis-ci = { repository = "CraneStation/cranelift" }
 
 [features]
 default = ["std"]
 std = ["cranelift-entity/std"]
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/ast.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/ast.rs
@@ -371,21 +371,27 @@ impl VarPool {
 pub struct Apply {
     pub inst: Instruction,
     pub args: Vec<Expr>,
     pub value_types: Vec<ValueType>,
 }
 
 impl Apply {
     pub fn new(target: InstSpec, args: Vec<Expr>) -> Self {
-        let (inst, value_types) = match target.into() {
+        let (inst, value_types) = match target {
             InstSpec::Inst(inst) => (inst, Vec::new()),
             InstSpec::Bound(bound_inst) => (bound_inst.inst, bound_inst.value_types),
         };
 
+        // Apply should only operate on concrete value types, not "any".
+        let value_types = value_types
+            .into_iter()
+            .map(|vt| vt.expect("shouldn't be Any"))
+            .collect();
+
         // Basic check on number of arguments.
         assert!(
             inst.operands_in.len() == args.len(),
             format!("incorrect number of arguments in instruction {}", inst.name)
         );
 
         // Check that the kinds of Literals arguments match the expected operand.
         for &imm_index in &inst.imm_opnums {
@@ -429,31 +435,32 @@ impl Apply {
             .args
             .iter()
             .map(|arg| arg.to_rust_code(var_pool))
             .collect::<Vec<_>>()
             .join(", ");
         format!("{}({})", self.inst.name, args)
     }
 
-    fn inst_predicate(
+    pub fn inst_predicate(
         &self,
         format_registry: &FormatRegistry,
         var_pool: &VarPool,
     ) -> InstructionPredicate {
         let iform = format_registry.get(self.inst.format);
 
         let mut pred = InstructionPredicate::new();
         for (format_field, &op_num) in iform.imm_fields.iter().zip(self.inst.imm_opnums.iter()) {
             let arg = &self.args[op_num];
             if arg.maybe_var().is_some() {
                 // Ignore free variables for now.
                 continue;
             }
-            pred = pred.and(InstructionPredicate::new_is_field_equal(
+            pred = pred.and(InstructionPredicate::new_is_field_equal_ast(
+                iform,
                 &format_field,
                 arg.to_rust_code(var_pool),
             ));
         }
 
         // Add checks for any bound secondary type variables.  We can't check the controlling type
         // variable this way since it may not appear as the type of an operand.
         if self.value_types.len() > 1 {
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/cpu_modes.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/cpu_modes.rs
@@ -1,74 +1,72 @@
-use crate::cdsl::types::LaneType;
-use crate::cdsl::xform::{TransformGroup, TransformGroupIndex, TransformGroups};
-
-use std::collections::{HashMap, HashSet};
+use std::collections::{hash_map, HashMap, HashSet};
 use std::iter::FromIterator;
 
+use crate::cdsl::encodings::Encoding;
+use crate::cdsl::types::{LaneType, ValueType};
+use crate::cdsl::xform::{TransformGroup, TransformGroupIndex};
+
 pub struct CpuMode {
-    _name: &'static str,
+    pub name: &'static str,
     default_legalize: Option<TransformGroupIndex>,
     monomorphic_legalize: Option<TransformGroupIndex>,
-    typed_legalize: HashMap<String, TransformGroupIndex>,
+    typed_legalize: HashMap<ValueType, TransformGroupIndex>,
+    pub encodings: Vec<Encoding>,
 }
 
 impl CpuMode {
     pub fn new(name: &'static str) -> Self {
         Self {
-            _name: name,
+            name,
             default_legalize: None,
             monomorphic_legalize: None,
             typed_legalize: HashMap::new(),
+            encodings: Vec::new(),
         }
     }
+
+    pub fn set_encodings(&mut self, encodings: Vec<Encoding>) {
+        assert!(self.encodings.is_empty(), "clobbering encodings");
+        self.encodings = encodings;
+    }
+
     pub fn legalize_monomorphic(&mut self, group: &TransformGroup) {
         assert!(self.monomorphic_legalize.is_none());
         self.monomorphic_legalize = Some(group.id);
     }
     pub fn legalize_default(&mut self, group: &TransformGroup) {
         assert!(self.default_legalize.is_none());
         self.default_legalize = Some(group.id);
     }
     pub fn legalize_type(&mut self, lane_type: impl Into<LaneType>, group: &TransformGroup) {
         assert!(self
             .typed_legalize
-            .insert(lane_type.into().to_string(), group.id)
+            .insert(lane_type.into().into(), group.id)
             .is_none());
     }
 
-    /// Returns a deterministically ordered, deduplicated list of TransformGroupIndex for the
-    /// transitive set of TransformGroup this TargetIsa uses.
-    pub fn transitive_transform_groups(
-        &self,
-        all_groups: &TransformGroups,
-    ) -> Vec<TransformGroupIndex> {
-        let mut roots = Vec::new();
-        if let Some(i) = &self.default_legalize {
-            roots.push(*i);
-        }
-        if let Some(i) = &self.monomorphic_legalize {
-            roots.push(*i);
+    pub fn get_default_legalize_code(&self) -> TransformGroupIndex {
+        self.default_legalize
+            .expect("a finished CpuMode must have a default legalize code")
+    }
+    pub fn get_legalize_code_for(&self, typ: &Option<ValueType>) -> TransformGroupIndex {
+        match typ {
+            Some(typ) => self
+                .typed_legalize
+                .get(typ)
+                .map(|x| *x)
+                .unwrap_or_else(|| self.get_default_legalize_code()),
+            None => self
+                .monomorphic_legalize
+                .unwrap_or_else(|| self.get_default_legalize_code()),
         }
-        roots.extend(self.typed_legalize.values().cloned());
-
-        let mut set = HashSet::new();
-        for root in roots {
-            set.insert(root);
-            let mut base = root;
-            // Follow the chain of chain_with.
-            while let Some(chain_with) = &all_groups.get(base).chain_with {
-                set.insert(*chain_with);
-                base = *chain_with;
-            }
-        }
-
-        let mut ret = Vec::from_iter(set);
-        ret.sort();
-        ret
+    }
+    pub fn get_legalized_types(&self) -> hash_map::Keys<ValueType, TransformGroupIndex> {
+        self.typed_legalize.keys()
     }
 
     /// Returns a deterministically ordered, deduplicated list of TransformGroupIndex for the directly
     /// reachable set of TransformGroup this TargetIsa uses.
     pub fn direct_transform_groups(&self) -> Vec<TransformGroupIndex> {
         let mut set = HashSet::new();
         if let Some(i) = &self.default_legalize {
             set.insert(*i);
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/encodings.rs
@@ -0,0 +1,160 @@
+use std::rc::Rc;
+
+use crate::cdsl::instructions::{
+    InstSpec, Instruction, InstructionPredicate, InstructionPredicateNode,
+    InstructionPredicateNumber, InstructionPredicateRegistry, ValueTypeOrAny,
+};
+use crate::cdsl::recipes::{EncodingRecipeNumber, Recipes};
+use crate::cdsl::settings::SettingPredicateNumber;
+use crate::cdsl::types::ValueType;
+
+/// Encoding for a concrete instruction.
+///
+/// An `Encoding` object ties an instruction opcode with concrete type variables together with an
+/// encoding recipe and encoding encbits.
+///
+/// The concrete instruction can be in three different forms:
+///
+/// 1. A naked opcode: `trap` for non-polymorphic instructions.
+/// 2. With bound type variables: `iadd.i32` for polymorphic instructions.
+/// 3. With operands providing constraints: `icmp.i32(intcc.eq, x, y)`.
+///
+/// If the instruction is polymorphic, all type variables must be provided.
+pub struct EncodingContent {
+    /// The `Instruction` or `BoundInstruction` being encoded.
+    inst: InstSpec,
+
+    /// The `EncodingRecipe` to use.
+    pub recipe: EncodingRecipeNumber,
+
+    /// Additional encoding bits to be interpreted by `recipe`.
+    pub encbits: u16,
+
+    /// An instruction predicate that must be true to allow selecting this encoding.
+    pub inst_predicate: Option<InstructionPredicateNumber>,
+
+    /// An ISA predicate that must be true to allow selecting this encoding.
+    pub isa_predicate: Option<SettingPredicateNumber>,
+
+    /// The value type this encoding has been bound to, for encodings of polymorphic instructions.
+    pub bound_type: Option<ValueType>,
+}
+
+impl EncodingContent {
+    pub fn inst(&self) -> &Instruction {
+        self.inst.inst()
+    }
+    pub fn to_rust_comment(&self, recipes: &Recipes) -> String {
+        format!("[{}#{:02x}]", recipes[self.recipe].name, self.encbits)
+    }
+}
+
+pub type Encoding = Rc<EncodingContent>;
+
+pub struct EncodingBuilder {
+    inst: InstSpec,
+    recipe: EncodingRecipeNumber,
+    encbits: u16,
+    inst_predicate: Option<InstructionPredicate>,
+    isa_predicate: Option<SettingPredicateNumber>,
+    bound_type: Option<ValueType>,
+}
+
+impl EncodingBuilder {
+    pub fn new(inst: InstSpec, recipe: EncodingRecipeNumber, encbits: u16) -> Self {
+        let (inst_predicate, bound_type) = match &inst {
+            InstSpec::Bound(inst) => {
+                let other_typevars = &inst.inst.polymorphic_info.as_ref().unwrap().other_typevars;
+
+                assert!(
+                    inst.value_types.len() == other_typevars.len() + 1,
+                    "partially bound polymorphic instruction"
+                );
+
+                // Add secondary type variables to the instruction predicate.
+                let value_types = &inst.value_types;
+                let mut inst_predicate = None;
+                for (typevar, value_type) in other_typevars.iter().zip(value_types.iter().skip(1)) {
+                    let value_type = match value_type {
+                        ValueTypeOrAny::Any => continue,
+                        ValueTypeOrAny::ValueType(vt) => vt,
+                    };
+                    let type_predicate =
+                        InstructionPredicate::new_typevar_check(&inst.inst, typevar, value_type);
+                    inst_predicate = Some(type_predicate.into());
+                }
+
+                let ctrl_type = value_types[0]
+                    .clone()
+                    .expect("Controlling type shouldn't be Any");
+                (inst_predicate, Some(ctrl_type))
+            }
+
+            InstSpec::Inst(inst) => {
+                assert!(
+                    inst.polymorphic_info.is_none(),
+                    "unbound polymorphic instruction"
+                );
+                (None, None)
+            }
+        };
+
+        Self {
+            inst,
+            recipe,
+            encbits,
+            inst_predicate,
+            isa_predicate: None,
+            bound_type,
+        }
+    }
+
+    pub fn inst_predicate(mut self, inst_predicate: InstructionPredicateNode) -> Self {
+        let inst_predicate = Some(match self.inst_predicate {
+            Some(node) => node.and(inst_predicate),
+            None => inst_predicate.into(),
+        });
+        self.inst_predicate = inst_predicate;
+        self
+    }
+
+    pub fn isa_predicate(mut self, isa_predicate: SettingPredicateNumber) -> Self {
+        assert!(self.isa_predicate.is_none());
+        self.isa_predicate = Some(isa_predicate);
+        self
+    }
+
+    pub fn build(
+        self,
+        recipes: &Recipes,
+        inst_pred_reg: &mut InstructionPredicateRegistry,
+    ) -> Encoding {
+        let inst_predicate = self.inst_predicate.map(|pred| inst_pred_reg.insert(pred));
+
+        let inst = self.inst.inst();
+        assert!(
+            inst.format == recipes[self.recipe].format,
+            format!(
+                "Inst {} and recipe {} must have the same format!",
+                inst.name, recipes[self.recipe].name
+            )
+        );
+
+        assert_eq!(
+            inst.is_branch && !inst.is_indirect_branch,
+            recipes[self.recipe].branch_range.is_some(),
+            "Inst {}'s is_branch contradicts recipe {} branch_range!",
+            inst.name,
+            recipes[self.recipe].name
+        );
+
+        Rc::new(EncodingContent {
+            inst: self.inst,
+            recipe: self.recipe,
+            encbits: self.encbits,
+            inst_predicate,
+            isa_predicate: self.isa_predicate,
+            bound_type: self.bound_type,
+        })
+    }
+}
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/formats.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/formats.rs
@@ -62,16 +62,30 @@ impl fmt::Display for InstructionFormat 
         fmt.write_fmt(format_args!(
             "{}(imms=({}), vals={})",
             self.name, args, self.num_value_operands
         ))?;
         Ok(())
     }
 }
 
+impl InstructionFormat {
+    pub fn imm_by_name(&self, name: &'static str) -> &FormatField {
+        self.imm_fields
+            .iter()
+            .find(|&field| field.member == name)
+            .unwrap_or_else(|| {
+                panic!(
+                    "unexpected immediate field named {} in instruction format {}",
+                    name, self.name
+                )
+            })
+    }
+}
+
 pub struct InstructionFormatBuilder {
     name: &'static str,
     num_value_operands: usize,
     has_value_list: bool,
     imm_fields: Vec<FormatField>,
     typevar_operand: Option<usize>,
 }
 
@@ -195,16 +209,24 @@ impl FormatRegistry {
 
         let sig = (imm_keys, num_values, has_varargs);
         *self
             .sig_to_index
             .get(&sig)
             .expect("unknown InstructionFormat; please define it in shared/formats.rs first")
     }
 
+    pub fn by_name(&self, name: &str) -> InstructionFormatIndex {
+        self.map
+            .iter()
+            .find(|(_key, value)| value.name == name)
+            .unwrap_or_else(|| panic!("format with name '{}' doesn't exist", name))
+            .0
+    }
+
     pub fn get(&self, index: InstructionFormatIndex) -> &InstructionFormat {
         self.map.get(index).unwrap()
     }
 
     pub fn insert(&mut self, inst_format: InstructionFormatBuilder) {
         let name = &inst_format.name;
         if !self.name_set.insert(name) {
             panic!(
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/instructions.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/instructions.rs
@@ -1,52 +1,68 @@
+use cranelift_entity::{entity_impl, PrimaryMap};
+
+use std::collections::HashMap;
+use std::fmt;
+use std::ops;
+use std::rc::Rc;
+use std::slice;
+
 use crate::cdsl::camel_case;
 use crate::cdsl::formats::{
     FormatField, FormatRegistry, InstructionFormat, InstructionFormatIndex,
 };
 use crate::cdsl::operands::Operand;
 use crate::cdsl::type_inference::Constraint;
-use crate::cdsl::types::{LaneType, ValueType};
+use crate::cdsl::types::{LaneType, ValueType, VectorType};
 use crate::cdsl::typevar::TypeVar;
 
-use std::fmt;
-use std::ops;
-use std::rc::Rc;
-use std::slice;
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct OpcodeNumber(u32);
+entity_impl!(OpcodeNumber);
 
-pub struct InstructionGroupBuilder<'format_reg> {
+pub type AllInstructions = PrimaryMap<OpcodeNumber, Instruction>;
+
+pub struct InstructionGroupBuilder<'format_reg, 'all_inst> {
     _name: &'static str,
     _doc: &'static str,
     format_registry: &'format_reg FormatRegistry,
-    instructions: Vec<Instruction>,
+    all_instructions: &'all_inst mut AllInstructions,
+    own_instructions: Vec<Instruction>,
 }
 
-impl<'format_reg> InstructionGroupBuilder<'format_reg> {
+impl<'format_reg, 'all_inst> InstructionGroupBuilder<'format_reg, 'all_inst> {
     pub fn new(
         name: &'static str,
         doc: &'static str,
+        all_instructions: &'all_inst mut AllInstructions,
         format_registry: &'format_reg FormatRegistry,
     ) -> Self {
         Self {
             _name: name,
             _doc: doc,
             format_registry,
-            instructions: Vec::new(),
+            all_instructions,
+            own_instructions: Vec::new(),
         }
     }
 
     pub fn push(&mut self, builder: InstructionBuilder) {
-        self.instructions.push(builder.build(self.format_registry));
+        let opcode_number = OpcodeNumber(self.all_instructions.next_key().as_u32());
+        let inst = builder.build(self.format_registry, opcode_number);
+        // Note this clone is cheap, since Instruction is a Rc<> wrapper for InstructionContent.
+        self.own_instructions.push(inst.clone());
+        self.all_instructions.push(inst);
     }
 
     pub fn build(self) -> InstructionGroup {
         InstructionGroup {
             _name: self._name,
             _doc: self._doc,
-            instructions: self.instructions,
+            instructions: self.own_instructions,
         }
     }
 }
 
 /// Every instruction must belong to exactly one instruction group. A given
 /// target architecture can support instructions from multiple groups, and it
 /// does not necessarily support all instructions in a group.
 pub struct InstructionGroup {
@@ -73,19 +89,20 @@ pub struct PolymorphicInfo {
     pub ctrl_typevar: TypeVar,
     pub other_typevars: Vec<TypeVar>,
 }
 
 pub struct InstructionContent {
     /// Instruction mnemonic, also becomes opcode name.
     pub name: String,
     pub camel_name: String,
+    pub opcode_number: OpcodeNumber,
 
     /// Documentation string.
-    doc: String,
+    pub doc: String,
 
     /// Input operands. This can be a mix of SSA value operands and other operand kinds.
     pub operands_in: Vec<Operand>,
     /// Output operands. The output operands must be SSA values or `variable_args`.
     pub operands_out: Vec<Operand>,
     /// Instruction-specific TypeConstraints.
     pub constraints: Vec<Constraint>,
 
@@ -140,39 +157,37 @@ impl Instruction {
     pub fn snake_name(&self) -> &str {
         if self.name == "return" {
             "return_"
         } else {
             &self.name
         }
     }
 
-    pub fn doc_comment_first_line(&self) -> &str {
-        for line in self.doc.split("\n") {
-            let stripped = line.trim();
-            if stripped.len() > 0 {
-                return stripped;
-            }
-        }
-        ""
-    }
-
     pub fn all_typevars(&self) -> Vec<&TypeVar> {
         match &self.polymorphic_info {
             Some(poly) => {
                 let mut result = vec![&poly.ctrl_typevar];
                 result.extend(&poly.other_typevars);
                 result
             }
             None => Vec::new(),
         }
     }
 
     pub fn bind(&self, lane_type: impl Into<LaneType>) -> BoundInstruction {
-        bind(self.clone(), lane_type.into(), Vec::new())
+        bind(self.clone(), Some(lane_type.into()), Vec::new())
+    }
+
+    pub fn bind_vector(&self, lane_type: impl Into<LaneType>, num_lanes: u64) -> BoundInstruction {
+        bind_vector(self.clone(), lane_type.into(), num_lanes, Vec::new())
+    }
+
+    pub fn bind_any(&self) -> BoundInstruction {
+        bind(self.clone(), None, Vec::new())
     }
 }
 
 impl fmt::Display for Instruction {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
         if self.operands_out.len() > 0 {
             let operands_out = self
                 .operands_out
@@ -295,17 +310,17 @@ impl InstructionBuilder {
         self.can_trap = val;
         self
     }
     pub fn other_side_effects(mut self, val: bool) -> Self {
         self.other_side_effects = val;
         self
     }
 
-    fn build(self, format_registry: &FormatRegistry) -> Instruction {
+    fn build(self, format_registry: &FormatRegistry, opcode_number: OpcodeNumber) -> Instruction {
         let operands_in = self.operands_in.unwrap_or_else(Vec::new);
         let operands_out = self.operands_out.unwrap_or_else(Vec::new);
 
         let format_index = format_registry.lookup(&operands_in);
 
         let mut value_opnums = Vec::new();
         let mut imm_opnums = Vec::new();
         for (i, op) in operands_in.iter().enumerate() {
@@ -328,20 +343,22 @@ impl InstructionBuilder {
         let format = format_registry.get(format_index);
         let polymorphic_info =
             verify_polymorphic(&operands_in, &operands_out, &format, &value_opnums);
 
         // Infer from output operands whether an instruciton clobbers CPU flags or not.
         let writes_cpu_flags = operands_out.iter().any(|op| op.is_cpu_flags());
 
         let camel_name = camel_case(&self.name);
+
         Instruction {
             content: Rc::new(InstructionContent {
                 name: self.name,
                 camel_name,
+                opcode_number,
                 doc: self.doc,
                 operands_in,
                 operands_out,
                 constraints: self.constraints.unwrap_or_else(Vec::new),
                 format: format_index,
                 polymorphic_info,
                 value_opnums,
                 value_results,
@@ -357,25 +374,49 @@ impl InstructionBuilder {
                 can_trap: self.can_trap,
                 other_side_effects: self.other_side_effects,
                 writes_cpu_flags,
             }),
         }
     }
 }
 
+/// A thin wrapper like Option<ValueType>, but with more precise semantics.
+#[derive(Clone)]
+pub enum ValueTypeOrAny {
+    ValueType(ValueType),
+    Any,
+}
+
+impl ValueTypeOrAny {
+    pub fn expect(self, msg: &str) -> ValueType {
+        match self {
+            ValueTypeOrAny::ValueType(vt) => vt,
+            ValueTypeOrAny::Any => panic!(format!("Unexpected Any: {}", msg)),
+        }
+    }
+}
+
 #[derive(Clone)]
 pub struct BoundInstruction {
     pub inst: Instruction,
-    pub value_types: Vec<ValueType>,
+    pub value_types: Vec<ValueTypeOrAny>,
 }
 
 impl BoundInstruction {
     pub fn bind(self, lane_type: impl Into<LaneType>) -> BoundInstruction {
-        bind(self.inst, lane_type.into(), self.value_types)
+        bind(self.inst, Some(lane_type.into()), self.value_types)
+    }
+
+    pub fn bind_vector(self, lane_type: impl Into<LaneType>, num_lanes: u64) -> BoundInstruction {
+        bind_vector(self.inst, lane_type.into(), num_lanes, self.value_types)
+    }
+
+    pub fn bind_any(self) -> BoundInstruction {
+        bind(self.inst, None, self.value_types)
     }
 }
 
 /// Check if this instruction is polymorphic, and verify its use of type variables.
 fn verify_polymorphic(
     operands_in: &Vec<Operand>,
     operands_out: &Vec<Operand>,
     format: &InstructionFormat,
@@ -528,128 +569,485 @@ fn verify_ctrl_typevar(
         }
 
         return Err("type variable in output not derived from ctrl_typevar".into());
     }
 
     Ok(other_typevars)
 }
 
-/// A basic node in an instruction predicate: either an atom, or an AND of two conditions.
-pub enum InstructionPredicateNode {
-    /// Is the field member (first member) equal to the actual argument (which name is the second
-    /// field)?
-    IsFieldEqual(String, String),
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub enum FormatPredicateKind {
+    /// Is the field member equal to the expected value (stored here)?
+    IsEqual(String),
+
+    /// Is the immediate instruction format field representable as an n-bit two's complement
+    /// integer? (with width: first member, scale: second member).
+    /// The predicate is true if the field is in the range: `-2^(width-1) -- 2^(width-1)-1` and a
+    /// multiple of `2^scale`.
+    IsSignedInt(usize, usize),
+
+    /// Is the immediate instruction format field representable as an n-bit unsigned integer? (with
+    /// width: first member, scale: second member).
+    /// The predicate is true if the field is in the range: `0 -- 2^width - 1` and a multiple of
+    /// `2^scale`.
+    IsUnsignedInt(usize, usize),
+
+    /// Is the immediate format field member equal to zero? (float32 version)
+    IsZero32BitFloat,
+
+    /// Is the immediate format field member equal to zero? (float64 version)
+    IsZero64BitFloat,
+
+    /// Has the value list (in member_name) the size specified in parameter?
+    LengthEquals(usize),
+
+    /// Is the referenced function colocated?
+    IsColocatedFunc,
+
+    /// Is the referenced data object colocated?
+    IsColocatedData,
+}
+
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub struct FormatPredicateNode {
+    format_name: &'static str,
+    member_name: &'static str,
+    kind: FormatPredicateKind,
+}
+
+impl FormatPredicateNode {
+    fn new(
+        format: &InstructionFormat,
+        field_name: &'static str,
+        kind: FormatPredicateKind,
+    ) -> Self {
+        let member_name = format.imm_by_name(field_name).member;
+        Self {
+            format_name: format.name,
+            member_name,
+            kind,
+        }
+    }
 
+    fn new_raw(
+        format: &InstructionFormat,
+        member_name: &'static str,
+        kind: FormatPredicateKind,
+    ) -> Self {
+        Self {
+            format_name: format.name,
+            member_name,
+            kind,
+        }
+    }
+
+    fn destructuring_member_name(&self) -> &'static str {
+        match &self.kind {
+            FormatPredicateKind::LengthEquals(_) => {
+                // Length operates on the argument value list.
+                assert!(self.member_name == "args");
+                "ref args"
+            }
+            _ => self.member_name,
+        }
+    }
+
+    fn rust_predicate(&self) -> String {
+        match &self.kind {
+            FormatPredicateKind::IsEqual(arg) => {
+                format!("predicates::is_equal({}, {})", self.member_name, arg)
+            }
+            FormatPredicateKind::IsSignedInt(width, scale) => format!(
+                "predicates::is_signed_int({}, {}, {})",
+                self.member_name, width, scale
+            ),
+            FormatPredicateKind::IsUnsignedInt(width, scale) => format!(
+                "predicates::is_unsigned_int({}, {}, {})",
+                self.member_name, width, scale
+            ),
+            FormatPredicateKind::IsZero32BitFloat => {
+                format!("predicates::is_zero_32_bit_float({})", self.member_name)
+            }
+            FormatPredicateKind::IsZero64BitFloat => {
+                format!("predicates::is_zero_64_bit_float({})", self.member_name)
+            }
+            FormatPredicateKind::LengthEquals(num) => format!(
+                "predicates::has_length_of({}, {}, func)",
+                self.member_name, num
+            ),
+            FormatPredicateKind::IsColocatedFunc => {
+                format!("predicates::is_colocated_func({}, func)", self.member_name,)
+            }
+            FormatPredicateKind::IsColocatedData => {
+                format!("predicates::is_colocated_data({}, func)", self.member_name)
+            }
+        }
+    }
+}
+
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub enum TypePredicateNode {
     /// Is the value argument (at the index designated by the first member) the same type as the
     /// type name (second member)?
     TypeVarCheck(usize, String),
 
     /// Is the controlling type variable the same type as the one designated by the type name
     /// (only member)?
     CtrlTypeVarCheck(String),
+}
 
-    /// A combination of two other predicates.
+impl TypePredicateNode {
+    fn rust_predicate(&self) -> String {
+        match self {
+            TypePredicateNode::TypeVarCheck(index, value_type_name) => format!(
+                "func.dfg.value_type(args[{}]) == {}",
+                index, value_type_name
+            ),
+            TypePredicateNode::CtrlTypeVarCheck(value_type_name) => {
+                format!("func.dfg.ctrl_typevar(inst) == {}", value_type_name)
+            }
+        }
+    }
+}
+
+/// A basic node in an instruction predicate: either an atom, or an AND of two conditions.
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub enum InstructionPredicateNode {
+    FormatPredicate(FormatPredicateNode),
+
+    TypePredicate(TypePredicateNode),
+
+    /// An AND-combination of two or more other predicates.
     And(Vec<InstructionPredicateNode>),
+
+    /// An OR-combination of two or more other predicates.
+    Or(Vec<InstructionPredicateNode>),
 }
 
 impl InstructionPredicateNode {
     fn rust_predicate(&self) -> String {
         match self {
-            InstructionPredicateNode::IsFieldEqual(field_name, arg) => {
-                let new_args = vec![field_name.clone(), arg.clone()];
-                format!("crate::predicates::is_equal({})", new_args.join(", "))
-            }
-            InstructionPredicateNode::TypeVarCheck(index, value_type_name) => format!(
-                "func.dfg.value_type(args[{}]) == {}",
-                index, value_type_name
-            ),
-            InstructionPredicateNode::CtrlTypeVarCheck(value_type_name) => {
-                format!("func.dfg.ctrl_typevar(inst) == {}", value_type_name)
-            }
+            InstructionPredicateNode::FormatPredicate(node) => node.rust_predicate(),
+            InstructionPredicateNode::TypePredicate(node) => node.rust_predicate(),
             InstructionPredicateNode::And(nodes) => nodes
                 .iter()
                 .map(|x| x.rust_predicate())
                 .collect::<Vec<_>>()
-                .join(" &&\n"),
+                .join(" && "),
+            InstructionPredicateNode::Or(nodes) => nodes
+                .iter()
+                .map(|x| x.rust_predicate())
+                .collect::<Vec<_>>()
+                .join(" || "),
+        }
+    }
+
+    pub fn format_destructuring_member_name(&self) -> &str {
+        match self {
+            InstructionPredicateNode::FormatPredicate(format_pred) => {
+                format_pred.destructuring_member_name()
+            }
+            _ => panic!("Only for leaf format predicates"),
+        }
+    }
+
+    pub fn format_name(&self) -> &str {
+        match self {
+            InstructionPredicateNode::FormatPredicate(format_pred) => format_pred.format_name,
+            _ => panic!("Only for leaf format predicates"),
         }
     }
+
+    pub fn is_type_predicate(&self) -> bool {
+        match self {
+            InstructionPredicateNode::FormatPredicate(_)
+            | InstructionPredicateNode::And(_)
+            | InstructionPredicateNode::Or(_) => false,
+            InstructionPredicateNode::TypePredicate(_) => true,
+        }
+    }
+
+    fn collect_leaves(&self) -> Vec<&InstructionPredicateNode> {
+        let mut ret = Vec::new();
+        match self {
+            InstructionPredicateNode::And(nodes) | InstructionPredicateNode::Or(nodes) => {
+                for node in nodes {
+                    ret.extend(node.collect_leaves());
+                }
+            }
+            _ => ret.push(&self),
+        }
+        ret
+    }
 }
 
+#[derive(Clone, Hash, PartialEq, Eq)]
 pub struct InstructionPredicate {
     node: Option<InstructionPredicateNode>,
 }
 
+impl Into<InstructionPredicate> for InstructionPredicateNode {
+    fn into(self) -> InstructionPredicate {
+        InstructionPredicate { node: Some(self) }
+    }
+}
+
 impl InstructionPredicate {
     pub fn new() -> Self {
         Self { node: None }
     }
 
+    pub fn unwrap(self) -> InstructionPredicateNode {
+        self.node.unwrap()
+    }
+
     pub fn new_typevar_check(
         inst: &Instruction,
         type_var: &TypeVar,
         value_type: &ValueType,
     ) -> InstructionPredicateNode {
         let index = inst
             .value_opnums
             .iter()
             .enumerate()
             .filter(|(_, &op_num)| inst.operands_in[op_num].type_var().unwrap() == type_var)
             .next()
             .unwrap()
             .0;
-        InstructionPredicateNode::TypeVarCheck(index, value_type.rust_name())
+        InstructionPredicateNode::TypePredicate(TypePredicateNode::TypeVarCheck(
+            index,
+            value_type.rust_name(),
+        ))
+    }
+
+    pub fn new_ctrl_typevar_check(value_type: &ValueType) -> InstructionPredicateNode {
+        InstructionPredicateNode::TypePredicate(TypePredicateNode::CtrlTypeVarCheck(
+            value_type.rust_name(),
+        ))
     }
 
     pub fn new_is_field_equal(
-        format_field: &FormatField,
+        format: &InstructionFormat,
+        field_name: &'static str,
+        imm_value: String,
+    ) -> InstructionPredicateNode {
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new(
+            format,
+            field_name,
+            FormatPredicateKind::IsEqual(imm_value),
+        ))
+    }
+
+    /// Used only for the AST module, which directly passes in the format field.
+    pub fn new_is_field_equal_ast(
+        format: &InstructionFormat,
+        field: &FormatField,
         imm_value: String,
     ) -> InstructionPredicateNode {
-        InstructionPredicateNode::IsFieldEqual(format_field.member.into(), imm_value)
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new_raw(
+            format,
+            field.member,
+            FormatPredicateKind::IsEqual(imm_value),
+        ))
+    }
+
+    pub fn new_is_signed_int(
+        format: &InstructionFormat,
+        field_name: &'static str,
+        width: usize,
+        scale: usize,
+    ) -> InstructionPredicateNode {
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new(
+            format,
+            field_name,
+            FormatPredicateKind::IsSignedInt(width, scale),
+        ))
+    }
+
+    pub fn new_is_unsigned_int(
+        format: &InstructionFormat,
+        field_name: &'static str,
+        width: usize,
+        scale: usize,
+    ) -> InstructionPredicateNode {
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new(
+            format,
+            field_name,
+            FormatPredicateKind::IsUnsignedInt(width, scale),
+        ))
     }
 
-    pub fn new_ctrl_typevar_check(value_type: &ValueType) -> InstructionPredicateNode {
-        InstructionPredicateNode::CtrlTypeVarCheck(value_type.rust_name())
+    pub fn new_is_zero_32bit_float(
+        format: &InstructionFormat,
+        field_name: &'static str,
+    ) -> InstructionPredicateNode {
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new(
+            format,
+            field_name,
+            FormatPredicateKind::IsZero32BitFloat,
+        ))
+    }
+
+    pub fn new_is_zero_64bit_float(
+        format: &InstructionFormat,
+        field_name: &'static str,
+    ) -> InstructionPredicateNode {
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new(
+            format,
+            field_name,
+            FormatPredicateKind::IsZero64BitFloat,
+        ))
+    }
+
+    pub fn new_length_equals(format: &InstructionFormat, size: usize) -> InstructionPredicateNode {
+        assert!(
+            format.has_value_list,
+            "the format must be variadic in number of arguments"
+        );
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new_raw(
+            format,
+            "args",
+            FormatPredicateKind::LengthEquals(size),
+        ))
+    }
+
+    pub fn new_is_colocated_func(
+        format: &InstructionFormat,
+        field_name: &'static str,
+    ) -> InstructionPredicateNode {
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new(
+            format,
+            field_name,
+            FormatPredicateKind::IsColocatedFunc,
+        ))
+    }
+
+    pub fn new_is_colocated_data(format_registry: &FormatRegistry) -> InstructionPredicateNode {
+        let format = format_registry.get(format_registry.by_name("UnaryGlobalValue"));
+        InstructionPredicateNode::FormatPredicate(FormatPredicateNode::new(
+            format,
+            "global_value",
+            FormatPredicateKind::IsColocatedData,
+        ))
     }
 
     pub fn and(mut self, new_node: InstructionPredicateNode) -> Self {
         let node = self.node;
         let mut and_nodes = match node {
             Some(node) => match node {
                 InstructionPredicateNode::And(nodes) => nodes,
+                InstructionPredicateNode::Or(_) => {
+                    panic!("Can't mix and/or without implementing operator precedence!")
+                }
                 _ => vec![node],
             },
             _ => Vec::new(),
         };
         and_nodes.push(new_node);
         self.node = Some(InstructionPredicateNode::And(and_nodes));
         self
     }
 
+    pub fn or(mut self, new_node: InstructionPredicateNode) -> Self {
+        let node = self.node;
+        let mut or_nodes = match node {
+            Some(node) => match node {
+                InstructionPredicateNode::Or(nodes) => nodes,
+                InstructionPredicateNode::And(_) => {
+                    panic!("Can't mix and/or without implementing operator precedence!")
+                }
+                _ => vec![node],
+            },
+            _ => Vec::new(),
+        };
+        or_nodes.push(new_node);
+        self.node = Some(InstructionPredicateNode::Or(or_nodes));
+        self
+    }
+
     pub fn rust_predicate(&self) -> String {
         match &self.node {
             Some(root) => root.rust_predicate(),
             None => "true".into(),
         }
     }
+
+    /// Returns true if the predicate only depends on type parameters (and not on an instruction
+    /// format).
+    pub fn is_type_predicate(&self) -> bool {
+        self.node.as_ref().unwrap().is_type_predicate()
+    }
+
+    /// Returns references to all the nodes that are leaves in the condition (i.e. by flattening
+    /// AND/OR).
+    pub fn collect_leaves(&self) -> Vec<&InstructionPredicateNode> {
+        self.node.as_ref().unwrap().collect_leaves()
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct InstructionPredicateNumber(u32);
+entity_impl!(InstructionPredicateNumber);
+
+pub type InstructionPredicateMap = PrimaryMap<InstructionPredicateNumber, InstructionPredicate>;
+
+/// A registry of predicates to help deduplicating them, during Encodings construction. When the
+/// construction process is over, it needs to be extracted with `extract` and associated to the
+/// TargetIsa.
+pub struct InstructionPredicateRegistry {
+    /// Maps a predicate number to its actual predicate.
+    map: InstructionPredicateMap,
+
+    /// Inverse map: maps a predicate to its predicate number. This is used before inserting a
+    /// predicate, to check whether it already exists.
+    inverted_map: HashMap<InstructionPredicate, InstructionPredicateNumber>,
+}
+
+impl InstructionPredicateRegistry {
+    pub fn new() -> Self {
+        Self {
+            map: PrimaryMap::new(),
+            inverted_map: HashMap::new(),
+        }
+    }
+    pub fn insert(&mut self, predicate: InstructionPredicate) -> InstructionPredicateNumber {
+        match self.inverted_map.get(&predicate) {
+            Some(&found) => found,
+            None => {
+                let key = self.map.push(predicate.clone());
+                self.inverted_map.insert(predicate, key);
+                key
+            }
+        }
+    }
+    pub fn extract(self) -> InstructionPredicateMap {
+        self.map
+    }
 }
 
 /// An instruction specification, containing an instruction that has bound types or not.
 pub enum InstSpec {
     Inst(Instruction),
     Bound(BoundInstruction),
 }
 
 impl InstSpec {
     pub fn inst(&self) -> &Instruction {
         match &self {
             InstSpec::Inst(inst) => inst,
             InstSpec::Bound(bound_inst) => &bound_inst.inst,
         }
     }
+    pub fn bind(&self, lane_type: impl Into<LaneType>) -> BoundInstruction {
+        match self {
+            InstSpec::Inst(inst) => inst.bind(lane_type),
+            InstSpec::Bound(inst) => inst.clone().bind(lane_type),
+        }
+    }
 }
 
 impl Into<InstSpec> for &Instruction {
     fn into(self) -> InstSpec {
         InstSpec::Inst(self.clone())
     }
 }
 
@@ -657,28 +1055,55 @@ impl Into<InstSpec> for BoundInstruction
     fn into(self) -> InstSpec {
         InstSpec::Bound(self)
     }
 }
 
 /// Helper bind reused by {Bound,}Instruction::bind.
 fn bind(
     inst: Instruction,
+    lane_type: Option<LaneType>,
+    mut value_types: Vec<ValueTypeOrAny>,
+) -> BoundInstruction {
+    match lane_type {
+        Some(lane_type) => {
+            value_types.push(ValueTypeOrAny::ValueType(lane_type.into()));
+        }
+        None => {
+            value_types.push(ValueTypeOrAny::Any);
+        }
+    }
+
+    verify_polymorphic_binding(&inst, &value_types);
+
+    BoundInstruction { inst, value_types }
+}
+
+/// Helper bind for vector types reused by {Bound,}Instruction::bind.
+fn bind_vector(
+    inst: Instruction,
     lane_type: LaneType,
-    mut value_types: Vec<ValueType>,
+    num_lanes: u64,
+    mut value_types: Vec<ValueTypeOrAny>,
 ) -> BoundInstruction {
-    value_types.push(ValueType::from(lane_type));
+    let vector_type = ValueType::Vector(VectorType::new(lane_type, num_lanes));
+    value_types.push(ValueTypeOrAny::ValueType(vector_type));
+    verify_polymorphic_binding(&inst, &value_types);
+    BoundInstruction { inst, value_types }
+}
+
+/// Helper to verify that binding types to the instruction does not violate polymorphic rules
+fn verify_polymorphic_binding(inst: &Instruction, value_types: &Vec<ValueTypeOrAny>) {
     match &inst.polymorphic_info {
         Some(poly) => {
             assert!(
                 value_types.len() <= 1 + poly.other_typevars.len(),
                 format!("trying to bind too many types for {}", inst.name)
             );
         }
         None => {
             panic!(format!(
                 "trying to bind a type for {} which is not a polymorphic instruction",
                 inst.name
             ));
         }
     }
-    BoundInstruction { inst, value_types }
 }
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/isa.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/isa.rs
@@ -1,61 +1,99 @@
+use std::collections::HashSet;
+use std::iter::FromIterator;
+
 use crate::cdsl::cpu_modes::CpuMode;
-use crate::cdsl::instructions::InstructionGroup;
+use crate::cdsl::instructions::{InstructionGroup, InstructionPredicateMap};
+use crate::cdsl::recipes::Recipes;
 use crate::cdsl::regs::IsaRegs;
 use crate::cdsl::settings::SettingGroup;
 use crate::cdsl::xform::{TransformGroupIndex, TransformGroups};
 
-use std::collections::HashSet;
-use std::iter::FromIterator;
-
 pub struct TargetIsa {
     pub name: &'static str,
     pub instructions: InstructionGroup,
     pub settings: SettingGroup,
     pub regs: IsaRegs,
+    pub recipes: Recipes,
     pub cpu_modes: Vec<CpuMode>,
+    pub encodings_predicates: InstructionPredicateMap,
+
+    /// TransformGroupIndex are global to all the ISAs, while we want to have indices into the
+    /// local array of transform groups that are directly used. We use this map to get this
+    /// information.
+    pub local_transform_groups: Vec<TransformGroupIndex>,
 }
 
 impl TargetIsa {
     pub fn new(
         name: &'static str,
         instructions: InstructionGroup,
         settings: SettingGroup,
         regs: IsaRegs,
+        recipes: Recipes,
         cpu_modes: Vec<CpuMode>,
+        encodings_predicates: InstructionPredicateMap,
     ) -> Self {
+        // Compute the local TransformGroup index.
+        let mut local_transform_groups = Vec::new();
+        for cpu_mode in &cpu_modes {
+            let transform_groups = cpu_mode.direct_transform_groups();
+            for group_index in transform_groups {
+                // find() is fine here: the number of transform group is < 5 as of June 2019.
+                if local_transform_groups
+                    .iter()
+                    .find(|&val| group_index == *val)
+                    .is_none()
+                {
+                    local_transform_groups.push(group_index);
+                }
+            }
+        }
+
         Self {
             name,
             instructions,
             settings,
             regs,
+            recipes,
             cpu_modes,
+            encodings_predicates,
+            local_transform_groups,
         }
     }
 
     /// Returns a deterministically ordered, deduplicated list of TransformGroupIndex for the
     /// transitive set of TransformGroup this TargetIsa uses.
     pub fn transitive_transform_groups(
         &self,
         all_groups: &TransformGroups,
     ) -> Vec<TransformGroupIndex> {
         let mut set = HashSet::new();
-        for cpu_mode in &self.cpu_modes {
-            set.extend(cpu_mode.transitive_transform_groups(all_groups));
+
+        for &root in self.local_transform_groups.iter() {
+            set.insert(root);
+            let mut base = root;
+            // Follow the chain of chain_with.
+            while let Some(chain_with) = &all_groups.get(base).chain_with {
+                set.insert(*chain_with);
+                base = *chain_with;
+            }
         }
+
         let mut vec = Vec::from_iter(set);
         vec.sort();
         vec
     }
 
     /// Returns a deterministically ordered, deduplicated list of TransformGroupIndex for the directly
     /// reachable set of TransformGroup this TargetIsa uses.
-    pub fn direct_transform_groups(&self) -> Vec<TransformGroupIndex> {
-        let mut set = HashSet::new();
-        for cpu_mode in &self.cpu_modes {
-            set.extend(cpu_mode.direct_transform_groups());
-        }
-        let mut vec = Vec::from_iter(set);
-        vec.sort();
-        vec
+    pub fn direct_transform_groups(&self) -> &Vec<TransformGroupIndex> {
+        &self.local_transform_groups
+    }
+
+    pub fn translate_group_index(&self, group_index: TransformGroupIndex) -> usize {
+        self.local_transform_groups
+            .iter()
+            .position(|&val| val == group_index)
+            .expect("TransformGroup unused by this TargetIsa!")
     }
 }
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/mod.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/mod.rs
@@ -1,20 +1,22 @@
 //! Cranelift DSL classes.
 //!
 //! This module defines the classes that are used to define Cranelift
 //! instructions and other entities.
 
 #[macro_use]
 pub mod ast;
 pub mod cpu_modes;
+pub mod encodings;
 pub mod formats;
 pub mod instructions;
 pub mod isa;
 pub mod operands;
+pub mod recipes;
 pub mod regs;
 pub mod settings;
 pub mod type_inference;
 pub mod types;
 pub mod typevar;
 pub mod xform;
 
 /// A macro that converts boolean settings into predicates to look more natural.
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/recipes.rs
@@ -0,0 +1,297 @@
+use cranelift_entity::{entity_impl, PrimaryMap};
+
+use crate::cdsl::formats::{FormatRegistry, InstructionFormatIndex};
+use crate::cdsl::instructions::InstructionPredicate;
+use crate::cdsl::regs::RegClassIndex;
+use crate::cdsl::settings::SettingPredicateNumber;
+
+/// A specific register in a register class.
+///
+/// A register is identified by the top-level register class it belongs to and
+/// its first register unit.
+///
+/// Specific registers are used to describe constraints on instructions where
+/// some operands must use a fixed register.
+///
+/// Register instances can be created with the constructor, or accessed as
+/// attributes on the register class: `GPR.rcx`.
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Register {
+    pub regclass: RegClassIndex,
+    pub unit: u8,
+}
+
+impl Register {
+    pub fn new(regclass: RegClassIndex, unit: u8) -> Self {
+        Self { regclass, unit }
+    }
+}
+
+/// An operand that must be in a stack slot.
+///
+/// A `Stack` object can be used to indicate an operand constraint for a value
+/// operand that must live in a stack slot.
+#[derive(Copy, Clone, Hash, PartialEq)]
+pub struct Stack {
+    pub regclass: RegClassIndex,
+}
+
+impl Stack {
+    pub fn new(regclass: RegClassIndex) -> Self {
+        Self { regclass }
+    }
+    pub fn stack_base_mask(&self) -> &'static str {
+        // TODO: Make this configurable instead of just using the SP.
+        "StackBaseMask(1)"
+    }
+}
+
+#[derive(Clone, Hash, PartialEq)]
+pub struct BranchRange {
+    pub inst_size: u64,
+    pub range: u64,
+}
+
+#[derive(Copy, Clone, Hash, PartialEq)]
+pub enum OperandConstraint {
+    RegClass(RegClassIndex),
+    FixedReg(Register),
+    TiedInput(usize),
+    Stack(Stack),
+}
+
+impl Into<OperandConstraint> for RegClassIndex {
+    fn into(self) -> OperandConstraint {
+        OperandConstraint::RegClass(self)
+    }
+}
+
+impl Into<OperandConstraint> for Register {
+    fn into(self) -> OperandConstraint {
+        OperandConstraint::FixedReg(self)
+    }
+}
+
+impl Into<OperandConstraint> for usize {
+    fn into(self) -> OperandConstraint {
+        OperandConstraint::TiedInput(self)
+    }
+}
+
+impl Into<OperandConstraint> for Stack {
+    fn into(self) -> OperandConstraint {
+        OperandConstraint::Stack(self)
+    }
+}
+
+/// A recipe for encoding instructions with a given format.
+///
+/// Many different instructions can be encoded by the same recipe, but they
+/// must all have the same instruction format.
+///
+/// The `operands_in` and `operands_out` arguments are tuples specifying the register
+/// allocation constraints for the value operands and results respectively. The
+/// possible constraints for an operand are:
+///
+/// - A `RegClass` specifying the set of allowed registers.
+/// - A `Register` specifying a fixed-register operand.
+/// - An integer indicating that this result is tied to a value operand, so
+///   they must use the same register.
+/// - A `Stack` specifying a value in a stack slot.
+///
+/// The `branch_range` argument must be provided for recipes that can encode
+/// branch instructions. It is an `(origin, bits)` tuple describing the exact
+/// range that can be encoded in a branch instruction.
+#[derive(Clone)]
+pub struct EncodingRecipe {
+    /// Short mnemonic name for this recipe.
+    pub name: String,
+
+    /// Associated instruction format.
+    pub format: InstructionFormatIndex,
+
+    /// Base number of bytes in the binary encoded instruction.
+    pub base_size: u64,
+
+    /// Tuple of register constraints for value operands.
+    pub operands_in: Vec<OperandConstraint>,
+
+    /// Tuple of register constraints for results.
+    pub operands_out: Vec<OperandConstraint>,
+
+    /// Function name to use when computing actual size.
+    pub compute_size: &'static str,
+
+    /// `(origin, bits)` range for branches.
+    pub branch_range: Option<BranchRange>,
+
+    /// This instruction clobbers `iflags` and `fflags`; true by default.
+    pub clobbers_flags: bool,
+
+    /// Instruction predicate.
+    pub inst_predicate: Option<InstructionPredicate>,
+
+    /// ISA predicate.
+    pub isa_predicate: Option<SettingPredicateNumber>,
+
+    /// Rust code for binary emission.
+    pub emit: Option<String>,
+}
+
+// Implement PartialEq ourselves: take all the fields into account but the name.
+impl PartialEq for EncodingRecipe {
+    fn eq(&self, other: &Self) -> bool {
+        self.format == other.format
+            && self.base_size == other.base_size
+            && self.operands_in == other.operands_in
+            && self.operands_out == other.operands_out
+            && self.compute_size == other.compute_size
+            && self.branch_range == other.branch_range
+            && self.clobbers_flags == other.clobbers_flags
+            && self.inst_predicate == other.inst_predicate
+            && self.isa_predicate == other.isa_predicate
+            && self.emit == other.emit
+    }
+}
+
+// To allow using it in a hashmap.
+impl Eq for EncodingRecipe {}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct EncodingRecipeNumber(u32);
+entity_impl!(EncodingRecipeNumber);
+
+pub type Recipes = PrimaryMap<EncodingRecipeNumber, EncodingRecipe>;
+
+#[derive(Clone)]
+pub struct EncodingRecipeBuilder {
+    pub name: String,
+    format: InstructionFormatIndex,
+    pub base_size: u64,
+    pub operands_in: Option<Vec<OperandConstraint>>,
+    pub operands_out: Option<Vec<OperandConstraint>>,
+    compute_size: Option<&'static str>,
+    pub branch_range: Option<BranchRange>,
+    pub emit: Option<String>,
+    clobbers_flags: Option<bool>,
+    inst_predicate: Option<InstructionPredicate>,
+    isa_predicate: Option<SettingPredicateNumber>,
+}
+
+impl EncodingRecipeBuilder {
+    pub fn new(name: impl Into<String>, format: InstructionFormatIndex, base_size: u64) -> Self {
+        Self {
+            name: name.into(),
+            format,
+            base_size,
+            operands_in: None,
+            operands_out: None,
+            compute_size: None,
+            branch_range: None,
+            emit: None,
+            clobbers_flags: None,
+            inst_predicate: None,
+            isa_predicate: None,
+        }
+    }
+
+    // Setters.
+    pub fn operands_in(mut self, constraints: Vec<impl Into<OperandConstraint>>) -> Self {
+        assert!(self.operands_in.is_none());
+        self.operands_in = Some(
+            constraints
+                .into_iter()
+                .map(|constr| constr.into())
+                .collect(),
+        );
+        self
+    }
+    pub fn operands_out(mut self, constraints: Vec<impl Into<OperandConstraint>>) -> Self {
+        assert!(self.operands_out.is_none());
+        self.operands_out = Some(
+            constraints
+                .into_iter()
+                .map(|constr| constr.into())
+                .collect(),
+        );
+        self
+    }
+    pub fn clobbers_flags(mut self, flag: bool) -> Self {
+        assert!(self.clobbers_flags.is_none());
+        self.clobbers_flags = Some(flag);
+        self
+    }
+    pub fn emit(mut self, code: impl Into<String>) -> Self {
+        assert!(self.emit.is_none());
+        self.emit = Some(code.into());
+        self
+    }
+    pub fn branch_range(mut self, range: (u64, u64)) -> Self {
+        assert!(self.branch_range.is_none());
+        self.branch_range = Some(BranchRange {
+            inst_size: range.0,
+            range: range.1,
+        });
+        self
+    }
+    pub fn isa_predicate(mut self, pred: SettingPredicateNumber) -> Self {
+        assert!(self.isa_predicate.is_none());
+        self.isa_predicate = Some(pred);
+        self
+    }
+    pub fn inst_predicate(mut self, inst_predicate: impl Into<InstructionPredicate>) -> Self {
+        assert!(self.inst_predicate.is_none());
+        self.inst_predicate = Some(inst_predicate.into());
+        self
+    }
+    pub fn compute_size(mut self, compute_size: &'static str) -> Self {
+        assert!(self.compute_size.is_none());
+        self.compute_size = Some(compute_size);
+        self
+    }
+
+    pub fn build(self, formats: &FormatRegistry) -> EncodingRecipe {
+        let operands_in = self.operands_in.unwrap_or(Vec::new());
+        let operands_out = self.operands_out.unwrap_or(Vec::new());
+
+        // The number of input constraints must match the number of format input operands.
+        if !formats.get(self.format).has_value_list {
+            let format = formats.get(self.format);
+            assert!(
+                operands_in.len() == format.num_value_operands,
+                format!(
+                    "missing operand constraints for recipe {} (format {})",
+                    self.name, format.name
+                )
+            );
+        }
+
+        // Ensure tied inputs actually refer to existing inputs.
+        for constraint in operands_in.iter().chain(operands_out.iter()) {
+            if let OperandConstraint::TiedInput(n) = *constraint {
+                assert!(n < operands_in.len());
+            }
+        }
+
+        let compute_size = match self.compute_size {
+            Some(compute_size) => compute_size,
+            None => "base_size",
+        };
+
+        let clobbers_flags = self.clobbers_flags.unwrap_or(true);
+
+        EncodingRecipe {
+            name: self.name.into(),
+            format: self.format,
+            base_size: self.base_size,
+            operands_in,
+            operands_out,
+            compute_size,
+            branch_range: self.branch_range,
+            clobbers_flags,
+            inst_predicate: self.inst_predicate,
+            isa_predicate: self.isa_predicate,
+            emit: self.emit,
+        }
+    }
+}
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/regs.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/regs.rs
@@ -30,19 +30,34 @@ impl RegBank {
             units,
             names,
             prefix,
             pressure_tracking,
             toprcs: Vec::new(),
             classes: Vec::new(),
         }
     }
+
+    fn unit_by_name(&self, name: &'static str) -> u8 {
+        let unit = if let Some(found) = self.names.iter().position(|&reg_name| reg_name == name) {
+            found
+        } else {
+            // Try to match without the bank prefix.
+            assert!(name.starts_with(self.prefix));
+            let name_without_prefix = &name[self.prefix.len()..];
+            self.names
+                .iter()
+                .position(|&reg_name| reg_name == name_without_prefix)
+                .expect(&format!("invalid register name {}", name))
+        };
+        self.first_unit + (unit as u8)
+    }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
 pub struct RegClassIndex(u32);
 entity_impl!(RegClassIndex);
 
 pub struct RegClass {
     pub name: &'static str,
     pub index: RegClassIndex,
     pub width: u8,
     pub bank: RegBankIndex,
@@ -347,9 +362,22 @@ pub struct IsaRegs {
 
 impl IsaRegs {
     fn new(
         banks: PrimaryMap<RegBankIndex, RegBank>,
         classes: PrimaryMap<RegClassIndex, RegClass>,
     ) -> Self {
         Self { banks, classes }
     }
+
+    pub fn class_by_name(&self, name: &str) -> RegClassIndex {
+        self.classes
+            .values()
+            .find(|&class| class.name == name)
+            .expect(&format!("register class {} not found", name))
+            .index
+    }
+
+    pub fn regunit_by_name(&self, class_index: RegClassIndex, name: &'static str) -> u8 {
+        let bank_index = self.classes.get(class_index).unwrap().bank;
+        self.banks.get(bank_index).unwrap().unit_by_name(name)
+    }
 }
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/settings.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/settings.rs
@@ -143,16 +143,24 @@ impl SettingGroup {
             if let SpecificSetting::Bool(_) = s.specific {
                 if s.name == name {
                     return (BoolSettingIndex(i), self);
                 }
             }
         }
         panic!("Should have found bool setting by name.");
     }
+
+    pub fn predicate_by_name(&self, name: &'static str) -> SettingPredicateNumber {
+        self.predicates
+            .iter()
+            .find(|pred| pred.name == name)
+            .unwrap_or_else(|| panic!("unknown predicate {}", name))
+            .number
+    }
 }
 
 /// This is the basic information needed to track the specific parts of a setting when building
 /// them.
 pub enum ProtoSpecificSetting {
     Bool(bool),
     Enum(Vec<&'static str>),
     Num(u8),
@@ -204,20 +212,22 @@ impl PredicateNode {
     }
 }
 
 struct ProtoPredicate {
     pub name: &'static str,
     node: PredicateNode,
 }
 
+pub type SettingPredicateNumber = u8;
+
 pub struct Predicate {
     pub name: &'static str,
     node: PredicateNode,
-    pub number: u8,
+    pub number: SettingPredicateNumber,
 }
 
 impl Predicate {
     pub fn render(&self, group: &SettingGroup) -> String {
         self.node.render(group)
     }
 }
 
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/type_inference.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/type_inference.rs
@@ -359,17 +359,16 @@ impl TypeEnvironment {
         for constraint in &self.constraints {
             let constraint = constraint.translate_with_env(&self);
             if constraint.is_trivial() || new_constraints.contains(&constraint) {
                 continue;
             }
 
             // Sanity check: translated constraints should refer only to real variables.
             for arg in constraint.typevar_args() {
-                assert!(vars_tv.contains(arg));
                 let arg_free_tv = arg.free_typevar();
                 assert!(arg_free_tv.is_none() || vars_tv.contains(&arg_free_tv.unwrap()));
             }
 
             new_constraints.insert(constraint);
         }
 
         TypeEnvironment {
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/types.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/types.rs
@@ -22,17 +22,17 @@ static LANE_BASE: u8 = 0x70;
 static _RUST_NAME_PREFIX: &'static str = "ir::types::";
 
 // ValueType variants (i8, i32, ...) are provided in `shared::types.rs`.
 
 /// A concrete SSA value type.
 ///
 /// All SSA values have a type that is described by an instance of `ValueType`
 /// or one of its subclasses.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub enum ValueType {
     BV(BVType),
     Lane(LaneType),
     Special(SpecialType),
     Vector(VectorType),
 }
 
 impl ValueType {
@@ -142,17 +142,17 @@ impl From<SpecialType> for ValueType {
 /// Create a ValueType from a given vector type.
 impl From<VectorType> for ValueType {
     fn from(vector: VectorType) -> Self {
         ValueType::Vector(vector)
     }
 }
 
 /// A concrete scalar type that can appear as a vector lane too.
-#[derive(Clone, Copy, PartialEq)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
 pub enum LaneType {
     BoolType(shared_types::Bool),
     FloatType(shared_types::Float),
     IntType(shared_types::Int),
 }
 
 impl LaneType {
     /// Return a string containing the documentation comment for this lane type.
@@ -322,17 +322,17 @@ impl Iterator for LaneTypeIterator {
         }
     }
 }
 
 /// A concrete SIMD vector type.
 ///
 /// A vector type has a lane type which is an instance of `LaneType`,
 /// and a positive number of lanes.
-#[derive(Clone, PartialEq)]
+#[derive(Clone, PartialEq, Eq, Hash)]
 pub struct VectorType {
     base: LaneType,
     lanes: u64,
 }
 
 impl VectorType {
     /// Initialize a new integer type with `n` bits.
     pub fn new(base: LaneType, lanes: u64) -> Self {
@@ -388,17 +388,17 @@ impl fmt::Debug for VectorType {
             "VectorType(base={}, lanes={})",
             self.base,
             self.lane_count()
         )
     }
 }
 
 /// A flat bitvector type. Used for semantics description only.
-#[derive(Clone, PartialEq)]
+#[derive(Clone, PartialEq, Eq, Hash)]
 pub struct BVType {
     bits: u64,
 }
 
 impl BVType {
     /// Initialize a new bitvector type with `n` bits.
     pub fn new(bits: u16) -> Self {
         Self { bits: bits.into() }
--- a/third_party/rust/cranelift-codegen-meta/src/cdsl/xform.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/xform.rs
@@ -399,21 +399,22 @@ impl TransformGroups {
         panic!(format!("transform group with name {} not found", name));
     }
 }
 
 #[test]
 #[should_panic]
 fn test_double_custom_legalization() {
     use crate::cdsl::formats::{FormatRegistry, InstructionFormatBuilder};
-    use crate::cdsl::instructions::{InstructionBuilder, InstructionGroupBuilder};
+    use crate::cdsl::instructions::{AllInstructions, InstructionBuilder, InstructionGroupBuilder};
 
+    let mut dummy_all = AllInstructions::new();
     let mut format = FormatRegistry::new();
     format.insert(InstructionFormatBuilder::new("nullary"));
-    let mut inst_group = InstructionGroupBuilder::new("test", "", &format);
+    let mut inst_group = InstructionGroupBuilder::new("test", "", &mut dummy_all, &format);
     inst_group.push(InstructionBuilder::new("dummy", "doc"));
     let inst_group = inst_group.build();
     let dummy_inst = inst_group.by_name("dummy");
 
     let mut transform_group = TransformGroupBuilder::new("test", "doc");
     transform_group.custom_legalize(&dummy_inst, "custom 1");
     transform_group.custom_legalize(&dummy_inst, "custom 2");
 }
--- a/third_party/rust/cranelift-codegen-meta/src/constant_hash.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/constant_hash.rs
@@ -1,47 +1,53 @@
+use std::iter;
+
 pub fn simple_hash(s: &str) -> usize {
     let mut h: u32 = 5381;
     for c in s.chars() {
         h = (h ^ c as u32).wrapping_add(h.rotate_right(6));
     }
     h as usize
 }
 
 /// Compute an open addressed, quadratically probed hash table containing
 /// `items`. The returned table is a list containing the elements of the
 /// iterable `items` and `None` in unused slots.
-pub fn generate_table<T, H: Fn(&T) -> usize>(items: &Vec<T>, hash_function: H) -> Vec<Option<&T>> {
-    let size = (1.20 * items.len() as f64) as usize;
+pub fn generate_table<'cont, T, I: iter::Iterator<Item = &'cont T>, H: Fn(&T) -> usize>(
+    items: I,
+    num_items: usize,
+    hash_function: H,
+) -> Vec<Option<&'cont T>> {
+    let size = (1.20 * num_items as f64) as usize;
     // TODO do we really need the multiply by two here?
     let size = if size.is_power_of_two() {
         size * 2
     } else {
         size.next_power_of_two()
     };
 
-    let mut table: Vec<Option<&T>> = vec![None; size];
+    let mut table = vec![None; size];
 
     for i in items {
-        let mut h = hash_function(i) % size;
+        let mut h = hash_function(&i) % size;
         let mut s = 0;
         while table[h].is_some() {
             s += 1;
             h = (h + s) % size;
         }
         table[h] = Some(i);
     }
 
     table
 }
 
 #[test]
 fn test_generate_table() {
     let v = vec!["Hello".to_string(), "world".to_string()];
-    let table = generate_table(&v, |s| simple_hash(&s));
+    let table = generate_table(v.iter(), v.len(), |s| simple_hash(&s));
     assert_eq!(
         table,
         vec![
             None,
             Some(&"Hello".to_string()),
             Some(&"world".to_string()),
             None
         ]
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/default_map.rs
@@ -0,0 +1,19 @@
+use std::collections::HashMap;
+use std::hash::Hash;
+
+pub trait MapWithDefault<K, V: Default> {
+    fn get_or_default(&mut self, k: K) -> &mut V;
+}
+
+impl<K: Eq + Hash, V: Default> MapWithDefault<K, V> for HashMap<K, V> {
+    fn get_or_default(&mut self, k: K) -> &mut V {
+        self.entry(k).or_insert_with(|| V::default())
+    }
+}
+
+#[test]
+fn test_default() {
+    let mut hash_map = HashMap::new();
+    hash_map.insert(42, "hello");
+    assert_eq!(*hash_map.get_or_default(43), "");
+}
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/gen_binemit.rs
@@ -0,0 +1,221 @@
+//! Generate binary emission code for each ISA.
+
+use cranelift_entity::EntityRef;
+
+use crate::error;
+use crate::srcgen::Formatter;
+
+use crate::cdsl::formats::FormatRegistry;
+use crate::cdsl::recipes::{EncodingRecipe, OperandConstraint, Recipes};
+
+/// Generate code to handle a single recipe.
+///
+/// - Unpack the instruction data, knowing the format.
+/// - Determine register locations for operands with register constraints.
+/// - Determine stack slot locations for operands with stack constraints.
+/// - Call hand-written code for the actual emission.
+fn gen_recipe(formats: &FormatRegistry, recipe: &EncodingRecipe, fmt: &mut Formatter) {
+    let inst_format = formats.get(recipe.format);
+    let num_value_ops = inst_format.num_value_operands;
+
+    let want_args = recipe.operands_in.iter().any(|c| match c {
+        OperandConstraint::RegClass(_) | OperandConstraint::Stack(_) => true,
+        OperandConstraint::FixedReg(_) | OperandConstraint::TiedInput(_) => false,
+    });
+    assert!(!want_args || num_value_ops > 0 || inst_format.has_value_list);
+
+    let want_outs = recipe.operands_out.iter().any(|c| match c {
+        OperandConstraint::RegClass(_) | OperandConstraint::Stack(_) => true,
+        OperandConstraint::FixedReg(_) | OperandConstraint::TiedInput(_) => false,
+    });
+
+    let is_regmove = ["RegMove", "RegSpill", "RegFill"].contains(&inst_format.name);
+
+    // Unpack the instruction data.
+    fmtln!(fmt, "if let &InstructionData::{} {{", inst_format.name);
+    fmt.indent(|fmt| {
+        fmt.line("opcode,");
+        for f in &inst_format.imm_fields {
+            fmtln!(fmt, "{},", f.member);
+        }
+        if want_args {
+            if inst_format.has_value_list || num_value_ops > 1 {
+                fmt.line("ref args,");
+            } else {
+                fmt.line("arg,");
+            }
+        }
+        fmt.line("..");
+
+        fmt.outdented_line("} = inst_data {");
+
+        // Pass recipe arguments in this order: inputs, imm_fields, outputs.
+        let mut args = String::new();
+
+        if want_args && !is_regmove {
+            if inst_format.has_value_list {
+                fmt.line("let args = args.as_slice(&func.dfg.value_lists);");
+            } else if num_value_ops == 1 {
+                fmt.line("let args = [arg];");
+            }
+            args += &unwrap_values(&recipe.operands_in, "in", "args", fmt);
+        }
+
+        for f in &inst_format.imm_fields {
+            args += &format!(", {}", f.member);
+        }
+
+        // Unwrap interesting output arguments.
+        if want_outs {
+            if recipe.operands_out.len() == 1 {
+                fmt.line("let results = [func.dfg.first_result(inst)];")
+            } else {
+                fmt.line("let results = func.dfg.inst_results(inst);");
+            }
+            args += &unwrap_values(&recipe.operands_out, "out", "results", fmt);
+        }
+
+        // Optimization: Only update the register diversion tracker for regmove instructions.
+        if is_regmove {
+            fmt.line("divert.apply(inst_data);")
+        }
+
+        match &recipe.emit {
+            Some(emit) => {
+                fmt.multi_line(emit);
+                fmt.line("return;");
+            }
+            None => {
+                fmtln!(
+                    fmt,
+                    "return recipe_{}(func, inst, sink, bits{});",
+                    recipe.name.to_lowercase(),
+                    args
+                );
+            }
+        }
+    });
+    fmt.line("}");
+}
+
+/// Emit code that unwraps values living in registers or stack slots.
+///
+/// :param args: Input or output constraints.
+/// :param prefix: Prefix to be used for the generated local variables.
+/// :param values: Name of slice containing the values to be unwrapped.
+/// :returns: Comma separated list of the generated variables
+fn unwrap_values(
+    args: &[OperandConstraint],
+    prefix: &str,
+    values_slice: &str,
+    fmt: &mut Formatter,
+) -> String {
+    let mut varlist = String::new();
+    for (i, cst) in args.iter().enumerate() {
+        match cst {
+            OperandConstraint::RegClass(_reg_class) => {
+                let v = format!("{}_reg{}", prefix, i);
+                varlist += &format!(", {}", v);
+                fmtln!(
+                    fmt,
+                    "let {} = divert.reg({}[{}], &func.locations);",
+                    v,
+                    values_slice,
+                    i
+                );
+            }
+            OperandConstraint::Stack(stack) => {
+                let v = format!("{}_stk{}", prefix, i);
+                varlist += &format!(", {}", v);
+                fmtln!(fmt, "let {} = StackRef::masked(", v);
+                fmt.indent(|fmt| {
+                    fmtln!(
+                        fmt,
+                        "divert.stack({}[{}], &func.locations),",
+                        values_slice,
+                        i
+                    );
+                    fmt.line(format!("{}, ", stack.stack_base_mask()));
+                    fmt.line("&func.stack_slots,");
+                });
+                fmt.line(").unwrap();");
+            }
+            _ => {}
+        }
+    }
+    varlist
+}
+
+fn gen_isa(formats: &FormatRegistry, isa_name: &str, recipes: &Recipes, fmt: &mut Formatter) {
+    fmt.doc_comment(format!(
+        "Emit binary machine code for `inst` for the {} ISA.",
+        isa_name
+    ));
+
+    if recipes.is_empty() {
+        fmt.line("pub fn emit_inst<CS: CodeSink + ?Sized>(");
+        fmt.indent(|fmt| {
+            fmt.line("func: &Function,");
+            fmt.line("inst: Inst,");
+            fmt.line("_divert: &mut RegDiversions,");
+            fmt.line("_sink: &mut CS,");
+        });
+        fmt.line(") {");
+        fmt.indent(|fmt| {
+            // No encoding recipes: Emit a stub.
+            fmt.line("bad_encoding(func, inst)");
+        });
+        fmt.line("}");
+        return;
+    }
+
+    fmt.line("#[allow(unused_variables, unreachable_code)]");
+    fmt.line("pub fn emit_inst<CS: CodeSink + ?Sized>(");
+    fmt.indent(|fmt| {
+        fmt.line("func: &Function,");
+        fmt.line("inst: Inst,");
+        fmt.line("divert: &mut RegDiversions,");
+        fmt.line("sink: &mut CS,");
+    });
+
+    fmt.line(") {");
+    fmt.indent(|fmt| {
+        fmt.line("let encoding = func.encodings[inst];");
+        fmt.line("let bits = encoding.bits();");
+        fmt.line("let inst_data = &func.dfg[inst];");
+        fmt.line("match encoding.recipe() {");
+        fmt.indent(|fmt| {
+            for (i, recipe) in recipes.iter() {
+                fmt.comment(format!("Recipe {}", recipe.name));
+                fmtln!(fmt, "{} => {{", i.index());
+                fmt.indent(|fmt| {
+                    gen_recipe(formats, recipe, fmt);
+                });
+                fmt.line("}");
+            }
+            fmt.line("_ => {},");
+        });
+        fmt.line("}");
+
+        // Allow for unencoded ghost instructions. The verifier will check details.
+        fmt.line("if encoding.is_legal() {");
+        fmt.indent(|fmt| {
+            fmt.line("bad_encoding(func, inst);");
+        });
+        fmt.line("}");
+    });
+    fmt.line("}");
+}
+
+pub fn generate(
+    formats: &FormatRegistry,
+    isa_name: &str,
+    recipes: &Recipes,
+    binemit_filename: &str,
+    out_dir: &str,
+) -> Result<(), error::Error> {
+    let mut fmt = Formatter::new();
+    gen_isa(formats, isa_name, recipes, &mut fmt);
+    fmt.update_file(binemit_filename, out_dir)?;
+    Ok(())
+}
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/gen_encodings.rs
@@ -0,0 +1,1072 @@
+//! Generate sources for instruction encoding.
+//!
+//! The tables and functions generated here support the `TargetISA::encode()` function which
+//! determines if a given instruction is legal, and if so, its `Encoding` data which consists of a
+//! *recipe* and some *encoding* bits.
+//!
+//! The `encode` function doesn't actually generate the binary machine bits. Each recipe has a
+//! corresponding hand-written function to do that after registers are allocated.
+//!
+//! This is the information available to us:
+//!
+//! - The instruction to be encoded as an `InstructionData` reference.
+//! - The controlling type variable.
+//! - The data-flow graph giving us access to the types of all values involved. This is needed for
+//! testing any secondary type variables.
+//! - A `PredicateView` reference for the ISA-specific settings for evaluating ISA predicates.
+//! - The currently active CPU mode is determined by the ISA.
+//!
+//! ## Level 1 table lookup
+//!
+//! The CPU mode provides the first table. The key is the instruction's controlling type variable.
+//! If the instruction is not polymorphic, use `INVALID` for the type variable. The table values
+//! are level 2 tables.
+//!
+//! ## Level 2 table lookup
+//!
+//! The level 2 table is keyed by the instruction's opcode. The table values are *encoding lists*.
+//!
+//! The two-level table lookup allows the level 2 tables to be much smaller with good locality.
+//! Code in any given function usually only uses a few different types, so many of the level 2
+//! tables will be cold.
+//!
+//! ## Encoding lists
+//!
+//! An encoding list is a non-empty sequence of list entries. Each entry has one of these forms:
+//!
+//! 1. Recipe + bits. Use this encoding if the recipe predicate is satisfied.
+//! 2. Recipe + bits, final entry. Use this encoding if the recipe predicate is satisfied.
+//!    Otherwise, stop with the default legalization code.
+//! 3. Stop with legalization code.
+//! 4. Predicate + skip count. Test predicate and skip N entries if it is false.
+//! 5. Predicate + stop. Test predicate and stop with the default legalization code if it is false.
+//!
+//! The instruction predicate is also used to distinguish between polymorphic instructions with
+//! different types for secondary type variables.
+
+use std::collections::btree_map;
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::convert::TryFrom;
+use std::iter::FromIterator;
+
+use cranelift_entity::EntityRef;
+
+use crate::error;
+use crate::srcgen::Formatter;
+
+use crate::cdsl::cpu_modes::CpuMode;
+use crate::cdsl::encodings::Encoding;
+use crate::cdsl::instructions::{Instruction, InstructionPredicate, InstructionPredicateNumber};
+use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::recipes::{EncodingRecipe, OperandConstraint, Recipes, Register};
+use crate::cdsl::regs::IsaRegs;
+use crate::cdsl::settings::SettingPredicateNumber;
+use crate::cdsl::types::ValueType;
+use crate::cdsl::xform::TransformGroupIndex;
+
+use crate::shared::Definitions as SharedDefinitions;
+
+use crate::constant_hash::generate_table;
+use crate::default_map::MapWithDefault;
+use crate::unique_table::UniqueSeqTable;
+
+/// Emit code for matching an instruction predicate against an `InstructionData` reference called
+/// `inst`.
+///
+/// The generated code is an `if let` pattern match that falls through if the instruction has an
+/// unexpected format. This should lead to a panic.
+fn emit_instp(instp: &InstructionPredicate, has_func: bool, fmt: &mut Formatter) {
+    if instp.is_type_predicate() {
+        fmt.line("let args = inst.arguments(&func.dfg.value_lists);");
+        fmt.line(instp.rust_predicate());
+        return;
+    }
+
+    let leaves = instp.collect_leaves();
+
+    let mut has_type_check = false;
+    let mut format_name = None;
+    let mut field_names = HashSet::new();
+
+    for leaf in leaves {
+        if leaf.is_type_predicate() {
+            has_type_check = true;
+        } else {
+            field_names.insert(leaf.format_destructuring_member_name());
+            let leaf_format_name = leaf.format_name();
+            match format_name {
+                None => format_name = Some(leaf_format_name),
+                Some(previous_format_name) => {
+                    assert!(
+                        previous_format_name == leaf_format_name,
+                        format!("Format predicate can only operate on a single InstructionFormat; trying to use both {} and {}", previous_format_name, leaf_format_name
+                    ));
+                }
+            }
+        }
+    }
+
+    let mut fields = Vec::from_iter(field_names);
+    fields.sort();
+    let fields = fields.join(", ");
+
+    let format_name = format_name.expect("There should be a format name!");
+
+    fmtln!(
+        fmt,
+        "if let crate::ir::InstructionData::{} {{ {}, .. }} = *inst {{",
+        format_name,
+        fields
+    );
+    fmt.indent(|fmt| {
+        if has_type_check {
+            // We could implement this.
+            assert!(has_func, "recipe predicates can't check type variables.");
+            fmt.line("let args = inst.arguments(&func.dfg.value_lists);");
+        } else if has_func {
+            // Silence dead argument.
+            fmt.line("let _ = func;");
+        }
+        fmtln!(fmt, "return {};", instp.rust_predicate());
+    });
+    fmtln!(fmt, "}");
+
+    fmt.line("unreachable!();");
+}
+
+/// Emit private functions for checking recipe predicates as well as a static `RECIPE_PREDICATES`
+/// array indexed by recipe number.
+///
+/// A recipe predicate is a combination of an ISA predicate and an instruction predicate. Many
+/// recipes have identical predicates.
+fn emit_recipe_predicates(isa: &TargetIsa, fmt: &mut Formatter) {
+    let mut predicate_names = HashMap::new();
+
+    for recipe in isa.recipes.values() {
+        let (isap, instp) = match (&recipe.isa_predicate, &recipe.inst_predicate) {
+            (None, None) => continue,
+            (isap, instp) if predicate_names.contains_key(&(isap, instp)) => continue,
+            (isap, instp) => (isap, instp),
+        };
+
+        let func_name = format!("recipe_predicate_{}", recipe.name.to_lowercase());
+        predicate_names.insert((isap, instp), func_name.clone());
+
+        // Generate the predicate function.
+        fmtln!(
+            fmt,
+            "fn {}({}: crate::settings::PredicateView, {}: &ir::InstructionData) -> bool {{",
+            func_name,
+            if let Some(_) = isap { "isap" } else { "_" },
+            if let Some(_) = instp { "inst" } else { "_" }
+        );
+        fmt.indent(|fmt| {
+            match (isap, instp) {
+                (Some(isap), None) => {
+                    fmtln!(fmt, "isap.test({})", isap);
+                }
+                (None, Some(instp)) => {
+                    emit_instp(instp, /* has func */ false, fmt);
+                }
+                (Some(isap), Some(instp)) => {
+                    fmtln!(fmt, "isap.test({}) &&", isap);
+                    emit_instp(instp, /* has func */ false, fmt);
+                }
+                _ => panic!("skipped above"),
+            }
+        });
+        fmtln!(fmt, "}");
+    }
+
+    // Generate the static table.
+    fmtln!(
+        fmt,
+        "pub static RECIPE_PREDICATES: [RecipePredicate; {}] = [",
+        isa.recipes.len()
+    );
+    fmt.indent(|fmt| {
+        for recipe in isa.recipes.values() {
+            match (&recipe.isa_predicate, &recipe.inst_predicate) {
+                (None, None) => fmt.line("None,"),
+                key => fmtln!(fmt, "Some({}),", predicate_names.get(&key).unwrap()),
+            }
+        }
+    });
+    fmtln!(fmt, "];");
+}
+
+/// Emit private functions for matching instruction predicates as well as a static
+/// `INST_PREDICATES` array indexed by predicate number.
+fn emit_inst_predicates(isa: &TargetIsa, fmt: &mut Formatter) {
+    for (id, instp) in isa.encodings_predicates.iter() {
+        fmtln!(fmt, "fn inst_predicate_{}(func: &crate::ir::Function, inst: &crate::ir::InstructionData) -> bool {{", id.index());
+        fmt.indent(|fmt| {
+            emit_instp(instp, /* has func */ true, fmt);
+        });
+        fmtln!(fmt, "}");
+    }
+
+    // Generate the static table.
+    fmtln!(
+        fmt,
+        "pub static INST_PREDICATES: [InstPredicate; {}] = [",
+        isa.encodings_predicates.len()
+    );
+    fmt.indent(|fmt| {
+        for id in isa.encodings_predicates.keys() {
+            fmtln!(fmt, "inst_predicate_{},", id.index());
+        }
+    });
+    fmtln!(fmt, "];");
+}
+
+/// Emit a table of encoding recipe names keyed by recipe number.
+///
+/// This is used for pretty-printing encodings.
+fn emit_recipe_names(isa: &TargetIsa, fmt: &mut Formatter) {
+    fmtln!(
+        fmt,
+        "static RECIPE_NAMES: [&str; {}] = [",
+        isa.recipes.len()
+    );
+    fmt.indent(|fmt| {
+        for recipe in isa.recipes.values() {
+            fmtln!(fmt, r#""{}","#, recipe.name);
+        }
+    });
+    fmtln!(fmt, "];");
+}
+
+/// Returns a set of all the registers involved in fixed register constraints.
+fn get_fixed_registers(operands_in: &Vec<OperandConstraint>) -> HashSet<Register> {
+    HashSet::from_iter(
+        operands_in
+            .iter()
+            .map(|constraint| {
+                if let OperandConstraint::FixedReg(reg) = &constraint {
+                    Some(reg.clone())
+                } else {
+                    None
+                }
+            })
+            .filter(|opt| opt.is_some())
+            .map(|opt| opt.unwrap()),
+    )
+}
+
+/// Emit a struct field initializer for an array of operand constraints.
+///
+/// Note "fixed_registers" must refer to the other kind of operands (i.e. if we're operating on
+/// inputs, fixed_registers must contain the fixed output registers).
+fn emit_operand_constraints(
+    registers: &IsaRegs,
+    recipe: &EncodingRecipe,
+    constraints: &Vec<OperandConstraint>,
+    field_name: &'static str,
+    tied_operands: &HashMap<usize, usize>,
+    fixed_registers: &HashSet<Register>,
+    fmt: &mut Formatter,
+) {
+    if constraints.len() == 0 {
+        fmtln!(fmt, "{}: &[],", field_name);
+        return;
+    }
+
+    fmtln!(fmt, "{}: &[", field_name);
+    fmt.indent(|fmt| {
+        for (n, constraint) in constraints.iter().enumerate() {
+            fmt.line("OperandConstraint {");
+            fmt.indent(|fmt| {
+                match constraint {
+                    OperandConstraint::RegClass(reg_class) => {
+                        if let Some(tied_input) = tied_operands.get(&n) {
+                            fmtln!(fmt, "kind: ConstraintKind::Tied({}),", tied_input);
+                        } else {
+                            fmt.line("kind: ConstraintKind::Reg,");
+                        }
+                        fmtln!(
+                            fmt,
+                            "regclass: &{}_DATA,",
+                            registers.classes[*reg_class].name
+                        );
+                    }
+                    OperandConstraint::FixedReg(reg) => {
+                        assert!(!tied_operands.contains_key(&n), "can't tie fixed registers");
+                        let constraint_kind = if fixed_registers.contains(&reg) {
+                            "FixedTied"
+                        } else {
+                            "FixedReg"
+                        };
+                        fmtln!(
+                            fmt,
+                            "kind: ConstraintKind::{}({}),",
+                            constraint_kind,
+                            reg.unit
+                        );
+                        fmtln!(
+                            fmt,
+                            "regclass: &{}_DATA,",
+                            registers.classes[reg.regclass].name
+                        );
+                    }
+                    OperandConstraint::TiedInput(tied_input) => {
+                        // This is a tied output constraint. It should never happen
+                        // for input constraints.
+                        assert!(
+                            tied_input == tied_operands.get(&n).unwrap(),
+                            "invalid tied constraint"
+                        );
+                        fmtln!(fmt, "kind: ConstraintKind::Tied({}),", tied_input);
+
+                        let tied_class = if let OperandConstraint::RegClass(tied_class) =
+                            recipe.operands_in[*tied_input]
+                        {
+                            tied_class
+                        } else {
+                            panic!("tied constraints relate only to register inputs");
+                        };
+
+                        fmtln!(
+                            fmt,
+                            "regclass: &{}_DATA,",
+                            registers.classes[tied_class].name
+                        );
+                    }
+                    OperandConstraint::Stack(stack) => {
+                        assert!(!tied_operands.contains_key(&n), "can't tie stack operand");
+                        fmt.line("kind: ConstraintKind::Stack,");
+                        fmtln!(
+                            fmt,
+                            "regclass: &{}_DATA,",
+                            registers.classes[stack.regclass].name
+                        );
+                    }
+                }
+            });
+            fmt.line("},");
+        }
+    });
+    fmtln!(fmt, "],");
+}
+
+/// Emit a table of encoding recipe operand constraints keyed by recipe number.
+///
+/// These are used by the register allocator to pick registers that can be properly encoded.
+fn emit_recipe_constraints(isa: &TargetIsa, fmt: &mut Formatter) {
+    fmtln!(
+        fmt,
+        "static RECIPE_CONSTRAINTS: [RecipeConstraints; {}] = [",
+        isa.recipes.len()
+    );
+    fmt.indent(|fmt| {
+        for recipe in isa.recipes.values() {
+            // Compute a mapping of tied operands in both directions (input tied to outputs and
+            // conversely).
+            let mut tied_in_to_out = HashMap::new();
+            let mut tied_out_to_in = HashMap::new();
+            for (out_index, constraint) in recipe.operands_out.iter().enumerate() {
+                if let OperandConstraint::TiedInput(in_index) = &constraint {
+                    tied_in_to_out.insert(*in_index, out_index);
+                    tied_out_to_in.insert(out_index, *in_index);
+                }
+            }
+
+            // Find the sets of registers involved in fixed register constraints.
+            let fixed_inputs = get_fixed_registers(&recipe.operands_in);
+            let fixed_outputs = get_fixed_registers(&recipe.operands_out);
+
+            fmt.comment(format!("Constraints for recipe {}:", recipe.name));
+            fmt.line("RecipeConstraints {");
+            fmt.indent(|fmt| {
+                emit_operand_constraints(
+                    &isa.regs,
+                    recipe,
+                    &recipe.operands_in,
+                    "ins",
+                    &tied_in_to_out,
+                    &fixed_outputs,
+                    fmt,
+                );
+                emit_operand_constraints(
+                    &isa.regs,
+                    recipe,
+                    &recipe.operands_out,
+                    "outs",
+                    &tied_out_to_in,
+                    &fixed_inputs,
+                    fmt,
+                );
+                fmtln!(
+                    fmt,
+                    "fixed_ins: {},",
+                    if !fixed_inputs.is_empty() {
+                        "true"
+                    } else {
+                        "false"
+                    }
+                );
+                fmtln!(
+                    fmt,
+                    "fixed_outs: {},",
+                    if !fixed_outputs.is_empty() {
+                        "true"
+                    } else {
+                        "false"
+                    }
+                );
+                fmtln!(
+                    fmt,
+                    "tied_ops: {},",
+                    if !tied_in_to_out.is_empty() {
+                        "true"
+                    } else {
+                        "false"
+                    }
+                );
+                fmtln!(
+                    fmt,
+                    "clobbers_flags: {},",
+                    if recipe.clobbers_flags {
+                        "true"
+                    } else {
+                        "false"
+                    }
+                );
+            });
+            fmt.line("},");
+        }
+    });
+    fmtln!(fmt, "];");
+}
+
+/// Emit a table of encoding recipe code size information.
+fn emit_recipe_sizing(isa: &TargetIsa, fmt: &mut Formatter) {
+    fmtln!(
+        fmt,
+        "static RECIPE_SIZING: [RecipeSizing; {}] = [",
+        isa.recipes.len()
+    );
+    fmt.indent(|fmt| {
+        for recipe in isa.recipes.values() {
+            fmt.comment(format!("Code size information for recipe {}:", recipe.name));
+            fmt.line("RecipeSizing {");
+            fmt.indent(|fmt| {
+                fmtln!(fmt, "base_size: {},", recipe.base_size);
+                fmtln!(fmt, "compute_size: {},", recipe.compute_size);
+                if let Some(range) = &recipe.branch_range {
+                    fmtln!(
+                        fmt,
+                        "branch_range: Some(BranchRange {{ origin: {}, bits: {} }}),",
+                        range.inst_size,
+                        range.range
+                    );
+                } else {
+                    fmt.line("branch_range: None,");
+                }
+            });
+            fmt.line("},");
+        }
+    });
+    fmtln!(fmt, "];");
+}
+
+/// Level 1 table mapping types to `Level2` objects.
+struct Level1Table<'cpu_mode> {
+    cpu_mode: &'cpu_mode CpuMode,
+    legalize_code: TransformGroupIndex,
+
+    table_map: HashMap<Option<ValueType>, usize>,
+    table_vec: Vec<Level2Table>,
+}
+
+impl<'cpu_mode> Level1Table<'cpu_mode> {
+    fn new(cpu_mode: &'cpu_mode CpuMode) -> Self {
+        Self {
+            cpu_mode,
+            legalize_code: cpu_mode.get_default_legalize_code(),
+            table_map: HashMap::new(),
+            table_vec: Vec::new(),
+        }
+    }
+
+    /// Returns the level2 table for the given type; None means monomorphic, in this context.
+    fn l2table_for(&mut self, typ: Option<ValueType>) -> &mut Level2Table {
+        let cpu_mode = &self.cpu_mode;
+        let index = match self.table_map.get(&typ) {
+            Some(&index) => index,
+            None => {
+                let legalize_code = cpu_mode.get_legalize_code_for(&typ);
+                let table = Level2Table::new(typ.clone(), legalize_code);
+                let index = self.table_vec.len();
+                self.table_map.insert(typ, index);
+                self.table_vec.push(table);
+                index
+            }
+        };
+        self.table_vec.get_mut(index).unwrap()
+    }
+
+    fn l2tables(&mut self) -> Vec<&mut Level2Table> {
+        self.table_vec
+            .iter_mut()
+            .filter(|table| !table.is_empty())
+            .collect::<Vec<_>>()
+    }
+}
+
+struct Level2HashTableEntry {
+    inst_name: String,
+    offset: usize,
+}
+
+/// Level 2 table mapping instruction opcodes to `EncList` objects.
+///
+/// A level 2 table can be completely empty if it only holds a custom legalization action for `ty`.
+struct Level2Table {
+    typ: Option<ValueType>,
+    legalize_code: TransformGroupIndex,
+    inst_to_encodings: BTreeMap<String, EncodingList>,
+    hash_table_offset: Option<usize>,
+    hash_table_len: Option<usize>,
+}
+
+impl Level2Table {
+    fn new(typ: Option<ValueType>, legalize_code: TransformGroupIndex) -> Self {
+        Self {
+            typ,
+            legalize_code,
+            inst_to_encodings: BTreeMap::new(),
+            hash_table_offset: None,
+            hash_table_len: None,
+        }
+    }
+
+    fn enclist_for(&mut self, inst: &Instruction) -> &mut EncodingList {
+        let copied_typ = self.typ.clone();
+        self.inst_to_encodings
+            .entry(inst.name.clone())
+            .or_insert_with(|| EncodingList::new(inst, copied_typ))
+    }
+
+    fn enclists(&mut self) -> btree_map::ValuesMut<'_, String, EncodingList> {
+        self.inst_to_encodings.values_mut()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.inst_to_encodings.is_empty()
+    }
+
+    fn layout_hashtable(
+        &mut self,
+        level2_hashtables: &mut Vec<Option<Level2HashTableEntry>>,
+        level2_doc: &mut HashMap<usize, Vec<String>>,
+    ) {
+        let hash_table = generate_table(
+            self.inst_to_encodings.values(),
+            self.inst_to_encodings.len(),
+            // TODO the Python code wanted opcode numbers to start from 1.
+            |enc_list| enc_list.inst.opcode_number.index() + 1,
+        );
+
+        let hash_table_offset = level2_hashtables.len();
+        let hash_table_len = hash_table.len();
+
+        assert!(self.hash_table_offset.is_none());
+        assert!(self.hash_table_len.is_none());
+        self.hash_table_offset = Some(hash_table_offset);
+        self.hash_table_len = Some(hash_table_len);
+
+        level2_hashtables.extend(hash_table.iter().map(|opt_enc_list| {
+            opt_enc_list.map(|enc_list| Level2HashTableEntry {
+                inst_name: enc_list.inst.camel_name.clone(),
+                offset: enc_list.offset.unwrap(),
+            })
+        }));
+
+        let typ_comment = match &self.typ {
+            Some(ty) => ty.to_string(),
+            None => "typeless".into(),
+        };
+
+        level2_doc.get_or_default(hash_table_offset).push(format!(
+            "{:06x}: {}, {} entries",
+            hash_table_offset, typ_comment, hash_table_len
+        ));
+    }
+}
+
+/// The u16 values in an encoding list entry are interpreted as follows:
+///
+/// NR = len(all_recipes)
+///
+/// entry < 2*NR
+///     Try Encoding(entry/2, next_entry) if the recipe predicate is satisfied.
+///     If bit 0 is set, stop with the default legalization code.
+///     If bit 0 is clear, keep going down the list.
+/// entry < PRED_START
+///     Stop with legalization code `entry - 2*NR`.
+///
+/// Remaining entries are interpreted as (skip, pred) pairs, where:
+///
+/// skip = (entry - PRED_START) >> PRED_BITS
+/// pred = (entry - PRED_START) & PRED_MASK
+///
+/// If the predicate is satisfied, keep going. Otherwise skip over the next
+/// `skip` entries. If skip == 0, stop with the default legalization code.
+///
+/// The `pred` predicate number is interpreted as an instruction predicate if it
+/// is in range, otherwise an ISA predicate.
+
+/// Encoding lists are represented as u16 arrays.
+const CODE_BITS: usize = 16;
+
+/// Beginning of the predicate code words.
+const PRED_START: u16 = 0x1000;
+
+/// Number of bits used to hold a predicate number (instruction + ISA predicates).
+const PRED_BITS: usize = 12;
+
+/// Mask for extracting the predicate number.
+const PRED_MASK: usize = (1 << PRED_BITS) - 1;
+
+/// Encoder for the list format above.
+struct Encoder {
+    num_instruction_predicates: usize,
+
+    /// u16 encoding list words.
+    words: Vec<u16>,
+
+    /// Documentation comments: Index into `words` + comment.
+    docs: Vec<(usize, String)>,
+}
+
+impl Encoder {
+    fn new(num_instruction_predicates: usize) -> Self {
+        Self {
+            num_instruction_predicates,
+            words: Vec::new(),
+            docs: Vec::new(),
+        }
+    }
+
+    /// Add a recipe+bits entry to the list.
+    fn recipe(&mut self, recipes: &Recipes, enc: &Encoding, is_final: bool) {
+        let code = (2 * enc.recipe.index() + if is_final { 1 } else { 0 }) as u16;
+        assert!(code < PRED_START);
+
+        let doc = format!(
+            "--> {}{}",
+            enc.to_rust_comment(recipes),
+            if is_final { " and stop" } else { "" }
+        );
+        self.docs.push((self.words.len(), doc));
+
+        self.words.push(code);
+        self.words.push(enc.encbits);
+    }
+
+    /// Add a predicate entry.
+    fn pred(&mut self, pred_comment: String, skip: usize, n: usize) {
+        assert!(n <= PRED_MASK);
+        let entry = (PRED_START as usize) + (n | (skip << PRED_BITS));
+        assert!(entry < (1 << CODE_BITS));
+        let entry = entry as u16;
+
+        let doc = if skip == 0 {
+            "stop".to_string()
+        } else {
+            format!("skip {}", skip)
+        };
+        let doc = format!("{} unless {}", doc, pred_comment);
+
+        self.docs.push((self.words.len(), doc));
+        self.words.push(entry);
+    }
+
+    /// Add an instruction predicate entry.
+    fn inst_predicate(&mut self, pred: InstructionPredicateNumber, skip: usize) {
+        let number = pred.index();
+        let pred_comment = format!("inst_predicate_{}", number);
+        self.pred(pred_comment, skip, number);
+    }
+
+    /// Add an ISA predicate entry.
+    fn isa_predicate(&mut self, pred: SettingPredicateNumber, skip: usize) {
+        // ISA predicates follow the instruction predicates.
+        let n = self.num_instruction_predicates + (pred as usize);
+        let pred_comment = format!("PredicateView({})", pred);
+        self.pred(pred_comment, skip, n);
+    }
+}
+
+/// List of instructions for encoding a given type + opcode pair.
+///
+/// An encoding list contains a sequence of predicates and encoding recipes, all encoded as u16
+/// values.
+struct EncodingList {
+    inst: Instruction,
+    typ: Option<ValueType>,
+    encodings: Vec<Encoding>,
+    offset: Option<usize>,
+}
+
+impl EncodingList {
+    fn new(inst: &Instruction, typ: Option<ValueType>) -> Self {
+        Self {
+            inst: inst.clone(),
+            typ,
+            encodings: Default::default(),
+            offset: None,
+        }
+    }
+
+    /// Encode this list as a sequence of u16 numbers.
+    ///
+    /// Adds the sequence to `enc_lists` and records the returned offset as
+    /// `self.offset`.
+    ///
+    /// Adds comment lines to `enc_lists_doc` keyed by enc_lists offsets.
+    fn encode(
+        &mut self,
+        isa: &TargetIsa,
+        cpu_mode: &CpuMode,
+        enc_lists: &mut UniqueSeqTable<u16>,
+        enc_lists_doc: &mut HashMap<usize, Vec<String>>,
+    ) {
+        assert!(!self.encodings.is_empty());
+
+        let mut encoder = Encoder::new(isa.encodings_predicates.len());
+
+        let mut index = 0;
+        while index < self.encodings.len() {
+            let encoding = &self.encodings[index];
+
+            // Try to see how many encodings are following and have the same ISA predicate and
+            // instruction predicate, so as to reduce the number of tests carried out by the
+            // encoding list interpreter..
+            //
+            // Encodings with similar tests are hereby called a group. The group includes the
+            // current encoding we're looking at.
+            let (isa_predicate, inst_predicate) =
+                (&encoding.isa_predicate, &encoding.inst_predicate);
+
+            let group_size = {
+                let mut group_size = 1;
+                while index + group_size < self.encodings.len() {
+                    let next_encoding = &self.encodings[index + group_size];
+                    if &next_encoding.inst_predicate != inst_predicate
+                        || &next_encoding.isa_predicate != isa_predicate
+                    {
+                        break;
+                    }
+                    group_size += 1;
+                }
+                group_size
+            };
+
+            let is_last_group = index + group_size == self.encodings.len();
+
+            // The number of entries to skip when a predicate isn't satisfied is the size of both
+            // predicates + the size of the group, minus one (for this predicate). Each recipe
+            // entry has a size of two u16 (recipe index + bits).
+            let mut skip = if is_last_group {
+                0
+            } else {
+                let isap_size = match isa_predicate {
+                    Some(_) => 1,
+                    None => 0,
+                };
+                let instp_size = match inst_predicate {
+                    Some(_) => 1,
+                    None => 0,
+                };
+                isap_size + instp_size + group_size * 2 - 1
+            };
+
+            if let Some(pred) = isa_predicate {
+                encoder.isa_predicate(*pred, skip);
+                if !is_last_group {
+                    skip -= 1;
+                }
+            }
+
+            if let Some(pred) = inst_predicate {
+                encoder.inst_predicate(*pred, skip);
+                // No need to update skip, it's dead after this point.
+            }
+
+            for i in 0..group_size {
+                let encoding = &self.encodings[index + i];
+                let is_last_encoding = index + i == self.encodings.len() - 1;
+                encoder.recipe(&isa.recipes, encoding, is_last_encoding);
+            }
+
+            index += group_size;
+        }
+
+        assert!(self.offset.is_none());
+        let offset = enc_lists.add(&encoder.words);
+        self.offset = Some(offset);
+
+        // Doc comments.
+        let recipe_typ_mode_name = format!(
+            "{}{} ({})",
+            self.inst.name,
+            if let Some(typ) = &self.typ {
+                format!(".{}", typ.to_string())
+            } else {
+                "".into()
+            },
+            cpu_mode.name
+        );
+
+        enc_lists_doc
+            .get_or_default(offset)
+            .push(format!("{:06x}: {}", offset, recipe_typ_mode_name));
+        for (pos, doc) in encoder.docs {
+            enc_lists_doc.get_or_default(offset + pos).push(doc);
+        }
+        enc_lists_doc
+            .get_or_default(offset + encoder.words.len())
+            .insert(0, format!("end of {}", recipe_typ_mode_name));
+    }
+}
+
+fn make_tables(cpu_mode: &CpuMode) -> Level1Table {
+    let mut table = Level1Table::new(cpu_mode);
+
+    for encoding in &cpu_mode.encodings {
+        table
+            .l2table_for(encoding.bound_type.clone())
+            .enclist_for(encoding.inst())
+            .encodings
+            .push(encoding.clone());
+    }
+
+    // Ensure there are level 1 table entries for all types with a custom legalize action.
+    for value_type in cpu_mode.get_legalized_types() {
+        table.l2table_for(Some(value_type.clone()));
+    }
+    // ... and also for monomorphic instructions.
+    table.l2table_for(None);
+
+    table
+}
+
+/// Compute encodings and doc comments for encoding lists in `level1`.
+fn encode_enclists(
+    isa: &TargetIsa,
+    cpu_mode: &CpuMode,
+    level1: &mut Level1Table,
+    enc_lists: &mut UniqueSeqTable<u16>,
+    enc_lists_doc: &mut HashMap<usize, Vec<String>>,
+) {
+    for level2 in level1.l2tables() {
+        for enclist in level2.enclists() {
+            enclist.encode(isa, cpu_mode, enc_lists, enc_lists_doc);
+        }
+    }
+}
+
+fn encode_level2_hashtables<'a>(
+    level1: &'a mut Level1Table,
+    level2_hashtables: &mut Vec<Option<Level2HashTableEntry>>,
+    level2_doc: &mut HashMap<usize, Vec<String>>,
+) {
+    for level2 in level1.l2tables() {
+        level2.layout_hashtable(level2_hashtables, level2_doc);
+    }
+}
+
+fn emit_tables(defs: &SharedDefinitions, isa: &TargetIsa, fmt: &mut Formatter) {
+    // Level 1 tables, one per CPU mode.
+    let mut level1_tables: HashMap<&'static str, Level1Table> = HashMap::new();
+
+    // Single table containing all the level2 hash tables.
+    let mut level2_hashtables = Vec::new();
+    let mut level2_doc: HashMap<usize, Vec<String>> = HashMap::new();
+
+    // Tables for encoding lists with comments.
+    let mut enc_lists = UniqueSeqTable::new();
+    let mut enc_lists_doc = HashMap::new();
+
+    for cpu_mode in &isa.cpu_modes {
+        level2_doc
+            .get_or_default(level2_hashtables.len())
+            .push(cpu_mode.name.into());
+
+        let mut level1 = make_tables(cpu_mode);
+
+        encode_enclists(
+            isa,
+            cpu_mode,
+            &mut level1,
+            &mut enc_lists,
+            &mut enc_lists_doc,
+        );
+        encode_level2_hashtables(&mut level1, &mut level2_hashtables, &mut level2_doc);
+
+        level1_tables.insert(cpu_mode.name, level1);
+    }
+
+    // Compute an appropriate Rust integer type to use for offsets into a table of the given length.
+    let offset_type = |length: usize| {
+        if length <= 0x10000 {
+            "u16"
+        } else {
+            assert!(u32::try_from(length).is_ok(), "table too big!");
+            "u32"
+        }
+    };
+
+    let level1_offset_type = offset_type(level2_hashtables.len());
+    let level2_offset_type = offset_type(enc_lists.len());
+
+    // Emit encoding lists.
+    fmtln!(fmt, "pub static ENCLISTS: [u16; {}] = [", enc_lists.len());
+    fmt.indent(|fmt| {
+        let mut line = Vec::new();
+        for (index, entry) in enc_lists.iter().enumerate() {
+            if let Some(comments) = enc_lists_doc.get(&index) {
+                if !line.is_empty() {
+                    fmtln!(fmt, "{},", line.join(", "));
+                    line.clear();
+                }
+                for comment in comments {
+                    fmt.comment(comment);
+                }
+            }
+            line.push(format!("{:#06x}", entry));
+        }
+        if !line.is_empty() {
+            fmtln!(fmt, "{},", line.join(", "));
+        }
+    });
+    fmtln!(fmt, "];");
+
+    // Emit the full concatenation of level 2 hash tables.
+    fmtln!(
+        fmt,
+        "pub static LEVEL2: [Level2Entry<{}>; {}] = [",
+        level2_offset_type,
+        level2_hashtables.len()
+    );
+    fmt.indent(|fmt| {
+        for (offset, entry) in level2_hashtables.iter().enumerate() {
+            if let Some(comments) = level2_doc.get(&offset) {
+                for comment in comments {
+                    fmt.comment(comment);
+                }
+            }
+            if let Some(entry) = entry {
+                fmtln!(
+                    fmt,
+                    "Level2Entry {{ opcode: Some(crate::ir::Opcode::{}), offset: {:#08x} }},",
+                    entry.inst_name,
+                    entry.offset
+                );
+            } else {
+                fmt.line("Level2Entry { opcode: None, offset: 0 },");
+            }
+        }
+    });
+    fmtln!(fmt, "];");
+
+    // Emit a level 1 hash table for each CPU mode.
+    for cpu_mode in &isa.cpu_modes {
+        let level1 = &level1_tables.get(cpu_mode.name).unwrap();
+        let hash_table = generate_table(
+            level1.table_vec.iter(),
+            level1.table_vec.len(),
+            |level2_table| {
+                if let Some(typ) = &level2_table.typ {
+                    typ.number().expect("type without a number") as usize
+                } else {
+                    0
+                }
+            },
+        );
+
+        fmtln!(
+            fmt,
+            "pub static LEVEL1_{}: [Level1Entry<{}>; {}] = [",
+            cpu_mode.name.to_uppercase(),
+            level1_offset_type,
+            hash_table.len()
+        );
+        fmt.indent(|fmt| {
+            for opt_level2 in hash_table {
+                let level2 = match opt_level2 {
+                    None => {
+                        // Empty hash table entry. Include the default legalization action.
+                        fmtln!(fmt, "Level1Entry {{ ty: ir::types::INVALID, log2len: !0, offset: 0, legalize: {} }},",
+                               isa.translate_group_index(level1.legalize_code));
+                        continue;
+                    }
+                    Some(level2) => level2,
+                };
+
+                let legalize_comment = defs.transform_groups.get(level2.legalize_code).name;
+                let legalize_code = isa.translate_group_index(level2.legalize_code);
+
+                let typ_name = if let Some(typ) = &level2.typ {
+                    typ.rust_name()
+                } else {
+                    "ir::types::INVALID".into()
+                };
+
+                if level2.is_empty() {
+                    // Empty level 2 table: Only a specialized legalization action, no actual
+                    // table.
+                    // Set an offset that is out of bounds, but make sure it doesn't overflow its
+                    // type when adding `1<<log2len`.
+                    fmtln!(fmt, "Level1Entry {{ ty: {}, log2len: 0, offset: !0 - 1, legalize: {} }}, // {}",
+                           typ_name, legalize_code, legalize_comment);
+                    continue;
+                }
+
+                // Proper level 2 hash table.
+                let l2l = (level2.hash_table_len.unwrap() as f64).log2() as i32;
+                assert!(l2l > 0, "Level2 hash table was too small.");
+                fmtln!(fmt, "Level1Entry {{ ty: {}, log2len: {}, offset: {:#08x}, legalize: {} }}, // {}",
+                       typ_name, l2l, level2.hash_table_offset.unwrap(), legalize_code, legalize_comment);
+            }
+        });
+        fmtln!(fmt, "];");
+    }
+}
+
+fn gen_isa(defs: &SharedDefinitions, isa: &TargetIsa, fmt: &mut Formatter) {
+    // Make the `RECIPE_PREDICATES` table.
+    emit_recipe_predicates(isa, fmt);
+
+    // Make the `INST_PREDICATES` table.
+    emit_inst_predicates(isa, fmt);
+
+    emit_tables(defs, isa, fmt);
+
+    emit_recipe_names(isa, fmt);
+    emit_recipe_constraints(isa, fmt);
+    emit_recipe_sizing(isa, fmt);
+
+    // Finally, tie it all together in an `EncInfo`.
+    fmt.line("pub static INFO: isa::EncInfo = isa::EncInfo {");
+    fmt.indent(|fmt| {
+        fmt.line("constraints: &RECIPE_CONSTRAINTS,");
+        fmt.line("sizing: &RECIPE_SIZING,");
+        fmt.line("names: &RECIPE_NAMES,");
+    });
+    fmt.line("};");
+}
+
+pub fn generate(
+    defs: &SharedDefinitions,
+    isa: &TargetIsa,
+    filename: &str,
+    out_dir: &str,
+) -> Result<(), error::Error> {
+    let mut fmt = Formatter::new();
+    gen_isa(defs, isa, &mut fmt);
+    fmt.update_file(filename, out_dir)?;
+    Ok(())
+}
--- a/third_party/rust/cranelift-codegen-meta/src/gen_inst.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/gen_inst.rs
@@ -1,20 +1,25 @@
+use std::fmt;
+
+use cranelift_entity::EntityRef;
+
 use crate::cdsl::camel_case;
 use crate::cdsl::formats::{FormatRegistry, InstructionFormat};
-use crate::cdsl::instructions::{Instruction, InstructionGroup};
+use crate::cdsl::instructions::{AllInstructions, Instruction};
 use crate::cdsl::operands::Operand;
 use crate::cdsl::typevar::{TypeSet, TypeVar};
+
+use crate::shared::Definitions as SharedDefinitions;
+
 use crate::constant_hash;
 use crate::error;
 use crate::srcgen::{Formatter, Match};
 use crate::unique_table::{UniqueSeqTable, UniqueTable};
 
-use std::fmt;
-
 // TypeSet indexes are encoded in 8 bits, with `0xff` reserved.
 const TYPESET_LIMIT: usize = 0xff;
 
 /// Generate an instruction format enumeration.
 fn gen_formats(registry: &FormatRegistry, fmt: &mut Formatter) {
     fmt.doc_comment(
         r#"
         An instruction format
@@ -363,231 +368,216 @@ fn gen_instruction_data_impl(registry: &
             fmt.line("}");
         });
         fmt.line("}");
     });
     fmt.line("}");
 }
 
 fn gen_bool_accessor<T: Fn(&Instruction) -> bool>(
-    instruction_groups: &Vec<&InstructionGroup>,
+    all_inst: &AllInstructions,
     get_attr: T,
     name: &'static str,
     doc: &'static str,
     fmt: &mut Formatter,
 ) {
     fmt.doc_comment(doc);
     fmtln!(fmt, "pub fn {}(self) -> bool {{", name);
     fmt.indent(|fmt| {
         let mut m = Match::new("self");
-        for group in instruction_groups.iter() {
-            for inst in group.iter() {
-                if get_attr(inst) {
-                    m.arm_no_fields(format!("Opcode::{}", inst.camel_name), "true");
-                }
+        for inst in all_inst.values() {
+            if get_attr(inst) {
+                m.arm_no_fields(format!("Opcode::{}", inst.camel_name), "true");
             }
         }
         m.arm_no_fields("_", "false");
         fmt.add_match(m);
     });
     fmtln!(fmt, "}");
     fmt.empty_line();
 }
 
-fn gen_opcodes<'a>(
-    formats: &FormatRegistry,
-    igroups: &Vec<&'a InstructionGroup>,
-    fmt: &mut Formatter,
-) -> Vec<&'a Instruction> {
-    let mut all_inst = Vec::new();
-    for group in igroups {
-        for inst in group.iter() {
-            all_inst.push(inst);
-        }
-    }
-
+fn gen_opcodes<'a>(all_inst: &AllInstructions, formats: &FormatRegistry, fmt: &mut Formatter) {
     fmt.doc_comment(
         r#"
         An instruction opcode.
 
         All instructions from all supported ISAs are present.
     "#,
     );
     fmt.line("#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]");
 
     // We explicitly set the discriminant of the first variant to 1, which allows us to take
     // advantage of the NonZero optimization, meaning that wrapping enums can use the 0
     // discriminant instead of increasing the size of the whole type, and so the size of
     // Option<Opcode> is the same as Opcode's.
     fmt.line("pub enum Opcode {");
     fmt.indent(|fmt| {
         let mut is_first_opcode = true;
-        for inst in &all_inst {
-            // TODO we might need to set an instruction number here. Probably can do in the
-            // InstructionGroup itself when adding instruction (would need to remember last
-            // instruction number in the SharedDefinitions or somewhere else).
+        for inst in all_inst.values() {
             let format = formats.get(inst.format);
             fmt.doc_comment(format!("`{}`. ({})", inst, format.name));
 
             // Document polymorphism.
             if let Some(poly) = &inst.polymorphic_info {
                 if poly.use_typevar_operand {
                     let op_num = inst.value_opnums[format.typevar_operand.unwrap()];
                     fmt.doc_comment(format!(
                         "Type inferred from `{}`.",
                         inst.operands_in[op_num].name
                     ));
                 }
             }
 
             // Enum variant itself.
             if is_first_opcode {
+                assert!(inst.opcode_number.index() == 0);
+                // TODO the python crate requires opcode numbers to start from one.
                 fmtln!(fmt, "{} = 1,", inst.camel_name);
                 is_first_opcode = false;
             } else {
                 fmtln!(fmt, "{},", inst.camel_name)
             }
         }
     });
     fmt.line("}");
     fmt.empty_line();
 
     fmt.line("impl Opcode {");
     fmt.indent(|fmt| {
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.is_terminator,
             "is_terminator",
             "True for instructions that terminate the EBB",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.is_branch,
             "is_branch",
             "True for all branch or jump instructions.",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.is_indirect_branch,
             "is_indirect_branch",
             "True for all indirect branch or jump instructions.",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.is_call,
             "is_call",
             "Is this a call instruction?",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.is_return,
             "is_return",
             "Is this a return instruction?",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.is_ghost,
             "is_ghost",
             "Is this a ghost instruction?",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.can_load,
             "can_load",
             "Can this instruction read from memory?",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.can_store,
             "can_store",
             "Can this instruction write to memory?",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.can_trap,
             "can_trap",
             "Can this instruction cause a trap?",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.other_side_effects,
             "other_side_effects",
             "Does this instruction have other side effects besides can_* flags?",
             fmt,
         );
         gen_bool_accessor(
-            igroups,
+            all_inst,
             |inst| inst.writes_cpu_flags,
             "writes_cpu_flags",
             "Does this instruction write to CPU flags?",
             fmt,
         );
     });
     fmt.line("}");
     fmt.empty_line();
 
     // Generate a private opcode_format table.
     fmtln!(
         fmt,
         "const OPCODE_FORMAT: [InstructionFormat; {}] = [",
         all_inst.len()
     );
     fmt.indent(|fmt| {
-        for inst in &all_inst {
+        for inst in all_inst.values() {
             let format = formats.get(inst.format);
             fmtln!(fmt, "InstructionFormat::{}, // {}", format.name, inst.name);
         }
     });
     fmtln!(fmt, "];");
     fmt.empty_line();
 
     // Generate a private opcode_name function.
     fmt.line("fn opcode_name(opc: Opcode) -> &\'static str {");
     fmt.indent(|fmt| {
         let mut m = Match::new("opc");
-        for inst in &all_inst {
+        for inst in all_inst.values() {
             m.arm_no_fields(
                 format!("Opcode::{}", inst.camel_name),
                 format!("\"{}\"", inst.name),
             );
         }
         fmt.add_match(m);
     });
     fmt.line("}");
     fmt.empty_line();
 
     // Generate an opcode hash table for looking up opcodes by name.
-    let hash_table =
-        constant_hash::generate_table(&all_inst, |inst| constant_hash::simple_hash(&inst.name));
+    let hash_table = constant_hash::generate_table(all_inst.values(), all_inst.len(), |inst| {
+        constant_hash::simple_hash(&inst.name)
+    });
     fmtln!(
         fmt,
         "const OPCODE_HASH_TABLE: [Option<Opcode>; {}] = [",
         hash_table.len()
     );
     fmt.indent(|fmt| {
         for i in hash_table {
             match i {
                 Some(i) => fmtln!(fmt, "Some(Opcode::{}),", i.camel_name),
                 None => fmtln!(fmt, "None,"),
             }
         }
     });
     fmtln!(fmt, "];");
     fmt.empty_line();
-
-    all_inst
 }
 
 /// Get the value type constraint for an SSA value operand, where
 /// `ctrl_typevar` is the controlling type variable.
 ///
 /// Each operand constraint is represented as a string, one of:
 /// - `Concrete(vt)`, where `vt` is a value type name.
 /// - `Free(idx)` where `idx` is an index into `type_sets`.
@@ -693,17 +683,17 @@ pub fn gen_typesets_table(type_sets: &Un
     });
     fmtln!(fmt, "];");
 }
 
 /// Generate value type constraints for all instructions.
 /// - Emit a compact constant table of ValueTypeSet objects.
 /// - Emit a compact constant table of OperandConstraint objects.
 /// - Emit an opcode-indexed table of instruction constraints.
-fn gen_type_constraints(all_inst: &Vec<&Instruction>, fmt: &mut Formatter) {
+fn gen_type_constraints(all_inst: &AllInstructions, fmt: &mut Formatter) {
     // Table of TypeSet instances.
     let mut type_sets = UniqueTable::new();
 
     // Table of operand constraint sequences (as tuples). Each operand
     // constraint is represented as a string, one of:
     // - `Concrete(vt)`, where `vt` is a value type name.
     // - `Free(idx)` where `idx` is an index into `type_sets`.
     // - `Same`, `Lane`, `AsBool` for controlling typevar-derived constraints.
@@ -714,17 +704,17 @@ fn gen_type_constraints(all_inst: &Vec<&
 
     fmt.comment("Table of opcode constraints.");
     fmtln!(
         fmt,
         "const OPCODE_CONSTRAINTS: [OpcodeConstraints; {}] = [",
         all_inst.len()
     );
     fmt.indent(|fmt| {
-        for inst in all_inst {
+        for inst in all_inst.values() {
             let (ctrl_typevar, ctrl_typeset) = if let Some(poly) = &inst.polymorphic_info {
                 let index = type_sets.add(&*poly.ctrl_typevar.get_raw_typeset());
                 (Some(&poly.ctrl_typevar), index)
             } else {
                 (None, TYPESET_LIMIT)
             };
 
             // Collect constraints for the value results, not including `variable_args` results
@@ -935,17 +925,17 @@ fn gen_inst_builder(inst: &Instruction, 
     let proto = format!(
         "{}{}({}) -> {}",
         inst.snake_name(),
         tmpl,
         args.join(", "),
         rtype
     );
 
-    fmt.doc_comment(format!("`{}`\n\n{}", inst, inst.doc_comment_first_line()));
+    fmt.doc_comment(&inst.doc);
     fmt.line("#[allow(non_snake_case)]");
     fmtln!(fmt, "fn {} {{", proto);
     fmt.indent(|fmt| {
         // Convert all of the `Into<>` arguments.
         for arg in &into_args {
             fmtln!(fmt, "let {} = {}.into();", arg, arg);
         }
 
@@ -1030,17 +1020,17 @@ fn gen_inst_builder(inst: &Instruction, 
                     .join(", ")
             );
         }
     });
     fmtln!(fmt, "}")
 }
 
 /// Generate a Builder trait with methods for all instructions.
-fn gen_builder(instructions: &Vec<&Instruction>, formats: &FormatRegistry, fmt: &mut Formatter) {
+fn gen_builder(instructions: &AllInstructions, formats: &FormatRegistry, fmt: &mut Formatter) {
     fmt.doc_comment(
         r#"
         Convenience methods for building instructions.
 
         The `InstBuilder` trait has one method per instruction opcode for
         conveniently constructing the instruction with minimum arguments.
         Polymorphic instructions infer their result types from the input
         arguments when possible. In some cases, an explicit `ctrl_typevar`
@@ -1050,43 +1040,45 @@ fn gen_builder(instructions: &Vec<&Instr
         the `Inst` itself for instructions that don't have any results.
 
         There is also a method per instruction format. These methods all
         return an `Inst`.
     "#,
     );
     fmt.line("pub trait InstBuilder<'f>: InstBuilderBase<'f> {");
     fmt.indent(|fmt| {
-        for inst in instructions {
+        for inst in instructions.values() {
             gen_inst_builder(inst, formats.get(inst.format), fmt);
         }
         for format in formats.iter() {
             gen_format_constructor(format, fmt);
         }
     });
     fmt.line("}");
 }
 
 pub fn generate(
-    all_inst_groups: Vec<&InstructionGroup>,
-    format_registry: &FormatRegistry,
+    shared_defs: &SharedDefinitions,
     opcode_filename: &str,
     inst_builder_filename: &str,
     out_dir: &str,
 ) -> Result<(), error::Error> {
+    let format_registry = &shared_defs.format_registry;
+    let all_inst = &shared_defs.all_instructions;
+
     // Opcodes.
     let mut fmt = Formatter::new();
     gen_formats(format_registry, &mut fmt);
     gen_instruction_data(format_registry, &mut fmt);
     fmt.empty_line();
     gen_instruction_data_impl(format_registry, &mut fmt);
     fmt.empty_line();
-    let all_inst = gen_opcodes(format_registry, &all_inst_groups, &mut fmt);
-    gen_type_constraints(&all_inst, &mut fmt);
+    gen_opcodes(all_inst, format_registry, &mut fmt);
+    gen_type_constraints(all_inst, &mut fmt);
     fmt.update_file(opcode_filename, out_dir)?;
 
     // Instruction builder.
     let mut fmt = Formatter::new();
-    gen_builder(&all_inst, format_registry, &mut fmt);
+    gen_builder(all_inst, format_registry, &mut fmt);
     fmt.update_file(inst_builder_filename, out_dir)?;
 
     Ok(())
 }
--- a/third_party/rust/cranelift-codegen-meta/src/gen_legalizer.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/gen_legalizer.rs
@@ -129,16 +129,29 @@ fn unwrap_inst(
         {
             // Special case: The instruction replacing node defines the exact same values.
             fmt.comment(format!(
                 "Results handled by {}.",
                 def_pool
                     .get(var_pool.get(def.defined_vars[0]).dst_def.unwrap())
                     .to_comment_string(var_pool)
             ));
+
+            fmt.line("let r = pos.func.dfg.inst_results(inst);");
+            for (i, &var_index) in def.defined_vars.iter().enumerate() {
+                let var = var_pool.get(var_index);
+                fmtln!(fmt, "let {} = &r[{}];", var.name, i);
+                fmtln!(
+                    fmt,
+                    "let typeof_{} = pos.func.dfg.value_type(*{});",
+                    var.name,
+                    var.name
+                );
+            }
+
             replace_inst = true;
         } else {
             // Boring case: Detach the result values, capture them in locals.
             for &var_index in &def.defined_vars {
                 fmtln!(fmt, "let {};", var_pool.get(var_index).name);
             }
 
             fmt.line("{");
@@ -522,17 +535,17 @@ fn gen_isa(
 
     let direct_groups = isa.direct_transform_groups();
     fmtln!(
         fmt,
         "pub static LEGALIZE_ACTIONS: [isa::Legalize; {}] = [",
         direct_groups.len()
     );
     fmt.indent(|fmt| {
-        for group_index in direct_groups {
+        for &group_index in direct_groups {
             fmtln!(fmt, "{},", transform_groups.get(group_index).rust_name());
         }
     });
     fmtln!(fmt, "];");
 }
 
 /// Generate the legalizer files.
 pub fn generate(
--- a/third_party/rust/cranelift-codegen-meta/src/gen_settings.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/gen_settings.rs
@@ -303,17 +303,19 @@ fn gen_descriptors(group: &SettingGroup,
     });
     fmtln!(fmt, "];");
 
     // Generate hash table.
     let mut hash_entries: Vec<SettingOrPreset> = Vec::new();
     hash_entries.extend(group.settings.iter().map(|x| SettingOrPreset::Setting(x)));
     hash_entries.extend(group.presets.iter().map(|x| SettingOrPreset::Preset(x)));
 
-    let hash_table = generate_table(&hash_entries, |entry| simple_hash(entry.name()));
+    let hash_table = generate_table(hash_entries.iter(), hash_entries.len(), |entry| {
+        simple_hash(entry.name())
+    });
     fmtln!(fmt, "static HASH_TABLE: [u16; {}] = [", hash_table.len());
     fmt.indent(|fmt| {
         for h in &hash_table {
             match *h {
                 Some(setting_or_preset) => fmtln!(
                     fmt,
                     "{},",
                     &descriptor_index_map
--- a/third_party/rust/cranelift-codegen-meta/src/isa/arm32/mod.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/arm32/mod.rs
@@ -1,11 +1,12 @@
 use crate::cdsl::cpu_modes::CpuMode;
-use crate::cdsl::instructions::InstructionGroupBuilder;
+use crate::cdsl::instructions::{InstructionGroupBuilder, InstructionPredicateMap};
 use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::recipes::Recipes;
 use crate::cdsl::regs::{IsaRegs, IsaRegsBuilder, RegBankBuilder, RegClassBuilder};
 use crate::cdsl::settings::{SettingGroup, SettingGroupBuilder};
 
 use crate::shared::Definitions as SharedDefinitions;
 
 fn define_settings(_shared: &SettingGroup) -> SettingGroup {
     let setting = SettingGroupBuilder::new("arm32");
     setting.build()
@@ -50,25 +51,40 @@ fn define_regs() -> IsaRegs {
 
 pub fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
     let settings = define_settings(&shared_defs.settings);
     let regs = define_regs();
 
     let inst_group = InstructionGroupBuilder::new(
         "arm32",
         "arm32 specific instruction set",
+        &mut shared_defs.all_instructions,
         &shared_defs.format_registry,
     )
     .build();
 
     // CPU modes for 32-bit ARM and Thumb2.
     let mut a32 = CpuMode::new("A32");
     let mut t32 = CpuMode::new("T32");
 
     // TODO refine these.
     let narrow = shared_defs.transform_groups.by_name("narrow");
     a32.legalize_default(narrow);
     t32.legalize_default(narrow);
 
     let cpu_modes = vec![a32, t32];
 
-    TargetIsa::new("arm32", inst_group, settings, regs, cpu_modes)
+    // TODO implement arm32 recipes.
+    let recipes = Recipes::new();
+
+    // TODO implement arm32 encodings and predicates.
+    let encodings_predicates = InstructionPredicateMap::new();
+
+    TargetIsa::new(
+        "arm32",
+        inst_group,
+        settings,
+        regs,
+        recipes,
+        cpu_modes,
+        encodings_predicates,
+    )
 }
--- a/third_party/rust/cranelift-codegen-meta/src/isa/arm64/mod.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/arm64/mod.rs
@@ -1,11 +1,12 @@
 use crate::cdsl::cpu_modes::CpuMode;
-use crate::cdsl::instructions::InstructionGroupBuilder;
+use crate::cdsl::instructions::{InstructionGroupBuilder, InstructionPredicateMap};
 use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::recipes::Recipes;
 use crate::cdsl::regs::{IsaRegs, IsaRegsBuilder, RegBankBuilder, RegClassBuilder};
 use crate::cdsl::settings::{SettingGroup, SettingGroupBuilder};
 
 use crate::shared::Definitions as SharedDefinitions;
 
 fn define_settings(_shared: &SettingGroup) -> SettingGroup {
     let setting = SettingGroupBuilder::new("arm64");
     setting.build()
@@ -46,22 +47,37 @@ fn define_registers() -> IsaRegs {
 
 pub fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
     let settings = define_settings(&shared_defs.settings);
     let regs = define_registers();
 
     let inst_group = InstructionGroupBuilder::new(
         "arm64",
         "arm64 specific instruction set",
+        &mut shared_defs.all_instructions,
         &shared_defs.format_registry,
     )
     .build();
 
     let mut a64 = CpuMode::new("A64");
 
     // TODO refine these.
     let narrow = shared_defs.transform_groups.by_name("narrow");
     a64.legalize_default(narrow);
 
     let cpu_modes = vec![a64];
 
-    TargetIsa::new("arm64", inst_group, settings, regs, cpu_modes)
+    // TODO implement arm64 recipes.
+    let recipes = Recipes::new();
+
+    // TODO implement arm64 encodings and predicates.
+    let encodings_predicates = InstructionPredicateMap::new();
+
+    TargetIsa::new(
+        "arm64",
+        inst_group,
+        settings,
+        regs,
+        recipes,
+        cpu_modes,
+        encodings_predicates,
+    )
 }
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/riscv/encodings.rs
@@ -0,0 +1,398 @@
+use crate::cdsl::ast::{Apply, Expr, Literal, VarPool};
+use crate::cdsl::encodings::{Encoding, EncodingBuilder};
+use crate::cdsl::instructions::{
+    BoundInstruction, InstSpec, InstructionPredicateNode, InstructionPredicateRegistry,
+};
+use crate::cdsl::recipes::{EncodingRecipeNumber, Recipes};
+use crate::cdsl::settings::SettingGroup;
+
+use crate::shared::types::Bool::B1;
+use crate::shared::types::Float::{F32, F64};
+use crate::shared::types::Int::{I16, I32, I64, I8};
+use crate::shared::Definitions as SharedDefinitions;
+
+use super::recipes::RecipeGroup;
+
+fn enc(inst: impl Into<InstSpec>, recipe: EncodingRecipeNumber, bits: u16) -> EncodingBuilder {
+    EncodingBuilder::new(inst.into(), recipe, bits)
+}
+
+pub struct PerCpuModeEncodings<'defs> {
+    pub inst_pred_reg: InstructionPredicateRegistry,
+    pub enc32: Vec<Encoding>,
+    pub enc64: Vec<Encoding>,
+    recipes: &'defs Recipes,
+}
+
+impl<'defs> PerCpuModeEncodings<'defs> {
+    fn new(recipes: &'defs Recipes) -> Self {
+        Self {
+            inst_pred_reg: InstructionPredicateRegistry::new(),
+            enc32: Vec::new(),
+            enc64: Vec::new(),
+            recipes,
+        }
+    }
+    fn add32(&mut self, encoding: EncodingBuilder) {
+        self.enc32
+            .push(encoding.build(self.recipes, &mut self.inst_pred_reg));
+    }
+    fn add64(&mut self, encoding: EncodingBuilder) {
+        self.enc64
+            .push(encoding.build(self.recipes, &mut self.inst_pred_reg));
+    }
+}
+
+// The low 7 bits of a RISC-V instruction is the base opcode. All 32-bit instructions have 11 as
+// the two low bits, with bits 6:2 determining the base opcode.
+//
+// Encbits for the 32-bit recipes are opcode[6:2] | (funct3 << 5) | ...
+// The functions below encode the encbits.
+
+fn load_bits(funct3: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b00000 | (funct3 << 5)
+}
+
+fn store_bits(funct3: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b01000 | (funct3 << 5)
+}
+
+fn branch_bits(funct3: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b11000 | (funct3 << 5)
+}
+
+fn jalr_bits() -> u16 {
+    // This was previously accepting an argument funct3 of 3 bits and used the following formula:
+    //0b11001 | (funct3 << 5)
+    0b11001
+}
+
+fn jal_bits() -> u16 {
+    0b11011
+}
+
+fn opimm_bits(funct3: u16, funct7: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b00100 | (funct3 << 5) | (funct7 << 8)
+}
+
+fn opimm32_bits(funct3: u16, funct7: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b00110 | (funct3 << 5) | (funct7 << 8)
+}
+
+fn op_bits(funct3: u16, funct7: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    assert!(funct7 <= 0b1111111);
+    0b01100 | (funct3 << 5) | (funct7 << 8)
+}
+
+fn op32_bits(funct3: u16, funct7: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    assert!(funct7 <= 0b1111111);
+    0b01110 | (funct3 << 5) | (funct7 << 8)
+}
+
+fn lui_bits() -> u16 {
+    0b01101
+}
+
+pub fn define<'defs>(
+    shared_defs: &'defs SharedDefinitions,
+    isa_settings: &SettingGroup,
+    recipes: &'defs RecipeGroup,
+) -> PerCpuModeEncodings<'defs> {
+    // Instructions shorthands.
+    let shared = &shared_defs.instructions;
+
+    let band = shared.by_name("band");
+    let band_imm = shared.by_name("band_imm");
+    let bor = shared.by_name("bor");
+    let bor_imm = shared.by_name("bor_imm");
+    let br_icmp = shared.by_name("br_icmp");
+    let brz = shared.by_name("brz");
+    let brnz = shared.by_name("brnz");
+    let bxor = shared.by_name("bxor");
+    let bxor_imm = shared.by_name("bxor_imm");
+    let call = shared.by_name("call");
+    let call_indirect = shared.by_name("call_indirect");
+    let copy = shared.by_name("copy");
+    let copy_nop = shared.by_name("copy_nop");
+    let fill = shared.by_name("fill");
+    let iadd = shared.by_name("iadd");
+    let iadd_imm = shared.by_name("iadd_imm");
+    let iconst = shared.by_name("iconst");
+    let icmp = shared.by_name("icmp");
+    let icmp_imm = shared.by_name("icmp_imm");
+    let imul = shared.by_name("imul");
+    let ishl = shared.by_name("ishl");
+    let ishl_imm = shared.by_name("ishl_imm");
+    let isub = shared.by_name("isub");
+    let jump = shared.by_name("jump");
+    let regmove = shared.by_name("regmove");
+    let spill = shared.by_name("spill");
+    let sshr = shared.by_name("sshr");
+    let sshr_imm = shared.by_name("sshr_imm");
+    let ushr = shared.by_name("ushr");
+    let ushr_imm = shared.by_name("ushr_imm");
+    let return_ = shared.by_name("return");
+
+    // Recipes shorthands, prefixed with r_.
+    let r_icall = recipes.by_name("Icall");
+    let r_icopy = recipes.by_name("Icopy");
+    let r_ii = recipes.by_name("Ii");
+    let r_iicmp = recipes.by_name("Iicmp");
+    let r_iret = recipes.by_name("Iret");
+    let r_irmov = recipes.by_name("Irmov");
+    let r_iz = recipes.by_name("Iz");
+    let r_gp_sp = recipes.by_name("GPsp");
+    let r_gp_fi = recipes.by_name("GPfi");
+    let r_r = recipes.by_name("R");
+    let r_ricmp = recipes.by_name("Ricmp");
+    let r_rshamt = recipes.by_name("Rshamt");
+    let r_sb = recipes.by_name("SB");
+    let r_sb_zero = recipes.by_name("SBzero");
+    let r_stacknull = recipes.by_name("stacknull");
+    let r_u = recipes.by_name("U");
+    let r_uj = recipes.by_name("UJ");
+    let r_uj_call = recipes.by_name("UJcall");
+
+    // Predicates shorthands.
+    let use_m = isa_settings.predicate_by_name("use_m");
+
+    // Definitions.
+    let mut e = PerCpuModeEncodings::new(&recipes.recipes);
+
+    // Basic arithmetic binary instructions are encoded in an R-type instruction.
+    for &(inst, inst_imm, f3, f7) in &[
+        (iadd, Some(iadd_imm), 0b000, 0b0000000),
+        (isub, None, 0b000, 0b0100000),
+        (bxor, Some(bxor_imm), 0b100, 0b0000000),
+        (bor, Some(bor_imm), 0b110, 0b0000000),
+        (band, Some(band_imm), 0b111, 0b0000000),
+    ] {
+        e.add32(enc(inst.bind(I32), r_r, op_bits(f3, f7)));
+        e.add64(enc(inst.bind(I64), r_r, op_bits(f3, f7)));
+
+        // Immediate versions for add/xor/or/and.
+        if let Some(inst_imm) = inst_imm {
+            e.add32(enc(inst_imm.bind(I32), r_ii, opimm_bits(f3, 0)));
+            e.add64(enc(inst_imm.bind(I64), r_ii, opimm_bits(f3, 0)));
+        }
+    }
+
+    // 32-bit ops in RV64.
+    e.add64(enc(iadd.bind(I32), r_r, op32_bits(0b000, 0b0000000)));
+    e.add64(enc(isub.bind(I32), r_r, op32_bits(0b000, 0b0100000)));
+    // There are no andiw/oriw/xoriw variations.
+    e.add64(enc(iadd_imm.bind(I32), r_ii, opimm32_bits(0b000, 0)));
+
+    // Use iadd_imm with %x0 to materialize constants.
+    e.add32(enc(iconst.bind(I32), r_iz, opimm_bits(0b0, 0)));
+    e.add64(enc(iconst.bind(I32), r_iz, opimm_bits(0b0, 0)));
+    e.add64(enc(iconst.bind(I64), r_iz, opimm_bits(0b0, 0)));
+
+    // Dynamic shifts have the same masking semantics as the clif base instructions.
+    for &(inst, inst_imm, f3, f7) in &[
+        (ishl, ishl_imm, 0b1, 0b0),
+        (ushr, ushr_imm, 0b101, 0b0),
+        (sshr, sshr_imm, 0b101, 0b100000),
+    ] {
+        e.add32(enc(inst.bind(I32).bind(I32), r_r, op_bits(f3, f7)));
+        e.add64(enc(inst.bind(I64).bind(I64), r_r, op_bits(f3, f7)));
+        e.add64(enc(inst.bind(I32).bind(I32), r_r, op32_bits(f3, f7)));
+        // Allow i32 shift amounts in 64-bit shifts.
+        e.add64(enc(inst.bind(I64).bind(I32), r_r, op_bits(f3, f7)));
+        e.add64(enc(inst.bind(I32).bind(I64), r_r, op32_bits(f3, f7)));
+
+        // Immediate shifts.
+        e.add32(enc(inst_imm.bind(I32), r_rshamt, opimm_bits(f3, f7)));
+        e.add64(enc(inst_imm.bind(I64), r_rshamt, opimm_bits(f3, f7)));
+        e.add64(enc(inst_imm.bind(I32), r_rshamt, opimm32_bits(f3, f7)));
+    }
+
+    // Signed and unsigned integer 'less than'. There are no 'w' variants for comparing 32-bit
+    // numbers in RV64.
+    {
+        let mut var_pool = VarPool::new();
+
+        // Helper that creates an instruction predicate for an instruction in the icmp family.
+        let mut icmp_instp = |bound_inst: &BoundInstruction,
+                              intcc_field: &'static str|
+         -> InstructionPredicateNode {
+            let x = var_pool.create("x");
+            let y = var_pool.create("y");
+            let cc =
+                Literal::enumerator_for(shared_defs.operand_kinds.by_name("intcc"), intcc_field);
+            Apply::new(
+                bound_inst.clone().into(),
+                vec![Expr::Literal(cc), Expr::Var(x), Expr::Var(y)],
+            )
+            .inst_predicate(&shared_defs.format_registry, &var_pool)
+            .unwrap()
+        };
+
+        let icmp_i32 = icmp.bind(I32);
+        let icmp_i64 = icmp.bind(I64);
+        e.add32(
+            enc(icmp_i32.clone(), r_ricmp, op_bits(0b010, 0b0000000))
+                .inst_predicate(icmp_instp(&icmp_i32, "slt")),
+        );
+        e.add64(
+            enc(icmp_i64.clone(), r_ricmp, op_bits(0b010, 0b0000000))
+                .inst_predicate(icmp_instp(&icmp_i64, "slt")),
+        );
+
+        e.add32(
+            enc(icmp_i32.clone(), r_ricmp, op_bits(0b011, 0b0000000))
+                .inst_predicate(icmp_instp(&icmp_i32, "ult")),
+        );
+        e.add64(
+            enc(icmp_i64.clone(), r_ricmp, op_bits(0b011, 0b0000000))
+                .inst_predicate(icmp_instp(&icmp_i64, "ult")),
+        );
+
+        // Immediate variants.
+        let icmp_i32 = icmp_imm.bind(I32);
+        let icmp_i64 = icmp_imm.bind(I64);
+        e.add32(
+            enc(icmp_i32.clone(), r_iicmp, opimm_bits(0b010, 0))
+                .inst_predicate(icmp_instp(&icmp_i32, "slt")),
+        );
+        e.add64(
+            enc(icmp_i64.clone(), r_iicmp, opimm_bits(0b010, 0))
+                .inst_predicate(icmp_instp(&icmp_i64, "slt")),
+        );
+
+        e.add32(
+            enc(icmp_i32.clone(), r_iicmp, opimm_bits(0b011, 0))
+                .inst_predicate(icmp_instp(&icmp_i32, "ult")),
+        );
+        e.add64(
+            enc(icmp_i64.clone(), r_iicmp, opimm_bits(0b011, 0))
+                .inst_predicate(icmp_instp(&icmp_i64, "ult")),
+        );
+    }
+
+    // Integer constants with the low 12 bits clear are materialized by lui.
+    e.add32(enc(iconst.bind(I32), r_u, lui_bits()));
+    e.add64(enc(iconst.bind(I32), r_u, lui_bits()));
+    e.add64(enc(iconst.bind(I64), r_u, lui_bits()));
+
+    // "M" Standard Extension for Integer Multiplication and Division.
+    // Gated by the `use_m` flag.
+    e.add32(enc(imul.bind(I32), r_r, op_bits(0b000, 0b00000001)).isa_predicate(use_m));
+    e.add64(enc(imul.bind(I64), r_r, op_bits(0b000, 0b00000001)).isa_predicate(use_m));
+    e.add64(enc(imul.bind(I32), r_r, op32_bits(0b000, 0b00000001)).isa_predicate(use_m));
+
+    // Control flow.
+
+    // Unconditional branches.
+    e.add32(enc(jump, r_uj, jal_bits()));
+    e.add64(enc(jump, r_uj, jal_bits()));
+    e.add32(enc(call, r_uj_call, jal_bits()));
+    e.add64(enc(call, r_uj_call, jal_bits()));
+
+    // Conditional branches.
+    {
+        let mut var_pool = VarPool::new();
+
+        // Helper that creates an instruction predicate for an instruction in the icmp family.
+        let mut br_icmp_instp = |bound_inst: &BoundInstruction,
+                                 intcc_field: &'static str|
+         -> InstructionPredicateNode {
+            let x = var_pool.create("x");
+            let y = var_pool.create("y");
+            let dest = var_pool.create("dest");
+            let args = var_pool.create("args");
+            let cc =
+                Literal::enumerator_for(shared_defs.operand_kinds.by_name("intcc"), intcc_field);
+            Apply::new(
+                bound_inst.clone().into(),
+                vec![
+                    Expr::Literal(cc),
+                    Expr::Var(x),
+                    Expr::Var(y),
+                    Expr::Var(dest),
+                    Expr::Var(args),
+                ],
+            )
+            .inst_predicate(&shared_defs.format_registry, &var_pool)
+            .unwrap()
+        };
+
+        let br_icmp_i32 = br_icmp.bind(I32);
+        let br_icmp_i64 = br_icmp.bind(I64);
+        for &(cond, f3) in &[
+            ("eq", 0b000),
+            ("ne", 0b001),
+            ("slt", 0b100),
+            ("sge", 0b101),
+            ("ult", 0b110),
+            ("uge", 0b111),
+        ] {
+            e.add32(
+                enc(br_icmp_i32.clone(), r_sb, branch_bits(f3))
+                    .inst_predicate(br_icmp_instp(&br_icmp_i32, cond)),
+            );
+            e.add64(
+                enc(br_icmp_i64.clone(), r_sb, branch_bits(f3))
+                    .inst_predicate(br_icmp_instp(&br_icmp_i64, cond)),
+            );
+        }
+    }
+
+    for &(inst, f3) in &[(brz, 0b000), (brnz, 0b001)] {
+        e.add32(enc(inst.bind(I32), r_sb_zero, branch_bits(f3)));
+        e.add64(enc(inst.bind(I64), r_sb_zero, branch_bits(f3)));
+        e.add32(enc(inst.bind(B1), r_sb_zero, branch_bits(f3)));
+        e.add64(enc(inst.bind(B1), r_sb_zero, branch_bits(f3)));
+    }
+
+    // Returns are a special case of jalr_bits using %x1 to hold the return address.
+    // The return address is provided by a special-purpose `link` return value that
+    // is added by legalize_signature().
+    e.add32(enc(return_, r_iret, jalr_bits()));
+    e.add64(enc(return_, r_iret, jalr_bits()));
+    e.add32(enc(call_indirect.bind(I32), r_icall, jalr_bits()));
+    e.add64(enc(call_indirect.bind(I64), r_icall, jalr_bits()));
+
+    // Spill and fill.
+    e.add32(enc(spill.bind(I32), r_gp_sp, store_bits(0b010)));
+    e.add64(enc(spill.bind(I32), r_gp_sp, store_bits(0b010)));
+    e.add64(enc(spill.bind(I64), r_gp_sp, store_bits(0b011)));
+    e.add32(enc(fill.bind(I32), r_gp_fi, load_bits(0b010)));
+    e.add64(enc(fill.bind(I32), r_gp_fi, load_bits(0b010)));
+    e.add64(enc(fill.bind(I64), r_gp_fi, load_bits(0b011)));
+
+    // Register copies.
+    e.add32(enc(copy.bind(I32), r_icopy, opimm_bits(0b000, 0)));
+    e.add64(enc(copy.bind(I64), r_icopy, opimm_bits(0b000, 0)));
+    e.add64(enc(copy.bind(I32), r_icopy, opimm32_bits(0b000, 0)));
+
+    e.add32(enc(regmove.bind(I32), r_irmov, opimm_bits(0b000, 0)));
+    e.add64(enc(regmove.bind(I64), r_irmov, opimm_bits(0b000, 0)));
+    e.add64(enc(regmove.bind(I32), r_irmov, opimm32_bits(0b000, 0)));
+
+    e.add32(enc(copy.bind(B1), r_icopy, opimm_bits(0b000, 0)));
+    e.add64(enc(copy.bind(B1), r_icopy, opimm_bits(0b000, 0)));
+    e.add32(enc(regmove.bind(B1), r_irmov, opimm_bits(0b000, 0)));
+    e.add64(enc(regmove.bind(B1), r_irmov, opimm_bits(0b000, 0)));
+
+    // Stack-slot-to-the-same-stack-slot copy, which is guaranteed to turn
+    // into a no-op.
+    // The same encoding is generated for both the 64- and 32-bit architectures.
+    for &ty in &[I64, I32, I16, I8] {
+        e.add32(enc(copy_nop.bind(ty), r_stacknull, 0));
+        e.add64(enc(copy_nop.bind(ty), r_stacknull, 0));
+    }
+    for &ty in &[F64, F32] {
+        e.add32(enc(copy_nop.bind(ty), r_stacknull, 0));
+        e.add64(enc(copy_nop.bind(ty), r_stacknull, 0));
+    }
+
+    e
+}
--- a/third_party/rust/cranelift-codegen-meta/src/isa/riscv/mod.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/riscv/mod.rs
@@ -3,16 +3,19 @@ use crate::cdsl::instructions::Instructi
 use crate::cdsl::isa::TargetIsa;
 use crate::cdsl::regs::{IsaRegs, IsaRegsBuilder, RegBankBuilder, RegClassBuilder};
 use crate::cdsl::settings::{PredicateNode, SettingGroup, SettingGroupBuilder};
 
 use crate::shared::types::Float::{F32, F64};
 use crate::shared::types::Int::{I32, I64};
 use crate::shared::Definitions as SharedDefinitions;
 
+mod encodings;
+mod recipes;
+
 fn define_settings(shared: &SettingGroup) -> SettingGroup {
     let mut setting = SettingGroupBuilder::new("riscv");
 
     let supports_m = setting.add_bool(
         "supports_m",
         "CPU supports the 'M' extension (mul/div)",
         false,
     );
@@ -84,16 +87,17 @@ fn define_registers() -> IsaRegs {
 
 pub fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
     let settings = define_settings(&shared_defs.settings);
     let regs = define_registers();
 
     let inst_group = InstructionGroupBuilder::new(
         "riscv",
         "riscv specific instruction set",
+        &mut shared_defs.all_instructions,
         &shared_defs.format_registry,
     )
     .build();
 
     // CPU modes for 32-bit and 64-bit operation.
     let mut rv_32 = CpuMode::new("RV32");
     let mut rv_64 = CpuMode::new("RV64");
 
@@ -107,12 +111,29 @@ pub fn define(shared_defs: &mut SharedDe
 
     rv_64.legalize_monomorphic(expand);
     rv_64.legalize_default(narrow);
     rv_64.legalize_type(I32, expand);
     rv_64.legalize_type(I64, expand);
     rv_64.legalize_type(F32, expand);
     rv_64.legalize_type(F64, expand);
 
+    let recipes = recipes::define(shared_defs, &regs);
+
+    let encodings = encodings::define(shared_defs, &settings, &recipes);
+    rv_32.set_encodings(encodings.enc32);
+    rv_64.set_encodings(encodings.enc64);
+    let encodings_predicates = encodings.inst_pred_reg.extract();
+
+    let recipes = recipes.collect();
+
     let cpu_modes = vec![rv_32, rv_64];
 
-    TargetIsa::new("riscv", inst_group, settings, regs, cpu_modes)
+    TargetIsa::new(
+        "riscv",
+        inst_group,
+        settings,
+        regs,
+        recipes,
+        cpu_modes,
+        encodings_predicates,
+    )
 }
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/riscv/recipes.rs
@@ -0,0 +1,275 @@
+use std::collections::HashMap;
+
+use crate::cdsl::formats::FormatRegistry;
+use crate::cdsl::instructions::InstructionPredicate;
+use crate::cdsl::recipes::{EncodingRecipeBuilder, EncodingRecipeNumber, Recipes, Stack};
+use crate::cdsl::regs::IsaRegs;
+use crate::shared::Definitions as SharedDefinitions;
+
+/// An helper to create recipes and use them when defining the RISCV encodings.
+pub struct RecipeGroup<'formats> {
+    /// Memoized format registry, to pass it to the builders.
+    formats: &'formats FormatRegistry,
+
+    /// The actualy list of recipes explicitly created in this file.
+    pub recipes: Recipes,
+
+    /// Provides fast lookup from a name to an encoding recipe.
+    name_to_recipe: HashMap<String, EncodingRecipeNumber>,
+}
+
+impl<'formats> RecipeGroup<'formats> {
+    fn new(formats: &'formats FormatRegistry) -> Self {
+        Self {
+            formats,
+            recipes: Recipes::new(),
+            name_to_recipe: HashMap::new(),
+        }
+    }
+
+    fn push(&mut self, builder: EncodingRecipeBuilder) {
+        assert!(
+            self.name_to_recipe.get(&builder.name).is_none(),
+            format!("riscv recipe '{}' created twice", builder.name)
+        );
+        let name = builder.name.clone();
+        let number = self.recipes.push(builder.build(self.formats));
+        self.name_to_recipe.insert(name, number);
+    }
+
+    pub fn by_name(&self, name: &str) -> EncodingRecipeNumber {
+        let number = *self
+            .name_to_recipe
+            .get(name)
+            .expect(&format!("unknown riscv recipe name {}", name));
+        number
+    }
+
+    pub fn collect(self) -> Recipes {
+        self.recipes
+    }
+}
+
+pub fn define<'formats>(
+    shared_defs: &'formats SharedDefinitions,
+    regs: &IsaRegs,
+) -> RecipeGroup<'formats> {
+    let formats = &shared_defs.format_registry;
+
+    // Format shorthands.
+    let f_binary = formats.by_name("Binary");
+    let f_binary_imm = formats.by_name("BinaryImm");
+    let f_branch = formats.by_name("Branch");
+    let f_branch_icmp = formats.by_name("BranchIcmp");
+    let f_call = formats.by_name("Call");
+    let f_call_indirect = formats.by_name("CallIndirect");
+    let f_int_compare = formats.by_name("IntCompare");
+    let f_int_compare_imm = formats.by_name("IntCompareImm");
+    let f_jump = formats.by_name("Jump");
+    let f_multiary = formats.by_name("MultiAry");
+    let f_regmove = formats.by_name("RegMove");
+    let f_unary = formats.by_name("Unary");
+    let f_unary_imm = formats.by_name("UnaryImm");
+
+    // Register classes shorthands.
+    let gpr = regs.class_by_name("GPR");
+
+    // Definitions.
+    let mut recipes = RecipeGroup::new(&shared_defs.format_registry);
+
+    // R-type 32-bit instructions: These are mostly binary arithmetic instructions.
+    // The encbits are `opcode[6:2] | (funct3 << 5) | (funct7 << 8)
+    recipes.push(
+        EncodingRecipeBuilder::new("R", f_binary, 4)
+            .operands_in(vec![gpr, gpr])
+            .operands_out(vec![gpr])
+            .emit("put_r(bits, in_reg0, in_reg1, out_reg0, sink);"),
+    );
+
+    // R-type with an immediate shift amount instead of rs2.
+    recipes.push(
+        EncodingRecipeBuilder::new("Rshamt", f_binary_imm, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .emit("put_rshamt(bits, in_reg0, imm.into(), out_reg0, sink);"),
+    );
+
+    // R-type encoding of an integer comparison.
+    recipes.push(
+        EncodingRecipeBuilder::new("Ricmp", f_int_compare, 4)
+            .operands_in(vec![gpr, gpr])
+            .operands_out(vec![gpr])
+            .emit("put_r(bits, in_reg0, in_reg1, out_reg0, sink);"),
+    );
+
+    let format = formats.get(f_binary_imm);
+    recipes.push(
+        EncodingRecipeBuilder::new("Ii", f_binary_imm, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .inst_predicate(InstructionPredicate::new_is_signed_int(
+                format, "imm", 12, 0,
+            ))
+            .emit("put_i(bits, in_reg0, imm.into(), out_reg0, sink);"),
+    );
+
+    // I-type instruction with a hardcoded %x0 rs1.
+    let format = formats.get(f_unary_imm);
+    recipes.push(
+        EncodingRecipeBuilder::new("Iz", f_unary_imm, 4)
+            .operands_out(vec![gpr])
+            .inst_predicate(InstructionPredicate::new_is_signed_int(
+                format, "imm", 12, 0,
+            ))
+            .emit("put_i(bits, 0, imm.into(), out_reg0, sink);"),
+    );
+
+    // I-type encoding of an integer comparison.
+    let format = formats.get(f_int_compare_imm);
+    recipes.push(
+        EncodingRecipeBuilder::new("Iicmp", f_int_compare_imm, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .inst_predicate(InstructionPredicate::new_is_signed_int(
+                format, "imm", 12, 0,
+            ))
+            .emit("put_i(bits, in_reg0, imm.into(), out_reg0, sink);"),
+    );
+
+    // I-type encoding for `jalr` as a return instruction. We won't use the immediate offset.  The
+    // variable return values are not encoded.
+    recipes.push(EncodingRecipeBuilder::new("Iret", f_multiary, 4).emit(
+        r#"
+                    // Return instructions are always a jalr to %x1.
+                    // The return address is provided as a special-purpose link argument.
+                    put_i(
+                        bits,
+                        1, // rs1 = %x1
+                        0, // no offset.
+                        0, // rd = %x0: no address written.
+                        sink,
+                    );
+                "#,
+    ));
+
+    // I-type encoding for `jalr` as a call_indirect.
+    recipes.push(
+        EncodingRecipeBuilder::new("Icall", f_call_indirect, 4)
+            .operands_in(vec![gpr])
+            .emit(
+                r#"
+                    // call_indirect instructions are jalr with rd=%x1.
+                    put_i(
+                        bits,
+                        in_reg0,
+                        0, // no offset.
+                        1, // rd = %x1: link register.
+                        sink,
+                    );
+                "#,
+            ),
+    );
+
+    // Copy of a GPR is implemented as addi x, 0.
+    recipes.push(
+        EncodingRecipeBuilder::new("Icopy", f_unary, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .emit("put_i(bits, in_reg0, 0, out_reg0, sink);"),
+    );
+
+    // Same for a GPR regmove.
+    recipes.push(
+        EncodingRecipeBuilder::new("Irmov", f_regmove, 4)
+            .operands_in(vec![gpr])
+            .emit("put_i(bits, src, 0, dst, sink);"),
+    );
+
+    // U-type instructions have a 20-bit immediate that targets bits 12-31.
+    let format = formats.get(f_unary_imm);
+    recipes.push(
+        EncodingRecipeBuilder::new("U", f_unary_imm, 4)
+            .operands_out(vec![gpr])
+            .inst_predicate(InstructionPredicate::new_is_signed_int(
+                format, "imm", 32, 12,
+            ))
+            .emit("put_u(bits, imm.into(), out_reg0, sink);"),
+    );
+
+    // UJ-type unconditional branch instructions.
+    recipes.push(
+        EncodingRecipeBuilder::new("UJ", f_jump, 4)
+            .branch_range((0, 21))
+            .emit(
+                r#"
+                    let dest = i64::from(func.offsets[destination]);
+                    let disp = dest - i64::from(sink.offset());
+                    put_uj(bits, disp, 0, sink);
+                "#,
+            ),
+    );
+
+    recipes.push(EncodingRecipeBuilder::new("UJcall", f_call, 4).emit(
+        r#"
+                    sink.reloc_external(Reloc::RiscvCall,
+                                        &func.dfg.ext_funcs[func_ref].name,
+                                        0);
+                    // rd=%x1 is the standard link register.
+                    put_uj(bits, 0, 1, sink);
+                "#,
+    ));
+
+    // SB-type branch instructions.
+    recipes.push(
+        EncodingRecipeBuilder::new("SB", f_branch_icmp, 4)
+            .operands_in(vec![gpr, gpr])
+            .branch_range((0, 13))
+            .emit(
+                r#"
+                    let dest = i64::from(func.offsets[destination]);
+                    let disp = dest - i64::from(sink.offset());
+                    put_sb(bits, disp, in_reg0, in_reg1, sink);
+                "#,
+            ),
+    );
+
+    // SB-type branch instruction with rs2 fixed to zero.
+    recipes.push(
+        EncodingRecipeBuilder::new("SBzero", f_branch, 4)
+            .operands_in(vec![gpr])
+            .branch_range((0, 13))
+            .emit(
+                r#"
+                    let dest = i64::from(func.offsets[destination]);
+                    let disp = dest - i64::from(sink.offset());
+                    put_sb(bits, disp, in_reg0, 0, sink);
+                "#,
+            ),
+    );
+
+    // Spill of a GPR.
+    recipes.push(
+        EncodingRecipeBuilder::new("GPsp", f_unary, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![Stack::new(gpr)])
+            .emit("unimplemented!();"),
+    );
+
+    // Fill of a GPR.
+    recipes.push(
+        EncodingRecipeBuilder::new("GPfi", f_unary, 4)
+            .operands_in(vec![Stack::new(gpr)])
+            .operands_out(vec![gpr])
+            .emit("unimplemented!();"),
+    );
+
+    // Stack-slot to same stack-slot copy, which is guaranteed to turn into a no-op.
+    recipes.push(
+        EncodingRecipeBuilder::new("stacknull", f_unary, 0)
+            .operands_in(vec![Stack::new(gpr)])
+            .operands_out(vec![Stack::new(gpr)])
+            .emit(""),
+    );
+
+    recipes
+}
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs
@@ -0,0 +1,1670 @@
+#![allow(non_snake_case)]
+
+use std::collections::HashMap;
+
+use crate::cdsl::encodings::{Encoding, EncodingBuilder};
+use crate::cdsl::instructions::{
+    BoundInstruction, InstSpec, Instruction, InstructionGroup, InstructionPredicate,
+    InstructionPredicateNode, InstructionPredicateRegistry,
+};
+use crate::cdsl::recipes::{EncodingRecipe, EncodingRecipeNumber, Recipes};
+use crate::cdsl::settings::{SettingGroup, SettingPredicateNumber};
+use crate::cdsl::types::ValueType;
+use crate::shared::types::Bool::{B1, B16, B32, B64, B8};
+use crate::shared::types::Float::{F32, F64};
+use crate::shared::types::Int::{I16, I32, I64, I8};
+use crate::shared::Definitions as SharedDefinitions;
+
+use super::recipes::{RecipeGroup, Template};
+
+pub struct PerCpuModeEncodings {
+    pub enc32: Vec<Encoding>,
+    pub enc64: Vec<Encoding>,
+    pub recipes: Recipes,
+    recipes_by_name: HashMap<String, EncodingRecipeNumber>,
+    pub inst_pred_reg: InstructionPredicateRegistry,
+}
+
+impl PerCpuModeEncodings {
+    fn new() -> Self {
+        Self {
+            enc32: Vec::new(),
+            enc64: Vec::new(),
+            recipes: Recipes::new(),
+            recipes_by_name: HashMap::new(),
+            inst_pred_reg: InstructionPredicateRegistry::new(),
+        }
+    }
+
+    fn add_recipe(&mut self, recipe: EncodingRecipe) -> EncodingRecipeNumber {
+        if let Some(found_index) = self.recipes_by_name.get(&recipe.name) {
+            assert!(
+                self.recipes[*found_index] == recipe,
+                format!(
+                    "trying to insert different recipes with a same name ({})",
+                    recipe.name
+                )
+            );
+            *found_index
+        } else {
+            let recipe_name = recipe.name.clone();
+            let index = self.recipes.push(recipe);
+            self.recipes_by_name.insert(recipe_name, index);
+            index
+        }
+    }
+
+    fn make_encoding<T>(
+        &mut self,
+        inst: InstSpec,
+        template: Template,
+        builder_closure: T,
+    ) -> Encoding
+    where
+        T: FnOnce(EncodingBuilder) -> EncodingBuilder,
+    {
+        let (recipe, bits) = template.build();
+        let recipe_number = self.add_recipe(recipe);
+        let builder = EncodingBuilder::new(inst.into(), recipe_number, bits);
+        builder_closure(builder).build(&self.recipes, &mut self.inst_pred_reg)
+    }
+
+    fn enc32_func<T>(&mut self, inst: impl Into<InstSpec>, template: Template, builder_closure: T)
+    where
+        T: FnOnce(EncodingBuilder) -> EncodingBuilder,
+    {
+        let encoding = self.make_encoding(inst.into(), template, builder_closure);
+        self.enc32.push(encoding);
+    }
+    fn enc32(&mut self, inst: impl Into<InstSpec>, template: Template) {
+        self.enc32_func(inst, template, |x| x);
+    }
+    fn enc32_isap(
+        &mut self,
+        inst: impl Into<InstSpec>,
+        template: Template,
+        isap: SettingPredicateNumber,
+    ) {
+        self.enc32_func(inst, template, |encoding| encoding.isa_predicate(isap));
+    }
+    fn enc32_instp(
+        &mut self,
+        inst: impl Into<InstSpec>,
+        template: Template,
+        instp: InstructionPredicateNode,
+    ) {
+        self.enc32_func(inst, template, |encoding| encoding.inst_predicate(instp));
+    }
+    fn enc32_rec(&mut self, inst: impl Into<InstSpec>, recipe: &EncodingRecipe, bits: u16) {
+        let recipe_number = self.add_recipe(recipe.clone());
+        let builder = EncodingBuilder::new(inst.into(), recipe_number, bits);
+        let encoding = builder.build(&self.recipes, &mut self.inst_pred_reg);
+        self.enc32.push(encoding);
+    }
+
+    fn enc64_func<T>(&mut self, inst: impl Into<InstSpec>, template: Template, builder_closure: T)
+    where
+        T: FnOnce(EncodingBuilder) -> EncodingBuilder,
+    {
+        let encoding = self.make_encoding(inst.into(), template, builder_closure);
+        self.enc64.push(encoding);
+    }
+    fn enc64(&mut self, inst: impl Into<InstSpec>, template: Template) {
+        self.enc64_func(inst, template, |x| x);
+    }
+    fn enc64_isap(
+        &mut self,
+        inst: impl Into<InstSpec>,
+        template: Template,
+        isap: SettingPredicateNumber,
+    ) {
+        self.enc64_func(inst, template, |encoding| encoding.isa_predicate(isap));
+    }
+    fn enc64_instp(
+        &mut self,
+        inst: impl Into<InstSpec>,
+        template: Template,
+        instp: InstructionPredicateNode,
+    ) {
+        self.enc64_func(inst, template, |encoding| encoding.inst_predicate(instp));
+    }
+    fn enc64_rec(&mut self, inst: impl Into<InstSpec>, recipe: &EncodingRecipe, bits: u16) {
+        let recipe_number = self.add_recipe(recipe.clone());
+        let builder = EncodingBuilder::new(inst.into(), recipe_number, bits);
+        let encoding = builder.build(&self.recipes, &mut self.inst_pred_reg);
+        self.enc64.push(encoding);
+    }
+
+    /// Add encodings for `inst.i32` to X86_32.
+    /// Add encodings for `inst.i32` to X86_64 with and without REX.
+    /// Add encodings for `inst.i64` to X86_64 with a REX.W prefix.
+    fn enc_i32_i64(&mut self, inst: impl Into<InstSpec>, template: Template) {
+        let inst: InstSpec = inst.into();
+        self.enc32(inst.bind(I32), template.nonrex());
+
+        // REX-less encoding must come after REX encoding so we don't use it by default. Otherwise
+        // reg-alloc would never use r8 and up.
+        self.enc64(inst.bind(I32), template.rex());
+        self.enc64(inst.bind(I32), template.nonrex());
+        self.enc64(inst.bind(I64), template.rex().w());
+    }
+
+    /// Add encodings for `inst.i32` to X86_32.
+    /// Add encodings for `inst.i32` to X86_64 with and without REX.
+    /// Add encodings for `inst.i64` to X86_64 with a REX.W prefix.
+    fn enc_i32_i64_instp(
+        &mut self,
+        inst: &Instruction,
+        template: Template,
+        instp: InstructionPredicateNode,
+    ) {
+        self.enc32_func(inst.bind(I32), template.nonrex(), |builder| {
+            builder.inst_predicate(instp.clone())
+        });
+
+        // REX-less encoding must come after REX encoding so we don't use it by default. Otherwise
+        // reg-alloc would never use r8 and up.
+        self.enc64_func(inst.bind(I32), template.rex(), |builder| {
+            builder.inst_predicate(instp.clone())
+        });
+        self.enc64_func(inst.bind(I32), template.nonrex(), |builder| {
+            builder.inst_predicate(instp.clone())
+        });
+        self.enc64_func(inst.bind(I64), template.rex().w(), |builder| {
+            builder.inst_predicate(instp)
+        });
+    }
+
+    /// Add encodings for `inst` to X86_64 with and without a REX prefix.
+    fn enc_x86_64(&mut self, inst: impl Into<InstSpec> + Clone, template: Template) {
+        // See above comment about the ordering of rex vs non-rex encodings.
+        self.enc64(inst.clone(), template.rex());
+        self.enc64(inst, template);
+    }
+
+    /// Add encodings for `inst` to X86_64 with and without a REX prefix.
+    fn enc_x86_64_instp(
+        &mut self,
+        inst: impl Clone + Into<InstSpec>,
+        template: Template,
+        instp: InstructionPredicateNode,
+    ) {
+        // See above comment about the ordering of rex vs non-rex encodings.
+        self.enc64_func(inst.clone(), template.rex(), |builder| {
+            builder.inst_predicate(instp.clone())
+        });
+        self.enc64_func(inst, template, |builder| builder.inst_predicate(instp));
+    }
+    fn enc_x86_64_isap(
+        &mut self,
+        inst: impl Clone + Into<InstSpec>,
+        template: Template,
+        isap: SettingPredicateNumber,
+    ) {
+        // See above comment about the ordering of rex vs non-rex encodings.
+        self.enc64_isap(inst.clone(), template.rex(), isap);
+        self.enc64_isap(inst, template, isap);
+    }
+
+    /// Add all three encodings for `inst`:
+    /// - X86_32
+    /// - X86_64 with and without the REX prefix.
+    fn enc_both(&mut self, inst: impl Clone + Into<InstSpec>, template: Template) {
+        self.enc32(inst.clone(), template.clone());
+        self.enc_x86_64(inst, template);
+    }
+    fn enc_both_isap(
+        &mut self,
+        inst: BoundInstruction,
+        template: Template,
+        isap: SettingPredicateNumber,
+    ) {
+        self.enc32_isap(inst.clone(), template.clone(), isap);
+        self.enc_x86_64_isap(inst, template, isap);
+    }
+    fn enc_both_instp(
+        &mut self,
+        inst: BoundInstruction,
+        template: Template,
+        instp: InstructionPredicateNode,
+    ) {
+        self.enc32_instp(inst.clone(), template.clone(), instp.clone());
+        self.enc_x86_64_instp(inst, template, instp);
+    }
+
+    /// Add encodings for `inst.i32` to X86_32.
+    /// Add encodings for `inst.i32` to X86_64 with and without REX.
+    /// Add encodings for `inst.i64` to X86_64 with a REX prefix, using the `w_bit`
+    /// argument to determine whether or not to set the REX.W bit.
+    fn enc_i32_i64_ld_st(&mut self, inst: &Instruction, w_bit: bool, template: Template) {
+        self.enc32(inst.clone().bind(I32).bind_any(), template.clone());
+
+        // REX-less encoding must come after REX encoding so we don't use it by
+        // default. Otherwise reg-alloc would never use r8 and up.
+        self.enc64(inst.clone().bind(I32).bind_any(), template.clone().rex());
+        self.enc64(inst.clone().bind(I32).bind_any(), template.clone());
+
+        if w_bit {
+            self.enc64(inst.clone().bind(I64).bind_any(), template.rex().w());
+        } else {
+            self.enc64(inst.clone().bind(I64).bind_any(), template.clone().rex());
+            self.enc64(inst.clone().bind(I64).bind_any(), template);
+        }
+    }
+
+    /// Add the same encoding to both X86_32 and X86_64; assumes configuration (e.g. REX, operand binding) has already happened
+    fn enc_32_64_isap(
+        &mut self,
+        inst: BoundInstruction,
+        template: Template,
+        isap: SettingPredicateNumber,
+    ) {
+        self.enc32_isap(inst.clone(), template.clone(), isap);
+        self.enc64_isap(inst, template, isap);
+    }
+}
+
+// Definitions.
+
+pub fn define(
+    shared_defs: &SharedDefinitions,
+    settings: &SettingGroup,
+    x86: &InstructionGroup,
+    r: &RecipeGroup,
+) -> PerCpuModeEncodings {
+    let shared = &shared_defs.instructions;
+    let formats = &shared_defs.format_registry;
+
+    // Shorthands for instructions.
+    let adjust_sp_down = shared.by_name("adjust_sp_down");
+    let adjust_sp_down_imm = shared.by_name("adjust_sp_down_imm");
+    let adjust_sp_up_imm = shared.by_name("adjust_sp_up_imm");
+    let band = shared.by_name("band");
+    let band_imm = shared.by_name("band_imm");
+    let band_not = shared.by_name("band_not");
+    let bconst = shared.by_name("bconst");
+    let bint = shared.by_name("bint");
+    let bitcast = shared.by_name("bitcast");
+    let bnot = shared.by_name("bnot");
+    let bor = shared.by_name("bor");
+    let bor_imm = shared.by_name("bor_imm");
+    let brff = shared.by_name("brff");
+    let brif = shared.by_name("brif");
+    let brnz = shared.by_name("brnz");
+    let brz = shared.by_name("brz");
+    let bxor = shared.by_name("bxor");
+    let bxor_imm = shared.by_name("bxor_imm");
+    let call = shared.by_name("call");
+    let call_indirect = shared.by_name("call_indirect");
+    let ceil = shared.by_name("ceil");
+    let clz = shared.by_name("clz");
+    let copy = shared.by_name("copy");
+    let copy_nop = shared.by_name("copy_nop");
+    let copy_special = shared.by_name("copy_special");
+    let ctz = shared.by_name("ctz");
+    let debugtrap = shared.by_name("debugtrap");
+    let f32const = shared.by_name("f32const");
+    let f64const = shared.by_name("f64const");
+    let fadd = shared.by_name("fadd");
+    let fcmp = shared.by_name("fcmp");
+    let fcvt_from_sint = shared.by_name("fcvt_from_sint");
+    let fdemote = shared.by_name("fdemote");
+    let fdiv = shared.by_name("fdiv");
+    let ffcmp = shared.by_name("ffcmp");
+    let fill = shared.by_name("fill");
+    let floor = shared.by_name("floor");
+    let fmul = shared.by_name("fmul");
+    let fpromote = shared.by_name("fpromote");
+    let fsub = shared.by_name("fsub");
+    let func_addr = shared.by_name("func_addr");
+    let iadd = shared.by_name("iadd");
+    let iadd_imm = shared.by_name("iadd_imm");
+    let icmp = shared.by_name("icmp");
+    let icmp_imm = shared.by_name("icmp_imm");
+    let iconst = shared.by_name("iconst");
+    let ifcmp = shared.by_name("ifcmp");
+    let ifcmp_imm = shared.by_name("ifcmp_imm");
+    let ifcmp_sp = shared.by_name("ifcmp_sp");
+    let imul = shared.by_name("imul");
+    let indirect_jump_table_br = shared.by_name("indirect_jump_table_br");
+    let insertlane = shared.by_name("insertlane");
+    let ireduce = shared.by_name("ireduce");
+    let ishl = shared.by_name("ishl");
+    let ishl_imm = shared.by_name("ishl_imm");
+    let istore16 = shared.by_name("istore16");
+    let istore16_complex = shared.by_name("istore16_complex");
+    let istore32 = shared.by_name("istore32");
+    let istore32_complex = shared.by_name("istore32_complex");
+    let istore8 = shared.by_name("istore8");
+    let istore8_complex = shared.by_name("istore8_complex");
+    let isub = shared.by_name("isub");
+    let jump = shared.by_name("jump");
+    let jump_table_base = shared.by_name("jump_table_base");
+    let jump_table_entry = shared.by_name("jump_table_entry");
+    let load = shared.by_name("load");
+    let load_complex = shared.by_name("load_complex");
+    let nearest = shared.by_name("nearest");
+    let popcnt = shared.by_name("popcnt");
+    let raw_bitcast = shared.by_name("raw_bitcast");
+    let regfill = shared.by_name("regfill");
+    let regmove = shared.by_name("regmove");
+    let regspill = shared.by_name("regspill");
+    let return_ = shared.by_name("return");
+    let rotl = shared.by_name("rotl");
+    let rotl_imm = shared.by_name("rotl_imm");
+    let rotr = shared.by_name("rotr");
+    let rotr_imm = shared.by_name("rotr_imm");
+    let scalar_to_vector = shared.by_name("scalar_to_vector");
+    let selectif = shared.by_name("selectif");
+    let sextend = shared.by_name("sextend");
+    let sload16 = shared.by_name("sload16");
+    let sload16_complex = shared.by_name("sload16_complex");
+    let sload32 = shared.by_name("sload32");
+    let sload32_complex = shared.by_name("sload32_complex");
+    let sload8 = shared.by_name("sload8");
+    let sload8_complex = shared.by_name("sload8_complex");
+    let spill = shared.by_name("spill");
+    let sqrt = shared.by_name("sqrt");
+    let sshr = shared.by_name("sshr");
+    let sshr_imm = shared.by_name("sshr_imm");
+    let stack_addr = shared.by_name("stack_addr");
+    let store = shared.by_name("store");
+    let store_complex = shared.by_name("store_complex");
+    let symbol_value = shared.by_name("symbol_value");
+    let trap = shared.by_name("trap");
+    let trapff = shared.by_name("trapff");
+    let trapif = shared.by_name("trapif");
+    let trueff = shared.by_name("trueff");
+    let trueif = shared.by_name("trueif");
+    let trunc = shared.by_name("trunc");
+    let uextend = shared.by_name("uextend");
+    let uload16 = shared.by_name("uload16");
+    let uload16_complex = shared.by_name("uload16_complex");
+    let uload32 = shared.by_name("uload32");
+    let uload32_complex = shared.by_name("uload32_complex");
+    let uload8 = shared.by_name("uload8");
+    let uload8_complex = shared.by_name("uload8_complex");
+    let ushr = shared.by_name("ushr");
+    let ushr_imm = shared.by_name("ushr_imm");
+    let x86_bsf = x86.by_name("x86_bsf");
+    let x86_bsr = x86.by_name("x86_bsr");
+    let x86_cvtt2si = x86.by_name("x86_cvtt2si");
+    let x86_fmax = x86.by_name("x86_fmax");
+    let x86_fmin = x86.by_name("x86_fmin");
+    let x86_pop = x86.by_name("x86_pop");
+    let x86_pshufd = x86.by_name("x86_pshufd");
+    let x86_pshufb = x86.by_name("x86_pshufb");
+    let x86_push = x86.by_name("x86_push");
+    let x86_sdivmodx = x86.by_name("x86_sdivmodx");
+    let x86_smulx = x86.by_name("x86_smulx");
+    let x86_udivmodx = x86.by_name("x86_udivmodx");
+    let x86_umulx = x86.by_name("x86_umulx");
+
+    // Shorthands for recipes.
+    let rec_adjustsp = r.template("adjustsp");
+    let rec_adjustsp_ib = r.template("adjustsp_ib");
+    let rec_adjustsp_id = r.template("adjustsp_id");
+    let rec_allones_fnaddr4 = r.template("allones_fnaddr4");
+    let rec_allones_fnaddr8 = r.template("allones_fnaddr8");
+    let rec_brfb = r.template("brfb");
+    let rec_brfd = r.template("brfd");
+    let rec_brib = r.template("brib");
+    let rec_brid = r.template("brid");
+    let rec_bsf_and_bsr = r.template("bsf_and_bsr");
+    let rec_call_id = r.template("call_id");
+    let rec_call_plt_id = r.template("call_plt_id");
+    let rec_call_r = r.template("call_r");
+    let rec_cmov = r.template("cmov");
+    let rec_copysp = r.template("copysp");
+    let rec_div = r.template("div");
+    let rec_debugtrap = r.recipe("debugtrap");
+    let rec_f32imm_z = r.template("f32imm_z");
+    let rec_f64imm_z = r.template("f64imm_z");
+    let rec_fa = r.template("fa");
+    let rec_fax = r.template("fax");
+    let rec_fcmp = r.template("fcmp");
+    let rec_fcscc = r.template("fcscc");
+    let rec_ffillSib32 = r.template("ffillSib32");
+    let rec_fillSib32 = r.template("fillSib32");
+    let rec_fld = r.template("fld");
+    let rec_fldDisp32 = r.template("fldDisp32");
+    let rec_fldDisp8 = r.template("fldDisp8");
+    let rec_fldWithIndex = r.template("fldWithIndex");
+    let rec_fldWithIndexDisp32 = r.template("fldWithIndexDisp32");
+    let rec_fldWithIndexDisp8 = r.template("fldWithIndexDisp8");
+    let rec_fnaddr4 = r.template("fnaddr4");
+    let rec_fnaddr8 = r.template("fnaddr8");
+    let rec_fregfill32 = r.template("fregfill32");
+    let rec_fregspill32 = r.template("fregspill32");
+    let rec_frmov = r.template("frmov");
+    let rec_frurm = r.template("frurm");
+    let rec_fspillSib32 = r.template("fspillSib32");
+    let rec_fst = r.template("fst");
+    let rec_fstDisp32 = r.template("fstDisp32");
+    let rec_fstDisp8 = r.template("fstDisp8");
+    let rec_fstWithIndex = r.template("fstWithIndex");
+    let rec_fstWithIndexDisp32 = r.template("fstWithIndexDisp32");
+    let rec_fstWithIndexDisp8 = r.template("fstWithIndexDisp8");
+    let rec_furm = r.template("furm");
+    let rec_furmi_rnd = r.template("furmi_rnd");
+    let rec_got_fnaddr8 = r.template("got_fnaddr8");
+    let rec_got_gvaddr8 = r.template("got_gvaddr8");
+    let rec_gvaddr4 = r.template("gvaddr4");
+    let rec_gvaddr8 = r.template("gvaddr8");
+    let rec_icscc = r.template("icscc");
+    let rec_icscc_ib = r.template("icscc_ib");
+    let rec_icscc_id = r.template("icscc_id");
+    let rec_indirect_jmp = r.template("indirect_jmp");
+    let rec_jmpb = r.template("jmpb");
+    let rec_jmpd = r.template("jmpd");
+    let rec_jt_base = r.template("jt_base");
+    let rec_jt_entry = r.template("jt_entry");
+    let rec_ld = r.template("ld");
+    let rec_ldDisp32 = r.template("ldDisp32");
+    let rec_ldDisp8 = r.template("ldDisp8");
+    let rec_ldWithIndex = r.template("ldWithIndex");
+    let rec_ldWithIndexDisp32 = r.template("ldWithIndexDisp32");
+    let rec_ldWithIndexDisp8 = r.template("ldWithIndexDisp8");
+    let rec_mulx = r.template("mulx");
+    let rec_null = r.recipe("null");
+    let rec_null_fpr = r.recipe("null_fpr");
+    let rec_pcrel_fnaddr8 = r.template("pcrel_fnaddr8");
+    let rec_pcrel_gvaddr8 = r.template("pcrel_gvaddr8");
+    let rec_popq = r.template("popq");
+    let rec_pu_id = r.template("pu_id");
+    let rec_pu_id_bool = r.template("pu_id_bool");
+    let rec_pu_iq = r.template("pu_iq");
+    let rec_pushq = r.template("pushq");
+    let rec_ret = r.template("ret");
+    let rec_r_ib = r.template("r_ib");
+    let rec_r_ib_unsigned = r.template("r_ib_unsigned");
+    let rec_r_ib_unsigned_r = r.template("r_ib_unsigned_r");
+    let rec_r_id = r.template("r_id");
+    let rec_rcmp = r.template("rcmp");
+    let rec_rcmp_ib = r.template("rcmp_ib");
+    let rec_rcmp_id = r.template("rcmp_id");
+    let rec_rcmp_sp = r.template("rcmp_sp");
+    let rec_regfill32 = r.template("regfill32");
+    let rec_regspill32 = r.template("regspill32");
+    let rec_rc = r.template("rc");
+    let rec_rfumr = r.template("rfumr");
+    let rec_rfurm = r.template("rfurm");
+    let rec_rmov = r.template("rmov");
+    let rec_rr = r.template("rr");
+    let rec_rrx = r.template("rrx");
+    let rec_setf_abcd = r.template("setf_abcd");
+    let rec_seti_abcd = r.template("seti_abcd");
+    let rec_spaddr4_id = r.template("spaddr4_id");
+    let rec_spaddr8_id = r.template("spaddr8_id");
+    let rec_spillSib32 = r.template("spillSib32");
+    let rec_st = r.template("st");
+    let rec_stacknull = r.recipe("stacknull");
+    let rec_stDisp32 = r.template("stDisp32");
+    let rec_stDisp32_abcd = r.template("stDisp32_abcd");
+    let rec_stDisp8 = r.template("stDisp8");
+    let rec_stDisp8_abcd = r.template("stDisp8_abcd");
+    let rec_stWithIndex = r.template("stWithIndex");
+    let rec_stWithIndexDisp32 = r.template("stWithIndexDisp32");
+    let rec_stWithIndexDisp32_abcd = r.template("stWithIndexDisp32_abcd");
+    let rec_stWithIndexDisp8 = r.template("stWithIndexDisp8");
+    let rec_stWithIndexDisp8_abcd = r.template("stWithIndexDisp8_abcd");
+    let rec_stWithIndex_abcd = r.template("stWithIndex_abcd");
+    let rec_st_abcd = r.template("st_abcd");
+    let rec_t8jccb_abcd = r.template("t8jccb_abcd");
+    let rec_t8jccd_abcd = r.template("t8jccd_abcd");
+    let rec_t8jccd_long = r.template("t8jccd_long");
+    let rec_tjccb = r.template("tjccb");
+    let rec_tjccd = r.template("tjccd");
+    let rec_trap = r.template("trap");
+    let rec_trapif = r.recipe("trapif");
+    let rec_trapff = r.recipe("trapff");
+    let rec_u_id = r.template("u_id");
+    let rec_umr = r.template("umr");
+    let rec_ur = r.template("ur");
+    let rec_urm = r.template("urm");
+    let rec_urm_noflags = r.template("urm_noflags");
+    let rec_urm_noflags_abcd = r.template("urm_noflags_abcd");
+
+    // Predicates shorthands.
+    let all_ones_funcaddrs_and_not_is_pic =
+        settings.predicate_by_name("all_ones_funcaddrs_and_not_is_pic");
+    let is_pic = settings.predicate_by_name("is_pic");
+    let not_all_ones_funcaddrs_and_not_is_pic =
+        settings.predicate_by_name("not_all_ones_funcaddrs_and_not_is_pic");
+    let not_is_pic = settings.predicate_by_name("not_is_pic");
+    let use_popcnt = settings.predicate_by_name("use_popcnt");
+    let use_lzcnt = settings.predicate_by_name("use_lzcnt");
+    let use_bmi1 = settings.predicate_by_name("use_bmi1");
+    let use_sse2 = settings.predicate_by_name("use_sse2");
+    let use_ssse3 = settings.predicate_by_name("use_ssse3");
+    let use_sse41 = settings.predicate_by_name("use_sse41");
+
+    // Definitions.
+    let mut e = PerCpuModeEncodings::new();
+
+    e.enc_i32_i64(iadd, rec_rr.opcodes(vec![0x01]));
+    e.enc_i32_i64(isub, rec_rr.opcodes(vec![0x29]));
+    e.enc_i32_i64(band, rec_rr.opcodes(vec![0x21]));
+    e.enc_i32_i64(bor, rec_rr.opcodes(vec![0x09]));
+    e.enc_i32_i64(bxor, rec_rr.opcodes(vec![0x31]));
+
+    // x86 has a bitwise not instruction NOT.
+    e.enc_i32_i64(bnot, rec_ur.opcodes(vec![0xf7]).rrr(2));
+
+    // Also add a `b1` encodings for the logic instructions.
+    // TODO: Should this be done with 8-bit instructions? It would improve partial register
+    // dependencies.
+    e.enc_both(band.bind(B1), rec_rr.opcodes(vec![0x21]));
+    e.enc_both(bor.bind(B1), rec_rr.opcodes(vec![0x09]));
+    e.enc_both(bxor.bind(B1), rec_rr.opcodes(vec![0x31]));
+
+    e.enc_i32_i64(imul, rec_rrx.opcodes(vec![0x0f, 0xaf]));
+    e.enc_i32_i64(x86_sdivmodx, rec_div.opcodes(vec![0xf7]).rrr(7));
+    e.enc_i32_i64(x86_udivmodx, rec_div.opcodes(vec![0xf7]).rrr(6));
+
+    e.enc_i32_i64(x86_smulx, rec_mulx.opcodes(vec![0xf7]).rrr(5));
+    e.enc_i32_i64(x86_umulx, rec_mulx.opcodes(vec![0xf7]).rrr(4));
+
+    e.enc_i32_i64(copy, rec_umr.opcodes(vec![0x89]));
+    e.enc_both(copy.bind(B1), rec_umr.opcodes(vec![0x89]));
+    e.enc_both(copy.bind(I8), rec_umr.opcodes(vec![0x89]));
+    e.enc_both(copy.bind(I16), rec_umr.opcodes(vec![0x89]));
+
+    // TODO For x86-64, only define REX forms for now, since we can't describe the
+    // special regunit immediate operands with the current constraint language.
+    for &ty in &[I8, I16, I32] {
+        e.enc32(regmove.bind(ty), rec_rmov.opcodes(vec![0x89]));
+        e.enc64(regmove.bind(ty), rec_rmov.opcodes(vec![0x89]).rex());
+    }
+    e.enc64(regmove.bind(I64), rec_rmov.opcodes(vec![0x89]).rex().w());
+    e.enc_both(regmove.bind(B1), rec_rmov.opcodes(vec![0x89]));
+    e.enc_both(regmove.bind(I8), rec_rmov.opcodes(vec![0x89]));
+
+    e.enc_i32_i64(iadd_imm, rec_r_ib.opcodes(vec![0x83]).rrr(0));
+    e.enc_i32_i64(iadd_imm, rec_r_id.opcodes(vec![0x81]).rrr(0));
+
+    e.enc_i32_i64(band_imm, rec_r_ib.opcodes(vec![0x83]).rrr(4));
+    e.enc_i32_i64(band_imm, rec_r_id.opcodes(vec![0x81]).rrr(4));
+
+    e.enc_i32_i64(bor_imm, rec_r_ib.opcodes(vec![0x83]).rrr(1));
+    e.enc_i32_i64(bor_imm, rec_r_id.opcodes(vec![0x81]).rrr(1));
+
+    e.enc_i32_i64(bxor_imm, rec_r_ib.opcodes(vec![0x83]).rrr(6));
+    e.enc_i32_i64(bxor_imm, rec_r_id.opcodes(vec![0x81]).rrr(6));
+
+    // TODO: band_imm.i64 with an unsigned 32-bit immediate can be encoded as band_imm.i32. Can
+    // even use the single-byte immediate for 0xffff_ffXX masks.
+
+    // Immediate constants.
+    e.enc32(iconst.bind(I32), rec_pu_id.opcodes(vec![0xb8]));
+
+    e.enc64(iconst.bind(I32), rec_pu_id.rex().opcodes(vec![0xb8]));
+    e.enc64(iconst.bind(I32), rec_pu_id.opcodes(vec![0xb8]));
+
+    // The 32-bit immediate movl also zero-extends to 64 bits.
+    let f_unary_imm = formats.get(formats.by_name("UnaryImm"));
+    let is_unsigned_int32 = InstructionPredicate::new_is_unsigned_int(f_unary_imm, "imm", 32, 0);
+
+    e.enc64_func(
+        iconst.bind(I64),
+        rec_pu_id.opcodes(vec![0xb8]).rex(),
+        |encoding| encoding.inst_predicate(is_unsigned_int32.clone()),
+    );
+    e.enc64_func(
+        iconst.bind(I64),
+        rec_pu_id.opcodes(vec![0xb8]),
+        |encoding| encoding.inst_predicate(is_unsigned_int32),
+    );
+
+    // Sign-extended 32-bit immediate.
+    e.enc64(
+        iconst.bind(I64),
+        rec_u_id.rex().opcodes(vec![0xc7]).rrr(0).w(),
+    );
+
+    // Finally, the 0xb8 opcode takes an 8-byte immediate with a REX.W prefix.
+    e.enc64(iconst.bind(I64), rec_pu_iq.opcodes(vec![0xb8]).rex().w());
+
+    // Bool constants (uses MOV)
+    for &ty in &[B1, B8, B16, B32] {
+        e.enc_both(bconst.bind(ty), rec_pu_id_bool.opcodes(vec![0xb8]));
+    }
+    e.enc64(
+        bconst.bind(B64),
+        rec_pu_id_bool.opcodes(vec![0xb8]).rex().w(),
+    );
+
+    // Shifts and rotates.
+    // Note that the dynamic shift amount is only masked by 5 or 6 bits; the 8-bit
+    // and 16-bit shifts would need explicit masking.
+
+    for &(inst, rrr) in &[(rotl, 0), (rotr, 1), (ishl, 4), (ushr, 5), (sshr, 7)] {
+        // Cannot use enc_i32_i64 for this pattern because instructions require
+        // to bind any.
+        e.enc32(
+            inst.bind(I32).bind_any(),
+            rec_rc.opcodes(vec![0xd3]).rrr(rrr),
+        );
+        e.enc64(
+            inst.bind(I64).bind_any(),
+            rec_rc.opcodes(vec![0xd3]).rrr(rrr).rex().w(),
+        );
+        e.enc64(
+            inst.bind(I32).bind_any(),
+            rec_rc.opcodes(vec![0xd3]).rrr(rrr).rex(),
+        );
+        e.enc64(
+            inst.bind(I32).bind_any(),
+            rec_rc.opcodes(vec![0xd3]).rrr(rrr),
+        );
+    }
+
+    for &(inst, rrr) in &[
+        (rotl_imm, 0),
+        (rotr_imm, 1),
+        (ishl_imm, 4),
+        (ushr_imm, 5),
+        (sshr_imm, 7),
+    ] {
+        e.enc_i32_i64(inst, rec_r_ib.opcodes(vec![0xc1]).rrr(rrr));
+    }
+
+    // Population count.
+    e.enc32_isap(
+        popcnt.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xb8]),
+        use_popcnt,
+    );
+    e.enc64_isap(
+        popcnt.bind(I64),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xb8]).rex().w(),
+        use_popcnt,
+    );
+    e.enc64_isap(
+        popcnt.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xb8]).rex(),
+        use_popcnt,
+    );
+    e.enc64_isap(
+        popcnt.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xb8]),
+        use_popcnt,
+    );
+
+    // Count leading zero bits.
+    e.enc32_isap(
+        clz.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xbd]),
+        use_lzcnt,
+    );
+    e.enc64_isap(
+        clz.bind(I64),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xbd]).rex().w(),
+        use_lzcnt,
+    );
+    e.enc64_isap(
+        clz.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xbd]).rex(),
+        use_lzcnt,
+    );
+    e.enc64_isap(
+        clz.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xbd]),
+        use_lzcnt,
+    );
+
+    // Count trailing zero bits.
+    e.enc32_isap(
+        ctz.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xbc]),
+        use_bmi1,
+    );
+    e.enc64_isap(
+        ctz.bind(I64),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xbc]).rex().w(),
+        use_bmi1,
+    );
+    e.enc64_isap(
+        ctz.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xbc]).rex(),
+        use_bmi1,
+    );
+    e.enc64_isap(
+        ctz.bind(I32),
+        rec_urm.opcodes(vec![0xf3, 0x0f, 0xbc]),
+        use_bmi1,
+    );
+
+    // Loads and stores.
+    let f_load_complex = formats.get(formats.by_name("LoadComplex"));
+    let is_load_complex_length_two = InstructionPredicate::new_length_equals(f_load_complex, 2);
+
+    for recipe in &[rec_ldWithIndex, rec_ldWithIndexDisp8, rec_ldWithIndexDisp32] {
+        e.enc_i32_i64_instp(
+            load_complex,
+            recipe.opcodes(vec![0x8b]),
+            is_load_complex_length_two.clone(),
+        );
+        e.enc_x86_64_instp(
+            uload32_complex,
+            recipe.opcodes(vec![0x8b]),
+            is_load_complex_length_two.clone(),
+        );
+
+        e.enc64_instp(
+            sload32_complex,
+            recipe.opcodes(vec![0x63]).rex().w(),
+            is_load_complex_length_two.clone(),
+        );
+
+        e.enc_i32_i64_instp(
+            uload16_complex,
+            recipe.opcodes(vec![0x0f, 0xb7]),
+            is_load_complex_length_two.clone(),
+        );
+        e.enc_i32_i64_instp(
+            sload16_complex,
+            recipe.opcodes(vec![0x0f, 0xbf]),
+            is_load_complex_length_two.clone(),
+        );
+
+        e.enc_i32_i64_instp(
+            uload8_complex,
+            recipe.opcodes(vec![0x0f, 0xb6]),
+            is_load_complex_length_two.clone(),
+        );
+
+        e.enc_i32_i64_instp(
+            sload8_complex,
+            recipe.opcodes(vec![0x0f, 0xbe]),
+            is_load_complex_length_two.clone(),
+        );
+    }
+
+    let f_store_complex = formats.get(formats.by_name("StoreComplex"));
+    let is_store_complex_length_three = InstructionPredicate::new_length_equals(f_store_complex, 3);
+
+    for recipe in &[rec_stWithIndex, rec_stWithIndexDisp8, rec_stWithIndexDisp32] {
+        e.enc_i32_i64_instp(
+            store_complex,
+            recipe.opcodes(vec![0x89]),
+            is_store_complex_length_three.clone(),
+        );
+        e.enc_x86_64_instp(
+            istore32_complex,
+            recipe.opcodes(vec![0x89]),
+            is_store_complex_length_three.clone(),
+        );
+        e.enc_both_instp(
+            istore16_complex.bind(I32),
+            recipe.opcodes(vec![0x66, 0x89]),
+            is_store_complex_length_three.clone(),
+        );
+        e.enc_x86_64_instp(
+            istore16_complex.bind(I64),
+            recipe.opcodes(vec![0x66, 0x89]),
+            is_store_complex_length_three.clone(),
+        );
+    }
+
+    for recipe in &[
+        rec_stWithIndex_abcd,
+        rec_stWithIndexDisp8_abcd,
+        rec_stWithIndexDisp32_abcd,
+    ] {
+        e.enc_both_instp(
+            istore8_complex.bind(I32),
+            recipe.opcodes(vec![0x88]),
+            is_store_complex_length_three.clone(),
+        );
+        e.enc_x86_64_instp(
+            istore8_complex.bind(I64),
+            recipe.opcodes(vec![0x88]),
+            is_store_complex_length_three.clone(),
+        );
+    }
+
+    for recipe in &[rec_st, rec_stDisp8, rec_stDisp32] {
+        e.enc_i32_i64_ld_st(store, true, recipe.opcodes(vec![0x89]));
+        e.enc_x86_64(istore32.bind(I64).bind_any(), recipe.opcodes(vec![0x89]));
+        e.enc_i32_i64_ld_st(istore16, false, recipe.opcodes(vec![0x66, 0x89]));
+    }
+
+    // Byte stores are more complicated because the registers they can address
+    // depends of the presence of a REX prefix. The st*_abcd recipes fall back to
+    // the corresponding st* recipes when a REX prefix is applied.
+
+    for recipe in &[rec_st_abcd, rec_stDisp8_abcd, rec_stDisp32_abcd] {
+        e.enc_both(istore8.bind(I32).bind_any(), recipe.opcodes(vec![0x88]));
+        e.enc_x86_64(istore8.bind(I64).bind_any(), recipe.opcodes(vec![0x88]));
+    }
+
+    e.enc_i32_i64(spill, rec_spillSib32.opcodes(vec![0x89]));
+    e.enc_i32_i64(regspill, rec_regspill32.opcodes(vec![0x89]));
+
+    // Use a 32-bit write for spilling `b1`, `i8` and `i16` to avoid
+    // constraining the permitted registers.
+    // See MIN_SPILL_SLOT_SIZE which makes this safe.
+
+    e.enc_both(spill.bind(B1), rec_spillSib32.opcodes(vec![0x89]));
+    e.enc_both(regspill.bind(B1), rec_regspill32.opcodes(vec![0x89]));
+    for &ty in &[I8, I16] {
+        e.enc_both(spill.bind(ty), rec_spillSib32.opcodes(vec![0x89]));
+        e.enc_both(regspill.bind(ty), rec_regspill32.opcodes(vec![0x89]));
+    }
+
+    for recipe in &[rec_ld, rec_ldDisp8, rec_ldDisp32] {
+        e.enc_i32_i64_ld_st(load, true, recipe.opcodes(vec![0x8b]));
+        e.enc_x86_64(uload32.bind(I64), recipe.opcodes(vec![0x8b]));
+        e.enc64(sload32.bind(I64), recipe.opcodes(vec![0x63]).rex().w());
+        e.enc_i32_i64_ld_st(uload16, true, recipe.opcodes(vec![0x0f, 0xb7]));
+        e.enc_i32_i64_ld_st(sload16, true, recipe.opcodes(vec![0x0f, 0xbf]));
+        e.enc_i32_i64_ld_st(uload8, true, recipe.opcodes(vec![0x0f, 0xb6]));
+        e.enc_i32_i64_ld_st(sload8, true, recipe.opcodes(vec![0x0f, 0xbe]));
+    }
+
+    e.enc_i32_i64(fill, rec_fillSib32.opcodes(vec![0x8b]));
+    e.enc_i32_i64(regfill, rec_regfill32.opcodes(vec![0x8b]));
+
+    // Load 32 bits from `b1`, `i8` and `i16` spill slots. See `spill.b1` above.
+
+    e.enc_both(fill.bind(B1), rec_fillSib32.opcodes(vec![0x8b]));
+    e.enc_both(regfill.bind(B1), rec_regfill32.opcodes(vec![0x8b]));
+    for &ty in &[I8, I16] {
+        e.enc_both(fill.bind(ty), rec_fillSib32.opcodes(vec![0x8b]));
+        e.enc_both(regfill.bind(ty), rec_regfill32.opcodes(vec![0x8b]));
+    }
+
+    // Push and Pop.
+    e.enc32(x86_push.bind(I32), rec_pushq.opcodes(vec![0x50]));
+    e.enc_x86_64(x86_push.bind(I64), rec_pushq.opcodes(vec![0x50]));
+
+    e.enc32(x86_pop.bind(I32), rec_popq.opcodes(vec![0x58]));
+    e.enc_x86_64(x86_pop.bind(I64), rec_popq.opcodes(vec![0x58]));
+
+    // Copy Special
+    // For x86-64, only define REX forms for now, since we can't describe the
+    // special regunit immediate operands with the current constraint language.
+    e.enc64(copy_special, rec_copysp.opcodes(vec![0x89]).rex().w());
+    e.enc32(copy_special, rec_copysp.opcodes(vec![0x89]));
+
+    // Stack-slot-to-the-same-stack-slot copy, which is guaranteed to turn
+    // into a no-op.
+    // The same encoding is generated for both the 64- and 32-bit architectures.
+    for &ty in &[I64, I32, I16, I8] {
+        e.enc64_rec(copy_nop.bind(ty), rec_stacknull, 0);
+        e.enc32_rec(copy_nop.bind(ty), rec_stacknull, 0);
+    }
+    for &ty in &[F64, F32] {
+        e.enc64_rec(copy_nop.bind(ty), rec_stacknull, 0);
+        e.enc32_rec(copy_nop.bind(ty), rec_stacknull, 0);
+    }
+
+    // Adjust SP down by a dynamic value (or up, with a negative operand).
+    e.enc32(adjust_sp_down.bind(I32), rec_adjustsp.opcodes(vec![0x29]));
+    e.enc64(
+        adjust_sp_down.bind(I64),
+        rec_adjustsp.opcodes(vec![0x29]).rex().w(),
+    );
+
+    // Adjust SP up by an immediate (or down, with a negative immediate).
+    e.enc32(adjust_sp_up_imm, rec_adjustsp_ib.opcodes(vec![0x83]));
+    e.enc32(adjust_sp_up_imm, rec_adjustsp_id.opcodes(vec![0x81]));
+    e.enc64(
+        adjust_sp_up_imm,
+        rec_adjustsp_ib.opcodes(vec![0x83]).rex().w(),
+    );
+    e.enc64(
+        adjust_sp_up_imm,
+        rec_adjustsp_id.opcodes(vec![0x81]).rex().w(),
+    );
+
+    // Adjust SP down by an immediate (or up, with a negative immediate).
+    e.enc32(
+        adjust_sp_down_imm,
+        rec_adjustsp_ib.opcodes(vec![0x83]).rrr(5),
+    );
+    e.enc32(
+        adjust_sp_down_imm,
+        rec_adjustsp_id.opcodes(vec![0x81]).rrr(5),
+    );
+    e.enc64(
+        adjust_sp_down_imm,
+        rec_adjustsp_ib.opcodes(vec![0x83]).rrr(5).rex().w(),
+    );
+    e.enc64(
+        adjust_sp_down_imm,
+        rec_adjustsp_id.opcodes(vec![0x81]).rrr(5).rex().w(),
+    );
+
+    // Float loads and stores.
+    e.enc_both(
+        load.bind(F32).bind_any(),
+        rec_fld.opcodes(vec![0xf3, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        load.bind(F32).bind_any(),
+        rec_fldDisp8.opcodes(vec![0xf3, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        load.bind(F32).bind_any(),
+        rec_fldDisp32.opcodes(vec![0xf3, 0x0f, 0x10]),
+    );
+
+    e.enc_both(
+        load_complex.bind(F32),
+        rec_fldWithIndex.opcodes(vec![0xf3, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        load_complex.bind(F32),
+        rec_fldWithIndexDisp8.opcodes(vec![0xf3, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        load_complex.bind(F32),
+        rec_fldWithIndexDisp32.opcodes(vec![0xf3, 0x0f, 0x10]),
+    );
+
+    e.enc_both(
+        load.bind(F64).bind_any(),
+        rec_fld.opcodes(vec![0xf2, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        load.bind(F64).bind_any(),
+        rec_fldDisp8.opcodes(vec![0xf2, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        load.bind(F64).bind_any(),
+        rec_fldDisp32.opcodes(vec![0xf2, 0x0f, 0x10]),
+    );
+
+    e.enc_both(
+        load_complex.bind(F64),
+        rec_fldWithIndex.opcodes(vec![0xf2, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        load_complex.bind(F64),
+        rec_fldWithIndexDisp8.opcodes(vec![0xf2, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        load_complex.bind(F64),
+        rec_fldWithIndexDisp32.opcodes(vec![0xf2, 0x0f, 0x10]),
+    );
+
+    e.enc_both(
+        store.bind(F32).bind_any(),
+        rec_fst.opcodes(vec![0xf3, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        store.bind(F32).bind_any(),
+        rec_fstDisp8.opcodes(vec![0xf3, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        store.bind(F32).bind_any(),
+        rec_fstDisp32.opcodes(vec![0xf3, 0x0f, 0x11]),
+    );
+
+    e.enc_both(
+        store_complex.bind(F32),
+        rec_fstWithIndex.opcodes(vec![0xf3, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        store_complex.bind(F32),
+        rec_fstWithIndexDisp8.opcodes(vec![0xf3, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        store_complex.bind(F32),
+        rec_fstWithIndexDisp32.opcodes(vec![0xf3, 0x0f, 0x11]),
+    );
+
+    e.enc_both(
+        store.bind(F64).bind_any(),
+        rec_fst.opcodes(vec![0xf2, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        store.bind(F64).bind_any(),
+        rec_fstDisp8.opcodes(vec![0xf2, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        store.bind(F64).bind_any(),
+        rec_fstDisp32.opcodes(vec![0xf2, 0x0f, 0x11]),
+    );
+
+    e.enc_both(
+        store_complex.bind(F64),
+        rec_fstWithIndex.opcodes(vec![0xf2, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        store_complex.bind(F64),
+        rec_fstWithIndexDisp8.opcodes(vec![0xf2, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        store_complex.bind(F64),
+        rec_fstWithIndexDisp32.opcodes(vec![0xf2, 0x0f, 0x11]),
+    );
+
+    e.enc_both(
+        fill.bind(F32),
+        rec_ffillSib32.opcodes(vec![0xf3, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        regfill.bind(F32),
+        rec_fregfill32.opcodes(vec![0xf3, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        fill.bind(F64),
+        rec_ffillSib32.opcodes(vec![0xf2, 0x0f, 0x10]),
+    );
+    e.enc_both(
+        regfill.bind(F64),
+        rec_fregfill32.opcodes(vec![0xf2, 0x0f, 0x10]),
+    );
+
+    e.enc_both(
+        spill.bind(F32),
+        rec_fspillSib32.opcodes(vec![0xf3, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        regspill.bind(F32),
+        rec_fregspill32.opcodes(vec![0xf3, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        spill.bind(F64),
+        rec_fspillSib32.opcodes(vec![0xf2, 0x0f, 0x11]),
+    );
+    e.enc_both(
+        regspill.bind(F64),
+        rec_fregspill32.opcodes(vec![0xf2, 0x0f, 0x11]),
+    );
+
+    // Function addresses.
+
+    // Non-PIC, all-ones funcaddresses.
+    e.enc32_isap(
+        func_addr.bind(I32),
+        rec_fnaddr4.opcodes(vec![0xb8]),
+        not_all_ones_funcaddrs_and_not_is_pic,
+    );
+    e.enc64_isap(
+        func_addr.bind(I64),
+        rec_fnaddr8.opcodes(vec![0xb8]).rex().w(),
+        not_all_ones_funcaddrs_and_not_is_pic,
+    );
+
+    // Non-PIC, all-zeros funcaddresses.
+    e.enc32_isap(
+        func_addr.bind(I32),
+        rec_allones_fnaddr4.opcodes(vec![0xb8]),
+        all_ones_funcaddrs_and_not_is_pic,
+    );
+    e.enc64_isap(
+        func_addr.bind(I64),
+        rec_allones_fnaddr8.opcodes(vec![0xb8]).rex().w(),
+        all_ones_funcaddrs_and_not_is_pic,
+    );
+
+    // 64-bit, colocated, both PIC and non-PIC. Use the lea instruction's pc-relative field.
+    let f_func_addr = formats.get(formats.by_name("FuncAddr"));
+    let is_colocated_func = InstructionPredicate::new_is_colocated_func(f_func_addr, "func_ref");
+    e.enc64_instp(
+        func_addr.bind(I64),
+        rec_pcrel_fnaddr8.opcodes(vec![0x8d]).rex().w(),
+        is_colocated_func,
+    );
+
+    // 64-bit, non-colocated, PIC.
+    e.enc64_isap(
+        func_addr.bind(I64),
+        rec_got_fnaddr8.opcodes(vec![0x8b]).rex().w(),
+        is_pic,
+    );
+
+    // Global addresses.
+
+    // Non-PIC.
+    e.enc32_isap(
+        symbol_value.bind(I32),
+        rec_gvaddr4.opcodes(vec![0xb8]),
+        not_is_pic,
+    );
+    e.enc64_isap(
+        symbol_value.bind(I64),
+        rec_gvaddr8.opcodes(vec![0xb8]).rex().w(),
+        not_is_pic,
+    );
+
+    // PIC, colocated.
+    e.enc64_func(
+        symbol_value.bind(I64),
+        rec_pcrel_gvaddr8.opcodes(vec![0x8d]).rex().w(),
+        |encoding| {
+            encoding
+                .isa_predicate(is_pic)
+                .inst_predicate(InstructionPredicate::new_is_colocated_data(formats))
+        },
+    );
+
+    // PIC, non-colocated.
+    e.enc64_isap(
+        symbol_value.bind(I64),
+        rec_got_gvaddr8.opcodes(vec![0x8b]).rex().w(),
+        is_pic,
+    );
+
+    // Stack addresses.
+    //
+    // TODO: Add encoding rules for stack_load and stack_store, so that they
+    // don't get legalized to stack_addr + load/store.
+    e.enc32(stack_addr.bind(I32), rec_spaddr4_id.opcodes(vec![0x8d]));
+    e.enc64(
+        stack_addr.bind(I64),
+        rec_spaddr8_id.opcodes(vec![0x8d]).rex().w(),
+    );
+
+    // Call/return
+
+    // 32-bit, both PIC and non-PIC.
+    e.enc32(call, rec_call_id.opcodes(vec![0xe8]));
+
+    // 64-bit, colocated, both PIC and non-PIC. Use the call instruction's pc-relative field.
+    let f_call = formats.get(formats.by_name("Call"));
+    let is_colocated_func = InstructionPredicate::new_is_colocated_func(f_call, "func_ref");
+    e.enc64_instp(call, rec_call_id.opcodes(vec![0xe8]), is_colocated_func);
+
+    // 64-bit, non-colocated, PIC. There is no 64-bit non-colocated non-PIC version, since non-PIC
+    // is currently using the large model, which requires calls be lowered to
+    // func_addr+call_indirect.
+    e.enc64_isap(call, rec_call_plt_id.opcodes(vec![0xe8]), is_pic);
+
+    e.enc32(
+        call_indirect.bind(I32),
+        rec_call_r.opcodes(vec![0xff]).rrr(2),
+    );
+    e.enc64(
+        call_indirect.bind(I64),
+        rec_call_r.opcodes(vec![0xff]).rrr(2).rex(),
+    );
+    e.enc64(
+        call_indirect.bind(I64),
+        rec_call_r.opcodes(vec![0xff]).rrr(2),
+    );
+
+    e.enc32(return_, rec_ret.opcodes(vec![0xc3]));
+    e.enc64(return_, rec_ret.opcodes(vec![0xc3]));
+
+    // Branches.
+    e.enc32(jump, rec_jmpb.opcodes(vec![0xeb]));
+    e.enc64(jump, rec_jmpb.opcodes(vec![0xeb]));
+    e.enc32(jump, rec_jmpd.opcodes(vec![0xe9]));
+    e.enc64(jump, rec_jmpd.opcodes(vec![0xe9]));
+
+    e.enc_both(brif, rec_brib.opcodes(vec![0x70]));
+    e.enc_both(brif, rec_brid.opcodes(vec![0x0f, 0x80]));
+
+    // Not all float condition codes are legal, see `supported_floatccs`.
+    e.enc_both(brff, rec_brfb.opcodes(vec![0x70]));
+    e.enc_both(brff, rec_brfd.opcodes(vec![0x0f, 0x80]));
+
+    // Note that the tjccd opcode will be prefixed with 0x0f.
+    e.enc_i32_i64(brz, rec_tjccb.opcodes(vec![0x74]));
+    e.enc_i32_i64(brz, rec_tjccd.opcodes(vec![0x84]));
+    e.enc_i32_i64(brnz, rec_tjccb.opcodes(vec![0x75]));
+    e.enc_i32_i64(brnz, rec_tjccd.opcodes(vec![0x85]));
+
+    // Branch on a b1 value in a register only looks at the low 8 bits. See also
+    // bint encodings below.
+    //
+    // Start with the worst-case encoding for X86_32 only. The register allocator
+    // can't handle a branch with an ABCD-constrained operand.
+    e.enc32(brz.bind(B1), rec_t8jccd_long.opcodes(vec![0x84]));
+    e.enc32(brnz.bind(B1), rec_t8jccd_long.opcodes(vec![0x85]));
+
+    e.enc_both(brz.bind(B1), rec_t8jccb_abcd.opcodes(vec![0x74]));
+    e.enc_both(brz.bind(B1), rec_t8jccd_abcd.opcodes(vec![0x84]));
+    e.enc_both(brnz.bind(B1), rec_t8jccb_abcd.opcodes(vec![0x75]));
+    e.enc_both(brnz.bind(B1), rec_t8jccd_abcd.opcodes(vec![0x85]));
+
+    // Jump tables.
+    e.enc64(
+        jump_table_entry.bind(I64),
+        rec_jt_entry.opcodes(vec![0x63]).rex().w(),
+    );
+    e.enc32(jump_table_entry.bind(I32), rec_jt_entry.opcodes(vec![0x8b]));
+
+    e.enc64(
+        jump_table_base.bind(I64),
+        rec_jt_base.opcodes(vec![0x8d]).rex().w(),
+    );
+    e.enc32(jump_table_base.bind(I32), rec_jt_base.opcodes(vec![0x8d]));
+
+    e.enc_x86_64(
+        indirect_jump_table_br.bind(I64),
+        rec_indirect_jmp.opcodes(vec![0xff]).rrr(4),
+    );
+    e.enc32(
+        indirect_jump_table_br.bind(I32),
+        rec_indirect_jmp.opcodes(vec![0xff]).rrr(4),
+    );
+
+    // Trap as ud2
+    e.enc32(trap, rec_trap.opcodes(vec![0x0f, 0x0b]));
+    e.enc64(trap, rec_trap.opcodes(vec![0x0f, 0x0b]));
+
+    // Debug trap as int3
+    e.enc32_rec(debugtrap, rec_debugtrap, 0);
+    e.enc64_rec(debugtrap, rec_debugtrap, 0);
+
+    e.enc32_rec(trapif, rec_trapif, 0);
+    e.enc64_rec(trapif, rec_trapif, 0);
+    e.enc32_rec(trapff, rec_trapff, 0);
+    e.enc64_rec(trapff, rec_trapff, 0);
+
+    // Comparisons
+    e.enc_i32_i64(icmp, rec_icscc.opcodes(vec![0x39]));
+    e.enc_i32_i64(icmp_imm, rec_icscc_ib.opcodes(vec![0x83]).rrr(7));
+    e.enc_i32_i64(icmp_imm, rec_icscc_id.opcodes(vec![0x81]).rrr(7));
+    e.enc_i32_i64(ifcmp, rec_rcmp.opcodes(vec![0x39]));
+    e.enc_i32_i64(ifcmp_imm, rec_rcmp_ib.opcodes(vec![0x83]).rrr(7));
+    e.enc_i32_i64(ifcmp_imm, rec_rcmp_id.opcodes(vec![0x81]).rrr(7));
+    // TODO: We could special-case ifcmp_imm(x, 0) to TEST(x, x).
+
+    e.enc32(ifcmp_sp.bind(I32), rec_rcmp_sp.opcodes(vec![0x39]));
+    e.enc64(
+        ifcmp_sp.bind(I64),
+        rec_rcmp_sp.opcodes(vec![0x39]).rex().w(),
+    );
+
+    // Convert flags to bool.
+    // This encodes `b1` as an 8-bit low register with the value 0 or 1.
+    e.enc_both(trueif, rec_seti_abcd.opcodes(vec![0x0f, 0x90]));
+    e.enc_both(trueff, rec_setf_abcd.opcodes(vec![0x0f, 0x90]));
+
+    // Conditional move (a.k.a integer select).
+    e.enc_i32_i64(selectif, rec_cmov.opcodes(vec![0x0f, 0x40]));
+
+    // Bit scan forwards and reverse
+    e.enc_i32_i64(x86_bsf, rec_bsf_and_bsr.opcodes(vec![0x0f, 0xbc]));
+    e.enc_i32_i64(x86_bsr, rec_bsf_and_bsr.opcodes(vec![0x0f, 0xbd]));
+
+    // Convert bool to int.
+    //
+    // This assumes that b1 is represented as an 8-bit low register with the value 0
+    // or 1.
+    //
+    // Encode movzbq as movzbl, because it's equivalent and shorter.
+    e.enc32(
+        bint.bind(I32).bind(B1),
+        rec_urm_noflags_abcd.opcodes(vec![0x0f, 0xb6]),
+    );
+
+    e.enc64(
+        bint.bind(I64).bind(B1),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb6]).rex(),
+    );
+    e.enc64(
+        bint.bind(I64).bind(B1),
+        rec_urm_noflags_abcd.opcodes(vec![0x0f, 0xb6]),
+    );
+    e.enc64(
+        bint.bind(I32).bind(B1),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb6]).rex(),
+    );
+    e.enc64(
+        bint.bind(I32).bind(B1),
+        rec_urm_noflags_abcd.opcodes(vec![0x0f, 0xb6]),
+    );
+
+    // Numerical conversions.
+
+    // Reducing an integer is a no-op.
+    e.enc32_rec(ireduce.bind(I8).bind(I16), rec_null, 0);
+    e.enc32_rec(ireduce.bind(I8).bind(I32), rec_null, 0);
+    e.enc32_rec(ireduce.bind(I16).bind(I32), rec_null, 0);
+
+    e.enc64_rec(ireduce.bind(I8).bind(I16), rec_null, 0);
+    e.enc64_rec(ireduce.bind(I8).bind(I32), rec_null, 0);
+    e.enc64_rec(ireduce.bind(I16).bind(I32), rec_null, 0);
+    e.enc64_rec(ireduce.bind(I8).bind(I64), rec_null, 0);
+    e.enc64_rec(ireduce.bind(I16).bind(I64), rec_null, 0);
+    e.enc64_rec(ireduce.bind(I32).bind(I64), rec_null, 0);
+
+    // TODO: Add encodings for cbw, cwde, cdqe, which are sign-extending
+    // instructions for %al/%ax/%eax to %ax/%eax/%rax.
+
+    // movsbl
+    e.enc32(
+        sextend.bind(I32).bind(I8),
+        rec_urm_noflags_abcd.opcodes(vec![0x0f, 0xbe]),
+    );
+    e.enc64(
+        sextend.bind(I32).bind(I8),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xbe]).rex(),
+    );
+    e.enc64(
+        sextend.bind(I32).bind(I8),
+        rec_urm_noflags_abcd.opcodes(vec![0x0f, 0xbe]),
+    );
+
+    // movswl
+    e.enc32(
+        sextend.bind(I32).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xbf]),
+    );
+    e.enc64(
+        sextend.bind(I32).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xbf]).rex(),
+    );
+    e.enc64(
+        sextend.bind(I32).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xbf]),
+    );
+
+    // movsbq
+    e.enc64(
+        sextend.bind(I64).bind(I8),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xbe]).rex().w(),
+    );
+
+    // movswq
+    e.enc64(
+        sextend.bind(I64).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xbf]).rex().w(),
+    );
+
+    // movslq
+    e.enc64(
+        sextend.bind(I64).bind(I32),
+        rec_urm_noflags.opcodes(vec![0x63]).rex().w(),
+    );
+
+    // movzbl
+    e.enc32(
+        uextend.bind(I32).bind(I8),
+        rec_urm_noflags_abcd.opcodes(vec![0x0f, 0xb6]),
+    );
+    e.enc64(
+        uextend.bind(I32).bind(I8),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb6]).rex(),
+    );
+    e.enc64(
+        uextend.bind(I32).bind(I8),
+        rec_urm_noflags_abcd.opcodes(vec![0x0f, 0xb6]),
+    );
+
+    // movzwl
+    e.enc32(
+        uextend.bind(I32).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb7]),
+    );
+    e.enc64(
+        uextend.bind(I32).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb7]).rex(),
+    );
+    e.enc64(
+        uextend.bind(I32).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb7]),
+    );
+
+    // movzbq, encoded as movzbl because it's equivalent and shorter.
+    e.enc64(
+        uextend.bind(I64).bind(I8),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb6]).rex(),
+    );
+    e.enc64(
+        uextend.bind(I64).bind(I8),
+        rec_urm_noflags_abcd.opcodes(vec![0x0f, 0xb6]),
+    );
+
+    // movzwq, encoded as movzwl because it's equivalent and shorter
+    e.enc64(
+        uextend.bind(I64).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb7]).rex(),
+    );
+    e.enc64(
+        uextend.bind(I64).bind(I16),
+        rec_urm_noflags.opcodes(vec![0x0f, 0xb7]),
+    );
+
+    // A 32-bit register copy clears the high 32 bits.
+    e.enc64(
+        uextend.bind(I64).bind(I32),
+        rec_umr.opcodes(vec![0x89]).rex(),
+    );
+    e.enc64(uextend.bind(I64).bind(I32), rec_umr.opcodes(vec![0x89]));
+
+    // Floating point
+
+    // Floating-point constants equal to 0.0 can be encoded using either `xorps` or `xorpd`, for
+    // 32-bit and 64-bit floats respectively.
+    let f_unary_ieee32 = formats.get(formats.by_name("UnaryIeee32"));
+    let is_zero_32_bit_float = InstructionPredicate::new_is_zero_32bit_float(f_unary_ieee32, "imm");
+    e.enc32_instp(
+        f32const,
+        rec_f32imm_z.opcodes(vec![0x0f, 0x57]),
+        is_zero_32_bit_float.clone(),
+    );
+
+    let f_unary_ieee64 = formats.get(formats.by_name("UnaryIeee64"));
+    let is_zero_64_bit_float = InstructionPredicate::new_is_zero_64bit_float(f_unary_ieee64, "imm");
+    e.enc32_instp(
+        f64const,
+        rec_f64imm_z.opcodes(vec![0x66, 0x0f, 0x57]),
+        is_zero_64_bit_float.clone(),
+    );
+
+    e.enc_x86_64_instp(
+        f32const,
+        rec_f32imm_z.opcodes(vec![0x0f, 0x57]),
+        is_zero_32_bit_float,
+    );
+    e.enc_x86_64_instp(
+        f64const,
+        rec_f64imm_z.opcodes(vec![0x66, 0x0f, 0x57]),
+        is_zero_64_bit_float,
+    );
+
+    // movd
+    e.enc_both(
+        bitcast.bind(F32).bind(I32),
+        rec_frurm.opcodes(vec![0x66, 0x0f, 0x6e]),
+    );
+    e.enc_both(
+        bitcast.bind(I32).bind(F32),
+        rec_rfumr.opcodes(vec![0x66, 0x0f, 0x7e]),
+    );
+
+    // movq
+    e.enc64(
+        bitcast.bind(F64).bind(I64),
+        rec_frurm.opcodes(vec![0x66, 0x0f, 0x6e]).rex().w(),
+    );
+    e.enc64(
+        bitcast.bind(I64).bind(F64),
+        rec_rfumr.opcodes(vec![0x66, 0x0f, 0x7e]).rex().w(),
+    );
+
+    // movaps
+    e.enc_both(copy.bind(F32), rec_furm.opcodes(vec![0x0f, 0x28]));
+    e.enc_both(copy.bind(F64), rec_furm.opcodes(vec![0x0f, 0x28]));
+
+    // TODO For x86-64, only define REX forms for now, since we can't describe the special regunit
+    // immediate operands with the current constraint language.
+    e.enc32(regmove.bind(F32), rec_frmov.opcodes(vec![0x0f, 0x28]));
+    e.enc64(regmove.bind(F32), rec_frmov.opcodes(vec![0x0f, 0x28]).rex());
+
+    // TODO For x86-64, only define REX forms for now, since we can't describe the special regunit
+    // immediate operands with the current constraint language.
+    e.enc32(regmove.bind(F64), rec_frmov.opcodes(vec![0x0f, 0x28]));
+    e.enc64(regmove.bind(F64), rec_frmov.opcodes(vec![0x0f, 0x28]).rex());
+
+    // cvtsi2ss
+    e.enc_i32_i64(
+        fcvt_from_sint.bind(F32),
+        rec_frurm.opcodes(vec![0xf3, 0x0f, 0x2a]),
+    );
+
+    // cvtsi2sd
+    e.enc_i32_i64(
+        fcvt_from_sint.bind(F64),
+        rec_frurm.opcodes(vec![0xf2, 0x0f, 0x2a]),
+    );
+
+    // cvtss2sd
+    e.enc_both(
+        fpromote.bind(F64).bind(F32),
+        rec_furm.opcodes(vec![0xf3, 0x0f, 0x5a]),
+    );
+
+    // cvtsd2ss
+    e.enc_both(
+        fdemote.bind(F32).bind(F64),
+        rec_furm.opcodes(vec![0xf2, 0x0f, 0x5a]),
+    );
+
+    // cvttss2si
+    e.enc_both(
+        x86_cvtt2si.bind(I32).bind(F32),
+        rec_rfurm.opcodes(vec![0xf3, 0x0f, 0x2c]),
+    );
+    e.enc64(
+        x86_cvtt2si.bind(I64).bind(F32),
+        rec_rfurm.opcodes(vec![0xf3, 0x0f, 0x2c]).rex().w(),
+    );
+
+    // cvttsd2si
+    e.enc_both(
+        x86_cvtt2si.bind(I32).bind(F64),
+        rec_rfurm.opcodes(vec![0xf2, 0x0f, 0x2c]),
+    );
+    e.enc64(
+        x86_cvtt2si.bind(I64).bind(F64),
+        rec_rfurm.opcodes(vec![0xf2, 0x0f, 0x2c]).rex().w(),
+    );
+
+    // Exact square roots.
+    e.enc_both(sqrt.bind(F32), rec_furm.opcodes(vec![0xf3, 0x0f, 0x51]));
+    e.enc_both(sqrt.bind(F64), rec_furm.opcodes(vec![0xf2, 0x0f, 0x51]));
+
+    // Rounding. The recipe looks at the opcode to pick an immediate.
+    for inst in &[nearest, floor, ceil, trunc] {
+        e.enc_both_isap(
+            inst.bind(F32),
+            rec_furmi_rnd.opcodes(vec![0x66, 0x0f, 0x3a, 0x0a]),
+            use_sse41,
+        );
+        e.enc_both_isap(
+            inst.bind(F64),
+            rec_furmi_rnd.opcodes(vec![0x66, 0x0f, 0x3a, 0x0b]),
+            use_sse41,
+        );
+    }
+
+    // Binary arithmetic ops.
+    for &(inst, opc) in &[
+        (fadd, 0x58),
+        (fsub, 0x5c),
+        (fmul, 0x59),
+        (fdiv, 0x5e),
+        (x86_fmin, 0x5d),
+        (x86_fmax, 0x5f),
+    ] {
+        e.enc_both(inst.bind(F32), rec_fa.opcodes(vec![0xf3, 0x0f, opc]));
+        e.enc_both(inst.bind(F64), rec_fa.opcodes(vec![0xf2, 0x0f, opc]));
+    }
+
+    // Binary bitwise ops.
+    for &(inst, opc) in &[(band, 0x54), (bor, 0x56), (bxor, 0x57)] {
+        e.enc_both(inst.bind(F32), rec_fa.opcodes(vec![0x0f, opc]));
+        e.enc_both(inst.bind(F64), rec_fa.opcodes(vec![0x0f, opc]));
+    }
+
+    // The `andnps(x,y)` instruction computes `~x&y`, while band_not(x,y)` is `x&~y.
+    e.enc_both(band_not.bind(F32), rec_fax.opcodes(vec![0x0f, 0x55]));
+    e.enc_both(band_not.bind(F64), rec_fax.opcodes(vec![0x0f, 0x55]));
+
+    // Comparisons.
+    //
+    // This only covers the condition codes in `supported_floatccs`, the rest are
+    // handled by legalization patterns.
+    e.enc_both(fcmp.bind(F32), rec_fcscc.opcodes(vec![0x0f, 0x2e]));
+    e.enc_both(fcmp.bind(F64), rec_fcscc.opcodes(vec![0x66, 0x0f, 0x2e]));
+    e.enc_both(ffcmp.bind(F32), rec_fcmp.opcodes(vec![0x0f, 0x2e]));
+    e.enc_both(ffcmp.bind(F64), rec_fcmp.opcodes(vec![0x66, 0x0f, 0x2e]));
+
+    // SIMD splat: before x86 can use vector data, it must be moved to XMM registers; see
+    // legalize.rs for how this is done; once there, x86_pshuf* (below) is used for broadcasting the
+    // value across the register
+
+    // PSHUFB, 8-bit shuffle using two XMM registers
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 8) {
+        let number_of_lanes = 128 / ty.lane_bits();
+        let instruction = x86_pshufb.bind_vector(ty, number_of_lanes);
+        let template = rec_fa.nonrex().opcodes(vec![0x66, 0x0f, 0x38, 0x00]);
+        e.enc32_isap(instruction.clone(), template.clone(), use_ssse3);
+        e.enc64_isap(instruction, template, use_ssse3);
+    }
+
+    // PSHUFD, 32-bit shuffle using one XMM register and a u8 immediate
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 32) {
+        let number_of_lanes = 128 / ty.lane_bits();
+        let instruction = x86_pshufd.bind_vector(ty, number_of_lanes);
+        let template = rec_r_ib_unsigned.nonrex().opcodes(vec![0x66, 0x0f, 0x70]);
+        e.enc32_isap(instruction.clone(), template.clone(), use_sse2);
+        e.enc64_isap(instruction, template, use_sse2);
+    }
+
+    // SIMD scalar_to_vector; this uses MOV to copy the scalar value to an XMM register; according
+    // to the Intel manual: "When the destination operand is an XMM register, the source operand is
+    // written to the low doubleword of the register and the regiser is zero-extended to 128 bits."
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() >= 8) {
+        let number_of_lanes = 128 / ty.lane_bits();
+        let instruction = scalar_to_vector.bind_vector(ty, number_of_lanes).bind(ty);
+        let template = rec_frurm.opcodes(vec![0x66, 0x0f, 0x6e]); // MOVD/MOVQ
+        if ty.lane_bits() < 64 {
+            // no 32-bit encodings for 64-bit widths
+            e.enc32_isap(instruction.clone(), template.clone(), use_sse2);
+        }
+        e.enc_x86_64_isap(instruction, template, use_sse2);
+    }
+
+    // SIMD insertlane
+    let mut insertlane_mapping: HashMap<u64, (Vec<u8>, SettingPredicateNumber)> = HashMap::new();
+    insertlane_mapping.insert(8, (vec![0x66, 0x0f, 0x3a, 0x20], use_sse41)); // PINSRB
+    insertlane_mapping.insert(16, (vec![0x66, 0x0f, 0xc4], use_sse2)); // PINSRW
+    insertlane_mapping.insert(32, (vec![0x66, 0x0f, 0x3a, 0x22], use_sse41)); // PINSRD
+    insertlane_mapping.insert(64, (vec![0x66, 0x0f, 0x3a, 0x22], use_sse41)); // PINSRQ, only x86_64
+
+    for ty in ValueType::all_lane_types() {
+        if let Some((opcode, isap)) = insertlane_mapping.get(&ty.lane_bits()) {
+            let number_of_lanes = 128 / ty.lane_bits();
+            let instruction = insertlane.bind_vector(ty, number_of_lanes);
+            let template = rec_r_ib_unsigned_r.opcodes(opcode.clone());
+            if ty.lane_bits() < 64 {
+                e.enc_32_64_isap(instruction, template.nonrex(), isap.clone());
+            } else {
+                // turns out the 64-bit widths have REX/W encodings and only are available on x86_64
+                e.enc64_isap(instruction, template.rex().w(), isap.clone());
+            }
+        }
+    }
+
+    // SIMD bitcast f64 to all 8-bit-lane vectors (for legalizing splat.x8x16); assumes that f64 is stored in an XMM register
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 8) {
+        let instruction = bitcast.bind_vector(ty, 16).bind(F64);
+        e.enc32_rec(instruction.clone(), rec_null_fpr, 0);
+        e.enc64_rec(instruction, rec_null_fpr, 0);
+    }
+
+    // SIMD bitcast all 128-bit vectors to each other (for legalizing splat.x16x8)
+    for from_type in ValueType::all_lane_types().filter(|t| t.lane_bits() >= 8) {
+        for to_type in ValueType::all_lane_types().filter(|t| t.lane_bits() >= 8 && *t != from_type)
+        {
+            let instruction = raw_bitcast
+                .bind_vector(to_type, 128 / to_type.lane_bits())
+                .bind_vector(from_type, 128 / from_type.lane_bits());
+            e.enc32_rec(instruction.clone(), rec_null_fpr, 0);
+            e.enc64_rec(instruction, rec_null_fpr, 0);
+        }
+    }
+
+    e
+}
--- a/third_party/rust/cranelift-codegen-meta/src/isa/x86/instructions.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/instructions.rs
@@ -1,22 +1,29 @@
 #![allow(non_snake_case)]
 
 use crate::cdsl::formats::FormatRegistry;
 use crate::cdsl::instructions::{
-    InstructionBuilder as Inst, InstructionGroup, InstructionGroupBuilder,
+    AllInstructions, InstructionBuilder as Inst, InstructionGroup, InstructionGroupBuilder,
 };
 use crate::cdsl::operands::{create_operand as operand, create_operand_doc as operand_doc};
 use crate::cdsl::types::ValueType;
 use crate::cdsl::typevar::{Interval, TypeSetBuilder, TypeVar};
-use crate::shared::types;
+use crate::shared::{immediates, types, OperandKinds};
 
-pub fn define(format_registry: &FormatRegistry) -> InstructionGroup {
-    let mut ig =
-        InstructionGroupBuilder::new("x86", "x86 specific instruction set", format_registry);
+pub fn define(
+    mut all_instructions: &mut AllInstructions,
+    format_registry: &FormatRegistry,
+) -> InstructionGroup {
+    let mut ig = InstructionGroupBuilder::new(
+        "x86",
+        "x86 specific instruction set",
+        &mut all_instructions,
+        format_registry,
+    );
 
     let iflags: &TypeVar = &ValueType::Special(types::Flag::IFlags.into()).into();
 
     let iWord = &TypeVar::new(
         "iWord",
         "A scalar integer machine word",
         TypeSetBuilder::new().ints(32..64).build(),
     );
@@ -139,17 +146,17 @@ pub fn define(format_registry: &FormatRe
 
     ig.push(
         Inst::new(
             "x86_fmin",
             r#"
         Floating point minimum with x86 semantics.
 
         This is equivalent to the C ternary operator `x < y ? x : y` which
-        differs from :inst:`fmin` when either operand is NaN or when comparing
+        differs from `fmin` when either operand is NaN or when comparing
         +0.0 to -0.0.
 
         When the two operands don't compare as LT, `y` is returned unchanged,
         even if it is a signalling NaN.
         "#,
         )
         .operands_in(vec![x, y])
         .operands_out(vec![a]),
@@ -157,17 +164,17 @@ pub fn define(format_registry: &FormatRe
 
     ig.push(
         Inst::new(
             "x86_fmax",
             r#"
         Floating point maximum with x86 semantics.
 
         This is equivalent to the C ternary operator `x > y ? x : y` which
-        differs from :inst:`fmax` when either operand is NaN or when comparing
+        differs from `fmax` when either operand is NaN or when comparing
         +0.0 to -0.0.
 
         When the two operands don't compare as GT, `y` is returned unchanged,
         even if it is a signalling NaN.
         "#,
         )
         .operands_in(vec![x, y])
         .operands_out(vec![a]),
@@ -237,10 +244,51 @@ pub fn define(format_registry: &FormatRe
     Bit Scan Forwards -- returns the bit-index of the least significant 1
     in the word. Is otherwise identical to 'bsr', just above.
     "#,
         )
         .operands_in(vec![x])
         .operands_out(vec![y, rflags]),
     );
 
+    let immediates = OperandKinds::from(immediates::define());
+    let uimm8 = immediates.by_name("uimm8");
+    let TxN = &TypeVar::new(
+        "TxN",
+        "A SIMD vector type",
+        TypeSetBuilder::new()
+            .ints(Interval::All)
+            .floats(Interval::All)
+            .bools(Interval::All)
+            .simd_lanes(Interval::All)
+            .includes_scalars(false)
+            .build(),
+    );
+    let a = &operand_doc("a", TxN, "A vector value (i.e. held in an XMM register)");
+    let b = &operand_doc("b", TxN, "A vector value (i.e. held in an XMM register)");
+    let i = &operand_doc("i", uimm8, "An ordering operand controlling the copying of data from the source to the destination; see PSHUFD in Intel manual for details");
+
+    ig.push(
+        Inst::new(
+            "x86_pshufd",
+            r#"
+    Packed Shuffle Doublewords -- copies data from either memory or lanes in an extended
+    register and re-orders the data according to the passed immediate byte.
+    "#,
+        )
+        .operands_in(vec![a, i]) // TODO allow copying from memory here (need more permissive type than TxN)
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_pshufb",
+            r#"
+    Packed Shuffle Bytes -- re-orders data in an extended register using a shuffle
+    mask from either memory or another extended register
+    "#,
+        )
+        .operands_in(vec![a, b]) // TODO allow re-ordering from memory here (need more permissive type than TxN)
+        .operands_out(vec![a]),
+    );
+
     ig.build()
 }
--- a/third_party/rust/cranelift-codegen-meta/src/isa/x86/legalize.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/legalize.rs
@@ -1,12 +1,13 @@
 use crate::cdsl::ast::{var, ExprBuilder, Literal};
 use crate::cdsl::instructions::InstructionGroup;
+use crate::cdsl::types::ValueType;
 use crate::cdsl::xform::TransformGroupBuilder;
-
+use crate::shared::types::Float::F64;
 use crate::shared::types::Int::{I32, I64};
 use crate::shared::Definitions as SharedDefinitions;
 
 pub fn define(shared: &mut SharedDefinitions, x86_instructions: &InstructionGroup) {
     let mut group = TransformGroupBuilder::new(
         "x86_expand",
         r#"
     Legalize instructions by expansion.
@@ -14,50 +15,60 @@ pub fn define(shared: &mut SharedDefinit
     Use x86-specific instructions if needed."#,
     )
     .isa("x86")
     .chain_with(shared.transform_groups.by_name("expand_flags").id);
 
     // List of instructions.
     let insts = &shared.instructions;
     let band = insts.by_name("band");
+    let bitcast = insts.by_name("bitcast");
     let bor = insts.by_name("bor");
     let clz = insts.by_name("clz");
     let ctz = insts.by_name("ctz");
+    let f64const = insts.by_name("f64const");
     let fcmp = insts.by_name("fcmp");
     let fcvt_from_uint = insts.by_name("fcvt_from_uint");
     let fcvt_to_sint = insts.by_name("fcvt_to_sint");
     let fcvt_to_uint = insts.by_name("fcvt_to_uint");
     let fcvt_to_sint_sat = insts.by_name("fcvt_to_sint_sat");
     let fcvt_to_uint_sat = insts.by_name("fcvt_to_uint_sat");
     let fmax = insts.by_name("fmax");
     let fmin = insts.by_name("fmin");
     let iadd = insts.by_name("iadd");
     let iconst = insts.by_name("iconst");
     let imul = insts.by_name("imul");
+    let insertlane = insts.by_name("insertlane");
     let isub = insts.by_name("isub");
     let popcnt = insts.by_name("popcnt");
+    let raw_bitcast = insts.by_name("raw_bitcast");
+    let scalar_to_vector = insts.by_name("scalar_to_vector");
     let sdiv = insts.by_name("sdiv");
     let selectif = insts.by_name("selectif");
     let smulhi = insts.by_name("smulhi");
+    let splat = insts.by_name("splat");
     let srem = insts.by_name("srem");
     let udiv = insts.by_name("udiv");
     let umulhi = insts.by_name("umulhi");
     let ushr_imm = insts.by_name("ushr_imm");
     let urem = insts.by_name("urem");
 
     let x86_bsf = x86_instructions.by_name("x86_bsf");
     let x86_bsr = x86_instructions.by_name("x86_bsr");
+    let x86_pshufb = x86_instructions.by_name("x86_pshufb");
+    let x86_pshufd = x86_instructions.by_name("x86_pshufd");
     let x86_umulx = x86_instructions.by_name("x86_umulx");
     let x86_smulx = x86_instructions.by_name("x86_smulx");
 
     // List of immediates.
     let floatcc = shared.operand_kinds.by_name("floatcc");
     let imm64 = shared.operand_kinds.by_name("imm64");
     let intcc = shared.operand_kinds.by_name("intcc");
+    let uimm8 = shared.operand_kinds.by_name("uimm8");
+    let ieee64 = shared.operand_kinds.by_name("ieee64");
 
     // Division and remainder.
     //
     // The srem expansion requires custom code because srem INT_MIN, -1 is not
     // allowed to trap. The other ops need to check avoid_div_traps.
     group.custom_legalize(sdiv, "expand_sdivrem");
     group.custom_legalize(srem, "expand_sdivrem");
     group.custom_legalize(udiv, "expand_udivrem");
@@ -285,9 +296,89 @@ pub fn define(shared: &mut SharedDefinit
             def!(lv14 = band(lv13, lc0F)),
             def!(lc01 = iconst(Literal::constant(imm64, 0x01010101))),
             def!(lv15 = imul(lv14, lc01)),
             def!(lv16 = ushr_imm(lv15, Literal::constant(imm64, 24))),
         ],
     );
 
     group.build_and_add_to(&mut shared.transform_groups);
+
+    let mut narrow = TransformGroupBuilder::new(
+        "x86_narrow",
+        r#"
+    Legalize instructions by narrowing.
+
+    Use x86-specific instructions if needed."#,
+    )
+    .isa("x86")
+    .chain_with(shared.transform_groups.by_name("narrow").id);
+
+    // SIMD
+    let uimm8_zero = Literal::constant(uimm8, 0x00);
+    let uimm8_one = Literal::constant(uimm8, 0x01);
+    let ieee64_zero = Literal::constant(ieee64, 0x00);
+    let b = var("b");
+    let c = var("c");
+    let d = var("d");
+
+    // SIMD splat: 8-bits
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 8) {
+        let splat_x8x16 = splat.bind_vector(ty, 128 / ty.lane_bits());
+        let bitcast_f64_to_any8x16 = bitcast.bind_vector(ty, 128 / ty.lane_bits()).bind(F64);
+        narrow.legalize(
+            def!(y = splat_x8x16(x)),
+            vec![
+                def!(a = scalar_to_vector(x)), // move into the lowest 8 bits of an XMM register
+                def!(b = f64const(ieee64_zero)), // zero out a different XMM register; the shuffle mask for moving the lowest byte to all other byte lanes is 0x0
+                def!(c = bitcast_f64_to_any8x16(b)), // no instruction emitted; informs the SSA that the 0 in b can be used as a vector of this type
+                def!(y = x86_pshufb(a, c)), // PSHUFB takes two XMM operands, one of which is a shuffle mask (i.e. b)
+            ],
+        );
+    }
+
+    // SIMD splat: 16-bits
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 16) {
+        let splat_x16x8 = splat.bind_vector(ty, 128 / ty.lane_bits());
+        let raw_bitcast_any16x8_to_i32x4 = raw_bitcast
+            .bind_vector(I32, 4)
+            .bind_vector(ty, 128 / ty.lane_bits());
+        let raw_bitcast_i32x4_to_any16x8 = raw_bitcast
+            .bind_vector(ty, 128 / ty.lane_bits())
+            .bind_vector(I32, 4);
+        narrow.legalize(
+            def!(y = splat_x16x8(x)),
+            vec![
+                def!(a = scalar_to_vector(x)), // move into the lowest 16 bits of an XMM register
+                def!(b = insertlane(a, uimm8_one, x)), // insert the value again but in the next lowest 16 bits
+                def!(c = raw_bitcast_any16x8_to_i32x4(b)), // no instruction emitted; pretend this is an I32x4 so we can use PSHUFD
+                def!(d = x86_pshufd(c, uimm8_zero)), // broadcast the bytes in the XMM register with PSHUFD
+                def!(y = raw_bitcast_i32x4_to_any16x8(d)), // no instruction emitted; pretend this is an X16x8 again
+            ],
+        );
+    }
+
+    // SIMD splat: 32-bits
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 32) {
+        let splat_any32x4 = splat.bind_vector(ty, 128 / ty.lane_bits());
+        narrow.legalize(
+            def!(y = splat_any32x4(x)),
+            vec![
+                def!(a = scalar_to_vector(x)), // translate to an x86 MOV to get the value in an XMM register
+                def!(y = x86_pshufd(a, uimm8_zero)), // broadcast the bytes in the XMM register with PSHUF
+            ],
+        );
+    }
+
+    // SIMD splat: 64-bits
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 64) {
+        let splat_any64x2 = splat.bind_vector(ty, 128 / ty.lane_bits());
+        narrow.legalize(
+            def!(y = splat_any64x2(x)),
+            vec![
+                def!(a = scalar_to_vector(x)), // move into the lowest 64 bits of an XMM register
+                def!(y = insertlane(a, uimm8_one, x)), // move into the highest 64 bits of the same XMM register
+            ],
+        );
+    }
+
+    narrow.build_and_add_to(&mut shared.transform_groups);
 }
--- a/third_party/rust/cranelift-codegen-meta/src/isa/x86/mod.rs
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/mod.rs
@@ -1,52 +1,75 @@
 use crate::cdsl::cpu_modes::CpuMode;
 use crate::cdsl::isa::TargetIsa;
 
 use crate::shared::types::Bool::B1;
 use crate::shared::types::Float::{F32, F64};
 use crate::shared::types::Int::{I16, I32, I64, I8};
 use crate::shared::Definitions as SharedDefinitions;
 
+mod encodings;
 mod instructions;
 mod legalize;
+mod recipes;
 mod registers;
 mod settings;
 
 pub fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
     let settings = settings::define(&shared_defs.settings);
     let regs = registers::define();
 
-    let inst_group = instructions::define(&shared_defs.format_registry);
+    let inst_group = instructions::define(
+        &mut shared_defs.all_instructions,
+        &shared_defs.format_registry,
+    );
     legalize::define(shared_defs, &inst_group);
 
     // CPU modes for 32-bit and 64-bit operations.
     let mut x86_64 = CpuMode::new("I64");
     let mut x86_32 = CpuMode::new("I32");
 
     let expand_flags = shared_defs.transform_groups.by_name("expand_flags");
     let narrow = shared_defs.transform_groups.by_name("narrow");
     let widen = shared_defs.transform_groups.by_name("widen");
+    let x86_narrow = shared_defs.transform_groups.by_name("x86_narrow");
     let x86_expand = shared_defs.transform_groups.by_name("x86_expand");
 
     x86_32.legalize_monomorphic(expand_flags);
     x86_32.legalize_default(narrow);
     x86_32.legalize_type(B1, expand_flags);
     x86_32.legalize_type(I8, widen);
     x86_32.legalize_type(I16, widen);
     x86_32.legalize_type(I32, x86_expand);
     x86_32.legalize_type(F32, x86_expand);
     x86_32.legalize_type(F64, x86_expand);
 
     x86_64.legalize_monomorphic(expand_flags);
-    x86_64.legalize_default(narrow);
+    x86_64.legalize_default(x86_narrow);
     x86_64.legalize_type(B1, expand_flags);
     x86_64.legalize_type(I8, widen);
     x86_64.legalize_type(I16, widen);
     x86_64.legalize_type(I32, x86_expand);
     x86_64.legalize_type(I64, x86_expand);
     x86_64.legalize_type(F32, x86_expand);
     x86_64.legalize_type(F64, x86_expand);
 
+    let recipes = recipes::define(shared_defs, &settings, &regs);
+
+    let encodings = encodings::define(shared_defs, &settings, &inst_group, &recipes);
+    x86_32.set_encodings(encodings.enc32);
+    x86_64.set_encodings(encodings.enc64);
+    let encodings_predicates = encodings.inst_pred_reg.extract();
+
+    let recipes = encodings.recipes;
+
     let cpu_modes = vec![x86_64, x86_32];
 
-    TargetIsa::new("x86", inst_group, settings, regs, cpu_modes)
+    TargetIsa::new(
+        "x86",
+        inst_group,
+        settings,
+        regs,
+        recipes,
+        cpu_modes,
+        encodings_predicates,
+    )
 }
new file mode 100644
--- /dev/null
+++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/recipes.rs
@@ -0,0 +1,2855 @@
+use std::rc::Rc;
+
+use crate::cdsl::ast::Literal;
+use crate::cdsl::formats::{FormatRegistry, InstructionFormat};
+use crate::cdsl::instructions::InstructionPredicate;
+use crate::cdsl::recipes::{
+    EncodingRecipe, EncodingRecipeBuilder, OperandConstraint, Register, Stack,
+};
+use crate::cdsl::regs::IsaRegs;
+use crate::cdsl::settings::SettingGroup;
+use crate::shared::Definitions as SharedDefinitions;
+
+/// Helper data structure to create recipes and template recipes.
+/// It contains all the recipes and recipe templates that might be used in the encodings crate of
+/// this same directory.
+pub struct RecipeGroup<'builder> {
+    /// Memoized format pointer, to pass it to builders later.
+    formats: &'builder FormatRegistry,
+
+    /// Memoized registers description, to pass it to builders later.
+    regs: &'builder IsaRegs,
+
+    /// All the recipes explicitly created in this file. This is different from the final set of
+    /// recipes, which is definitive only once encodings have generated new recipes on the fly.
+    recipes: Vec<EncodingRecipe>,
+
+    /// All the recipe templates created in this file.
+    templates: Vec<Rc<Template<'builder>>>,
+}
+
+impl<'builder> RecipeGroup<'builder> {
+    fn new(formats: &'builder FormatRegistry, regs: &'builder IsaRegs) -> Self {
+        Self {
+            formats,
+            regs,
+            recipes: Vec::new(),
+            templates: Vec::new(),
+        }
+    }
+    fn add_recipe(&mut self, recipe: EncodingRecipeBuilder) {
+        self.recipes.push(recipe.build(self.formats));
+    }
+    fn add_template_recipe(&mut self, recipe: EncodingRecipeBuilder) -> Rc<Template<'builder>> {
+        let template = Rc::new(Template::new(recipe, self.formats, self.regs));
+        self.templates.push(template.clone());
+        template
+    }
+    fn add_template(&mut self, template: Template<'builder>) -> Rc<Template<'builder>> {
+        let template = Rc::new(template);
+        self.templates.push(template.clone());
+        template
+    }
+    pub fn recipe(&self, name: &str) -> &EncodingRecipe {
+        self.recipes
+            .iter()
+            .find(|recipe| recipe.name == name)
+            .expect(&format!("unknown recipe name: {}. Try template?", name))
+    }
+    pub fn template(&self, name: &str) -> &Template {
+        self.templates
+            .iter()
+            .find(|recipe| recipe.name() == name)
+            .expect(&format!("unknown tail recipe name: {}. Try recipe?", name))
+    }
+}
+
+// Opcode representation.
+//
+// Cranelift requires each recipe to have a single encoding size in bytes, and x86 opcodes are
+// variable length, so we use separate recipes for different styles of opcodes and prefixes. The
+// opcode format is indicated by the recipe name prefix.
+//
+// The match case below does not include the REX prefix which goes after the mandatory prefix.
+// VEX/XOP and EVEX prefixes are not yet supported. Encodings using any of these prefixes are
+// represented by separate recipes.
+//
+// The encoding bits are:
+//
+// 0-7:   The opcode byte <op>.
+// 8-9:   pp, mandatory prefix:
+//        00 none (Op*)
+//        01 66   (Mp*)
+//        10 F3   (Mp*)
+//        11 F2   (Mp*)
+// 10-11: mm, opcode map:
+//        00 <op>        (Op1/Mp1)
+//        01 0F <op>     (Op2/Mp2)
+//        10 0F 38 <op>  (Op3/Mp3)
+//        11 0F 3A <op>  (Op3/Mp3)
+// 12-14  rrr, opcode bits for the ModR/M byte for certain opcodes.
+// 15:    REX.W bit (or VEX.W/E)
+//
+// There is some redundancy between bits 8-11 and the recipe names, but we have enough bits, and
+// the pp+mm format is ready for supporting VEX prefixes.
+//
+// TODO Cranelift doesn't actually require recipe to have different encoding sizes anymore, so this
+// could be simplified.
+
+/// Given a sequence of opcode bytes, compute the recipe name prefix and encoding bits.
+fn decode_opcodes(op_bytes: &[u8], rrr: u16, w: u16) -> (&'static str, u16) {
+    assert!(op_bytes.len() >= 1, "at least one opcode byte");
+
+    let prefix_bytes = &op_bytes[..op_bytes.len() - 1];
+    let (name, mmpp) = match prefix_bytes {
+        [] => ("Op1", 0b000),
+        [0x66] => ("Mp1", 0b0001),
+        [0xf3] => ("Mp1", 0b0010),
+        [0xf2] => ("Mp1", 0b0011),
+        [0x0f] => ("Op2", 0b0100),
+        [0x66, 0x0f] => ("Mp2", 0b0101),
+        [0xf3, 0x0f] => ("Mp2", 0b0110),
+        [0xf2, 0x0f] => ("Mp2", 0b0111),
+        [0x0f, 0x38] => ("Op3", 0b1000),
+        [0x66, 0x0f, 0x38] => ("Mp3", 0b1001),
+        [0xf3, 0x0f, 0x38] => ("Mp3", 0b1010),
+        [0xf2, 0x0f, 0x38] => ("Mp3", 0b1011),
+        [0x0f, 0x3a] => ("Op3", 0b1100),
+        [0x66, 0x0f, 0x3a] => ("Mp3", 0b1101),
+        [0xf3, 0x0f, 0x3a] => ("Mp3", 0b1110),
+        [0xf2, 0x0f, 0x3a] => ("Mp3", 0b1111),
+        _ => {
+            panic!("unexpected opcode sequence: {:?}", op_bytes);
+        }
+    };
+
+    let opcode_byte = op_bytes[op_bytes.len() - 1] as u16;
+    (name, opcode_byte | (mmpp << 8) | (rrr << 12) | w << 15)
+}
+
+/// Given a snippet of Rust code (or None), replace the `PUT_OP` macro with the
+/// corresponding `put_*` function from the `binemit.rs` module.
+fn replace_put_op(code: Option<String>, prefix: &str) -> Option<String> {
+    code.map(|code| code.replace("{{PUT_OP}}", &format!("put_{}", prefix.to_lowercase())))
+}
+
+/// Replaces constraints to a REX-prefixed register class by the equivalent non-REX register class.
+fn replace_nonrex_constraints(
+    regs: &IsaRegs,
+    constraints: Vec<OperandConstraint>,
+) -> Vec<OperandConstraint> {
+    constraints
+        .into_iter()
+        .map(|constraint| match constraint {
+            OperandConstraint::RegClass(rc_index) => {
+                let new_rc_index = if rc_index == regs.class_by_name("GPR") {
+                    regs.class_by_name("GPR8")
+                } else if rc_index == regs.class_by_name("FPR") {
+                    regs.class_by_name("FPR8")
+                } else {
+                    rc_index
+                };
+                OperandConstraint::RegClass(new_rc_index)
+            }
+            _ => constraint,
+        })
+        .collect()
+}
+
+/// Previously called a TailRecipe in the Python meta language, this allows to create multiple
+/// variants of a single base EncodingRecipe (rex prefix, specialized w/rrr bits, different
+/// opcodes). It serves as a prototype of an EncodingRecipe, which is then used when actually creating
+/// Encodings, in encodings.rs. This is an idiosyncrasy of the x86 meta-language, and could be
+/// reconsidered later.
+#[derive(Clone)]
+pub struct Template<'builder> {
+    /// Mapping of format indexes to format data, used in the build() method.
+    formats: &'builder FormatRegistry,
+
+    /// Description of registers, used in the build() method.
+    regs: &'builder IsaRegs,
+
+    /// The recipe template, which is to be specialized (by copy).
+    recipe: EncodingRecipeBuilder,
+
+    /// Does this recipe requires a REX prefix?
+    requires_prefix: bool,
+
+    /// Other recipe to use when REX-prefixed.
+    when_prefixed: Option<Rc<Template<'builder>>>,
+
+    // Specialized parameters.
+    /// Should we include the REX prefix?
+    rex: bool,
+    /// Value of the W bit (0 or 1).
+    w_bit: u16,
+    /// Value of the RRR bits (between 0 and 0b111).
+    rrr_bits: u16,
+    /// Opcode bytes.
+    op_bytes: Vec<u8>,
+}
+
+impl<'builder> Template<'builder> {
+    fn new(
+        recipe: EncodingRecipeBuilder,
+        formats: &'builder FormatRegistry,
+        regs: &'builder IsaRegs,
+    ) -> Self {
+        Self {
+            formats,
+            regs,
+            recipe,
+            requires_prefix: false,
+            when_prefixed: None,
+            rex: false,
+            w_bit: 0,
+            rrr_bits: 0,
+            op_bytes: Vec::new(),
+        }
+    }
+
+    fn name(&self) -> &str {
+        &self.recipe.name
+    }
+    fn requires_prefix(self, value: bool) -> Self {
+        Self {
+            requires_prefix: value,
+            ..self
+        }
+    }
+    fn when_prefixed(self, template: Rc<Template<'builder>>) -> Self {
+        assert!(self.when_prefixed.is_none());
+        Self {
+            when_prefixed: Some(template),
+            ..self
+        }
+    }
+
+    // Copy setters.
+    pub fn opcodes(&self, op_bytes: Vec<u8>) -> Self {
+        assert!(!op_bytes.is_empty());
+        let mut copy = self.clone();
+        copy.op_bytes = op_bytes;
+        copy
+    }
+    pub fn w(&self) -> Self {
+        let mut copy = self.clone();
+        copy.w_bit = 1;
+        copy
+    }
+    pub fn rrr(&self, value: u16) -> Self {
+        assert!(value <= 0b111);
+        let mut copy = self.clone();
+        copy.rrr_bits = value;
+        copy
+    }
+    pub fn nonrex(&self) -> Self {
+        assert!(!self.requires_prefix, "Tail recipe requires REX prefix.");
+        let mut copy = self.clone();
+        copy.rex = false;
+        copy
+    }
+    pub fn rex(&self) -> Self {
+        if let Some(prefixed) = &self.when_prefixed {
+            let mut ret = prefixed.rex();
+            // Forward specialized parameters.
+            ret.op_bytes = self.op_bytes.clone();
+            ret.w_bit = self.w_bit;
+            ret.rrr_bits = self.rrr_bits;
+            return ret;
+        }
+        let mut copy = self.clone();
+        copy.rex = true;
+        copy
+    }
+
+    pub fn build(mut self) -> (EncodingRecipe, u16) {
+        let (name, bits) = decode_opcodes(&self.op_bytes, self.rrr_bits, self.w_bit);
+
+        let (name, rex_prefix_size) = if self.rex {
+            ("Rex".to_string() + name, 1)
+        } else {
+            (name.into(), 0)
+        };
+
+        let size_addendum = self.op_bytes.len() as u64 + rex_prefix_size;
+        self.recipe.base_size += size_addendum;
+
+        // Branch ranges are relative to the end of the instruction.
+        self.recipe
+            .branch_range
+            .as_mut()
+            .map(|range| range.inst_size += size_addendum);
+
+        self.recipe.emit = replace_put_op(self.recipe.emit, &name);
+        self.recipe.name = name + &self.recipe.name;
+
+        if !self.rex {
+            let operands_in = self.recipe.operands_in.unwrap_or(Vec::new());
+            self.recipe.operands_in = Some(replace_nonrex_constraints(self.regs, operands_in));
+            let operands_out = self.recipe.operands_out.unwrap_or(Vec::new());
+            self.recipe.operands_out = Some(replace_nonrex_constraints(self.regs, operands_out));
+        }
+
+        (self.recipe.build(self.formats), bits)
+    }
+}
+
+/// Returns a predicate checking that the "cond" field of the instruction contains one of the
+/// directly supported floating point condition codes.
+fn supported_floatccs_predicate(
+    supported_cc: &[Literal],
+    format: &InstructionFormat,
+) -> InstructionPredicate {
+    supported_cc
+        .iter()
+        .fold(InstructionPredicate::new(), |pred, literal| {
+            pred.or(InstructionPredicate::new_is_field_equal(
+                format,
+                "cond",
+                literal.to_rust_code(),
+            ))
+        })
+}
+
+/// Return an instruction predicate that checks if `iform.imm` is a valid `scale` for a SIB byte.
+fn valid_scale(format: &InstructionFormat) -> InstructionPredicate {
+    ["1", "2", "4", "8"]
+        .iter()
+        .fold(InstructionPredicate::new(), |pred, &literal| {
+            pred.or(InstructionPredicate::new_is_field_equal(
+                format,
+                "imm",
+                literal.into(),
+            ))
+        })
+}
+
+pub fn define<'shared>(
+    shared_defs: &'shared SharedDefinitions,
+    settings: &'shared SettingGroup,
+    regs: &'shared IsaRegs,
+) -> RecipeGroup<'shared> {
+    // The set of floating point condition codes that are directly supported.
+    // Other condition codes need to be reversed or expressed as two tests.
+    let floatcc = shared_defs.operand_kinds.by_name("floatcc");
+    let supported_floatccs: Vec<Literal> = ["ord", "uno", "one", "ueq", "gt", "ge", "ult", "ule"]
+        .iter()
+        .map(|name| Literal::enumerator_for(floatcc, name))
+        .collect();
+
+    let formats = &shared_defs.format_registry;
+
+    // Register classes shorthands.
+    let abcd = regs.class_by_name("ABCD");
+    let gpr = regs.class_by_name("GPR");
+    let fpr = regs.class_by_name("FPR");
+    let flag = regs.class_by_name("FLAG");
+
+    // Operand constraints shorthands.
+    let reg_rflags = Register::new(flag, regs.regunit_by_name(flag, "rflags"));
+    let reg_rax = Register::new(gpr, regs.regunit_by_name(gpr, "rax"));
+    let reg_rcx = Register::new(gpr, regs.regunit_by_name(gpr, "rcx"));
+    let reg_rdx = Register::new(gpr, regs.regunit_by_name(gpr, "rdx"));
+
+    // Stack operand with a 32-bit signed displacement from either RBP or RSP.
+    let stack_gpr32 = Stack::new(gpr);
+    let stack_fpr32 = Stack::new(fpr);
+
+    // Format shorthands, prefixed with f_.
+    let f_binary = formats.by_name("Binary");
+    let f_binary_imm = formats.by_name("BinaryImm");
+    let f_branch = formats.by_name("Branch");
+    let f_branch_float = formats.by_name("BranchFloat");
+    let f_branch_int = formats.by_name("BranchInt");
+    let f_branch_table_entry = formats.by_name("BranchTableEntry");
+    let f_branch_table_base = formats.by_name("BranchTableBase");
+    let f_call = formats.by_name("Call");
+    let f_call_indirect = formats.by_name("CallIndirect");
+    let f_copy_special = formats.by_name("CopySpecial");
+    let f_extract_lane = formats.by_name("ExtractLane"); // TODO this would preferably retrieve a BinaryImm8 format but because formats are compared structurally and ExtractLane has the same structure this is impossible--if we rename ExtractLane, it may even impact parsing
+    let f_float_compare = formats.by_name("FloatCompare");
+    let f_float_cond = formats.by_name("FloatCond");
+    let f_float_cond_trap = formats.by_name("FloatCondTrap");
+    let f_func_addr = formats.by_name("FuncAddr");
+    let f_indirect_jump = formats.by_name("IndirectJump");
+    let f_insert_lane = formats.by_name("InsertLane");
+    let f_int_compare = formats.by_name("IntCompare");
+    let f_int_compare_imm = formats.by_name("IntCompareImm");
+    let f_int_cond = formats.by_name("IntCond");
+    let f_int_cond_trap = formats.by_name("IntCondTrap");
+    let f_int_select = formats.by_name("IntSelect");
+    let f_jump = formats.by_name("Jump");
+    let f_load = formats.by_name("Load");
+    let f_load_complex = formats.by_name("LoadComplex");
+    let f_multiary = formats.by_name("MultiAry");
+    let f_nullary = formats.by_name("NullAry");
+    let f_reg_fill = formats.by_name("RegFill");
+    let f_reg_move = formats.by_name("RegMove");
+    let f_reg_spill = formats.by_name("RegSpill");
+    let f_stack_load = formats.by_name("StackLoad");
+    let f_store = formats.by_name("Store");
+    let f_store_complex = formats.by_name("StoreComplex");
+    let f_ternary = formats.by_name("Ternary");
+    let f_trap = formats.by_name("Trap");
+    let f_unary = formats.by_name("Unary");
+    let f_unary_bool = formats.by_name("UnaryBool");
+    let f_unary_global_value = formats.by_name("UnaryGlobalValue");
+    let f_unary_ieee32 = formats.by_name("UnaryIeee32");
+    let f_unary_ieee64 = formats.by_name("UnaryIeee64");
+    let f_unary_imm = formats.by_name("UnaryImm");
+
+    // Predicates shorthands.
+    let use_sse41 = settings.predicate_by_name("use_sse41");
+
+    // Definitions.
+    let mut recipes = RecipeGroup::new(formats, regs);
+
+    // A null unary instruction that takes a GPR register. Can be used for identity copies and
+    // no-op conversions.
+    recipes.add_recipe(
+        EncodingRecipeBuilder::new("null", f_unary, 0)
+            .operands_in(vec![gpr])
+            .operands_out(vec![0])
+            .emit(""),
+    );
+    recipes.add_recipe(
+        EncodingRecipeBuilder::new("null_fpr", f_unary, 0)
+            .operands_in(vec![fpr])
+            .operands_out(vec![0])
+            .emit(""),
+    );
+    recipes.add_recipe(
+        EncodingRecipeBuilder::new("stacknull", f_unary, 0)
+            .operands_in(vec![stack_gpr32])
+            .operands_out(vec![stack_gpr32])
+            .emit(""),
+    );
+
+    recipes
+        .add_recipe(EncodingRecipeBuilder::new("debugtrap", f_nullary, 1).emit("sink.put1(0xcc);"));
+
+    // XX opcode, no ModR/M.
+    recipes.add_template_recipe(EncodingRecipeBuilder::new("trap", f_trap, 0).emit(
+        r#"
+            sink.trap(code, func.srclocs[inst]);
+            {{PUT_OP}}(bits, BASE_REX, sink);
+        "#,
+    ));
+
+    // Macro: conditional jump over a ud2.
+    recipes.add_recipe(
+        EncodingRecipeBuilder::new("trapif", f_int_cond_trap, 4)
+            .operands_in(vec![reg_rflags])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    // Jump over a 2-byte ud2.
+                    sink.put1(0x70 | (icc2opc(cond.inverse()) as u8));
+                    sink.put1(2);
+                    // ud2.
+                    sink.trap(code, func.srclocs[inst]);
+                    sink.put1(0x0f);
+                    sink.put1(0x0b);
+                "#,
+            ),
+    );
+
+    recipes.add_recipe(
+        EncodingRecipeBuilder::new("trapff", f_float_cond_trap, 4)
+            .operands_in(vec![reg_rflags])
+            .clobbers_flags(false)
+            .inst_predicate(supported_floatccs_predicate(
+                &supported_floatccs,
+                formats.get(f_float_cond_trap),
+            ))
+            .emit(
+                r#"
+                    // Jump over a 2-byte ud2.
+                    sink.put1(0x70 | (fcc2opc(cond.inverse()) as u8));
+                    sink.put1(2);
+                    // ud2.
+                    sink.trap(code, func.srclocs[inst]);
+                    sink.put1(0x0f);
+                    sink.put1(0x0b);
+                "#,
+            ),
+    );
+
+    // XX /r
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("rr", f_binary, 1)
+            .operands_in(vec![gpr, gpr])
+            .operands_out(vec![0])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, in_reg1), sink);
+                    modrm_rr(in_reg0, in_reg1, sink);
+                "#,
+            ),
+    );
+
+    // XX /r with operands swapped. (RM form).
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("rrx", f_binary, 1)
+            .operands_in(vec![gpr, gpr])
+            .operands_out(vec![0])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg1, in_reg0), sink);
+                    modrm_rr(in_reg1, in_reg0, sink);
+                "#,
+            ),
+    );
+
+    // XX /r with FPR ins and outs. A form.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("fa", f_binary, 1)
+            .operands_in(vec![fpr, fpr])
+            .operands_out(vec![0])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg1, in_reg0), sink);
+                    modrm_rr(in_reg1, in_reg0, sink);
+                "#,
+            ),
+    );
+
+    // XX /r with FPR ins and outs. A form with input operands swapped.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("fax", f_binary, 1)
+            .operands_in(vec![fpr, fpr])
+            .operands_out(vec![1])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, in_reg1), sink);
+                    modrm_rr(in_reg0, in_reg1, sink);
+                "#,
+            ),
+    );
+
+    // XX /n for a unary operation with extension bits.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("ur", f_unary, 1)
+            .operands_in(vec![gpr])
+            .operands_out(vec![0])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex1(in_reg0), sink);
+                    modrm_r_bits(in_reg0, bits, sink);
+                "#,
+            ),
+    );
+
+    // XX /r, but for a unary operator with separate input/output register, like
+    // copies. MR form, preserving flags.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("umr", f_unary, 1)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(out_reg0, in_reg0), sink);
+                    modrm_rr(out_reg0, in_reg0, sink);
+                "#,
+            ),
+    );
+
+    // Same as umr, but with FPR -> GPR registers.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("rfumr", f_unary, 1)
+            .operands_in(vec![fpr])
+            .operands_out(vec![gpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(out_reg0, in_reg0), sink);
+                    modrm_rr(out_reg0, in_reg0, sink);
+                "#,
+            ),
+    );
+
+    // XX /r, but for a unary operator with separate input/output register.
+    // RM form. Clobbers FLAGS.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("urm", f_unary, 1)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, out_reg0), sink);
+                    modrm_rr(in_reg0, out_reg0, sink);
+                "#,
+            ),
+    );
+
+    // XX /r. Same as urm, but doesn't clobber FLAGS.
+    let urm_noflags = recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("urm_noflags", f_unary, 1)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, out_reg0), sink);
+                    modrm_rr(in_reg0, out_reg0, sink);
+                "#,
+            ),
+    );
+
+    // XX /r. Same as urm_noflags, but input limited to ABCD.
+    recipes.add_template(
+        Template::new(
+            EncodingRecipeBuilder::new("urm_noflags_abcd", f_unary, 1)
+                .operands_in(vec![abcd])
+                .operands_out(vec![gpr])
+                .clobbers_flags(false)
+                .emit(
+                    r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, out_reg0), sink);
+                    modrm_rr(in_reg0, out_reg0, sink);
+                "#,
+                ),
+            formats,
+            regs,
+        )
+        .when_prefixed(urm_noflags),
+    );
+
+    // XX /r, RM form, FPR -> FPR.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("furm", f_unary, 1)
+            .operands_in(vec![fpr])
+            .operands_out(vec![fpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, out_reg0), sink);
+                    modrm_rr(in_reg0, out_reg0, sink);
+                "#,
+            ),
+    );
+
+    // XX /r, RM form, GPR -> FPR.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("frurm", f_unary, 1)
+            .operands_in(vec![gpr])
+            .operands_out(vec![fpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, out_reg0), sink);
+                    modrm_rr(in_reg0, out_reg0, sink);
+                "#,
+            ),
+    );
+
+    // XX /r, RM form, FPR -> GPR.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("rfurm", f_unary, 1)
+            .operands_in(vec![fpr])
+            .operands_out(vec![gpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, out_reg0), sink);
+                    modrm_rr(in_reg0, out_reg0, sink);
+                "#,
+            ),
+    );
+
+    // XX /r, RMI form for one of the roundXX SSE 4.1 instructions.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("furmi_rnd", f_unary, 2)
+            .operands_in(vec![fpr])
+            .operands_out(vec![fpr])
+            .isa_predicate(use_sse41)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, out_reg0), sink);
+                    modrm_rr(in_reg0, out_reg0, sink);
+                    sink.put1(match opcode {
+                        Opcode::Nearest => 0b00,
+                        Opcode::Floor => 0b01,
+                        Opcode::Ceil => 0b10,
+                        Opcode::Trunc => 0b11,
+                        x => panic!("{} unexpected for furmi_rnd", opcode),
+                    });
+                "#,
+            ),
+    );
+
+    // XX /r, for regmove instructions.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("rmov", f_reg_move, 1)
+            .operands_in(vec![gpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(dst, src), sink);
+                    modrm_rr(dst, src, sink);
+                "#,
+            ),
+    );
+
+    // XX /r, for regmove instructions (FPR version, RM encoded).
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("frmov", f_reg_move, 1)
+            .operands_in(vec![fpr])
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(src, dst), sink);
+                    modrm_rr(src, dst, sink);
+                "#,
+            ),
+    );
+
+    // XX /n with one arg in %rcx, for shifts.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("rc", f_binary, 1)
+            .operands_in(vec![
+                OperandConstraint::RegClass(gpr),
+                OperandConstraint::FixedReg(reg_rcx),
+            ])
+            .operands_out(vec![0])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex1(in_reg0), sink);
+                    modrm_r_bits(in_reg0, bits, sink);
+                "#,
+            ),
+    );
+
+    // XX /n for division: inputs in %rax, %rdx, r. Outputs in %rax, %rdx.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("div", f_ternary, 1)
+            .operands_in(vec![
+                OperandConstraint::FixedReg(reg_rax),
+                OperandConstraint::FixedReg(reg_rdx),
+                OperandConstraint::RegClass(gpr),
+            ])
+            .operands_out(vec![reg_rax, reg_rdx])
+            .emit(
+                r#"
+                    sink.trap(TrapCode::IntegerDivisionByZero, func.srclocs[inst]);
+                    {{PUT_OP}}(bits, rex1(in_reg2), sink);
+                    modrm_r_bits(in_reg2, bits, sink);
+                "#,
+            ),
+    );
+
+    // XX /n for {s,u}mulx: inputs in %rax, r. Outputs in %rdx(hi):%rax(lo)
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("mulx", f_binary, 1)
+            .operands_in(vec![
+                OperandConstraint::FixedReg(reg_rax),
+                OperandConstraint::RegClass(gpr),
+            ])
+            .operands_out(vec![
+                OperandConstraint::FixedReg(reg_rax),
+                OperandConstraint::FixedReg(reg_rdx),
+            ])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex1(in_reg1), sink);
+                    modrm_r_bits(in_reg1, bits, sink);
+                "#,
+            ),
+    );
+
+    // XX /n ib with 8-bit immediate sign-extended.
+    {
+        let format = formats.get(f_binary_imm);
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("r_ib", f_binary_imm, 2)
+                .operands_in(vec![gpr])
+                .operands_out(vec![0])
+                .inst_predicate(InstructionPredicate::new_is_signed_int(format, "imm", 8, 0))
+                .emit(
+                    r#"
+                        {{PUT_OP}}(bits, rex1(in_reg0), sink);
+                        modrm_r_bits(in_reg0, bits, sink);
+                        let imm: i64 = imm.into();
+                        sink.put1(imm as u8);
+                    "#,
+                ),
+        );
+
+        // XX /n id with 32-bit immediate sign-extended.
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("r_id", f_binary_imm, 5)
+                .operands_in(vec![gpr])
+                .operands_out(vec![0])
+                .inst_predicate(InstructionPredicate::new_is_signed_int(
+                    format, "imm", 32, 0,
+                ))
+                .emit(
+                    r#"
+                        {{PUT_OP}}(bits, rex1(in_reg0), sink);
+                        modrm_r_bits(in_reg0, bits, sink);
+                        let imm: i64 = imm.into();
+                        sink.put4(imm as u32);
+                    "#,
+                ),
+        );
+    }
+
+    // XX /r ib with 8-bit unsigned immediate (e.g. for pshufd)
+    {
+        let format = formats.get(f_extract_lane);
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("r_ib_unsigned", f_extract_lane, 2)
+                .operands_in(vec![fpr])
+                .operands_out(vec![fpr])
+                .inst_predicate(InstructionPredicate::new_is_unsigned_int(
+                    format, "lane", 8, 0,
+                )) // TODO if the format name is changed then "lane" should be renamed to something more appropriate--ordering mask? broadcast immediate?
+                .emit(
+                    r#"
+                    {{PUT_OP}}(bits, rex2(in_reg0, out_reg0), sink);
+                    modrm_rr(in_reg0, out_reg0, sink);
+                    let imm:i64 = lane.into();
+                    sink.put1(imm as u8);
+                "#,
+                ),
+        );
+    }
+
+    // XX /r ib with 8-bit unsigned immediate (e.g. for insertlane)
+    {
+        let format = formats.get(f_insert_lane);
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("r_ib_unsigned_r", f_insert_lane, 2)
+                .operands_in(vec![fpr, gpr])
+                .operands_out(vec![0])
+                .inst_predicate(InstructionPredicate::new_is_unsigned_int(
+                    format, "lane", 8, 0,
+                ))
+                .emit(
+                    r#"
+                    {{PUT_OP}}(bits, rex2(in_reg1, in_reg0), sink);
+                    modrm_rr(in_reg1, in_reg0, sink);
+                    let imm:i64 = lane.into();
+                    sink.put1(imm as u8);
+                "#,
+                ),
+        );
+    }
+
+    {
+        // XX /n id with 32-bit immediate sign-extended. UnaryImm version.
+        let format = formats.get(f_unary_imm);
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("u_id", f_unary_imm, 5)
+                .operands_out(vec![gpr])
+                .inst_predicate(InstructionPredicate::new_is_signed_int(
+                    format, "imm", 32, 0,
+                ))
+                .emit(
+                    r#"
+                        {{PUT_OP}}(bits, rex1(out_reg0), sink);
+                        modrm_r_bits(out_reg0, bits, sink);
+                        let imm: i64 = imm.into();
+                        sink.put4(imm as u32);
+                    "#,
+                ),
+        );
+    }
+
+    // XX+rd id unary with 32-bit immediate. Note no recipe predicate.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("pu_id", f_unary_imm, 4)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    // The destination register is encoded in the low bits of the opcode.
+                    // No ModR/M.
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    let imm: i64 = imm.into();
+                    sink.put4(imm as u32);
+                "#,
+            ),
+    );
+
+    // XX+rd id unary with bool immediate. Note no recipe predicate.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("pu_id_bool", f_unary_bool, 4)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    // The destination register is encoded in the low bits of the opcode.
+                    // No ModR/M.
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    let imm: u32 = if imm { 1 } else { 0 };
+                    sink.put4(imm);
+                "#,
+            ),
+    );
+
+    // XX+rd iq unary with 64-bit immediate.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("pu_iq", f_unary_imm, 8)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    let imm: i64 = imm.into();
+                    sink.put8(imm as u64);
+                "#,
+            ),
+    );
+
+    // XX /n Unary with floating point 32-bit immediate equal to zero.
+    {
+        let format = formats.get(f_unary_ieee32);
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("f32imm_z", f_unary_ieee32, 1)
+                .operands_out(vec![fpr])
+                .inst_predicate(InstructionPredicate::new_is_zero_32bit_float(format, "imm"))
+                .emit(
+                    r#"
+                        {{PUT_OP}}(bits, rex2(out_reg0, out_reg0), sink);
+                        modrm_rr(out_reg0, out_reg0, sink);
+                    "#,
+                ),
+        );
+    }
+
+    // XX /n Unary with floating point 64-bit immediate equal to zero.
+    {
+        let format = formats.get(f_unary_ieee64);
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("f64imm_z", f_unary_ieee64, 1)
+                .operands_out(vec![fpr])
+                .inst_predicate(InstructionPredicate::new_is_zero_64bit_float(format, "imm"))
+                .emit(
+                    r#"
+                        {{PUT_OP}}(bits, rex2(out_reg0, out_reg0), sink);
+                        modrm_rr(out_reg0, out_reg0, sink);
+                    "#,
+                ),
+        );
+    }
+
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("pushq", f_unary, 0)
+            .operands_in(vec![gpr])
+            .emit(
+                r#"
+                    sink.trap(TrapCode::StackOverflow, func.srclocs[inst]);
+                    {{PUT_OP}}(bits | (in_reg0 & 7), rex1(in_reg0), sink);
+                "#,
+            ),
+    );
+
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("popq", f_nullary, 0)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                "#,
+            ),
+    );
+
+    // XX /r, for regmove instructions.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("copysp", f_copy_special, 1)
+            .clobbers_flags(false)
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(dst, src), sink);
+                    modrm_rr(dst, src, sink);
+                "#,
+            ),
+    );
+
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("adjustsp", f_unary, 1)
+            .operands_in(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(RU::rsp.into(), in_reg0), sink);
+                    modrm_rr(RU::rsp.into(), in_reg0, sink);
+                "#,
+            ),
+    );
+
+    {
+        let format = formats.get(f_unary_imm);
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("adjustsp_ib", f_unary_imm, 2)
+                .inst_predicate(InstructionPredicate::new_is_signed_int(format, "imm", 8, 0))
+                .emit(
+                    r#"
+                        {{PUT_OP}}(bits, rex1(RU::rsp.into()), sink);
+                        modrm_r_bits(RU::rsp.into(), bits, sink);
+                        let imm: i64 = imm.into();
+                        sink.put1(imm as u8);
+                    "#,
+                ),
+        );
+
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("adjustsp_id", f_unary_imm, 5)
+                .inst_predicate(InstructionPredicate::new_is_signed_int(
+                    format, "imm", 32, 0,
+                ))
+                .emit(
+                    r#"
+                        {{PUT_OP}}(bits, rex1(RU::rsp.into()), sink);
+                        modrm_r_bits(RU::rsp.into(), bits, sink);
+                        let imm: i64 = imm.into();
+                        sink.put4(imm as u32);
+                    "#,
+                ),
+        );
+    }
+
+    // XX+rd id with Abs4 function relocation.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("fnaddr4", f_func_addr, 4)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    sink.reloc_external(Reloc::Abs4,
+                                        &func.dfg.ext_funcs[func_ref].name,
+                                        0);
+                    sink.put4(0);
+                "#,
+            ),
+    );
+
+    // XX+rd iq with Abs8 function relocation.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("fnaddr8", f_func_addr, 8)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    sink.reloc_external(Reloc::Abs8,
+                                        &func.dfg.ext_funcs[func_ref].name,
+                                        0);
+                    sink.put8(0);
+                "#,
+            ),
+    );
+
+    // Similar to fnaddr4, but writes !0 (this is used by BaldrMonkey).
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("allones_fnaddr4", f_func_addr, 4)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    sink.reloc_external(Reloc::Abs4,
+                                        &func.dfg.ext_funcs[func_ref].name,
+                                        0);
+                    // Write the immediate as `!0` for the benefit of BaldrMonkey.
+                    sink.put4(!0);
+                "#,
+            ),
+    );
+
+    // Similar to fnaddr8, but writes !0 (this is used by BaldrMonkey).
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("allones_fnaddr8", f_func_addr, 8)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    sink.reloc_external(Reloc::Abs8,
+                                        &func.dfg.ext_funcs[func_ref].name,
+                                        0);
+                    // Write the immediate as `!0` for the benefit of BaldrMonkey.
+                    sink.put8(!0);
+                "#,
+            ),
+    );
+
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("pcrel_fnaddr8", f_func_addr, 5)
+            .operands_out(vec![gpr])
+            // rex2 gets passed 0 for r/m register because the upper bit of
+            // r/m doesn't get decoded when in rip-relative addressing mode.
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(0, out_reg0), sink);
+                    modrm_riprel(out_reg0, sink);
+                    // The addend adjusts for the difference between the end of the
+                    // instruction and the beginning of the immediate field.
+                    sink.reloc_external(Reloc::X86PCRel4,
+                                        &func.dfg.ext_funcs[func_ref].name,
+                                        -4);
+                    sink.put4(0);
+                "#,
+            ),
+    );
+
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("got_fnaddr8", f_func_addr, 5)
+            .operands_out(vec![gpr])
+            // rex2 gets passed 0 for r/m register because the upper bit of
+            // r/m doesn't get decoded when in rip-relative addressing mode.
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(0, out_reg0), sink);
+                    modrm_riprel(out_reg0, sink);
+                    // The addend adjusts for the difference between the end of the
+                    // instruction and the beginning of the immediate field.
+                    sink.reloc_external(Reloc::X86GOTPCRel4,
+                                        &func.dfg.ext_funcs[func_ref].name,
+                                        -4);
+                    sink.put4(0);
+                "#,
+            ),
+    );
+
+    // XX+rd id with Abs4 globalsym relocation.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("gvaddr4", f_unary_global_value, 4)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    sink.reloc_external(Reloc::Abs4,
+                                        &func.global_values[global_value].symbol_name(),
+                                        0);
+                    sink.put4(0);
+                "#,
+            ),
+    );
+
+    // XX+rd iq with Abs8 globalsym relocation.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("gvaddr8", f_unary_global_value, 8)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits | (out_reg0 & 7), rex1(out_reg0), sink);
+                    sink.reloc_external(Reloc::Abs8,
+                                        &func.global_values[global_value].symbol_name(),
+                                        0);
+                    sink.put8(0);
+                "#,
+            ),
+    );
+
+    // XX+rd iq with PCRel4 globalsym relocation.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("pcrel_gvaddr8", f_unary_global_value, 5)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(0, out_reg0), sink);
+                    modrm_rm(5, out_reg0, sink);
+                    // The addend adjusts for the difference between the end of the
+                    // instruction and the beginning of the immediate field.
+                    sink.reloc_external(Reloc::X86PCRel4,
+                                        &func.global_values[global_value].symbol_name(),
+                                        -4);
+                    sink.put4(0);
+                "#,
+            ),
+    );
+
+    // XX+rd iq with Abs8 globalsym relocation.
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("got_gvaddr8", f_unary_global_value, 5)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    {{PUT_OP}}(bits, rex2(0, out_reg0), sink);
+                    modrm_rm(5, out_reg0, sink);
+                    // The addend adjusts for the difference between the end of the
+                    // instruction and the beginning of the immediate field.
+                    sink.reloc_external(Reloc::X86GOTPCRel4,
+                                        &func.global_values[global_value].symbol_name(),
+                                        -4);
+                    sink.put4(0);
+                "#,
+            ),
+    );
+
+    // Stack addresses.
+    //
+    // TODO Alternative forms for 8-bit immediates, when applicable.
+
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("spaddr4_id", f_stack_load, 6)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    let sp = StackRef::sp(stack_slot, &func.stack_slots);
+                    let base = stk_base(sp.base);
+                    {{PUT_OP}}(bits, rex2(out_reg0, base), sink);
+                    modrm_sib_disp8(out_reg0, sink);
+                    sib_noindex(base, sink);
+                    let imm : i32 = offset.into();
+                    sink.put4(sp.offset.checked_add(imm).unwrap() as u32);
+                "#,
+            ),
+    );
+
+    recipes.add_template_recipe(
+        EncodingRecipeBuilder::new("spaddr8_id", f_stack_load, 6)
+            .operands_out(vec![gpr])
+            .emit(
+                r#"
+                    let sp = StackRef::sp(stack_slot, &func.stack_slots);
+                    let base = stk_base(sp.base);
+                    {{PUT_OP}}(bits, rex2(base, out_reg0), sink);
+                    modrm_sib_disp32(out_reg0, sink);
+                    sib_noindex(base, sink);
+                    let imm : i32 = offset.into();
+                    sink.put4(sp.offset.checked_add(imm).unwrap() as u32);
+                "#,
+            ),
+    );
+
+    // Store recipes.
+
+    {
+        // Simple stores.
+        let format = formats.get(f_store);
+
+        // A predicate asking if the offset is zero.
+        let has_no_offset = InstructionPredicate::new_is_field_equal(format, "offset", "0".into());
+
+        // XX /r register-indirect store with no offset.
+        let st = recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("st", f_store, 1)
+                .operands_in(vec![gpr, gpr])
+                .inst_predicate(has_no_offset.clone())
+                .clobbers_flags(false)
+                .compute_size("size_plus_maybe_sib_or_offset_for_in_reg_1")
+                .emit(
+                    r#"
+                        if !flags.notrap() {
+                            sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
+                        }
+                        {{PUT_OP}}(bits, rex2(in_reg1, in_reg0), sink);
+                        if needs_sib_byte(in_reg1) {
+                            modrm_sib(in_reg0, sink);
+                            sib_noindex(in_reg1, sink);
+                        } else if needs_offset(in_reg1) {
+                            modrm_disp8(in_reg1, in_reg0, sink);
+                            sink.put1(0);
+                        } else {
+                            modrm_rm(in_reg1, in_reg0, sink);
+                        }
+                    "#,
+                ),
+        );
+
+        // XX /r register-indirect store with no offset.
+        // Only ABCD allowed for stored value. This is for byte stores with no REX.
+        recipes.add_template(
+            Template::new(
+                EncodingRecipeBuilder::new("st_abcd", f_store, 1)
+                    .operands_in(vec![abcd, gpr])
+                    .inst_predicate(has_no_offset.clone())
+                    .clobbers_flags(false)
+                    .compute_size("size_plus_maybe_sib_or_offset_for_in_reg_1")
+                    .emit(
+                        r#"
+                        if !flags.notrap() {
+                            sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
+                        }
+                        {{PUT_OP}}(bits, rex2(in_reg1, in_reg0), sink);
+                        if needs_sib_byte(in_reg1) {
+                            modrm_sib(in_reg0, sink);
+                            sib_noindex(in_reg1, sink);
+                        } else if needs_offset(in_reg1) {
+                            modrm_disp8(in_reg1, in_reg0, sink);
+                            sink.put1(0);
+                        } else {
+                            modrm_rm(in_reg1, in_reg0, sink);
+                        }
+                    "#,
+                    ),
+                formats,
+                regs,
+            )
+            .when_prefixed(st),
+        );
+
+        // XX /r register-indirect store of FPR with no offset.
+        recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("fst", f_store, 1)
+                .operands_in(vec![fpr, gpr])
+                .inst_predicate(has_no_offset.clone())
+                .clobbers_flags(false)
+                .compute_size("size_plus_maybe_sib_or_offset_for_in_reg_1")
+                .emit(
+                    r#"
+                        if !flags.notrap() {
+                            sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
+                        }
+                        {{PUT_OP}}(bits, rex2(in_reg1, in_reg0), sink);
+                        if needs_sib_byte(in_reg1) {
+                            modrm_sib(in_reg0, sink);
+                            sib_noindex(in_reg1, sink);
+                        } else if needs_offset(in_reg1) {
+                            modrm_disp8(in_reg1, in_reg0, sink);
+                            sink.put1(0);
+                        } else {
+                            modrm_rm(in_reg1, in_reg0, sink);
+                        }
+                    "#,
+                ),
+        );
+
+        let has_small_offset = InstructionPredicate::new_is_signed_int(format, "offset", 8, 0);
+
+        // XX /r register-indirect store with 8-bit offset.
+        let st_disp8 = recipes.add_template_recipe(
+            EncodingRecipeBuilder::new("stDisp8", f_store, 2)
+                .operands_in(vec![gpr, gpr])