mirror of
https://github.com/odin-lang/Odin.git
synced 2026-10-09 06:12:05 -04:00
Move the lowering for LLVM's fast instruction selector to llvm_backend_opt.cpp
This commit is contained in:
gingerBill committed
1 parent
ddcf115f6f
commit
ccf1ae4fe9
4 files changed
+623
-622
No files matched your search
@@ -2546,627 +2546,6 @@ gb_internal bool lb_is_module_empty(lbModule *m) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// NOTE(bill, 2026-10-03)
|
||||
//
|
||||
// LLVM's fast instruction selector cannot select a first class aggregate `load`, `store`, `insertvalue`, or `select`,
|
||||
// and hands the rest of the block to SelectionDAG.
|
||||
// SROA leaves many behind so it stores/loads the fields instead
|
||||
enum {
|
||||
LB_SCALARIZE_MAX_LEAVES = 32,
|
||||
LB_SCALARIZE_MAX_DEPTH = 8,
|
||||
};
|
||||
|
||||
struct lbAggregateLeaf {
|
||||
unsigned path[LB_SCALARIZE_MAX_DEPTH];
|
||||
unsigned depth;
|
||||
};
|
||||
|
||||
struct lbScalarizedPhi {
|
||||
LLVMValueRef aggregate;
|
||||
unsigned path[LB_SCALARIZE_MAX_DEPTH];
|
||||
unsigned depth;
|
||||
LLVMValueRef field;
|
||||
bool ok;
|
||||
};
|
||||
|
||||
struct lbFastIselLowering {
|
||||
lbModule * m;
|
||||
LLVMBuilderRef builder;
|
||||
LLVMValueRef store;
|
||||
Array<lbScalarizedPhi> phis;
|
||||
};
|
||||
|
||||
gb_internal i64 lb_aggregate_path_offset(LLVMTypeRef type, unsigned const *path, unsigned depth, LLVMTypeRef *leaf_type_) {
|
||||
i64 offset = 0;
|
||||
for (unsigned d = 0; d < depth; d++) {
|
||||
if (LLVMGetTypeKind(type) == LLVMStructTypeKind) {
|
||||
bool is_packed = LLVMIsPackedStruct(type);
|
||||
i64 field_offset = 0;
|
||||
for (unsigned i = 0; i <= path[d]; i++) {
|
||||
LLVMTypeRef field = LLVMStructGetTypeAtIndex(type, i);
|
||||
if (!is_packed) {
|
||||
field_offset = llvm_align_formula(field_offset, lb_alignof(field));
|
||||
}
|
||||
if (i == path[d]) {
|
||||
type = field;
|
||||
break;
|
||||
}
|
||||
field_offset += lb_sizeof(field);
|
||||
}
|
||||
offset += field_offset;
|
||||
} else {
|
||||
type = OdinLLVMGetArrayElementType(type);
|
||||
offset += cast(i64)path[d] * lb_sizeof(type);
|
||||
}
|
||||
}
|
||||
if (leaf_type_) *leaf_type_ = type;
|
||||
return offset;
|
||||
}
|
||||
|
||||
gb_internal bool lb_aggregate_leaves(LLVMTypeRef type, unsigned *path, unsigned depth, lbAggregateLeaf *leaves, isize *leaf_count) {
|
||||
LLVMTypeKind kind = LLVMGetTypeKind(type);
|
||||
if (kind == LLVMStructTypeKind || kind == LLVMArrayTypeKind) {
|
||||
if (depth >= LB_SCALARIZE_MAX_DEPTH) {
|
||||
return false;
|
||||
}
|
||||
if (kind == LLVMStructTypeKind && LLVMIsOpaqueStruct(type)) {
|
||||
return false;
|
||||
}
|
||||
unsigned count = kind == LLVMStructTypeKind ? LLVMCountStructElementTypes(type) : cast(unsigned)LLVMGetArrayLength(type);
|
||||
for (unsigned i = 0; i < count; i++) {
|
||||
path[depth] = i;
|
||||
LLVMTypeRef elem = kind == LLVMStructTypeKind ? LLVMStructGetTypeAtIndex(type, i) : OdinLLVMGetArrayElementType(type);
|
||||
if (!lb_aggregate_leaves(elem, path, depth+1, leaves, leaf_count)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (*leaf_count >= LB_SCALARIZE_MAX_LEAVES) {
|
||||
return false;
|
||||
}
|
||||
|
||||
lbAggregateLeaf *leaf = &leaves[(*leaf_count)++];
|
||||
gb_memmove(leaf->path, path, depth*gb_size_of(unsigned));
|
||||
leaf->depth = depth;
|
||||
return true;
|
||||
}
|
||||
|
||||
gb_internal unsigned lb_alignment_at_offset(unsigned alignment, i64 offset) {
|
||||
unsigned a = gb_max(alignment, 1u);
|
||||
while (offset % a != 0) {
|
||||
a >>= 1;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
gb_internal LLVMValueRef lb_aggregate_field_gep(lbModule *m, LLVMBuilderRef b, LLVMTypeRef type, LLVMValueRef ptr, unsigned const *path, unsigned depth) {
|
||||
LLVMValueRef indices[LB_SCALARIZE_MAX_DEPTH+1] = {};
|
||||
LLVMTypeRef i32 = LLVMInt32TypeInContext(m->ctx);
|
||||
indices[0] = LLVMConstInt(i32, 0, false);
|
||||
for (unsigned d = 0; d < depth; d++) {
|
||||
indices[d+1] = LLVMConstInt(i32, path[d], false);
|
||||
}
|
||||
return LLVMBuildInBoundsGEP2(b, type, ptr, indices, depth+1, "");
|
||||
}
|
||||
|
||||
// returns false if the field cannot be reached without an aggregate value, and sets `*field_` to nullptr when it is undefined
|
||||
gb_internal bool lb_aggregate_field_value(lbFastIselLowering *s, LLVMValueRef v, unsigned const *path, unsigned depth, LLVMValueRef *field_) {
|
||||
if (depth == 0) {
|
||||
*field_ = (LLVMIsUndef(v) || LLVMIsPoison(v)) ? nullptr : v;
|
||||
return true;
|
||||
}
|
||||
if (LLVMIsUndef(v) || LLVMIsPoison(v)) {
|
||||
*field_ = nullptr;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (LLVMIsAConstant(v)) {
|
||||
LLVMValueRef elem = LLVMGetAggregateElement(v, path[0]);
|
||||
if (elem != nullptr) {
|
||||
return lb_aggregate_field_value(s, elem, path+1, depth-1, field_);
|
||||
}
|
||||
} else if (LLVMIsAInsertValueInst(v)) {
|
||||
unsigned n = LLVMGetNumIndices(v);
|
||||
unsigned const *indices = LLVMGetIndices(v);
|
||||
|
||||
unsigned k = 0;
|
||||
while (k < n && k < depth && indices[k] == path[k]) {
|
||||
k += 1;
|
||||
}
|
||||
|
||||
if (k == n) {
|
||||
return lb_aggregate_field_value(s, LLVMGetOperand(v, 1), path+n, depth-n, field_);
|
||||
} else if (k < depth) {
|
||||
return lb_aggregate_field_value(s, LLVMGetOperand(v, 0), path, depth, field_);
|
||||
}
|
||||
return false;
|
||||
} else if (LLVMIsAExtractValueInst(v)) {
|
||||
unsigned n = LLVMGetNumIndices(v);
|
||||
if (n + depth <= LB_SCALARIZE_MAX_DEPTH) {
|
||||
unsigned full[LB_SCALARIZE_MAX_DEPTH];
|
||||
gb_memmove(full, LLVMGetIndices(v), n*gb_size_of(unsigned));
|
||||
gb_memmove(full+n, path, depth*gb_size_of(unsigned));
|
||||
return lb_aggregate_field_value(s, LLVMGetOperand(v, 0), full, n+depth, field_);
|
||||
}
|
||||
return false;
|
||||
} else if (LLVMIsASelectInst(v)) {
|
||||
LLVMValueRef x = nullptr;
|
||||
LLVMValueRef y = nullptr;
|
||||
if (!lb_aggregate_field_value(s, LLVMGetOperand(v, 1), path, depth, &x) ||
|
||||
!lb_aggregate_field_value(s, LLVMGetOperand(v, 2), path, depth, &y)) {
|
||||
return false;
|
||||
}
|
||||
if (x == nullptr || y == nullptr) {
|
||||
*field_ = x ? x : y;
|
||||
return true;
|
||||
}
|
||||
LLVMPositionBuilderBefore(s->builder, s->store);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(s->store));
|
||||
*field_ = LLVMBuildSelect(s->builder, LLVMGetOperand(v, 0), x, y, "");
|
||||
return true;
|
||||
} else if (LLVMIsAPHINode(v)) {
|
||||
for (lbScalarizedPhi const &e : s->phis) {
|
||||
if (e.aggregate == v &&
|
||||
e.depth == depth &&
|
||||
gb_memcompare(e.path, path, depth*gb_size_of(unsigned)) == 0) {
|
||||
*field_ = e.field;
|
||||
return e.ok;
|
||||
}
|
||||
}
|
||||
|
||||
LLVMTypeRef field_type = nullptr;
|
||||
lb_aggregate_path_offset(LLVMTypeOf(v), path, depth, &field_type);
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, v);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, nullptr);
|
||||
|
||||
|
||||
// NOTE(bill): cached before its incoming values, as a loop reaches it again
|
||||
isize index = s->phis.count;
|
||||
lbScalarizedPhi entry = {};
|
||||
entry.aggregate = v;
|
||||
gb_memmove(entry.path, path, depth*gb_size_of(unsigned));
|
||||
|
||||
LLVMValueRef phi = LLVMBuildPhi(s->builder, field_type, "");
|
||||
entry.depth = depth;
|
||||
entry.field = phi;
|
||||
entry.ok = true;
|
||||
array_add(&s->phis, entry);
|
||||
|
||||
LLVMValueRef saved_store = s->store;
|
||||
bool ok = true;
|
||||
unsigned incoming_count = LLVMCountIncoming(v);
|
||||
for (unsigned k = 0; k < incoming_count; k++) {
|
||||
LLVMBasicBlockRef block = LLVMGetIncomingBlock(v, k);
|
||||
s->store = LLVMGetBasicBlockTerminator(block);
|
||||
LLVMValueRef field = nullptr;
|
||||
if (!lb_aggregate_field_value(s, LLVMGetIncomingValue(v, k), path, depth, &field)) {
|
||||
ok = false;
|
||||
field = nullptr;
|
||||
}
|
||||
if (field == nullptr) {
|
||||
field = LLVMGetUndef(field_type);
|
||||
}
|
||||
LLVMAddIncoming(phi, &field, &block, 1);
|
||||
}
|
||||
s->store = saved_store;
|
||||
s->phis[index].ok = ok;
|
||||
*field_ = phi;
|
||||
return ok;
|
||||
} else if (LLVMIsALoadInst(v) && !LLVMGetVolatile(v) && LLVMGetOrdering(v) == LLVMAtomicOrderingNotAtomic) {
|
||||
// NOTE(bill): Read the field where the aggregate was read, as the memory may change before the store
|
||||
LLVMTypeRef type = LLVMTypeOf(v);
|
||||
LLVMTypeRef field_type = nullptr;
|
||||
i64 offset = lb_aggregate_path_offset(type, path, depth, &field_type);
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, LLVMGetNextInstruction(v));
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(v));
|
||||
|
||||
LLVMValueRef ptr = lb_aggregate_field_gep(s->m, s->builder, type, LLVMGetOperand(v, 0), path, depth);
|
||||
LLVMValueRef field = LLVMBuildLoad2(s->builder, field_type, ptr, "");
|
||||
LLVMSetAlignment(field, lb_alignment_at_offset(LLVMGetAlignment(v), offset));
|
||||
|
||||
*field_ = field;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (depth == 1) {
|
||||
LLVMPositionBuilderBefore(s->builder, s->store);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(s->store));
|
||||
*field_ = LLVMBuildExtractValue(s->builder, v, path[0], "");
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
gb_internal bool lb_scalarize_aggregate_store(lbFastIselLowering *s, LLVMValueRef store) {
|
||||
LLVMValueRef value = LLVMGetOperand(store, 0);
|
||||
LLVMValueRef ptr = LLVMGetOperand(store, 1);
|
||||
LLVMTypeRef type = LLVMTypeOf(value);
|
||||
|
||||
lbAggregateLeaf leaves[LB_SCALARIZE_MAX_LEAVES] = {};
|
||||
isize leaf_count = 0;
|
||||
unsigned path[LB_SCALARIZE_MAX_DEPTH] = {};
|
||||
|
||||
if (!lb_aggregate_leaves(type, path, 0, leaves, &leaf_count)) {
|
||||
return false;
|
||||
}
|
||||
s->store = store;
|
||||
|
||||
LLVMValueRef fields[LB_SCALARIZE_MAX_LEAVES] = {};
|
||||
for (isize i = 0; i < leaf_count; i++) {
|
||||
if (!lb_aggregate_field_value(s, value, leaves[i].path, leaves[i].depth, &fields[i])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
unsigned alignment = LLVMGetAlignment(store);
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, store);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(store));
|
||||
|
||||
for (isize i = 0; i < leaf_count; i++) {
|
||||
if (fields[i] == nullptr) {
|
||||
continue;
|
||||
}
|
||||
i64 offset = lb_aggregate_path_offset(type, leaves[i].path, leaves[i].depth, nullptr);
|
||||
LLVMValueRef field_ptr = lb_aggregate_field_gep(s->m, s->builder, type, ptr, leaves[i].path, leaves[i].depth);
|
||||
LLVMValueRef field_store = LLVMBuildStore(s->builder, fields[i], field_ptr);
|
||||
LLVMSetAlignment(field_store, lb_alignment_at_offset(alignment, offset));
|
||||
}
|
||||
LLVMInstructionEraseFromParent(store);
|
||||
return true;
|
||||
}
|
||||
|
||||
// NOTE(bill): If a a constant has too many fields to store one at a time is set or copied from a constant instead
|
||||
gb_internal void lb_lower_large_constant_store(lbFastIselLowering *s, LLVMValueRef store) {
|
||||
lbModule *m = s->m;
|
||||
LLVMValueRef value = LLVMGetOperand(store, 0);
|
||||
LLVMValueRef ptr = LLVMGetOperand(store, 1);
|
||||
LLVMTypeRef type = LLVMTypeOf(value);
|
||||
|
||||
LLVMTargetDataRef data_layout = LLVMGetModuleDataLayout(m->mod);
|
||||
unsigned alignment = gb_max(LLVMGetAlignment(store), 1u);
|
||||
if (!LLVMIsAConstant(value) || lb_const_has_misaligned_pointer(data_layout, value, 0, alignment)) {
|
||||
return;
|
||||
}
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, store);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(store));
|
||||
|
||||
LLVMValueRef size = LLVMConstInt(LLVMInt64TypeInContext(m->ctx), LLVMStoreSizeOfType(data_layout, type), false);
|
||||
if (LLVMIsNull(value)) {
|
||||
LLVMBuildMemSet(s->builder, ptr, LLVMConstInt(LLVMInt8TypeInContext(m->ctx), 0, false), size, alignment);
|
||||
LLVMInstructionEraseFromParent(store);
|
||||
return;
|
||||
}
|
||||
|
||||
LLVMValueRef global = LLVMAddGlobal(m->mod, type, "");
|
||||
LLVMSetInitializer(global, value);
|
||||
LLVMSetGlobalConstant(global, true);
|
||||
LLVMSetAlignment(global, alignment);
|
||||
|
||||
LLVMSetLinkage(global, LLVMPrivateLinkage);
|
||||
LLVMSetUnnamedAddress(global, LLVMGlobalUnnamedAddr);
|
||||
|
||||
LLVMBuildMemCpy(s->builder, ptr, alignment, global, alignment, size);
|
||||
|
||||
LLVMInstructionEraseFromParent(store);
|
||||
}
|
||||
|
||||
// NOTE(bill): The fast instruction selector in LLVM cannot select a call to `memmove` nor to the floating point `minnum` and `maxnum`.
|
||||
// To improve things we can them through a function of the module which calls them
|
||||
gb_internal void lb_redirect_unselectable_call(lbFastIselLowering *s, LLVMValueRef call) {
|
||||
lbModule *m = s->m;
|
||||
|
||||
LLVMValueRef callee = LLVMGetCalledValue(call);
|
||||
if (!LLVMIsAFunction(callee)) {
|
||||
return;
|
||||
}
|
||||
size_t name_len = 0;
|
||||
char const *name_text = LLVMGetValueName2(callee, &name_len);
|
||||
String name = make_string(cast(u8 const *)name_text, name_len);
|
||||
|
||||
LLVMTypeRef fn_type = LLVMGlobalGetValueType(callee);
|
||||
if (LLVMGetCalledFunctionType(call) != fn_type || LLVMIsFunctionVarArg(fn_type)) {
|
||||
return;
|
||||
}
|
||||
unsigned param_count = LLVMCountParamTypes(fn_type);
|
||||
LLVMTypeKind return_kind = LLVMGetTypeKind(LLVMGetReturnType(fn_type));
|
||||
bool is_memmove = name == "memmove" && param_count == 3 && return_kind == LLVMPointerTypeKind;
|
||||
bool is_minmax = (string_starts_with(name, str_lit("llvm.minnum.")) ||
|
||||
string_starts_with(name, str_lit("llvm.maxnum."))) &&
|
||||
(return_kind == LLVMFloatTypeKind ||
|
||||
return_kind == LLVMDoubleTypeKind);
|
||||
if (!is_memmove && !is_minmax) {
|
||||
return;
|
||||
}
|
||||
|
||||
gbString wrapper_name = gb_string_make(heap_allocator(), "__$fast_isel$");
|
||||
wrapper_name = gb_string_append_length(wrapper_name, name.text, name.len);
|
||||
defer (gb_string_free(wrapper_name));
|
||||
|
||||
LLVMValueRef wrapper = LLVMGetNamedFunction(m->mod, wrapper_name);
|
||||
defer (LLVMSetOperand(call, cast(unsigned)LLVMGetNumOperands(call) - 1, wrapper));
|
||||
|
||||
if (wrapper != nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
LLVMCallConv cc = cast(LLVMCallConv)LLVMGetFunctionCallConv(callee);
|
||||
wrapper = LLVMAddFunction(m->mod, wrapper_name, fn_type);
|
||||
LLVMSetLinkage(wrapper, LLVMInternalLinkage);
|
||||
LLVMSetFunctionCallConv(wrapper, cc);
|
||||
lb_add_attribute_to_proc(m, wrapper, "nounwind");
|
||||
|
||||
LLVMValueRef params[3] = {};
|
||||
LLVMGetParams(wrapper, params);
|
||||
|
||||
LLVMPositionBuilderAtEnd(s->builder, LLVMAppendBasicBlockInContext(m->ctx, wrapper, ""));
|
||||
LLVMSetCurrentDebugLocation2(s->builder, nullptr);
|
||||
|
||||
LLVMValueRef inner = LLVMBuildCall2(s->builder, fn_type, callee, params, param_count, "");
|
||||
LLVMSetInstructionCallConv(inner, cc);
|
||||
LLVMBuildRet(s->builder, inner);
|
||||
}
|
||||
|
||||
gb_internal bool lb_is_aggregate_type(LLVMTypeRef type) {
|
||||
LLVMTypeKind kind = LLVMGetTypeKind(type);
|
||||
return kind == LLVMStructTypeKind || kind == LLVMArrayTypeKind;
|
||||
}
|
||||
|
||||
gb_internal bool lb_is_plain_access(LLVMValueRef inst) {
|
||||
return !LLVMGetVolatile(inst) && LLVMGetOrdering(inst) == LLVMAtomicOrderingNotAtomic;
|
||||
}
|
||||
|
||||
gb_internal void lb_lower_bool_select(lbFastIselLowering *s, LLVMValueRef select) {
|
||||
LLVMValueRef c = LLVMGetOperand(select, 0);
|
||||
LLVMValueRef x = LLVMGetOperand(select, 1);
|
||||
LLVMValueRef y = LLVMGetOperand(select, 2);
|
||||
if (LLVMGetTypeKind(LLVMTypeOf(c)) != LLVMIntegerTypeKind) {
|
||||
return;
|
||||
}
|
||||
LLVMPositionBuilderBefore(s->builder, select);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(select));
|
||||
LLVMValueRef t = LLVMConstInt(LLVMTypeOf(c), 1, false);
|
||||
|
||||
LLVMValueRef res = nullptr;
|
||||
if (LLVMIsAConstantInt(x)) {
|
||||
res = LLVMConstIntGetZExtValue(x) ? LLVMBuildOr(s->builder, c, y, "") : LLVMBuildAnd(s->builder, LLVMBuildXor(s->builder, c, t, ""), y, "");
|
||||
} else if (LLVMIsAConstantInt(y)) {
|
||||
res = LLVMConstIntGetZExtValue(y) ? LLVMBuildOr(s->builder, LLVMBuildXor(s->builder, c, t, ""), x, "") : LLVMBuildAnd(s->builder, c, x, "");
|
||||
} else {
|
||||
LLVMValueRef a = LLVMBuildAnd(s->builder, c, x, "");
|
||||
LLVMValueRef b = LLVMBuildAnd(s->builder, LLVMBuildXor(s->builder, c, t, ""), y, "");
|
||||
res = LLVMBuildOr(s->builder, a, b, "");
|
||||
}
|
||||
LLVMReplaceAllUsesWith(select, res);
|
||||
LLVMInstructionEraseFromParent(select);
|
||||
}
|
||||
|
||||
gb_internal void lb_truncate_bool_arguments(lbFastIselLowering *s, LLVMValueRef call) {
|
||||
LLVMTypeRef i1 = LLVMInt1TypeInContext(s->m->ctx);
|
||||
LLVMTypeRef i8 = LLVMInt8TypeInContext(s->m->ctx);
|
||||
LLVMBasicBlockRef block = LLVMGetInstructionParent(call);
|
||||
unsigned arg_count = LLVMGetNumArgOperands(call);
|
||||
for (unsigned a = 0; a < arg_count; a++) {
|
||||
LLVMValueRef arg = LLVMGetOperand(call, a);
|
||||
if (LLVMTypeOf(arg) != i1 || LLVMIsAConstantInt(arg)) {
|
||||
continue;
|
||||
}
|
||||
LLVMUseRef first_use = LLVMGetFirstUse(arg);
|
||||
if (LLVMIsATruncInst(arg) && LLVMGetInstructionParent(arg) == block &&
|
||||
first_use != nullptr && LLVMGetNextUse(first_use) == nullptr) {
|
||||
continue;
|
||||
}
|
||||
LLVMPositionBuilderBefore(s->builder, call);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(call));
|
||||
LLVMValueRef byte = LLVMBuildZExt(s->builder, arg, i8, "");
|
||||
LLVMSetOperand(call, a, LLVMBuildTrunc(s->builder, byte, i1, ""));
|
||||
}
|
||||
}
|
||||
|
||||
gb_internal void lb_lower_small_switch(lbFastIselLowering *s, LLVMValueRef sw) {
|
||||
enum {MAX_CASES = 3};
|
||||
LLVMValueRef cond = LLVMGetOperand(sw, 0);
|
||||
unsigned case_count = (cast(unsigned)LLVMGetNumOperands(sw) - 2) / 2;
|
||||
if (case_count == 0 || case_count > MAX_CASES || LLVMGetIntTypeWidth(LLVMTypeOf(cond)) > 64) {
|
||||
return;
|
||||
}
|
||||
|
||||
LLVMBasicBlockRef block = LLVMGetInstructionParent(sw);
|
||||
LLVMBasicBlockRef next_block = LLVMGetNextBasicBlock(block);
|
||||
|
||||
LLVMValueRef fn = LLVMGetBasicBlockParent(block);
|
||||
LLVMMetadataRef loc = LLVMInstructionGetDebugLoc(sw);
|
||||
|
||||
LLVMBasicBlockRef from[MAX_CASES+1] = {};
|
||||
LLVMBasicBlockRef to [MAX_CASES+1] = {};
|
||||
|
||||
LLVMBasicBlockRef curr = block;
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, sw);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, loc);
|
||||
|
||||
for (unsigned j = 0; j < case_count; j++) {
|
||||
LLVMBasicBlockRef dest = LLVMGetSuccessor(sw, j+1);
|
||||
LLVMBasicBlockRef else_block = LLVMGetSwitchDefaultDest(sw);
|
||||
if (j+1 < case_count) {
|
||||
if (next_block != nullptr) {
|
||||
else_block = LLVMInsertBasicBlockInContext(s->m->ctx, next_block, "");
|
||||
} else {
|
||||
else_block = LLVMAppendBasicBlockInContext(s->m->ctx, fn, "");
|
||||
}
|
||||
}
|
||||
|
||||
LLVMValueRef cmp = LLVMBuildICmp(s->builder, LLVMIntEQ, cond, LLVMGetOperand(sw, 2 + 2*j), "");
|
||||
LLVMBuildCondBr(s->builder, cmp, dest, else_block);
|
||||
|
||||
from[j] = curr;
|
||||
to[j] = dest;
|
||||
|
||||
if (j+1 < case_count) {
|
||||
curr = else_block;
|
||||
LLVMPositionBuilderAtEnd(s->builder, curr);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, loc);
|
||||
}
|
||||
}
|
||||
from[case_count] = curr;
|
||||
to [case_count] = LLVMGetSwitchDefaultDest(sw);
|
||||
LLVMInstructionEraseFromParent(sw);
|
||||
|
||||
// the incoming entries for `block` become one for each new edge
|
||||
for (unsigned e = 0; e <= case_count; e++) {
|
||||
bool seen = false;
|
||||
for (unsigned k = 0; k < e; k++) {
|
||||
seen |= to[k] == to[e];
|
||||
}
|
||||
if (seen) {
|
||||
continue;
|
||||
}
|
||||
|
||||
LLVMBasicBlockRef dest = to[e];
|
||||
for (LLVMValueRef phi = LLVMGetFirstInstruction(dest); phi != nullptr && LLVMIsAPHINode(phi); /**/) {
|
||||
LLVMValueRef next = LLVMGetNextInstruction(phi);
|
||||
LLVMValueRef value_from_block = nullptr;
|
||||
|
||||
unsigned incoming_count = LLVMCountIncoming(phi);
|
||||
for (unsigned k = 0; k < incoming_count; k++) {
|
||||
if (LLVMGetIncomingBlock(phi, k) == block) {
|
||||
value_from_block = LLVMGetIncomingValue(phi, k);
|
||||
}
|
||||
}
|
||||
|
||||
if (value_from_block != nullptr) {
|
||||
LLVMPositionBuilderBefore(s->builder, phi);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, nullptr);
|
||||
|
||||
LLVMValueRef new_phi = LLVMBuildPhi(s->builder, LLVMTypeOf(phi), "");
|
||||
for (unsigned k = 0; k < incoming_count; k++) {
|
||||
LLVMBasicBlockRef incoming_block = LLVMGetIncomingBlock(phi, k);
|
||||
if (incoming_block != block) {
|
||||
LLVMValueRef incoming_value = LLVMGetIncomingValue(phi, k);
|
||||
LLVMAddIncoming(new_phi, &incoming_value, &incoming_block, 1);
|
||||
}
|
||||
}
|
||||
for (unsigned k = 0; k <= case_count; k++) {
|
||||
if (to[k] == dest) {
|
||||
LLVMAddIncoming(new_phi, &value_from_block, &from[k], 1);
|
||||
}
|
||||
}
|
||||
LLVMReplaceAllUsesWith(phi, new_phi);
|
||||
LLVMInstructionEraseFromParent(phi);
|
||||
}
|
||||
|
||||
phi = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gb_internal void lb_lower_for_fast_isel(lbModule *m) {
|
||||
lbFastIselLowering s = {};
|
||||
s.m = m;
|
||||
|
||||
s.builder = LLVMCreateBuilderInContext(m->ctx);
|
||||
defer (LLVMDisposeBuilder(s.builder));
|
||||
|
||||
array_init(&s.phis, heap_allocator());
|
||||
defer (array_free(&s.phis));
|
||||
|
||||
auto work = array_make<LLVMValueRef>(heap_allocator(), 0, 64);
|
||||
defer (array_free(&work));
|
||||
|
||||
LLVMValueRef last_fn = LLVMGetLastFunction(m->mod);
|
||||
for (LLVMValueRef fn = LLVMGetFirstFunction(m->mod); fn != nullptr; fn = fn == last_fn ? nullptr : LLVMGetNextFunction(fn)) {
|
||||
array_clear(&work);
|
||||
array_clear(&s.phis);
|
||||
|
||||
for (LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(fn); bb != nullptr; bb = LLVMGetNextBasicBlock(bb)) {
|
||||
for (LLVMValueRef i = LLVMGetFirstInstruction(bb); i != nullptr; i = LLVMGetNextInstruction(i)) {
|
||||
if (LLVMIsAStoreInst(i) && lb_is_plain_access(i) && lb_is_aggregate_type(LLVMTypeOf(LLVMGetOperand(i, 0)))) {
|
||||
array_add(&work, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (LLVMValueRef store : work) {
|
||||
if (!lb_scalarize_aggregate_store(&s, store)) {
|
||||
lb_lower_large_constant_store(&s, store);
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE(bill): A field read out of an aggregate value is read where that aggregate came from
|
||||
array_clear(&work);
|
||||
for (LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(fn); bb != nullptr; bb = LLVMGetNextBasicBlock(bb)) {
|
||||
for (LLVMValueRef i = LLVMGetFirstInstruction(bb); i != nullptr; i = LLVMGetNextInstruction(i)) {
|
||||
if (LLVMIsAExtractValueInst(i) && !lb_is_aggregate_type(LLVMTypeOf(i))) {
|
||||
array_add(&work, i);
|
||||
} else if (LLVMIsASelectInst(i) && LLVMTypeOf(i) == LLVMInt1TypeInContext(m->ctx)) {
|
||||
array_add(&work, i);
|
||||
} else if (LLVMIsACallInst(i)) {
|
||||
array_add(&work, i);
|
||||
} else if (LLVMIsASwitchInst(i)) {
|
||||
array_add(&work, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (LLVMValueRef i : work) {
|
||||
if (LLVMIsASelectInst(i)) {
|
||||
lb_lower_bool_select(&s, i);
|
||||
continue;
|
||||
}
|
||||
if (LLVMIsACallInst(i)) {
|
||||
lb_truncate_bool_arguments(&s, i);
|
||||
lb_redirect_unselectable_call(&s, i);
|
||||
continue;
|
||||
}
|
||||
if (LLVMIsASwitchInst(i)) {
|
||||
lb_lower_small_switch(&s, i);
|
||||
continue;
|
||||
}
|
||||
LLVMValueRef agg = LLVMGetOperand(i, 0);
|
||||
unsigned n = LLVMGetNumIndices(i);
|
||||
if (n > LB_SCALARIZE_MAX_DEPTH) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(LLVMIsAConstant(agg) ||
|
||||
LLVMIsAInsertValueInst(agg) ||
|
||||
LLVMIsASelectInst(agg) ||
|
||||
LLVMIsAPHINode(agg) ||
|
||||
(LLVMIsALoadInst(agg) && lb_is_plain_access(agg)))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
s.store = i;
|
||||
|
||||
LLVMValueRef field = nullptr;
|
||||
if (lb_aggregate_field_value(&s, agg, LLVMGetIndices(i), n, &field) && field != nullptr) {
|
||||
LLVMReplaceAllUsesWith(i, field);
|
||||
LLVMInstructionEraseFromParent(i);
|
||||
}
|
||||
}
|
||||
|
||||
bool removed = true;
|
||||
while (removed) {
|
||||
removed = false;
|
||||
for (LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(fn); bb != nullptr; bb = LLVMGetNextBasicBlock(bb)) {
|
||||
for (LLVMValueRef i = LLVMGetFirstInstruction(bb); i != nullptr; /**/) {
|
||||
LLVMValueRef next = LLVMGetNextInstruction(i);
|
||||
if (LLVMGetFirstUse(i) == nullptr &&
|
||||
(LLVMIsAInsertValueInst(i) || LLVMIsAExtractValueInst(i) || LLVMIsASelectInst(i) || LLVMIsAPHINode(i) ||
|
||||
LLVMIsAGetElementPtrInst(i) || (LLVMIsALoadInst(i) && lb_is_plain_access(i)))) {
|
||||
LLVMInstructionEraseFromParent(i);
|
||||
removed = true;
|
||||
}
|
||||
i = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct lbLLVMEmitWorker {
|
||||
LLVMTargetMachineRef target_machine;
|
||||
LLVMCodeGenFileType code_gen_file_type;
|
||||
|
||||
@@ -630,6 +630,9 @@ gb_internal void lb_mem_copy_non_overlapping(lbProcedure *p, lbValue dst, lbValu
|
||||
gb_internal LLVMValueRef lb_mem_zero_ptr_internal(lbProcedure *p, LLVMValueRef ptr, LLVMValueRef len, unsigned alignment, bool is_volatile);
|
||||
gb_internal LLVMValueRef lb_mem_zero_ptr_internal(lbProcedure *p, LLVMValueRef ptr, usize len, unsigned alignment, bool is_volatile);
|
||||
|
||||
gb_internal bool lb_const_has_misaligned_pointer(LLVMTargetDataRef td, LLVMValueRef c, u64 offset, u64 base_align);
|
||||
gb_internal void lb_add_attribute_to_proc(lbModule *m, LLVMValueRef proc_value, char const *name, u64 value=0);
|
||||
|
||||
gb_internal gb_inline i64 lb_max_zero_init_size(void) {
|
||||
if (build_context.metrics.os == TargetOs_darwin && build_context.metrics.arch == TargetArch_arm64) {
|
||||
// https://github.com/odin-lang/Odin/issues/6347
|
||||
|
||||
@@ -3514,7 +3514,7 @@ gb_internal void lb_add_nocapture_proc_attribute_at_index(lbProcedure *p, isize
|
||||
LLVMAddAttributeAtIndex(p->value, cast(unsigned)index, lb_create_nocapture_attribute(p->module->ctx));
|
||||
}
|
||||
|
||||
gb_internal void lb_add_attribute_to_proc(lbModule *m, LLVMValueRef proc_value, char const *name, u64 value=0) {
|
||||
gb_internal void lb_add_attribute_to_proc(lbModule *m, LLVMValueRef proc_value, char const *name, u64 value) {
|
||||
LLVMAddAttributeAtIndex(proc_value, LLVMAttributeIndex_FunctionIndex, lb_create_enum_attribute(m->ctx, name, value));
|
||||
}
|
||||
|
||||
|
||||
@@ -488,4 +488,623 @@ gb_internal void lb_run_remove_unused_globals_pass(lbModule *m) {
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE(bill, 2026-10-03)
|
||||
//
|
||||
// LLVM's fast instruction selector cannot select a first class aggregate `load`, `store`, `insertvalue`, or `select`,
|
||||
// and hands the rest of the block to SelectionDAG.
|
||||
// SROA leaves many behind so it stores/loads the fields instead
|
||||
enum {
|
||||
LB_SCALARIZE_MAX_LEAVES = 32,
|
||||
LB_SCALARIZE_MAX_DEPTH = 8,
|
||||
};
|
||||
|
||||
struct lbAggregateLeaf {
|
||||
unsigned path[LB_SCALARIZE_MAX_DEPTH];
|
||||
unsigned depth;
|
||||
};
|
||||
|
||||
struct lbScalarizedPhi {
|
||||
LLVMValueRef aggregate;
|
||||
unsigned path[LB_SCALARIZE_MAX_DEPTH];
|
||||
unsigned depth;
|
||||
LLVMValueRef field;
|
||||
bool ok;
|
||||
};
|
||||
|
||||
struct lbFastIselLowering {
|
||||
lbModule * m;
|
||||
LLVMBuilderRef builder;
|
||||
LLVMValueRef store;
|
||||
Array<lbScalarizedPhi> phis;
|
||||
};
|
||||
|
||||
gb_internal i64 lb_aggregate_path_offset(LLVMTypeRef type, unsigned const *path, unsigned depth, LLVMTypeRef *leaf_type_) {
|
||||
i64 offset = 0;
|
||||
for (unsigned d = 0; d < depth; d++) {
|
||||
if (LLVMGetTypeKind(type) == LLVMStructTypeKind) {
|
||||
bool is_packed = LLVMIsPackedStruct(type);
|
||||
i64 field_offset = 0;
|
||||
for (unsigned i = 0; i <= path[d]; i++) {
|
||||
LLVMTypeRef field = LLVMStructGetTypeAtIndex(type, i);
|
||||
if (!is_packed) {
|
||||
field_offset = llvm_align_formula(field_offset, lb_alignof(field));
|
||||
}
|
||||
if (i == path[d]) {
|
||||
type = field;
|
||||
break;
|
||||
}
|
||||
field_offset += lb_sizeof(field);
|
||||
}
|
||||
offset += field_offset;
|
||||
} else {
|
||||
type = OdinLLVMGetArrayElementType(type);
|
||||
offset += cast(i64)path[d] * lb_sizeof(type);
|
||||
}
|
||||
}
|
||||
if (leaf_type_) *leaf_type_ = type;
|
||||
return offset;
|
||||
}
|
||||
|
||||
gb_internal bool lb_aggregate_leaves(LLVMTypeRef type, unsigned *path, unsigned depth, lbAggregateLeaf *leaves, isize *leaf_count) {
|
||||
LLVMTypeKind kind = LLVMGetTypeKind(type);
|
||||
if (kind == LLVMStructTypeKind || kind == LLVMArrayTypeKind) {
|
||||
if (depth >= LB_SCALARIZE_MAX_DEPTH) {
|
||||
return false;
|
||||
}
|
||||
if (kind == LLVMStructTypeKind && LLVMIsOpaqueStruct(type)) {
|
||||
return false;
|
||||
}
|
||||
unsigned count = kind == LLVMStructTypeKind ? LLVMCountStructElementTypes(type) : cast(unsigned)LLVMGetArrayLength(type);
|
||||
for (unsigned i = 0; i < count; i++) {
|
||||
path[depth] = i;
|
||||
LLVMTypeRef elem = kind == LLVMStructTypeKind ? LLVMStructGetTypeAtIndex(type, i) : OdinLLVMGetArrayElementType(type);
|
||||
if (!lb_aggregate_leaves(elem, path, depth+1, leaves, leaf_count)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (*leaf_count >= LB_SCALARIZE_MAX_LEAVES) {
|
||||
return false;
|
||||
}
|
||||
|
||||
lbAggregateLeaf *leaf = &leaves[(*leaf_count)++];
|
||||
gb_memmove(leaf->path, path, depth*gb_size_of(unsigned));
|
||||
leaf->depth = depth;
|
||||
return true;
|
||||
}
|
||||
|
||||
gb_internal unsigned lb_aggregate_field_alignment(unsigned alignment, i64 offset) {
|
||||
unsigned a = gb_max(alignment, 1u);
|
||||
while (offset % a != 0) {
|
||||
a >>= 1;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
gb_internal LLVMValueRef lb_aggregate_field_gep(lbModule *m, LLVMBuilderRef b, LLVMTypeRef type, LLVMValueRef ptr, unsigned const *path, unsigned depth) {
|
||||
LLVMValueRef indices[LB_SCALARIZE_MAX_DEPTH+1] = {};
|
||||
LLVMTypeRef i32 = LLVMInt32TypeInContext(m->ctx);
|
||||
indices[0] = LLVMConstInt(i32, 0, false);
|
||||
for (unsigned d = 0; d < depth; d++) {
|
||||
indices[d+1] = LLVMConstInt(i32, path[d], false);
|
||||
}
|
||||
return LLVMBuildInBoundsGEP2(b, type, ptr, indices, depth+1, "");
|
||||
}
|
||||
|
||||
// returns false if the field cannot be reached without an aggregate value, and sets `*field_` to nullptr when it is undefined
|
||||
gb_internal bool lb_aggregate_field_value(lbFastIselLowering *s, LLVMValueRef v, unsigned const *path, unsigned depth, LLVMValueRef *field_) {
|
||||
if (depth == 0) {
|
||||
*field_ = (LLVMIsUndef(v) || LLVMIsPoison(v)) ? nullptr : v;
|
||||
return true;
|
||||
}
|
||||
if (LLVMIsUndef(v) || LLVMIsPoison(v)) {
|
||||
*field_ = nullptr;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (LLVMIsAConstant(v)) {
|
||||
LLVMValueRef elem = LLVMGetAggregateElement(v, path[0]);
|
||||
if (elem != nullptr) {
|
||||
return lb_aggregate_field_value(s, elem, path+1, depth-1, field_);
|
||||
}
|
||||
} else if (LLVMIsAInsertValueInst(v)) {
|
||||
unsigned n = LLVMGetNumIndices(v);
|
||||
unsigned const *indices = LLVMGetIndices(v);
|
||||
|
||||
unsigned k = 0;
|
||||
while (k < n && k < depth && indices[k] == path[k]) {
|
||||
k += 1;
|
||||
}
|
||||
|
||||
if (k == n) {
|
||||
return lb_aggregate_field_value(s, LLVMGetOperand(v, 1), path+n, depth-n, field_);
|
||||
} else if (k < depth) {
|
||||
return lb_aggregate_field_value(s, LLVMGetOperand(v, 0), path, depth, field_);
|
||||
}
|
||||
return false;
|
||||
} else if (LLVMIsAExtractValueInst(v)) {
|
||||
unsigned n = LLVMGetNumIndices(v);
|
||||
if (n + depth <= LB_SCALARIZE_MAX_DEPTH) {
|
||||
unsigned full[LB_SCALARIZE_MAX_DEPTH];
|
||||
gb_memmove(full, LLVMGetIndices(v), n*gb_size_of(unsigned));
|
||||
gb_memmove(full+n, path, depth*gb_size_of(unsigned));
|
||||
return lb_aggregate_field_value(s, LLVMGetOperand(v, 0), full, n+depth, field_);
|
||||
}
|
||||
return false;
|
||||
} else if (LLVMIsASelectInst(v)) {
|
||||
LLVMValueRef x = nullptr;
|
||||
LLVMValueRef y = nullptr;
|
||||
if (!lb_aggregate_field_value(s, LLVMGetOperand(v, 1), path, depth, &x) ||
|
||||
!lb_aggregate_field_value(s, LLVMGetOperand(v, 2), path, depth, &y)) {
|
||||
return false;
|
||||
}
|
||||
if (x == nullptr || y == nullptr) {
|
||||
*field_ = x ? x : y;
|
||||
return true;
|
||||
}
|
||||
LLVMPositionBuilderBefore(s->builder, s->store);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(s->store));
|
||||
*field_ = LLVMBuildSelect(s->builder, LLVMGetOperand(v, 0), x, y, "");
|
||||
return true;
|
||||
} else if (LLVMIsAPHINode(v)) {
|
||||
for (lbScalarizedPhi const &e : s->phis) {
|
||||
if (e.aggregate == v &&
|
||||
e.depth == depth &&
|
||||
gb_memcompare(e.path, path, depth*gb_size_of(unsigned)) == 0) {
|
||||
*field_ = e.field;
|
||||
return e.ok;
|
||||
}
|
||||
}
|
||||
|
||||
LLVMTypeRef field_type = nullptr;
|
||||
lb_aggregate_path_offset(LLVMTypeOf(v), path, depth, &field_type);
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, v);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, nullptr);
|
||||
|
||||
|
||||
// NOTE(bill): cached before its incoming values, as a loop reaches it again
|
||||
isize index = s->phis.count;
|
||||
lbScalarizedPhi entry = {};
|
||||
entry.aggregate = v;
|
||||
gb_memmove(entry.path, path, depth*gb_size_of(unsigned));
|
||||
|
||||
LLVMValueRef phi = LLVMBuildPhi(s->builder, field_type, "");
|
||||
entry.depth = depth;
|
||||
entry.field = phi;
|
||||
entry.ok = true;
|
||||
array_add(&s->phis, entry);
|
||||
|
||||
LLVMValueRef saved_store = s->store;
|
||||
bool ok = true;
|
||||
unsigned incoming_count = LLVMCountIncoming(v);
|
||||
for (unsigned k = 0; k < incoming_count; k++) {
|
||||
LLVMBasicBlockRef block = LLVMGetIncomingBlock(v, k);
|
||||
s->store = LLVMGetBasicBlockTerminator(block);
|
||||
LLVMValueRef field = nullptr;
|
||||
if (!lb_aggregate_field_value(s, LLVMGetIncomingValue(v, k), path, depth, &field)) {
|
||||
ok = false;
|
||||
field = nullptr;
|
||||
}
|
||||
if (field == nullptr) {
|
||||
field = LLVMGetUndef(field_type);
|
||||
}
|
||||
LLVMAddIncoming(phi, &field, &block, 1);
|
||||
}
|
||||
s->store = saved_store;
|
||||
s->phis[index].ok = ok;
|
||||
*field_ = phi;
|
||||
return ok;
|
||||
} else if (LLVMIsALoadInst(v) && !LLVMGetVolatile(v) && LLVMGetOrdering(v) == LLVMAtomicOrderingNotAtomic) {
|
||||
// NOTE(bill): Read the field where the aggregate was read, as the memory may change before the store
|
||||
LLVMTypeRef type = LLVMTypeOf(v);
|
||||
LLVMTypeRef field_type = nullptr;
|
||||
i64 offset = lb_aggregate_path_offset(type, path, depth, &field_type);
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, LLVMGetNextInstruction(v));
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(v));
|
||||
|
||||
LLVMValueRef ptr = lb_aggregate_field_gep(s->m, s->builder, type, LLVMGetOperand(v, 0), path, depth);
|
||||
LLVMValueRef field = LLVMBuildLoad2(s->builder, field_type, ptr, "");
|
||||
LLVMSetAlignment(field, lb_aggregate_field_alignment(LLVMGetAlignment(v), offset));
|
||||
|
||||
*field_ = field;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (depth == 1) {
|
||||
LLVMPositionBuilderBefore(s->builder, s->store);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(s->store));
|
||||
*field_ = LLVMBuildExtractValue(s->builder, v, path[0], "");
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
gb_internal bool lb_scalarize_aggregate_store(lbFastIselLowering *s, LLVMValueRef store) {
|
||||
LLVMValueRef value = LLVMGetOperand(store, 0);
|
||||
LLVMValueRef ptr = LLVMGetOperand(store, 1);
|
||||
LLVMTypeRef type = LLVMTypeOf(value);
|
||||
|
||||
lbAggregateLeaf leaves[LB_SCALARIZE_MAX_LEAVES] = {};
|
||||
isize leaf_count = 0;
|
||||
unsigned path[LB_SCALARIZE_MAX_DEPTH] = {};
|
||||
|
||||
if (!lb_aggregate_leaves(type, path, 0, leaves, &leaf_count)) {
|
||||
return false;
|
||||
}
|
||||
s->store = store;
|
||||
|
||||
LLVMValueRef fields[LB_SCALARIZE_MAX_LEAVES] = {};
|
||||
for (isize i = 0; i < leaf_count; i++) {
|
||||
if (!lb_aggregate_field_value(s, value, leaves[i].path, leaves[i].depth, &fields[i])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
unsigned alignment = LLVMGetAlignment(store);
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, store);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(store));
|
||||
|
||||
for (isize i = 0; i < leaf_count; i++) {
|
||||
if (fields[i] == nullptr) {
|
||||
continue;
|
||||
}
|
||||
i64 offset = lb_aggregate_path_offset(type, leaves[i].path, leaves[i].depth, nullptr);
|
||||
LLVMValueRef field_ptr = lb_aggregate_field_gep(s->m, s->builder, type, ptr, leaves[i].path, leaves[i].depth);
|
||||
LLVMValueRef field_store = LLVMBuildStore(s->builder, fields[i], field_ptr);
|
||||
LLVMSetAlignment(field_store, lb_aggregate_field_alignment(alignment, offset));
|
||||
}
|
||||
LLVMInstructionEraseFromParent(store);
|
||||
return true;
|
||||
}
|
||||
|
||||
// NOTE(bill): If a a constant has too many fields to store one at a time is set or copied from a constant instead
|
||||
gb_internal void lb_lower_large_constant_store(lbFastIselLowering *s, LLVMValueRef store) {
|
||||
lbModule *m = s->m;
|
||||
LLVMValueRef value = LLVMGetOperand(store, 0);
|
||||
LLVMValueRef ptr = LLVMGetOperand(store, 1);
|
||||
LLVMTypeRef type = LLVMTypeOf(value);
|
||||
|
||||
LLVMTargetDataRef data_layout = LLVMGetModuleDataLayout(m->mod);
|
||||
unsigned alignment = gb_max(LLVMGetAlignment(store), 1u);
|
||||
if (!LLVMIsAConstant(value) || lb_const_has_misaligned_pointer(data_layout, value, 0, alignment)) {
|
||||
return;
|
||||
}
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, store);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(store));
|
||||
|
||||
LLVMValueRef size = LLVMConstInt(LLVMInt64TypeInContext(m->ctx), LLVMStoreSizeOfType(data_layout, type), false);
|
||||
if (LLVMIsNull(value)) {
|
||||
LLVMBuildMemSet(s->builder, ptr, LLVMConstInt(LLVMInt8TypeInContext(m->ctx), 0, false), size, alignment);
|
||||
LLVMInstructionEraseFromParent(store);
|
||||
return;
|
||||
}
|
||||
|
||||
LLVMValueRef global = LLVMAddGlobal(m->mod, type, "");
|
||||
LLVMSetInitializer(global, value);
|
||||
LLVMSetGlobalConstant(global, true);
|
||||
LLVMSetAlignment(global, alignment);
|
||||
|
||||
LLVMSetLinkage(global, LLVMPrivateLinkage);
|
||||
LLVMSetUnnamedAddress(global, LLVMGlobalUnnamedAddr);
|
||||
|
||||
LLVMBuildMemCpy(s->builder, ptr, alignment, global, alignment, size);
|
||||
|
||||
LLVMInstructionEraseFromParent(store);
|
||||
}
|
||||
|
||||
// NOTE(bill): The fast instruction selector in LLVM cannot select a call to `memmove` nor to the floating point `minnum` and `maxnum`.
|
||||
// To improve things we can them through a function of the module which calls them
|
||||
gb_internal void lb_redirect_unselectable_call(lbFastIselLowering *s, LLVMValueRef call) {
|
||||
lbModule *m = s->m;
|
||||
|
||||
LLVMValueRef callee = LLVMGetCalledValue(call);
|
||||
if (!LLVMIsAFunction(callee)) {
|
||||
return;
|
||||
}
|
||||
size_t name_len = 0;
|
||||
char const *name_text = LLVMGetValueName2(callee, &name_len);
|
||||
String name = make_string(cast(u8 const *)name_text, name_len);
|
||||
|
||||
LLVMTypeRef fn_type = LLVMGlobalGetValueType(callee);
|
||||
if (LLVMGetCalledFunctionType(call) != fn_type || LLVMIsFunctionVarArg(fn_type)) {
|
||||
return;
|
||||
}
|
||||
unsigned param_count = LLVMCountParamTypes(fn_type);
|
||||
LLVMTypeKind return_kind = LLVMGetTypeKind(LLVMGetReturnType(fn_type));
|
||||
bool is_memmove = name == "memmove" && param_count == 3 && return_kind == LLVMPointerTypeKind;
|
||||
bool is_minmax = (string_starts_with(name, str_lit("llvm.minnum.")) ||
|
||||
string_starts_with(name, str_lit("llvm.maxnum."))) &&
|
||||
(return_kind == LLVMFloatTypeKind ||
|
||||
return_kind == LLVMDoubleTypeKind);
|
||||
if (!is_memmove && !is_minmax) {
|
||||
return;
|
||||
}
|
||||
|
||||
gbString wrapper_name = gb_string_make(heap_allocator(), "__$fast_isel$");
|
||||
wrapper_name = gb_string_append_length(wrapper_name, name.text, name.len);
|
||||
defer (gb_string_free(wrapper_name));
|
||||
|
||||
LLVMValueRef wrapper = LLVMGetNamedFunction(m->mod, wrapper_name);
|
||||
defer (LLVMSetOperand(call, cast(unsigned)LLVMGetNumOperands(call) - 1, wrapper));
|
||||
|
||||
if (wrapper != nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
LLVMCallConv cc = cast(LLVMCallConv)LLVMGetFunctionCallConv(callee);
|
||||
wrapper = LLVMAddFunction(m->mod, wrapper_name, fn_type);
|
||||
LLVMSetLinkage(wrapper, LLVMInternalLinkage);
|
||||
LLVMSetFunctionCallConv(wrapper, cc);
|
||||
lb_add_attribute_to_proc(m, wrapper, "nounwind");
|
||||
|
||||
LLVMValueRef params[3] = {};
|
||||
LLVMGetParams(wrapper, params);
|
||||
|
||||
LLVMPositionBuilderAtEnd(s->builder, LLVMAppendBasicBlockInContext(m->ctx, wrapper, ""));
|
||||
LLVMSetCurrentDebugLocation2(s->builder, nullptr);
|
||||
|
||||
LLVMValueRef inner = LLVMBuildCall2(s->builder, fn_type, callee, params, param_count, "");
|
||||
LLVMSetInstructionCallConv(inner, cc);
|
||||
LLVMBuildRet(s->builder, inner);
|
||||
}
|
||||
|
||||
gb_internal bool lb_is_aggregate_type(LLVMTypeRef type) {
|
||||
LLVMTypeKind kind = LLVMGetTypeKind(type);
|
||||
return kind == LLVMStructTypeKind || kind == LLVMArrayTypeKind;
|
||||
}
|
||||
|
||||
gb_internal bool lb_is_plain_access(LLVMValueRef inst) {
|
||||
return !LLVMGetVolatile(inst) && LLVMGetOrdering(inst) == LLVMAtomicOrderingNotAtomic;
|
||||
}
|
||||
|
||||
gb_internal void lb_lower_bool_select(lbFastIselLowering *s, LLVMValueRef select) {
|
||||
LLVMValueRef c = LLVMGetOperand(select, 0);
|
||||
LLVMValueRef x = LLVMGetOperand(select, 1);
|
||||
LLVMValueRef y = LLVMGetOperand(select, 2);
|
||||
if (LLVMGetTypeKind(LLVMTypeOf(c)) != LLVMIntegerTypeKind) {
|
||||
return;
|
||||
}
|
||||
LLVMPositionBuilderBefore(s->builder, select);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(select));
|
||||
LLVMValueRef t = LLVMConstInt(LLVMTypeOf(c), 1, false);
|
||||
|
||||
LLVMValueRef res = nullptr;
|
||||
if (LLVMIsAConstantInt(x)) {
|
||||
res = LLVMConstIntGetZExtValue(x) ? LLVMBuildOr(s->builder, c, y, "") : LLVMBuildAnd(s->builder, LLVMBuildXor(s->builder, c, t, ""), y, "");
|
||||
} else if (LLVMIsAConstantInt(y)) {
|
||||
res = LLVMConstIntGetZExtValue(y) ? LLVMBuildOr(s->builder, LLVMBuildXor(s->builder, c, t, ""), x, "") : LLVMBuildAnd(s->builder, c, x, "");
|
||||
} else {
|
||||
LLVMValueRef a = LLVMBuildAnd(s->builder, c, x, "");
|
||||
LLVMValueRef b = LLVMBuildAnd(s->builder, LLVMBuildXor(s->builder, c, t, ""), y, "");
|
||||
res = LLVMBuildOr(s->builder, a, b, "");
|
||||
}
|
||||
LLVMReplaceAllUsesWith(select, res);
|
||||
LLVMInstructionEraseFromParent(select);
|
||||
}
|
||||
|
||||
gb_internal void lb_truncate_bool_arguments(lbFastIselLowering *s, LLVMValueRef call) {
|
||||
LLVMTypeRef i1 = LLVMInt1TypeInContext(s->m->ctx);
|
||||
LLVMTypeRef i8 = LLVMInt8TypeInContext(s->m->ctx);
|
||||
LLVMBasicBlockRef block = LLVMGetInstructionParent(call);
|
||||
unsigned arg_count = LLVMGetNumArgOperands(call);
|
||||
for (unsigned a = 0; a < arg_count; a++) {
|
||||
LLVMValueRef arg = LLVMGetOperand(call, a);
|
||||
if (LLVMTypeOf(arg) != i1 || LLVMIsAConstantInt(arg)) {
|
||||
continue;
|
||||
}
|
||||
LLVMUseRef first_use = LLVMGetFirstUse(arg);
|
||||
if (LLVMIsATruncInst(arg) && LLVMGetInstructionParent(arg) == block &&
|
||||
first_use != nullptr && LLVMGetNextUse(first_use) == nullptr) {
|
||||
continue;
|
||||
}
|
||||
LLVMPositionBuilderBefore(s->builder, call);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, LLVMInstructionGetDebugLoc(call));
|
||||
LLVMValueRef byte = LLVMBuildZExt(s->builder, arg, i8, "");
|
||||
LLVMSetOperand(call, a, LLVMBuildTrunc(s->builder, byte, i1, ""));
|
||||
}
|
||||
}
|
||||
|
||||
gb_internal void lb_lower_small_switch(lbFastIselLowering *s, LLVMValueRef sw) {
|
||||
enum {MAX_CASES = 3};
|
||||
LLVMValueRef cond = LLVMGetOperand(sw, 0);
|
||||
unsigned case_count = (cast(unsigned)LLVMGetNumOperands(sw) - 2) / 2;
|
||||
if (case_count == 0 || case_count > MAX_CASES || LLVMGetIntTypeWidth(LLVMTypeOf(cond)) > 64) {
|
||||
return;
|
||||
}
|
||||
|
||||
LLVMBasicBlockRef block = LLVMGetInstructionParent(sw);
|
||||
LLVMBasicBlockRef next_block = LLVMGetNextBasicBlock(block);
|
||||
|
||||
LLVMValueRef fn = LLVMGetBasicBlockParent(block);
|
||||
LLVMMetadataRef loc = LLVMInstructionGetDebugLoc(sw);
|
||||
|
||||
LLVMBasicBlockRef from[MAX_CASES+1] = {};
|
||||
LLVMBasicBlockRef to [MAX_CASES+1] = {};
|
||||
|
||||
LLVMBasicBlockRef curr = block;
|
||||
|
||||
LLVMPositionBuilderBefore(s->builder, sw);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, loc);
|
||||
|
||||
for (unsigned j = 0; j < case_count; j++) {
|
||||
LLVMBasicBlockRef dest = LLVMGetSuccessor(sw, j+1);
|
||||
LLVMBasicBlockRef else_block = LLVMGetSwitchDefaultDest(sw);
|
||||
if (j+1 < case_count) {
|
||||
if (next_block != nullptr) {
|
||||
else_block = LLVMInsertBasicBlockInContext(s->m->ctx, next_block, "");
|
||||
} else {
|
||||
else_block = LLVMAppendBasicBlockInContext(s->m->ctx, fn, "");
|
||||
}
|
||||
}
|
||||
|
||||
LLVMValueRef cmp = LLVMBuildICmp(s->builder, LLVMIntEQ, cond, LLVMGetOperand(sw, 2 + 2*j), "");
|
||||
LLVMBuildCondBr(s->builder, cmp, dest, else_block);
|
||||
|
||||
from[j] = curr;
|
||||
to[j] = dest;
|
||||
|
||||
if (j+1 < case_count) {
|
||||
curr = else_block;
|
||||
LLVMPositionBuilderAtEnd(s->builder, curr);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, loc);
|
||||
}
|
||||
}
|
||||
from[case_count] = curr;
|
||||
to [case_count] = LLVMGetSwitchDefaultDest(sw);
|
||||
LLVMInstructionEraseFromParent(sw);
|
||||
|
||||
// the incoming entries for `block` become one for each new edge
|
||||
for (unsigned e = 0; e <= case_count; e++) {
|
||||
bool seen = false;
|
||||
for (unsigned k = 0; k < e; k++) {
|
||||
seen |= to[k] == to[e];
|
||||
}
|
||||
if (seen) {
|
||||
continue;
|
||||
}
|
||||
|
||||
LLVMBasicBlockRef dest = to[e];
|
||||
for (LLVMValueRef phi = LLVMGetFirstInstruction(dest); phi != nullptr && LLVMIsAPHINode(phi); /**/) {
|
||||
LLVMValueRef next = LLVMGetNextInstruction(phi);
|
||||
LLVMValueRef value_from_block = nullptr;
|
||||
|
||||
unsigned incoming_count = LLVMCountIncoming(phi);
|
||||
for (unsigned k = 0; k < incoming_count; k++) {
|
||||
if (LLVMGetIncomingBlock(phi, k) == block) {
|
||||
value_from_block = LLVMGetIncomingValue(phi, k);
|
||||
}
|
||||
}
|
||||
|
||||
if (value_from_block != nullptr) {
|
||||
LLVMPositionBuilderBefore(s->builder, phi);
|
||||
LLVMSetCurrentDebugLocation2(s->builder, nullptr);
|
||||
|
||||
LLVMValueRef new_phi = LLVMBuildPhi(s->builder, LLVMTypeOf(phi), "");
|
||||
for (unsigned k = 0; k < incoming_count; k++) {
|
||||
LLVMBasicBlockRef incoming_block = LLVMGetIncomingBlock(phi, k);
|
||||
if (incoming_block != block) {
|
||||
LLVMValueRef incoming_value = LLVMGetIncomingValue(phi, k);
|
||||
LLVMAddIncoming(new_phi, &incoming_value, &incoming_block, 1);
|
||||
}
|
||||
}
|
||||
for (unsigned k = 0; k <= case_count; k++) {
|
||||
if (to[k] == dest) {
|
||||
LLVMAddIncoming(new_phi, &value_from_block, &from[k], 1);
|
||||
}
|
||||
}
|
||||
LLVMReplaceAllUsesWith(phi, new_phi);
|
||||
LLVMInstructionEraseFromParent(phi);
|
||||
}
|
||||
|
||||
phi = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gb_internal void lb_lower_for_fast_isel(lbModule *m) {
|
||||
lbFastIselLowering s = {};
|
||||
s.m = m;
|
||||
|
||||
s.builder = LLVMCreateBuilderInContext(m->ctx);
|
||||
defer (LLVMDisposeBuilder(s.builder));
|
||||
|
||||
array_init(&s.phis, heap_allocator());
|
||||
defer (array_free(&s.phis));
|
||||
|
||||
auto work = array_make<LLVMValueRef>(heap_allocator(), 0, 64);
|
||||
defer (array_free(&work));
|
||||
|
||||
LLVMValueRef last_fn = LLVMGetLastFunction(m->mod);
|
||||
for (LLVMValueRef fn = LLVMGetFirstFunction(m->mod); fn != nullptr; fn = fn == last_fn ? nullptr : LLVMGetNextFunction(fn)) {
|
||||
array_clear(&work);
|
||||
array_clear(&s.phis);
|
||||
|
||||
for (LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(fn); bb != nullptr; bb = LLVMGetNextBasicBlock(bb)) {
|
||||
for (LLVMValueRef i = LLVMGetFirstInstruction(bb); i != nullptr; i = LLVMGetNextInstruction(i)) {
|
||||
if (LLVMIsAStoreInst(i) && lb_is_plain_access(i) && lb_is_aggregate_type(LLVMTypeOf(LLVMGetOperand(i, 0)))) {
|
||||
array_add(&work, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (LLVMValueRef store : work) {
|
||||
if (!lb_scalarize_aggregate_store(&s, store)) {
|
||||
lb_lower_large_constant_store(&s, store);
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE(bill): A field read out of an aggregate value is read where that aggregate came from
|
||||
array_clear(&work);
|
||||
for (LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(fn); bb != nullptr; bb = LLVMGetNextBasicBlock(bb)) {
|
||||
for (LLVMValueRef i = LLVMGetFirstInstruction(bb); i != nullptr; i = LLVMGetNextInstruction(i)) {
|
||||
if (LLVMIsAExtractValueInst(i) && !lb_is_aggregate_type(LLVMTypeOf(i))) {
|
||||
array_add(&work, i);
|
||||
} else if (LLVMIsASelectInst(i) && LLVMTypeOf(i) == LLVMInt1TypeInContext(m->ctx)) {
|
||||
array_add(&work, i);
|
||||
} else if (LLVMIsACallInst(i)) {
|
||||
array_add(&work, i);
|
||||
} else if (LLVMIsASwitchInst(i)) {
|
||||
array_add(&work, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (LLVMValueRef i : work) {
|
||||
if (LLVMIsASelectInst(i)) {
|
||||
lb_lower_bool_select(&s, i);
|
||||
continue;
|
||||
}
|
||||
if (LLVMIsACallInst(i)) {
|
||||
lb_truncate_bool_arguments(&s, i);
|
||||
lb_redirect_unselectable_call(&s, i);
|
||||
continue;
|
||||
}
|
||||
if (LLVMIsASwitchInst(i)) {
|
||||
lb_lower_small_switch(&s, i);
|
||||
continue;
|
||||
}
|
||||
LLVMValueRef agg = LLVMGetOperand(i, 0);
|
||||
unsigned n = LLVMGetNumIndices(i);
|
||||
if (n > LB_SCALARIZE_MAX_DEPTH) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(LLVMIsAConstant(agg) ||
|
||||
LLVMIsAInsertValueInst(agg) ||
|
||||
LLVMIsASelectInst(agg) ||
|
||||
LLVMIsAPHINode(agg) ||
|
||||
(LLVMIsALoadInst(agg) && lb_is_plain_access(agg)))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
s.store = i;
|
||||
|
||||
LLVMValueRef field = nullptr;
|
||||
if (lb_aggregate_field_value(&s, agg, LLVMGetIndices(i), n, &field) && field != nullptr) {
|
||||
LLVMReplaceAllUsesWith(i, field);
|
||||
LLVMInstructionEraseFromParent(i);
|
||||
}
|
||||
}
|
||||
|
||||
bool removed = true;
|
||||
while (removed) {
|
||||
removed = false;
|
||||
for (LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(fn); bb != nullptr; bb = LLVMGetNextBasicBlock(bb)) {
|
||||
for (LLVMValueRef i = LLVMGetFirstInstruction(bb); i != nullptr; /**/) {
|
||||
LLVMValueRef next = LLVMGetNextInstruction(i);
|
||||
if (LLVMGetFirstUse(i) == nullptr &&
|
||||
(LLVMIsAInsertValueInst(i) || LLVMIsAExtractValueInst(i) || LLVMIsASelectInst(i) || LLVMIsAPHINode(i) ||
|
||||
LLVMIsAGetElementPtrInst(i) || (LLVMIsALoadInst(i) && lb_is_plain_access(i)))) {
|
||||
LLVMInstructionEraseFromParent(i);
|
||||
removed = true;
|
||||
}
|
||||
i = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user