asm {
//************************************
//See /Doc/Credits.DD.
_MALLOC::
// Throws 'OutMem'
PUSH RBP
MOV RBP,RSP
PUSH RSI
PUSH RDI
XOR RBX,RBX
MOV RDX,U64 SF_ARG2[RBP]
TEST RDX,RDX
JNZ @@05
MOV RDX,U64 FS:CTask.addr[RBX]
@@05: CMP U32 CTask.task_signature[RDX],TASK_SIGNATURE_VAL
#assert CTask.task_signature==CHeapCtrl.hc_signature //location signature same
JNE @@10
MOV RDX,U64 CTask.data_heap[RDX]
@@10: CMP U32 CHeapCtrl.hc_signature[RDX],HEAP_CTRL_SIGNATURE_VAL
JE @@15
PUSH RDX
CALL &SysBadMAlloc
JMP I32 _SYS_HLT
@@15: MOV RAX,U64 SF_ARG1[RBP]
PUSHFD
ADD RAX,CMemUsed.start+7 //round-up to I64
AND AL,0xF8
#assert CMemUsed.start>=sizeof(CMemUnused)
CMP RAX,CMemUsed.start
JAE @@20
MOV RAX,CMemUsed.start
@@20:
CLI
@@25: LOCK
BTS U32 CHeapCtrl.locked_flags[RDX],HClf_LOCKED
PAUSE //don't know if this inst helps
JC @@25
CMP RAX,MEM_HEAP_HASH_SIZE
JAE @@30
MOV RSI,U64 CHeapCtrl.heap_hash[RAX+RDX]
TEST RSI,RSI
JZ @@35
MOV RCX,U64 CMemUnused.next[RSI]
MOV U64 CHeapCtrl.heap_hash[RAX+RDX],RCX
JMP I32 MALLOC_ALMOST_DONE
//Big allocation
@@30: ADD RAX,sizeof(CMemBlk)+MEM_PAG_SIZE-1
SHR RAX,MEM_PAG_BITS
PUSH RDX //preserve HeapCtrl
PUSH RDX
PUSH RAX
CALL &MemPagTaskAlloc
POP RDX
TEST RAX,RAX
JZ @@45 //Out of memory
MOV RSI,RAX
MOV EAX,U32 CMemBlk.pags[RSI]
SHL RAX,MEM_PAG_BITS
SUB RAX,sizeof(CMemBlk)
ADD RSI,sizeof(CMemBlk)
JMP I32 MALLOC_ALMOST_DONE
//Little allocation, chunk-off piece from free lst chunks
@@35: LEA RSI,U64 CHeapCtrl.malloc_free_lst-CMemUnused.next[RDX]
@@40: MOV RBX,RSI
MOV RSI,U64 CMemUnused.next[RBX]
TEST RSI,RSI
JNZ I32 @@60
PUSH RAX //-**** save byte size
ADD RAX,16*MEM_PAG_SIZE-1
SHR RAX,MEM_PAG_BITS
PUSH RDX //preserve HeapCtrl
PUSH RDX
PUSH RAX
CALL &MemPagTaskAlloc
POP RDX
TEST RAX,RAX
JNZ @@50
//Out of memory
@@45: LOCK
BTR U32 CHeapCtrl.locked_flags[RDX],HClf_LOCKED
POPFD
PUSH TRUE
MOV RAX,'OutMem'
PUSH RAX
CALL I32 &throw
JMP I32 MALLOC_FINAL_EXIT //Never gets here, hopefully.
@@50: MOV RSI,RAX
MOV EAX,U32 CMemBlk.pags[RSI]
SHL RAX,MEM_PAG_BITS
//Can it be combined with last chunk? (Never Free these chunks.)
MOV RDI,U64 CHeapCtrl.last_mergable[RDX]
LEA RBX,U64 [RSI+RAX]
CMP RDI,RBX
JNE @@55
PUSH RAX
MOV EAX,U32 CMemBlk.pags[RDI]
ADD U32 CMemBlk.pags[RSI],EAX
//QueRem
MOV RAX,U64 CMemBlk.next[RDI]
MOV RBX,U64 CMemBlk.last[RDI]
MOV U64 CMemBlk.last[RAX],RBX
MOV U64 CMemBlk.next[RBX],RAX
POP RAX
@@55: MOV U64 CHeapCtrl.last_mergable[RDX],RSI
LEA RSI,U64 sizeof(CMemBlk)[RSI]
SUB RAX,sizeof(CMemBlk)
LEA RBX,U64 CHeapCtrl.malloc_free_lst-CMemUnused.next[RDX]
MOV RDI,U64 CMemUnused.next[RBX]
MOV U64 CMemUnused.next[RSI],RDI
MOV U64 CMemUnused.size[RSI],RAX
MOV U64 CMemUnused.next[RBX],RSI
POP RAX //+****
JMP @@70
@@60: CMP U64 CMemUnused.size[RSI],RAX
JB I32 @@40
JNE @@70
@@65: MOV RDI,U64 CMemUnused.next[RSI]
MOV U64 CMemUnused.next[RBX],RDI
JMP MALLOC_ALMOST_DONE
@@70: SUB U64 CMemUnused.size[RSI],RAX //UPDATE FREE ENTRY
CMP U64 CMemUnused.size[RSI],sizeof(CMemUnused)
JAE @@75 //take from top of block
ADD U64 CMemUnused.size[RSI],RAX //doesn't fit, undo
JMP I32 @@40
@@75: ADD RSI,U64 CMemUnused.size[RSI]
MALLOC_ALMOST_DONE:
//RSI=res-CMemUsed.size
//RAX=size+CMemUsed.size
//RDX=HeapCtrl
ADD U64 CHeapCtrl.used_u8s[RDX],RAX
#if _CFG_HEAP_DBG
//QueIns
MOV RDI,U64 CHeapCtrl.last_um[RDX]
MOV U64 CMemUsed.next[RDI],RSI
MOV U64 CHeapCtrl.last_um[RDX],RSI
MOV U64 CMemUsed.last[RSI],RDI
LEA RDI,U64 CHeapCtrl.next_um-CMemUsed.next[RDX]
MOV U64 CMemUsed.next[RSI],RDI
//Caller1/Caller2
PUSH RDX
MOV RDX,U64 [MEM_HEAP_LIMIT]
MOV RDI,U64 SF_RIP[RBP]
CMP RDI,RDX
JB @@80
XOR RDI,RDI
MOV U64 CMemUsed.caller1[RSI],RDI
JMP @@90
@@80: MOV U64 CMemUsed.caller1[RSI],RDI
MOV RDI,U64 SF_RBP[RBP]
CMP RDI,RDX
JB @@85
XOR RDI,RDI
JMP @@90
@@85: MOV RDI,U64 SF_RIP[RDI]
CMP RDI,RDX
JB @@90
XOR RDI,RDI
@@90: MOV U64 CMemUsed.caller2[RSI],RDI
POP RDX
#endif
LOCK
BTR U32 CHeapCtrl.locked_flags[RDX],HClf_LOCKED
POPFD
MOV U64 CMemUsed.size[RSI],RAX
MOV U64 CMemUsed.hc[RSI],RDX
LEA RAX,U64 CMemUsed.start[RSI]
TEST U8 [SYS_SEMAS+SEMA_HEAPLOG_ACTIVE*DFT_CACHE_LINE_WIDTH],1
JZ @@105
PUSH RAX
PUSH RAX
MOV RAX,U64 [SYS_EXTERN_TABLE]
MOV RAX,U64 EXT_HEAPLOG_MALLOC*8[RAX]
TEST RAX,RAX
JZ @@95
CALL RAX
JMP @@100
@@95: ADD RSP,8
@@100: POP RAX
@@105: TEST U8 [SYS_HEAP_INIT_FLAG],1
JZ MALLOC_FINAL_EXIT
PUSH RAX
MOV RCX,U64 CMemUsed.size-CMemUsed.start[RAX]
SUB RCX,CMemUsed.start
MOV RDI,RAX
MOV AL,U8 [SYS_HEAP_INIT_VAL]
REP_STOSB
POP RAX
MALLOC_FINAL_EXIT:
POP RDI
POP RSI
POP RBP
RET1 16
//************************************
_FREE::
//Be aware of heap_hash in MemPagTaskAlloc().
PUSH RBP
MOV RBP,RSP
PUSH RSI
PUSH RDI
TEST U8 [SYS_SEMAS+SEMA_HEAPLOG_ACTIVE*DFT_CACHE_LINE_WIDTH],1
JZ @@15
MOV RBX,U64 SF_ARG1[RBP]
TEST RBX,RBX
JZ @@05
MOV RAX,U64 CMemUsed.size-CMemUsed.start[RBX]
TEST RAX,RAX
JGE @@05 //Aligned alloced chunks have neg size
ADD RBX,RAX
@@05: PUSH RBX
MOV RAX,U64 [SYS_EXTERN_TABLE]
MOV RAX,U64 EXT_HEAPLOG_FREE*8[RAX]
TEST RAX,RAX
JZ @@10
CALL RAX
JMP @@15
@@10: ADD RSP,8
@@15: MOV RSI,U64 SF_ARG1[RBP]
TEST RSI,RSI
#if _CFG_HEAP_DBG
JZ I32 FREE_DONE
#else
JZ FREE_DONE
#endif
MOV RAX,U64 CMemUsed.size-CMemUsed.start[RSI]
TEST RAX,RAX
JGE @@20 //Aligned alloced chunks have neg size.
//The neg size is offset to start of CMemUsed struct.
ADD RSI,RAX
@@20: PUSHFD
SUB RSI,CMemUsed.start
MOV RDX,U64 CMemUsed.hc[RSI]
CMP U32 CHeapCtrl.hc_signature[RDX],HEAP_CTRL_SIGNATURE_VAL
JE @@25
ADD RSI,CMemUsed.start
PUSH RSI
CALL &SysBadFree
JMP I32 _SYS_HLT
@@25: MOV RAX,U64 CMemUsed.size[RSI]
SUB U64 CHeapCtrl.used_u8s[RDX],RAX
CLI
@@30: LOCK
BTS U32 CHeapCtrl.locked_flags[RDX],HClf_LOCKED
PAUSE
JC @@30
#if _CFG_HEAP_DBG
//QueRem
MOV RDX,U64 CMemUsed.next[RSI]
MOV RDI,U64 CMemUsed.last[RSI]
MOV U64 CMemUsed.last[RDX],RDI
MOV U64 CMemUsed.next[RDI],RDX
//Caller1/Caller2
MOV RDX,U64 [MEM_HEAP_LIMIT]
MOV RDI,U64 SF_RIP[RBP]
CMP RDI,RDX
JB @@35
XOR RDI,RDI
MOV U64 CMemUnused.caller1[RSI],RDI
JMP @@45
@@35: MOV U64 CMemUnused.caller1[RSI],RDI
MOV RDI,U64 SF_RBP[RBP]
CMP RDI,RDX
JB @@40
XOR RDI,RDI
JMP @@45
@@40: MOV RDI,U64 SF_RIP[RDI]
CMP RDI,RDX
JB @@45
XOR RDI,RDI
@@45: MOV U64 CMemUnused.caller2[RSI],RDI
MOV RDX,U64 CMemUsed.hc[RSI]
#endif
CMP RAX,MEM_HEAP_HASH_SIZE
JAE @@50
#assert CMemUnused.size==CMemUsed.size
// MOV U64 CMemUnused.size[RSI],RAX
MOV RBX,U64 CHeapCtrl.heap_hash[RAX+RDX]
MOV U64 CMemUnused.next[RSI],RBX
MOV U64 CHeapCtrl.heap_hash[RAX+RDX],RSI
JMP @@55
@@50: SUB RSI,sizeof(CMemBlk)
PUSH RDX
PUSH RDX
PUSH RSI
CALL &MemPagTaskFree
POP RDX
@@55: LOCK
BTR U32 CHeapCtrl.locked_flags[RDX],HClf_LOCKED
POPFD
FREE_DONE:
POP RDI
POP RSI
POP RBP
RET1 8
//************************************
_MSIZE::
PUSH RBP
MOV RBP,RSP
MOV RBX,U64 SF_ARG1[RBP]
XOR RAX,RAX
TEST RBX,RBX
JZ @@10
MOV RAX,U64 CMemUsed.size-CMemUsed.start[RBX]
TEST RAX,RAX
JGE @@05 //Aligned alloced chunks have neg size
ADD RBX,RAX
MOV RAX,U64 CMemUsed.size-CMemUsed.start[RBX]
@@05: SUB RAX,CMemUsed.start
@@10: POP RBP
RET1 8
//************************************
_MSIZE2::
PUSH RBP
MOV RBP,RSP
MOV RBX,U64 SF_ARG1[RBP]
XOR RAX,RAX
TEST RBX,RBX
JZ @@10
MOV RAX,U64 CMemUsed.size-CMemUsed.start[RBX]
TEST RAX,RAX
JGE @@05 //Aligned alloced chunks have neg size
ADD RBX,RAX
@@05: MOV RAX,U64 CMemUsed.size-CMemUsed.start[RBX]
@@10: POP RBP
RET1 8
//************************************
_MHEAP_CTRL::
PUSH RBP
MOV RBP,RSP
MOV RBX,U64 SF_ARG1[RBP]
XOR RAX,RAX
TEST RBX,RBX
JZ @@10
MOV RAX,U64 CMemUsed.size-CMemUsed.start[RBX]
TEST RAX,RAX
JGE @@05 //Aligned alloced chunks have neg size
ADD RBX,RAX
@@05: MOV RAX,U64 CMemUsed.hc-CMemUsed.start[RBX]
@@10: POP RBP
RET1 8
}
_extern _FREE U0 Free(U8 *addr); //Free MAlloc()ed memory chunk.
_extern _MSIZE I64 MSize(U8 *src); //Size of heap object.
_extern _MSIZE2 I64 MSize2(U8 *src); //Internal size of heap object.
_extern _MHEAP_CTRL CHeapCtrl *MHeapCtrl(U8 *src); //CHeapCtrl of object.
_extern _MALLOC U8 *MAlloc(I64 size,CTask *mem_task=NULL); //Alloc memory chunk.
//Accepts a CTask or CHeapCtrl. NULL allocs off current task's heap.
U8 *AMAlloc(I64 size)
{//Alloc memory in Adam's heap.
return MAlloc(size,adam_task);
}
U8 *CAlloc(I64 size,CTask *mem_task=NULL)
{//Accepts a CTask or CHeapCtrl.NULL allocs off current task's heap.
U8 *res=MAlloc(size,mem_task);
MemSet(res,0,size);
return res;
}
U8 *ACAlloc(I64 size)
{//Alloc and set to zero memory in Adam's heap.
return CAlloc(size,adam_task);
}
U8 *MAllocIdent(U8 *src,CTask *mem_task=NULL)
{//Accepts a CTask or CHeapCtrl.NULL allocs off current task's heap.
U8 *res;
I64 size;
if (!src) return NULL;
size=MSize(src);
res=MAlloc(size,mem_task);
MemCpy(res,src,size);
return res;
}
U8 *AMAllocIdent(U8 *src)
{//Alloc in Adam's heap, ident copy of heap node.
return MAllocIdent(src,adam_task);
}
U8 *MAllocAligned(I64 size,I64 alignment,
CTask *mem_task=NULL,I64 misalignment=0)
{//Only powers of two alignment. This is awful.
I64 mask=alignment-1;
U8 *ptr=MAlloc(size+mask+sizeof(I64)+misalignment,mem_task),
*res=(ptr+sizeof(I64)+mask)&~mask+misalignment;
res(I64 *)[-1]=ptr-res;
#assert offset(CMemUsed.size)==offset(CMemUsed.start)-sizeof(I64)
return res;
}
U8 *CAllocAligned(I64 size,I64 alignment,
CTask *mem_task=NULL,I64 misalignment=0)
{//Only powers of two alignment. This is awful.
I64 mask=alignment-1;
U8 *ptr=MAlloc(size+mask+sizeof(I64)+misalignment,mem_task),
*res=(ptr+sizeof(I64)+mask)&~mask+misalignment;
res(I64 *)[-1]=ptr-res;
#assert offset(CMemUsed.size)==offset(CMemUsed.start)-sizeof(I64)
MemSet(res,0,size);
return res;
}
U8 *StrNew(U8 *buf,CTask *mem_task=NULL)
{//Accepts a CTask or CHeapCtrl.NULL allocs off current task's heap.
U8 *res;
I64 size;
if (buf) {
size=StrLen(buf)+1;
res=MAlloc(size,mem_task);
MemCpy(res,buf,size);
} else {
res=MAlloc(1,mem_task);
*res=0;
}
return res;
}
U8 *AStrNew(U8 *buf)
{//Alloc copy of string in Adam's heap.
return StrNew(buf,adam_task);
}