30 #if defined (__APPLE__) 31 #define AARCH64_STACK_ALIGN 1 33 #define AARCH64_STACK_ALIGN 16 39 #define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT) 56 #if defined (__clang__) && defined (__APPLE__) 58 sys_icache_invalidate (
void *start,
size_t len);
64 #if defined (__clang__) && defined (__APPLE__) 65 sys_icache_invalidate (start, (
char *)end - (
char *)start);
66 #elif defined (__GNUC__) 67 __builtin___clear_cache (start, end);
69 #error "Missing builtin to flush instruction cache" 76 return &context->
x[n];
82 #if defined __AARCH64EB__ 83 return &context->
v[n].
d[1].
s[1];
85 return &context->
v[n].
d[0].
s[0];
92 #if defined __AARCH64EB__ 93 return &context->
v[n].
d[1];
95 return &context->
v[n].
d[0];
102 return &context->
v[n];
116 case FFI_TYPE_DOUBLE:
118 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 124 case FFI_TYPE_UINT16:
125 case FFI_TYPE_SINT16:
126 case FFI_TYPE_UINT32:
127 case FFI_TYPE_SINT32:
129 case FFI_TYPE_POINTER:
130 case FFI_TYPE_UINT64:
131 case FFI_TYPE_SINT64:
149 #if defined (__APPLE__) 150 return sizeof (UINT32);
152 case FFI_TYPE_DOUBLE:
153 return sizeof (UINT64);
154 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 156 return sizeof (
long double);
160 #if defined (__APPLE__) 161 return sizeof (UINT8);
163 case FFI_TYPE_UINT16:
164 case FFI_TYPE_SINT16:
165 #if defined (__APPLE__) 166 return sizeof (UINT16);
168 case FFI_TYPE_UINT32:
170 case FFI_TYPE_SINT32:
171 #if defined (__APPLE__) 172 return sizeof (UINT32);
174 case FFI_TYPE_POINTER:
175 case FFI_TYPE_UINT64:
176 case FFI_TYPE_SINT64:
177 return sizeof (UINT64);
193 return sizeof (UINT32);
194 case FFI_TYPE_DOUBLE:
195 return sizeof (UINT64);
196 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 198 return sizeof (
long double);
201 return sizeof (UINT8);
203 return sizeof (SINT8);
204 case FFI_TYPE_UINT16:
205 return sizeof (UINT16);
206 case FFI_TYPE_SINT16:
207 return sizeof (SINT16);
208 case FFI_TYPE_UINT32:
209 return sizeof (UINT32);
211 case FFI_TYPE_SINT32:
212 return sizeof (SINT32);
213 case FFI_TYPE_POINTER:
214 case FFI_TYPE_UINT64:
215 return sizeof (UINT64);
216 case FFI_TYPE_SINT64:
217 return sizeof (SINT64);
241 return (type == FFI_TYPE_FLOAT || type == FFI_TYPE_DOUBLE
247 static unsigned short 250 if (ty->type == FFI_TYPE_STRUCT && ty->elements)
253 unsigned short candidate_type
255 for (i =1; ty->elements[i]; i++)
257 unsigned short iteration_type = 0;
261 if (ty->elements[i]->type == FFI_TYPE_STRUCT
262 && ty->elements[i]->elements)
268 iteration_type = ty->elements[i]->type;
272 if (candidate_type != iteration_type)
273 return FFI_TYPE_STRUCT;
275 return candidate_type;
292 if (ty->type == FFI_TYPE_STRUCT && ty->elements)
296 for (n = 0; ty->elements[n]; n++)
298 if (ty->elements[n]->type == FFI_TYPE_STRUCT
299 && ty->elements[n]->elements)
319 if (ty->type == FFI_TYPE_STRUCT
324 return n >= 1 && n <= 4;
348 case FFI_TYPE_DOUBLE:
349 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 353 case FFI_TYPE_UINT16:
354 case FFI_TYPE_UINT32:
355 case FFI_TYPE_UINT64:
356 case FFI_TYPE_POINTER:
358 case FFI_TYPE_SINT16:
359 case FFI_TYPE_SINT32:
361 case FFI_TYPE_SINT64:
364 case FFI_TYPE_STRUCT:
369 else if (ty->size > 16)
400 || (ty->type == FFI_TYPE_STRUCT &&
is_hfa (ty));
415 #if defined (__APPLE__) 416 unsigned allocating_variadic;
428 #if defined (__APPLE__) 429 state->allocating_variadic = 0;
490 #if defined (__APPLE__) 491 if (state->allocating_variadic)
497 allocation = stack + state->
nsaa;
511 *(
float *) dest = *(
float *) source;
513 case FFI_TYPE_DOUBLE:
514 *(
double *) dest = *(
double *) source;
516 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 518 *(
long double *) dest = *(
long double *) source;
522 *(
ffi_arg *) dest = *(UINT8 *) source;
525 *(
ffi_sarg *) dest = *(SINT8 *) source;
527 case FFI_TYPE_UINT16:
528 *(
ffi_arg *) dest = *(UINT16 *) source;
530 case FFI_TYPE_SINT16:
531 *(
ffi_sarg *) dest = *(SINT16 *) source;
533 case FFI_TYPE_UINT32:
534 *(
ffi_arg *) dest = *(UINT32 *) source;
537 case FFI_TYPE_SINT32:
538 *(
ffi_sarg *) dest = *(SINT32 *) source;
540 case FFI_TYPE_POINTER:
541 case FFI_TYPE_UINT64:
542 *(
ffi_arg *) dest = *(UINT64 *) source;
544 case FFI_TYPE_SINT64:
545 *(
ffi_sarg *) dest = *(SINT64 *) source;
559 unsigned char *stack,
577 for (i = 0; i < elems; i++)
592 unsigned char *stack,
597 size_t size = alignment;
603 size =
sizeof (UINT32);
605 case FFI_TYPE_DOUBLE:
610 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 619 case FFI_TYPE_UINT16:
620 case FFI_TYPE_SINT16:
621 case FFI_TYPE_UINT32:
622 case FFI_TYPE_SINT32:
624 case FFI_TYPE_POINTER:
625 case FFI_TYPE_UINT64:
626 case FFI_TYPE_SINT64:
643 unsigned char *stack,
666 for (i = 0; i < ecif->
cif->nargs; i++)
668 ffi_type *ty = ecif->
cif->arg_types[i];
678 case FFI_TYPE_DOUBLE:
679 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 684 case FFI_TYPE_UINT16:
685 case FFI_TYPE_SINT16:
686 case FFI_TYPE_UINT32:
688 case FFI_TYPE_SINT32:
689 case FFI_TYPE_POINTER:
690 case FFI_TYPE_UINT64:
691 case FFI_TYPE_SINT64:
693 ecif->
avalue[i], ty->type);
696 case FFI_TYPE_STRUCT:
702 else if (ty->size > 16)
709 &(ecif->
avalue[i]), FFI_TYPE_POINTER);
711 else if (
available_x (&state) >= (ty->size + 7) / 8)
718 for (j = 0; j < (ty->size + 7) / 8; j++)
721 &(((UINT64 *) ecif->
avalue[i])[j]),
734 ty->size), ecif->
avalue + i, ty->size);
743 #if defined (__APPLE__) 744 if (i + 1 == ecif->
cif->aarch64_nfixedargs)
749 state.allocating_variadic = 1;
754 return ecif->
cif->aarch64_flags;
768 cif->aarch64_flags = 0;
777 for (i = 0; i < cif->nargs; i++)
785 #if defined (__APPLE__) 786 cif->aarch64_nfixedargs = 0;
792 #if defined (__APPLE__) 796 unsigned int nfixedargs,
797 unsigned int ntotalargs)
803 cif->aarch64_nfixedargs = nfixedargs;
813 ffi_call (ffi_cif *cif,
void (*fn)(
void),
void *rvalue,
void **avalue)
832 stack_bytes =
ALIGN(cif->bytes, 16);
834 memset (&context, 0,
sizeof (context));
838 switch (cif->rtype->type)
842 case FFI_TYPE_DOUBLE:
843 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 848 case FFI_TYPE_UINT16:
849 case FFI_TYPE_SINT16:
850 case FFI_TYPE_UINT32:
851 case FFI_TYPE_SINT32:
852 case FFI_TYPE_POINTER:
853 case FFI_TYPE_UINT64:
855 case FFI_TYPE_SINT64:
863 case FFI_TYPE_STRUCT:
869 for (j = 0; j < elems; j++)
878 size_t size =
ALIGN (cif->rtype->size, sizeof (UINT64));
908 { 0x70, 0x00, 0x00, 0x58,
909 0x91, 0x00, 0x00, 0x10,
910 0x00, 0x02, 0x1f, 0xd6
915 #define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS) \ 916 ({unsigned char *__tramp = (unsigned char*)(TRAMP); \ 917 UINT64 __fun = (UINT64)(FUN); \ 918 UINT64 __ctx = (UINT64)(CTX); \ 919 UINT64 __flags = (UINT64)(FLAGS); \ 920 memcpy (__tramp, trampoline, sizeof (trampoline)); \ 921 memcpy (__tramp + 12, &__fun, sizeof (__fun)); \ 922 memcpy (__tramp + 20, &__ctx, sizeof (__ctx)); \ 923 memcpy (__tramp + 28, &__flags, sizeof (__flags)); \ 924 ffi_clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE); \ 930 void (*fun)(ffi_cif*,
void*,
void**,
void*),
941 closure->user_data = user_data;
967 ffi_cif *cif = closure->cif;
968 void **avalue = (
void**)
alloca (cif->nargs * sizeof (
void*));
975 for (i = 0; i < cif->nargs; i++)
977 ffi_type *ty = cif->arg_types[i];
987 case FFI_TYPE_UINT16:
988 case FFI_TYPE_SINT16:
989 case FFI_TYPE_UINT32:
990 case FFI_TYPE_SINT32:
992 case FFI_TYPE_POINTER:
993 case FFI_TYPE_UINT64:
994 case FFI_TYPE_SINT64:
996 case FFI_TYPE_DOUBLE:
997 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 1004 case FFI_TYPE_STRUCT:
1018 case FFI_TYPE_FLOAT:
1031 UINT32 *p = avalue[i] =
alloca (ty->size);
1039 case FFI_TYPE_DOUBLE:
1052 UINT64 *p = avalue[i] =
alloca (ty->size);
1060 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 1074 else if (ty->size > 16)
1080 &state, FFI_TYPE_POINTER),
1081 sizeof (avalue[i]));
1083 else if (
available_x (&state) >= (ty->size + 7) / 8)
1086 state.
ngrn += (ty->size + 7) / 8;
1115 rvalue =
alloca (cif->rtype->size);
1116 (closure->fun) (cif, rvalue, avalue, closure->user_data);
1120 switch (cif->rtype->type)
1125 case FFI_TYPE_UINT8:
1126 case FFI_TYPE_UINT16:
1127 case FFI_TYPE_UINT32:
1128 case FFI_TYPE_POINTER:
1129 case FFI_TYPE_UINT64:
1130 case FFI_TYPE_SINT8:
1131 case FFI_TYPE_SINT16:
1133 case FFI_TYPE_SINT32:
1134 case FFI_TYPE_SINT64:
1135 case FFI_TYPE_FLOAT:
1136 case FFI_TYPE_DOUBLE:
1137 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE 1145 case FFI_TYPE_STRUCT:
1151 for (j = 0; j < elems; j++)
1158 else if ((cif->rtype->size + 7) / 8 <
N_X_ARG_REG)
1160 size_t size =
ALIGN (cif->rtype->size, sizeof (UINT64)) ;
1176 (closure->fun) (cif, rvalue, avalue, closure->user_data);
static unsigned is_floating_type(unsigned short type)
static void * allocate_to_v(struct call_context *context, struct arg_state *state)
static void * get_d_addr(struct call_context *context, unsigned n)
static void * allocate_to_d(struct call_context *context, struct arg_state *state)
static int is_v_register_candidate(ffi_type *ty)
static unsigned aarch64_prep_args(struct call_context *context, unsigned char *stack, extended_cif *ecif)
static size_t get_basic_type_alignment(unsigned short type)
void FFI_HIDDEN ffi_closure_SYSV_inner(ffi_closure *closure, struct call_context *context, void *stack)
static void * allocate_to_x(struct call_context *context, struct arg_state *state)
static void * allocate_to_s(struct call_context *context, struct arg_state *state)
static unsigned element_count(ffi_type *ty)
static void copy_basic_type(void *dest, void *source, unsigned short type)
static unsigned char trampoline[]
void ffi_call_SYSV(unsigned(*)(struct call_context *context, unsigned char *, extended_cif *), struct call_context *context, extended_cif *, size_t, void(*fn)(void))
static void * get_basic_type_addr(unsigned short type, struct call_context *context, unsigned n)
#define FFI_INIT_TRAMPOLINE(TRAMP, FUN, CTX, FLAGS)
static void arg_init(struct arg_state *state, size_t call_frame_size)
static void * allocate_to_stack(struct arg_state *state, void *stack, size_t alignment, size_t size)
static void copy_to_register_or_stack(struct call_context *context, unsigned char *stack, struct arg_state *state, void *value, unsigned short type)
static void * get_s_addr(struct call_context *context, unsigned n)
struct call_context::@10 v[AARCH64_N_VREG]
static unsigned available_x(struct arg_state *state)
#define AARCH64_FFI_WITH_V
#define FFI_TYPE_LONGDOUBLE
#define AARCH64_STACK_ALIGN
ffi_status ffi_prep_closure_loc(ffi_closure *closure, ffi_cif *cif, void(*fun)(ffi_cif *, void *, void **, void *), void *user_data, void *codeloc)
static size_t get_basic_type_size(unsigned short type)
ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs)
static int is_hfa(ffi_type *ty)
void ffi_call(ffi_cif *cif, void(*fn)(void), void *rvalue, void **avalue)
static void * get_v_addr(struct call_context *context, unsigned n)
const UINT64 ffi_template_tramp_tile [] FFI_HIDDEN
static void copy_hfa_to_reg_or_stack(void *memory, ffi_type *ty, struct call_context *context, unsigned char *stack, struct arg_state *state)
register unsigned int len
static void * allocate_to_register_or_stack(struct call_context *context, unsigned char *stack, struct arg_state *state, unsigned short type)
static int is_register_candidate(ffi_type *ty)
static void * get_x_addr(struct call_context *context, unsigned n)
ffi_status ffi_prep_cif_machdep(ffi_cif *cif)
static unsigned available_v(struct arg_state *state)
void ffi_closure_SYSV(ffi_closure *)
static void ffi_clear_cache(void *start, void *end)
static unsigned short get_homogeneous_type(ffi_type *ty)