Skip to content

Add prototype inline cache #551

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 14 commits into from
Closed

Add prototype inline cache #551

Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
51b90ba
feat: add prototype inline cache for basic get/set op
Sep 25, 2024
7989a42
Merge branch 'master' into prototype-inline-cache
Sep 25, 2024
25df155
fix: cr->index should be changed when ic missing
Sep 25, 2024
2ed62d9
feat: enable ic for OP_get_field2 and force_inline ic get/set methods
Sep 25, 2024
badda37
Merge branch 'master' into prototype-inline-cache
Oct 8, 2024
d43f4ef
feat: prototype ICs code modifications and optimizations
Oct 9, 2024
d7b0901
Merge branch 'master' into prototype-inline-cache
Nov 28, 2024
e2ec966
fix: repair the ref_count release issue caused by add_ic_slot
Nov 28, 2024
db482ee
fix: remove ic watchpoints free function call
Nov 28, 2024
9f1ce56
fix: remove force_inline and watchpoint free function call
Nov 29, 2024
7826ad8
fix: should use BOOL instead of bool at ic_delete_shape_proto_watchpo…
Nov 29, 2024
3639c24
Merge branch 'master' into prototype-inline-cache
Jan 16, 2025
617dbe0
fix BOOL to bool type macro
Jan 21, 2025
88d8f4b
Merge branch 'master' into prototype-inline-cache
Jan 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 152 additions & 22 deletions quickjs.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,10 +568,19 @@ typedef enum JSFunctionKindEnum {

#define IC_CACHE_ITEM_CAPACITY 4

typedef struct JSInlineCacheWatchpoint {
struct list_head link;
struct JSInlineCacheRingSlot *ref;
uint32_t offset;
JSAtom atom;
} JSInlineCacheWatchpoint;

typedef struct JSInlineCacheRingSlot {
/* SoA for space optimization: 56 bytes */
/* SoA for space optimization: 120 bytes */
JSShape* shape[IC_CACHE_ITEM_CAPACITY];
uint32_t prop_offset[IC_CACHE_ITEM_CAPACITY];
JSObject* proto[IC_CACHE_ITEM_CAPACITY];
JSInlineCacheWatchpoint* watchpoint_ref[IC_CACHE_ITEM_CAPACITY];
JSAtom atom;
uint8_t index;
} JSInlineCacheRingSlot;
Expand Down Expand Up @@ -604,11 +613,15 @@ static JSInlineCache *init_ic(JSContext *ctx);
static int rebuild_ic(JSContext *ctx, JSInlineCache *ic);
static int resize_ic_hash(JSContext *ctx, JSInlineCache *ic);
static int free_ic(JSRuntime *rt, JSInlineCache *ic);
static int ic_delete_shape_proto_watchpoints(JSRuntime *rt, JSShape *shape, JSAtom atom, bool force);
static int ic_watchpoint_delete_handler(JSRuntime* rt, JSInlineCacheRingSlot *ref, uint8_t offset, JSAtom atom, JSInlineCacheRingSlot *target);
static int js_shape_delete_watchpoints(JSRuntime *rt, JSShape *shape, JSInlineCacheRingSlot *target);
static JSInlineCacheWatchpoint *js_shape_create_watchpoint(JSRuntime *rt, JSShape *shape, JSInlineCacheRingSlot *ptr, uint8_t offset, JSAtom atom);
static void add_ic_slot(JSContext *ctx, JSInlineCacheUpdate *icu,
JSAtom atom, JSObject *object, uint32_t prop_offset);
JSAtom atom, JSObject *object, uint32_t prop_offset, JSObject* prototype);

static uint32_t get_ic_prop_offset(const JSInlineCacheUpdate *icu,
JSShape *shape)
JSShape *shape, JSObject **prototype)
{
uint32_t i, cache_offset = icu->offset;
JSInlineCache *ic = icu->ic;
Expand All @@ -621,6 +634,7 @@ static uint32_t get_ic_prop_offset(const JSInlineCacheUpdate *icu,
shape_slot = *(cr->shape + i);
if (likely(shape_slot == shape)) {
cr->index = i;
*prototype = cr->proto[i];
return cr->prop_offset[i];
}

Expand All @@ -629,7 +643,8 @@ static uint32_t get_ic_prop_offset(const JSInlineCacheUpdate *icu,
break;
}
}


*prototype = NULL;
return INLINE_CACHE_MISS;
}

Expand Down Expand Up @@ -902,6 +917,7 @@ struct JSShape {
int deleted_prop_count;
JSShape *shape_hash_next; /* in JSRuntime.shape_hash[h] list */
JSObject *proto;
struct list_head *watchpoint; /* watchpoint for prototype inline cache */
JSShapeProperty prop[]; /* prop_size elements */
};

Expand Down Expand Up @@ -4523,6 +4539,7 @@ static no_inline JSShape *js_new_shape2(JSContext *ctx, JSObject *proto,
sh->hash = shape_initial_hash(proto);
sh->is_hashed = true;
sh->has_small_array_index = false;
sh->watchpoint = NULL;
js_shape_hash_link(ctx->rt, sh);
return sh;
}
Expand Down Expand Up @@ -4554,6 +4571,7 @@ static JSShape *js_clone_shape(JSContext *ctx, JSShape *sh1)
sh->header.ref_count = 1;
add_gc_object(ctx->rt, &sh->header, JS_GC_OBJ_TYPE_SHAPE);
sh->is_hashed = false;
sh->watchpoint = NULL;
if (sh->proto) {
js_dup(JS_MKPTR(JS_TAG_OBJECT, sh->proto));
}
Expand All @@ -4580,6 +4598,7 @@ static void js_free_shape0(JSRuntime *rt, JSShape *sh)
if (sh->proto != NULL) {
JS_FreeValueRT(rt, JS_MKPTR(JS_TAG_OBJECT, sh->proto));
}
js_shape_delete_watchpoints(rt, sh, NULL);
pr = get_shape_prop(sh);
for(i = 0; i < sh->prop_count; i++) {
JS_FreeAtomRT(rt, pr->atom);
Expand Down Expand Up @@ -5785,6 +5804,7 @@ static void mark_children(JSRuntime *rt, JSGCObjectHeader *gp,
/* the template objects can be part of a cycle */
{
JSShape **shape, *(*shapes)[IC_CACHE_ITEM_CAPACITY];
JSObject **proto, *(*protos)[IC_CACHE_ITEM_CAPACITY];
JSFunctionBytecode *b = (JSFunctionBytecode *)gp;
int i;
for(i = 0; i < b->cpool_count; i++) {
Expand All @@ -5798,6 +5818,10 @@ static void mark_children(JSRuntime *rt, JSGCObjectHeader *gp,
for (shape = *shapes; shape != endof(*shapes); shape++)
if (*shape)
mark_func(rt, &(*shape)->header);
protos = &b->ic->cache[i].proto;
for (proto = *protos; proto != endof(*protos); proto++)
if (*proto)
mark_func(rt, &(*proto)->header);
}
}
}
Expand Down Expand Up @@ -7143,6 +7167,7 @@ static int JS_SetPrototypeInternal(JSContext *ctx, JSValue obj,
if (js_shape_prepare_update(ctx, p, NULL))
return -1;
sh = p->shape;
ic_delete_shape_proto_watchpoints(ctx->rt, sh, JS_ATOM_NULL, true);
if (sh->proto)
JS_FreeValue(ctx, JS_MKPTR(JS_TAG_OBJECT, sh->proto));
sh->proto = proto;
Expand Down Expand Up @@ -7360,7 +7385,7 @@ static JSValue JS_GetPropertyInternal2(JSContext *ctx, JSValue obj,
JSInlineCacheUpdate *icu,
bool throw_ref_error)
{
JSObject *p;
JSObject *p, *p1;
JSProperty *pr;
JSShapeProperty *prs;
uint32_t tag, offset, proto_depth;
Expand Down Expand Up @@ -7401,6 +7426,7 @@ static JSValue JS_GetPropertyInternal2(JSContext *ctx, JSValue obj,
p = JS_VALUE_GET_OBJ(obj);
}

p1 = p;
for(;;) {
prs = find_own_property_ic(&pr, p, prop, &offset);
if (prs) {
Expand All @@ -7427,8 +7453,7 @@ static JSValue JS_GetPropertyInternal2(JSContext *ctx, JSValue obj,
continue;
}
} else {
if (proto_depth == 0)
add_ic_slot(ctx, icu, prop, p, offset);
add_ic_slot(ctx, icu, prop, p1, offset, proto_depth > 0 ? p : NULL);
return js_dup(pr->u.value);
}
}
Expand Down Expand Up @@ -7515,20 +7540,22 @@ JSValue JS_GetProperty(JSContext *ctx, JSValue this_obj, JSAtom prop)
return JS_GetPropertyInternal2(ctx, this_obj, prop, this_obj, NULL, false);
}

static JSValue JS_GetPropertyInternalWithIC(JSContext *ctx, JSValue obj,
static inline JSValue JS_GetPropertyInternalWithIC(JSContext *ctx, JSValue obj,
JSAtom prop, JSValue this_obj,
JSInlineCacheUpdate *icu,
bool throw_ref_error)
{
uint32_t tag, offset;
JSObject *p;
JSObject *p, *proto;
tag = JS_VALUE_GET_TAG(obj);
if (unlikely(tag != JS_TAG_OBJECT))
goto slow_path;
p = JS_VALUE_GET_OBJ(obj);
offset = get_ic_prop_offset(icu, p->shape);
if (likely(offset != INLINE_CACHE_MISS))
offset = get_ic_prop_offset(icu, p->shape, &proto);
if (likely(offset != INLINE_CACHE_MISS)) {
if (proto) p = proto;
return js_dup(p->prop[offset].u.value);
}
slow_path:
return JS_GetPropertyInternal2(ctx, obj, prop, this_obj, icu, throw_ref_error);
}
Expand Down Expand Up @@ -8463,7 +8490,7 @@ static int delete_property(JSContext *ctx, JSObject *p, JSAtom atom)
pr->flags = 0;
pr->atom = JS_ATOM_NULL;
pr1->u.value = JS_UNDEFINED;

ic_delete_shape_proto_watchpoints(ctx->rt, sh, atom, false);
/* compact the properties if too many deleted properties */
if (sh->deleted_prop_count >= 8 &&
sh->deleted_prop_count >= ((unsigned)sh->prop_count / 2)) {
Expand Down Expand Up @@ -8722,7 +8749,7 @@ static int JS_SetPropertyInternal2(JSContext *ctx, JSValue obj, JSAtom prop,
if (likely((prs->flags & (JS_PROP_TMASK | JS_PROP_WRITABLE |
JS_PROP_LENGTH)) == JS_PROP_WRITABLE)) {
/* fast case */
add_ic_slot(ctx, icu, prop, p, offset);
add_ic_slot(ctx, icu, prop, p, offset, NULL);
set_value(ctx, &pr->u.value, val);
return true;
} else if (prs->flags & JS_PROP_LENGTH) {
Expand Down Expand Up @@ -8884,10 +8911,13 @@ static int JS_SetPropertyInternal2(JSContext *ctx, JSValue obj, JSAtom prop,
}
goto generic_create_prop;
} else {
ic_delete_shape_proto_watchpoints(ctx->rt, p->shape, prop, false);
pr = add_property(ctx, p, prop, JS_PROP_C_W_E);
if (!pr)
if (unlikely(!pr))
goto fail;
pr->u.value = val;
/* fast case */
add_ic_slot(ctx, icu, prop, p, p->shape->prop_count - 1, NULL);
return true;
}
}
Expand Down Expand Up @@ -8947,17 +8977,18 @@ int JS_SetProperty(JSContext *ctx, JSValue this_obj, JSAtom prop, JSValue val)
}

// XXX(bnoordhuis) only used by OP_put_field_ic, maybe inline at call site
static int JS_SetPropertyInternalWithIC(JSContext *ctx, JSValue this_obj,
static inline int JS_SetPropertyInternalWithIC(JSContext *ctx, JSValue this_obj,
JSAtom prop, JSValue val, int flags,
JSInlineCacheUpdate *icu) {
uint32_t tag, offset;
JSObject *p;
JSObject *p, *proto;
tag = JS_VALUE_GET_TAG(this_obj);
if (unlikely(tag != JS_TAG_OBJECT))
goto slow_path;
p = JS_VALUE_GET_OBJ(this_obj);
offset = get_ic_prop_offset(icu, p->shape);
offset = get_ic_prop_offset(icu, p->shape, &proto);
if (likely(offset != INLINE_CACHE_MISS)) {
if (proto) goto slow_path;
set_value(ctx, &p->prop[offset].u.value, val);
return true;
}
Expand Down Expand Up @@ -9335,6 +9366,44 @@ static int js_shape_prepare_update(JSContext *ctx, JSObject *p,
return 0;
}

int js_shape_delete_watchpoints(JSRuntime *rt, JSShape *shape, JSInlineCacheRingSlot *target) {
struct list_head *el, *el1;
if (unlikely(!shape || !shape->watchpoint))
goto end;
list_for_each_safe(el, el1, shape->watchpoint) {
JSInlineCacheWatchpoint *o = list_entry(el, JSInlineCacheWatchpoint, link);
if (!ic_watchpoint_delete_handler(rt, o->ref, o->offset, o->atom, target)) {
list_del(el);
js_free_rt(rt, o);
break;
}
}
if (!target) {
list_empty(shape->watchpoint);
js_free_rt(rt, shape->watchpoint);
}
end:
return 0;
}

JSInlineCacheWatchpoint *js_shape_create_watchpoint(JSRuntime *rt, JSShape *shape, JSInlineCacheRingSlot *ptr, uint8_t offset, JSAtom atom)
{
JSInlineCacheWatchpoint *o;
o = js_malloc_rt(rt, sizeof(*o));
if (unlikely(!o))
return NULL;
o->ref = ptr;
o->atom = atom;
o->offset = offset;
if (!shape->watchpoint) {
shape->watchpoint = js_malloc_rt(rt, sizeof(*shape->watchpoint));
init_list_head(shape->watchpoint);
}
init_list_head(&o->link);
list_add_tail(&o->link, shape->watchpoint);
return o;
}

static int js_update_property_flags(JSContext *ctx, JSObject *p,
JSShapeProperty **pprs, int flags)
{
Expand Down Expand Up @@ -55916,12 +55985,22 @@ int free_ic(JSRuntime* rt, JSInlineCache *ic)
uint32_t i;
JSInlineCacheHashSlot *ch, *ch_next;
JSShape **shape, *(*shapes)[IC_CACHE_ITEM_CAPACITY];
JSInlineCacheWatchpoint **ref, *(*refs)[IC_CACHE_ITEM_CAPACITY], *o;
if (ic->cache) {
for (i = 0; i < ic->count; i++) {
shapes = &ic->cache[i].shape;
JS_FreeAtomRT(rt, ic->cache[i].atom);
for (shape = *shapes; shape != endof(*shapes); shape++)
js_free_shape_null(rt, *shape);
refs = &ic->cache[i].watchpoint_ref;
for (ref = *refs; ref != endof(*refs); ref++) {
o = *ref;
if (o) {
ic_watchpoint_delete_handler(rt, o->ref ,o->offset, o->atom, NULL);
list_del(&o->link);
js_free_rt(rt, o);
}
}
}
}
for (i = 0; i < ic->capacity; i++) {
Expand All @@ -55939,9 +56018,9 @@ int free_ic(JSRuntime* rt, JSInlineCache *ic)
}

static void add_ic_slot(JSContext *ctx, JSInlineCacheUpdate *icu,
JSAtom atom, JSObject *object, uint32_t prop_offset)
JSAtom atom, JSObject *object, uint32_t prop_offset, JSObject* prototype)
{
int32_t i;
uint8_t i;
uint32_t h;
JSInlineCacheHashSlot *ch;
JSInlineCacheRingSlot *cr;
Expand All @@ -55956,6 +56035,8 @@ static void add_ic_slot(JSContext *ctx, JSInlineCacheUpdate *icu,
sh = object->shape;
if (!sh->is_hashed)
return;
if (prototype && !prototype->shape->is_hashed)
return;
cr = NULL;
h = get_index_hash(atom, ic->hash_bits);
for (ch = ic->hash[h]; ch != NULL; ch = ch->next) {
Expand All @@ -55967,19 +56048,68 @@ static void add_ic_slot(JSContext *ctx, JSInlineCacheUpdate *icu,
assert(cr != NULL);
i = cr->index;
do {
if (sh == cr->shape[i]) {
if (sh == cr->shape[i] && prototype == cr->proto[i]) {
cr->prop_offset[i] = prop_offset;
goto end;
}
i = (i + 1) % countof(cr->shape);
if (unlikely(i == cr->index)) {
cr->index = (cr->index + 1) % countof(cr->shape);
break;
}
} while (i != cr->index);
js_free_shape_null(ctx->rt, cr->shape[i]);
cr->shape[i] = js_dup_shape(sh);
sh = cr->shape[i];
if (cr->watchpoint_ref[i])
js_shape_delete_watchpoints(ctx->rt, sh, cr);
cr->prop_offset[i] = prop_offset;
cr->shape[i] = js_dup_shape(object->shape);
js_free_shape_null(ctx->rt, sh);
if (prototype) {
JS_DupValue(ctx, JS_MKPTR(JS_TAG_OBJECT, prototype));
cr->proto[i] = prototype;
cr->watchpoint_ref[i] = js_shape_create_watchpoint(ctx->rt, cr->shape[i], cr, i, JS_DupAtom(ctx, atom));
}
end:
icu->offset = ch->index;
}

int ic_watchpoint_delete_handler(JSRuntime* rt, JSInlineCacheRingSlot *ref, uint8_t offset, JSAtom atom, JSInlineCacheRingSlot *target) {
if (target && ref != target)
return 1;
assert(ref->proto[offset] != NULL);
JS_FreeValueRT(rt, JS_MKPTR(JS_TAG_OBJECT, ref->proto[offset]));
JS_FreeAtomRT(rt, atom);
ref->watchpoint_ref[offset] = NULL;
ref->proto[offset] = NULL;
ref->prop_offset[offset] = 0;
ref->shape[offset] = NULL;
return 0;
}

int ic_delete_shape_proto_watchpoints(JSRuntime *rt, JSShape *shape, JSAtom atom, bool force) {
struct list_head *el, *el1;
JSObject *p;
JSInlineCacheRingSlot *cr;
JSShape *sh;
p = shape->proto;
while(likely(p)) {
if (p->shape->watchpoint)
list_for_each_safe(el, el1, p->shape->watchpoint) {
JSInlineCacheWatchpoint *o = list_entry(el, JSInlineCacheWatchpoint, link);
if (force || o->atom == atom) {
cr = (JSInlineCacheRingSlot *)o->ref;
sh = cr->shape[o->offset];
ic_watchpoint_delete_handler(rt, cr, o->offset, o->atom, NULL);
js_free_shape_null(rt, shape);
list_del(el);
js_free_rt(rt, o);
}
}
p = p->shape->proto;
}
return 0;
}

/* CallSite */

static void js_callsite_finalizer(JSRuntime *rt, JSValue val)
Expand Down
Loading