From 3028da4d1d91560f362626480b8308d39b4bf078 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Wed, 8 Jul 2020 22:07:49 -0700 Subject: [PATCH 01/18] libdrgn: compare language in drgn_type_eq() Signed-off-by: Omar Sandoval --- libdrgn/type.c | 1 + tests/test_type.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/libdrgn/type.c b/libdrgn/type.c index c27863a6a..b9e537efd 100644 --- a/libdrgn/type.c +++ b/libdrgn/type.c @@ -651,6 +651,7 @@ static struct drgn_error *drgn_type_eq_impl(struct drgn_type *a, (*depth)++; if (drgn_type_kind(a) != drgn_type_kind(b) || + drgn_type_language(a) != drgn_type_language(b) || drgn_type_is_complete(a) != drgn_type_is_complete(b)) goto out_false; diff --git a/tests/test_type.py b/tests/test_type.py index 31a389b4a..cdd81b666 100644 --- a/tests/test_type.py +++ b/tests/test_type.py @@ -1172,6 +1172,11 @@ def test_language(self): int_type("int", 4, True, language=Language.CPP).language, Language.CPP ) + self.assertNotEqual( + int_type("int", 4, True, language=Language.C), + int_type("int", 4, True, language=Language.CPP), + ) + def test_cmp(self): self.assertEqual(void_type(), void_type()) self.assertEqual(void_type(Qualifiers.CONST), void_type(Qualifiers.CONST)) From f1eaf5b14cbe76fa2d71f54ed1185fc7e0047a66 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Fri, 10 Jul 2020 15:58:40 -0700 Subject: [PATCH 02/18] libdrgn: add load_debug_info example program Really it's more of a test program than an example program. It's useful for benchmarking, testing with valgrind, etc. It's not built by default, but it can be built manually with: $ make -C build/temp.* examples/load_debug_info And run with: $ ./build/temp.*/examples/load_debug_info Signed-off-by: Omar Sandoval --- libdrgn/Makefile.am | 7 ++- libdrgn/examples/load_debug_info.c | 90 ++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 libdrgn/examples/load_debug_info.c diff --git a/libdrgn/Makefile.am b/libdrgn/Makefile.am index 74a0128ad..426175991 100644 --- a/libdrgn/Makefile.am +++ b/libdrgn/Makefile.am @@ -84,7 +84,7 @@ drgn.h: configure.ac build-aux/gen_drgn_h.awk build-aux/parse_arch.awk $(ARCH_IN -v version=@PACKAGE_VERSION@ \ $(wordlist 4, $(words $^), $^) > $@ -elfutils_LIBS = elfutils/libdw/libdw.a elfutils/libelf/libelf.a -lz -llzma -lbz2 +elfutils_LIBS = elfutils/libdw/libdw.a elfutils/libelf/libelf.a -lz -llzma -lbz2 -ldl lib_LTLIBRARIES = libdrgn.la @@ -142,3 +142,8 @@ python/docstrings.h: ../_drgn.pyi $(drgndoc_docstrings_deps) EXTRA_DIST = $(ARCH_INS) build-aux/gen_arch.awk build-aux/gen_constants.py \ build-aux/gen_drgn_h.awk build-aux/parse_arch.awk + +EXTRA_PROGRAMS = examples/load_debug_info + +examples_load_debug_info_SOURCES = examples/load_debug_info.c +examples_load_debug_info_LDADD = libdrgnimpl.la $(elfutils_LIBS) diff --git a/libdrgn/examples/load_debug_info.c b/libdrgn/examples/load_debug_info.c new file mode 100644 index 000000000..247c1ebae --- /dev/null +++ b/libdrgn/examples/load_debug_info.c @@ -0,0 +1,90 @@ +#include +#include +#include +#include + +#include "drgn.h" + +static void usage(bool error) +{ + fprintf(error ? stderr : stdout, + "usage: load_debug_info [-k|-c CORE|-p PID]\n" + "\n" + "Example libdrgn program that loads default debug information\n" + "\n" + "Options:\n" + " -k, --kernel debug the running kernel (default)\n" + " -c PATH, --core PATH debug the given core dump\n" + " -p PID, --pid PID debug the running process with the given PID\n" + " -h, --help display this help message and exit\n"); + exit(error ? EXIT_FAILURE : EXIT_SUCCESS); +} + +int main(int argc, char **argv) +{ + struct option long_options[] = { + {"kernel", no_argument, NULL, 'k'}, + {"core", required_argument, NULL, 'c'}, + {"pid", required_argument, NULL, 'p'}, + {"help", no_argument, NULL, 'h'}, + {}, + }; + bool kernel = false; + const char *core = NULL; + const char *pid = NULL; + for (;;) { + int c = getopt_long(argc, argv, "kc:p:h", long_options, NULL); + if (c == -1) + break; + switch (c) { + case 'k': + kernel = true; + break; + case 'c': + core = optarg; + break; + case 'p': + pid = optarg; + break; + case 'h': + usage(false); + default: + usage(true); + } + } + if (optind != argc || kernel + !!core + !!pid > 1) + usage(true); + + struct drgn_program *prog; + struct drgn_error *err = drgn_program_create(NULL, &prog); + if (err) { + prog = NULL; + goto out; + } + + if (core) + err = drgn_program_set_core_dump(prog, core); + else if (pid) + err = drgn_program_set_pid(prog, atoi(pid) ?: getpid()); + else + err = drgn_program_set_kernel(prog); + if (err) + goto out; + + err = drgn_program_load_debug_info(prog, NULL, 0, true, true); + +out:; + int status; + if (err) { + if (err->code == DRGN_ERROR_MISSING_DEBUG_INFO) + status = EXIT_SUCCESS; + else + status = EXIT_FAILURE; + drgn_error_fwrite(stderr, err); + drgn_error_destroy(err); + } else { + status = EXIT_SUCCESS; + } + drgn_program_destroy(prog); + return status; +} From c840072d05b4f2935916086b188ad9ef14fdf603 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 13 Jul 2020 10:25:03 -0700 Subject: [PATCH 03/18] libdrgn: make drgn_object_set_buffer() take a void * It's awkward to make callers cast to char *. Signed-off-by: Omar Sandoval --- libdrgn/drgn.h.in | 2 +- libdrgn/object.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libdrgn/drgn.h.in b/libdrgn/drgn.h.in index cc435c46a..9c3804e95 100644 --- a/libdrgn/drgn.h.in +++ b/libdrgn/drgn.h.in @@ -1807,7 +1807,7 @@ drgn_object_set_float(struct drgn_object *res, struct drgn_error * drgn_object_set_buffer(struct drgn_object *res, struct drgn_qualified_type qualified_type, - const char *buf, uint8_t bit_offset, + const void *buf, uint8_t bit_offset, uint64_t bit_field_size, enum drgn_byte_order byte_order); diff --git a/libdrgn/object.c b/libdrgn/object.c index c35d9932c..950d68ba6 100644 --- a/libdrgn/object.c +++ b/libdrgn/object.c @@ -244,7 +244,7 @@ struct drgn_error *sanity_check_object(enum drgn_object_kind kind, } } -static void drgn_value_deserialize(union drgn_value *value, const char *buf, +static void drgn_value_deserialize(union drgn_value *value, const void *buf, uint8_t bit_offset, enum drgn_object_kind kind, uint64_t bit_size, bool little_endian) @@ -300,7 +300,7 @@ static struct drgn_error * drgn_object_set_buffer_internal(struct drgn_object *res, const struct drgn_object_type *type, enum drgn_object_kind kind, uint64_t bit_size, - const char *buf, uint8_t bit_offset, + const void *buf, uint8_t bit_offset, bool little_endian) { struct drgn_error *err; @@ -352,7 +352,7 @@ drgn_object_set_buffer_internal(struct drgn_object *res, LIBDRGN_PUBLIC struct drgn_error * drgn_object_set_buffer(struct drgn_object *res, struct drgn_qualified_type qualified_type, - const char *buf, uint8_t bit_offset, + const void *buf, uint8_t bit_offset, uint64_t bit_field_size, enum drgn_byte_order byte_order) { struct drgn_error *err; From 213c148ce6ba14b830b7c466724d8317e2355583 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 13 Jul 2020 13:30:36 -0700 Subject: [PATCH 04/18] libdrgn: dwarf_info_cache: handle DW_AT_endianity Variables can have a non-default endianity. Handle it and clean up variable endian handling. Signed-off-by: Omar Sandoval --- libdrgn/dwarf_info_cache.c | 92 +++++++++++++++++++++++++++++++------- libdrgn/internal.h | 39 +--------------- libdrgn/object.h | 7 +-- 3 files changed, 79 insertions(+), 59 deletions(-) diff --git a/libdrgn/dwarf_info_cache.c b/libdrgn/dwarf_info_cache.c index 51b1f8a3c..746eb435c 100644 --- a/libdrgn/dwarf_info_cache.c +++ b/libdrgn/dwarf_info_cache.c @@ -58,6 +58,63 @@ static void drgn_dwarf_type_free(struct drgn_dwarf_type *dwarf_type) } } +/** + * Return whether a DWARF DIE is little-endian. + * + * @param[in] check_attr Whether to check the DW_AT_endianity attribute. If @c + * false, only the ELF header is checked and this function cannot fail. + * @return @c NULL on success, non-@c NULL on error. + */ +static struct drgn_error *dwarf_die_is_little_endian(Dwarf_Die *die, + bool check_attr, bool *ret) +{ + Dwarf_Attribute endianity_attr_mem, *endianity_attr; + Dwarf_Word endianity; + if (check_attr && + (endianity_attr = dwarf_attr_integrate(die, DW_AT_endianity, + &endianity_attr_mem))) { + if (dwarf_formudata(endianity_attr, &endianity)) { + return drgn_error_create(DRGN_ERROR_OTHER, + "invalid DW_AT_endianity"); + } + } else { + endianity = DW_END_default; + } + switch (endianity) { + case DW_END_default: { + Elf *elf = dwarf_getelf(dwarf_cu_getdwarf(die->cu)); + *ret = elf_getident(elf, NULL)[EI_DATA] == ELFDATA2LSB; + return NULL; + } + case DW_END_little: + *ret = true; + return NULL; + case DW_END_big: + *ret = false; + return NULL; + default: + return drgn_error_create(DRGN_ERROR_OTHER, + "unknown DW_AT_endianity"); + } +} + +/** Like dwarf_die_is_little_endian(), but returns a @ref drgn_byte_order. */ +static struct drgn_error *dwarf_die_byte_order(Dwarf_Die *die, + bool check_attr, + enum drgn_byte_order *ret) +{ + bool little_endian; + struct drgn_error *err = dwarf_die_is_little_endian(die, check_attr, + &little_endian); + /* + * dwarf_die_is_little_endian() can't fail if check_attr is false, so + * the !check_attr test suppresses maybe-uninitialized warnings. + */ + if (!err || !check_attr) + *ret = little_endian ? DRGN_LITTLE_ENDIAN : DRGN_BIG_ENDIAN; + return err; +} + static int dwarf_type(Dwarf_Die *die, Dwarf_Die *ret) { Dwarf_Attribute attr_mem; @@ -618,7 +675,8 @@ drgn_compound_type_from_dwarf(struct drgn_dwarf_info_cache *dicache, goto err; } - bool little_endian = dwarf_die_is_little_endian(die); + bool little_endian; + dwarf_die_is_little_endian(die, false, &little_endian); Dwarf_Die child; int r = dwarf_child(die, &child); while (r == 0) { @@ -1453,20 +1511,21 @@ drgn_object_from_dwarf_subprogram(struct drgn_dwarf_info_cache *dicache, Dwarf_Die *die, uint64_t bias, const char *name, struct drgn_object *ret) { - struct drgn_error *err; struct drgn_qualified_type qualified_type; - Dwarf_Addr low_pc; - - err = drgn_type_from_dwarf(dicache, die, &qualified_type); + struct drgn_error *err = drgn_type_from_dwarf(dicache, die, + &qualified_type); if (err) return err; + Dwarf_Addr low_pc; if (dwarf_lowpc(die, &low_pc) == -1) { return drgn_error_format(DRGN_ERROR_LOOKUP, "could not find address of '%s'", name); } + enum drgn_byte_order byte_order; + dwarf_die_byte_order(die, false, &byte_order); return drgn_object_set_reference(ret, qualified_type, low_pc + bias, 0, - 0, dwarf_die_byte_order(die)); + 0, byte_order); } static struct drgn_error * @@ -1474,31 +1533,34 @@ drgn_object_from_dwarf_variable(struct drgn_dwarf_info_cache *dicache, Dwarf_Die *die, uint64_t bias, const char *name, struct drgn_object *ret) { - struct drgn_error *err; struct drgn_qualified_type qualified_type; - Dwarf_Attribute attr_mem; - Dwarf_Attribute *attr; - Dwarf_Op *loc; - size_t nloc; - - err = drgn_type_from_dwarf_child(dicache, die, NULL, "DW_TAG_variable", - true, true, NULL, &qualified_type); + struct drgn_error *err = drgn_type_from_dwarf_child(dicache, die, NULL, + "DW_TAG_variable", + true, true, NULL, + &qualified_type); if (err) return err; + Dwarf_Attribute attr_mem, *attr; if (!(attr = dwarf_attr_integrate(die, DW_AT_location, &attr_mem))) { return drgn_error_format(DRGN_ERROR_LOOKUP, "could not find address of '%s'", name); } + Dwarf_Op *loc; + size_t nloc; if (dwarf_getlocation(attr, &loc, &nloc)) return drgn_error_libdw(); if (nloc != 1 || loc[0].atom != DW_OP_addr) { return drgn_error_create(DRGN_ERROR_OTHER, "DW_AT_location has unimplemented operation"); } + enum drgn_byte_order byte_order; + err = dwarf_die_byte_order(die, true, &byte_order); + if (err) + return err; return drgn_object_set_reference(ret, qualified_type, loc[0].number + bias, 0, 0, - dwarf_die_byte_order(die)); + byte_order); } struct drgn_error * diff --git a/libdrgn/internal.h b/libdrgn/internal.h index 84fc814d2..0c4d37d1b 100644 --- a/libdrgn/internal.h +++ b/libdrgn/internal.h @@ -42,46 +42,9 @@ struct drgn_error *read_elf_section(Elf_Scn *scn, Elf_Data **ret); struct drgn_error *elf_address_range(Elf *elf, uint64_t bias, uint64_t *start_ret, uint64_t *end_ret); -/** - * Return the word size of a program based on an ELF file. - * - * Note that this looks at the ELF header rather than determining this based on - * machine type, but the file format @e should correspond to the architecture - * word size. - */ -static inline uint8_t elf_word_size(Elf *elf) -{ - return elf_getident(elf, NULL)[EI_CLASS] == ELFCLASS64 ? 8 : 4; -} - -/** - * Return the endianness of a program based on an ELF file. - * - * Like @ref elf_word_size(), this only looks at the ELF header. - */ -static inline bool elf_is_little_endian(Elf *elf) -{ - return elf_getident(elf, NULL)[EI_DATA] == ELFDATA2LSB; -} - -static inline bool dwarf_die_is_little_endian(Dwarf_Die *die) -{ - return elf_is_little_endian(dwarf_getelf(dwarf_cu_getdwarf(die->cu))); -} - -static inline enum drgn_byte_order dwarf_die_byte_order(Dwarf_Die *die) -{ - return (dwarf_die_is_little_endian(die) ? - DRGN_LITTLE_ENDIAN : DRGN_BIG_ENDIAN); -} - bool die_matches_filename(Dwarf_Die *die, const char *filename); -/** - * Path iterator input component. - * - * - */ +/** Path iterator input component. */ struct path_iterator_component { /** * Path component. diff --git a/libdrgn/object.h b/libdrgn/object.h index ca9b3c580..c0a44ada5 100644 --- a/libdrgn/object.h +++ b/libdrgn/object.h @@ -129,12 +129,7 @@ struct drgn_error *sanity_check_object(enum drgn_object_kind kind, uint64_t bit_field_size, uint64_t bit_size); -/** - * Convert a @ref drgn_byte_order to a boolean. - * - * @return @c true if the byte order is little endian, @c false if the byte - * order is big endian. - */ +/** Convert a @ref drgn_byte_order to a boolean. */ struct drgn_error * drgn_byte_order_to_little_endian(struct drgn_program *prog, enum drgn_byte_order byte_order, bool *ret); From 6d4af7e17efa90c963f72cba60a73e982ba051b6 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 13 Jul 2020 15:21:55 -0700 Subject: [PATCH 05/18] libdrgn: dwarf_info_cache: handle variables DW_AT_const_value Compile-time constants have DW_AT_const_value instead of DW_AT_location. We can translate those to a value object. Signed-off-by: Omar Sandoval --- libdrgn/dwarf_info_cache.c | 90 ++++++++++++++++++++++++++------ libdrgn/object.c | 6 +-- libdrgn/object.h | 29 +++++++++++ tests/dwarfwriter.py | 9 ++++ tests/test_dwarf.py | 104 ++++++++++++++++++++++++++++++++++++- 5 files changed, 217 insertions(+), 21 deletions(-) diff --git a/libdrgn/dwarf_info_cache.c b/libdrgn/dwarf_info_cache.c index 746eb435c..536d99a2e 100644 --- a/libdrgn/dwarf_info_cache.c +++ b/libdrgn/dwarf_info_cache.c @@ -10,6 +10,7 @@ #include "dwarf_index.h" #include "dwarf_info_cache.h" #include "hash_table.h" +#include "object.h" #include "object_index.h" #include "type_index.h" #include "vector.h" @@ -1528,6 +1529,55 @@ drgn_object_from_dwarf_subprogram(struct drgn_dwarf_info_cache *dicache, 0, byte_order); } +static struct drgn_error * +drgn_object_from_dwarf_constant(struct drgn_dwarf_info_cache *dicache, + Dwarf_Die *die, + struct drgn_qualified_type qualified_type, + Dwarf_Attribute *attr, struct drgn_object *ret) +{ + struct drgn_object_type type; + enum drgn_object_kind kind; + uint64_t bit_size; + struct drgn_error *err = drgn_object_set_common(qualified_type, 0, + &type, &kind, + &bit_size); + if (err) + return err; + Dwarf_Block block; + if (dwarf_formblock(attr, &block) == 0) { + bool little_endian; + err = dwarf_die_is_little_endian(die, true, &little_endian); + if (err) + return err; + if (block.length < drgn_value_size(bit_size, 0)) { + return drgn_error_create(DRGN_ERROR_OTHER, + "DW_AT_const_value block is too small"); + } + return drgn_object_set_buffer_internal(ret, &type, kind, + bit_size, block.data, 0, + little_endian); + } else if (kind == DRGN_OBJECT_SIGNED) { + Dwarf_Sword svalue; + if (dwarf_formsdata(attr, &svalue)) { + return drgn_error_create(DRGN_ERROR_OTHER, + "invalid DW_AT_const_value"); + } + return drgn_object_set_signed_internal(ret, &type, bit_size, + svalue); + } else if (kind == DRGN_OBJECT_UNSIGNED) { + Dwarf_Word uvalue; + if (dwarf_formudata(attr, &uvalue)) { + return drgn_error_create(DRGN_ERROR_OTHER, + "invalid DW_AT_const_value"); + } + return drgn_object_set_unsigned_internal(ret, &type, bit_size, + uvalue); + } else { + return drgn_error_create(DRGN_ERROR_OTHER, + "unknown DW_AT_const_value form"); + } +} + static struct drgn_error * drgn_object_from_dwarf_variable(struct drgn_dwarf_info_cache *dicache, Dwarf_Die *die, uint64_t bias, const char *name, @@ -1541,26 +1591,32 @@ drgn_object_from_dwarf_variable(struct drgn_dwarf_info_cache *dicache, if (err) return err; Dwarf_Attribute attr_mem, *attr; - if (!(attr = dwarf_attr_integrate(die, DW_AT_location, &attr_mem))) { + if ((attr = dwarf_attr_integrate(die, DW_AT_location, &attr_mem))) { + Dwarf_Op *loc; + size_t nloc; + if (dwarf_getlocation(attr, &loc, &nloc)) + return drgn_error_libdw(); + if (nloc != 1 || loc[0].atom != DW_OP_addr) { + return drgn_error_create(DRGN_ERROR_OTHER, + "DW_AT_location has unimplemented operation"); + } + enum drgn_byte_order byte_order; + err = dwarf_die_byte_order(die, true, &byte_order); + if (err) + return err; + return drgn_object_set_reference(ret, qualified_type, + loc[0].number + bias, 0, 0, + byte_order); + } else if ((attr = dwarf_attr_integrate(die, DW_AT_const_value, + &attr_mem))) { + return drgn_object_from_dwarf_constant(dicache, die, + qualified_type, attr, + ret); + } else { return drgn_error_format(DRGN_ERROR_LOOKUP, - "could not find address of '%s'", + "could not find address or value of '%s'", name); } - Dwarf_Op *loc; - size_t nloc; - if (dwarf_getlocation(attr, &loc, &nloc)) - return drgn_error_libdw(); - if (nloc != 1 || loc[0].atom != DW_OP_addr) { - return drgn_error_create(DRGN_ERROR_OTHER, - "DW_AT_location has unimplemented operation"); - } - enum drgn_byte_order byte_order; - err = dwarf_die_byte_order(die, true, &byte_order); - if (err) - return err; - return drgn_object_set_reference(ret, qualified_type, - loc[0].number + bias, 0, 0, - byte_order); } struct drgn_error * diff --git a/libdrgn/object.c b/libdrgn/object.c index 950d68ba6..4f0d57c02 100644 --- a/libdrgn/object.c +++ b/libdrgn/object.c @@ -104,7 +104,7 @@ drgn_object_set_common(struct drgn_qualified_type qualified_type, return drgn_object_type_kind_and_size(type_ret, kind_ret, bit_size_ret); } -static struct drgn_error * +struct drgn_error * drgn_object_set_signed_internal(struct drgn_object *res, const struct drgn_object_type *type, uint64_t bit_size, int64_t svalue) @@ -140,7 +140,7 @@ drgn_object_set_signed(struct drgn_object *res, return drgn_object_set_signed_internal(res, &type, bit_size, svalue); } -static struct drgn_error * +struct drgn_error * drgn_object_set_unsigned_internal(struct drgn_object *res, const struct drgn_object_type *type, uint64_t bit_size, uint64_t uvalue) @@ -296,7 +296,7 @@ drgn_byte_order_to_little_endian(struct drgn_program *prog, } } -static struct drgn_error * +struct drgn_error * drgn_object_set_buffer_internal(struct drgn_object *res, const struct drgn_object_type *type, enum drgn_object_kind kind, uint64_t bit_size, diff --git a/libdrgn/object.h b/libdrgn/object.h index c0a44ada5..6e6258c1b 100644 --- a/libdrgn/object.h +++ b/libdrgn/object.h @@ -129,6 +129,35 @@ struct drgn_error *sanity_check_object(enum drgn_object_kind kind, uint64_t bit_field_size, uint64_t bit_size); +/** + * Like @ref drgn_object_set_signed() but @ref drgn_object_set_common() was + * already called. + */ +struct drgn_error * +drgn_object_set_signed_internal(struct drgn_object *res, + const struct drgn_object_type *type, + uint64_t bit_size, int64_t svalue); + +/** + * Like @ref drgn_object_set_unsigned() but @ref drgn_object_set_common() was + * already called. + */ +struct drgn_error * +drgn_object_set_unsigned_internal(struct drgn_object *res, + const struct drgn_object_type *type, + uint64_t bit_size, uint64_t uvalue); + +/** + * Like @ref drgn_object_set_buffer() but @ref drgn_object_set_common() was + * already called. + */ +struct drgn_error * +drgn_object_set_buffer_internal(struct drgn_object *res, + const struct drgn_object_type *type, + enum drgn_object_kind kind, uint64_t bit_size, + const void *buf, uint8_t bit_offset, + bool little_endian); + /** Convert a @ref drgn_byte_order to a boolean. */ struct drgn_error * drgn_byte_order_to_little_endian(struct drgn_program *prog, diff --git a/tests/dwarfwriter.py b/tests/dwarfwriter.py index 8073d0ed9..268a7ba44 100644 --- a/tests/dwarfwriter.py +++ b/tests/dwarfwriter.py @@ -90,10 +90,19 @@ def aux(die, depth): buf.extend(value.to_bytes(bits // 8, byteorder)) elif attrib.form == DW_FORM.data1: buf.append(value) + elif attrib.form == DW_FORM.data2: + buf.extend(value.to_bytes(2, byteorder)) + elif attrib.form == DW_FORM.data4: + buf.extend(value.to_bytes(4, byteorder)) + elif attrib.form == DW_FORM.data8: + buf.extend(value.to_bytes(8, byteorder)) elif attrib.form == DW_FORM.udata: _append_uleb128(buf, value) elif attrib.form == DW_FORM.sdata: _append_sleb128(buf, value) + elif attrib.form == DW_FORM.block1: + buf.append(len(value)) + buf.extend(value) elif attrib.form == DW_FORM.string: buf.extend(value.encode()) buf.append(0) diff --git a/tests/test_dwarf.py b/tests/test_dwarf.py index 27527bdb5..ceecb5039 100644 --- a/tests/test_dwarf.py +++ b/tests/test_dwarf.py @@ -2033,12 +2033,114 @@ def test_variable(self): del dies[1].attribs[2] prog = dwarf_program(dies) - self.assertRaisesRegex(LookupError, "could not find address", prog.object, "x") + self.assertRaisesRegex( + LookupError, "could not find address or value", prog.object, "x" + ) dies[1].attribs.insert(2, DwarfAttrib(DW_AT.location, DW_FORM.exprloc, b"\xe0")) prog = dwarf_program(dies) self.assertRaisesRegex(Exception, "unimplemented operation", prog.object, "x") + def test_const_signed(self): + for form in ( + DW_FORM.data1, + DW_FORM.data2, + DW_FORM.data4, + DW_FORM.data8, + DW_FORM.sdata, + ): + dies = [ + int_die, + DwarfDie( + DW_TAG.variable, + [ + DwarfAttrib(DW_AT.name, DW_FORM.string, "x"), + DwarfAttrib(DW_AT.type, DW_FORM.ref4, 0), + DwarfAttrib(DW_AT.const_value, form, 1,), + ], + ), + ] + prog = dwarf_program(dies) + self.assertEqual( + prog["x"], Object(prog, int_type("int", 4, True), 1), + ) + + def test_const_unsigned(self): + for form in ( + DW_FORM.data1, + DW_FORM.data2, + DW_FORM.data4, + DW_FORM.data8, + DW_FORM.udata, + ): + dies = [ + unsigned_int_die, + DwarfDie( + DW_TAG.variable, + [ + DwarfAttrib(DW_AT.name, DW_FORM.string, "x"), + DwarfAttrib(DW_AT.type, DW_FORM.ref4, 0), + DwarfAttrib(DW_AT.const_value, form, 1), + ], + ), + ] + prog = dwarf_program(dies) + self.assertEqual( + prog["x"], Object(prog, int_type("unsigned int", 4, False), 1), + ) + + def test_const_block(self): + dies = [ + int_die, + DwarfDie( + DW_TAG.structure_type, + [ + DwarfAttrib(DW_AT.name, DW_FORM.string, "point"), + DwarfAttrib(DW_AT.byte_size, DW_FORM.data1, 8), + ], + [ + DwarfDie( + DW_TAG.member, + [ + DwarfAttrib(DW_AT.name, DW_FORM.string, "x"), + DwarfAttrib(DW_AT.data_member_location, DW_FORM.data1, 0), + DwarfAttrib(DW_AT.type, DW_FORM.ref4, 0), + ], + ), + DwarfDie( + DW_TAG.member, + [ + DwarfAttrib(DW_AT.name, DW_FORM.string, "y"), + DwarfAttrib(DW_AT.data_member_location, DW_FORM.data1, 4), + DwarfAttrib(DW_AT.type, DW_FORM.ref4, 0), + ], + ), + ], + ), + DwarfDie( + DW_TAG.variable, + [ + DwarfAttrib(DW_AT.name, DW_FORM.string, "p"), + DwarfAttrib(DW_AT.type, DW_FORM.ref4, 1), + DwarfAttrib( + DW_AT.const_value, + DW_FORM.block1, + b"\x01\x00\x00\x00\x02\x00\x00\x00", + ), + ], + ), + ] + prog = dwarf_program(dies) + self.assertEqual( + prog["p"], Object(prog, point_type, {"x": 1, "y": 2}), + ) + + dies[2].attribs[2] = DwarfAttrib( + DW_AT.const_value, DW_FORM.block1, b"\x01\x00\x00\x00\x02\x00\x00", + ) + prog = dwarf_program(dies) + self.assertRaisesRegex(Exception, "too small", prog.variable, "p") + def test_not_found(self): prog = dwarf_program([int_die]) self.assertRaisesRegex(LookupError, "could not find", prog.object, "y") From 209eaee4858b0abad9d97241226fdddefc4caafb Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Sat, 18 Jul 2020 01:36:50 -0700 Subject: [PATCH 06/18] setup.py: import setuptools before distutils setuptools recently started warning if distutils is imported before it. Signed-off-by: Omar Sandoval --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 15fc4a53d..f3c9ee6cf 100755 --- a/setup.py +++ b/setup.py @@ -2,6 +2,9 @@ # Copyright (c) Facebook, Inc. and its affiliates. # SPDX-License-Identifier: GPL-3.0+ +# setuptools must be imported before distutils (see pypa/setuptools#2230). +from setuptools import setup, find_packages, Command + import contextlib from distutils import log from distutils.command.build import build as _build @@ -12,7 +15,6 @@ import os.path import re import pkg_resources -from setuptools import setup, find_packages, Command from setuptools.command.build_ext import build_ext as _build_ext from setuptools.command.egg_info import egg_info as _egg_info from setuptools.extension import Extension From 2409868409501be7dc6a8d4f5b7faf6972be1c5a Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Fri, 17 Jul 2020 12:06:14 -0700 Subject: [PATCH 07/18] libdrgn: hash_table: define chunk alignment constant Signed-off-by: Omar Sandoval --- libdrgn/hash_table.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libdrgn/hash_table.h b/libdrgn/hash_table.h index ccfbfac63..5f752e9dc 100644 --- a/libdrgn/hash_table.h +++ b/libdrgn/hash_table.h @@ -18,6 +18,7 @@ #ifdef __SSE4_2__ #include #endif +#include #include #include #include @@ -300,6 +301,11 @@ struct hash_table_iterator hash_table_first(struct hash_table *table); struct hash_table_iterator hash_table_next(struct hash_table_iterator it); #endif +enum { + hash_table_chunk_alignment = + alignof(max_align_t) > 16 ? alignof(max_align_t) : 16, +}; + static inline size_t hash_table_probe_delta(struct hash_pair hp) { return 2 * hp.second + 1; @@ -421,7 +427,7 @@ struct table##_chunk { \ */ \ uint8_t outbound_overflow_count; \ table##_entry_type entries[table##_chunk_allocated_capacity]; \ -} __attribute__((aligned(16))); \ +} __attribute__((aligned(hash_table_chunk_alignment))); \ \ struct table##_iterator { \ table##_entry_type *entry; \ @@ -627,7 +633,8 @@ static bool table##_rehash(struct table *table, size_t new_chunk_count, \ * aligned_alloc() requires that the allocation size is aligned to the \ * allocation alignment. \ */ \ - table->chunks = aligned_alloc(16, (alloc_size + 0xf) & ~(size_t)0xf); \ + table->chunks = aligned_alloc(hash_table_chunk_alignment, \ + (alloc_size + 0xf) & ~(size_t)0xf); \ if (!table->chunks) \ goto err; \ memset(table->chunks, 0, alloc_size); \ From 2eab47ce9e81d0d9781c5add687084d2932275ec Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Jul 2020 16:44:42 -0700 Subject: [PATCH 08/18] libdrgn: hash_table: use posix_memalign() instead of aligned_alloc() posix_memalign() doesn't have the restriction that the size must be a multiple of the alignment like aligned_alloc() does in C11. Signed-off-by: Omar Sandoval --- libdrgn/hash_table.h | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/libdrgn/hash_table.h b/libdrgn/hash_table.h index 5f752e9dc..b2889bd96 100644 --- a/libdrgn/hash_table.h +++ b/libdrgn/hash_table.h @@ -628,15 +628,10 @@ static bool table##_rehash(struct table *table, size_t new_chunk_count, \ size_t orig_chunk_mask = table->chunk_mask; \ size_t orig_chunk_count = orig_chunk_mask + 1; \ size_t alloc_size = table##_alloc_size(new_chunk_count, new_max_size); \ - \ - /* \ - * aligned_alloc() requires that the allocation size is aligned to the \ - * allocation alignment. \ - */ \ - table->chunks = aligned_alloc(hash_table_chunk_alignment, \ - (alloc_size + 0xf) & ~(size_t)0xf); \ - if (!table->chunks) \ - goto err; \ + void *new_chunks; \ + if (posix_memalign(&new_chunks, hash_table_chunk_alignment, alloc_size))\ + return false; \ + table->chunks = new_chunks; \ memset(table->chunks, 0, alloc_size); \ table->chunks[0].chunk0_capacity = \ new_chunk_count == 1 ? new_max_size : 1; \ From 9ea11a7c260d865e435d2f46e3e05661ba211f71 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Jul 2020 15:21:37 -0700 Subject: [PATCH 09/18] libdrgn: hash_table: port reserve optimization The only major change to the folly F14 implementation since I originally ported it is commit 3d169f4365cf ("memory savings for F14 tables with explicit reserve()"). That is a small improvement for small tables and a large improvement for vector tables, which are about to be added. Signed-off-by: Omar Sandoval --- libdrgn/hash_table.h | 372 ++++++++++++++++++++++++++----------------- 1 file changed, 230 insertions(+), 142 deletions(-) diff --git a/libdrgn/hash_table.h b/libdrgn/hash_table.h index b2889bd96..37398377c 100644 --- a/libdrgn/hash_table.h +++ b/libdrgn/hash_table.h @@ -311,6 +311,9 @@ static inline size_t hash_table_probe_delta(struct hash_pair hp) return 2 * hp.second + 1; } +static const uint8_t hosted_overflow_count_inc = 0x10; +static const uint8_t hosted_overflow_count_dec = -0x10; + /* * We could represent an empty hash table with chunks set to NULL. However, then * we would need a branch to check for this in insert, search, and delete. We @@ -401,25 +404,26 @@ enum { \ table##_chunk_allocated_capacity = \ (table##_chunk_capacity + \ (sizeof(table##_entry_type) == 16 ? 1 : 0)), \ + /* \ + * If the chunk capacity is 12, we can use tags 12 and 13 for 16 bits. \ + * Otherwise, we only get 4 from control. \ + */ \ + table##_capacity_scale_bits = table##_chunk_capacity == 12 ? 16 : 4, \ + table##_capacity_scale_shift = table##_capacity_scale_bits - 4, \ table##_chunk_full_mask = (1 << table##_chunk_capacity) - 1, \ }; \ \ struct table##_chunk { \ uint8_t tags[14]; \ /* \ - * If this is the first chunk, the capacity of the table if it is also \ - * the only chunk, and one otherwise. Zero if this is not the first \ - * chunk. \ - */ \ - uint8_t chunk0_capacity : 4; \ - /* \ - * The number of entries in this chunk that overflowed their desired \ - * chunk. \ + * The lower 4 bits are capacity_scale: for the first chunk, this is \ + * the scaling factor between the chunk count and the capacity; for \ + * other chunks, this is zero. \ * \ - * Note that this bit field and chunk0_capacity are combined into a \ - * single uint8_t member named "control" in the folly implementation. \ + * The upper 4 bits are hosted_overflow_count: the number of entries in \ + * this chunk that overflowed their desired chunk. \ */ \ - uint8_t hosted_overflow_count : 4; \ + uint8_t control; \ /* \ * The number of entries that would have been in this chunk if it were \ * not full. This value saturates if it hits 255, after which it will \ @@ -442,7 +446,7 @@ struct table { \ size_t size; \ /* Cached first iterator. */ \ uintptr_t first_packed; \ -}; \ +}; /** * Define the functions for a hash table. @@ -528,6 +532,58 @@ table##_chunk_last_occupied(struct table##_chunk *chunk) \ return mask ? fls(mask) - 1 : (unsigned int)-1; \ } \ \ +static inline size_t \ +table##_chunk_hosted_overflow_count(struct table##_chunk *chunk) \ +{ \ + return chunk->control >> 4; \ +} \ + \ +static inline void \ +table##_chunk_adjust_hosted_overflow_count(struct table##_chunk *chunk, \ + size_t op) \ +{ \ + chunk->control += op; \ +} \ + \ +static inline size_t table##_chunk_capacity_scale(struct table##_chunk *chunk) \ +{ \ + if (table##_capacity_scale_bits == 4) { \ + return chunk->control & 0xf; \ + } else { \ + uint16_t val; \ + memcpy(&val, &chunk->tags[12], 2); \ + return val; \ + } \ +} \ + \ +static inline void \ +table##_chunk_set_capacity_scale(struct table##_chunk *chunk, \ + size_t capacity_scale) \ +{ \ + if (table##_capacity_scale_bits == 4) { \ + chunk->control = (chunk->control & ~0xf) | capacity_scale; \ + } else { \ + uint16_t val = capacity_scale; \ + memcpy(&chunk->tags[12], &val, 2); \ + } \ +} \ + \ +static inline bool table##_chunk_eof(struct table##_chunk *chunk) \ +{ \ + return table##_chunk_capacity_scale(chunk) != 0; \ +} \ + \ +static inline void table##_chunk_mark_eof(struct table##_chunk *chunk, \ + size_t capacity_scale) \ +{ \ + if (table##_capacity_scale_bits == 4) { \ + chunk->control = capacity_scale; \ + } else { \ + uint16_t val = capacity_scale; \ + memcpy(&chunk->tags[12], &val, 2); \ + } \ +} \ + \ static inline void \ table##_chunk_inc_outbound_overflow_count(struct table##_chunk *chunk) \ { \ @@ -574,24 +630,22 @@ static table##_entry_type *table##_allocate_tag(struct table *table, \ uint8_t *fullness, \ struct hash_pair hp) \ { \ - struct table##_chunk *chunk; \ + const size_t delta = hash_table_probe_delta(hp); \ size_t index = hp.first; \ - size_t delta = hash_table_probe_delta(hp); \ - uint8_t hosted_inc = 0; \ - size_t entry_index; \ - \ + struct table##_chunk *chunk; \ + uint8_t hosted_op = 0; \ for (;;) { \ index &= table->chunk_mask; \ chunk = &table->chunks[index]; \ if (likely(fullness[index] < table##_chunk_capacity)) \ break; \ table##_chunk_inc_outbound_overflow_count(chunk); \ - hosted_inc = 1; \ + hosted_op = hosted_overflow_count_inc; \ index += delta; \ } \ - entry_index = fullness[index]++; \ + size_t entry_index = fullness[index]++; \ chunk->tags[entry_index] = hp.second; \ - chunk->hosted_overflow_count += hosted_inc; \ + table##_chunk_adjust_hosted_overflow_count(chunk, hosted_op); \ return &chunk->entries[entry_index]; \ } \ \ @@ -607,7 +661,44 @@ static void table##_set_first_packed_after_rehash(struct table *table, \ fullness[i] - 1); \ } \ \ -static inline size_t table##_alloc_size(size_t chunk_count, size_t max_size) \ +static size_t table##_compute_capacity(size_t chunk_count, size_t scale) \ +{ \ + return (((chunk_count - 1) >> table##_capacity_scale_shift) + 1) * scale;\ +} \ + \ +static bool \ +table##_compute_chunk_count_and_scale(size_t capacity, \ + bool continuous_single_chunk_capacity, \ + size_t *chunk_count_ret, \ + size_t *scale_ret) \ +{ \ + if (capacity <= table##_chunk_capacity) { \ + if (!continuous_single_chunk_capacity) { \ + if (capacity <= 2) \ + capacity = 2; \ + else if (capacity <= 6) \ + capacity = 6; \ + else \ + capacity = table##_chunk_capacity; \ + } \ + *chunk_count_ret = 1; \ + *scale_ret = capacity; \ + } else { \ + size_t min_chunks = \ + (capacity - 1) / table##_chunk_desired_capacity + 1; \ + size_t chunk_pow = fls(min_chunks - 1); \ + if (chunk_pow == 8 * sizeof(size_t)) \ + return false; \ + *chunk_count_ret = (size_t)1 << chunk_pow; \ + size_t ss = (chunk_pow >= table##_capacity_scale_shift ? \ + chunk_pow - table##_capacity_scale_shift : 0); \ + *scale_ret = table##_chunk_desired_capacity << (chunk_pow - ss);\ + } \ + return true; \ +} \ + \ +static inline size_t table##_alloc_size(size_t chunk_count, \ + size_t capacity_scale) \ { \ /* \ * Small hash tables are common, so for capacities of less than a full \ @@ -615,36 +706,35 @@ static inline size_t table##_alloc_size(size_t chunk_count, size_t max_size) \ */ \ if (chunk_count == 1) { \ return (offsetof(struct table##_chunk, entries) + \ - max_size * sizeof(table##_entry_type)); \ + table##_compute_capacity(1, capacity_scale) * \ + sizeof(table##_entry_type)); \ } else { \ return chunk_count * sizeof(struct table##_chunk); \ } \ } \ \ -static bool table##_rehash(struct table *table, size_t new_chunk_count, \ - size_t new_max_size) \ +static bool table##_rehash(struct table *table, size_t orig_chunk_count, \ + size_t orig_capacity_scale, size_t new_chunk_count, \ + size_t new_capacity_scale) \ { \ struct table##_chunk *orig_chunks = table->chunks; \ - size_t orig_chunk_mask = table->chunk_mask; \ - size_t orig_chunk_count = orig_chunk_mask + 1; \ - size_t alloc_size = table##_alloc_size(new_chunk_count, new_max_size); \ + size_t alloc_size = table##_alloc_size(new_chunk_count, \ + new_capacity_scale); \ + \ void *new_chunks; \ if (posix_memalign(&new_chunks, hash_table_chunk_alignment, alloc_size))\ return false; \ table->chunks = new_chunks; \ memset(table->chunks, 0, alloc_size); \ - table->chunks[0].chunk0_capacity = \ - new_chunk_count == 1 ? new_max_size : 1; \ + table##_chunk_mark_eof(table->chunks, new_capacity_scale); \ table->chunk_mask = new_chunk_count - 1; \ \ if (table->size == 0) { \ /* Nothing to do. */ \ } else if (orig_chunk_count == 1 && new_chunk_count == 1) { \ - struct table##_chunk *src, *dst; \ + struct table##_chunk *src = orig_chunks; \ + struct table##_chunk *dst = table->chunks; \ size_t src_i = 0, dst_i = 0; \ - \ - src = &orig_chunks[0]; \ - dst = &table->chunks[0]; \ while (dst_i < table->size) { \ if (likely(src->tags[src_i])) { \ dst->tags[dst_i] = src->tags[src_i]; \ @@ -657,11 +747,8 @@ static bool table##_rehash(struct table *table, size_t new_chunk_count, \ } \ table->first_packed = table##_pack_iterator(dst, dst_i - 1); \ } else { \ - struct table##_chunk *src; \ uint8_t stack_fullness[256]; \ uint8_t *fullness; \ - size_t remaining; \ - \ if (new_chunk_count <= sizeof(stack_fullness)) { \ memset(stack_fullness, 0, sizeof(stack_fullness)); \ fullness = stack_fullness; \ @@ -671,25 +758,22 @@ static bool table##_rehash(struct table *table, size_t new_chunk_count, \ goto err; \ } \ \ - src = &orig_chunks[orig_chunk_count - 1]; \ - remaining = table->size; \ + struct table##_chunk *src = &orig_chunks[orig_chunk_count - 1]; \ + size_t remaining = table->size; \ while (remaining) { \ - unsigned int mask, i; \ - \ - mask = table##_chunk_occupied(src); \ + unsigned int mask = table##_chunk_occupied(src), i; \ for_each_bit(i, mask) { \ - table##_entry_type *src_entry; \ - table##_entry_type *dst_entry; \ - table##_key_type key; \ - struct hash_pair hp; \ - \ remaining--; \ - src_entry = &src->entries[i]; \ - key = table##_entry_to_key(src_entry); \ - hp = table##_hash(&key); \ - dst_entry = table##_allocate_tag(table, \ - fullness, \ - hp); \ + \ + table##_entry_type *src_entry = \ + &src->entries[i]; \ + table##_key_type key = \ + table##_entry_to_key(src_entry); \ + struct hash_pair hp = table##_hash(&key); \ + table##_entry_type *dst_entry = \ + table##_allocate_tag(table, fullness, \ + hp); \ + \ memcpy(dst_entry, src_entry, \ sizeof(*dst_entry)); \ } \ @@ -709,84 +793,80 @@ static bool table##_rehash(struct table *table, size_t new_chunk_count, \ err: \ free(table->chunks); \ table->chunks = orig_chunks; \ - table->chunk_mask = orig_chunk_mask; \ + table->chunk_mask = orig_chunk_count - 1; \ return false; \ } \ \ -static bool table##_do_reserve(struct table *table, size_t capacity, \ - size_t orig_max_size) \ +static void table##_do_clear(struct table *table, bool reset) \ { \ - static const size_t initial_capacity = 2; \ - static const size_t half_chunk_capacity = \ - (table##_chunk_desired_capacity / 2) & ~(size_t)1; \ - size_t new_chunk_count, new_max_size; \ - \ - if (capacity <= half_chunk_capacity) { \ - new_chunk_count = 1; \ - new_max_size = (capacity < initial_capacity ? \ - initial_capacity : half_chunk_capacity); \ - } else { \ - new_chunk_count = ((capacity - 1) / \ - table##_chunk_desired_capacity + 1); \ - new_chunk_count = next_power_of_two(new_chunk_count); \ - new_max_size = (new_chunk_count * \ - table##_chunk_desired_capacity); \ - \ - if (new_chunk_count > \ - SIZE_MAX / table##_chunk_desired_capacity) \ - return false; \ - } \ - \ - if (new_max_size != orig_max_size) \ - return table##_rehash(table, new_chunk_count, new_max_size); \ - else \ - return true; \ -} \ + if (table->chunks == hash_table_empty_chunk) \ + return; \ \ -static size_t table##_max_size(struct table *table) \ -{ \ - if (table->chunk_mask == 0) { \ - return table->chunks[0].chunk0_capacity; \ - } else { \ - return ((table->chunk_mask + 1) * \ - table##_chunk_desired_capacity); \ + size_t chunk_count = table->chunk_mask + 1; \ + /* Always reset large tables. */ \ + if (chunk_count >= 16) \ + reset = true; \ + if (table->size) { \ + if (!reset) { \ + size_t capacity_scale = \ + table##_chunk_capacity_scale(table->chunks); \ + memset(table->chunks, 0, \ + table##_alloc_size(chunk_count, capacity_scale));\ + table##_chunk_mark_eof(table->chunks, capacity_scale); \ + } \ + table->size = 0; \ + table->first_packed = 0; \ + } \ + if (reset) { \ + free(table->chunks); \ + table->chunks = hash_table_empty_chunk; \ + table->chunk_mask = 0; \ } \ } \ \ __attribute__((unused)) \ static bool table##_reserve(struct table *table, size_t capacity) \ { \ - if (table->size > capacity) \ - capacity = table->size; \ - return table##_do_reserve(table, capacity, table##_max_size(table)); \ + capacity = max(capacity, table->size); \ + if (!capacity) { \ + table##_do_clear(table, true); \ + return true; \ + } \ + \ + size_t orig_chunk_count = table->chunk_mask + 1; \ + size_t orig_capacity_scale = table##_chunk_capacity_scale(table->chunks);\ + size_t orig_capacity = table##_compute_capacity(orig_chunk_count, \ + orig_capacity_scale); \ + \ + /* \ + * To avoid pathological behavior, ignore decreases that aren't at \ + * least a 1/8 decrease, and double for increases that aren't at least \ + * a 1/8 increase. \ + */ \ + if (capacity <= orig_capacity && \ + capacity >= orig_capacity - orig_capacity / 8) \ + return true; \ + bool attempt_exact = !(capacity > orig_capacity && \ + capacity < orig_capacity + orig_capacity / 8); \ + \ + size_t new_chunk_count; \ + size_t new_capacity_scale; \ + if (!table##_compute_chunk_count_and_scale(capacity, attempt_exact, \ + &new_chunk_count, \ + &new_capacity_scale)) \ + return false; \ + size_t new_capacity = table##_compute_capacity(new_chunk_count, \ + new_capacity_scale); \ + if (new_capacity == orig_capacity) \ + return true; \ + return table##_rehash(table, orig_chunk_count, orig_capacity_scale, \ + new_chunk_count, new_capacity_scale); \ } \ \ __attribute__((unused)) \ static void table##_clear(struct table *table) \ { \ - size_t chunk_count; \ - \ - if (table->chunks == hash_table_empty_chunk) \ - return; \ - \ - /* For large tables, free the chunks. For small tables, zero them. */ \ - chunk_count = table->chunk_mask + 1; \ - if (chunk_count >= 16) { \ - free(table->chunks); \ - table->chunks = hash_table_empty_chunk; \ - table->chunk_mask = 0; \ - } else if (table->size) { \ - uint8_t chunk0_capacity; \ - size_t alloc_size; \ - \ - chunk0_capacity = table->chunks[0].chunk0_capacity; \ - alloc_size = table##_alloc_size(chunk_count, \ - table##_max_size(table)); \ - memset(table->chunks, 0, alloc_size); \ - table->chunks[0].chunk0_capacity = chunk0_capacity; \ - } \ - table->size = 0; \ - table->first_packed = 0; \ + table##_do_clear(table, false); \ } \ \ static struct table##_iterator \ @@ -834,14 +914,26 @@ table##_search(struct table *table, const table##_key_type *key) \ \ static bool table##_reserve_for_insert(struct table *table) \ { \ - size_t capacity, max_size; \ - \ - capacity = table->size + 1; \ - max_size = table##_max_size(table); \ - if (capacity - 1 >= max_size) \ - return table##_do_reserve(table, capacity, max_size); \ - else \ + size_t orig_chunk_count = table->chunk_mask + 1; \ + size_t orig_capacity_scale = table##_chunk_capacity_scale(table->chunks);\ + size_t orig_capacity = table##_compute_capacity(orig_chunk_count, \ + orig_capacity_scale); \ + size_t capacity = table->size + 1; \ + if (capacity <= orig_capacity) \ return true; \ + /* Grow by at least orig_capacity * 2^0.5. */ \ + size_t min_growth = (orig_capacity + \ + (orig_capacity >> 2) + \ + (orig_capacity >> 3) + \ + (orig_capacity >> 5)); \ + capacity = max(capacity, min_growth); \ + size_t new_chunk_count, new_capacity_scale; \ + if (!table##_compute_chunk_count_and_scale(capacity, false, \ + &new_chunk_count, \ + &new_capacity_scale)) \ + return false; \ + return table##_rehash(table, orig_chunk_count, orig_capacity_scale, \ + new_chunk_count, new_capacity_scale); \ } \ \ static void \ @@ -862,25 +954,22 @@ static int table##_insert_searched(struct table *table, \ struct hash_pair hp, \ struct table##_iterator *it_ret) \ { \ - size_t index = hp.first; \ - struct table##_chunk *chunk; \ - unsigned int first_empty; \ - \ if (!table##_reserve_for_insert(table)) \ return -1; \ \ - chunk = &table->chunks[index & table->chunk_mask]; \ - first_empty = table##_chunk_first_empty(chunk); \ + size_t index = hp.first; \ + struct table##_chunk *chunk = &table->chunks[index & table->chunk_mask];\ + unsigned int first_empty = table##_chunk_first_empty(chunk); \ if (first_empty == (unsigned int)-1) { \ - size_t delta = hash_table_probe_delta(hp); \ - \ + const size_t delta = hash_table_probe_delta(hp); \ do { \ table##_chunk_inc_outbound_overflow_count(chunk); \ index += delta; \ chunk = &table->chunks[index & table->chunk_mask]; \ first_empty = table##_chunk_first_empty(chunk); \ } while (first_empty == (unsigned int)-1); \ - chunk->hosted_overflow_count++; \ + table##_chunk_adjust_hosted_overflow_count(chunk, \ + hosted_overflow_count_inc);\ } \ chunk->tags[first_empty] = hp.second; \ memcpy(&chunk->entries[first_empty], entry, sizeof(*entry)); \ @@ -998,7 +1087,7 @@ table##_next_impl(struct table##_iterator it, bool likely_dead) \ for (i = 1; !likely_dead || i != 0; i++) { \ unsigned int last; \ \ - if (unlikely(chunk->chunk0_capacity != 0)) \ + if (unlikely(table##_chunk_eof(chunk))) \ break; \ \ chunk--; \ @@ -1017,26 +1106,25 @@ table##_next_impl(struct table##_iterator it, bool likely_dead) \ static void table##_do_delete(struct table *table, struct table##_iterator it, \ struct hash_pair hp) \ { \ - struct table##_chunk *it_chunk, *chunk; \ - \ - it_chunk = table##_iterator_chunk(it); \ + struct table##_chunk *it_chunk = table##_iterator_chunk(it); \ it_chunk->tags[it.index] = 0; \ \ table##_adjust_size_and_first_before_delete(table, it_chunk, it.index); \ \ - if (it_chunk->hosted_overflow_count) { \ + if (table##_chunk_hosted_overflow_count(it_chunk)) { \ + const size_t delta = hash_table_probe_delta(hp); \ size_t index = hp.first; \ - size_t delta = hash_table_probe_delta(hp); \ - uint8_t hosted_dec = 0; \ - \ + uint8_t hosted_op = 0; \ for (;;) { \ - chunk = &table->chunks[index & table->chunk_mask]; \ + struct table##_chunk *chunk = \ + &table->chunks[index & table->chunk_mask]; \ if (chunk == it_chunk) { \ - chunk->hosted_overflow_count -= hosted_dec; \ + table##_chunk_adjust_hosted_overflow_count(chunk,\ + hosted_op);\ break; \ } \ table##_chunk_dec_outbound_overflow_count(chunk); \ - hosted_dec = -1; \ + hosted_op = hosted_overflow_count_dec; \ index += delta; \ } \ } \ @@ -1062,7 +1150,7 @@ table##_delete_iterator(struct table *table, struct table##_iterator it) \ struct hash_pair hp = {}; \ \ /* We only need the hash if the chunk hosts an overflowed entry. */ \ - if (table##_iterator_chunk(it)->hosted_overflow_count) { \ + if (table##_chunk_hosted_overflow_count(table##_iterator_chunk(it))) { \ table##_key_type key = table##_entry_to_key(it.entry); \ \ hp = table##_hash(&key); \ From f94b0262c6aa3b4f0e86b265bb98e867cb837911 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Sat, 18 Jul 2020 00:15:37 -0700 Subject: [PATCH 10/18] libdrgn: hash_table: implement vector storage policy The folly F14 implementation provides 3 storage policies: value, node, and vector. The default F14FastMap/F14FastSet chooses between the value and vector policies based on the value size. We currently only implement the value policy, as the node policy is easy to emulate and the vector policy would've added more complexity. This adds support for the vector policy (adding even more C abuse :) and automatically chooses the policy the same way as folly. It'd be easy to add a way to choose the policy if needed. Signed-off-by: Omar Sandoval --- libdrgn/hash_table.h | 612 ++++++++++++++++++++++++++++++------------- 1 file changed, 424 insertions(+), 188 deletions(-) diff --git a/libdrgn/hash_table.h b/libdrgn/hash_table.h index 37398377c..15dbdadb7 100644 --- a/libdrgn/hash_table.h +++ b/libdrgn/hash_table.h @@ -112,11 +112,11 @@ struct hash_table; * * Several functions return an iterator or take one as an argument. This * iterator has a reference to an entry, which can be @c NULL to indicate that - * there is no such entry. It may also contain private bookkeeping which should - * not be used. + * there is no such entry. It also contains private bookkeeping which should not + * be used. * - * An iterator remains valid as long as the entry is not deleted and the table - * is not rehashed. + * An iterator remains valid until the table is rehashed or the entry or one + * before it is deleted. */ struct hash_table_iterator { /** Pointer to the entry in the hash table. */ @@ -176,8 +176,8 @@ void hash_table_clear(struct hash_table *table); /** * Reserve entries in a @ref hash_table. * - * This allocates space up front to ensure that the table will not be rehashed - * until the table contains the given number of entries. + * This allocates space up front and rehashes the table to ensure that it will + * not be rehashed until it contains the given number of entries. * * @return @c true on success, @c false on failure. */ @@ -390,20 +390,50 @@ table##_entry_to_key(const table##_entry_type *entry) \ \ enum { \ /* \ - * The number of entries per chunk. 14 is the most space efficient, but \ - * if an entry is 4 bytes, 12 entries makes a chunk exactly one cache \ + * Whether this table uses the vector storage policy. \ + * \ + * The vector policy provides the best performance and memory \ + * efficiency for medium and large entries. \ + */ \ + table##_vector_policy = sizeof(table##_entry_type) >= 24, \ +}; \ + \ +/* \ + * Item stored in a chunk. \ + * \ + * When using the basic policy, the entry is stored directly in the item. When \ + * using the vector policy, the item is an index to an out-of-band vector of \ + * entries. \ + * \ + * C doesn't make it easy to define a type conditionally, so we use a nasty \ + * hack: the member for the used policy is an array of length 1, and the unused \ + * member is an array of length 0. We also have to force the struct to be \ + * aligned only for the used member. \ + */ \ +typedef struct { \ + uint32_t index[table##_vector_policy]; \ + table##_entry_type entry[!table##_vector_policy]; \ +} __attribute__((packed, \ + aligned(table##_vector_policy ? \ + alignof(uint32_t) : alignof(table##_entry_type)))) \ +table##_item_type; \ + \ +enum { \ + /* \ + * The number of items per chunk. 14 is the most space efficient, but \ + * if an item is 4 bytes, 12 items makes a chunk exactly one cache \ * line. \ */ \ - table##_chunk_capacity = sizeof(table##_entry_type) == 4 ? 12 : 14, \ - /* The maximum load factor in terms of entries per chunk. */ \ + table##_chunk_capacity = sizeof(table##_item_type) == 4 ? 12 : 14, \ + /* The maximum load factor in terms of items per chunk. */ \ table##_chunk_desired_capacity = table##_chunk_capacity - 2, \ /* \ - * If an entry is 16 bytes, add an extra 16 bytes of padding to make a \ + * If an item is 16 bytes, add an extra 16 bytes of padding to make a \ * chunk exactly four cache lines. \ */ \ table##_chunk_allocated_capacity = \ (table##_chunk_capacity + \ - (sizeof(table##_entry_type) == 16 ? 1 : 0)), \ + (sizeof(table##_item_type) == 16 ? 1 : 0)), \ /* \ * If the chunk capacity is 12, we can use tags 12 and 13 for 16 bits. \ * Otherwise, we only get 4 from control. \ @@ -430,24 +460,93 @@ struct table##_chunk { \ * not be updated. \ */ \ uint8_t outbound_overflow_count; \ - table##_entry_type entries[table##_chunk_allocated_capacity]; \ + table##_item_type items[table##_chunk_allocated_capacity]; \ } __attribute__((aligned(hash_table_chunk_alignment))); \ \ +/* \ + * This may be a "public iterator" (used by the public interface to refer to an \ + * entry) or an "item iterator" (used by certain internal functions to refer to \ + * an item regardless of the storage policy). \ + */ \ struct table##_iterator { \ - table##_entry_type *entry; \ - size_t index; \ + union { \ + /* Entry if public iterator. */ \ + table##_entry_type *entry; \ + /* \ + * Item if item iterator. Interchangable with entry when using \ + * the basic storage policy. \ + */ \ + table##_item_type *item; \ + }; \ + union { \ + /* \ + * Lowest entry if public iterator and using the vector storage \ + * policy (i.e., table->vector->entries). \ + */ \ + table##_entry_type *lowest; \ + /* \ + * Index of item in its containing chunk if item iterator or \ + * using the basic storage policy. \ + */ \ + size_t index; \ + }; \ }; \ \ struct table { \ struct table##_chunk *chunks; \ - /* Number of chunks minus one. */ \ - size_t chunk_mask; \ - /* Number of used values. */ \ - size_t size; \ - /* Cached first iterator. */ \ - uintptr_t first_packed; \ + struct { \ + /* \ + * The vector storage policy stores 32-bit indices, so we only \ + * need 32-bit sizes. \ + */ \ + uint32_t chunk_mask; \ + uint32_t size; \ + /* Allocated together with chunks. */ \ + table##_entry_type *entries; \ + } vector[table##_vector_policy]; \ + struct { \ + size_t chunk_mask; \ + size_t size; \ + uintptr_t first_packed; \ + } basic[!table##_vector_policy]; \ }; +/* + * Common search function implementation returning an item iterator. This is + * shared by key lookups and index lookups. + */ +#define HASH_TABLE_SEARCH_IMPL(table, func, key_type, item_to_key, eq_func) \ +static struct table##_iterator table##_##func(struct table *table, \ + const key_type *key, \ + struct hash_pair hp) \ +{ \ + const size_t delta = hash_table_probe_delta(hp); \ + size_t index = hp.first; \ + for (size_t tries = 0; tries <= table##_chunk_mask(table); tries++) { \ + struct table##_chunk *chunk = \ + &table->chunks[index & table##_chunk_mask(table)]; \ + if (sizeof(*chunk) > 64) \ + __builtin_prefetch(&chunk->items[8]); \ + unsigned int mask = table##_chunk_match(chunk, hp.second), i; \ + for_each_bit(i, mask) { \ + table##_item_type *item = &chunk->items[i]; \ + key_type item_key = item_to_key(table, item); \ + if (likely(eq_func(key, &item_key))) { \ + return (struct table##_iterator){ \ + .item = item, \ + .index = i, \ + }; \ + } \ + } \ + if (likely(chunk->outbound_overflow_count == 0)) \ + break; \ + index += delta; \ + } \ + return (struct table##_iterator){}; \ +} + +#define HASH_TABLE_SEARCH_BY_INDEX_ITEM_TO_KEY(table, item) (*(item)->index) + /** * Define the functions for a hash table. * @@ -465,6 +564,27 @@ static inline struct hash_pair table##_hash(const table##_key_type *key) \ return hash_func(key); \ } \ \ +static inline table##_entry_type * \ +table##_item_to_entry(struct table *table, table##_item_type *item) \ +{ \ + if (table##_vector_policy) { \ + return &table->vector->entries[*item->index]; \ + } else { \ + /* \ + * Returning item->entry directly results in a false positive \ + * -Waddress-of-packed-member warning. \ + */ \ + void *entry = item->entry; \ + return entry; \ + } \ +} \ + \ +static inline table##_key_type \ +table##_item_to_key(struct table *table, table##_item_type *item) \ +{ \ + return table##_entry_to_key(table##_item_to_entry(table, item)); \ +} \ + \ /* \ * We cache the first position in the table as a tagged pointer: we steal the \ * bottom bits of the chunk pointer for the entry index. We can do this because \ @@ -493,13 +613,10 @@ static inline size_t table##_unpack_index(uintptr_t packed) \ \ static inline struct table##_iterator table##_unpack_iterator(uintptr_t packed) \ { \ - struct table##_chunk *chunk; \ - size_t index; \ - \ - chunk = table##_unpack_chunk(packed); \ - index = table##_unpack_index(packed); \ - return (struct table##_iterator){ \ - .entry = chunk ? &chunk->entries[index] : NULL, \ + struct table##_chunk *chunk = table##_unpack_chunk(packed); \ + size_t index = table##_unpack_index(packed); \ + return (struct table##_iterator) { \ + .item = chunk ? &chunk->items[index] : NULL, \ .index = index, \ }; \ } \ @@ -507,8 +624,7 @@ static inline struct table##_iterator table##_unpack_iterator(uintptr_t packed) static inline struct table##_chunk * \ table##_iterator_chunk(struct table##_iterator it) \ { \ - return container_of(it.entry - it.index, struct table##_chunk, \ - entries[0]); \ + return container_of(it.item - it.index, struct table##_chunk, items[0]);\ } \ \ HASH_TABLE_CHUNK_MATCH(table) \ @@ -602,9 +718,15 @@ __attribute__((unused)) \ static void table##_init(struct table *table) \ { \ table->chunks = hash_table_empty_chunk; \ - table->chunk_mask = 0; \ - table->size = 0; \ - table->first_packed = 0; \ + if (table##_vector_policy) { \ + table->vector->chunk_mask = 0; \ + table->vector->size = 0; \ + table->vector->entries = NULL; \ + } else { \ + table->basic->chunk_mask = 0; \ + table->basic->size = 0; \ + table->basic->first_packed = 0; \ + } \ } \ \ __attribute__((unused)) \ @@ -614,28 +736,55 @@ static void table##_deinit(struct table *table) \ free(table->chunks); \ } \ \ -__attribute__((unused)) \ -static inline bool table##_empty(struct table *table) \ +static inline size_t table##_size(struct table *table) \ { \ - return table->size == 0; \ + if (table##_vector_policy) \ + return table->vector->size; \ + else \ + return table->basic->size; \ +} \ + \ +static inline void table##_set_size(struct table *table, size_t size) \ +{ \ + if (table##_vector_policy) \ + table->vector->size = size; \ + else \ + table->basic->size = size; \ +} \ + \ +static inline size_t table##_chunk_mask(struct table *table) \ +{ \ + if (table##_vector_policy) \ + return table->vector->chunk_mask; \ + else \ + return table->basic->chunk_mask; \ +} \ + \ +static inline void table##_set_chunk_mask(struct table *table, \ + size_t chunk_mask) \ +{ \ + if (table##_vector_policy) \ + table->vector->chunk_mask = chunk_mask; \ + else \ + table->basic->chunk_mask = chunk_mask; \ } \ \ __attribute__((unused)) \ -static inline size_t table##_size(struct table *table) \ +static inline bool table##_empty(struct table *table) \ { \ - return table->size; \ + return table##_size(table) == 0; \ } \ \ -static table##_entry_type *table##_allocate_tag(struct table *table, \ - uint8_t *fullness, \ - struct hash_pair hp) \ +static table##_item_type *table##_allocate_tag(struct table *table, \ + uint8_t *fullness, \ + struct hash_pair hp) \ { \ const size_t delta = hash_table_probe_delta(hp); \ size_t index = hp.first; \ struct table##_chunk *chunk; \ uint8_t hosted_op = 0; \ for (;;) { \ - index &= table->chunk_mask; \ + index &= table##_chunk_mask(table); \ chunk = &table->chunks[index]; \ if (likely(fullness[index] < table##_chunk_capacity)) \ break; \ @@ -643,22 +792,10 @@ static table##_entry_type *table##_allocate_tag(struct table *table, \ hosted_op = hosted_overflow_count_inc; \ index += delta; \ } \ - size_t entry_index = fullness[index]++; \ - chunk->tags[entry_index] = hp.second; \ + size_t item_index = fullness[index]++; \ + chunk->tags[item_index] = hp.second; \ table##_chunk_adjust_hosted_overflow_count(chunk, hosted_op); \ - return &chunk->entries[entry_index]; \ -} \ - \ -static void table##_set_first_packed_after_rehash(struct table *table, \ - uint8_t *fullness) \ -{ \ - size_t i; \ - \ - i = table->chunk_mask; \ - while (fullness[i] == 0) \ - i--; \ - table->first_packed = table##_pack_iterator(&table->chunks[i], \ - fullness[i] - 1); \ + return &chunk->items[item_index]; \ } \ \ static size_t table##_compute_capacity(size_t chunk_count, size_t scale) \ @@ -669,6 +806,7 @@ static size_t table##_compute_capacity(size_t chunk_count, size_t scale) \ static bool \ table##_compute_chunk_count_and_scale(size_t capacity, \ bool continuous_single_chunk_capacity, \ + bool continuous_multi_chunk_capacity, \ size_t *chunk_count_ret, \ size_t *scale_ret) \ { \ @@ -689,25 +827,33 @@ table##_compute_chunk_count_and_scale(size_t capacity, \ size_t chunk_pow = fls(min_chunks - 1); \ if (chunk_pow == 8 * sizeof(size_t)) \ return false; \ - *chunk_count_ret = (size_t)1 << chunk_pow; \ + size_t chunk_count = (size_t)1 << chunk_pow; \ size_t ss = (chunk_pow >= table##_capacity_scale_shift ? \ chunk_pow - table##_capacity_scale_shift : 0); \ - *scale_ret = table##_chunk_desired_capacity << (chunk_pow - ss);\ + size_t scale = \ + continuous_multi_chunk_capacity ? \ + ((capacity - 1) >> ss) + 1 : \ + table##_chunk_desired_capacity << (chunk_pow - ss); \ + if (table##_vector_policy && \ + table##_compute_capacity(chunk_count, scale) > UINT32_MAX) \ + return false; \ + *chunk_count_ret = chunk_count; \ + *scale_ret = scale; \ } \ return true; \ } \ \ -static inline size_t table##_alloc_size(size_t chunk_count, \ - size_t capacity_scale) \ +static inline size_t table##_chunk_alloc_size(size_t chunk_count, \ + size_t capacity_scale) \ { \ /* \ * Small hash tables are common, so for capacities of less than a full \ - * chunk we only allocate the required entries. \ + * chunk, we only allocate the required items. \ */ \ if (chunk_count == 1) { \ - return (offsetof(struct table##_chunk, entries) + \ + return (offsetof(struct table##_chunk, items) + \ table##_compute_capacity(1, capacity_scale) * \ - sizeof(table##_entry_type)); \ + sizeof(table##_item_type)); \ } else { \ return chunk_count * sizeof(struct table##_chunk); \ } \ @@ -717,35 +863,64 @@ static bool table##_rehash(struct table *table, size_t orig_chunk_count, \ size_t orig_capacity_scale, size_t new_chunk_count, \ size_t new_capacity_scale) \ { \ - struct table##_chunk *orig_chunks = table->chunks; \ - size_t alloc_size = table##_alloc_size(new_chunk_count, \ - new_capacity_scale); \ + size_t chunk_alloc_size = table##_chunk_alloc_size(new_chunk_count, \ + new_capacity_scale); \ + size_t alloc_size, entries_offset; \ + if (table##_vector_policy) { \ + entries_offset = chunk_alloc_size; \ + if (alignof(table##_entry_type) > alignof(table##_item_type)) { \ + entries_offset = -(-entries_offset & \ + ~(alignof(table##_entry_type) - 1)); \ + } \ + size_t new_capacity = \ + table##_compute_capacity(new_chunk_count, \ + new_capacity_scale); \ + alloc_size = (entries_offset + \ + new_capacity * sizeof(table##_entry_type)); \ + } else { \ + alloc_size = chunk_alloc_size; \ + } \ \ void *new_chunks; \ if (posix_memalign(&new_chunks, hash_table_chunk_alignment, alloc_size))\ return false; \ + \ + struct table##_chunk *orig_chunks = table->chunks; \ table->chunks = new_chunks; \ - memset(table->chunks, 0, alloc_size); \ + table##_entry_type *orig_entries; \ + if (table##_vector_policy) { \ + orig_entries = table->vector->entries; \ + table->vector->entries = new_chunks + entries_offset; \ + if (table##_size(table) > 0) { \ + memcpy(table->vector->entries, orig_entries, \ + table##_size(table) * \ + sizeof(table##_entry_type)); \ + } \ + } \ + \ + memset(table->chunks, 0, chunk_alloc_size); \ table##_chunk_mark_eof(table->chunks, new_capacity_scale); \ - table->chunk_mask = new_chunk_count - 1; \ + table##_set_chunk_mask(table, new_chunk_count - 1); \ \ - if (table->size == 0) { \ + if (table##_size(table) == 0) { \ /* Nothing to do. */ \ } else if (orig_chunk_count == 1 && new_chunk_count == 1) { \ struct table##_chunk *src = orig_chunks; \ struct table##_chunk *dst = table->chunks; \ size_t src_i = 0, dst_i = 0; \ - while (dst_i < table->size) { \ + while (dst_i < table##_size(table)) { \ if (likely(src->tags[src_i])) { \ dst->tags[dst_i] = src->tags[src_i]; \ - memcpy(&dst->entries[dst_i], \ - &src->entries[src_i], \ - sizeof(dst->entries[dst_i])); \ + memcpy(&dst->items[dst_i], &src->items[src_i], \ + sizeof(dst->items[dst_i])); \ dst_i++; \ } \ src_i++; \ } \ - table->first_packed = table##_pack_iterator(dst, dst_i - 1); \ + if (!table##_vector_policy) { \ + table->basic->first_packed = \ + table##_pack_iterator(dst, dst_i - 1); \ + } \ } else { \ uint8_t stack_fullness[256]; \ uint8_t *fullness; \ @@ -759,28 +934,37 @@ static bool table##_rehash(struct table *table, size_t orig_chunk_count, \ } \ \ struct table##_chunk *src = &orig_chunks[orig_chunk_count - 1]; \ - size_t remaining = table->size; \ + size_t remaining = table##_size(table); \ while (remaining) { \ unsigned int mask = table##_chunk_occupied(src), i; \ + if (table##_vector_policy) { \ + unsigned int pmask = mask; \ + for_each_bit(i, pmask) \ + __builtin_prefetch(&src->items[i]); \ + } \ for_each_bit(i, mask) { \ remaining--; \ \ - table##_entry_type *src_entry = \ - &src->entries[i]; \ + table##_item_type *src_item = &src->items[i]; \ table##_key_type key = \ - table##_entry_to_key(src_entry); \ + table##_item_to_key(table, src_item); \ struct hash_pair hp = table##_hash(&key); \ - table##_entry_type *dst_entry = \ + table##_item_type *dst_item = \ table##_allocate_tag(table, fullness, \ hp); \ - \ - memcpy(dst_entry, src_entry, \ - sizeof(*dst_entry)); \ + memcpy(dst_item, src_item, sizeof(*dst_item)); \ } \ src--; \ } \ \ - table##_set_first_packed_after_rehash(table, fullness); \ + if (!table##_vector_policy) { \ + size_t i = table##_chunk_mask(table); \ + while (fullness[i] == 0) \ + i--; \ + table->basic->first_packed = \ + table##_pack_iterator(&table->chunks[i], \ + fullness[i] - 1); \ + } \ \ if (fullness != stack_fullness) \ free(fullness); \ @@ -793,7 +977,9 @@ static bool table##_rehash(struct table *table, size_t orig_chunk_count, \ err: \ free(table->chunks); \ table->chunks = orig_chunks; \ - table->chunk_mask = orig_chunk_count - 1; \ + table##_set_chunk_mask(table, orig_chunk_count - 1); \ + if (table##_vector_policy) \ + table->vector->entries = orig_entries; \ return false; \ } \ \ @@ -802,38 +988,42 @@ static void table##_do_clear(struct table *table, bool reset) \ if (table->chunks == hash_table_empty_chunk) \ return; \ \ - size_t chunk_count = table->chunk_mask + 1; \ + size_t chunk_count = table##_chunk_mask(table) + 1; \ /* Always reset large tables. */ \ if (chunk_count >= 16) \ reset = true; \ - if (table->size) { \ + if (!table##_empty(table)) { \ if (!reset) { \ size_t capacity_scale = \ table##_chunk_capacity_scale(table->chunks); \ memset(table->chunks, 0, \ - table##_alloc_size(chunk_count, capacity_scale));\ + table##_chunk_alloc_size(chunk_count, \ + capacity_scale)); \ table##_chunk_mark_eof(table->chunks, capacity_scale); \ } \ - table->size = 0; \ - table->first_packed = 0; \ + if (!table##_vector_policy) \ + table->basic->first_packed = 0; \ + table##_set_size(table, 0); \ } \ if (reset) { \ free(table->chunks); \ table->chunks = hash_table_empty_chunk; \ - table->chunk_mask = 0; \ + table##_set_chunk_mask(table, 0); \ + if (table##_vector_policy) \ + table->vector->entries = NULL; \ } \ } \ \ __attribute__((unused)) \ static bool table##_reserve(struct table *table, size_t capacity) \ { \ - capacity = max(capacity, table->size); \ + capacity = max(capacity, table##_size(table)); \ if (!capacity) { \ table##_do_clear(table, true); \ return true; \ } \ \ - size_t orig_chunk_count = table->chunk_mask + 1; \ + size_t orig_chunk_count = table##_chunk_mask(table) + 1; \ size_t orig_capacity_scale = table##_chunk_capacity_scale(table->chunks);\ size_t orig_capacity = table##_compute_capacity(orig_chunk_count, \ orig_capacity_scale); \ @@ -852,6 +1042,8 @@ static bool table##_reserve(struct table *table, size_t capacity) \ size_t new_chunk_count; \ size_t new_capacity_scale; \ if (!table##_compute_chunk_count_and_scale(capacity, attempt_exact, \ + table##_vector_policy && \ + attempt_exact, \ &new_chunk_count, \ &new_capacity_scale)) \ return false; \ @@ -869,40 +1061,24 @@ static void table##_clear(struct table *table) \ table##_do_clear(table, false); \ } \ \ + \ +HASH_TABLE_SEARCH_IMPL(table, search_by_key, table##_key_type, \ + table##_item_to_key, eq_func) \ +HASH_TABLE_SEARCH_IMPL(table, search_by_index, uint32_t, \ + HASH_TABLE_SEARCH_BY_INDEX_ITEM_TO_KEY, \ + hash_table_scalar_eq) \ + \ static struct table##_iterator \ table##_search_hashed(struct table *table, const table##_key_type *key, \ struct hash_pair hp) \ { \ - size_t index = hp.first; \ - size_t delta = hash_table_probe_delta(hp); \ - size_t tries; \ - \ - for (tries = 0; tries <= table->chunk_mask; tries++) { \ - struct table##_chunk *chunk; \ - unsigned int mask, i; \ - \ - chunk = &table->chunks[index & table->chunk_mask]; \ - if (sizeof(*chunk) > 64) \ - __builtin_prefetch(&chunk->entries[8]); \ - mask = table##_chunk_match(chunk, hp.second); \ - for_each_bit(i, mask) { \ - table##_entry_type *entry; \ - table##_key_type entry_key; \ - \ - entry = &chunk->entries[i]; \ - entry_key = table##_entry_to_key(entry); \ - if (likely(eq_func(key, &entry_key))) { \ - return (struct table##_iterator){ \ - .entry = entry, \ - .index = i, \ - }; \ - } \ - } \ - if (likely(chunk->outbound_overflow_count == 0)) \ - break; \ - index += delta; \ + struct table##_iterator it = table##_search_by_key(table, key, hp); \ + /* Convert the item iterator to a public iterator. */ \ + if (table##_vector_policy && it.item) { \ + it.entry = table##_item_to_entry(table, it.item); \ + it.lowest = table->vector->entries; \ } \ - return (struct table##_iterator){}; \ + return it; \ } \ \ __attribute__((unused)) \ @@ -914,11 +1090,11 @@ table##_search(struct table *table, const table##_key_type *key) \ \ static bool table##_reserve_for_insert(struct table *table) \ { \ - size_t orig_chunk_count = table->chunk_mask + 1; \ + size_t orig_chunk_count = table##_chunk_mask(table) + 1; \ size_t orig_capacity_scale = table##_chunk_capacity_scale(table->chunks);\ size_t orig_capacity = table##_compute_capacity(orig_chunk_count, \ orig_capacity_scale); \ - size_t capacity = table->size + 1; \ + size_t capacity = table##_size(table) + 1; \ if (capacity <= orig_capacity) \ return true; \ /* Grow by at least orig_capacity * 2^0.5. */ \ @@ -928,7 +1104,7 @@ static bool table##_reserve_for_insert(struct table *table) \ (orig_capacity >> 5)); \ capacity = max(capacity, min_growth); \ size_t new_chunk_count, new_capacity_scale; \ - if (!table##_compute_chunk_count_and_scale(capacity, false, \ + if (!table##_compute_chunk_count_and_scale(capacity, false, false, \ &new_chunk_count, \ &new_capacity_scale)) \ return false; \ @@ -941,12 +1117,12 @@ table##_adjust_size_and_first_after_insert(struct table *table, \ struct table##_chunk *chunk, \ size_t index) \ { \ - uintptr_t first_packed; \ - \ - first_packed = table##_pack_iterator(chunk, index); \ - if (first_packed > table->first_packed) \ - table->first_packed = first_packed; \ - table->size++; \ + if (!table##_vector_policy) { \ + uintptr_t first_packed = table##_pack_iterator(chunk, index); \ + if (first_packed > table->basic->first_packed) \ + table->basic->first_packed = first_packed; \ + } \ + table##_set_size(table, table##_size(table) + 1); \ } \ \ static int table##_insert_searched(struct table *table, \ @@ -958,25 +1134,38 @@ static int table##_insert_searched(struct table *table, \ return -1; \ \ size_t index = hp.first; \ - struct table##_chunk *chunk = &table->chunks[index & table->chunk_mask];\ + struct table##_chunk *chunk = \ + &table->chunks[index & table##_chunk_mask(table)]; \ unsigned int first_empty = table##_chunk_first_empty(chunk); \ if (first_empty == (unsigned int)-1) { \ - const size_t delta = hash_table_probe_delta(hp); \ + size_t delta = hash_table_probe_delta(hp); \ do { \ table##_chunk_inc_outbound_overflow_count(chunk); \ index += delta; \ - chunk = &table->chunks[index & table->chunk_mask]; \ + chunk = &table->chunks[index & table##_chunk_mask(table)];\ first_empty = table##_chunk_first_empty(chunk); \ } while (first_empty == (unsigned int)-1); \ table##_chunk_adjust_hosted_overflow_count(chunk, \ hosted_overflow_count_inc);\ } \ chunk->tags[first_empty] = hp.second; \ - memcpy(&chunk->entries[first_empty], entry, sizeof(*entry)); \ + if (table##_vector_policy) { \ + *chunk->items[first_empty].index = table##_size(table); \ + memcpy(&table->vector->entries[table##_size(table)], entry, \ + sizeof(*entry)); \ + } else { \ + memcpy(&chunk->items[first_empty], entry, sizeof(*entry)); \ + } \ table##_adjust_size_and_first_after_insert(table, chunk, first_empty); \ if (it_ret) { \ - it_ret->entry = &chunk->entries[first_empty]; \ - it_ret->index = first_empty; \ + if (table##_vector_policy) { \ + it_ret->entry = \ + &table->vector->entries[table##_size(table) - 1];\ + it_ret->lowest = table->vector->entries; \ + } else { \ + it_ret->item = &chunk->items[first_empty]; \ + it_ret->index = first_empty; \ + } \ } \ return 1; \ } \ @@ -1011,16 +1200,14 @@ static int table##_insert(struct table *table, \ /* Similar to table##_next_impl() but for the cached first position. */ \ static void table##_advance_first_packed(struct table *table) \ { \ - uintptr_t packed = table->first_packed; \ - struct table##_chunk *chunk; \ - size_t index; \ - \ - chunk = table##_unpack_chunk(packed); \ - index = table##_unpack_index(packed); \ + uintptr_t packed = table->basic->first_packed; \ + struct table##_chunk *chunk = table##_unpack_chunk(packed); \ + size_t index = table##_unpack_index(packed); \ while (index > 0) { \ index--; \ if (chunk->tags[index]) { \ - table->first_packed = table##_pack_iterator(chunk, index);\ + table->basic->first_packed = \ + table##_pack_iterator(chunk, index); \ return; \ } \ } \ @@ -1030,12 +1217,11 @@ static void table##_advance_first_packed(struct table *table) \ * don't need to check if we hit the end. \ */ \ for (;;) { \ - unsigned int last; \ - \ chunk--; \ - last = table##_chunk_last_occupied(chunk); \ + unsigned int last = table##_chunk_last_occupied(chunk); \ if (last != (unsigned int)-1) { \ - table->first_packed = table##_pack_iterator(chunk, last);\ + table->basic->first_packed = \ + table##_pack_iterator(chunk, last); \ return; \ } \ } \ @@ -1046,13 +1232,11 @@ table##_adjust_size_and_first_before_delete(struct table *table, \ struct table##_chunk *chunk, \ size_t index) \ { \ - uintptr_t packed; \ - \ - table->size--; \ - packed = table##_pack_iterator(chunk, index); \ - if (packed == table->first_packed) { \ - if (table->size == 0) \ - table->first_packed = 0; \ + table##_set_size(table, table##_size(table) - 1); \ + if (!table##_vector_policy && \ + table##_pack_iterator(chunk, index) == table->basic->first_packed) {\ + if (table##_empty(table)) \ + table->basic->first_packed = 0; \ else \ table##_advance_first_packed(table); \ } \ @@ -1067,10 +1251,7 @@ __attribute__((always_inline)) \ static inline struct table##_iterator \ table##_next_impl(struct table##_iterator it, bool likely_dead) \ { \ - struct table##_chunk *chunk; \ - size_t i; \ - \ - chunk = table##_iterator_chunk(it); \ + struct table##_chunk *chunk = table##_iterator_chunk(it); \ while (it.index > 0) { \ it.index--; \ it.entry--; \ @@ -1084,32 +1265,32 @@ table##_next_impl(struct table##_iterator it, bool likely_dead) \ * table##_delete_iterator() is often ignored), but the compiler needs \ * some help proving that the following loop terminates. \ */ \ - for (i = 1; !likely_dead || i != 0; i++) { \ - unsigned int last; \ - \ + for (size_t i = 1; !likely_dead || i != 0; i++) { \ if (unlikely(table##_chunk_eof(chunk))) \ break; \ \ chunk--; \ - last = table##_chunk_last_occupied(chunk); \ + unsigned int last = table##_chunk_last_occupied(chunk); \ if (!likely_dead) \ __builtin_prefetch(chunk - 1); \ if (likely(last != (unsigned int)-1)) { \ it.index = last; \ - it.entry = &chunk->entries[last]; \ + it.item = &chunk->items[last]; \ return it; \ } \ } \ return (struct table##_iterator){}; \ } \ \ -static void table##_do_delete(struct table *table, struct table##_iterator it, \ - struct hash_pair hp) \ +static void table##_delete_impl(struct table *table, \ + struct table##_iterator item_it, \ + struct hash_pair hp) \ { \ - struct table##_chunk *it_chunk = table##_iterator_chunk(it); \ - it_chunk->tags[it.index] = 0; \ + struct table##_chunk *it_chunk = table##_iterator_chunk(item_it); \ + it_chunk->tags[item_it.index] = 0; \ \ - table##_adjust_size_and_first_before_delete(table, it_chunk, it.index); \ + table##_adjust_size_and_first_before_delete(table, it_chunk, \ + item_it.index); \ \ if (table##_chunk_hosted_overflow_count(it_chunk)) { \ const size_t delta = hash_table_probe_delta(hp); \ @@ -1117,7 +1298,7 @@ static void table##_do_delete(struct table *table, struct table##_iterator it, \ uint8_t hosted_op = 0; \ for (;;) { \ struct table##_chunk *chunk = \ - &table->chunks[index & table->chunk_mask]; \ + &table->chunks[index & table##_chunk_mask(table)];\ if (chunk == it_chunk) { \ table##_chunk_adjust_hosted_overflow_count(chunk,\ hosted_op);\ @@ -1130,6 +1311,27 @@ static void table##_do_delete(struct table *table, struct table##_iterator it, \ } \ } \ \ +static void table##_vector_delete_impl(struct table *table, \ + struct table##_iterator item_it, \ + struct hash_pair hp) \ +{ \ + /* Delete the index from the table. */ \ + uint32_t index = *item_it.item->index; \ + table##_delete_impl(table, item_it, hp); \ + \ + /* Replace it with the last entry and update its index in the table. */ \ + uint32_t tail_index = table##_size(table); \ + if (tail_index != index) { \ + table##_entry_type *tail = \ + &table->vector->entries[tail_index]; \ + table##_key_type tail_key = table##_entry_to_key(tail); \ + item_it = table##_search_by_index(table, &tail_index, \ + table##_hash(&tail_key)); \ + *item_it.item->index = index; \ + memcpy(&table->vector->entries[index], tail, sizeof(*tail)); \ + } \ +} \ + \ /* \ * We want this inlined so that the call to table##_next_impl() can be \ * optimized away. \ @@ -1139,8 +1341,21 @@ static inline struct table##_iterator \ table##_delete_iterator_hashed(struct table *table, struct table##_iterator it, \ struct hash_pair hp) \ { \ - table##_do_delete(table, it, hp); \ - return table##_next_impl(it, true); \ + if (table##_vector_policy) { \ + uint32_t index = it.entry - it.lowest; \ + struct table##_iterator item_it = \ + table##_search_by_index(table, &index, hp); \ + table##_vector_delete_impl(table, item_it, hp); \ + if (index == 0) { \ + return (struct table##_iterator){}; \ + } else { \ + it.entry--; \ + return it; \ + } \ + } else { \ + table##_delete_impl(table, it, hp); \ + return table##_next_impl(it, true); \ + } \ } \ \ __attribute__((always_inline)) \ @@ -1148,30 +1363,30 @@ static inline struct table##_iterator \ table##_delete_iterator(struct table *table, struct table##_iterator it) \ { \ struct hash_pair hp = {}; \ - \ - /* We only need the hash if the chunk hosts an overflowed entry. */ \ - if (table##_chunk_hosted_overflow_count(table##_iterator_chunk(it))) { \ + /* \ + * The basic policy only needs the hash if the chunk hosts an \ + * overflowed entry. \ + */ \ + if (table##_vector_policy || \ + table##_chunk_hosted_overflow_count(table##_iterator_chunk(it))) { \ table##_key_type key = table##_entry_to_key(it.entry); \ - \ hp = table##_hash(&key); \ } \ - table##_do_delete(table, it, hp); \ - return table##_next_impl(it, true); \ + return table##_delete_iterator_hashed(table, it, hp); \ } \ \ static bool table##_delete_hashed(struct table *table, \ const table##_key_type *key, \ struct hash_pair hp) \ { \ - struct table##_iterator it; \ - \ - it = table##_search_hashed(table, key, hp); \ - if (it.entry) { \ - table##_do_delete(table, it, hp); \ - return true; \ - } else { \ + struct table##_iterator item_it = table##_search_by_key(table, key, hp);\ + if (!item_it.item) \ return false; \ - } \ + if (table##_vector_policy) \ + table##_vector_delete_impl(table, item_it, hp); \ + else \ + table##_delete_impl(table, item_it, hp); \ + return true; \ } \ \ __attribute__((unused)) \ @@ -1183,13 +1398,34 @@ static bool table##_delete(struct table *table, const table##_key_type *key) \ __attribute__((unused)) \ static struct table##_iterator table##_first(struct table *table) \ { \ - return table##_unpack_iterator(table->first_packed); \ + if (table##_vector_policy) { \ + table##_entry_type *entry; \ + if (table##_empty(table)) \ + entry = NULL; \ + else \ + entry = &table->vector->entries[table##_size(table) - 1];\ + return (struct table##_iterator){ \ + .entry = entry, \ + .lowest = table->vector->entries, \ + }; \ + } else { \ + return table##_unpack_iterator(table->basic->first_packed); \ + } \ } \ \ __attribute__((unused)) \ static struct table##_iterator table##_next(struct table##_iterator it) \ { \ - return table##_next_impl(it, false); \ + if (table##_vector_policy) { \ + if (it.entry == it.lowest) { \ + return (struct table##_iterator){}; \ + } else { \ + it.entry--; \ + return it; \ + } \ + } else { \ + return table##_next_impl(it, false); \ + } \ } /** From e7f353c118960632fe6184c30e20396dc75bc55a Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Sat, 18 Jul 2020 01:19:35 -0700 Subject: [PATCH 11/18] libdrgn: hash_table: clean up coding style Clean up the coding style of the remaining few places that the last couple of changes didn't rewrite. Signed-off-by: Omar Sandoval --- libdrgn/hash_table.h | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/libdrgn/hash_table.h b/libdrgn/hash_table.h index 15dbdadb7..732232796 100644 --- a/libdrgn/hash_table.h +++ b/libdrgn/hash_table.h @@ -347,7 +347,6 @@ static inline unsigned int table##_chunk_match(struct table##_chunk *chunk, \ size_t needle) \ { \ unsigned int mask, i; \ - \ for (mask = 0, i = 0; i < table##_chunk_capacity; i++) { \ if (chunk->tags[i] == needle) \ mask |= 1U << i; \ @@ -359,7 +358,6 @@ static inline unsigned int table##_chunk_match(struct table##_chunk *chunk, \ static inline unsigned int table##_chunk_occupied(struct table##_chunk *chunk) \ { \ unsigned int mask, i; \ - \ for (mask = 0, i = 0; i < table##_chunk_capacity; i++) { \ if (chunk->tags[i]) \ mask |= 1U << i; \ @@ -633,18 +631,15 @@ HASH_TABLE_CHUNK_OCCUPIED(table) \ static inline unsigned int \ table##_chunk_first_empty(struct table##_chunk *chunk) \ { \ - unsigned int mask; \ - \ - mask = table##_chunk_occupied(chunk) ^ table##_chunk_full_mask; \ + unsigned int mask = \ + table##_chunk_occupied(chunk) ^ table##_chunk_full_mask; \ return mask ? ctz(mask) : (unsigned int)-1; \ } \ \ static inline unsigned int \ table##_chunk_last_occupied(struct table##_chunk *chunk) \ { \ - unsigned int mask; \ - \ - mask = table##_chunk_occupied(chunk); \ + unsigned int mask = table##_chunk_occupied(chunk); \ return mask ? fls(mask) - 1 : (unsigned int)-1; \ } \ \ @@ -1177,7 +1172,6 @@ static int table##_insert_hashed(struct table *table, \ { \ table##_key_type key = table##_entry_to_key(entry); \ struct table##_iterator it = table##_search_hashed(table, &key, hp); \ - \ if (it.entry) { \ if (it_ret) \ *it_ret = it; \ @@ -1193,7 +1187,6 @@ static int table##_insert(struct table *table, \ struct table##_iterator *it_ret) \ { \ table##_key_type key = table##_entry_to_key(entry); \ - \ return table##_insert_hashed(table, entry, table##_hash(&key), it_ret); \ } \ \ @@ -1565,7 +1558,6 @@ static inline struct hash_pair hash_pair_from_non_avalanching_hash(size_t hash) #ifdef __SSE4_2__ /* 64-bit with SSE4.2 uses CRC32 */ size_t c = _mm_crc32_u64(0, hash); - return (struct hash_pair){ .first = hash + c, .second = (c >> 24) | 0x80, @@ -1573,10 +1565,8 @@ static inline struct hash_pair hash_pair_from_non_avalanching_hash(size_t hash) #else /* 64-bit without SSE4.2 uses a 128-bit multiplication-based mixer */ static const uint64_t multiplier = UINT64_C(0xc4ceb9fe1a85ec53); - uint64_t hi, lo; - - hi = ((unsigned __int128)hash * multiplier) >> 64; - lo = hash * multiplier; + uint64_t hi = ((unsigned __int128)hash * multiplier) >> 64; + uint64_t lo = hash * multiplier; hash = hi ^ lo; hash *= multiplier; return (struct hash_pair){ @@ -1588,7 +1578,6 @@ static inline struct hash_pair hash_pair_from_non_avalanching_hash(size_t hash) #ifdef __SSE4_2__ /* 32-bit with SSE4.2 uses CRC32 */ size_t c = _mm_crc32_u32(0, hash); - return (struct hash_pair){ .first = hash + c, .second = (uint8_t)(~(c >> 25)), @@ -1626,7 +1615,6 @@ static inline uint64_t hash_128_to_64(unsigned __int128 hash) #define hash_pair_int_type(key) ({ \ __auto_type _key = *(key); \ - \ sizeof(_key) > sizeof(size_t) ? \ hash_pair_from_avalanching_hash(hash_128_to_64(_key)) : \ hash_pair_from_non_avalanching_hash(_key); \ @@ -1646,7 +1634,6 @@ static inline uint32_t hash_64_to_32(uint64_t hash) #define hash_pair_int_type(key) ({ \ __auto_type _key = *(key); \ - \ sizeof(_key) > sizeof(size_t) ? \ hash_pair_from_avalanching_hash(hash_64_to_32(_key)) : \ hash_pair_from_non_avalanching_hash(_key); \ @@ -1665,7 +1652,6 @@ struct hash_pair hash_pair_ptr_type(T * const *key); #else #define hash_pair_ptr_type(key) ({ \ uintptr_t _ptr = (uintptr_t)*key; \ - \ hash_pair_int_type(&_ptr); \ }) #endif @@ -1707,7 +1693,6 @@ struct hash_pair c_string_hash(const char * const *key); #define c_string_hash(key) ({ \ const char *_key = *(key); \ size_t _hash = cityhash_size_t(_key, strlen(_key)); \ - \ hash_pair_from_avalanching_hash(_hash); \ }) #endif @@ -1716,11 +1701,7 @@ struct hash_pair c_string_hash(const char * const *key); /** Compare two null-terminated string keys for equality. */ bool c_string_eq(const char * const *a, const char * const *b); #else -#define c_string_eq(a, b) ({ \ - const char *_a = *(a), *_b = *(b); \ - \ - (bool)(strcmp(_a, _b) == 0); \ -}) +#define c_string_eq(a, b) ((bool)(strcmp(*(a), *(b)) == 0)) #endif /** A string with a given length. */ @@ -1738,7 +1719,6 @@ struct string { static inline struct hash_pair string_hash(const struct string *key) { size_t hash = cityhash_size_t(key->str, key->len); - return hash_pair_from_avalanching_hash(hash); } From 27e73fb84b085aff81760bb4c91ed87f0b3a07db Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 27 Jul 2020 11:56:36 -0700 Subject: [PATCH 12/18] docs: fix broken link to drgn.h drgn.h is generated from drgn.h.in since commit d60c6a1d683e ("libdrgn: add register information to platform"). Signed-off-by: Omar Sandoval --- docs/advanced_usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index bc09c6fe4..6584aeaa1 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -35,7 +35,7 @@ The core functionality of drgn is implemented in C and is available as a C library, ``libdrgn``. See |drgn.h|_. .. |drgn.h| replace:: ``drgn.h`` -.. _drgn.h: https://github.com/osandov/drgn/blob/master/libdrgn/drgn.h +.. _drgn.h: https://github.com/osandov/drgn/blob/master/libdrgn/drgn.h.in Full documentation can be generated by running ``doxygen`` in the ``libdrgn`` directory of the source code. Note that the API and ABI are not yet stable. From d45aafe43ce4d2994212c98ecd4086f037c77afd Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 27 Jul 2020 12:00:35 -0700 Subject: [PATCH 13/18] vmtest: manage: remove 3.16 blacklist 3.16 is EOL and no longer included in the list of releases. Signed-off-by: Omar Sandoval --- vmtest/manage.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vmtest/manage.py b/vmtest/manage.py index 10058bba9..d74cbd6d7 100644 --- a/vmtest/manage.py +++ b/vmtest/manage.py @@ -97,11 +97,6 @@ async def get_kernel_org_versions(http_client: aiohttp.ClientSession) -> List[st release["version"] for release in releases if release["moniker"] in {"mainline", "stable", "longterm"} - # 3.16 seems to be missing "x86/build/64: Force the linker to use - # 2MB page size", so it doesn't even boot. It's projected to be EOL - # in June 2020 (https://www.kernel.org/category/releases.html), and - # 3.x is ancient anyways, so don't bother. - and not release["version"].startswith("3.") ] From d118fda740f1ebc0c5651d3866b891b1fbbc360e Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 27 Jul 2020 12:19:46 -0700 Subject: [PATCH 14/18] vmtest: check that downloaded file is not truncated My work VPN is apparently closing HTTP connections prematurely, which exposed that urllib won't catch incomplete reads if copied through shutil.copyfileobj(). Check it explicitly. Signed-off-by: Omar Sandoval --- vmtest/resolver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vmtest/resolver.py b/vmtest/resolver.py index ffa862ea7..25ea539ec 100644 --- a/vmtest/resolver.py +++ b/vmtest/resolver.py @@ -3,6 +3,7 @@ import fnmatch import glob +import http.client import os import os.path import queue @@ -110,6 +111,8 @@ def _download_file(self, name: str, *, compressed: bool = False) -> str: ) else: shutil.copyfileobj(u, f) + if u.length: + raise http.client.IncompleteRead(b"", u.length) # Passing dst_dir_fd forces Python to use linkat() with # AT_SYMLINK_FOLLOW instead of link(). See # https://bugs.python.org/msg348086. From e3309765f96b4b881ec31b953772d7a9c83745c2 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 27 Jul 2020 13:55:54 -0700 Subject: [PATCH 15/18] helpers: add kaslr_offset() and move pgtable_l5_enabled() Make the KASLR offset available to Python in a new drgn.helpers.linux.boot module, and move pgtable_l5_enabled() there, too. Signed-off-by: Omar Sandoval --- _drgn.pyi | 1 + drgn/helpers/linux/boot.py | 37 ++++++++++++++++++++++++++++++++ drgn/helpers/linux/mm.py | 12 +---------- libdrgn/python/drgnpy.h | 2 ++ libdrgn/python/helpers.c | 21 +++++++++++++++--- libdrgn/python/module.c | 3 +++ tests/helpers/linux/test_boot.py | 19 ++++++++++++++++ tests/helpers/linux/test_mm.py | 10 --------- 8 files changed, 81 insertions(+), 24 deletions(-) create mode 100644 drgn/helpers/linux/boot.py create mode 100644 tests/helpers/linux/test_boot.py diff --git a/_drgn.pyi b/_drgn.pyi index 2d9de196f..0dccb1251 100644 --- a/_drgn.pyi +++ b/_drgn.pyi @@ -1643,4 +1643,5 @@ def _linux_helper_find_pid(ns, pid): ... def _linux_helper_pid_task(pid, pid_type): ... def _linux_helper_find_task(ns, pid): ... def _linux_helper_task_state_to_char(task): ... +def _linux_helper_kaslr_offset(prog): ... def _linux_helper_pgtable_l5_enabled(prog): ... diff --git a/drgn/helpers/linux/boot.py b/drgn/helpers/linux/boot.py new file mode 100644 index 000000000..7d064960d --- /dev/null +++ b/drgn/helpers/linux/boot.py @@ -0,0 +1,37 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# SPDX-License-Identifier: GPL-3.0+ + +""" +Boot +---- + +The ``drgn.helpers.linux.boot`` module provides helpers for inspecting the +Linux kernel boot configuration. +""" + +from _drgn import _linux_helper_kaslr_offset, _linux_helper_pgtable_l5_enabled + + +__all__ = ( + "kaslr_offset", + "pgtable_l5_enabled", +) + + +def kaslr_offset(prog): + """ + .. c:function:: unsigned long kaslr_offset(void) + + Get the kernel address space layout randomization offset (zero if it is + disabled). + """ + return _linux_helper_kaslr_offset(prog) + + +def pgtable_l5_enabled(prog): + """ + .. c:function:: bool pgtable_l5_enabled(void) + + Return whether 5-level paging is enabled. + """ + return _linux_helper_pgtable_l5_enabled(prog) diff --git a/drgn/helpers/linux/mm.py b/drgn/helpers/linux/mm.py index 38d5ed712..684b2444f 100644 --- a/drgn/helpers/linux/mm.py +++ b/drgn/helpers/linux/mm.py @@ -12,7 +12,7 @@ from typing import List -from _drgn import _linux_helper_read_vm, _linux_helper_pgtable_l5_enabled +from _drgn import _linux_helper_read_vm from drgn import Object, cast @@ -26,21 +26,11 @@ "page_to_virt", "pfn_to_page", "pfn_to_virt", - "pgtable_l5_enabled", "virt_to_page", "virt_to_pfn", ) -def pgtable_l5_enabled(prog): - """ - .. c:function:: bool pgtable_l5_enabled(void) - - Return whether 5-level paging is enabled. - """ - return _linux_helper_pgtable_l5_enabled(prog) - - def for_each_page(prog): """ Iterate over all pages in the system. diff --git a/libdrgn/python/drgnpy.h b/libdrgn/python/drgnpy.h index 96c7d303a..0dd47174e 100644 --- a/libdrgn/python/drgnpy.h +++ b/libdrgn/python/drgnpy.h @@ -302,6 +302,8 @@ DrgnObject *drgnpy_linux_helper_find_task(PyObject *self, PyObject *args, PyObject *kwds); PyObject *drgnpy_linux_helper_task_state_to_char(PyObject *self, PyObject *args, PyObject *kwds); +PyObject *drgnpy_linux_helper_kaslr_offset(PyObject *self, PyObject *args, + PyObject *kwds); PyObject *drgnpy_linux_helper_pgtable_l5_enabled(PyObject *self, PyObject *args, PyObject *kwds); diff --git a/libdrgn/python/helpers.c b/libdrgn/python/helpers.c index 805255066..433e78310 100644 --- a/libdrgn/python/helpers.c +++ b/libdrgn/python/helpers.c @@ -237,19 +237,34 @@ PyObject *drgnpy_linux_helper_task_state_to_char(PyObject *self, PyObject *args, return PyUnicode_FromStringAndSize(&c, 1); } +PyObject *drgnpy_linux_helper_kaslr_offset(PyObject *self, PyObject *args, + PyObject *kwds) + +{ + static char *keywords[] = {"prog", NULL}; + Program *prog; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!:kaslr_offset", + keywords, &Program_type, &prog)) + return NULL; + + if (!(prog->prog.flags & DRGN_PROGRAM_IS_LINUX_KERNEL)) + return PyErr_Format(PyExc_ValueError, "not Linux kernel"); + return PyLong_FromUnsignedLongLong(prog->prog.vmcoreinfo.kaslr_offset); +} + PyObject *drgnpy_linux_helper_pgtable_l5_enabled(PyObject *self, PyObject *args, PyObject *kwds) { static char *keywords[] = {"prog", NULL}; Program *prog; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!:pgtable_l5_enabled", keywords, &Program_type, &prog)) return NULL; - if ((prog->prog.flags & DRGN_PROGRAM_IS_LINUX_KERNEL) && - prog->prog.vmcoreinfo.pgtable_l5_enabled) + if (!(prog->prog.flags & DRGN_PROGRAM_IS_LINUX_KERNEL)) + return PyErr_Format(PyExc_ValueError, "not Linux kernel"); + if (prog->prog.vmcoreinfo.pgtable_l5_enabled) Py_RETURN_TRUE; else Py_RETURN_FALSE; diff --git a/libdrgn/python/module.c b/libdrgn/python/module.c index 92d5185f4..43f182213 100644 --- a/libdrgn/python/module.c +++ b/libdrgn/python/module.c @@ -131,6 +131,9 @@ static PyMethodDef drgn_methods[] = { {"_linux_helper_task_state_to_char", (PyCFunction)drgnpy_linux_helper_task_state_to_char, METH_VARARGS | METH_KEYWORDS}, + {"_linux_helper_kaslr_offset", + (PyCFunction)drgnpy_linux_helper_kaslr_offset, + METH_VARARGS | METH_KEYWORDS}, {"_linux_helper_pgtable_l5_enabled", (PyCFunction)drgnpy_linux_helper_pgtable_l5_enabled, METH_VARARGS | METH_KEYWORDS}, diff --git a/tests/helpers/linux/test_boot.py b/tests/helpers/linux/test_boot.py new file mode 100644 index 000000000..ec2946fa2 --- /dev/null +++ b/tests/helpers/linux/test_boot.py @@ -0,0 +1,19 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# SPDX-License-Identifier: GPL-3.0+ + +import platform +import re +import unittest + +from drgn.helpers.linux.boot import pgtable_l5_enabled +from tests.helpers.linux import LinuxHelperTestCase + + +class TestBoot(LinuxHelperTestCase): + @unittest.skipUnless(platform.machine() == "x86_64", "machine is not x86_64") + def test_pgtable_l5_enabled(self): + with open("/proc/cpuinfo", "r") as f: + self.assertEqual( + pgtable_l5_enabled(self.prog), + bool(re.search(r"flags\s*:.*\bla57\b", f.read())), + ) diff --git a/tests/helpers/linux/test_mm.py b/tests/helpers/linux/test_mm.py index edd8dca89..114430183 100644 --- a/tests/helpers/linux/test_mm.py +++ b/tests/helpers/linux/test_mm.py @@ -6,7 +6,6 @@ import mmap import os import platform -import re import struct import tempfile import unittest @@ -20,7 +19,6 @@ page_to_pfn, pfn_to_page, pfn_to_virt, - pgtable_l5_enabled, virt_to_pfn, ) from drgn.helpers.linux.pid import find_task @@ -130,11 +128,3 @@ def test_environ(self): proc_environ = f.read().split(b"\0")[:-1] task = find_task(self.prog, os.getpid()) self.assertEqual(environ(task), proc_environ) - - @unittest.skipUnless(platform.machine() == "x86_64", "machine is not x86_64") - def test_pgtable_l5_enabled(self): - with open("/proc/cpuinfo", "r") as f: - self.assertEqual( - pgtable_l5_enabled(self.prog), - bool(re.search(r"flags\s*:.*\bla57\b", f.read())), - ) From 025989871b394ee5be47c0f25b6a552b8351d2db Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 27 Jul 2020 17:25:54 -0700 Subject: [PATCH 16/18] drgn 0.0.6 Signed-off-by: Omar Sandoval --- libdrgn/configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdrgn/configure.ac b/libdrgn/configure.ac index 650a4e8a3..0cbee4ab1 100644 --- a/libdrgn/configure.ac +++ b/libdrgn/configure.ac @@ -1,7 +1,7 @@ dnl Copyright (c) Facebook, Inc. and its affiliates. dnl SPDX-License-Identifier: GPL-3.0+ -AC_INIT([drgn], [0.0.5], +AC_INIT([drgn], [0.0.6], [https://github.com/osandov/drgn/issues],, [https://github.com/osandov/drgn]) From 9f3fadd3de8009ed6bf04d49cd4d5000ff2fcc0e Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 27 Jul 2020 23:28:23 -0700 Subject: [PATCH 17/18] README: use code-block instead of highlight PyPI's RST parser apparently doesn't know the highlight directive, which snuck into the README in commit 4de147e478cc ("Add CONTRIBUTING.rst"). Use code-block instead. Signed-off-by: Omar Sandoval --- README.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index c1d888465..eff3b1166 100644 --- a/README.rst +++ b/README.rst @@ -56,15 +56,17 @@ Installation .. start-install-dependencies -.. highlight:: console - Install dependencies: -Arch Linux:: +Arch Linux: + +.. code-block:: console $ sudo pacman -S --needed autoconf automake bison bzip2 flex gawk gcc libtool make pkgconf python python-setuptools xz zlib -Debian/Ubuntu:: +Debian/Ubuntu: + +.. code-block:: console $ sudo apt-get install autoconf automake bison flex gawk gcc libbz2-dev liblzma-dev libtool make pkgconf python3 python3-dev python3-setuptools zlib1g-dev @@ -72,7 +74,9 @@ Note that Debian Stretch, Ubuntu Trusty, and Ubuntu Xenial (and older) ship Python versions which are too old. Python 3.6 or newer must be installed manually. -Fedora:: +Fedora: + +.. code-block:: console $ sudo dnf install autoconf automake bison bzip2-devel flex gawk gcc libtool make pkgconf python3 python3-devel python3-setuptools xz-devel zlib-devel @@ -83,7 +87,9 @@ Optionally, install: .. end-install-dependencies -Then, run:: +Then, run: + +.. code-block:: console $ sudo pip3 install drgn From 20bcde1f1d27ccb17efb47c54fff9afa262464dc Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Mon, 27 Jul 2020 23:32:32 -0700 Subject: [PATCH 18/18] drgn 0.0.7 Signed-off-by: Omar Sandoval --- libdrgn/configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdrgn/configure.ac b/libdrgn/configure.ac index 0cbee4ab1..ef1d7d7f0 100644 --- a/libdrgn/configure.ac +++ b/libdrgn/configure.ac @@ -1,7 +1,7 @@ dnl Copyright (c) Facebook, Inc. and its affiliates. dnl SPDX-License-Identifier: GPL-3.0+ -AC_INIT([drgn], [0.0.6], +AC_INIT([drgn], [0.0.7], [https://github.com/osandov/drgn/issues],, [https://github.com/osandov/drgn])