From 9e3eac9243ef508f46ed99973905a5c71601a4dc Mon Sep 17 00:00:00 2001 From: Lionel Perrin Date: Fri, 6 Dec 2013 17:10:58 +0100 Subject: [PATCH 01/10] first naive implementation of virtual table * see test/test_vtable.rb for example * only select is supported (update/insert is not yet implemented) --- ext/sqlite3/database.c | 2 +- ext/sqlite3/database.h | 3 + ext/sqlite3/module.c | 244 +++++++++++++++++++++++++++++++++++++ ext/sqlite3/module.h | 16 +++ ext/sqlite3/sqlite3.c | 1 + ext/sqlite3/sqlite3_ruby.h | 1 + test/test_vtable.rb | 77 ++++++++++++ 7 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 ext/sqlite3/module.c create mode 100644 ext/sqlite3/module.h create mode 100644 test/test_vtable.rb diff --git a/ext/sqlite3/database.c b/ext/sqlite3/database.c index dea01dc0..697c10b7 100644 --- a/ext/sqlite3/database.c +++ b/ext/sqlite3/database.c @@ -334,7 +334,7 @@ static VALUE sqlite3val2rb(sqlite3_value * val) } } -static void set_sqlite3_func_result(sqlite3_context * ctx, VALUE result) +void set_sqlite3_func_result(sqlite3_context * ctx, VALUE result) { switch(TYPE(result)) { case T_NIL: diff --git a/ext/sqlite3/database.h b/ext/sqlite3/database.h index 63e5e961..bb8c3236 100644 --- a/ext/sqlite3/database.h +++ b/ext/sqlite3/database.h @@ -3,6 +3,9 @@ #include +// used by module.c too +void set_sqlite3_func_result(sqlite3_context * ctx, VALUE result); + struct _sqlite3Ruby { sqlite3 *db; }; diff --git a/ext/sqlite3/module.c b/ext/sqlite3/module.c new file mode 100644 index 00000000..7e87953f --- /dev/null +++ b/ext/sqlite3/module.c @@ -0,0 +1,244 @@ +#include +#include + +#undef ENABLE_TRACE + +#ifdef ENABLE_TRACE +static FILE* pf; +# define TRACE(str) \ + fprintf(pf, "%s:%d:%s\n", __FILE__, __LINE__, str); \ + fflush(pf); +#else +# define TRACE(str) ; +#endif + +VALUE cSqlite3Module; + +/** structure for ruby virtual table: inherits from sqlite3_vtab */ +typedef struct { + // mandatory sqlite3 fields + const sqlite3_module* pModule; + int nRef; + char *zErrMsg; + // Ruby fields + VALUE vtable; +} ruby_sqlite3_vtab; + +/** structure for ruby cursors: inherits from sqlite3_vtab_cursor */ +typedef struct { + ruby_sqlite3_vtab* pVTab; + VALUE row; + int rowid; +} ruby_sqlite3_vtab_cursor; + + +/** + * lookup for a ruby class :: and create an instance of this class + * This instance is then used to bind sqlite vtab callbacks + */ +static int xCreate(sqlite3* db, VALUE *module_name, + int argc, char **argv, + ruby_sqlite3_vtab **ppVTab, + char **pzErr) +{ + VALUE sql_stmt, module, ruby_class; + ID table_id, module_id; + VALUE ruby_class_args[0]; + const char* module_name_cstr = (const char*)StringValuePtr(*module_name); + const char* table_name_cstr = (const char*)argv[2]; + TRACE("xCreate"); + + + // lookup for ruby class named like :: + module_id = rb_intern( module_name_cstr ); + module = rb_const_get(rb_cObject, module_id); + table_id = rb_intern( table_name_cstr ); + ruby_class = rb_const_get(module, table_id); + + // alloc a new ruby_sqlite3_vtab object + // and store related attributes + (*ppVTab) = (ruby_sqlite3_vtab*)malloc(sizeof(ruby_sqlite3_vtab)); + + // create a new instance + (*ppVTab)->vtable = rb_class_new_instance(0, ruby_class_args, ruby_class); + + // call the create function + sql_stmt = rb_funcall((*ppVTab)->vtable, rb_intern("create_statement"), 0); + +#ifdef HAVE_RUBY_ENCODING_H + if(!UTF8_P(sql_stmt)) { + sql_stmt = rb_str_export_to_enc(sql_stmt, rb_utf8_encoding()); + } +#endif + if ( sqlite3_declare_vtab(db, StringValuePtr(sql_stmt)) ) + rb_raise(rb_eArgError, "fail to declare virtual table"); + + TRACE("xCreate done"); + return SQLITE_OK; +} + +static int xConnect(sqlite3* db, void *pAux, + int argc, char **argv, + ruby_sqlite3_vtab **ppVTab, + char **pzErr) +{ + TRACE("xConnect"); + return xCreate(db, pAux, argc, argv, ppVTab, pzErr); +} + +static int xBestIndex(ruby_sqlite3_vtab *pVTab, sqlite3_index_info* info) +{ + TRACE("xBestIndex"); + return SQLITE_OK; +} + +static int xDestroy(ruby_sqlite3_vtab *pVTab) +{ + TRACE("xDestroy"); + free(pVTab); + return SQLITE_OK; +} + +static int xDisconnect(ruby_sqlite3_vtab *pVTab) +{ + TRACE("xDisconnect"); + return xDestroy(pVTab); +} + +static int xOpen(ruby_sqlite3_vtab *pVTab, ruby_sqlite3_vtab_cursor **ppCursor) +{ + TRACE("xOpen"); + rb_funcall( pVTab->vtable, rb_intern("open"), 0 ); + *ppCursor = (ruby_sqlite3_vtab_cursor*)malloc(sizeof(ruby_sqlite3_vtab_cursor)); + (*ppCursor)->pVTab = pVTab; + (*ppCursor)->rowid = 0; + return SQLITE_OK; +} + +static int xClose(ruby_sqlite3_vtab_cursor* cursor) +{ + TRACE("xClose"); + rb_funcall( cursor->pVTab->vtable, rb_intern("close"), 0 ); + free(cursor); + return SQLITE_OK; +} + +static int xNext(ruby_sqlite3_vtab_cursor* cursor) +{ + TRACE("xNext"); + cursor->row = rb_funcall(cursor->pVTab->vtable, rb_intern("next"), 0); + ++(cursor->rowid); + return SQLITE_OK; +} + +static int xFilter(ruby_sqlite3_vtab_cursor* cursor, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv) +{ + TRACE("xFilter"); + cursor->rowid = 0; + return xNext(cursor); +} + +static int xEof(ruby_sqlite3_vtab_cursor* cursor) +{ + TRACE("xEof"); + return (cursor->row == Qnil); +} + +static int xColumn(ruby_sqlite3_vtab_cursor* cursor, sqlite3_context* context, int i) +{ + VALUE val = rb_ary_entry(cursor->row, i); + TRACE("xColumn(%d)"); + + set_sqlite3_func_result(context, val); + return SQLITE_OK; +} + +static int xRowid(ruby_sqlite3_vtab_cursor* cursor, sqlite_int64 *pRowid) +{ + TRACE("xRowid"); + *pRowid = cursor->rowid; + return SQLITE_OK; +} + +static sqlite3_module ruby_proxy_module = +{ + 0, /* iVersion */ + xCreate, /* xCreate - create a vtable */ + xConnect, /* xConnect - associate a vtable with a connection */ + xBestIndex, /* xBestIndex - best index */ + xDisconnect, /* xDisconnect - disassociate a vtable with a connection */ + xDestroy, /* xDestroy - destroy a vtable */ + xOpen, /* xOpen - open a cursor */ + xClose, /* xClose - close a cursor */ + xFilter, /* xFilter - configure scan constraints */ + xNext, /* xNext - advance a cursor */ + xEof, /* xEof - indicate end of result set*/ + xColumn, /* xColumn - read data */ + xRowid, /* xRowid - read data */ + NULL, /* xUpdate - write data */ + NULL, /* xBegin - begin transaction */ + NULL, /* xSync - sync transaction */ + NULL, /* xCommit - commit transaction */ + NULL, /* xRollback - rollback transaction */ + NULL, /* xFindFunction - function overloading */ +}; + +static void deallocate(void * ctx) +{ + sqlite3ModuleRubyPtr c = (sqlite3ModuleRubyPtr)ctx; + xfree(c); +} + +static VALUE allocate(VALUE klass) +{ + sqlite3ModuleRubyPtr ctx = xcalloc((size_t)1, sizeof(sqlite3ModuleRuby)); + ctx->module = NULL; + + return Data_Wrap_Struct(klass, NULL, deallocate, ctx); +} + +static VALUE initialize(VALUE self, VALUE db, VALUE name) +{ + sqlite3RubyPtr db_ctx; + sqlite3ModuleRubyPtr ctx; + + StringValue(name); + + Data_Get_Struct(db, sqlite3Ruby, db_ctx); + Data_Get_Struct(self, sqlite3ModuleRuby, ctx); + + + if(!db_ctx->db) + rb_raise(rb_eArgError, "initializing a module on a closed database"); + +#ifdef HAVE_RUBY_ENCODING_H + if(!UTF8_P(name)) { + name = rb_str_export_to_enc(name, rb_utf8_encoding()); + } +#endif + + // make possible to access to ruby object from c + ctx->module_name = name; + + sqlite3_create_module( + db_ctx->db, + (const char *)StringValuePtr(name), + &ruby_proxy_module, + &(ctx->module_name) //the vtable required the module name + ); + + TRACE("module initialized"); + return self; +} + +void init_sqlite3_module() +{ +#ifdef ENABLE_TRACE + pf = fopen("trace.log", "w"); +#endif + cSqlite3Module = rb_define_class_under(mSqlite3, "Module", rb_cObject); + rb_define_alloc_func(cSqlite3Module, allocate); + rb_define_method(cSqlite3Module, "initialize", initialize, 2); +} + diff --git a/ext/sqlite3/module.h b/ext/sqlite3/module.h new file mode 100644 index 00000000..2638d2a3 --- /dev/null +++ b/ext/sqlite3/module.h @@ -0,0 +1,16 @@ +#ifndef SQLITE3_MODULE_RUBY +#define SQLITE3_MODULE_RUBY + +#include + +struct _sqlite3ModuleRuby { + sqlite3_module *module; + VALUE module_name; // so that sqlite can bring the module_name up to the vtable +}; + +typedef struct _sqlite3ModuleRuby sqlite3ModuleRuby; +typedef sqlite3ModuleRuby * sqlite3ModuleRubyPtr; + +void init_sqlite3_module(); + +#endif diff --git a/ext/sqlite3/sqlite3.c b/ext/sqlite3/sqlite3.c index 1c02011f..0bc3deff 100644 --- a/ext/sqlite3/sqlite3.c +++ b/ext/sqlite3/sqlite3.c @@ -135,6 +135,7 @@ void Init_sqlite3_native() #ifdef HAVE_SQLITE3_BACKUP_INIT init_sqlite3_backup(); #endif + init_sqlite3_module(); rb_define_singleton_method(mSqlite3, "libversion", libversion, 0); rb_define_singleton_method(mSqlite3, "threadsafe", threadsafe_p, 0); diff --git a/ext/sqlite3/sqlite3_ruby.h b/ext/sqlite3/sqlite3_ruby.h index 9e9e1751..fc475b2c 100644 --- a/ext/sqlite3/sqlite3_ruby.h +++ b/ext/sqlite3/sqlite3_ruby.h @@ -46,6 +46,7 @@ extern VALUE cSqlite3Blob; #include #include #include +#include int bignum_to_int64(VALUE big, sqlite3_int64 *result); diff --git a/test/test_vtable.rb b/test/test_vtable.rb new file mode 100644 index 00000000..3bb6cdb5 --- /dev/null +++ b/test/test_vtable.rb @@ -0,0 +1,77 @@ +require "helper" +require "sqlite3" +require "sqlite3_native" + +#the ruby module name should be the one given to sqlite when creating the virtual table. +module RubyModule + class TestVTable + def initialize + @str = "A"*1500 + end + + #required method for vtable + #this method is needed to declare the type of each column to sqlite + def create_statement + "create table TestVTable(s text, x integer, y int)" + end + + #required method for vtable + #called before each statement + def open + @count = 0 + end + + #required method for vtable + #called to retrieve a new row + def next + + #produce up to 100000 lines + @count += 1 + if @count <= 100000 + [@str, rand(10), rand] + else + nil + end + + end + + #required method for vtable + #called after each statement + def close + end + end +end + +module SQLite3 + class TestVTable < SQLite3::TestCase + def setup + @db = SQLite3::Database.new(":memory:") + @m = SQLite3::Module.new(@db, "RubyModule") + end + + def test_exception_module + #the following line throws an exception because NonExistingModule is not valid ruby module + assert_raise SQLite3::SQLException do + @db.execute("create virtual table TestVTable using NonExistingModule") + end + end + + def test_exception_table + #the following line throws an exception because no ruby class RubyModule::NonExistingVTable is found as vtable implementation + assert_raise NameError do + @db.execute("create virtual table NonExistingVTable using RubyModule") + end + end + + def test_working + #this will instantiate a new virtual table using implementation from RubyModule::TestVTable + @db.execute("create virtual table if not exists TestVTable using RubyModule") + + #execute an sql statement + nb_row = @db.execute("select x, sum(y), avg(y), avg(y*y), min(y), max(y), count(y) from TestVTable group by x").each.count + assert( nb_row > 0 ) + end + + end if defined?(SQLite3::Module) +end + From 0fcff838737e26614c9063774a83f16ecf151167 Mon Sep 17 00:00:00 2001 From: lionel Date: Mon, 9 Dec 2013 21:32:31 +0100 Subject: [PATCH 02/10] - remove non necessary macro TRACE from module.c - fix unit test so that it can be launched using 'rake' --- ext/sqlite3/module.c | 29 ----------------------------- test/test_vtable.rb | 2 -- 2 files changed, 31 deletions(-) diff --git a/ext/sqlite3/module.c b/ext/sqlite3/module.c index 7e87953f..0a3954fa 100644 --- a/ext/sqlite3/module.c +++ b/ext/sqlite3/module.c @@ -1,17 +1,6 @@ #include #include -#undef ENABLE_TRACE - -#ifdef ENABLE_TRACE -static FILE* pf; -# define TRACE(str) \ - fprintf(pf, "%s:%d:%s\n", __FILE__, __LINE__, str); \ - fflush(pf); -#else -# define TRACE(str) ; -#endif - VALUE cSqlite3Module; /** structure for ruby virtual table: inherits from sqlite3_vtab */ @@ -46,8 +35,6 @@ static int xCreate(sqlite3* db, VALUE *module_name, VALUE ruby_class_args[0]; const char* module_name_cstr = (const char*)StringValuePtr(*module_name); const char* table_name_cstr = (const char*)argv[2]; - TRACE("xCreate"); - // lookup for ruby class named like :: module_id = rb_intern( module_name_cstr ); @@ -73,7 +60,6 @@ static int xCreate(sqlite3* db, VALUE *module_name, if ( sqlite3_declare_vtab(db, StringValuePtr(sql_stmt)) ) rb_raise(rb_eArgError, "fail to declare virtual table"); - TRACE("xCreate done"); return SQLITE_OK; } @@ -82,32 +68,27 @@ static int xConnect(sqlite3* db, void *pAux, ruby_sqlite3_vtab **ppVTab, char **pzErr) { - TRACE("xConnect"); return xCreate(db, pAux, argc, argv, ppVTab, pzErr); } static int xBestIndex(ruby_sqlite3_vtab *pVTab, sqlite3_index_info* info) { - TRACE("xBestIndex"); return SQLITE_OK; } static int xDestroy(ruby_sqlite3_vtab *pVTab) { - TRACE("xDestroy"); free(pVTab); return SQLITE_OK; } static int xDisconnect(ruby_sqlite3_vtab *pVTab) { - TRACE("xDisconnect"); return xDestroy(pVTab); } static int xOpen(ruby_sqlite3_vtab *pVTab, ruby_sqlite3_vtab_cursor **ppCursor) { - TRACE("xOpen"); rb_funcall( pVTab->vtable, rb_intern("open"), 0 ); *ppCursor = (ruby_sqlite3_vtab_cursor*)malloc(sizeof(ruby_sqlite3_vtab_cursor)); (*ppCursor)->pVTab = pVTab; @@ -117,7 +98,6 @@ static int xOpen(ruby_sqlite3_vtab *pVTab, ruby_sqlite3_vtab_cursor **ppCursor) static int xClose(ruby_sqlite3_vtab_cursor* cursor) { - TRACE("xClose"); rb_funcall( cursor->pVTab->vtable, rb_intern("close"), 0 ); free(cursor); return SQLITE_OK; @@ -125,7 +105,6 @@ static int xClose(ruby_sqlite3_vtab_cursor* cursor) static int xNext(ruby_sqlite3_vtab_cursor* cursor) { - TRACE("xNext"); cursor->row = rb_funcall(cursor->pVTab->vtable, rb_intern("next"), 0); ++(cursor->rowid); return SQLITE_OK; @@ -134,21 +113,18 @@ static int xNext(ruby_sqlite3_vtab_cursor* cursor) static int xFilter(ruby_sqlite3_vtab_cursor* cursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv) { - TRACE("xFilter"); cursor->rowid = 0; return xNext(cursor); } static int xEof(ruby_sqlite3_vtab_cursor* cursor) { - TRACE("xEof"); return (cursor->row == Qnil); } static int xColumn(ruby_sqlite3_vtab_cursor* cursor, sqlite3_context* context, int i) { VALUE val = rb_ary_entry(cursor->row, i); - TRACE("xColumn(%d)"); set_sqlite3_func_result(context, val); return SQLITE_OK; @@ -156,7 +132,6 @@ static int xColumn(ruby_sqlite3_vtab_cursor* cursor, sqlite3_context* context, i static int xRowid(ruby_sqlite3_vtab_cursor* cursor, sqlite_int64 *pRowid) { - TRACE("xRowid"); *pRowid = cursor->rowid; return SQLITE_OK; } @@ -228,15 +203,11 @@ static VALUE initialize(VALUE self, VALUE db, VALUE name) &(ctx->module_name) //the vtable required the module name ); - TRACE("module initialized"); return self; } void init_sqlite3_module() { -#ifdef ENABLE_TRACE - pf = fopen("trace.log", "w"); -#endif cSqlite3Module = rb_define_class_under(mSqlite3, "Module", rb_cObject); rb_define_alloc_func(cSqlite3Module, allocate); rb_define_method(cSqlite3Module, "initialize", initialize, 2); diff --git a/test/test_vtable.rb b/test/test_vtable.rb index 3bb6cdb5..33c1f7a1 100644 --- a/test/test_vtable.rb +++ b/test/test_vtable.rb @@ -1,6 +1,4 @@ require "helper" -require "sqlite3" -require "sqlite3_native" #the ruby module name should be the one given to sqlite when creating the virtual table. module RubyModule From fc2f818f6a928ef8504c4349e5c8b5fc5bac0bb6 Mon Sep 17 00:00:00 2001 From: Lionel Perrin Date: Tue, 29 Mar 2016 14:13:29 +0200 Subject: [PATCH 03/10] vtable version --- lib/sqlite3/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlite3/version.rb b/lib/sqlite3/version.rb index 1c193d4b..da66d243 100644 --- a/lib/sqlite3/version.rb +++ b/lib/sqlite3/version.rb @@ -1,6 +1,6 @@ module SQLite3 - VERSION = '1.3.11' + VERSION = '1.3.11.vtable' module VersionProxy From 410d6b156e8ad9228e36764ead75debd22960a6c Mon Sep 17 00:00:00 2001 From: Lionel Perrin Date: Tue, 29 Mar 2016 14:55:05 +0200 Subject: [PATCH 04/10] add missing files to manifest.txt --- Manifest.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Manifest.txt b/Manifest.txt index ca812a88..61c663f3 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -13,6 +13,8 @@ ext/sqlite3/database.h ext/sqlite3/exception.c ext/sqlite3/exception.h ext/sqlite3/extconf.rb +ext/sqlite3/module.c +ext/sqlite3/module.h ext/sqlite3/sqlite3.c ext/sqlite3/sqlite3_ruby.h ext/sqlite3/statement.c @@ -38,7 +40,9 @@ test/helper.rb test/test_backup.rb test/test_collation.rb test/test_database.rb +test/test_database_flags.rb test/test_database_readonly.rb +test/test_database_readwrite.rb test/test_deprecated.rb test/test_encoding.rb test/test_integration.rb @@ -50,3 +54,4 @@ test/test_result_set.rb test/test_sqlite3.rb test/test_statement.rb test/test_statement_execute.rb +test/test_vtable.rb From 2054e2ec2fbc05793ebee0df6cf90f61d7d72500 Mon Sep 17 00:00:00 2001 From: Lionel Perrin Date: Thu, 31 Mar 2016 09:53:26 +0200 Subject: [PATCH 05/10] ease vtable declaration using SQLite3::vtable --- ext/sqlite3/module.c | 6 ++++- lib/sqlite3/vtable.rb | 53 +++++++++++++++++++++++++++++++++++++++++++ test/test_vtable.rb | 8 ++----- 3 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 lib/sqlite3/vtable.rb diff --git a/ext/sqlite3/module.c b/ext/sqlite3/module.c index 0a3954fa..dd79ffea 100644 --- a/ext/sqlite3/module.c +++ b/ext/sqlite3/module.c @@ -36,6 +36,9 @@ static int xCreate(sqlite3* db, VALUE *module_name, const char* module_name_cstr = (const char*)StringValuePtr(*module_name); const char* table_name_cstr = (const char*)argv[2]; + // method will raise in case of error: no need to use pzErr + *pzErr = NULL; + // lookup for ruby class named like :: module_id = rb_intern( module_name_cstr ); module = rb_const_get(rb_cObject, module_id); @@ -57,8 +60,9 @@ static int xCreate(sqlite3* db, VALUE *module_name, sql_stmt = rb_str_export_to_enc(sql_stmt, rb_utf8_encoding()); } #endif - if ( sqlite3_declare_vtab(db, StringValuePtr(sql_stmt)) ) + if ( sqlite3_declare_vtab(db, StringValuePtr(sql_stmt)) ) { rb_raise(rb_eArgError, "fail to declare virtual table"); + } return SQLITE_OK; } diff --git a/lib/sqlite3/vtable.rb b/lib/sqlite3/vtable.rb new file mode 100644 index 00000000..65f68af8 --- /dev/null +++ b/lib/sqlite3/vtable.rb @@ -0,0 +1,53 @@ +module SQLite3Vtable + # this module contains the vtable classes generated + # using SQLite3::vtable method +end + +module SQLite3 + class VTableInterface + #this method is needed to declare the type of each column to sqlite + def create_statement + fail 'VTableInterface#create_statement not implemented' + end + + #called before each statement + def open + # do nothing by default + end + + #called before each statement + def close + # do nothing by default + end + + # called to retrieve a new row + def next + fail 'VTableInterface#next not implemented' + end + end + + def self.vtable(db, table_name, table_columns) + if SQLite3Vtable.const_defined?(table_name, inherit = false) + raise "'#{table_name}' already declared" + end + + klass = Class.new(VTableInterface) do + def initialize(enumerable) + @enumerable = enumerable + end + def create_statement + "create table #{table_name}(#{table_columns})" + end + def next + @enumerable.next + end + end + + begin + SQLite3Vtable.const_set(table_name, klass) + rescue NameError + raise "'#{table_name}' must be a valid ruby constant name" + end + db.execute("create virtual table #{table_name} using SQLite3Vtable") + end +end diff --git a/test/test_vtable.rb b/test/test_vtable.rb index 33c1f7a1..ab0e184d 100644 --- a/test/test_vtable.rb +++ b/test/test_vtable.rb @@ -1,8 +1,9 @@ require "helper" +require 'sqlite3/vtable' #the ruby module name should be the one given to sqlite when creating the virtual table. module RubyModule - class TestVTable + class TestVTable < VTableInterface def initialize @str = "A"*1500 end @@ -32,11 +33,6 @@ def next end end - - #required method for vtable - #called after each statement - def close - end end end From 4367569d11764af0e64bb40ed2c362118b75953e Mon Sep 17 00:00:00 2001 From: Lionel Perrin Date: Fri, 1 Apr 2016 14:59:38 +0200 Subject: [PATCH 06/10] support for best_index/filter --- Manifest.txt | 1 + ext/sqlite3/database.c | 2 +- ext/sqlite3/database.h | 1 + ext/sqlite3/module.c | 134 +++++++++++++++++++++++++++++++++++++++++ lib/sqlite3/vtable.rb | 44 ++++++++++---- test/test_vtable.rb | 67 ++++++++++++++++++++- 6 files changed, 234 insertions(+), 15 deletions(-) diff --git a/Manifest.txt b/Manifest.txt index 61c663f3..30d9d226 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -31,6 +31,7 @@ lib/sqlite3/statement.rb lib/sqlite3/translator.rb lib/sqlite3/value.rb lib/sqlite3/version.rb +lib/sqlite3/vtable.rb setup.rb tasks/faq.rake tasks/gem.rake diff --git a/ext/sqlite3/database.c b/ext/sqlite3/database.c index 697c10b7..6f7810de 100644 --- a/ext/sqlite3/database.c +++ b/ext/sqlite3/database.c @@ -297,7 +297,7 @@ static VALUE last_insert_row_id(VALUE self) return LL2NUM(sqlite3_last_insert_rowid(ctx->db)); } -static VALUE sqlite3val2rb(sqlite3_value * val) +VALUE sqlite3val2rb(sqlite3_value * val) { switch(sqlite3_value_type(val)) { case SQLITE_INTEGER: diff --git a/ext/sqlite3/database.h b/ext/sqlite3/database.h index bb8c3236..0c5df531 100644 --- a/ext/sqlite3/database.h +++ b/ext/sqlite3/database.h @@ -5,6 +5,7 @@ // used by module.c too void set_sqlite3_func_result(sqlite3_context * ctx, VALUE result); +VALUE sqlite3val2rb(sqlite3_value * val); struct _sqlite3Ruby { sqlite3 *db; diff --git a/ext/sqlite3/module.c b/ext/sqlite3/module.c index dd79ffea..240722c2 100644 --- a/ext/sqlite3/module.c +++ b/ext/sqlite3/module.c @@ -75,8 +75,136 @@ static int xConnect(sqlite3* db, void *pAux, return xCreate(db, pAux, argc, argv, ppVTab, pzErr); } +static VALUE constraint_op_as_symbol(unsigned char op) +{ + ID op_id; + switch(op) { + case SQLITE_INDEX_CONSTRAINT_EQ: + op_id = rb_intern("=="); + break; + case SQLITE_INDEX_CONSTRAINT_GT: + op_id = rb_intern(">"); + break; + case SQLITE_INDEX_CONSTRAINT_LE: + op_id = rb_intern("<="); + break; + case SQLITE_INDEX_CONSTRAINT_LT: + op_id = rb_intern("<"); + break; + case SQLITE_INDEX_CONSTRAINT_GE: + op_id = rb_intern(">="); + break; + case SQLITE_INDEX_CONSTRAINT_MATCH: + op_id = rb_intern("match"); + break; +#if SQLITE_VERSION_NUMBER>=3010000 + case SQLITE_INDEX_CONSTRAINT_LIKE: + op_id = rb_intern("like"); + break; + case SQLITE_INDEX_CONSTRAINT_GLOB: + op_id = rb_intern("glob"); + break; + case SQLITE_INDEX_CONSTRAINT_REGEXP: + op_id = rb_intern("regexp"); + break; +#endif +#if SQLITE_VERSION_NUMBER>=3009000 + case SQLITE_INDEX_SCAN_UNIQUE: + op_id = rb_intern("unique"); + break; +#endif + default: + op_id = rb_intern("unsupported"); + } + return ID2SYM(op_id); +} + +static VALUE constraint_to_ruby(const struct sqlite3_index_constraint* c) +{ + VALUE cons = rb_ary_new2(2); + rb_ary_store(cons, 0, LONG2FIX(c->iColumn)); + rb_ary_store(cons, 1, constraint_op_as_symbol(c->op)); + return cons; +} + +static VALUE order_by_to_ruby(const struct sqlite3_index_orderby* c) +{ + VALUE order_by = rb_ary_new2(2); + rb_ary_store(order_by, 0, LONG2FIX(c->iColumn)); + rb_ary_store(order_by, 1, LONG2FIX(1-2*c->desc)); + return order_by; +} + static int xBestIndex(ruby_sqlite3_vtab *pVTab, sqlite3_index_info* info) { + int i; + VALUE constraint = rb_ary_new(); + VALUE order_by = rb_ary_new2(info->nOrderBy); + VALUE ret, idx_num, estimated_cost, order_by_consumed, omit_all; +#if SQLITE_VERSION_NUMBER >= 3008002 + VALUE estimated_rows; +#endif +#if SQLITE_VERSION_NUMBER >= 3009000 + VALUE idx_flags; +#endif +#if SQLITE_VERSION_NUMBER >= 3010000 + VALUE col_used; +#endif + + // convert constraints to ruby + for (i = 0; i < info->nConstraint; ++i) { + if (info->aConstraint[i].usable) { + rb_ary_push(constraint, constraint_to_ruby(info->aConstraint + i)); + } else { + printf("ignoring %d %d\n", info->aConstraint[i].iColumn, info->aConstraint[i].op); + } + } + + // convert order_by to ruby + for (i = 0; i < info->nOrderBy; ++i) { + rb_ary_store(order_by, i, order_by_to_ruby(info->aOrderBy + i)); + } + + + ret = rb_funcall( pVTab->vtable, rb_intern("best_index"), 2, constraint, order_by ); + if (ret != Qnil ) { + if (!RB_TYPE_P(ret, T_HASH)) { + rb_raise(rb_eTypeError, "best_index: expect returned value to be a Hash"); + } + idx_num = rb_hash_aref(ret, ID2SYM(rb_intern("idxNum"))); + if (idx_num == Qnil ) { + rb_raise(rb_eKeyError, "best_index: mandatory key 'idxNum' not found"); + } + info->idxNum = FIX2INT(idx_num); + estimated_cost = rb_hash_aref(ret, ID2SYM(rb_intern("estimatedCost"))); + if (estimated_cost != Qnil) { info->estimatedCost = NUM2DBL(estimated_cost); } + order_by_consumed = rb_hash_aref(ret, ID2SYM(rb_intern("orderByConsumed"))); + info->orderByConsumed = RTEST(order_by_consumed); +#if SQLITE_VERSION_NUMBER >= 3008002 + estimated_rows = rb_hash_aref(ret, ID2SYM(rb_intern("estimatedRows"))); + if (estimated_rows != Qnil) { bignum_to_int64(estimated_rows, &info->estimatedRows); } +#endif +#if SQLITE_VERSION_NUMBER >= 3009000 + idx_flags = rb_hash_aref(ret, ID2SYM(rb_intern("idxFlags"))); + if (idx_flags != Qnil) { info->idxFlags = FIX2INT(idx_flags); } +#endif +#if SQLITE_VERSION_NUMBER >= 3010000 + col_used = rb_hash_aref(ret, ID2SYM(rb_intern("colUsed"))); + if (col_used != Qnil) { bignum_to_int64(col_used, &info->colUsed); } +#endif + + // make sure that expression are given to filter + omit_all = rb_hash_aref(ret, ID2SYM(rb_intern("omitAllConstraint"))); + for (i = 0; i < info->nConstraint; ++i) { + if (RTEST(omit_all)) { + info->aConstraintUsage[i].omit = 1; + } + if (info->aConstraint[i].usable) { + info->aConstraintUsage[i].argvIndex = (i+1); + } + } + } + return SQLITE_OK; } @@ -117,6 +245,12 @@ static int xNext(ruby_sqlite3_vtab_cursor* cursor) static int xFilter(ruby_sqlite3_vtab_cursor* cursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv) { + int i; + VALUE argv_ruby = rb_ary_new2(argc); + for (i = 0; i < argc; ++i) { + rb_ary_store(argv_ruby, i, sqlite3val2rb(argv[i])); + } + rb_funcall( cursor->pVTab->vtable, rb_intern("filter"), 2, LONG2FIX(idxNum), argv_ruby ); cursor->rowid = 0; return xNext(cursor); } diff --git a/lib/sqlite3/vtable.rb b/lib/sqlite3/vtable.rb index 65f68af8..5f0b1189 100644 --- a/lib/sqlite3/vtable.rb +++ b/lib/sqlite3/vtable.rb @@ -1,4 +1,4 @@ -module SQLite3Vtable +module SQLite3_VTables # this module contains the vtable classes generated # using SQLite3::vtable method end @@ -20,34 +20,52 @@ def close # do nothing by default end + #called to define the best suitable index + def best_index(constraint, order_by) + # one can return an evaluation of the index as shown below + # { idxNum: 1, estimatedCost: 10.0, orderByConsumed: true } + # see sqlite documentation for more details + end + + # may be called several times between open/close + # it initialize/reset cursor + def filter(idxNum, args) + fail 'VTableInterface#filter not implemented' + end + # called to retrieve a new row def next fail 'VTableInterface#next not implemented' end end - def self.vtable(db, table_name, table_columns) - if SQLite3Vtable.const_defined?(table_name, inherit = false) + def self.vtable(db, table_name, table_columns, enumerable) + Module.new(db, 'SQLite3_VTables') + if SQLite3_VTables.const_defined?(table_name, inherit = false) raise "'#{table_name}' already declared" end - klass = Class.new(VTableInterface) do - def initialize(enumerable) - @enumerable = enumerable - end - def create_statement - "create table #{table_name}(#{table_columns})" - end - def next + klass = Class.new(VTableInterface) + klass.send(:define_method, :filter) do |idxNum, args| + @enumerable = enumerable.to_enum + end + klass.send(:define_method, :create_statement) do + "create table #{table_name}(#{table_columns})" + end + klass.send(:define_method, :next) do + begin @enumerable.next + rescue StopIteration + nil end end begin - SQLite3Vtable.const_set(table_name, klass) + SQLite3_VTables.const_set(table_name, klass) rescue NameError raise "'#{table_name}' must be a valid ruby constant name" end - db.execute("create virtual table #{table_name} using SQLite3Vtable") + db.execute("create virtual table #{table_name} using SQLite3_VTables") + klass end end diff --git a/test/test_vtable.rb b/test/test_vtable.rb index ab0e184d..3336d848 100644 --- a/test/test_vtable.rb +++ b/test/test_vtable.rb @@ -3,7 +3,7 @@ #the ruby module name should be the one given to sqlite when creating the virtual table. module RubyModule - class TestVTable < VTableInterface + class TestVTable < SQLite3::VTableInterface def initialize @str = "A"*1500 end @@ -17,6 +17,10 @@ def create_statement #required method for vtable #called before each statement def open + end + + # this method initialize/reset cursor + def filter(id, args) @count = 0 end @@ -66,6 +70,67 @@ def test_working assert( nb_row > 0 ) end + def test_vtable + SQLite3.vtable(@db, 'TestVTable2', 'a, b, c', [ + [1, 2, 3], + [2, 4, 6], + [3, 6, 9] + ]) + nb_row = @db.execute('select count(*) from TestVTable2').each.first[0] + assert( nb_row == 3 ) + sum_a, sum_b, sum_c = *@db.execute('select sum(a), sum(b), sum(c) from TestVTable2').each.first + assert( sum_a = 6 ) + assert( sum_b == 12 ) + assert( sum_c == 18 ) + end + + def test_multiple_vtable + SQLite3.vtable(@db, 'TestVTable3', 'col1', [['a'], ['b']]) + SQLite3.vtable(@db, 'TestVTable4', 'col2', [['c'], ['d']]) + rows = @db.execute('select col1, col2 from TestVTable3, TestVTable4').each.to_a + assert( rows.include?(['a', 'c']) ) + assert( rows.include?(['a', 'd']) ) + assert( rows.include?(['b', 'c']) ) + assert( rows.include?(['b', 'd']) ) + end + + def test_best_filter + test = self + SQLite3.vtable(@db, 'TestVTable5', 'col1, col2', [['a', 1], ['b', 2]]).tap do |vtable| + vtable.send(:define_method, :best_index) do |constraint, order_by| + # check constraint + test.assert( constraint.include?([0, :<=]) ) # col1 <= 'c' + test.assert( constraint.include?([0, :>]) ) # col1 > 'a' + test.assert( constraint.include?([1, :<]) ) # col2 < 3 + @constraint = constraint + + # check order by + test.assert( order_by == [ + [1, 1], # col2 + [0, -1], # col1 desc + ] ) + + { idxNum: 45 } + end + vtable.send(:alias_method, :orig_filter, :filter) + vtable.send(:define_method, :filter) do |idxNum, args| + # idxNum should be the one returned by best_index + test.assert( idxNum == 45 ) + + # args should be consistent with the constraint given to best_index + test.assert( args.size == @constraint.size ) + filters = @constraint.zip(args) + test.assert( filters.include?([[0, :<=], 'c']) ) # col1 <= 'c' + test.assert( filters.include?([[0, :>], 'a']) ) # col1 > 'a' + test.assert( filters.include?([[1, :<], 3]) ) # col2 < 3 + + orig_filter(idxNum, args) + end + end + rows = @db.execute('select col1 from TestVTable5 where col1 <= \'c\' and col1 > \'a\' and col2 < 3 order by col2, col1 desc').each.to_a + assert( rows == [['b']] ) + end + end if defined?(SQLite3::Module) end From 06e1f924de2ed6d2c85dc97036a47708263c2e78 Mon Sep 17 00:00:00 2001 From: perrinl Date: Mon, 4 Apr 2016 09:04:14 +0200 Subject: [PATCH 07/10] fix naming conflict between SQLite3::Module and ::Module --- lib/sqlite3/database.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlite3/database.rb b/lib/sqlite3/database.rb index 6c02bb15..5d897ecd 100644 --- a/lib/sqlite3/database.rb +++ b/lib/sqlite3/database.rb @@ -393,7 +393,7 @@ def self.finalize( &block ) end proxy = factory.new - proxy.extend(Module.new { + proxy.extend(::Module.new { attr_accessor :ctx def step( *args ) From 2bea1807dca208a346fca3cdeead5ce9ad7f0681 Mon Sep 17 00:00:00 2001 From: Lionel Perrin Date: Mon, 4 Apr 2016 09:45:33 +0200 Subject: [PATCH 08/10] fix tests on windows: change path comparison so that c:\tmp can be seen as equal to c:/tmp --- test/helper.rb | 6 ++++++ test/test_database.rb | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test/helper.rb b/test/helper.rb index efa4a39d..b50a94f4 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -1,5 +1,6 @@ require 'sqlite3' require 'minitest/autorun' +require 'pathname' unless RUBY_VERSION >= "1.9" require 'iconv' @@ -11,6 +12,11 @@ class TestCase < Minitest::Test alias :assert_not_nil :refute_nil alias :assert_raise :assert_raises + + def assert_path_equal(p1, p2) + assert_equal( Pathname.new(p1).realpath, Pathname.new(p2).realpath ) + end + def assert_nothing_raised yield end diff --git a/test/test_database.rb b/test/test_database.rb index c50e05d3..539bad31 100644 --- a/test/test_database.rb +++ b/test/test_database.rb @@ -19,7 +19,7 @@ def test_db_filename assert_equal '', @db.filename('main') tf = Tempfile.new 'thing' @db = SQLite3::Database.new tf.path - assert_equal tf.path, @db.filename('main') + assert_path_equal tf.path, @db.filename('main') ensure tf.unlink if tf end @@ -29,7 +29,7 @@ def test_filename assert_equal '', @db.filename tf = Tempfile.new 'thing' @db = SQLite3::Database.new tf.path - assert_equal tf.path, @db.filename + assert_path_equal tf.path, @db.filename ensure tf.unlink if tf end @@ -39,7 +39,7 @@ def test_filename_with_attachment assert_equal '', @db.filename tf = Tempfile.new 'thing' @db.execute "ATTACH DATABASE '#{tf.path}' AS 'testing'" - assert_equal tf.path, @db.filename('testing') + assert_path_equal tf.path, @db.filename('testing') ensure tf.unlink if tf end From 65717fa5f9311b9b5293ad46ffbe09e9c0d0cf13 Mon Sep 17 00:00:00 2001 From: Lionel Perrin Date: Mon, 4 Apr 2016 10:33:57 +0200 Subject: [PATCH 09/10] test_vtable: use assert_operator/assert_equal instead of simple assert for better error messages --- test/test_vtable.rb | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/test_vtable.rb b/test/test_vtable.rb index 3336d848..c5b34638 100644 --- a/test/test_vtable.rb +++ b/test/test_vtable.rb @@ -67,7 +67,7 @@ def test_working #execute an sql statement nb_row = @db.execute("select x, sum(y), avg(y), avg(y*y), min(y), max(y), count(y) from TestVTable group by x").each.count - assert( nb_row > 0 ) + assert_operator nb_row, :>, 0 end def test_vtable @@ -77,21 +77,21 @@ def test_vtable [3, 6, 9] ]) nb_row = @db.execute('select count(*) from TestVTable2').each.first[0] - assert( nb_row == 3 ) + assert_equal( 3, nb_row ) sum_a, sum_b, sum_c = *@db.execute('select sum(a), sum(b), sum(c) from TestVTable2').each.first - assert( sum_a = 6 ) - assert( sum_b == 12 ) - assert( sum_c == 18 ) + assert_equal( 6, sum_a ) + assert_equal( 12, sum_b ) + assert_equal( 18, sum_c ) end def test_multiple_vtable SQLite3.vtable(@db, 'TestVTable3', 'col1', [['a'], ['b']]) SQLite3.vtable(@db, 'TestVTable4', 'col2', [['c'], ['d']]) rows = @db.execute('select col1, col2 from TestVTable3, TestVTable4').each.to_a - assert( rows.include?(['a', 'c']) ) - assert( rows.include?(['a', 'd']) ) - assert( rows.include?(['b', 'c']) ) - assert( rows.include?(['b', 'd']) ) + assert_includes rows, ['a', 'c'] + assert_includes rows, ['a', 'd'] + assert_includes rows, ['b', 'c'] + assert_includes rows, ['b', 'd'] end def test_best_filter @@ -99,36 +99,36 @@ def test_best_filter SQLite3.vtable(@db, 'TestVTable5', 'col1, col2', [['a', 1], ['b', 2]]).tap do |vtable| vtable.send(:define_method, :best_index) do |constraint, order_by| # check constraint - test.assert( constraint.include?([0, :<=]) ) # col1 <= 'c' - test.assert( constraint.include?([0, :>]) ) # col1 > 'a' - test.assert( constraint.include?([1, :<]) ) # col2 < 3 + test.assert_includes constraint, [0, :<=] # col1 <= 'c' + test.assert_includes constraint, [0, :>] # col1 > 'a' + test.assert_includes constraint, [1, :<] # col2 < 3 @constraint = constraint # check order by - test.assert( order_by == [ + test.assert_equal( [ [1, 1], # col2 [0, -1], # col1 desc - ] ) + ], order_by ) { idxNum: 45 } end vtable.send(:alias_method, :orig_filter, :filter) vtable.send(:define_method, :filter) do |idxNum, args| # idxNum should be the one returned by best_index - test.assert( idxNum == 45 ) + test.assert_equal( 45, idxNum ) # args should be consistent with the constraint given to best_index - test.assert( args.size == @constraint.size ) + test.assert_equal( @constraint.size, args.size ) filters = @constraint.zip(args) - test.assert( filters.include?([[0, :<=], 'c']) ) # col1 <= 'c' - test.assert( filters.include?([[0, :>], 'a']) ) # col1 > 'a' - test.assert( filters.include?([[1, :<], 3]) ) # col2 < 3 + test.assert_includes filters, [[0, :<=], 'c'] # col1 <= 'c' + test.assert_includes filters, [[0, :>], 'a'] # col1 > 'a' + test.assert_includes filters, [[1, :<], 3] # col2 < 3 orig_filter(idxNum, args) end end rows = @db.execute('select col1 from TestVTable5 where col1 <= \'c\' and col1 > \'a\' and col2 < 3 order by col2, col1 desc').each.to_a - assert( rows == [['b']] ) + assert_equal( [['b']], rows ) end end if defined?(SQLite3::Module) From 06c6659f30b6a5b04f3bf28b7b74f4e92bff0208 Mon Sep 17 00:00:00 2001 From: Lionel Perrin Date: Tue, 5 Apr 2016 16:52:53 +0200 Subject: [PATCH 10/10] simplify implementation (and solve GC issue) * remove class SQLite3::Module and create class SQLite3::VTable * when declared, store VTables in db.vtables --- Manifest.txt | 4 +- ext/sqlite3/database.h | 4 +- ext/sqlite3/module.h | 16 --- ext/sqlite3/sqlite3.c | 2 +- ext/sqlite3/sqlite3_ruby.h | 2 +- ext/sqlite3/statement.c | 6 +- ext/sqlite3/{module.c => vtable.c} | 119 +++++++++++----------- ext/sqlite3/vtable.h | 8 ++ lib/sqlite3/database.rb | 1 + lib/sqlite3/vtable.rb | 63 +++++++----- test/test_vtable.rb | 158 +++++++++++++++++++---------- 11 files changed, 221 insertions(+), 162 deletions(-) delete mode 100644 ext/sqlite3/module.h rename ext/sqlite3/{module.c => vtable.c} (80%) create mode 100644 ext/sqlite3/vtable.h diff --git a/Manifest.txt b/Manifest.txt index 30d9d226..6385cfcd 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -13,8 +13,8 @@ ext/sqlite3/database.h ext/sqlite3/exception.c ext/sqlite3/exception.h ext/sqlite3/extconf.rb -ext/sqlite3/module.c -ext/sqlite3/module.h +ext/sqlite3/vtable.c +ext/sqlite3/vtable.h ext/sqlite3/sqlite3.c ext/sqlite3/sqlite3_ruby.h ext/sqlite3/statement.c diff --git a/ext/sqlite3/database.h b/ext/sqlite3/database.h index 0c5df531..eb5f940d 100644 --- a/ext/sqlite3/database.h +++ b/ext/sqlite3/database.h @@ -3,7 +3,9 @@ #include -// used by module.c too +#define DEBUG_LOG(...) (void)(0) +//#define DEBUG_LOG(...) printf(__VA_ARGS__); fflush(stdout) +// used by vtable.c too void set_sqlite3_func_result(sqlite3_context * ctx, VALUE result); VALUE sqlite3val2rb(sqlite3_value * val); diff --git a/ext/sqlite3/module.h b/ext/sqlite3/module.h deleted file mode 100644 index 2638d2a3..00000000 --- a/ext/sqlite3/module.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef SQLITE3_MODULE_RUBY -#define SQLITE3_MODULE_RUBY - -#include - -struct _sqlite3ModuleRuby { - sqlite3_module *module; - VALUE module_name; // so that sqlite can bring the module_name up to the vtable -}; - -typedef struct _sqlite3ModuleRuby sqlite3ModuleRuby; -typedef sqlite3ModuleRuby * sqlite3ModuleRubyPtr; - -void init_sqlite3_module(); - -#endif diff --git a/ext/sqlite3/sqlite3.c b/ext/sqlite3/sqlite3.c index 0bc3deff..5888d426 100644 --- a/ext/sqlite3/sqlite3.c +++ b/ext/sqlite3/sqlite3.c @@ -135,7 +135,7 @@ void Init_sqlite3_native() #ifdef HAVE_SQLITE3_BACKUP_INIT init_sqlite3_backup(); #endif - init_sqlite3_module(); + init_sqlite3_vtable(); rb_define_singleton_method(mSqlite3, "libversion", libversion, 0); rb_define_singleton_method(mSqlite3, "threadsafe", threadsafe_p, 0); diff --git a/ext/sqlite3/sqlite3_ruby.h b/ext/sqlite3/sqlite3_ruby.h index fc475b2c..35a00640 100644 --- a/ext/sqlite3/sqlite3_ruby.h +++ b/ext/sqlite3/sqlite3_ruby.h @@ -46,7 +46,7 @@ extern VALUE cSqlite3Blob; #include #include #include -#include +#include int bignum_to_int64(VALUE big, sqlite3_int64 *result); diff --git a/ext/sqlite3/statement.c b/ext/sqlite3/statement.c index cf5956c2..a9281aed 100644 --- a/ext/sqlite3/statement.c +++ b/ext/sqlite3/statement.c @@ -118,7 +118,9 @@ static VALUE step(VALUE self) REQUIRE_OPEN_STMT(ctx); - if(ctx->done_p) return Qnil; + if(ctx->done_p) { + return Qnil; + } #ifdef HAVE_RUBY_ENCODING_H { @@ -128,8 +130,8 @@ static VALUE step(VALUE self) } #endif + stmt = ctx->st; - value = sqlite3_step(stmt); length = sqlite3_column_count(stmt); list = rb_ary_new2((long)length); diff --git a/ext/sqlite3/module.c b/ext/sqlite3/vtable.c similarity index 80% rename from ext/sqlite3/module.c rename to ext/sqlite3/vtable.c index 240722c2..d6198a2e 100644 --- a/ext/sqlite3/module.c +++ b/ext/sqlite3/vtable.c @@ -1,7 +1,7 @@ #include #include -VALUE cSqlite3Module; +VALUE cVTable; /** structure for ruby virtual table: inherits from sqlite3_vtab */ typedef struct { @@ -20,39 +20,64 @@ typedef struct { int rowid; } ruby_sqlite3_vtab_cursor; - /** * lookup for a ruby class :: and create an instance of this class * This instance is then used to bind sqlite vtab callbacks */ -static int xCreate(sqlite3* db, VALUE *module_name, +static int xCreate(sqlite3* db, VALUE *db_ruby, int argc, char **argv, ruby_sqlite3_vtab **ppVTab, char **pzErr) { - VALUE sql_stmt, module, ruby_class; - ID table_id, module_id; - VALUE ruby_class_args[0]; - const char* module_name_cstr = (const char*)StringValuePtr(*module_name); + VALUE sql_stmt, tables; + VALUE module_name, module; + VALUE table_name, table; + const char* module_name_cstr = (const char*)argv[0]; const char* table_name_cstr = (const char*)argv[2]; // method will raise in case of error: no need to use pzErr *pzErr = NULL; - // lookup for ruby class named like :: - module_id = rb_intern( module_name_cstr ); - module = rb_const_get(rb_cObject, module_id); - table_id = rb_intern( table_name_cstr ); - ruby_class = rb_const_get(module, table_id); + // lookup for hash db.vtables + tables = rb_funcall(*db_ruby, rb_intern("vtables"), 0); + if (!RB_TYPE_P(tables, T_HASH)) { + rb_raise(rb_eTypeError, "xCreate: expect db.vtables to be a Hash"); + } + module_name = rb_str_new2(module_name_cstr); + module = rb_hash_aref(tables, module_name); + if (module == Qnil ) { + rb_raise( + rb_eKeyError, + "xCreate: module %s is declared in sqlite3 but cant be found in db.vtables.", + module_name_cstr + ); + } + + table_name = rb_str_new2(table_name_cstr); + table = rb_hash_aref(module, table_name); + if (table == Qnil) { + rb_raise( + rb_eKeyError, + "no such table: %s in module %s", + table_name_cstr, + module_name_cstr + ); + } + if (rb_obj_is_kind_of(table, cVTable) != Qtrue) { + VALUE table_inspect = rb_funcall(table, rb_intern("inspect"), 0); + rb_raise( + rb_eTypeError, + "Object %s must inherit from VTable", + StringValuePtr(table_inspect) + ); + } // alloc a new ruby_sqlite3_vtab object // and store related attributes (*ppVTab) = (ruby_sqlite3_vtab*)malloc(sizeof(ruby_sqlite3_vtab)); + (*ppVTab)->vtable = table; - // create a new instance - (*ppVTab)->vtable = rb_class_new_instance(0, ruby_class_args, ruby_class); - - // call the create function + // get the create statement sql_stmt = rb_funcall((*ppVTab)->vtable, rb_intern("create_statement"), 0); #ifdef HAVE_RUBY_ENCODING_H @@ -61,7 +86,7 @@ static int xCreate(sqlite3* db, VALUE *module_name, } #endif if ( sqlite3_declare_vtab(db, StringValuePtr(sql_stmt)) ) { - rb_raise(rb_eArgError, "fail to declare virtual table"); + rb_raise(rb_path2class("SQLite3::Exception"), "fail to declare virtual table with \"%s\": %s", StringValuePtr(sql_stmt), sqlite3_errmsg(db)); } return SQLITE_OK; @@ -155,9 +180,7 @@ static int xBestIndex(ruby_sqlite3_vtab *pVTab, sqlite3_index_info* info) for (i = 0; i < info->nConstraint; ++i) { if (info->aConstraint[i].usable) { rb_ary_push(constraint, constraint_to_ruby(info->aConstraint + i)); - } else { - printf("ignoring %d %d\n", info->aConstraint[i].iColumn, info->aConstraint[i].op); - } + } } // convert order_by to ruby @@ -165,7 +188,6 @@ static int xBestIndex(ruby_sqlite3_vtab *pVTab, sqlite3_index_info* info) rb_ary_store(order_by, i, order_by_to_ruby(info->aOrderBy + i)); } - ret = rb_funcall( pVTab->vtable, rb_intern("best_index"), 2, constraint, order_by ); if (ret != Qnil ) { if (!RB_TYPE_P(ret, T_HASH)) { @@ -263,7 +285,6 @@ static int xEof(ruby_sqlite3_vtab_cursor* cursor) static int xColumn(ruby_sqlite3_vtab_cursor* cursor, sqlite3_context* context, int i) { VALUE val = rb_ary_entry(cursor->row, i); - set_sqlite3_func_result(context, val); return SQLITE_OK; } @@ -297,57 +318,41 @@ static sqlite3_module ruby_proxy_module = NULL, /* xFindFunction - function overloading */ }; -static void deallocate(void * ctx) -{ - sqlite3ModuleRubyPtr c = (sqlite3ModuleRubyPtr)ctx; - xfree(c); -} - -static VALUE allocate(VALUE klass) -{ - sqlite3ModuleRubyPtr ctx = xcalloc((size_t)1, sizeof(sqlite3ModuleRuby)); - ctx->module = NULL; - - return Data_Wrap_Struct(klass, NULL, deallocate, ctx); -} - -static VALUE initialize(VALUE self, VALUE db, VALUE name) +static VALUE create_module(VALUE self, VALUE db, VALUE name) { + VALUE *db_ruby; sqlite3RubyPtr db_ctx; - sqlite3ModuleRubyPtr ctx; - StringValue(name); - Data_Get_Struct(db, sqlite3Ruby, db_ctx); - Data_Get_Struct(self, sqlite3ModuleRuby, ctx); - - if(!db_ctx->db) - rb_raise(rb_eArgError, "initializing a module on a closed database"); + if(!db_ctx->db) { + rb_raise(rb_eArgError, "create_module on a closed database"); + } #ifdef HAVE_RUBY_ENCODING_H if(!UTF8_P(name)) { - name = rb_str_export_to_enc(name, rb_utf8_encoding()); + name = rb_str_export_to_enc(name, rb_utf8_encoding()); } #endif - - // make possible to access to ruby object from c - ctx->module_name = name; - sqlite3_create_module( + db_ruby = xcalloc(1, sizeof(VALUE)); + *db_ruby = db; + if (sqlite3_create_module_v2( db_ctx->db, - (const char *)StringValuePtr(name), + StringValuePtr(name), &ruby_proxy_module, - &(ctx->module_name) //the vtable required the module name - ); + db_ruby, + xfree + ) != SQLITE_OK) { + rb_raise(rb_path2class("SQLite3::Exception"), sqlite3_errmsg(db_ctx->db)); + } - return self; + return Qnil; } -void init_sqlite3_module() +void init_sqlite3_vtable() { - cSqlite3Module = rb_define_class_under(mSqlite3, "Module", rb_cObject); - rb_define_alloc_func(cSqlite3Module, allocate); - rb_define_method(cSqlite3Module, "initialize", initialize, 2); + cVTable = rb_define_class_under(mSqlite3, "VTable", rb_cObject); + rb_define_singleton_method(cVTable, "create_module", create_module, 2); } diff --git a/ext/sqlite3/vtable.h b/ext/sqlite3/vtable.h new file mode 100644 index 00000000..a6efbcec --- /dev/null +++ b/ext/sqlite3/vtable.h @@ -0,0 +1,8 @@ +#ifndef SQLITE3_MODULE_RUBY +#define SQLITE3_MODULE_RUBY + +#include + +void init_sqlite3_vtable(); + +#endif diff --git a/lib/sqlite3/database.rb b/lib/sqlite3/database.rb index 5d897ecd..dd33e945 100644 --- a/lib/sqlite3/database.rb +++ b/lib/sqlite3/database.rb @@ -34,6 +34,7 @@ module SQLite3 # hashes, then the results will all be indexible by field name. class Database attr_reader :collations + attr_accessor :vtables include Pragmas diff --git a/lib/sqlite3/vtable.rb b/lib/sqlite3/vtable.rb index 5f0b1189..1a74d602 100644 --- a/lib/sqlite3/vtable.rb +++ b/lib/sqlite3/vtable.rb @@ -4,10 +4,23 @@ module SQLite3_VTables end module SQLite3 - class VTableInterface + class VTable + def register(db, module_name, table_name) + tables = (db.vtables ||= {}) + m = tables[module_name] + raise "VTable #{table_name} for module #{module_name} is already registered" if m && m[table_name] + unless m + self.class.create_module(db, module_name) + m = tables[module_name] = {} + end + m[table_name] = self + end + def initialize(db, module_name, table_name = nil) + register(db, module_name, table_name || self.class.name.split('::').last) + end #this method is needed to declare the type of each column to sqlite def create_statement - fail 'VTableInterface#create_statement not implemented' + fail 'VTable#create_statement not implemented' end #called before each statement @@ -30,42 +43,40 @@ def best_index(constraint, order_by) # may be called several times between open/close # it initialize/reset cursor def filter(idxNum, args) - fail 'VTableInterface#filter not implemented' + fail 'VTable#filter not implemented' end # called to retrieve a new row def next - fail 'VTableInterface#next not implemented' + fail 'VTable#next not implemented' end end - def self.vtable(db, table_name, table_columns, enumerable) - Module.new(db, 'SQLite3_VTables') - if SQLite3_VTables.const_defined?(table_name, inherit = false) - raise "'#{table_name}' already declared" + class VTableFromEnumerable < VTable + DEFAULT_MODULE = 'DEFAULT_MODULE' + def initialize(db, table_name, table_columns, enumerable) + super(db, DEFAULT_MODULE, table_name) + @table_name = table_name + @table_columns = table_columns + @enumerable = enumerable + db.execute("create virtual table #{table_name} using #{DEFAULT_MODULE}") end - klass = Class.new(VTableInterface) - klass.send(:define_method, :filter) do |idxNum, args| - @enumerable = enumerable.to_enum - end - klass.send(:define_method, :create_statement) do - "create table #{table_name}(#{table_columns})" + def filter(idxNum, args) + @enum = @enumerable.to_enum end - klass.send(:define_method, :next) do - begin - @enumerable.next - rescue StopIteration - nil - end + + def create_statement + "create table #{@table_name}(#{@table_columns})" end - begin - SQLite3_VTables.const_set(table_name, klass) - rescue NameError - raise "'#{table_name}' must be a valid ruby constant name" + def next + @enum.next + rescue StopIteration + nil end - db.execute("create virtual table #{table_name} using SQLite3_VTables") - klass end + def self.vtable(db, table_name, table_columns, enumerable) + VTableFromEnumerable.new(db, table_name, table_columns, enumerable) + end end diff --git a/test/test_vtable.rb b/test/test_vtable.rb index c5b34638..76f51e9b 100644 --- a/test/test_vtable.rb +++ b/test/test_vtable.rb @@ -1,42 +1,40 @@ require "helper" require 'sqlite3/vtable' -#the ruby module name should be the one given to sqlite when creating the virtual table. -module RubyModule - class TestVTable < SQLite3::VTableInterface - def initialize - @str = "A"*1500 - end - - #required method for vtable - #this method is needed to declare the type of each column to sqlite - def create_statement - "create table TestVTable(s text, x integer, y int)" - end +class VTableTest < SQLite3::VTable + def initialize(db, module_name) + super(db, module_name) + @str = "A"*1500 + end - #required method for vtable - #called before each statement - def open - end + #required method for vtable + #this method is needed to declare the type of each column to sqlite + def create_statement + "create table VTableTest(s text, x integer, y int)" + end - # this method initialize/reset cursor - def filter(id, args) - @count = 0 - end + #required method for vtable + #called before each statement + def open + end - #required method for vtable - #called to retrieve a new row - def next + # this method initialize/reset cursor + def filter(id, args) + @count = 0 + end - #produce up to 100000 lines - @count += 1 - if @count <= 100000 - [@str, rand(10), rand] - else - nil - end + #required method for vtable + #called to retrieve a new row + def next + #produce up to 100000 lines + @count += 1 + if @count <= 50 + [@str, rand(10), rand] + else + nil end + end end @@ -44,50 +42,77 @@ module SQLite3 class TestVTable < SQLite3::TestCase def setup @db = SQLite3::Database.new(":memory:") - @m = SQLite3::Module.new(@db, "RubyModule") + GC.stress = true + end + + def teardown + GC.stress = false end def test_exception_module - #the following line throws an exception because NonExistingModule is not valid ruby module - assert_raise SQLite3::SQLException do - @db.execute("create virtual table TestVTable using NonExistingModule") + #the following line throws an exception because NonExistingModule has not been created in sqlite + err = assert_raise SQLite3::SQLException do + @db.execute("create virtual table VTableTest using NonExistingModule") end + assert_includes(err.message, 'no such module: NonExistingModule') end def test_exception_table - #the following line throws an exception because no ruby class RubyModule::NonExistingVTable is found as vtable implementation - assert_raise NameError do - @db.execute("create virtual table NonExistingVTable using RubyModule") + #the following line throws an exception because no ruby class NonExistingVTable has been registered + VTableTest.new(@db, 'TestModule') + err = assert_raise KeyError do + @db.execute("create virtual table NonExistingVTable using TestModule") end + assert_includes(err.message, 'no such table: NonExistingVTable in module TestModule') end - def test_working - #this will instantiate a new virtual table using implementation from RubyModule::TestVTable - @db.execute("create virtual table if not exists TestVTable using RubyModule") + def test_exception_bad_create_statement + t = VTableTest.new(@db, 'TestModule2').tap do |vtable| + vtable.define_singleton_method(:create_statement) { + 'create tab with a bad statement' + } + end + # this will fail because create_statement is not valid statement such as "create virtual table t(col1, col2)" + err = assert_raises SQLite3::Exception do + @db.execute('create virtual table VTableTest using TestModule2') + end + assert_includes(err.message, 'fail to declare virtual table') + assert_includes(err.message, t.create_statement) + end - #execute an sql statement - nb_row = @db.execute("select x, sum(y), avg(y), avg(y*y), min(y), max(y), count(y) from TestVTable group by x").each.count - assert_operator nb_row, :>, 0 + def test_working + # register vtable implementation under module RubyModule. RubyModule will be created in sqlite3 if not already existing + VTableTest.new(@db, 'RubyModule') + 2.times do |i| + #this will instantiate a new virtual table using implementation from VTableTest + @db.execute("create virtual table if not exists VTableTest using RubyModule") + + #execute an sql statement + nb_row = @db.execute("select x, sum(y), avg(y), avg(y*y), min(y), max(y), count(y) from VTableTest group by x").each.count + assert_operator nb_row, :>, 0 + end end def test_vtable - SQLite3.vtable(@db, 'TestVTable2', 'a, b, c', [ + # test compact declaration of virtual table. The last parameter should be an enumerable of Array. + SQLite3.vtable(@db, 'VTableTest2', 'a, b, c', [ [1, 2, 3], [2, 4, 6], [3, 6, 9] ]) - nb_row = @db.execute('select count(*) from TestVTable2').each.first[0] + nb_row = @db.execute('select count(*) from VTableTest2').each.first[0] assert_equal( 3, nb_row ) - sum_a, sum_b, sum_c = *@db.execute('select sum(a), sum(b), sum(c) from TestVTable2').each.first + sum_a, sum_b, sum_c = *@db.execute('select sum(a), sum(b), sum(c) from VTableTest2').each.first assert_equal( 6, sum_a ) assert_equal( 12, sum_b ) assert_equal( 18, sum_c ) end def test_multiple_vtable - SQLite3.vtable(@db, 'TestVTable3', 'col1', [['a'], ['b']]) - SQLite3.vtable(@db, 'TestVTable4', 'col2', [['c'], ['d']]) - rows = @db.execute('select col1, col2 from TestVTable3, TestVTable4').each.to_a + # make sure it is possible to join virtual table using sqlite + SQLite3.vtable(@db, 'VTableTest3', 'col1', [['a'], ['b']]) + SQLite3.vtable(@db, 'VTableTest4', 'col2', [['c'], ['d']]) + rows = @db.execute('select col1, col2 from VTableTest3, VTableTest4').each.to_a assert_includes rows, ['a', 'c'] assert_includes rows, ['a', 'd'] assert_includes rows, ['b', 'c'] @@ -95,9 +120,10 @@ def test_multiple_vtable end def test_best_filter + # one can provide a best_filter implementation see SQLite3 documentation about best_filter test = self - SQLite3.vtable(@db, 'TestVTable5', 'col1, col2', [['a', 1], ['b', 2]]).tap do |vtable| - vtable.send(:define_method, :best_index) do |constraint, order_by| + SQLite3.vtable(@db, 'VTableTest5', 'col1, col2', [['a', 1], ['b', 2]]).tap do |vtable| + vtable.define_singleton_method(:best_index) do |constraint, order_by| # check constraint test.assert_includes constraint, [0, :<=] # col1 <= 'c' test.assert_includes constraint, [0, :>] # col1 > 'a' @@ -112,8 +138,8 @@ def test_best_filter { idxNum: 45 } end - vtable.send(:alias_method, :orig_filter, :filter) - vtable.send(:define_method, :filter) do |idxNum, args| + vtable.singleton_class.send(:alias_method, :orig_filter, :filter) + vtable.define_singleton_method(:filter) do |idxNum, args| # idxNum should be the one returned by best_index test.assert_equal( 45, idxNum ) @@ -123,14 +149,34 @@ def test_best_filter test.assert_includes filters, [[0, :<=], 'c'] # col1 <= 'c' test.assert_includes filters, [[0, :>], 'a'] # col1 > 'a' test.assert_includes filters, [[1, :<], 3] # col2 < 3 - + orig_filter(idxNum, args) end end - rows = @db.execute('select col1 from TestVTable5 where col1 <= \'c\' and col1 > \'a\' and col2 < 3 order by col2, col1 desc').each.to_a + rows = @db.execute('select col1 from VTableTest5 where col1 <= \'c\' and col1 > \'a\' and col2 < 3 order by col2, col1 desc').each.to_a assert_equal( [['b']], rows ) end - end if defined?(SQLite3::Module) + def test_garbage_collection + # this test will check that everything is working even if rows are getting collected during the execution of the statement + started = false + n_deleted_during_request = 0 + finalizer = proc do |id| + n_deleted_during_request += 1 if started + end + SQLite3.vtable(@db, 'VTableTest6', 'col1 number, col2 number, col3 text', (1..Float::INFINITY).lazy.map do |i| + r = [i, i*5, "some text #{i}"] + ObjectSpace.define_finalizer(r, finalizer) + r + end) + started = true + @db.execute('select col1, col2 from VTableTest6 limit 10') do |row| + assert_equal(row[1], row[0]*5) + end + started = false + assert_operator(n_deleted_during_request, :>, 0) + end + + end if defined?(SQLite3::VTable) end