diff --git a/Gemfile b/Gemfile index 6b5dbaa..ba44849 100644 --- a/Gemfile +++ b/Gemfile @@ -2,5 +2,5 @@ source "https://rubygems.org" git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } -# Specify your gem's dependencies in pattern_query_helper.gemspec +# Specify your gem's dependencies in query_helper.gemspec gemspec diff --git a/Gemfile.lock b/Gemfile.lock index 0d476d6..55559d8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,13 +1,20 @@ PATH remote: . specs: - pattern_query_helper (0.2.9) + query_helper (0.0.0) activerecord (~> 5.0) - kaminari (~> 1.1.1) + activesupport (~> 5.0) GEM remote: https://rubygems.org/ specs: + actionpack (5.2.3) + actionview (= 5.2.3) + activesupport (= 5.2.3) + rack (~> 2.0) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.0.2) actionview (5.2.3) activesupport (= 5.2.3) builder (~> 3.1) @@ -36,30 +43,28 @@ GEM i18n (>= 0.7) i18n (1.6.0) concurrent-ruby (~> 1.0) - kaminari (1.1.1) - activesupport (>= 4.1.0) - kaminari-actionview (= 1.1.1) - kaminari-activerecord (= 1.1.1) - kaminari-core (= 1.1.1) - kaminari-actionview (1.1.1) - actionview - kaminari-core (= 1.1.1) - kaminari-activerecord (1.1.1) - activerecord - kaminari-core (= 1.1.1) - kaminari-core (1.1.1) loofah (2.2.3) crass (~> 1.0.2) nokogiri (>= 1.5.9) + method_source (0.9.2) mini_portile2 (2.4.0) minitest (5.11.3) nokogiri (1.10.3) mini_portile2 (~> 2.4.0) + rack (2.0.7) + rack-test (1.1.0) + rack (>= 1.0, < 3) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.0.4) loofah (~> 2.2, >= 2.2.2) + railties (5.2.3) + actionpack (= 5.2.3) + activesupport (= 5.2.3) + method_source + rake (>= 0.8.7) + thor (>= 0.19.0, < 2.0) rake (10.5.0) rspec (3.8.0) rspec-core (~> 3.8.0) @@ -73,8 +78,17 @@ GEM rspec-mocks (3.8.0) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.8.0) + rspec-rails (3.8.2) + actionpack (>= 3.0) + activesupport (>= 3.0) + railties (>= 3.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-support (~> 3.8.0) rspec-support (3.8.0) sqlite3 (1.3.13) + thor (0.20.3) thread_safe (0.3.6) tzinfo (1.2.5) thread_safe (~> 0.1) @@ -83,12 +97,15 @@ PLATFORMS ruby DEPENDENCIES + actionpack + activesupport bundler (~> 1.16) byebug faker (~> 1.9.3) - pattern_query_helper! + query_helper! rake (~> 10.0) rspec (~> 3.0) + rspec-rails sqlite3 (~> 1.3.6) BUNDLED WITH diff --git a/README.md b/README.md index b6c1998..8e309eb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# PatternQueryHelper -[![TravisCI](https://travis-ci.org/iserve-products/pattern_query_helper.svg?branch=master)](https://travis-ci.org/iserve-products/pattern_query_helper) -[![Gem Version](https://badge.fury.io/rb/pattern_query_helper.svg)](https://badge.fury.io/rb/pattern_query_helper) +# QueryHelper +[![TravisCI](https://travis-ci.org/iserve-products/query_helper.svg?branch=master)](https://travis-ci.org/iserve-products/query_helper) +[![Gem Version](https://badge.fury.io/rb/query_helper.svg)](https://badge.fury.io/rb/query_helper) Ruby Gem developed and used at Pattern to paginate, sort, filter, and include associations on sql and active record queries. @@ -9,7 +9,7 @@ Ruby Gem developed and used at Pattern to paginate, sort, filter, and include as Add this line to your application's Gemfile: ```ruby -gem 'pattern_query_helper' +gem 'query_helper' ``` And then execute: @@ -18,27 +18,175 @@ And then execute: Or install it yourself as: - $ gem install pattern_query_helper + $ gem install query_helper ## Use +### SQL Queries + +#### Initialize + +To create a new sql query object run + +```ruby +QueryHelper::Sql.new( + model:, # required + query:, # required + query_params: , # optional + column_mappings: , # optional + filters: , # optional + sorts: , # optional + page: , # optional + per_page: , # optional + single_record: , # optional, default: false + associations: , # optional + as_json_options: , # optional + run: # optional, default: true +) +``` + +The following arguments are accepted when creating a new objects + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ArgumentDescriptionExample
modelthe model to run the query against +
+Parent
+
+
querythe custom sql string to be executed +
+'select * from parents'
+
+
query_paramsa hash of bind variables to be embedded into the sql query +
+{
+  age: 20,
+  name: 'John'
+}
+
+
column_mappingsA hash that translates aliases to sql expressions +
+{
+  "age" => "parents.age"
+  "children_count" => {
+    sql_expression: "count(children.id)",
+    aggregate: true
+  }
+}
+
+
filtersa list of filters in the form of `{"comparate_alias"=>{"operator_code"=>"value"}}` +
+{
+  "age" => { "lt" => 100 },
+  "children_count" => { "gt" => 0 }
+}
+
+
sortsa comma separated string with a list of sort values +
+"age:desc,name:asc:lowercase"
+
+
pagethe page you want returned +
+5
+
+
per_pagethe number of results per page +
+20
+
+
single_recordwhether or not you expect the record to return a single result, if toggled, only the first result will be returned +
+false
+
+
associationsa list of activerecord associations you'd like included in the payload +
+
+
+
as_json_optionsa list of as_json options you'd like run before returning the payload +
+
+
+
runwhether or not you'd like to run the query on initilization +
+false
+
+
+ ### Active Record Queries To run an active record query execute ```ruby -PatternQueryHelper.run_active_record_query(active_record_call, query_helpers, valid_columns, single_record) +QueryHelper.run_active_record_query(active_record_call, query_helpers, valid_columns, single_record) ``` active_record_call: Valid active record syntax (i.e. ```Object.where(state: 'Active')```) query_helpers: See docs below valid_columns: Default is []. Pass in an array of columns you want to allow sorting and filtering on. single_record: Default is false. Pass in true to format payload as a single object instead of a list of objects -### Custom SQL Queries -To run a custom sql query execute -```ruby -PatternQueryHelper.run_sql_query(model, query, query_params, query_helpers, valid_columns, single_record) -``` model: A valid ActiveRecord model query: A string containing your custom SQL query query_params: a symbolized hash of binds to be included in your SQL query @@ -143,7 +291,7 @@ query_helpers = { ## Payload Formats -The PatternQueryHelper gem will return results in one of three formats +The QueryHelper gem will return results in one of three formats ### Paginated List Payload ```json @@ -222,7 +370,7 @@ The PatternQueryHelper gem will return results in one of three formats ## Contributing -Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pattern_query_helper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. +Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/query_helper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. ## License @@ -230,4 +378,4 @@ The gem is available as open source under the terms of the [MIT License](https:/ ## Code of Conduct -Everyone interacting in the PatternQueryHelper project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/pattern_query_helper/blob/master/CODE_OF_CONDUCT.md). +Everyone interacting in the QueryHelper project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/query_helper/blob/master/CODE_OF_CONDUCT.md). diff --git a/bin/console b/bin/console index 57136d3..9492f54 100755 --- a/bin/console +++ b/bin/console @@ -1,7 +1,7 @@ #!/usr/bin/env ruby require "bundler/setup" -require "pattern_query_helper" +require "query_helper" # You can add fixtures and/or initialization code here to make experimenting # with your gem easier. You can also use a different console, if you like. diff --git a/lib/pattern_query_helper.rb b/lib/pattern_query_helper.rb deleted file mode 100644 index 8c464f5..0000000 --- a/lib/pattern_query_helper.rb +++ /dev/null @@ -1,115 +0,0 @@ -require "pattern_query_helper/version" -require "pattern_query_helper/pagination" -require "pattern_query_helper/filtering" -require "pattern_query_helper/associations" -require "pattern_query_helper/sorting" -require "pattern_query_helper/sql" - -module PatternQueryHelper - - def self.run_sql_query(model, query, query_params, query_helpers, valid_columns=[], single_record=false) - if single_record - single_record_sql_query(model, query, query_params, query_helpers, valid_columns) - elsif query_helpers[:per_page] || query_helpers[:page] - paginated_sql_query(model, query, query_params, query_helpers, valid_columns) - else - sql_query(model, query, query_params, query_helpers, valid_columns) - end - end - - def self.run_active_record_query(active_record_call, query_helpers, valid_columns=[], single_record=false) - run_sql_query(active_record_call.model, active_record_call.to_sql, {}, query_helpers, valid_columns, single_record) - end - - private - - def self.paginated_sql_query(model, query, query_params, query_helpers, valid_columns) - query_helpers = parse_helpers(query_helpers, valid_columns) - - query_config = { - model: model, - query: query, - query_params: query_params, - page: query_helpers[:pagination][:page], - per_page: query_helpers[:pagination][:per_page], - filter_string: query_helpers[:filters][:filter_string], - filter_params: query_helpers[:filters][:filter_params], - sort_string: query_helpers[:sorting], - } - - data = PatternQueryHelper::Sql.sql_query(query_config) - data = PatternQueryHelper::Associations.load_associations(data, query_helpers[:associations], query_helpers[:as_json]) - count = PatternQueryHelper::Sql.sql_query_count(data) - data.map! { |d| d.except(PatternQueryHelper::Sql::QUERY_COUNT_COLUMN) } if query_config[:page] or query_config[:per_page] - pagination = PatternQueryHelper::Pagination.create_pagination_payload(count, query_helpers[:pagination]) - - { - pagination: pagination, - data: data - } - end - - def self.sql_query(model, query, query_params, query_helpers, valid_columns) - query_helpers = parse_helpers(query_helpers, valid_columns) - - query_config = { - model: model, - query: query, - query_params: query_params, - filter_string: query_helpers[:filters][:filter_string], - filter_params: query_helpers[:filters][:filter_params], - sort_string: query_helpers[:sorting], - } - - data = PatternQueryHelper::Sql.sql_query(query_config) - data = PatternQueryHelper::Associations.load_associations(data, query_helpers[:associations], query_helpers[:as_json]) - - { - data: data - } - end - - def self.single_record_sql_query(model, query, query_params, query_helpers, valid_columns) - query_helpers = parse_helpers(query_helpers, valid_columns) - - query_config = { - model: model, - query: query, - query_params: query_params, - filter_string: query_helpers[:filters][:filter_string], - filter_params: query_helpers[:filters][:filter_params], - sort_string: query_helpers[:sorting], - } - - data = PatternQueryHelper::Sql.single_record_query(query_config) - data = PatternQueryHelper::Associations.load_associations(data, query_helpers[:associations], query_helpers[:as_json]) - - { - data: data - } - end - - def self.parse_helpers(query_helpers, valid_columns) - valid_columns_map = {} - valid_columns.each do |c| - valid_columns_map["#{c}"] = c - end - filtering = PatternQueryHelper::Filtering.create_filters(filters: query_helpers[:filter], valid_columns_map: valid_columns_map) - sorting = PatternQueryHelper::Sorting.parse_sorting_params(query_helpers[:sort], valid_columns) - associations = PatternQueryHelper::Associations.process_association_params(query_helpers[:include]) - pagination = PatternQueryHelper::Pagination.parse_pagination_params(query_helpers[:page], query_helpers[:per_page]) - as_json = query_helpers[:as_json] - - { - filters: filtering, - sorting: sorting, - associations: associations, - pagination: pagination, - as_json: as_json - } - end - - class << self - attr_accessor :active_record_adapter - end -end diff --git a/lib/pattern_query_helper/filtering.rb b/lib/pattern_query_helper/filtering.rb deleted file mode 100644 index 51cda3a..0000000 --- a/lib/pattern_query_helper/filtering.rb +++ /dev/null @@ -1,84 +0,0 @@ -module PatternQueryHelper - class Filtering - def self.create_filters(filters:, valid_columns_map: nil, symbol_prefix: "", include_where: true) - filters ||= {} - all_conditions = [] - filter_params = {} - filter_array = [] - filters.each do |filter_attribute, criteria| - if valid_columns_map - raise ArgumentError.new("Invalid filter '#{filter_attribute}'") unless valid_columns_map[filter_attribute] - filter_column = valid_columns_map[filter_attribute] - else - filter_column = filter_attribute - end - criteria.each do |operator_code, criterion| - filter_symbol = "#{symbol_prefix}#{filter_attribute}_#{operator_code}" - case operator_code - when "gte" - operator = ">=" - when "lte" - operator = "<=" - when "gt" - operator = ">" - when "lt" - operator = "<" - when "eql" - operator = "=" - when "noteql" - operator = "!=" - when "like" - modified_filter_column = "lower(#{filter_column})" - operator = "like" - criterion.downcase! - when "in" - operator = "in (:#{filter_symbol})" - # if criterion are anything but numbers, downcase the filter_column - if criterion.scan(/[^\d|,|\s]/).any? - modified_filter_column = "lower(#{filter_column})" - end - criterion = criterion.downcase.split(",") - filter_symbol_already_embedded = true - when "notin" - operator = "not in (:#{filter_symbol})" - # if criterion are anything but numbers, downcase the filter_column - if criterion.scan(/[^\d|,|\s]/).any? - modified_filter_column = "lower(#{filter_column})" - end - criterion = criterion.downcase.split(",") - filter_symbol_already_embedded = true - when "null" - operator = criterion.to_s == "true" ? "is null" : "is not null" - filter_symbol = "" - else - raise ArgumentError.new("Invalid operator code '#{operator_code}' on '#{filter_attribute}' filter") - end - filter_column = modified_filter_column || filter_column - condition = "#{filter_column} #{operator}" - condition << " :#{filter_symbol}" unless filter_symbol_already_embedded or filter_symbol.blank? - all_conditions << condition - filter_params["#{filter_symbol}"] = criterion unless filter_symbol.blank? - filter_array << { - column: filter_attribute, - operator: operator, - value: criterion, - symbol: filter_symbol - } - end - end - - if include_where - filter_string = "where " + all_conditions.join("\n and ") unless all_conditions.empty? - else - filter_string = all_conditions.empty? ? "1 = 1" : all_conditions.join("\n and ") - end - - { - filter_string: filter_string, - filter_params: filter_params, - filter_array: filter_array - } - - end - end -end diff --git a/lib/pattern_query_helper/pagination.rb b/lib/pattern_query_helper/pagination.rb deleted file mode 100644 index 5bbd397..0000000 --- a/lib/pattern_query_helper/pagination.rb +++ /dev/null @@ -1,47 +0,0 @@ -require 'active_record' -require 'kaminari' - -module PatternQueryHelper - class Pagination - def self.parse_pagination_params(page, per_page) - page = page ? page.to_i : 1 - per_page = per_page ? per_page.to_i : 20 - raise RangeError.new("page must be greater than 0") unless page > 0 - raise RangeError.new("per_page must be greater than 0") unless per_page > 0 - - { - page: page, - per_page: per_page - } - end - - def self.create_pagination_payload(count, pagination_params) - page = pagination_params[:page] - per_page = pagination_params[:per_page] - total_pages = (count/(per_page.nonzero? || 1).to_f).ceil - next_page = page + 1 if page.between?(1, total_pages - 1) - previous_page = page - 1 if page.between?(2, total_pages) - first_page = page == 1 - last_page = page == total_pages - out_of_range = !page.between?(1,total_pages) - - { - count: count, - current_page: page, - next_page: next_page, - previous_page: previous_page, - total_pages: total_pages, - per_page: per_page, - first_page: first_page, - last_page: last_page, - out_of_range: out_of_range - } - end - - def self.paginate_active_record_query(active_record_call, pagination_params) - page = pagination_params[:page] - per_page = pagination_params[:per_page] - active_record_call.page(page).per(per_page) - end - end -end diff --git a/lib/pattern_query_helper/sorting.rb b/lib/pattern_query_helper/sorting.rb deleted file mode 100644 index 01ad449..0000000 --- a/lib/pattern_query_helper/sorting.rb +++ /dev/null @@ -1,40 +0,0 @@ -module PatternQueryHelper - class Sorting - def self.parse_sorting_params(sort, valid_columns) - sort_sql = [] - if sort - sorts = sort.split(",") - sorts.each_with_index do |sort, index| - column = sort.split(":")[0] - direction = sort.split(":")[1] - modifier = sort.split(":")[2] - - raise ArgumentError.new("Sorting not allowed on column '#{column}'") unless valid_columns.include? column - - if direction == "desc" - case PatternQueryHelper.active_record_adapter - when "sqlite3" - direction = "desc" - else - direction = "desc nulls last" - end - else - direction = "asc" - end - - case modifier - when "lowercase" - column = "lower(#{column})" - end - - sort_sql << "#{column} #{direction}" - end - end - sort_sql.join(", ") - end - - def self.sort_active_record_query(active_record_call, sort_string) - active_record_call.order(sort_string) - end - end -end diff --git a/lib/pattern_query_helper/sql.rb b/lib/pattern_query_helper/sql.rb deleted file mode 100644 index bda7300..0000000 --- a/lib/pattern_query_helper/sql.rb +++ /dev/null @@ -1,57 +0,0 @@ -module PatternQueryHelper - class Sql - QUERY_COUNT_COLUMN = "_query_full_count".freeze - def self.sql_query(config) - model = config[:model] - query_params = config[:query_params] || {} - page = config[:page] - per_page = config[:per_page] - filter_params = config[:filter_params] || {} - - if page && per_page - query_params[:limit] = per_page - query_params[:offset] = (page - 1) * per_page - limit = "limit :limit offset :offset" - end - - full_count_join = "cross join (select count(*) as #{QUERY_COUNT_COLUMN} from filtered_query) as filtered_query_count" if page || per_page - query_params = query_params.merge(filter_params).symbolize_keys - - sql = %( - with filtered_query as (#{filtered_query(config)}) - select * - from filtered_query - #{full_count_join} - #{limit} - ) - - model.find_by_sql([sql, query_params]) - end - - def self.sql_query_count(sql_query_results) - sql_query_results.empty? ? 0 : sql_query_results.first[QUERY_COUNT_COLUMN] - end - - def self.single_record_query(config) - results = sql_query(config) - results.first - end - - private - - def self.filtered_query(config) - query = config[:query] - filter_string = config[:filter_string] - sort_string = config[:sort_string] - sort_string = "order by #{sort_string}" if !sort_string.blank? - - sql = %( - with query as (#{query}) - select * - from query - #{filter_string} - #{sort_string} - ) - end - end -end diff --git a/lib/pattern_query_helper/version.rb b/lib/pattern_query_helper/version.rb deleted file mode 100644 index 1c65dd5..0000000 --- a/lib/pattern_query_helper/version.rb +++ /dev/null @@ -1,3 +0,0 @@ -module PatternQueryHelper - VERSION = "0.2.10" -end diff --git a/lib/query_helper.rb b/lib/query_helper.rb new file mode 100644 index 0000000..e8413b0 --- /dev/null +++ b/lib/query_helper.rb @@ -0,0 +1,173 @@ +require "active_record" + +require "query_helper/version" +require "query_helper/filter" +require "query_helper/column_map" +require "query_helper/associations" +require "query_helper/query_helper_concern" +require "query_helper/sql_parser" +require "query_helper/sql_manipulator" +require "query_helper/sql_filter" +require "query_helper/sql_sort" +require "query_helper/invalid_query_error" + +class QueryHelper + + attr_accessor :model, :query, :bind_variables, :sql_filter, :sql_sort, :page, :per_page, :single_record, :associations, :as_json_options, :executed_query, :api_payload + + def initialize( + model: nil, # the model to run the query against + query: nil, # a sql string or an active record query + bind_variables: {}, # a list of bind variables to be embedded into the query + sql_filter: SqlFilter.new(), # a SqlFilter object + sql_sort: SqlSort.new(), # a SqlSort object + page: nil, # define the page you want returned + per_page: nil, # define how many results you want per page + single_record: false, # whether or not you expect the record to return a single result, if toggled, only the first result will be returned + associations: nil, # a list of activerecord associations you'd like included in the payload + as_json_options: nil, # a list of as_json options you'd like run before returning the payload + custom_mappings: {}, # custom keyword => sql_expression mappings + api_payload: false # Return the paginated payload or simply return the result array + ) + @model = model + @query = query + @bind_variables = bind_variables + @sql_filter = sql_filter + @sql_sort = sql_sort + @page = page.to_i if page + @per_page = per_page.to_i if per_page + @single_record = single_record + @associations = associations + @as_json_options = as_json_options + @custom_mappings = custom_mappings + @api_payload = api_payload + + if @page && @per_page + # Determine limit and offset + limit = @per_page + offset = (@page - 1) * @per_page + + # Merge limit/offset variables into bind_variables + @bind_variables.merge!({limit: limit, offset: offset}) + end + end + + def update_query(query: nil, model:nil, bind_variables: {}) + @model = model if model + @query = query if query + @bind_variables.merge!(bind_variables) + end + + def add_filter(operator_code:, criterion:, comparate:) + @sql_filter.filter_values["comparate"] = { operator_code => criterion } + end + + def execute_query + # Correctly set the query and model based on query type + determine_query_type() + + # Create column maps to be used by the filter and sort objects + column_maps = create_column_maps() + @sql_filter.column_maps = column_maps + @sql_sort.column_maps = column_maps + + # create the filters from the column maps + @sql_filter.create_filters() + + # merge the filter bind variables into the query bind variables + @bind_variables.merge!(@sql_filter.bind_variables) + + # Execute Sql Query + manipulator = SqlManipulator.new( + sql: @query, + where_clauses: @sql_filter.where_clauses, + having_clauses: @sql_filter.having_clauses, + order_by_clauses: @sql_sort.parse_sort_string, + include_limit_clause: @page && @per_page ? true : false + ) + @executed_query = manipulator.build() + @results = @model.find_by_sql([@executed_query, @bind_variables]) # Execute Sql Query + @results = @results.first if @single_record # Return a single result if requested + + determine_count() + load_associations() + clean_results() + end + + def results() + execute_query() + return paginated_results() if @api_payload + return @results + end + + + + private + + def paginated_results + { pagination: pagination_results(), + data: @results } + end + + def determine_query_type + # If a custom sql string is passed in, make sure a valid model is passed in as well + if @query.class == String + raise InvalidQueryError.new("a valid model must be included to run a custom SQL query") unless @model < ActiveRecord::Base + # If an active record query is passed in, find the model and sql from the query + elsif @query.class < ActiveRecord::Relation + @model = @query.model + @query = @query.to_sql + else + raise InvalidQueryError.new("unable to determine query type") + end + end + + def determine_count + # Determine total result count (unpaginated) + @count = @page && @per_page && @results.length > 0 ? @results.first["_query_full_count"] : @results.length + end + + def load_associations + @results = Associations.load_associations( + payload: @results, + associations: @associations, + as_json_options: @as_json_options + ) + end + + def clean_results + @results.map!{ |r| r.except("_query_full_count") } if @page && @per_page + end + + def pagination_results + # Set pagination params if they aren't provided + @per_page = @count unless @per_page + @page = 1 unless @page + + total_pages = (@count/(@per_page.nonzero? || 1).to_f).ceil + next_page = @page + 1 if @page.between?(1, total_pages - 1) + previous_page = @page - 1 if @page.between?(2, total_pages) + first_page = @page == 1 + last_page = @page == total_pages + out_of_range = !@page.between?(1,total_pages) + + { count: @count, + current_page: @page, + next_page: next_page, + previous_page: previous_page, + total_pages: total_pages, + per_page: @per_page, + first_page: first_page, + last_page: last_page, + out_of_range: out_of_range } + end + + def create_column_maps + ColumnMap.create_column_mappings( + query: @query, + custom_mappings: @custom_mappings, + model: @model + ) + end + +end diff --git a/lib/pattern_query_helper/associations.rb b/lib/query_helper/associations.rb similarity index 92% rename from lib/pattern_query_helper/associations.rb rename to lib/query_helper/associations.rb index f2e8bc7..a14a2c3 100644 --- a/lib/pattern_query_helper/associations.rb +++ b/lib/query_helper/associations.rb @@ -1,11 +1,11 @@ -module PatternQueryHelper +class QueryHelper class Associations def self.process_association_params(associations) associations ||= [] associations.class == String ? [associations.to_sym] : associations end - def self.load_associations(payload, associations, as_json_options) + def self.load_associations(payload:, associations: [], as_json_options: {}) as_json_options ||= {} as_json_options[:include] = as_json_options[:include] || json_associations(associations) ActiveRecord::Associations::Preloader.new.preload(payload, associations) diff --git a/lib/query_helper/column_map.rb b/lib/query_helper/column_map.rb new file mode 100644 index 0000000..caa17f3 --- /dev/null +++ b/lib/query_helper/column_map.rb @@ -0,0 +1,56 @@ +require "query_helper/sql_parser" + +class QueryHelper + class ColumnMap + + def self.create_column_mappings(custom_mappings:, query:, model:) + parser = SqlParser.new(query) + maps = create_from_hash(custom_mappings) + + parser.find_aliases.each do |m| + maps << m if maps.select{|x| x.alias_name == m.alias_name}.empty? + end + + model.attribute_names.each do |attribute| + if maps.select{|x| x.alias_name == attribute}.empty? + maps << ColumnMap.new(alias_name: attribute, sql_expression: "#{model.to_s.downcase.pluralize}.#{attribute}") + end + end + + maps + end + + def self.create_from_hash(hash) + map = [] + hash.each do |k,v| + alias_name = k + aggregate = false + if v.class == String + sql_expression = v + elsif v.class == Hash + sql_expression = v[:sql_expression] + aggregate = v[:aggregate] + end + map << self.new( + alias_name: alias_name, + sql_expression: sql_expression, + aggregate: aggregate + ) + end + map + end + + attr_accessor :alias_name, :sql_expression, :aggregate + + def initialize( + alias_name:, + sql_expression:, + aggregate: false + ) + @alias_name = alias_name + @sql_expression = sql_expression + @aggregate = aggregate + end + + end +end diff --git a/lib/query_helper/filter.rb b/lib/query_helper/filter.rb new file mode 100644 index 0000000..8094560 --- /dev/null +++ b/lib/query_helper/filter.rb @@ -0,0 +1,112 @@ +require "query_helper/invalid_query_error" + +class QueryHelper + class Filter + + attr_accessor :operator, :criterion, :comparate, :operator_code, :bind_variable, :aggregate + + def initialize( + operator_code:, + criterion:, + comparate:, + aggregate: false + ) + @operator_code = operator_code + @criterion = criterion # Converts to a string to be inserted into sql. + @comparate = comparate + @aggregate = aggregate + @bind_variable = ('a'..'z').to_a.shuffle[0,20].join.to_sym + + translate_operator_code() + mofify_criterion() + modify_comparate() + validate_criterion() + end + + def sql_string + case operator_code + when "in", "notin" + "#{comparate} #{operator} (:#{bind_variable})" + when "null" + "#{comparate} #{operator}" + else + "#{comparate} #{operator} :#{bind_variable}" + end + + end + + private + + def translate_operator_code + @operator = case operator_code + when "gte" + ">=" + when "lte" + "<=" + when "gt" + ">" + when "lt" + "<" + when "eql" + "=" + when "noteql" + "!=" + when "in" + "in" + when "like" + "like" + when "notin" + "not in" + when "null" + if criterion.to_s == "true" + "is null" + else + "is not null" + end + else + raise InvalidQueryError.new("Invalid operator code: '#{operator_code}'") + end + end + + def mofify_criterion + # lowercase strings for comparison + @criterion.downcase! if criterion.class == String && criterion.scan(/[a-zA-Z]/).any? + + # turn the criterion into an array for in and notin comparisons + @criterion = criterion.split(",") if ["in", "notin"].include?(operator_code) && criterion.class == String + end + + def modify_comparate + # lowercase strings for comparison + @comparate = "lower(#{@comparate})" if criterion.class == String && criterion.scan(/[a-zA-Z]/).any? && !["true", "false"].include?(criterion) + end + + def validate_criterion + case operator_code + when "gte", "lte", "gt", "lt" + begin + Time.parse(criterion.to_s) + rescue + begin + Date.parse(criterion.to_s) + rescue + begin + Float(criterion.to_s) + rescue + invalid_criterion_error() + end + end + end + when "in", "notin" + invalid_criterion_error() unless criterion.class == Array + when "null" + invalid_criterion_error() unless ["true", "false"].include?(criterion.to_s) + end + true + end + + def invalid_criterion_error + raise InvalidQueryError.new("'#{criterion}' is not a valid criterion for the '#{@operator}' operator") + end + end +end diff --git a/lib/query_helper/invalid_query_error.rb b/lib/query_helper/invalid_query_error.rb new file mode 100644 index 0000000..a394eba --- /dev/null +++ b/lib/query_helper/invalid_query_error.rb @@ -0,0 +1,3 @@ +class QueryHelper + class InvalidQueryError < StandardError; end +end diff --git a/lib/query_helper/query_helper_concern.rb b/lib/query_helper/query_helper_concern.rb new file mode 100644 index 0000000..04094c7 --- /dev/null +++ b/lib/query_helper/query_helper_concern.rb @@ -0,0 +1,41 @@ +require 'active_support/concern' +require "query_helper/sql_filter" + +class QueryHelper + module QueryHelperConcern + extend ActiveSupport::Concern + + included do + def query_helper + @query_helper + end + + def create_query_helper + @query_helper = QueryHelper.new(**query_helper_params, api_payload: true) + end + + def create_query_helper_filter + filter_values = params[:filter].permit!.to_h + QueryHelper::SqlFilter.new(filter_values: filter_values) + end + + def create_query_helper_sort + QueryHelper::SqlSort.new(sort_string: params[:sort]) + end + + def create_query_helper_associations + + end + + def query_helper_params + helpers = {} + helpers[:page] = params[:page] if params[:page] + helpers[:per_page] = params[:per_page] if params[:per_page] + helpers[:sql_filter] = create_query_helper_filter() if params[:filter] + helpers[:sql_sort] = create_query_helper_sort() if params[:sort] + helpers[:associations] = create_query_helper_associations() if params[:include] + helpers + end + end + end +end diff --git a/lib/query_helper/sql_filter.rb b/lib/query_helper/sql_filter.rb new file mode 100644 index 0000000..a557ea4 --- /dev/null +++ b/lib/query_helper/sql_filter.rb @@ -0,0 +1,43 @@ +require "query_helper/invalid_query_error" + +class QueryHelper + class SqlFilter + + attr_accessor :filter_values, :column_maps + + def initialize(filter_values: [], column_maps: []) + @column_maps = column_maps + @filter_values = filter_values + end + + def create_filters + @filters = [] + + @filter_values.each do |comparate_alias, criteria| + # Find the sql mapping if it exists + map = @column_maps.find { |m| m.alias_name == comparate_alias } + raise InvalidQueryError.new("cannot filter by #{comparate_alias}") unless map + + # create the filter + @filters << QueryHelper::Filter.new( + operator_code: criteria.keys.first, + criterion: criteria.values.first, + comparate: map.sql_expression, + aggregate: map.aggregate + ) + end + end + + def where_clauses + @filters.select{ |f| f.aggregate == false }.map(&:sql_string) + end + + def having_clauses + @filters.select{ |f| f.aggregate == true }.map(&:sql_string) + end + + def bind_variables + Hash[@filters.collect { |f| [f.bind_variable, f.criterion] }] + end + end +end diff --git a/lib/query_helper/sql_manipulator.rb b/lib/query_helper/sql_manipulator.rb new file mode 100644 index 0000000..dc352d1 --- /dev/null +++ b/lib/query_helper/sql_manipulator.rb @@ -0,0 +1,62 @@ +require "query_helper/sql_parser" + +class QueryHelper + class SqlManipulator + + attr_accessor :sql + + def initialize( + sql:, + where_clauses: nil, + having_clauses: nil, + order_by_clauses: nil, + include_limit_clause: false + ) + @parser = SqlParser.new(sql) + @sql = @parser.sql.dup + @where_clauses = where_clauses + @having_clauses = having_clauses + @order_by_clauses = order_by_clauses + @include_limit_clause = include_limit_clause + end + + def build + insert_having_clauses() + insert_where_clauses() + insert_total_count_select_clause() + insert_order_by_and_limit_clause() + @sql.squish + end + + private + + def insert_total_count_select_clause + return unless @include_limit_clause + total_count_clause = " ,count(*) over () as _query_full_count " + @sql.insert(@parser.insert_select_index, total_count_clause) + # Potentially update parser here + end + + def insert_where_clauses + return unless @where_clauses.length > 0 + begin_string = @parser.where_included? ? "and" : "where" + filter_string = @where_clauses.join(" and ") + " #{begin_string} #{filter_string} " + @sql.insert(@parser.insert_where_index, " #{begin_string} #{filter_string} ") + end + + def insert_having_clauses + return unless @having_clauses.length > 0 + begin_string = @parser.having_included? ? "and" : "having" + filter_string = @having_clauses.join(" and ") + @sql.insert(@parser.insert_having_index, " #{begin_string} #{filter_string} ") + end + + def insert_order_by_and_limit_clause + @sql.slice!(@parser.limit_clause) if @parser.limit_included? # remove existing limit clause + @sql.slice!(@parser.order_by_clause) if @parser.order_by_included? # remove existing order by clause + @sql += " order by #{@order_by_clauses.join(", ")} " if @order_by_clauses.length > 0 + @sql += " limit :limit offset :offset " if @include_limit_clause + end + end +end diff --git a/lib/query_helper/sql_parser.rb b/lib/query_helper/sql_parser.rb new file mode 100644 index 0000000..65c0864 --- /dev/null +++ b/lib/query_helper/sql_parser.rb @@ -0,0 +1,189 @@ +require "query_helper/invalid_query_error" +require "query_helper/column_map" + +class QueryHelper + class SqlParser + + attr_accessor :sql + + def initialize(sql) + update(sql) + end + + def update(sql) + @sql = sql + remove_comments() + white_out() + end + + def remove_comments + # Remove SQL inline comments (/* */) and line comments (--) + @sql = @sql.gsub(/\/\*(.*?)\*\//, '').gsub(/--(.*)$/, '') + @sql.squish! + end + + def white_out + # Replace everything between () and '' and "" + # This will allow us to ignore subqueries, common table expressions, + # regex, custom strings, etc. when determining injection points + # and performing other manipulations + @white_out_sql = @sql.dup + while @white_out_sql.scan(/\"[^""]*\"|\'[^'']*\'|\([^()]*\)/).length > 0 do + @white_out_sql.scan(/\"[^""]*\"|\'[^'']*\'|\([^()]*\)/).each { |s| @white_out_sql.gsub!(s,s.gsub(/./, '*')) } + end + end + + def select_index(position=:start) + regex = /( |^)[Ss][Ee][Ll][Ee][Cc][Tt] / # space or new line at beginning of select + find_index(regex, position) + end + + def from_index(position=:start) + regex = / [Ff][Rr][Oo][Mm] / + find_index(regex, position) + end + + def where_index(position=:start) + regex = / [Ww][Hh][Ee][Rr][Ee] / + find_index(regex, position) + end + + def group_by_index(position=:start) + regex = / [Gg][Rr][Oo][Uu][Pp] [Bb][Yy] / + find_index(regex, position) + end + + def having_index(position=:start) + regex = / [Hh][Aa][Vv][Ii][Nn][Gg] / + find_index(regex, position) + end + + def order_by_index(position=:start) + regex = / [Oo][Rr][Dd][Ee][Rr] [Bb][Yy] / + find_index(regex, position) + end + + def limit_index(position=:start) + regex = / [Ll][Ii][Mm][Ii][Tt] / + find_index(regex, position) + end + + def select_included? + !select_index.nil? + end + + def from_included? + !from_index.nil? + end + + def where_included? + !where_index.nil? + end + + def group_by_included? + !group_by_index.nil? + end + + def having_included? + !having_index.nil? + end + + def order_by_included? + !order_by_index.nil? + end + + def limit_included? + !limit_index.nil? + end + + def insert_select_index + from_index() || where_index() || group_by_index() || order_by_index() || limit_index() || @sql.length + end + + def insert_join_index + where_index() || group_by_index() || order_by_index() || limit_index() || @sql.length + end + + def insert_where_index + group_by_index() || order_by_index() || limit_index() || @sql.length + end + + def insert_having_index + # raise InvalidQueryError.new("Cannot calculate insert_having_index because the query has no group by clause") unless group_by_included? + order_by_index() || limit_index() || @sql.length + end + + def insert_order_by_index + # raise InvalidQueryError.new("This query already includes an order by clause") if order_by_included? + limit_index() || @sql.length + end + + def insert_limit_index + # raise InvalidQueryError.new("This query already includes a limit clause") if limit_included? + @sql.length + end + + def select_clause + @sql[select_index()..insert_select_index()].strip if select_included? + end + + def from_clause + @sql[from_index()..insert_join_index()].strip if from_included? + end + + def where_clause + @sql[where_index()..insert_where_index()].strip if where_included? + end + + # def group_by_clause + # @sql[group_by_index()..insert_group_by_index()] if group_by_included? + # end + + def having_clause + @sql[having_index()..insert_having_index()].strip if having_included? + end + + def order_by_clause + @sql[order_by_index()..insert_order_by_index()].strip if order_by_included? + end + + def limit_clause + @sql[limit_index()..insert_limit_index()].strip if limit_included? + end + + def find_aliases + # Determine alias expression combos. White out sql used in case there + # are any custom strings or subqueries in the select clause + white_out_selects = @white_out_sql[select_index(:end)..from_index()] + selects = @sql[select_index(:end)..from_index()] + comma_split_points = white_out_selects.each_char.with_index.map{|char, i| i if char == ','}.compact + comma_split_points.unshift(-1) # We need the first select clause to start out with a 'split' + column_maps = white_out_selects.split(",").each_with_index.map do |x,i| + sql_alias = x.squish.split(" as ")[1] || x.squish.split(" AS ")[1] || x.squish.split(".")[1] # look for custom defined aliases or table.column notation + # sql_alias = nil unless /^[a-zA-Z_]+$/.match?(sql_alias) # only allow aliases with letters and underscores + sql_expression = if x.split(" as ")[1] + expression_length = x.split(" as ")[0].length + selects[comma_split_points[i] + 1, expression_length] + elsif x.squish.split(" AS ")[1] + expression_length = x.split(" AS ")[0].length + selects[comma_split_points[i] + 1, expression_length] + elsif x.squish.split(".")[1] + selects[comma_split_points[i] + 1, x.length] + end + ColumnMap.new( + alias_name: sql_alias, + sql_expression: sql_expression.squish, + aggregate: /(array_agg|avg|bit_and|bit_or|bool_and|bool_or|count|every|json_agg|jsonb_agg|json_object_agg|jsonb_object_agg|max|min|string_agg|sum|xmlagg)\((.*)\)/.match?(sql_expression) + ) if sql_alias + end + column_maps.compact + end + + private + + def find_index(regex, position=:start) + start_position = @white_out_sql.rindex(regex) + return position == :start ? start_position : start_position + @white_out_sql[regex].size() + end + end +end diff --git a/lib/query_helper/sql_sort.rb b/lib/query_helper/sql_sort.rb new file mode 100644 index 0000000..78dd710 --- /dev/null +++ b/lib/query_helper/sql_sort.rb @@ -0,0 +1,49 @@ +require "query_helper/invalid_query_error" + +class QueryHelper + class SqlSort + + attr_accessor :column_maps + + def initialize(sort_string: "", column_maps: []) + @sort_string = sort_string + @column_maps = column_maps + end + + def parse_sort_string + sql_strings = [] + sorts = @sort_string.split(",") + sorts.each_with_index do |sort, index| + sort_alias = sort.split(":")[0] + direction = sort.split(":")[1] + modifier = sort.split(":")[2] + + begin + sql_expression = @column_maps.find{ |m| m.alias_name == sort_alias }.sql_expression + rescue NoMethodError => e + raise InvalidQueryError.new("Sorting not allowed on column '#{sort_alias}'") + end + + if direction == "desc" + case ActiveRecord::Base.connection.adapter_name + when "SQLite" # SQLite is used in the test suite + direction = "desc" + else + direction = "desc nulls last" + end + else + direction = "asc" + end + + case modifier + when "lowercase" + sql_expression = "lower(#{sql_expression})" + end + + sql_strings << "#{sql_expression} #{direction}" + end + + return sql_strings + end + end +end diff --git a/lib/query_helper/version.rb b/lib/query_helper/version.rb new file mode 100644 index 0000000..336a062 --- /dev/null +++ b/lib/query_helper/version.rb @@ -0,0 +1,3 @@ +class QueryHelper + VERSION = "0.0.0" +end diff --git a/pattern_query_helper.gemspec b/query_helper.gemspec similarity index 83% rename from pattern_query_helper.gemspec rename to query_helper.gemspec index f5d318c..897c06d 100644 --- a/pattern_query_helper.gemspec +++ b/query_helper.gemspec @@ -1,17 +1,17 @@ lib = File.expand_path("../lib", __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require "pattern_query_helper/version" +require "query_helper/version" Gem::Specification.new do |spec| - spec.name = "pattern_query_helper" - spec.version = PatternQueryHelper::VERSION + spec.name = "query_helper" + spec.version = QueryHelper::VERSION spec.authors = ["Evan McDaniel"] spec.email = ["eamigo13@gmail.com"] spec.summary = %q{Ruby Gem to help with pagination and data formatting at Pattern, Inc.} spec.description = %q{Ruby gem developed to help with pagination, filtering, sorting, and including associations on both active record queries and custom sql queries} - spec.homepage = "https://github.com/iserve-products/pattern_query_helper" + spec.homepage = "https://github.com/iserve-products/query_helper" spec.license = "MIT" # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host' @@ -42,7 +42,10 @@ Gem::Specification.new do |spec| spec.add_development_dependency "sqlite3", "~> 1.3.6" spec.add_development_dependency "faker", "~> 1.9.3" spec.add_development_dependency "byebug" + spec.add_development_dependency 'rspec-rails' + spec.add_development_dependency 'actionpack' + spec.add_development_dependency 'activesupport' spec.add_dependency "activerecord", "~> 5.0" - spec.add_dependency "kaminari", "~> 1.1.1" + spec.add_dependency "activesupport", "~> 5.0" end diff --git a/spec/fixtures/application.rb b/spec/fixtures/application.rb new file mode 100644 index 0000000..58e3c46 --- /dev/null +++ b/spec/fixtures/application.rb @@ -0,0 +1,26 @@ +require 'active_support/all' +require 'action_controller' +require 'action_dispatch' +require 'active_record' + +module Rails + class App + def env_config; {} end + def routes + return @routes if defined?(@routes) + @routes = ActionDispatch::Routing::RouteSet.new + @routes.draw do + resources :parents do + collection do + get 'test' + end + end + end + @routes + end + end + + def self.application + @app ||= App.new + end +end diff --git a/spec/fixtures/controllers.rb b/spec/fixtures/controllers.rb new file mode 100644 index 0000000..7b0bac3 --- /dev/null +++ b/spec/fixtures/controllers.rb @@ -0,0 +1,24 @@ +require 'fixtures/application' +require 'fixtures/models' + +class ApplicationController < ActionController::API + include Rails.application.routes.url_helpers + include QueryHelper::QueryHelperConcern + before_action :create_query_helper +end + +class ParentsController < ApplicationController + def index + @query_helper.query = Parent.all + render json: @query_helper.results() + end + + def test + test_query = ExampleQueries::SQL_QUERIES[params[:test_number].to_i] + @query_helper.query = test_query[:query] + @query_helper.model = test_query[:model] + results = @query_helper.results() + puts "EXECUTED QUERY: #{@query_helper.executed_query()}" + render json: @query_helper.results() + end +end diff --git a/spec/fixtures/example_queries.rb b/spec/fixtures/example_queries.rb new file mode 100644 index 0000000..025b201 --- /dev/null +++ b/spec/fixtures/example_queries.rb @@ -0,0 +1,198 @@ +module ExampleQueries + SQL_QUERIES = [ + { + query: %( + select c.id, c.name, c.age + from children c + ), + model: Child, + expected_sorts: ["id", "name", "age"], + expected_filters: [ + { + alias: "id", + operator_codes: ["gte", "gt", "lte", "lt", "eql", "noteql"], + class: Integer + }, + { + alias: "age", + operator_codes: ["gte", "gt", "lte", "lt", "eql", "noteql"], + class: Integer + }, + { + alias: "name", + operator_codes: ["like"], + class: String + }, + { + alias: "name", + operator_codes: ["in", "notin"], + class: String + }, + { + alias: "age", + operator_codes: ["null"], + class: TrueClass + }, + ] + }, + { + query: %( + select p.name, p.age, count(c.id) as children_count + from parents p join children c on c.parent_id = p.id + where p.age > 30 + group by p.name + having count(c.id) > 3 + order by p.name + limit 1 + ), + model: Parent, + expected_sorts: ["name", "age", "children_count"], + expected_filters: [ + { + alias: "age", + operator_codes: ["gte", "gt", "lte", "lt", "eql", "noteql"], + class: Integer + }, + { + alias: "name", + operator_codes: ["like"], + class: String + }, + { + alias: "name", + operator_codes: ["in", "notin"], + class: String + }, + { + alias: "age", + operator_codes: ["null"], + class: TrueClass + }, + { + alias: "children_count", + operator_codes: ["gte", "gt", "lte", "lt", "eql", "noteql"], + class: Integer + }, + ] + }, + { + query: %( + with children_count as ( + select p.id as parent_id, p.age, count(c.id) as children_count + from parents p join children c on c.parent_id = p.id + group by p.id + ), + max_age as ( + select p.id as parent_id, max(c.age) as max_age + from parents p join children c on c.parent_id = p.id + group by p.id + ) + select p.name, c.children_count, m.max_age + from parents p + join children_count c on p.id = c.parent_id + join max_age m on p.id = m.parent_id + ), + model: Child, + expected_sorts: ["name", "children_count", "max_age"] + }, + { + query: %( + select + p.name, + ( + select max(c2.age) + from parents p2 join children c2 on c2.parent_id = p2.id + where p2.id = p.id + group by p2.id + ) as max_age + from parents p + ), + model: Child, + expected_sorts: ["name", "max_age"] + }, + { + query: %( + select c.id, c.name, c.age, p.name as parent_name + from children c + join parents p on p.id = c.parent_id + ), + model: Child, + expected_sorts: ["id", "name", "age", "parent_name"] + }, + { + query: %( + select + p.name as parent_name, + count(c.id) as count, + COUNT(c.age) as count2, + -- array_agg(c.name) as children_names, + -- ARRAY_AGG(c.name) as children_names2, + avg(c.age) as average, + AVG(c.id) as average2, + -- bit_and(c.age) as bit_and, + -- BIT_AND(c.age) as bit_and2, + -- bit_or(c.age) as bit_or, + -- BIT_OR(c.age) as bit_or2, + -- bool_and(true) as bool_and, + -- BOOL_AND(true) as bool_and2, + -- bool_or(true) as bool_or, + -- BOOL_OR(true) as bool_or2, + -- every(false) as every, + -- EVERY(false) as every2, + -- json_agg(c.*) as json_agg, + -- JSON_AGG(c.*) as json_agg2, + -- jsonb_agg(c.*) as jsonb_agg, + -- JSONB_AGG(c.*) as jsonb_agg2, + -- json_object_agg(c.*) as json_object_agg, + -- JSON_OBJECT_AGG(c.*) as json_object_agg2, + -- jsonb_object_agg(c.*) as jsonb_object_agg, + -- JSONB_OBJECT_AGG(c.*) as jsonb_object_agg2, + max(c.age) as max, + MAX(c.id) as max2, + min(c.age) as min, + MIN(c.id) as min2, + sum(c.age) as sum, + SUM(c.id) as sum2 + from children c + join parents p on p.id = c.parent_id + group by p.name + ), + model: Child, + expected_sorts: [ + "parent_name", + "count", + "count2", + # "children_names", + # "children_names2", + "average", + "average2", + # "bit_and", + # "bit_and2", + # "bit_or", + # "bit_or2", + # "bool_and", + # "bool_and2", + # "bool_or", + # "bool_or2", + # "every", + # "every2", + # "every2", + # "json_agg", + # "json_agg2", + # "jsonb_agg", + # "jsonb_agg2", + # "json_object_agg", + # "json_object_agg2", + # "jsonb_object_agg", + # "jsonb_object_agg2", + "max", + "max2", + "min", + "min2", + "sum", + "sum2" + ] + }, + ] + +end diff --git a/spec/fixtures/models.rb b/spec/fixtures/models.rb new file mode 100644 index 0000000..3b7a68a --- /dev/null +++ b/spec/fixtures/models.rb @@ -0,0 +1,26 @@ +# class Post +# extend ActiveModel::Naming +# include ActiveModel::Conversion +# attr_accessor :id +# +# def initialize(attributes={}) +# self.id = attributes +# end +# +# def persisted? +# true +# end +# end +class ApplicationRecord < ActiveRecord::Base + self.abstract_class = true +end +class Parent < ApplicationRecord + has_many :children + + def favorite_star_wars_character + Faker::Movies::StarWars.character + end +end +class Child < ApplicationRecord + belongs_to :parent +end diff --git a/spec/fixtures/routes.rb b/spec/fixtures/routes.rb new file mode 100644 index 0000000..d5ee432 --- /dev/null +++ b/spec/fixtures/routes.rb @@ -0,0 +1,6 @@ +Rails.application.routes.draw do + # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html + # Serve websocket cable requests in-process + + resources :parents, only: [:index, :show] +end diff --git a/spec/pattern_query_helper/associations_spec.rb b/spec/pattern_query_helper/associations_spec.rb deleted file mode 100644 index d6d1057..0000000 --- a/spec/pattern_query_helper/associations_spec.rb +++ /dev/null @@ -1,41 +0,0 @@ -require "spec_helper" - -RSpec.describe PatternQueryHelper::Associations do - - describe "process_association_params" do - it "parses association params" do - associations = PatternQueryHelper::Associations.process_association_params("parent") - expect(associations).to eq([:parent]) - end - end - - describe "load_associations" do - it "loads associations" do - associations = PatternQueryHelper::Associations.process_association_params("parent") - payload = Child.all - results = PatternQueryHelper::Associations.load_associations(payload, associations, nil) - results.each do |child| - expect(child["parent_id"]).to eq(child["parent"]["id"]) - end - end - end - - describe 'json_associations' do - subject { described_class.json_associations(associations) } - - context 'nested associations' do - let(:associations) do - [:parent, - children: [:grand_children], - pets: :grand_pets, - messages: { author: [:avatar, :profile] }] - end - - it 'translates to as_json format' do - expect(subject).to eq([:parent, children: { include: [:grand_children] }, - pets: { include: [:grand_pets] }, - messages: { include: [ author: { include: [:avatar, :profile] }]}]) - end - end - end -end diff --git a/spec/pattern_query_helper/filtering_spec.rb b/spec/pattern_query_helper/filtering_spec.rb deleted file mode 100644 index af210c9..0000000 --- a/spec/pattern_query_helper/filtering_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -require "spec_helper" - -RSpec.describe PatternQueryHelper::Filtering do - - describe "parse_pagination_params" do - it "create filters" do - filters = PatternQueryHelper::Filtering.create_filters(filters: { - "id" => { - "gte" => 20, - "lt" => 40, - "in" => "20,25,30", - "null" => false - }, - "name" => { - "like" => "my_name%" - } - }, valid_columns_map: {"id" => "id", "name" => "name"}) - expect(filters[:filter_string]).to eq("where id >= :id_gte\n and id < :id_lt\n and id in (:id_in)\n and id is not null\n and lower(name) like :name_like") - expect(filters[:filter_params]).to eq({"id_gte"=>20, "id_in"=>["20","25","30"], "id_lt"=>40, "name_like"=>"my_name%"}) - expect(filters[:filter_array]).to eq([ - {:column=>"id", :operator=>">=", :value=>20, :symbol=>"id_gte"}, - {:column=>"id", :operator=>"<", :value=>40, :symbol=>"id_lt"}, - {:column=>"id", :operator=>"in (:id_in)", :value=>["20","25","30"], :symbol=>"id_in"}, - {:column=>"id", :operator=>"is not null", :value=>false, :symbol=>""}, - {:column=>"name", :operator=>"like", :value=>"my_name%", :symbol=>"name_like"}, - ]) - end - it "handles a single filter" do - filters = PatternQueryHelper::Filtering.create_filters(filters: { - "id" => { - "gte" => 20 - } - }, valid_columns_map: {"id" => "id"}) - expect(filters[:filter_string]).to eq("where id >= :id_gte") - expect(filters[:filter_params]).to eq({"id_gte"=>20}) - expect(filters[:filter_array]).to eq([ - {:column=>"id", :operator=>">=", :value=>20, :symbol=>"id_gte"}, - ]) - end - it "handles no filter" do - filters = PatternQueryHelper::Filtering.create_filters(filters: { - }, include_where: false) - expect(filters[:filter_string]).to eq("1 = 1") - expect(filters[:filter_params]).to eq({}) - expect(filters[:filter_array]).to eq([]) - end - end -end diff --git a/spec/pattern_query_helper/pagination_spec.rb b/spec/pattern_query_helper/pagination_spec.rb deleted file mode 100644 index 20d3623..0000000 --- a/spec/pattern_query_helper/pagination_spec.rb +++ /dev/null @@ -1,73 +0,0 @@ -require "spec_helper" - -RSpec.describe PatternQueryHelper::Pagination do - before(:each) do - @per_page = Faker::Number.between(5,15) - @page = Faker::Number.between(2,5) - end - - describe "parse_pagination_params" do - - it "creates pagination params from url params" do - pagination_params = PatternQueryHelper::Pagination.parse_pagination_params(@page, @per_page) - expect(pagination_params[:per_page]).to eq(@per_page) - expect(pagination_params[:page]).to eq(@page) - end - - it "fails if page <= 0" do - expect { PatternQueryHelper::Pagination.parse_pagination_params(0, @per_page) }.to raise_error(RangeError) - end - - it "fails if per_page <= 0" do - expect { PatternQueryHelper::Pagination.parse_pagination_params(@page, 0) }.to raise_error(RangeError) - end - - it "default per page is 20" do - pagination_params = PatternQueryHelper::Pagination.parse_pagination_params(@page, nil) - expect(pagination_params[:per_page]).to eq(20) - end - - it "default page is 1" do - pagination_params = PatternQueryHelper::Pagination.parse_pagination_params(nil, @per_page) - expect(pagination_params[:page]).to eq(1) - end - - end - - describe "create_pagination_payload" do - - before(:each) do - @pagination_params = PatternQueryHelper::Pagination.parse_pagination_params(@page, @per_page) - @count = Faker::Number.between(100, 500) - @pagination_payload = PatternQueryHelper::Pagination.create_pagination_payload(@count, @pagination_params) - expect(@pagination_payload[:previous_page]).to eq(@page - 1) - expect(@pagination_payload[:next_page]).to eq(@page + 1) - end - - it "calculate total_pages correctly" do - expect(@pagination_payload[:total_pages]).to eq((@count/@per_page.to_f).ceil) - end - - it "next_page is null if on last page" do - @pagination_params[:page] = @pagination_payload[:total_pages] - @pagination_payload = PatternQueryHelper::Pagination.create_pagination_payload(@count, @pagination_params) - expect(@pagination_payload[:next_page]).to eq(nil) - expect(@pagination_payload[:last_page]).to eq(true) - end - - it "previous_page is null if on first page" do - @pagination_params[:page] = 1 - @pagination_payload = PatternQueryHelper::Pagination.create_pagination_payload(@count, @pagination_params) - expect(@pagination_payload[:previous_page]).to eq(nil) - expect(@pagination_payload[:first_page]).to eq(true) - end - - it "out_of_range determined correctly" do - @pagination_params[:page] = @pagination_payload[:total_pages] + 1 - @pagination_payload = PatternQueryHelper::Pagination.create_pagination_payload(@count, @pagination_params) - expect(@pagination_payload[:out_of_range]).to eq(true) - end - - end - -end diff --git a/spec/pattern_query_helper/sorting_spec.rb b/spec/pattern_query_helper/sorting_spec.rb deleted file mode 100644 index cee0772..0000000 --- a/spec/pattern_query_helper/sorting_spec.rb +++ /dev/null @@ -1,16 +0,0 @@ -require "spec_helper" - -RSpec.describe PatternQueryHelper::Sorting do - - describe "parse_sorting_params" do - it "parses params into sorting sql string" do - sort_string = PatternQueryHelper::Sorting.parse_sorting_params("name:desc", ["name"]) - expect(sort_string).to eq("name desc") - end - - it "lowercase if asked for" do - sort_string = PatternQueryHelper::Sorting.parse_sorting_params("name:desc:lowercase", ["name"]) - expect(sort_string).to eq("lower(name) desc") - end - end -end diff --git a/spec/pattern_query_helper/sql_spec.rb b/spec/pattern_query_helper/sql_spec.rb deleted file mode 100644 index 89013de..0000000 --- a/spec/pattern_query_helper/sql_spec.rb +++ /dev/null @@ -1,103 +0,0 @@ -require "spec_helper" - -RSpec.describe PatternQueryHelper::Sql do - - describe "sql_query" do - it "query returns the same number of results as designated by per_page" do - per_page = Faker::Number.between(2,10) - results = PatternQueryHelper::Sql.sql_query( - model: Child, - query: "select * from children c join parents p on p.id = c.parent_id", - page: Faker::Number.between(1,10), - per_page: per_page - ) - - expect(results.length).to eq(per_page) - end - - it "query sorts correctly" do - sort_string = PatternQueryHelper::Sorting.parse_sorting_params("id:desc", ["id"]) - results = PatternQueryHelper::Sql.sql_query( - model: Child, - query: "select * from children c join parents p on p.id = c.parent_id", - page: 1, - per_page: 500, - sort_string: sort_string - ) - previous_id = 1000000000 - results.each do |result| - expect(result.id).to be < previous_id - previous_id = result.id - end - end - - it "query filters correctly" do - filters = PatternQueryHelper::Filtering.create_filters(filters: { - "id" => { - "gte" => 20, - "lt" => 40 - } - }) - results = PatternQueryHelper::Sql.sql_query( - model: Child, - query: "select * from children c join parents p on p.id = c.parent_id", - page: 1, - per_page: 500, - filter_string: filters[:filter_string], - filter_params: filters[:filter_params] - ) - results.each do |result| - expect(result.id).to be >= 20 - expect(result.id).to be < 40 - end - end - - it "query returns all results if no pagination info" do - results = PatternQueryHelper::Sql.sql_query( - model: Child, - query: "select * from children c join parents p on p.id = c.parent_id", - ) - expect(results.length).to eq(Child.all.length) - end - end - - describe "sql_query_count" do - it "should count the number of rows correctly" do - results = PatternQueryHelper::Sql.sql_query( - model: Child, - query: "select * from children c join parents p on p.id = c.parent_id", - page: 1 - ) - count = PatternQueryHelper::Sql.sql_query_count(results) - expect(count).to eq(Child.all.length) - end - - it "should count the number of rows correctly with filters in place" do - filters = PatternQueryHelper::Filtering.create_filters(filters: { - "id" => { - "gte" => 20, - "lt" => 40 - } - }) - results = PatternQueryHelper::Sql.sql_query( - model: Child, - query: "select * from children c join parents p on p.id = c.parent_id", - filter_string: filters[:filter_string], - filter_params: filters[:filter_params], - page: 1, - ) - count = PatternQueryHelper::Sql.sql_query_count(results) - expect(count).to eq(Child.all.where("id >= 20 and id < 40").length) - end - end - - describe "single_record_query" do - it "returns one result" do - result = PatternQueryHelper::Sql.single_record_query( - model: Child, - query: "select * from children c join parents p on p.id = c.parent_id where c.id = 1", - ) - expect(result.class).to eq(Child) - end - end -end diff --git a/spec/pattern_query_helper_spec.rb b/spec/pattern_query_helper_spec.rb deleted file mode 100644 index a102c44..0000000 --- a/spec/pattern_query_helper_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -RSpec.describe PatternQueryHelper do - it "returns a paginated sql query" do - results = PatternQueryHelper.run_sql_query(Child, "select * from children", {}, @url_params, ["name", "id"]) - expect(results[:pagination]).to_not be nil - expect(results[:data]).to_not be nil - end - - it "returns a sql query" do - results = PatternQueryHelper.run_sql_query(Child, "select * from children", {}, @url_params, ["name", "id"]) - expect(results[:data]).to_not be nil - end - - it "returns a single record sql query" do - @url_params[:filter] = nil - results = PatternQueryHelper.run_sql_query(Child, "select * from children where id=#{Child.first.id}", {}, @url_params, ["name", "id"], true) - expect(results[:data]).to_not be nil - end - - it "returns an active record query" do - results = PatternQueryHelper.run_active_record_query(Child.all, @url_params, ["name", "id"]) - expect(results[:data]).to_not be nil - end - - it "returns a paginated active record query" do - results = PatternQueryHelper.run_active_record_query(Child.all, @url_params, ["name", "id"]) - expect(results[:pagination]).to_not be nil - expect(results[:data]).to_not be nil - end - - it "has a version number" do - expect(PatternQueryHelper::VERSION).not_to be nil - end - - it "sets up the test database correctly" do - expect(Parent.all.count).to eq(100) - # Every parent has between 2 and 6 children - expect(Child.all.count).to be_between(200, 600).inclusive - end -end diff --git a/spec/rails_integration_spec.rb b/spec/rails_integration_spec.rb new file mode 100644 index 0000000..6c84e8d --- /dev/null +++ b/spec/rails_integration_spec.rb @@ -0,0 +1,112 @@ +require 'fixtures/application' +require 'fixtures/controllers' +require 'fixtures/models' +require 'fixtures/example_queries' +require 'rspec/rails' + + RSpec.describe ParentsController, type: :controller do + describe '#index' do + # url_params = { + # filter: { + # "id" => { + # "gte" => 20, + # "lt" => 40 + # } + # }, + # page: Faker::Number.between(5,15).to_s, + # per_page: Faker::Number.between(2,5).to_s, + # sort: "name:desc", + # include: "parent" + # } + # + # it "text" do + # get :index, params: url_params + # byebug + # end + + it "test example queries" do + ExampleQueries::SQL_QUERIES.each_with_index do |q, index| + + q[:expected_sorts].each do |sort| + sort_direction = ["asc","desc"].sample + + url_params = { + page: Faker::Number.between(5,15).to_s, + per_page: Faker::Number.between(2,5).to_s, + sort: "#{sort}:#{sort_direction}", + test_number: index + } + + puts "INDEX: #{index} --- QUERY: #{q[:query].squish} --- SORT: #{sort}" + + get :test, params: url_params + + rsp = JSON.parse(response.body) + data = rsp["data"] + pagination = rsp["pagination"] + + previous_value = nil + data.each_with_index do |d, index| + if index > 0 + if sort_direction == "desc" + expect(d[sort] <= previous_value).to eql(true) + else + expect(d[sort] >= previous_value).to eql(true) + end + end + previous_value = d[sort] + end + + end + + q[:expected_filters].each do |filter| + filter[:operator_codes].each do |oc| + filter_value = if filter[:class] == Integer + Faker::Number.between(0,100).to_s + elsif filter[:class] == String + case oc + when "in", "notin" + "#{q[:model].all.pluck(:name).sample},#{q[:model].all.pluck(:name).sample},#{q[:model].all.pluck(:name).sample}" + when "like" + q[:model].all.pluck(:name).sample[0..4] + end + elsif filter[:class] == TrueClass + case oc + when "null" + ['true', 'false'].sample + end + end + filter_url_param = { + filter[:alias] => { + oc => filter_value + } + } + + url_params = { + page: Faker::Number.between(5,15).to_s, + per_page: Faker::Number.between(2,5).to_s, + filter: filter_url_param, + test_number: index + } + + puts "INDEX: #{index} --- QUERY: #{q[:query].squish} --- filter: #{filter_url_param}" + + get :test, params: url_params + + # TODO: Add some sort of expectation + # Perhaps expect the result to be filtered (have less values than previously) + + end + end if q[:expected_filters] + + end + end + end +end + +# RSpec.describe 'Requests', type: :request do +# it "text" do +# get '/parents' +# byebug +# end +# end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 45556db..bcca9a1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,9 +1,10 @@ -require "bundler/setup" -require "pattern_query_helper" +require 'bundler/setup' +require 'query_helper' require 'sqlite3' require 'active_record' require 'faker' require 'byebug' +require 'fixtures/models' RSpec.configure do |config| # Enable flags like --only-failures and --next-failure @@ -36,11 +37,9 @@ } end - PatternQueryHelper.active_record_adapter = "sqlite3" - # Set up a database that resides in RAM ActiveRecord::Base.establish_connection( - adapter: PatternQueryHelper.active_record_adapter, + adapter: "sqlite3", database: ':memory:' ) @@ -48,29 +47,20 @@ ActiveRecord::Schema.define do create_table :parents, force: true do |t| t.string :name + t.integer :age end create_table :children, force: true do |t| t.string :name t.references :parent + t.integer :age end end - # Set up model classes - class ApplicationRecord < ActiveRecord::Base - self.abstract_class = true - end - class Parent < ApplicationRecord - has_many :children - end - class Child < ApplicationRecord - belongs_to :parent - end - # Load data into databases (0..99).each do - parent = Parent.create(name: Faker::Name.name) + parent = Parent.create(name: Faker::Name.name, age: Faker::Number.between(25, 55)) (0..Faker::Number.between(1, 5)).each do - Child.create(name: Faker::Name.name, parent: parent) + Child.create(name: Faker::Name.name, parent: parent, age: Faker::Number.between(1, 25)) end end end