The big old postgres refactor

Gemfile

@@ -29,6 +29,7 @@ gem 'jbuilder', '~> 2.9'
 gem 'sdoc', '~> 2.1', group: :doc
 
 gem 'foreman', '~> 0.87'
+gem 'pg', '~> 0.18'
 
 group :production do
   gem 'thin', '~> 1.8'
@@ -53,8 +54,7 @@ group :development, :test do
   gem 'poltergeist', '~> 1.6'
   gem 'vcr', '~> 6.0'
   gem 'cucumber-api-steps', require: false, git: 'https://github.com/Data-Liberation-Front/cucumber-api-steps.git', branch: 'feature-test-content-type'
-  # Use sqlite3 as the database for Active Record
-  gem 'sqlite3', '~> 1.4'
+  gem 'sqlite3', '~> 1.3.0'
 end
 
 group :development do
@@ -77,7 +77,6 @@ gem 'rack-google-analytics', '~> 1.2'
 gem 'mongo', '~> 2.1', git: 'https://github.com/Data-Liberation-Front/mongo-ruby-driver.git', ref: "212-with-authsource-fix"
 gem 'mongoid', '~> 5.1'
 gem 'kaminari-mongoid', '~> 1.0'
-gem 'mongoid-grid_fs', '~> 2.4'
 gem 'kaminari', '~> 1.2'
 gem 'bootstrap-kaminari-views', '~> 0.0'
 gem 'data_kitten', git: 'https://github.com/Data-Liberation-Front/data_kitten.git', ref: "e343510bd15e3329c1f2fab35035e248195348be", require: false

Gemfile.lock

@@ -504,9 +504,6 @@ GEM
       mongo (~> 2.1)
       origin (~> 2.2)
       tzinfo (>= 0.3.37)
-    mongoid-grid_fs (2.4.0)
-      mime-types (>= 1.0, < 4.0)
-      mongoid (>= 3.0, < 8.0)
     multi_json (1.15.0)
     multi_test (0.1.2)
     multipart-post (2.1.1)
@@ -525,6 +522,7 @@ GEM
     open_uri_redirections (0.2.1)
     optimist (3.0.1)
     origin (2.3.1)
+    pg (0.21.0)
     poltergeist (1.6.0)
       capybara (~> 2.1)
       cliver (~> 0.3.1)
@@ -729,7 +727,7 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    sqlite3 (1.4.2)
+    sqlite3 (1.3.13)
     sxp (1.1.0)
       rdf (~> 3.1)
     sync (0.5.0)
@@ -823,8 +821,8 @@ DEPENDENCIES
   kaminari-mongoid (~> 1.0)
   mongo (~> 2.1)!
   mongoid (~> 5.1)
-  mongoid-grid_fs (~> 2.4)
   nokogiri (~> 1.10)
+  pg (~> 0.18)
   poltergeist (~> 1.6)
   pry (~> 0.14)
   puma (~> 5.2)
@@ -843,7 +841,7 @@ DEPENDENCIES
   sidekiq (~> 4.2)
   simplecov (~> 0.16)
   spring (~> 2.1)
-  sqlite3 (~> 1.4)
+  sqlite3 (~> 1.3.0)
   therubyracer (~> 0.12)
   thin (~> 1.8)
   timecop (~> 0.9)

app/controllers/application_controller.rb

@@ -8,10 +8,10 @@ def standard_csv_options
 
   def index
     if params[:uri]
-      validator = Validation.where(:url => params[:uri]).first
+      validator = Legacy::Validation.where(:url => params[:uri]).first
       if validator.nil?
 
-        Validation.delay.create_validation(params[:uri])
+        Legacy::Validation.delay.create_validation(params[:uri])
         respond_to do |wants|
           wants.html { render status: 202 }
           wants.json { render status: 202 }

app/controllers/package_controller.rb

@@ -6,11 +6,11 @@ class PackageController < ApplicationController
   before_filter(:only => [:show]) { alternate_formats [:json] }
 
   def create
-    @package = Package.create
+    @package = Legacy::Package.create
     if params[:format] == "json"
-      Package.delay.process(@package.id, params)
+      Legacy::Package.delay.process(@package.id, params)
     else
-      Package.process(@package.id, params)
+      Legacy::Package.process(@package.id, params)
       if @package.validations.count == 1
         redirect_to validation_path(@package.validations.first)
       else
@@ -20,7 +20,7 @@ def create
   end
 
   def show
-    @package = Package.find( params[:id] )
+    @package = Legacy::Package.find( params[:id] )
 
     if @package.validations.count == 1 && params[:format] != "json"
       redirect_to validation_path(@package.validations.first)

app/controllers/schemas_controller.rb

@@ -6,17 +6,17 @@ class SchemasController < ApplicationController
 
   def index
     if params[:uri]
-      schema = Schema.where(url: params[:uri]).first
+      schema = Legacy::Schema.where(url: params[:uri]).first
       render status: 404 and return if schema.nil?
       redirect_to schema, status: 303
     else
-      schemas = Schema.all
+      schemas = Legacy::Schema.all
       @schemas = Kaminari.paginate_array(schemas).page(params[:page])
     end
   end
 
   def show
-    @db_schema = Schema.where(id: params[:id]).first
+    @db_schema = Legacy::Schema.where(id: params[:id]).first
     @schema = Csvlint::Schema.load_from_uri(@db_schema.url)
     respond_to do |wants|
       wants.html

app/controllers/summary_controller.rb

@@ -1,9 +1,9 @@
 class SummaryController < ApplicationController
-  
+
   before_filter(:only => [:index]) { alternate_formats [:json] }
-    
+
   def index
-    @summary = Summary.last
+    @summary = Legacy::Summary.last
   end
-  
-end
+
+end

app/controllers/validation_controller.rb

@@ -6,13 +6,13 @@ class ValidationController < ApplicationController
   before_filter(:only => [:show]) { alternate_formats [:json] }
 
   def show
-    @validation = Validation.fetch_validation(params[:id], params[:format], params[:revalidate])
+    @validation = Legacy::Validation.fetch_validation(params[:id], params[:format], params[:revalidate])
 
     raise ActionController::RoutingError.new('Not Found') if @validation.state.nil?
     # @result stores all the validation errors, warnings and information messages
     @result = @validation.validator
     # byebug
-    @dialect = @result.dialect || Validation.standard_dialect
+    @dialect = @result.dialect || Legacy::Validation.standard_dialect
     # Responses
     respond_to do |wants|
       wants.html
@@ -27,7 +27,7 @@ def update
     # this method is triggered when user revalidates a schema
     dialect = build_dialect(params)
     # build a fresh dialect for comparing a file against
-    v = Validation.find(params[:id])
+    v = Legacy::Validation.find(params[:id])
     if v.has_attribute?(:expirable_created_at)
       # has expirable_created_at value
       v.update_validation(dialect, expiry=true)

app/models/legacy.rb

@@ -0,0 +1,5 @@
+module Legacy
+  def self.use_relative_model_naming?
+    true
+  end
+end

app/models/package.rb → app/models/legacy/package.rb

@@ -36,15 +36,16 @@ def detect_publishing_format
   end
 end
 
-class Package
+class Legacy::Package
   include Mongoid::Document
+  store_in collection: "packages"
   include Mongoid::Timestamps
 
   field :url, type: String
   field :dataset, type: String
   field :type, type: String
 
-  has_many :validations
+  has_many :validations, class_name: 'Legacy::Validation'
 
   def parse_package(dataset, validations)
     attributes = {
@@ -72,7 +73,7 @@ def create_package(sources, schema_url = nil, schema = nil)
     update_attributes({ type: set_type(sources) })
 
     sources.each do |source|
-      validations << Validation.create_validation(source, schema_url, schema)
+      validations << Legacy::Validation.create_validation(source, schema_url, schema)
     end
 
     save
@@ -103,7 +104,7 @@ def create_validations(dataset)
     validations = []
     dataset.distributions.each do |distribution|
       if can_validate?(distribution)
-        validations << Validation.create_validation(distribution.access_url, nil, create_schema(distribution) )
+        validations << Legacy::Validation.create_validation(distribution.access_url, nil, create_schema(distribution) )
       end
     end
    validations

app/models/schema.rb → app/models/legacy/schema.rb

@@ -1,14 +1,14 @@
-class Schema
+class Legacy::Schema
   include Mongoid::Document
+  store_in collection: "schemas"
   include Mongoid::Timestamps
-  
+
   field :url, type: String
 
-  has_many :validations
+  has_many :validations, class_name: 'Legacy::Validation'
 
   def to_param
     id.to_s
   end
 
 end
-

app/models/summary.rb → app/models/legacy/summary.rb

@@ -14,8 +14,9 @@ class CategorySummary
   field :context_breakdown, type: Hash
 end
 
-class Summary
+class Legacy::Summary
   include Mongoid::Document
+  store_in collection: "summaries"
   include Mongoid::Timestamps
 
   field :sources, type: Integer
@@ -26,9 +27,9 @@ class Summary
   embeds_one :category_summary
 
   def self.generate
-    summary = Summary.create
+    summary = Legacy::Summary.create
 
-    validations = Validation.where(:url.ne => nil).order_by(:created_at.desc)
+    validations = Legacy::Validation.where(:url.ne => nil).order_by(:created_at.desc)
     # retrieve validations from Mongo Datastore, ordered in reverse by date created
 
     summary.sources = validations.length

app/models/validation.rb → app/models/legacy/validation.rb

@@ -1,5 +1,6 @@
-class Validation
+class Legacy::Validation
   include Mongoid::Document
+  store_in collection: "validations"
   include Mongoid::Timestamps
 
   field :filename, type: String
@@ -15,10 +16,10 @@ class Validation
   # invoke the mongo time-to-live feature which will automatically expire entries
   # - this index is only enabled for a subset of validations, which are validations uploaded as file
 
-  belongs_to :schema
-  accepts_nested_attributes_for :schema
+  belongs_to :schema, class_name: 'Legacy::Schema'
+  accepts_nested_attributes_for :schema, class_name: 'Legacy::Schema'
 
-  belongs_to :package
+  belongs_to :package, class_name: 'Legacy::Package'
 
   def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry)
     if io.respond_to?(:tempfile)
@@ -71,7 +72,7 @@ def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry)
       :filename => filename,
       :state => state,
       :result => Marshal.dump(validator).force_encoding("UTF-8"),
-      :parse_options => Validation.generate_options(validator.dialect)
+      :parse_options => Legacy::Validation.generate_options(validator.dialect)
     }
 
     attributes[:expirable_created_at] = Time.now if expiry.eql?(true)
@@ -82,7 +83,7 @@ def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry)
 
     if schema_url.present?
       # Find matching schema if possible and retrieve
-      schema = Schema.where(url: schema_url).first
+      schema = Legacy::Schema.where(url: schema_url).first
       attributes[:schema] = schema || { :url => schema_url }
     end
     # byebug
@@ -98,7 +99,7 @@ def self.fetch_validation(id, format, revalidate = nil)
     unless revalidate.to_s == "false"
       if ["png", "svg"].include?(format)
         # suspect the above functions tied to the use of badges as hyperlinks to valid schemas & csvs
-        Validation.delay.check_validation(v.id)
+        Legacy::Validation.delay.check_validation(v.id)
       else
         v.check_validation
       end
@@ -156,10 +157,10 @@ def self.create_validation(io, schema_url = nil, schema = nil)
     # this method invokes the validate method below rather than self.validate
     # returns validation object
     if io.class == String
-      validation = Validation.find_or_initialize_by(url: io)
+      validation = Legacy::Validation.find_or_initialize_by(url: io)
       expiry = false
     else
-      validation = Validation.create
+      validation = Legacy::Validation.create
       expiry = true
     end
     validation.validate(io, schema_url, schema, expiry)
@@ -168,15 +169,15 @@ def self.create_validation(io, schema_url = nil, schema = nil)
   end
 
   def validate(io, schema_url = nil, schema = nil, expiry)
-    validation = Validation.validate(io, schema_url, schema, nil, expiry)
+    validation = Legacy::Validation.validate(io, schema_url, schema, nil, expiry)
     self.update_attributes(validation)
     # update_attributes is a method from Mongoid
   end
 
   def update_validation(dialect = nil, expiry=nil)
     loaded_schema = schema ? Csvlint::Schema.load_from_uri(schema.url) : nil
     io = self.url.nil? ? StoredCSV.fetch(self.filename) : self.url
-    validation = Validation.validate(io, schema.try(:url), loaded_schema, dialect, expiry)
+    validation = Legacy::Validation.validate(io, schema.try(:url), loaded_schema, dialect, expiry)
     self.update_attributes(validation)
     # update mongoDB record
     self
@@ -202,14 +203,14 @@ def csv
 
   def parse_options
     if self.read_attribute(:parse_options).nil?
-      self.parse_options = Validation.generate_options(self.validator.dialect)
+      self.parse_options = Legacy::Validation.generate_options(self.validator.dialect)
       save
     end
     self.read_attribute(:parse_options)
   end
 
   def self.check_validation(id)
-    Validation.find(id).check_validation
+    Legacy::Validation.find(id).check_validation
   end
 
   def check_validation
@@ -236,7 +237,7 @@ def badge
   end
 
   def self.clean_up(hours)
-    delete_validations Validation.where(:created_at.lte => hours.hours.ago, :url => nil)
+    delete_validations Legacy::Validation.where(:created_at.lte => hours.hours.ago, :url => nil)
   rescue => e
     Airbrake.notify(e) if ENV['CSVLINT_AIRBRAKE_KEY'] # Exit cleanly, but still notify airbrake
   end

app/views/summary/index.json.jbuilder

@@ -1,8 +1,8 @@
 json.created @summary.created_at
 json.sources @summary.sources
 json.date_range do
-  json.from Validation.first.created_at
-  json.to Validation.last.created_at
+  json.from Legacy::Validation.first.created_at
+  json.to Legacy::Validation.last.created_at
 end
 json.results do
   @summary.states.each do |state, count|
@@ -15,21 +15,20 @@ json.hosts do
   end
 end
 json.severity do
-  [:errors, :warnings, :info_messages].each do |level| 
+  [:errors, :warnings, :info_messages].each do |level|
     json.set! level do
       @summary.level_summary.send("#{level}_breakdown").sort_by{|k,v| v}.reverse.each_with_index do |(type,count), index|
-        json.set! type, count        
+        json.set! type, count
       end
     end
   end
 end
 json.category do
-  [:structure, :schema, :context].each do |category| 
+  [:structure, :schema, :context].each do |category|
     json.set! category do
       @summary.category_summary.send("#{category}_breakdown").sort_by{|k,v| v}.reverse.each_with_index do |(type,count), index|
-        json.set! type, count        
+        json.set! type, count
       end
     end
   end
 end
-