diff --git a/.circleci/config.yml b/.circleci/config.yml index f81f3d27..d0fef7e8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,7 +91,7 @@ jobs: command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Database setup - command: bundle exec rake db:setup + command: RAILS_ENV=test bundle exec rake db:migrate - ruby/install-deps - ruby/rspec-test: include: spec/**/*_spec.rb diff --git a/app/models/library_staff.rb b/app/models/library_staff.rb index 3fdcf471..f200953b 100644 --- a/app/models/library_staff.rb +++ b/app/models/library_staff.rb @@ -9,7 +9,7 @@ class LibraryStaff attr_reader :query_terms - def initialize(query_terms:, rom: Rails.application.config.rom) + def initialize(query_terms:, rom: ALLSEARCH_ROM) @query_terms = query_terms @rom = rom end diff --git a/app/repositories/repository_factory.rb b/app/repositories/repository_factory.rb index 3412524e..251cb325 100644 --- a/app/repositories/repository_factory.rb +++ b/app/repositories/repository_factory.rb @@ -5,7 +5,7 @@ class RepositoryFactory define_singleton_method method_name do return instance_variable_get("@#{method_name}") if instance_variable_defined?("@#{method_name}") - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! instance_variable_set("@#{method_name}", new(rom).send(method_name)) end end diff --git a/app/services/csv_loading_service.rb b/app/services/csv_loading_service.rb index e27ed776..0fb18621 100644 --- a/app/services/csv_loading_service.rb +++ b/app/services/csv_loading_service.rb @@ -12,7 +12,7 @@ class CSVLoadingService # rom_container: ALLSEARCH_ROM and remove the offending line def initialize(logger: ALLSEARCH_LOGGER, rom_container: nil) @logger = logger - @rom_container = rom_container || Rails.application.config&.rom || RomFactory.new.require_rom! + @rom_container = rom_container || rails_rom_container || RomFactory.new.require_rom! end def run @@ -24,6 +24,13 @@ def run attr_reader :csv, :logger, :rom_container + # :reek:UtilityFunction + def rails_rom_container + Rails.application.config.rom + rescue NoMethodError + nil + end + def fetch_data contents = uri.open @csv = CSV.new(contents) diff --git a/config/db_connection.rb b/config/db_connection.rb index 9f3740a2..2a8a50bb 100644 --- a/config/db_connection.rb +++ b/config/db_connection.rb @@ -2,13 +2,14 @@ require_relative 'allsearch_configs' require 'erb' -require 'sequel' require 'yaml' +require 'sequel' begin db_config = ALLSEARCH_CONFIGS[:database] + DB = Sequel.postgres(db_config[:database], user: db_config[:username], password: db_config[:password], host: db_config[:host], port: db_config[:port]) rescue StandardError - nil + DB = nil end diff --git a/config/initializers/rom_setup.rb b/config/initializers/rom_setup.rb index fa05ea8b..9f24950f 100644 --- a/config/initializers/rom_setup.rb +++ b/config/initializers/rom_setup.rb @@ -6,7 +6,6 @@ # This initializer is responsible for making a ROM container available # to the application -# Initialize rom config attribute to ensure it always exists Rails.application.config.rom = nil result = RomFactory.new.rom_if_available diff --git a/db/migrate/20230918174143_create_best_bet_documents.rb b/db/old_rails_migrate/20230918174143_create_best_bet_documents.rb similarity index 100% rename from db/migrate/20230918174143_create_best_bet_documents.rb rename to db/old_rails_migrate/20230918174143_create_best_bet_documents.rb diff --git a/db/migrate/20230918230048_create_oauth_tokens.rb b/db/old_rails_migrate/20230918230048_create_oauth_tokens.rb similarity index 100% rename from db/migrate/20230918230048_create_oauth_tokens.rb rename to db/old_rails_migrate/20230918230048_create_oauth_tokens.rb diff --git a/db/migrate/20230919194126_rename_best_best_documents_to_best_bet_records.rb b/db/old_rails_migrate/20230919194126_rename_best_best_documents_to_best_bet_records.rb similarity index 100% rename from db/migrate/20230919194126_rename_best_best_documents_to_best_bet_records.rb rename to db/old_rails_migrate/20230919194126_rename_best_best_documents_to_best_bet_records.rb diff --git a/db/migrate/20230919194551_remove_not_null_constraints_from_oauth_token.rb b/db/old_rails_migrate/20230919194551_remove_not_null_constraints_from_oauth_token.rb similarity index 100% rename from db/migrate/20230919194551_remove_not_null_constraints_from_oauth_token.rb rename to db/old_rails_migrate/20230919194551_remove_not_null_constraints_from_oauth_token.rb diff --git a/db/migrate/20230920214343_create_library_databases.rb b/db/old_rails_migrate/20230920214343_create_library_databases.rb similarity index 100% rename from db/migrate/20230920214343_create_library_databases.rb rename to db/old_rails_migrate/20230920214343_create_library_databases.rb diff --git a/db/migrate/20230921193904_add_weights_to_searchable.rb b/db/old_rails_migrate/20230921193904_add_weights_to_searchable.rb similarity index 100% rename from db/migrate/20230921193904_add_weights_to_searchable.rb rename to db/old_rails_migrate/20230921193904_add_weights_to_searchable.rb diff --git a/db/migrate/20231120153410_create_library_staff_documents.rb b/db/old_rails_migrate/20231120153410_create_library_staff_documents.rb similarity index 100% rename from db/migrate/20231120153410_create_library_staff_documents.rb rename to db/old_rails_migrate/20231120153410_create_library_staff_documents.rb diff --git a/db/migrate/20231121205208_staff_search_indexing.rb b/db/old_rails_migrate/20231121205208_staff_search_indexing.rb similarity index 100% rename from db/migrate/20231121205208_staff_search_indexing.rb rename to db/old_rails_migrate/20231121205208_staff_search_indexing.rb diff --git a/db/migrate/20240710210913_remove_not_null_constraint_from_staff_department.rb b/db/old_rails_migrate/20240710210913_remove_not_null_constraint_from_staff_department.rb similarity index 100% rename from db/migrate/20240710210913_remove_not_null_constraint_from_staff_department.rb rename to db/old_rails_migrate/20240710210913_remove_not_null_constraint_from_staff_department.rb diff --git a/db/migrate/20240716210532_add_staff_records_fields.rb b/db/old_rails_migrate/20240716210532_add_staff_records_fields.rb similarity index 100% rename from db/migrate/20240716210532_add_staff_records_fields.rb rename to db/old_rails_migrate/20240716210532_add_staff_records_fields.rb diff --git a/db/migrate/20240716214838_update_searchable_library_staff_records.rb b/db/old_rails_migrate/20240716214838_update_searchable_library_staff_records.rb similarity index 100% rename from db/migrate/20240716214838_update_searchable_library_staff_records.rb rename to db/old_rails_migrate/20240716214838_update_searchable_library_staff_records.rb diff --git a/db/migrate/20240717164314_create_flipper_tables.rb b/db/old_rails_migrate/20240717164314_create_flipper_tables.rb similarity index 100% rename from db/migrate/20240717164314_create_flipper_tables.rb rename to db/old_rails_migrate/20240717164314_create_flipper_tables.rb diff --git a/db/migrate/20240717173433_create_banners.rb b/db/old_rails_migrate/20240717173433_create_banners.rb similarity index 100% rename from db/migrate/20240717173433_create_banners.rb rename to db/old_rails_migrate/20240717173433_create_banners.rb diff --git a/db/migrate/20240813174823_add_pronouns_to_library_staff_record.rb b/db/old_rails_migrate/20240813174823_add_pronouns_to_library_staff_record.rb similarity index 100% rename from db/migrate/20240813174823_add_pronouns_to_library_staff_record.rb rename to db/old_rails_migrate/20240813174823_add_pronouns_to_library_staff_record.rb diff --git a/db/migrate/20240815164656_enable_unaccent_extension.rb b/db/old_rails_migrate/20240815164656_enable_unaccent_extension.rb similarity index 100% rename from db/migrate/20240815164656_enable_unaccent_extension.rb rename to db/old_rails_migrate/20240815164656_enable_unaccent_extension.rb diff --git a/db/migrate/20240819142401_add_unaccented_dictionary.rb b/db/old_rails_migrate/20240819142401_add_unaccented_dictionary.rb similarity index 100% rename from db/migrate/20240819142401_add_unaccented_dictionary.rb rename to db/old_rails_migrate/20240819142401_add_unaccented_dictionary.rb diff --git a/db/migrate/20240819150118_use_unaccented_dict_library_db.rb b/db/old_rails_migrate/20240819150118_use_unaccented_dict_library_db.rb similarity index 100% rename from db/migrate/20240819150118_use_unaccented_dict_library_db.rb rename to db/old_rails_migrate/20240819150118_use_unaccented_dict_library_db.rb diff --git a/db/migrate/20240820184147_add_unaccented_simple_dictionary.rb b/db/old_rails_migrate/20240820184147_add_unaccented_simple_dictionary.rb similarity index 100% rename from db/migrate/20240820184147_add_unaccented_simple_dictionary.rb rename to db/old_rails_migrate/20240820184147_add_unaccented_simple_dictionary.rb diff --git a/db/migrate/20240820185228_use_unaccented_simple_dict_library_staff.rb b/db/old_rails_migrate/20240820185228_use_unaccented_simple_dict_library_staff.rb similarity index 100% rename from db/migrate/20240820185228_use_unaccented_simple_dict_library_staff.rb rename to db/old_rails_migrate/20240820185228_use_unaccented_simple_dict_library_staff.rb diff --git a/db/migrate/20250627142651_rank_searchable_staff.rb b/db/old_rails_migrate/20250627142651_rank_searchable_staff.rb similarity index 100% rename from db/migrate/20250627142651_rank_searchable_staff.rb rename to db/old_rails_migrate/20250627142651_rank_searchable_staff.rb diff --git a/db/rom_migrate/20251212143343_create_best_bet_records.rb b/db/rom_migrate/20251212143343_create_best_bet_records.rb new file mode 100644 index 00000000..10cde97b --- /dev/null +++ b/db/rom_migrate/20251212143343_create_best_bet_records.rb @@ -0,0 +1,15 @@ + +Sequel.migration do + change do + create_table(:best_bet_records) do + primary_key :id + String :title + String :description + String :url + column :search_terms, 'text[]' + Date :last_update + DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + end + end +end diff --git a/db/rom_migrate/20251212150000_create_oauth_tokens.rb b/db/rom_migrate/20251212150000_create_oauth_tokens.rb new file mode 100644 index 00000000..0d67f716 --- /dev/null +++ b/db/rom_migrate/20251212150000_create_oauth_tokens.rb @@ -0,0 +1,16 @@ + +Sequel.migration do + change do + create_table(:oauth_tokens) do + primary_key :id + String :service, null: false + String :endpoint, null: false + String :token, null: true + DateTime :expiration_time, null: true + DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + end + add_index :oauth_tokens, :service, unique: true + add_index :oauth_tokens, :endpoint, unique: true + end +end diff --git a/db/rom_migrate/20251212150500_create_banners.rb b/db/rom_migrate/20251212150500_create_banners.rb new file mode 100644 index 00000000..d1fe2d87 --- /dev/null +++ b/db/rom_migrate/20251212150500_create_banners.rb @@ -0,0 +1,15 @@ + +Sequel.migration do + change do + create_table(:banners) do + primary_key :id + String :text, text: true, default: '' + Boolean :display_banner, default: false + Integer :alert_status, default: 1 + Boolean :dismissible, default: true + Boolean :autoclear, default: false + DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + end + end +end diff --git a/db/rom_migrate/20251212161222_create_library_database_records.rb b/db/rom_migrate/20251212161222_create_library_database_records.rb new file mode 100644 index 00000000..58448931 --- /dev/null +++ b/db/rom_migrate/20251212161222_create_library_database_records.rb @@ -0,0 +1,29 @@ + +Sequel.migration do + change do + create_table(:library_database_records) do + primary_key :id + bigint :libguides_id, null: false + String :name, null: false + String :description + column :alt_names, 'text[]' + String :alt_names_concat + String :url + String :friendly_url + column :subjects, 'text[]' + String :subjects_concat + DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + + tsvector :searchable, + generated_always_as: Sequel.lit( + "setweight(to_tsvector('english', coalesce(name,'')), 'A') || " \ + "setweight(to_tsvector('english', coalesce(alt_names_concat,'')), 'B') || " \ + "setweight(to_tsvector('english', coalesce(description,'')), 'C') || " \ + "setweight(to_tsvector('english', coalesce(subjects_concat,'')), 'D') ", + ) + end + add_index :library_database_records, :searchable, + using: :gin, name: 'searchable_idx' + end +end diff --git a/db/rom_migrate/20251215120000_create_library_staff_records.rb b/db/rom_migrate/20251215120000_create_library_staff_records.rb new file mode 100644 index 00000000..f79eec2b --- /dev/null +++ b/db/rom_migrate/20251215120000_create_library_staff_records.rb @@ -0,0 +1,62 @@ + +Sequel.migration do + change do + execute "CREATE ExTENSION IF NOT EXISTS unaccent;" + execute <<~SQL + CREATE TEXT SEARCH CONFIGURATION unaccented_dict ( COPY = english ); + ALTER TEXT SEARCH CONFIGURATION unaccented_dict ALTER MAPPING FOR hword, hword_part, word WITH unaccent, simple; + SQL + execute <<~SQL + CREATE TEXT SEARCH CONFIGURATION unaccented_simple_dict ( COPY = simple ); + ALTER TEXT SEARCH CONFIGURATION unaccented_simple_dict ALTER MAPPING FOR hword, hword_part, word WITH unaccent, simple; + SQL + create_table(:library_staff_records) do + primary_key :id + bigint :puid, null: false + String :netid, null: false + String :phone + String :name, null: false + String :last_name + String :first_name + String :middle_name + String :title, null: false + String :library_title, null: false + String :email, null: false + String :team + String :division + String :department + String :unit + String :office + String :building + DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + String :areas_of_study + String :other_entities + String :my_scheduler_link + String :pronouns + + tsvector :name_searchable, + generated_always_as: Sequel.function( + :to_tsvector, + 'public.unaccented_simple_dict', + Sequel.lit("coalesce(name, '') || ' ' || coalesce(first_name, '') || ' ' || coalesce(middle_name, '') || ' ' || coalesce(last_name, '')") + ) + + String :bio + + tsvector :searchable, + generated_always_as: Sequel.function( + :to_tsvector, + 'public.unaccented_dict', + Sequel.lit("coalesce(title, '') || ' ' || coalesce(email, '') || ' ' || coalesce(department, '') || ' ' || coalesce(office, '') || ' ' || coalesce(building, '') || ' ' || coalesce(team, '') || ' ' || coalesce(division, '') || ' ' || coalesce(unit, '') || ' ' || coalesce(areas_of_study, '') || ' ' || coalesce(bio, '') || ' ' || coalesce(other_entities, '')") + ) + + end + + add_index :library_staff_records, :name_searchable, + using: :gin, name: 'staff_name_search_idx' + + add_index :library_staff_records, :searchable, + using: :gin, name: 'staff_search_idx' + end +end diff --git a/db/rom_migrate/20251215121000_create_flipper_tables.rb b/db/rom_migrate/20251215121000_create_flipper_tables.rb new file mode 100644 index 00000000..cc6bac1d --- /dev/null +++ b/db/rom_migrate/20251215121000_create_flipper_tables.rb @@ -0,0 +1,24 @@ + +Sequel.migration do + change do + create_table(:flipper_features) do + primary_key :id + String :key, null: false + DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + end + + add_index :flipper_features, :key, unique: true + + create_table(:flipper_gates) do + primary_key :id + String :feature_key, null: false + String :key, null: false + column :value, 'text' + DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + end + + add_index :flipper_gates, [:feature_key, :key, :value], unique: true, name: 'flipper_gates_feature_key_key_value_idx' + end +end diff --git a/init/rom_factory.rb b/init/rom_factory.rb index 5cfd3671..ea1a32da 100644 --- a/init/rom_factory.rb +++ b/init/rom_factory.rb @@ -4,6 +4,7 @@ require 'rom-sql' require 'dry-monads' require_relative 'environment' +require_relative '../config/db_connection' require allsearch_path 'init/logger' require allsearch_path 'app/relations/banner_relation' require allsearch_path 'app/relations/best_bet_relation' @@ -29,13 +30,16 @@ def database_if_available def rom_if_available db_connection - .bind { |connection| verify_database_ready(connection) } + .bind { |connection| verify_db_connection_is_ready(connection) } + .bind { |connection| verify_required_rom_tables(connection) } .bind { |connection| rom_container(connection) } end private def db_connection + return Success(DB) if defined?(DB) && DB + db_config = ALLSEARCH_CONFIGS[:database] Success(Sequel.postgres(db_config[:database], user: db_config[:username], password: db_config[:password], host: db_config[:host], port: db_config[:port])) @@ -43,16 +47,35 @@ def db_connection Failure(error) end + def rom_container(db_connection) + rom_config = ROM::Configuration.new(:sql, db_connection) + rom_config.register_relation BannerRelation + rom_config.register_relation BestBetRelation + rom_config.register_relation LibraryDatabaseRelation + rom_config.register_relation LibraryStaffRelation + rom_config.register_relation OAuthTokenRelation + rom_config.default.use_logger ALLSEARCH_LOGGER + Success(ROM.container(rom_config)) + end + # rubocop:disable Metrics/MethodLength - def verify_database_ready(connection) - required_tables = [:schema_migrations, :ar_internal_metadata] + def verify_required_rom_tables(connection) + # Check forschema_migrations table (created by ROM) + unless connection.table_exists?(:schema_migrations) + return Failure(StandardError.new( + 'ROM migration tracking table does not exist. Run: bundle exec rake db:migrate' + )) + end + + # Check for required tables that ROM relations depend on + required_tables = [:best_bet_records, :oauth_tokens, :library_database_records, :library_staff_records, :banners] missing_tables = required_tables.reject { |table| connection.table_exists?(table) } if missing_tables.empty? Success(connection) else Failure(StandardError.new( - "Database is missing tables: #{missing_tables.join(', ')}. Please run migrations/load the db structure." + "Required tables missing: #{missing_tables.join(', ')}. Run: bundle exec rake db:migrate" )) end rescue StandardError => error @@ -60,14 +83,9 @@ def verify_database_ready(connection) end # rubocop:enable Metrics/MethodLength - def rom_container(db_connection) - rom_config = ROM::Configuration.new(:sql, db_connection) - rom_config.register_relation BannerRelation - rom_config.register_relation BestBetRelation - rom_config.register_relation LibraryDatabaseRelation - rom_config.register_relation LibraryStaffRelation - rom_config.register_relation OAuthTokenRelation - rom_config.default.use_logger ALLSEARCH_LOGGER - Success(ROM.container(rom_config)) + def verify_db_connection_is_ready(connection) + Success(connection) + rescue StandardError => error + Failure(error) end end diff --git a/lib/capistrano/tasks/rom_migrate.rake b/lib/capistrano/tasks/rom_migrate.rake new file mode 100644 index 00000000..6f5c1749 --- /dev/null +++ b/lib/capistrano/tasks/rom_migrate.rake @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Override default Rails migrate step to run ROM/Sequel migrations instead +namespace :deploy do + Rake::Task['deploy:migrate'].clear if Rake::Task.task_defined?('deploy:migrate') + + desc 'Run ROM/Sequel migrations' + task :migrate do + on roles(:db) do + within release_path do + with rails_env: fetch(:rails_env) do + execute :bundle, :exec, :rake, 'db:migrate_to_rom' + end + end + end + end +end diff --git a/lib/tasks/fix_sequences.rake b/lib/tasks/fix_sequences.rake new file mode 100644 index 00000000..2ab18b21 --- /dev/null +++ b/lib/tasks/fix_sequences.rake @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +namespace :rom do + desc 'Fix missing sequences for primary keys' + task :fix_sequences do + require_relative '../../init/rom_factory' + + conn = RomFactory.new.database_if_available.value! + + tables = [:best_bet_records, :oauth_tokens, :library_databases, :library_staff_documents, :banners] + + tables.each do |table| + next unless conn.table_exists?(table) + + sequence_name = "#{table}_id_seq" + + conn.run("CREATE SEQUENCE IF NOT EXISTS #{sequence_name}") + + conn.run("ALTER TABLE #{table} ALTER COLUMN id SET DEFAULT nextval('#{sequence_name}')") + + conn.run("SELECT setval('#{sequence_name}', (SELECT COALESCE(MAX(id), 0) + 1 FROM #{table}))") + + puts "Fixed sequence for #{table}" + end + + puts 'All sequences fixed.' + end +end diff --git a/lib/tasks/migrate_to_rom.rake b/lib/tasks/migrate_to_rom.rake new file mode 100644 index 00000000..5bc44091 --- /dev/null +++ b/lib/tasks/migrate_to_rom.rake @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +namespace :db do + desc 'One-time task: Convert Rails schema_migrations to ROM format' + task :migrate_to_rom do + require_relative '../../init/rom_factory' + + conn = RomFactory.new.database_if_available.value! + + # Check if version column is string type (Rails format) + column_type = conn.schema(:schema_migrations).find { |col| col[0] == :version }&.dig(1, :db_type) + + if column_type == 'character varying' + puts 'Converting schema_migrations from Rails format to ROM format...' + + # Truncate table + conn.run('TRUNCATE TABLE schema_migrations') + + # Change column type from varchar to bigint with explicit casting + conn.run('ALTER TABLE schema_migrations ALTER COLUMN version TYPE bigint USING version::bigint') + + puts 'Populating ROM migrations...' + Rake::Task['db:migrations_applied'].invoke + + puts 'Migration to ROM complete!' + else + puts 'Already using ROM format, running normal migrations...' + Rake::Task['db:migrate'].invoke + end + end +end diff --git a/lib/tasks/rom_migrations.rake b/lib/tasks/rom_migrations.rake new file mode 100644 index 00000000..dc34701b --- /dev/null +++ b/lib/tasks/rom_migrations.rake @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +# rubocop:disable Metrics/BlockLength +namespace :db do + # rubocop:enable Metrics/BlockLength + + # Clear Rails default db:migrate task if it exists + Rake::Task['db:migrate'].clear if Rake::Task.task_defined?('db:migrate') + + desc 'Run ROM/Sequel migrations' + task :migrate do + require_relative '../../init/rom_factory' + + conn = RomFactory.new.database_if_available.value! + Sequel.extension :migration + + unless conn.table_exists?(:schema_migrations) + conn.create_table(:schema_migrations) do + column :version, :bigint, null: false, primary_key: true + end + end + + migration_dir = File.join(Dir.pwd, 'db', 'rom_migrate') + applied = conn[:schema_migrations].select_map(:version) + + Dir.glob(File.join(migration_dir, '*.rb')).each do |file| + version = File.basename(file).split('_').first.to_i + + next if applied.include?(version) + + puts "Applying migration: #{File.basename(file)}" + load file + migration = Sequel::Migration.descendants.last + migration.apply(conn, :up) + Sequel::Migration.descendants.pop + conn[:schema_migrations].insert(version: version) + end + + puts 'ROM/Sequel migrations run.' + end + + desc 'Rollback ROM/Sequel migrations to a target (provide TARGET=timestamp)' + task :rollback do + require_relative '../../init/rom_factory' + + target = ENV['TARGET']&.to_i + unless target&.positive? + puts 'Please provide TARGET=timestamp to rollback to (e.g. TARGET=20251211000001)' + exit 1 + end + + conn = RomFactory.new.database_if_available.value! + Sequel.extension :migration + Sequel::Migrator.run(conn, File.join(Dir.pwd, 'db', 'rom_migrate'), + target: target, + table: :schema_migrations, + column: :version) + puts "Rolled back ROM/Sequel migrations to #{target}." + end + + desc 'Reset migration tracking table (clears all migration history)' + task :reset_tracking do + require_relative '../../init/rom_factory' + + conn = RomFactory.new.database_if_available.value! + + if conn.table_exists?(:schema_migrations) + conn[:schema_migrations].delete + puts 'Cleared schema_migrations table' + else + puts 'schema_migrations table does not exist' + end + end + + desc 'Mark existing migrations as applied without running them (first time setup)' + task :migrations_applied do + require_relative '../../init/rom_factory' + + conn = RomFactory.new.database_if_available.value! + Sequel.extension :migration + + unless conn.table_exists?(:schema_migrations) + conn.create_table(:schema_migrations) do + column :version, :bigint, null: false, primary_key: true + end + end + + migration_dir = File.join(Dir.pwd, 'db', 'rom_migrate') + migration_files = Dir.glob(File.join(migration_dir, '*.rb')).map { |f| File.basename(f).split('_').first.to_i }.sort + + migration_files.each do |version| + if conn[:schema_migrations].where(version: version).any? + puts "Already applied: #{version}" + else + conn[:schema_migrations].insert(version: version) + puts "Applied: #{version}" + end + end + + puts 'All existing migrations are applied.' + end +end diff --git a/lib/tasks/servers.rake b/lib/tasks/servers.rake index aadb0b10..292d8618 100644 --- a/lib/tasks/servers.rake +++ b/lib/tasks/servers.rake @@ -2,18 +2,19 @@ namespace :servers do desc 'Start development servers' - task :initialize do + task initialize: :environment do + Rake::Task['db:drop'].invoke Rake::Task['db:create'].invoke Rake::Task['db:migrate'].invoke end desc 'Start the Apache Solr and PostgreSQL container services using Lando.' - task :start do + task start: :environment do Rake::Task['solr:update_configs'].invoke system('lando start') - system('rake servers:initialize') - system('rake servers:initialize RAILS_ENV=test') - system('rake solr:load_sample_data') - system('rake best_bets:sync') + system('bundle exec rake servers:initialize') + system('RAILS_ENV=test bundle exec rake db:migrate') + system('bundle exec rake solr:load_sample_data') + system('bundle exec rake best_bets:sync') end end diff --git a/spec/init/rom_factory_spec.rb b/spec/init/rom_factory_spec.rb index b6607856..da3b4745 100644 --- a/spec/init/rom_factory_spec.rb +++ b/spec/init/rom_factory_spec.rb @@ -4,6 +4,35 @@ require allsearch_path 'init/rom_factory' RSpec.describe RomFactory do + describe '#db_connection' do + it 'returns success with existing DB constant if already defined' do + result = described_class.new.send(:db_connection) + expect(result).to be_success + expect(result.success).to be_a(Sequel::Database) + end + + context 'when DB constant is not defined' do + before do + hide_const('DB') + end + + it 'returns success with database connection when connection succeeds' do + db = instance_double(Sequel::Postgres::Database, to_s: 'postgres://my-connection-string') + allow(Sequel).to receive(:postgres).and_return(db) + result = described_class.new.send(:db_connection) + expect(result).to be_success + expect(result.success).to eq(db) + end + + it 'returns failure when database connection fails' do + allow(Sequel).to receive(:postgres).and_raise(Sequel::DatabaseConnectionError.new('Connection failed')) + result = described_class.new.send(:db_connection) + expect(result).to be_failure + expect(result.failure).to be_a(Sequel::DatabaseConnectionError) + end + end + end + describe '#require_rom!' do # rubocop: disable RSpec/VerifiedDoubles it 'returns a rom if the database connection is good' do @@ -18,23 +47,55 @@ # rubocop: enable RSpec/VerifiedDoubles it 'raises an error if the database connection is bad' do - allow(Sequel).to receive(:postgres).and_raise Sequel::Error - expect { described_class.new.require_rom! }.to raise_exception(/Could not connect to the database/) + rom_factory = described_class.new + allow(rom_factory).to receive(:db_connection).and_return(Failure(Sequel::Error.new)) + expect { rom_factory.require_rom! }.to raise_exception(/Could not connect to the database/) end end describe 'rom_if_available' do it 'returns failure if we cannot connect to the database' do - allow(Sequel).to receive(:postgres).and_raise Sequel::DatabaseConnectionError - expect(described_class.new.rom_if_available).to be_failure + rom_factory = described_class.new + allow(rom_factory).to receive(:db_connection).and_return(Failure(Sequel::DatabaseConnectionError.new)) + expect(rom_factory.rom_if_available).to be_failure end - it 'returns failure if required tables do not exist' do + it 'returns failure if database connection is not ready' do db = instance_double(Sequel::Postgres::Database, to_s: 'postgres://my-connection-string') - allow(Sequel).to receive(:postgres).and_return(db) - allow(db).to receive(:table_exists?).with(:schema_migrations).and_return(false) - allow(db).to receive(:table_exists?).with(:ar_internal_metadata).and_return(true) - expect(described_class.new.rom_if_available).to be_failure + rom_factory = described_class.new + allow(rom_factory).to receive(:db_connection).and_return(Success(db)) + allow(rom_factory).to receive(:verify_db_connection_is_ready).and_call_original + allow(rom_factory).to receive(:Success).and_raise(StandardError.new) + expect(rom_factory.rom_if_available).to be_failure + end + + describe 'when required tables do not exist' do + # rubocop: disable RSpec/NestedGroups + context 'when schema_migrations table is missing' do + it 'returns failure' do + db = instance_double(Sequel::Postgres::Database, to_s: 'postgres://my-connection-string') + allow(db).to receive(:table_exists?).with(:schema_migrations).and_return(false) + rom_factory = described_class.new + allow(rom_factory).to receive(:db_connection).and_return(Success(db)) + expect(rom_factory.rom_if_available).to be_failure + end + end + + context 'when other required tables are missing' do + it 'returns failure' do + db = instance_double(Sequel::Postgres::Database, to_s: 'postgres://my-connection-string') + allow(db).to receive(:table_exists?).with(:schema_migrations).and_return(true) + allow(db).to receive(:table_exists?).with(:best_bet_records).and_return(true) + allow(db).to receive(:table_exists?).with(:library_database_records).and_return(true) + allow(db).to receive(:table_exists?).with(:library_staff_records).and_return(true) + allow(db).to receive(:table_exists?).with(:banners).and_return(true) + allow(db).to receive(:table_exists?).with(:oauth_tokens).and_return(false) + rom_factory = described_class.new + allow(rom_factory).to receive(:db_connection).and_return(Success(db)) + expect(rom_factory.rom_if_available).to be_failure + end + end + # rubocop: enable RSpec/NestedGroups end it 'returns success when database is ready and ROM can be initialized' do diff --git a/spec/requests/library_staff_spec.rb b/spec/requests/library_staff_spec.rb index a8ec690d..09673c17 100644 --- a/spec/requests/library_staff_spec.rb +++ b/spec/requests/library_staff_spec.rb @@ -33,7 +33,7 @@ before do stub_request(:get, 'https://lib-jobs.princeton.edu/pul-staff-report.csv') .to_return(status: 200, body: libjobs_response) - LibraryStaffLoadingService.new.run + LibraryStaffLoadingService.new(rom_container: ALLSEARCH_ROM).run end it 'returns json' do diff --git a/spec/services/best_bet_loading_service_spec.rb b/spec/services/best_bet_loading_service_spec.rb index 7898bb4f..4863bcc5 100644 --- a/spec/services/best_bet_loading_service_spec.rb +++ b/spec/services/best_bet_loading_service_spec.rb @@ -8,7 +8,7 @@ let(:search_terms) { '{some, terms}' } let(:url) { 'https://example.com' } - let(:rom) { Rails.application.config.rom } + let(:rom) { RomFactory.new.require_rom! } let(:repo) { RepositoryFactory.best_bet } let(:best_bet) { rom.relations[:best_bet_records] } @@ -52,8 +52,8 @@ context 'when a best bet in postgres is no longer in the CSV' do it 'removes it from the database' do repo = RepositoryFactory.best_bet - old_record = repo.create(id: 123, title:, search_terms:, url:) - expect(best_bet.where(id: 123)).to contain_exactly(old_record) + repo.create(id: 123, title:, search_terms:, url:) + expect(best_bet.where(id: 123).count).to eq 1 described_class.new.run expect(best_bet.where(id: 123).count).to eq 0 end @@ -83,4 +83,11 @@ 'The original length was 30 rows, the new length is 8 rows.') end end + + context 'when Rails raises NoMethodError during CSVLoadingService initialization' do + it 'falls back to RomFactory and still works' do + allow(Rails).to receive(:application).and_raise(NoMethodError) + expect { described_class.new.run }.to change(best_bet, :count).by(7) + end + end end diff --git a/spec/services/library_database_loading_service_spec.rb b/spec/services/library_database_loading_service_spec.rb index b0a8c216..5800d7dd 100644 --- a/spec/services/library_database_loading_service_spec.rb +++ b/spec/services/library_database_loading_service_spec.rb @@ -11,7 +11,7 @@ end it 'creates a new row in the library_database_records table for each CSV row' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! expect { described_class.new.run }.to change { rom.relations[:library_database_records].count }.by(14) third_record = rom.relations[:library_database_records].to_a[2] expect(third_record[:name]).to eq('Abzu') @@ -23,7 +23,7 @@ end it 'is idempotent' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! described_class.new.run expect { described_class.new.run }.not_to(change { rom.relations[:library_database_records].count }) end @@ -32,7 +32,7 @@ let(:libjobs_response) { 'bad response' } it 'does not proceed' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! repo = RepositoryFactory.library_database repo.create(libguides_id: 123, name: 'JSTOR') expect { described_class.new.run }.not_to(change { rom.relations[:library_database_records].count }) @@ -41,7 +41,7 @@ context 'when a library database in postgres is no longer in the CSV' do it 'removes it from the database' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! repo = RepositoryFactory.library_database repo.create(libguides_id: 123, name: 'JSTOR') expect(rom.relations[:library_database_records].where(libguides_id: 123).count).to eq 1 @@ -52,7 +52,7 @@ context 'when a library database has updated info in the CSV' do it 'updates the relevant fields' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! repo = RepositoryFactory.library_database repo.create(libguides_id: 2_938_694, name: 'JSTOR') expect( @@ -67,7 +67,7 @@ context 'when the CSV is suspiciously small relative to the number of database rows' do it 'does not proceed' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! repo = RepositoryFactory.library_database 30.times { |number| repo.create(libguides_id: number, name: 'JSTOR') } expect { described_class.new.run }.not_to(change { rom.relations[:library_database_records].count }) diff --git a/spec/services/library_staff_loading_service_spec.rb b/spec/services/library_staff_loading_service_spec.rb index af3214ee..e9cf4a65 100644 --- a/spec/services/library_staff_loading_service_spec.rb +++ b/spec/services/library_staff_loading_service_spec.rb @@ -11,7 +11,7 @@ end it 'creates new rows in the library_staff table for each CSV row' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! expect { described_class.new.run }.to change { rom.relations[:library_staff_records].count }.by(5) third_record = rom.relations[:library_staff_records].to_a[2] fourth_record = rom.relations[:library_staff_records].to_a[3] @@ -40,7 +40,7 @@ it 'is idempotent' do described_class.new.run - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! expect { described_class.new.run }.not_to(change { rom.relations[:library_staff_records].count }) end @@ -51,7 +51,7 @@ described_class.new.run # Create a record that would typically be deleted as part of running # this service, since it is not in the CSV data - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! repo = RepositoryFactory.library_staff repo.create(puid: 0o00000004, netid: 'notourcat', name: 'Cat, Not Our', title: 'Outside Specialist', library_title: 'Outside Specialist', @@ -67,7 +67,7 @@ described_class.new.run # Create a record that would typically be deleted as part of running # this service, since it is not in the CSV data - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! repo = RepositoryFactory.library_staff repo.create(puid: 0o00000004, netid: 'notourcat', name: 'Cat, Not Our', title: 'Outside Specialist', library_title: 'Outside Specialist', @@ -78,7 +78,7 @@ context 'when a staff member in postgres is no longer in the CSV' do it 'removes them from the database' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! repo = RepositoryFactory.library_staff repo.create(puid: 0o00000004, netid: 'notourcat', name: 'Cat, Not Our', title: 'Outside Specialist', library_title: 'Outside Specialist', @@ -91,7 +91,7 @@ context 'when a staff member has updated info in the CSV' do it 'updates the relevant fields' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! repo = RepositoryFactory.library_staff repo.create(puid: 0o00000003, netid: 'tiberius', first_name: 'Spot Tiberius', name: 'Adams, Spot', title: 'Lead Hairball Engineer', library_title: 'Lead Hairball Engineer', @@ -108,7 +108,7 @@ let(:libjobs_response) { file_fixture('library_staff/staff-directory-blank-lines.csv') } it 'creates records for any complete lines in the CSV' do - rom = Rails.application.config.rom + rom = RomFactory.new.require_rom! expect { described_class.new.run }.to change { rom.relations[:library_staff_records].count }.by(1) end end