Class: DB::DBPool

Inherits:
Object
  • Object
show all
Defined in:
backend/app/model/db.rb

Defined Under Namespace

Classes: DBAttempt

Constant Summary collapse

DATABASE_READ_ONLY_REGEX = 
/is read only|server is running with the --read-only option/

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(pool_size = AppConfig[:db_max_connections], pool_timeout = AppConfig[:db_pool_timeout], opts = {}) ⇒ DBPool

Returns a new instance of DBPool.



29
30
31
32
33
34
35
 
# File 'backend/app/model/db.rb', line 29

def initialize(pool_size = AppConfig[:db_max_connections], pool_timeout = AppConfig[:db_pool_timeout], opts = {})
  @pool_size = pool_size
  @pool_timeout = pool_timeout
  @opts = opts

  @lock = Mutex.new
end

Instance Attribute Details

#pool_size ⇒ Object (readonly)

Returns the value of attribute pool_size



27
28
29
 
# File 'backend/app/model/db.rb', line 27

def pool_size
  @pool_size
end

#pool_timeout ⇒ Object (readonly)

Returns the value of attribute pool_timeout



27
28
29
 
# File 'backend/app/model/db.rb', line 27

def pool_timeout
  @pool_timeout
end

Instance Method Details

#after_commit(&block) ⇒ Object 



152
153
154
155
156
157
158
159
160
 
# File 'backend/app/model/db.rb', line 152

def after_commit(&block)
  if @pool.in_transaction?
    @pool.after_commit do
      block.call
    end
  else
    block.call
  end
end

#attempt(&block) ⇒ Object 



325
326
327
 
# File 'backend/app/model/db.rb', line 325

def attempt(&block)
  DBAttempt.new(block)
end

#check_supported(url) ⇒ Object 



360
361
362
363
 
# File 'backend/app/model/db.rb', line 360

def check_supported(url)
      if !SUPPORTED_DATABASES.any? {|db| url =~ db[:pattern]}

msg = "\n          =======================================================================\n          UNSUPPORTED DATABASE\n          =======================================================================\n\n          The database listed in your configuration:\n\n            \#{url}\n\n          is not officially supported by ArchivesSpace.  Although the system may\n          still work, there's no guarantee that future versions will continue to\n          work, or that it will be possible to upgrade without losing your data.\n\n          It is strongly recommended that you run ArchivesSpace against one of\n          these supported databases:\n\n        eof\n\n        SUPPORTED_DATABASES.each do |db|\n          msg += \"  * \#{db[:name]}\\n\"\n        end\n\n        msg += \"\\n\"\n        msg += <<~eof\n\n          To ignore this (very good) advice, you can set the configuration option:\n\n            AppConfig[:allow_unsupported_database] = true\n\n\n          =======================================================================\n\n        eof\n\n        Log.error(msg)\n\n        raise \"Database not supported\"\n      end\n    end\n\n\n    def backups_dir\n      AppConfig[:backup_directory]\n    end\n\n\n    def expire_backups\n      backups = []\n      Dir.foreach(backups_dir) do |filename|\n        if filename =~ /^demo_db_backup_[0-9]+_[0-9]+$/\n          backups << File.join(backups_dir, filename)\n        end\n      end\n\n      expired_backups = backups.sort.reverse.drop(AppConfig[:demo_db_backup_number_to_keep])\n\n      expired_backups.each do |backup_dir|\n        # Proudly paranoid\n        if File.exist?(File.join(backup_dir, \"archivesspace_demo_db\", \"BACKUP.HISTORY\"))\n          Log.info(\"Expiring old backup: \#{backup_dir}\")\n          FileUtils.rm_rf(backup_dir)\n        else\n          Log.warn(\"Too cowardly to delete: \#{backup_dir}\")\n        end\n      end\n    end\n\n\n    def demo_db_backup\n      # Timestamp must come first here for filenames to sort chronologically\n      this_backup = File.join(backups_dir, \"demo_db_backup_\#{Time.now.to_i}_\#{$$}\")\n\n      Log.info(\"Writing backup to '\#{this_backup}'\")\n\n      @pool.pool.hold do |c|\n        cs = c.prepare_call(\"CALL SYSCS_UTIL.SYSCS_BACKUP_DATABASE(?)\")\n        cs.set_string(1, this_backup.to_s)\n        cs.execute\n        cs.close\n      end\n\n      expire_backups\n    end\n\n\n    def increase_lock_version_or_fail(obj)\n      updated_rows = obj.class.dataset.filter(:id => obj.id, :lock_version => obj.lock_version).\n                     update(:lock_version => obj.lock_version + 1,\n                            :system_mtime => Time.now)\n\n      if updated_rows != 1\n        raise Sequel::Plugins::OptimisticLocking::Error.new(\"Couldn't create version of: \#{obj}\")\n      end\n    end\n\n\n    def supports_mvcc?\n      ![:derby, :h2].include?(@pool.database_type)\n    end\n\n\n    def supports_join_updates?\n      ![:derby, :h2].include?(@pool.database_type)\n    end\n\n\n    def needs_blob_hack?\n      (@pool.database_type == :derby)\n    end\n\n    def blobify(s)\n      (@pool.database_type == :derby) ? s.to_sequel_blob : s\n    end\n\n\n    def concat(s1, s2)\n      if @pool.database_type == :derby\n        \"\#{s1} || \#{s2}\"\n      else\n        \"CONCAT(\#{s1}, \#{s2})\"\n      end\n    end\n\n\n    def ensure_tables_are_utf8(db)\n      non_utf8_tables = db[:information_schema__tables].\n                        join(:information_schema__collation_character_set_applicability, :collation_name => :table_collation).\n                        filter(:table_schema => Sequel.function(:database)).\n                        filter(~Sequel.like(:character_set_name, 'utf8%')).all\n\n      unless (non_utf8_tables.empty?)\n        msg = <<~EOF\n\n          The following MySQL database tables are not set to use UTF-8 for their character\n          encoding:\n\n          \#{non_utf8_tables.map {|t| \"  * \" + t[:TABLE_NAME]}.join(\"\\n\")}\n\n          Please refer to README.md for instructions on configuring your database to use\n          UTF-8.\n\n          If you want to override this restriction (not recommended!) you can set the\n          following option in your config.rb file:\n\n            AppConfig[:allow_non_utf8_mysql_database] = true\n\n          But note that ArchivesSpace largely assumes that your data will be UTF-8\n          encoded.  Running in a non-UTF-8 configuration is not supported.\n\n        EOF\n\n        Log.warn(msg)\n        raise msg\n      end\n\n      Log.info(\"All tables checked and confirmed set to UTF-8.  Nice job!\")\n    end\n  end\n\n\n  # Create our default connection pool\n  @default_pool = :not_connected\n\n  def self.connect\n    if @default_pool == :not_connected\n      @default_pool = DBPool.new.connect\n    else\n      @default_pool\n    end\n  end\n\n  def self.connected?\n    if @default_pool == :not_connected\n      false\n    else\n      @default_pool.connected?\n    end\n  end\n\n  # Any method called on DB is dispatched to our default pool.\n  DBPool.instance_methods(false).each do |method|\n    if self.singleton_methods(false).include?(method)\n      next\n    end\n\n    self.define_singleton_method(method) do |*args, &block|\n      if block\n        @default_pool.send(method, *args, &block)\n      else\n        @default_pool.send(method, *args)\n      end\n    end\n  end\n\nend\n"

#connect ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'backend/app/model/db.rb', line 37

def connect
  if not @pool

    if !AppConfig[:allow_unsupported_database]
      check_supported(AppConfig[:db_url])
    end

    begin
      Log.info("Connecting to database: #{AppConfig[:db_url_redacted]}. Max connections: #{pool_size}")
      pool = Sequel.connect(AppConfig[:db_url],
                            :max_connections => pool_size,
                            :pool_timeout => pool_timeout,
                            :test => true,
                            :loggers => (AppConfig[:db_debug_log] ? [Logger.new($stderr)] : [])
                           )

      # Test if any tables exist
      pool[:schema_info].all

      if !@opts[:skip_utf8_check] && pool.database_type == :mysql && !AppConfig[:allow_non_utf8_mysql_database]
        ensure_tables_are_utf8(pool)
      end

      @pool = pool
    rescue Sequel::DatabaseConnectionError
      Log.error("DB connection failed: #{$!}")

      exceptions = [$!.wrapped_exception].compact

      while !exceptions.empty?
        exception = exceptions.shift
        Log.error("Additional DB info: #{exception.inspect}: #{exception}")
        exceptions << exception.get_cause if exception.get_cause
      end

      raise
    rescue
      Log.error("DB connection failed: #{$!}")
      raise
    end
  end

  self
end

#connected? ⇒ Boolean

Returns:

  • (Boolean) 


83
84
85
 
# File 'backend/app/model/db.rb', line 83

def connected?
  not @pool.nil?
end

#disconnect ⇒ Object 



355
356
357
 
# File 'backend/app/model/db.rb', line 355

def disconnect
  @pool.disconnect
end

#in_transaction? ⇒ Boolean

Returns:

  • (Boolean) 


268
269
270
 
# File 'backend/app/model/db.rb', line 268

def in_transaction?
  @pool.in_transaction?
end

#is_integrity_violation(exception) ⇒ Object

Yeesh.



331
332
333
 
# File 'backend/app/model/db.rb', line 331

def is_integrity_violation(exception)
  (exception.wrapped_exception.cause or exception.wrapped_exception).getSQLState() =~ /^23/
end

#is_retriable_exception(exception, opts = {}) ⇒ Object 



336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
 
# File 'backend/app/model/db.rb', line 336

def is_retriable_exception(exception, opts = {})
  # Transaction was rolled back, but we can retry
  return true if exception.instance_of?(RetryTransaction)

  return true if (opts[:retry_on_optimistic_locking_fail] && exception.instance_of?(Sequel::Plugins::OptimisticLocking::Error))

  if (inner_exception = exception.wrapped_exception)
    if inner_exception.cause
      inner_exception = inner_exception.cause
    end

    if inner_exception.is_a?(java.sql.SQLException)
      return inner_exception.getSQLState =~ /^(40|41)/
    end
  end

  false
end

#jdbc_metadata ⇒ Object 



277
278
279
280
281
 
# File 'backend/app/model/db.rb', line 277

def jdbc_metadata
  md = open { |p| p.synchronize { |c| c.getMetaData }}
  { "databaseProductName" => md.getDatabaseProductName,
    "databaseProductVersion" => md.getDatabaseProductVersion }
end

#needs_savepoint? ⇒ Boolean

Returns:

  • (Boolean) 


291
292
293
294
295
296
 
# File 'backend/app/model/db.rb', line 291

def needs_savepoint?
  # Postgres needs a savepoint for any statement that might fail
  # (otherwise the whole transaction becomes invalid).  Use a savepoint to
  # run the happy case, since we're half expecting it to fail.
  [:postgres].include?(@pool.database_type)
end

#open(transaction = true, opts = {}) ⇒ Object 



167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
 
# File 'backend/app/model/db.rb', line 167

def open(transaction = true, opts = {})
  # Give us a place to hang storage that relates to the current database
  # session.
  Thread.current[:db_session_storage] ||= {}
  Thread.current[:nesting_level] ||= 0
  Thread.current[:nesting_level] += 1

  Thread.current[:in_transaction] ||= false

  begin
    if Thread.current[:in_transaction] && ASpaceEnvironment.environment != :unit_test
      # We are already inside another DB.open that will handle all
      # exceptions and retries for us.  We want to avoid a situation like
      # this:
      #
      # transaction scope / DB.open do |db|
      #                   |   db[:sometable].insert(foo)
      #                   |
      #                   |   DB.open do |db|                                                    #                   |     db[:sometable].insert(something_that_depends_on_foo)  | retry scope
      #                   |   end                                                     /
      #                   \ end
      #
      # Despite the nested DB.open calls, Sequel's default behavior is to
      # merge the inner call to DB.transaction with the already active
      # transaction.
      #
      # If the inner "retry scope" hits an exception, the whole transaction
      # is rolled back (all of "transaction scope", including the insert of
      # `foo`), but only the inner "retry scope" is retried.  If that
      # succeeds on the retry, we end up losing first insert and keeping the
      # second.
      #
      # So the fix here is to let the outermost DB.open take responsibility
      # for everything: make the retry scope and the transaction scope line
      # up with each other.

      return yield @pool
    end

    last_err = false
    retries = opts[:retries] || 10

    retries.times do |attempt|
      begin
        if transaction
          self.transaction(:isolation => opts.fetch(:isolation_level, :repeatable)) do
            Thread.current[:in_transaction] = true
            begin
              return yield @pool
            ensure
              Thread.current[:in_transaction] = false
            end
          end

          # Sometimes we'll make it to here.  That means we threw a
          # Sequel::Rollback which has been quietly caught.
          return nil
        else
          begin
            return yield @pool
          rescue Sequel::Rollback
            # If we're not in a transaction we can't roll back, but no need to blow up.
            Log.warn("Sequel::Rollback caught but we're not inside of a transaction")
            return nil
          end
        end


      rescue Sequel::DatabaseDisconnectError => e
        # MySQL might have been restarted.
        last_err = e
        Log.info("Connecting to the database failed.  Retrying...")
        sleep(opts[:db_failed_retry_delay] || 3)


      rescue Sequel::NoExistingObject, Sequel::DatabaseError => e
        if (attempt + 1) < retries && is_retriable_exception(e, opts) && transaction
          Log.info("Retrying transaction after retriable exception (#{e})")
          sleep(opts[:retry_delay] || 1)
        else
          raise e
        end
      end

      if last_err
        Log.error("Failed to connect to the database")
        Log.exception(last_err)

        raise "Failed to connect to the database: #{last_err}"
      end
    end
  ensure
    Thread.current[:nesting_level] -= 1

    if Thread.current[:nesting_level] <= 0
      Thread.current[:db_session_storage] = nil
    end
  end
end

#session_storage ⇒ Object 



163
164
165
 
# File 'backend/app/model/db.rb', line 163

def session_storage
  Thread.current[:db_session_storage] or raise "Not inside transaction!"
end

#sysinfo ⇒ Object 



272
273
274
 
# File 'backend/app/model/db.rb', line 272

def sysinfo
  jdbc_metadata.merge(system_metadata).merge({ "archivesSpaceVersion" => ASConstants.VERSION})
end

#system_metadata ⇒ Object 



283
284
285
286
287
288
289
 
# File 'backend/app/model/db.rb', line 283

def system_metadata
  RbConfig.const_get("CONFIG").select { |key| ['host_os', 'host_cpu',
                                               'build', 'ruby_version'].include? key }.merge({
                                                  'java.runtime.name' => java.lang.System.getProperty('java.runtime.name'),
                                                  'java.version' => java.lang.System.getProperty('java.version')
                                                })
end

#transaction(*args) ⇒ Object 



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'backend/app/model/db.rb', line 87

def transaction(*args)
  retry_count = 0

  begin
    # @pool might be nil if we're in the middle of a reconnect.  Spin for a
    # bit before giving up.
    pool = nil

    60.times do
      pool = @pool
      break if pool
      sleep 1
    end

    if pool.nil?
      Log.info("DB connection failed: unable to get a connection")
      raise
    end

    pool.transaction(*args) do
      yield(pool)
    end
  rescue Sequel::DatabaseError, java.sql.SQLException => e
    if retry_count > 0
      Log.warn("DB connection failure: #{e}.  Retry count is #{retry_count}")
    end

    if retry_count > 6
      # We give up
      raise e
    end

    if e.to_s =~ DATABASE_READ_ONLY_REGEX
      sleep rand * 10

      # Reset the pool...
      old_pool = @pool

      @lock.synchronize do
        if @pool == old_pool
          # If we got the lock and nobody has reset the pool yet, it's time to do our thing.
          @pool = nil

          # We retry the connection indefinitely here.  The system isn't
          # going to function until the pool is restored, so either return
          # successful or don't return at all.
          begin
            connect
          rescue
            Log.warn("DB connection failure on reconnect: #{$!}.  Retrying indefinitely...")
            sleep 1
            retry
          end
        end
      end

      retry_count += 1
      retry
    else
      raise e
    end
  end
end