Class: DB::DBPool

Inherits:
Object
  • Object
show all
Defined in:
backend/app/model/db.rb

Defined Under Namespace

Classes: DBAttempt

Constant Summary collapse

DATABASE_READ_ONLY_REGEX =
/is read only|server is running with the --read-only option/

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(pool_size = AppConfig[:db_max_connections], opts = {}) ⇒ DBPool

Returns a new instance of DBPool.



25
26
27
28
29
30
# File 'backend/app/model/db.rb', line 25

def initialize(pool_size = AppConfig[:db_max_connections], opts = {})
  @pool_size = pool_size
  @opts = opts

  @lock = Mutex.new
end

Instance Attribute Details

#pool_sizeObject (readonly)

Returns the value of attribute pool_size



23
24
25
# File 'backend/app/model/db.rb', line 23

def pool_size
  @pool_size
end

Class Method Details

.connectObject



506
507
508
509
510
511
512
# File 'backend/app/model/db.rb', line 506

def self.connect
  if @default_pool == :not_connected
    @default_pool = DBPool.new.connect
  else
    @default_pool
  end
end

.connected?Boolean

Returns:

  • (Boolean)


514
515
516
517
518
519
520
# File 'backend/app/model/db.rb', line 514

def self.connected?
  if @default_pool == :not_connected
    false
  else
    @default_pool.connected?
  end
end

Instance Method Details

#after_commit(&block) ⇒ Object



134
135
136
137
138
139
140
141
142
# File 'backend/app/model/db.rb', line 134

def after_commit(&block)
  if @pool.in_transaction?
    @pool.after_commit do
      block.call
    end
  else
    block.call
  end
end

#attempt(&block) ⇒ Object



304
305
306
# File 'backend/app/model/db.rb', line 304

def attempt(&block)
  DBAttempt.new(block)
end

#blobify(s) ⇒ Object



453
454
455
# File 'backend/app/model/db.rb', line 453

def blobify(s)
  (@pool.database_type == :derby) ? s.to_sequel_blob : s
end

#check_supported(url) ⇒ Object



339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
# File 'backend/app/model/db.rb', line 339

def check_supported(url)
      if !SUPPORTED_DATABASES.any? {|db| url =~ db[:pattern]}

        msg = <<~eof
          
          =======================================================================
          UNSUPPORTED DATABASE
          =======================================================================
          
          The database listed in your configuration:
          
            #{url}
          
          is not officially supported by ArchivesSpace.  Although the system may
          still work, there's no guarantee that future versions will continue to
          work, or that it will be possible to upgrade without losing your data.
          
          It is strongly recommended that you run ArchivesSpace against one of
          these supported databases:
          
        eof

        SUPPORTED_DATABASES.each do |db|
          msg += "  * #{db[:name]}\n"
        end

        msg += "\n"
        msg += <<~eof
          
          To ignore this (very good) advice, you can set the configuration option:
          
            AppConfig[:allow_unsupported_database] = true
          
          
          =======================================================================
          
        eof

        Log.error(msg)

        raise "Database not supported"
      end
    end


    def backups_dir
      AppConfig[:backup_directory]
    end


    def expire_backups
      backups = []
      Dir.foreach(backups_dir) do |filename|
        if filename =~ /^demo_db_backup_[0-9]+_[0-9]+$/
          backups << File.join(backups_dir, filename)
        end
      end

      victims = backups.sort.reverse.drop(AppConfig[:demo_db_backup_number_to_keep])

      victims.each do |backup_dir|
        # Proudly paranoid
        if File.exist?(File.join(backup_dir, "archivesspace_demo_db", "BACKUP.HISTORY"))
          Log.info("Expiring old backup: #{backup_dir}")
          FileUtils.rm_rf(backup_dir)
        else
          Log.warn("Too cowardly to delete: #{backup_dir}")
        end
      end
    end


    def demo_db_backup
      # Timestamp must come first here for filenames to sort chronologically
      this_backup = File.join(backups_dir, "demo_db_backup_#{Time.now.to_i}_#{$$}")

      Log.info("Writing backup to '#{this_backup}'")

      @pool.pool.hold do |c|
        cs = c.prepare_call("CALL SYSCS_UTIL.SYSCS_BACKUP_DATABASE(?)")
        cs.set_string(1, this_backup.to_s)
        cs.execute
        cs.close
      end

      expire_backups
    end


    def increase_lock_version_or_fail(obj)
      updated_rows = obj.class.dataset.filter(:id => obj.id, :lock_version => obj.lock_version).
                     update(:lock_version => obj.lock_version + 1,
                            :system_mtime => Time.now)

      if updated_rows != 1
        raise Sequel::Plugins::OptimisticLocking::Error.new("Couldn't create version of: #{obj}")
      end
    end


    def supports_mvcc?
      ![:derby, :h2].include?(@pool.database_type)
    end


    def supports_join_updates?
      ![:derby, :h2].include?(@pool.database_type)
    end


    def needs_blob_hack?
      (@pool.database_type == :derby)
    end

    def blobify(s)
      (@pool.database_type == :derby) ? s.to_sequel_blob : s
    end


    def concat(s1, s2)
      if @pool.database_type == :derby
        "#{s1} || #{s2}"
      else
        "CONCAT(#{s1}, #{s2})"
      end
    end


    def ensure_tables_are_utf8(db)
      non_utf8_tables = db[:information_schema__tables].
                        join(:information_schema__collation_character_set_applicability, :collation_name => :table_collation).
                        filter(:table_schema => Sequel.function(:database)).
                        filter(~Sequel.like(:character_set_name, 'utf8%')).all

      unless (non_utf8_tables.empty?)
        msg = <<~EOF
          
          The following MySQL database tables are not set to use UTF-8 for their character
          encoding:
          
          #{non_utf8_tables.map {|t| "  * " + t[:TABLE_NAME]}.join("\n")}
          
          Please refer to README.md for instructions on configuring your database to use
          UTF-8.
          
          If you want to override this restriction (not recommended!) you can set the
          following option in your config.rb file:
          
            AppConfig[:allow_non_utf8_mysql_database] = true
          
          But note that ArchivesSpace largely assumes that your data will be UTF-8
          encoded.  Running in a non-UTF-8 configuration is not supported.
          
        EOF

        Log.warn(msg)
        raise msg
      end

      Log.info("All tables checked and confirmed set to UTF-8.  Nice job!")
    end
  end


  # Create our default connection pool
  @default_pool = :not_connected

  def self.connect
    if @default_pool == :not_connected
      @default_pool = DBPool.new.connect
    else
      @default_pool
    end
  end

  def self.connected?
    if @default_pool == :not_connected
      false
    else
      @default_pool.connected?
    end
  end

  # Any method called on DB is dispatched to our default pool.
  DBPool.instance_methods(false).each do |method|
    if self.singleton_methods(false).include?(method)
      next
    end

    self.define_singleton_method(method) do |*args, &block|
      if block
        @default_pool.send(method, *args, &block)
      else
        @default_pool.send(method, *args)
      end
    end
  end

end

#concat(s1, s2) ⇒ Object



458
459
460
461
462
463
464
# File 'backend/app/model/db.rb', line 458

def concat(s1, s2)
  if @pool.database_type == :derby
    "#{s1} || #{s2}"
  else
    "CONCAT(#{s1}, #{s2})"
  end
end

#connectObject



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# File 'backend/app/model/db.rb', line 32

def connect
  if not @pool

    if !AppConfig[:allow_unsupported_database]
      check_supported(AppConfig[:db_url])
    end

    begin
      Log.info("Connecting to database: #{AppConfig[:db_url_redacted]}. Max connections: #{pool_size}")
      pool = Sequel.connect(AppConfig[:db_url],
                            :max_connections => pool_size,
                            :test => true,
                            :loggers => (AppConfig[:db_debug_log] ? [Logger.new($stderr)] : [])
                           )

      # Test if any tables exist
      pool[:schema_info].all

      if !@opts[:skip_utf8_check] && pool.database_type == :mysql && !AppConfig[:allow_non_utf8_mysql_database]
        ensure_tables_are_utf8(pool)
      end

      @pool = pool
    rescue
      Log.error("DB connection failed: #{$!}")
      raise
    end
  end

  self
end

#connected?Boolean

Returns:

  • (Boolean)


65
66
67
# File 'backend/app/model/db.rb', line 65

def connected?
  not @pool.nil?
end

#disconnectObject



334
335
336
# File 'backend/app/model/db.rb', line 334

def disconnect
  @pool.disconnect
end

#ensure_tables_are_utf8(db) ⇒ Object



467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
# File 'backend/app/model/db.rb', line 467

def ensure_tables_are_utf8(db)
      non_utf8_tables = db[:information_schema__tables].
                        join(:information_schema__collation_character_set_applicability, :collation_name => :table_collation).
                        filter(:table_schema => Sequel.function(:database)).
                        filter(~Sequel.like(:character_set_name, 'utf8%')).all

      unless (non_utf8_tables.empty?)
        msg = <<~EOF
          
          The following MySQL database tables are not set to use UTF-8 for their character
          encoding:
          
          #{non_utf8_tables.map {|t| "  * " + t[:TABLE_NAME]}.join("\n")}
          
          Please refer to README.md for instructions on configuring your database to use
          UTF-8.
          
          If you want to override this restriction (not recommended!) you can set the
          following option in your config.rb file:
          
            AppConfig[:allow_non_utf8_mysql_database] = true
          
          But note that ArchivesSpace largely assumes that your data will be UTF-8
          encoded.  Running in a non-UTF-8 configuration is not supported.
          
        EOF

        Log.warn(msg)
        raise msg
      end

      Log.info("All tables checked and confirmed set to UTF-8.  Nice job!")
    end
  end


  # Create our default connection pool
  @default_pool = :not_connected

  def self.connect
    if @default_pool == :not_connected
      @default_pool = DBPool.new.connect
    else
      @default_pool
    end
  end

  def self.connected?
    if @default_pool == :not_connected
      false
    else
      @default_pool.connected?
    end
  end

  # Any method called on DB is dispatched to our default pool.
  DBPool.instance_methods(false).each do |method|
    if self.singleton_methods(false).include?(method)
      next
    end

    self.define_singleton_method(method) do |*args, &block|
      if block
        @default_pool.send(method, *args, &block)
      else
        @default_pool.send(method, *args)
      end
    end
  end

end

#in_transaction?Boolean

Returns:

  • (Boolean)


250
251
252
# File 'backend/app/model/db.rb', line 250

def in_transaction?
  @pool.in_transaction?
end

#increase_lock_version_or_fail(obj) ⇒ Object



428
429
430
431
432
433
434
435
436
# File 'backend/app/model/db.rb', line 428

def increase_lock_version_or_fail(obj)
  updated_rows = obj.class.dataset.filter(:id => obj.id, :lock_version => obj.lock_version).
                 update(:lock_version => obj.lock_version + 1,
                        :system_mtime => Time.now)

  if updated_rows != 1
    raise Sequel::Plugins::OptimisticLocking::Error.new("Couldn't create version of: #{obj}")
  end
end

#is_integrity_violation(exception) ⇒ Object

Yeesh.



310
311
312
# File 'backend/app/model/db.rb', line 310

def is_integrity_violation(exception)
  (exception.wrapped_exception.cause or exception.wrapped_exception).getSQLState() =~ /^23/
end

#is_retriable_exception(exception, opts = {}) ⇒ Object



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
# File 'backend/app/model/db.rb', line 315

def is_retriable_exception(exception, opts = {})
  # Transaction was rolled back, but we can retry
  return true if exception.instance_of?(RetryTransaction)

  return true if (opts[:retry_on_optimistic_locking_fail] && exception.instance_of?(Sequel::Plugins::OptimisticLocking::Error))

  if (inner_exception = exception.wrapped_exception)
    if inner_exception.cause
      inner_exception = inner_exception.cause
    end

    if inner_exception.is_a?(java.sql.SQLException)
      return inner_exception.getSQLState =~ /^(40|41)/
    end
  end

  false
end

#jdbc_metadataObject



259
260
261
262
263
# File 'backend/app/model/db.rb', line 259

def 
  md = open { |p| p.synchronize { |c| c. }}
  { "databaseProductName" => md.getDatabaseProductName,
    "databaseProductVersion" => md.getDatabaseProductVersion }
end

#needs_blob_hack?Boolean

Returns:

  • (Boolean)


449
450
451
# File 'backend/app/model/db.rb', line 449

def needs_blob_hack?
  (@pool.database_type == :derby)
end

#needs_savepoint?Boolean

Returns:

  • (Boolean)


270
271
272
273
274
275
# File 'backend/app/model/db.rb', line 270

def needs_savepoint?
  # Postgres needs a savepoint for any statement that might fail
  # (otherwise the whole transaction becomes invalid).  Use a savepoint to
  # run the happy case, since we're half expecting it to fail.
  [:postgres].include?(@pool.database_type)
end

#open(transaction = true, opts = {}) ⇒ Object



149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
# File 'backend/app/model/db.rb', line 149

def open(transaction = true, opts = {})
  # Give us a place to hang storage that relates to the current database
  # session.
  Thread.current[:db_session_storage] ||= {}
  Thread.current[:nesting_level] ||= 0
  Thread.current[:nesting_level] += 1

  Thread.current[:in_transaction] ||= false

  begin
    if Thread.current[:in_transaction] && ASpaceEnvironment.environment != :unit_test
      # We are already inside another DB.open that will handle all
      # exceptions and retries for us.  We want to avoid a situation like
      # this:
      #
      # transaction scope / DB.open do |db|
      #                   |   db[:sometable].insert(foo)
      #                   |
      #                   |   DB.open do |db|                                                    #                   |     db[:sometable].insert(something_that_depends_on_foo)  | retry scope
      #                   |   end                                                     /
      #                   \ end
      #
      # Despite the nested DB.open calls, Sequel's default behavior is to
      # merge the inner call to DB.transaction with the already active
      # transaction.
      #
      # If the inner "retry scope" hits an exception, the whole transaction
      # is rolled back (all of "transaction scope", including the insert of
      # `foo`), but only the inner "retry scope" is retried.  If that
      # succeeds on the retry, we end up losing first insert and keeping the
      # second.
      #
      # So the fix here is to let the outermost DB.open take responsibility
      # for everything: make the retry scope and the transaction scope line
      # up with each other.

      return yield @pool
    end

    last_err = false
    retries = opts[:retries] || 10

    retries.times do |attempt|
      begin
        if transaction
          self.transaction(:isolation => opts.fetch(:isolation_level, :repeatable)) do
            Thread.current[:in_transaction] = true
            begin
              return yield @pool
            ensure
              Thread.current[:in_transaction] = false
            end
          end

          # Sometimes we'll make it to here.  That means we threw a
          # Sequel::Rollback which has been quietly caught.
          return nil
        else
          begin
            return yield @pool
          rescue Sequel::Rollback
            # If we're not in a transaction we can't roll back, but no need to blow up.
            Log.warn("Sequel::Rollback caught but we're not inside of a transaction")
            return nil
          end
        end


      rescue Sequel::DatabaseDisconnectError => e
        # MySQL might have been restarted.
        last_err = e
        Log.info("Connecting to the database failed.  Retrying...")
        sleep(opts[:db_failed_retry_delay] || 3)


      rescue Sequel::NoExistingObject, Sequel::DatabaseError => e
        if (attempt + 1) < retries && is_retriable_exception(e, opts) && transaction
          Log.info("Retrying transaction after retriable exception (#{e})")
          sleep(opts[:retry_delay] || 1)
        else
          raise e
        end
      end

      if last_err
        Log.error("Failed to connect to the database")
        Log.exception(last_err)

        raise "Failed to connect to the database: #{last_err}"
      end
    end
  ensure
    Thread.current[:nesting_level] -= 1

    if Thread.current[:nesting_level] <= 0
      Thread.current[:db_session_storage] = nil
    end
  end
end

#session_storageObject



145
146
147
# File 'backend/app/model/db.rb', line 145

def session_storage
  Thread.current[:db_session_storage] or raise "Not inside transaction!"
end

#supports_join_updates?Boolean

Returns:

  • (Boolean)


444
445
446
# File 'backend/app/model/db.rb', line 444

def supports_join_updates?
  ![:derby, :h2].include?(@pool.database_type)
end

#supports_mvcc?Boolean

Returns:

  • (Boolean)


439
440
441
# File 'backend/app/model/db.rb', line 439

def supports_mvcc?
  ![:derby, :h2].include?(@pool.database_type)
end

#sysinfoObject



254
255
256
# File 'backend/app/model/db.rb', line 254

def sysinfo
  .merge()
end

#system_metadataObject



265
266
267
268
# File 'backend/app/model/db.rb', line 265

def 
  RbConfig.const_get("CONFIG").select { |key| ['host_os', 'host_cpu',
                                               'build', 'ruby_version'].include? key }
end

#transaction(*args) ⇒ Object



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
# File 'backend/app/model/db.rb', line 69

def transaction(*args)
  retry_count = 0

  begin
    # @pool might be nil if we're in the middle of a reconnect.  Spin for a
    # bit before giving up.
    pool = nil

    60.times do
      pool = @pool
      break if pool
      sleep 1
    end

    if pool.nil?
      Log.info("DB connection failed: unable to get a connection")
      raise
    end

    pool.transaction(*args) do
      yield(pool)
    end
  rescue Sequel::DatabaseError, java.sql.SQLException => e
    if retry_count > 0
      Log.warn("DB connection failure: #{e}.  Retry count is #{retry_count}")
    end

    if retry_count > 6
      # We give up
      raise e
    end

    if e.to_s =~ DATABASE_READ_ONLY_REGEX
      sleep rand * 10

      # Reset the pool...
      old_pool = @pool

      @lock.synchronize do
        if @pool == old_pool
          # If we got the lock and nobody has reset the pool yet, it's time to do our thing.
          @pool = nil

          # We retry the connection indefinitely here.  The system isn't
          # going to function until the pool is restored, so either return
          # successful or don't return at all.
          begin
            connect
          rescue
            Log.warn("DB connection failure on reconnect: #{$!}.  Retrying indefinitely...")
            sleep 1
            retry
          end
        end
      end

      retry_count += 1
      retry
    else
      raise e
    end
  end
end