Class: BulkImportParser

Inherits:
Object
  • Object
show all
Includes:
BulkImportMixins, URIResolver
Defined in:
backend/app/lib/bulk_import/bulk_import_parser.rb

Overview

Base class for bulk import via spreadsheet; handles both CSV’s and excel spreadsheets This class is designed for spreadsheets with the following features: 1. There may be multiple rows of headers, each of which have something in the 0th column 2. The header row containing the internal (machine-readble) labels will have in its 0th column some defined string; this string shall be used in the sub-class’s START_MARKER constant 3. Only header rows contain strings in their 0th column; data rows will have an empty 0th column. This means that there can be an arbitrary (including 0) number of header rows after the internal labels row; the code can handle it! 4. The various handler classes are now required in the bulk_import_mixins.rb file. 5. The methods that must be implemented in the sub class are process_row and log_row

Constant Summary

Constants included from JSONModel

JSONModel::REFERENCE_KEY_REGEX

Instance Method Summary collapse

Methods included from BulkImportMixins

#ao_save, #archival_object_from_ref, #archival_object_from_ref_or_uri, #archival_object_from_uri, #create_date, #created, #find_top_container, #handle_notes, #indicator_and_type_exist_for_resource?, #resolves, #resource_from_ref, #resource_match, #sub_container_from_barcode, #test_exceptions, #valid, #value_check

Methods included from CrudHelpers

#handle_create, #handle_delete, #handle_listing, #handle_raw_listing, #handle_unlimited_listing, #handle_update, scoped_dataset, with_record_conflict_reporting, #with_record_conflict_reporting

Methods included from URIResolver

add_resolve_wrapper, ensure_reference_is_valid, register_resolver, resolve_references, resolve_wrappers

Methods included from JSONModel

JSONModel, #JSONModel, add_error_handler, all, allow_unmapped_enum_value, backend_url, check_valid_refs, client_mode?, custom_validations, destroy_model, enum_default_value, enum_values, handle_error, init, load_schema, #models, models, parse_jsonmodel_ref, parse_reference, repository, repository_for, schema_src, set_publish_flags!, set_repository, strict_mode, strict_mode?, validate_schema, with_repository

Constructor Details

#initialize(input_file, content_type, current_user, opts, log_method) ⇒ BulkImportParser

MAX_FILE_SIZE = Integer(AppConfig[:bulk_import_size]) MAX_FILE_ROWS = Integer(AppConfig[:bulk_import_rows]) MAX_FILE_INFO = I18n.t(“bulk_import.max_file_info”, :rows => MAX_FILE_ROWS, :size => MAX_FILE_SIZE)



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'backend/app/lib/bulk_import/bulk_import_parser.rb', line 32

def initialize(input_file, content_type, current_user, opts, log_method)
  @created_refs = []
  @input_file = input_file
  @file_content_type = content_type
  @opts = opts
  @current_user = current_user
  @report_out = []
  @report = BulkImportReport.new
  @start_position
  @need_to_move = false
  @log_method = log_method
  @is_xslx = @file_content_type == "xlsx"
  @is_csv = @file_content_type == "csv"
  @validate_only = opts[:validate]
end

Instance Method Details

#initialize_handler_enums ⇒ Object 



104
105
106
 
# File 'backend/app/lib/bulk_import/bulk_import_parser.rb', line 104

def initialize_handler_enums
  #initialize handlers, if needed
end

#initialize_info ⇒ Object

set up all the @ variables

Raises:



109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'backend/app/lib/bulk_import/bulk_import_parser.rb', line 109

def initialize_info
  @orig_filename = @opts[:filename]
  @report_out = []
  @report = BulkImportReport.new
  @headers
  @report.set_file_name(@orig_filename)
  initialize_handler_enums
  jsonresource = Resource.to_jsonmodel(Integer(@opts[:rid]))
  @resource = resolve_references(jsonresource, ["repository"])
  @repository = @resource["repository"]["ref"]
  @hier = 1
  @counter = 0
  @rows_processed = 0
  @error_rows = 0
  raise StopBulkImportException.new(I18n.t("bulk_import.error.wrong_file_type")) if !@is_csv && !@is_xslx
  #XSLX
  if @is_xslx
    workbook = RubyXL::Parser.parse(@input_file)
    sheet = workbook[0]
    @rows = sheet.enum_for(:each)
    #CSV
  elsif @is_csv
    table = CSV.read(@input_file)
    @rows = table.enum_for(:each)
  end
  find_headers
end

#record_uris ⇒ Object 



48
49
50
 
# File 'backend/app/lib/bulk_import/bulk_import_parser.rb', line 48

def record_uris
  @created_refs
end

#run ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'backend/app/lib/bulk_import/bulk_import_parser.rb', line 52

def run
  begin
    initialize_info
    begin
      while (row = @rows.next)
        @counter += 1
        values = row_values(row)
        next if !values[0].nil?  # header rows all have something in the first column
        next if values.reject(&:nil?).empty?
        @row_hash = Hash[@headers.zip(values)]
        begin
          @report.new_row(@counter)
          process_row
          @rows_processed += 1
          @error_level = nil
        rescue StopBulkImportException => se
          @report.add_errors(I18n.t("bulk_import.error.stopped", :row => @counter, :msg => se.message))
          raise StopIteration.new
        rescue BulkImportException => e
          @error_rows += 1
          @report.add_errors(e.message)
          @error_level = @hier
        end
        current = @report.current_row
        log_row(current) unless @log_method.nil?
        @report.end_row
      end
    rescue StopIteration
      # we just want to catch this without processing further
    end
    if @rows_processed == 0
      message = I18n.t("bulk_import.error.no_data") # default message (no data)
      if @report.current_row && @report.current_row.errors.any?
        # if we have a row error message put that into the exception so it more closely matches the csv report
        message = @report.current_row.errors.first.match(/\[(.*)\]/)[1]
      end
      raise BulkImportException.new(message)
    end
  rescue Exception => e
    if e.is_a?(BulkImportException) || e.is_a?(StopBulkImportException)
      @report.add_terminal_error(I18n.t("bulk_import.error.spreadsheet", :errs => e.message), @counter)
    elsif e.is_a?(StopIteration) && @headers.nil?
      @report.add_terminal_error(I18n.t("bulk_import.error.no_header"), @counter)
    else # something else went wrong
      @report.add_terminal_error(I18n.t("bulk_import.error.system", :msg => e.message), @counter)
      Log.error("UNEXPECTED EXCEPTION on bulkimport load! #{e.message}")
      Log.error(e.backtrace.pretty_inspect)
    end
  end
  return @report
end