Class: IndexBatch

Inherits:
Object
  • Object
show all
Defined in:
indexer/app/lib/index_batch.rb

Constant Summary collapse

SEPARATORS = 
[",\n", "]\n"]

Instance Method Summary collapse

Constructor Details

#initialize ⇒ IndexBatch

Returns a new instance of IndexBatch.



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'indexer/app/lib/index_batch.rb', line 11

def initialize
  @bytes = 0
  @record_count = 0
  @closed = false

  # Store basic metadata about what's in this batch to avoid having to reparse
  # everything.
  @record_info_by_primary_type = {}

  @filestore = ASUtils.tempfile('index_batch')

  # Don't mess up our line breaks under Windows!
  @filestore.binmode

  self.write("[\n")
end

Instance Method Details

#<<(doc) ⇒ Object 



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'indexer/app/lib/index_batch.rb', line 48

def <<(doc)
  primary_type = doc['primary_type'].to_s
  unless primary_type.empty?
    @record_info_by_primary_type[primary_type] ||= []
    @record_info_by_primary_type[primary_type] << {id: doc['id']}
  end

  json = ASUtils.to_json(doc)

  if @record_count > 0
    self.write(",\n")
  end

  self.write(json)
  self.write("\n")

  @record_count += 1
end

#byte_count ⇒ Object 



104
105
106
 
# File 'indexer/app/lib/index_batch.rb', line 104

def byte_count
  @bytes
end

#close ⇒ Object 



29
30
31
32
33
34
 
# File 'indexer/app/lib/index_batch.rb', line 29

def close
  unless @closed
    @closed = true
    self.write("]\n")
  end
end

#concat(docs) ⇒ Object 



108
109
110
111
112
 
# File 'indexer/app/lib/index_batch.rb', line 108

def concat(docs)
  docs.each do |doc|
    self << doc
  end
end

#destroy ⇒ Object 



125
126
127
128
129
 
# File 'indexer/app/lib/index_batch.rb', line 125

def destroy
  self.close
  @filestore.close
  @filestore.unlink
end

#each(&block) ⇒ Object 



84
85
86
87
88
89
90
91
92
 
# File 'indexer/app/lib/index_batch.rb', line 84

def each(&block)
  self.rewind

  @filestore.each_line("\n") do |line|
    block.call(ASUtils.json_parse(line)) if !SEPARATORS.include?(line)
  end

  self
end

#empty? ⇒ Boolean

Returns:

  • (Boolean) 


115
116
117
 
# File 'indexer/app/lib/index_batch.rb', line 115

def empty?
  @record_count == 0
end

#length ⇒ Object 



120
121
122
 
# File 'indexer/app/lib/index_batch.rb', line 120

def length
  @record_count
end

#map(&block) ⇒ Object 



72
73
74
75
76
77
78
79
80
81
 
# File 'indexer/app/lib/index_batch.rb', line 72

def map(&block)
  self.rewind

  result = []
  @filestore.each_line("\n") do |line|
    result << block.call(ASUtils.json_parse(line)) if !SEPARATORS.include?(line)
  end

  result
end

#record_info_for_type(record_type) ⇒ Object 



44
45
46
 
# File 'indexer/app/lib/index_batch.rb', line 44

def record_info_for_type(record_type)
  @record_info_by_primary_type.fetch(record_type, [])
end

#rewind ⇒ Object 



67
68
69
70
 
# File 'indexer/app/lib/index_batch.rb', line 67

def rewind
  @filestore.rewind
  @filestore.readline         # skip the opening [
end

#to_json_stream ⇒ Object 



95
96
97
98
99
100
101
 
# File 'indexer/app/lib/index_batch.rb', line 95

def to_json_stream
  self.close
  @filestore.close

  # Open with "b" to avoid converting \n to \r\n on Windows
  File.open(@filestore.path, "rb")
end

#write(s) ⇒ Object 



37
38
39
40
41
 
# File 'indexer/app/lib/index_batch.rb', line 37

def write(s)
  @bytes += s.bytes.count
  @filestore.write(s)
  @filestore.flush
end