-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathhelpers.rb
More file actions
217 lines (186 loc) · 5.49 KB
/
Copy pathhelpers.rb
File metadata and controls
217 lines (186 loc) · 5.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
# frozen_string_literal: true
require "kiba/extend"
module Helpers
module_function
def fixtures_dir = File.join(Bundler.root, "spec", "support", "fixtures")
def tmp_dir = File.join(Bundler.root, "spec", "tmp")
class TestJob
include Kiba::Extend::Jobs::Parser
attr_reader :control, :context, :accumulator
def initialize(input:, accumulator:, transforms:)
@accumulator = accumulator
@control = Kiba.parse do
source Kiba::Extend::Sources::Enumerable, input
destination Kiba::Extend::Destinations::Lambda,
on_write: ->(r) { accumulator << r }
end
@context = Kiba::Context.new(control)
parse_job(control, context, [transforms])
Kiba.run(control)
end
end
def marc_file
File.join(fixtures_dir, "harvard_open_data.mrc")
end
# @param path [String] path to MARC binary file (.mrc, .dat, etc)
# @param index [Integer] which record from file you want to use in your
# test. Count starts at 0.
def get_marc_record(index:, path: marc_file)
recs = []
MARC::Reader.new(path).each { |rec| recs << rec }
recs[index]
end
def populate_registry(more_entries: {})
fkeypath = File.join(fixtures_dir, "existing.csv")
nofilepath = File.join(fixtures_dir, "not_here.csv")
entries = {
fkey: {path: fkeypath, supplied: true, lookup_on: :id},
invalid: {},
fee: {path: fkeypath, lookup_on: :foo, supplied: true},
foo: {
path: fkeypath,
creator: Helpers.method(:test_csv),
tags: %i[test]
},
bar: {
path: fkeypath,
creator: Helpers.method(:lookup_csv),
tags: %i[test report]
},
baz: {
path: fkeypath,
creator: Kiba::Extend::Utils::Lookup.method(:csv_to_hash),
tags: %i[report]
},
warn: {
path: fkeypath,
dest_class: Kiba::Extend::Destinations::CSV,
creator: Kiba::Extend.method(:csvopts),
dest_special_opts: {initial_headers: %i[objectnumber
briefdescription]}
},
json_arr: {
path: File.join(fixtures_dir, "output.json"),
dest_class: Kiba::Extend::Destinations::JsonArray,
creator: Helpers.method(:fake_creator_method)
},
noresultfile: {
path: nofilepath,
creator: Helpers.method(:fake_creator_with_no_job_output)
},
resultfile: {
path: nofilepath,
creator: Helpers.method(:fake_creator_with_job_output)
}
}.merge(more_entries)
entries.each { |key, data| Kiba::Extend.registry.register(key, data) }
Kiba::Extend.registry.namespace(:ns) do
namespace(:sub) do
register(:fkey, {path: fkeypath, supplied: true})
end
end
end
def transform_registry
Kiba::Extend.registry.transform
end
def prepare_registry
populate_registry
transform_registry
end
def fake_creator_method
FileUtils.touch(File.join(fixtures_dir, "base_job_missing.csv"))
end
def dependency_graph_registry
Kiba::Extend.config.registry = Kiba::Extend::Registry::FileRegistry
extra_entries = {
b: {path: File.join(fixtures_dir, "b.csv"),
creator: B},
c: {path: File.join(fixtures_dir, "c.csv"),
creator: C},
d: {path: File.join(fixtures_dir, "d.csv"),
creator: D},
e: {path: File.join(fixtures_dir, "e.csv"),
creator: E},
f: {path: File.join(fixtures_dir, "f.csv"),
creator: F}
}
populate_registry(more_entries: extra_entries)
transform_registry
end
class OutputJob
attr_reader :outrows
def initialize(rowct)
@outrows = rowct
end
def run = nil
end
def fake_creator_with_no_job_output
OutputJob.new(0)
end
def fake_creator_with_job_output
OutputJob.new(101)
end
# for test in Kiba::Extend::Jobs::BaseJobsSpec that I can't get working
# class BaseJob
# include Kiba::Extend::Jobs::Runner
# attr_reader :files
# def initialize(files:)
# @files = setup_files(files)
# end
# def creator=(arg)
# @creator = arg
# end
# def creator
# @creator
# end
# end
def test_csv
File.join(tmp_dir, "test.csv")
end
def lookup_csv
File.join(tmp_dir, "lkup.csv")
end
def generate_csv(rows)
CSV.open(test_csv, "w") do |csv|
rows.each { |row| csv << row }
end
end
def generate_lookup_csv(rows)
CSV.open(lookup_csv, "w") do |csv|
rows.each { |row| csv << row }
end
end
def execute_job(filename:, xform:, csvopt: {}, xformopt: {})
output_rows = []
settings = {filename: filename,
csv_options: Kiba::Extend.csvopts.merge(csvopt)}
job = Kiba.parse do
source Kiba::Common::Sources::CSV, **settings
transform(&:to_h)
transform xform, **xformopt
transform { |row| output_rows << row }
end
Kiba.run(job)
output_rows
end
def job_csv(filename:, incsvopt: {}, outcsvopt: {})
insettings = {filename: filename,
csv_options: Kiba::Extend.csvopts.merge(incsvopt)}
outsettings = {filename: filename,
csv_options: Kiba::Extend.csvopts.merge(outcsvopt)}
job = Kiba.parse do
source Kiba::Common::Sources::CSV, **insettings
transform(&:to_h)
destination Kiba::Common::Destinations::CSV, **outsettings
end
Kiba.run(job)
output_rows = []
job2 = Kiba.parse do
source Kiba::Common::Sources::CSV, **insettings
transform(&:to_h)
transform { |row| output_rows << row }
end
Kiba.run(job2)
output_rows
end
end