Skip to content

Commit 1c4dadc

Browse files
lawrence-mbflawrence-mbf
authored
Merge pull request #267 from NeurodataWithoutBorders/getrow-table
getRow/addRow table support
2 parents 00758c6 + f31129c commit 1c4dadc

File tree

10 files changed

+358
-227
lines changed

10 files changed

+358
-227
lines changed

‎+tests/+system/DynamicTableTest.m‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,23 +6,21 @@ function addContainer(~, file)
66
'description', 'test dynamic table column',...
77
'colnames', colnames);
88

9+
id = primes(2000) .';
910
for i = 1:100
1011
start_time = i;
1112
stop_time = i + 1;
1213
rand_data = rand(5,1);
13-
id = primes(i);
14-
if isempty(id)
15-
id = 0;
16-
else
17-
id = id(end);
18-
end
1914
file.intervals_trials.addRow(...
2015
'start_time', start_time,...
2116
'stop_time', stop_time,...
2217
'randomvalues', rand_data,...
23-
'id', id,...
18+
'id', id(i),...
2419
'tablepath', '/intervals/trials');
2520
end
21+
t = table(id(101:200), (101:200) .', (102:201) .', mat2cell(rand(500,1), repmat(5, 100, 1)),...
22+
'VariableNames', {'id', 'start_time', 'stop_time', 'randomvalues'});
23+
file.intervals_trials.addRow(t);
2624
end
2725

2826
function c = getContainer(~, file)
@@ -31,12 +29,12 @@ function addContainer(~, file)
3129

3230
function appendContainer(testCase, file)
3331
container = testCase.getContainer(file);
34-
container.data = rand(500, 1); % new random values.
32+
container.data = rand(1000, 1); % new random values.
3533
file.intervals_trials.colnames{end+1} = 'newcolumn';
3634
file.intervals_trials.vectordata.set('newcolumn',...
3735
types.hdmf_common.VectorData(...
3836
'description', 'newly added column',...
39-
'data', 100:-1:1));
37+
'data', 200:-1:1));
4038
end
4139
end
4240

@@ -48,8 +46,9 @@ function getRowTest(testCase)
4846
ActualTable = ActualFile.intervals_trials;
4947
ExpectedTable = testCase.file.intervals_trials;
5048
testCase.verifyEqual(ExpectedTable.getRow(5), ActualTable.getRow(5));
51-
testCase.verifyEqual(ExpectedTable.getRow(97, 'useId', true),...
52-
ActualTable.getRow(97, 'useId', true));
49+
testCase.verifyEqual(ExpectedTable.getRow([5 6]), ActualTable.getRow([5 6]));
50+
testCase.verifyEqual(ExpectedTable.getRow([1153, 1217], 'useId', true),...
51+
ActualTable.getRow([1153, 1217], 'useId', true));
5352
end
5453
end
5554
end

‎+types/+util/+dynamictable/addRawData.m‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
function addRawData(DynamicTable, column, data, index)
2+
%ADDRAWDATA Internal method for adding data to DynamicTable given column
3+
% name, data and an optional index.
4+
validateattributes(column, {'char'}, {'scalartext'});
5+
if nargin < 4
6+
% indicates an index column. Note we assume that the index name is correct.
7+
% Validation of this index name must occur upstream.
8+
index = '';
9+
end
10+
11+
% Don't set the data until after indices are updated.
12+
VecData = types.hdmf_common.VectorData(...
13+
'description', sprintf('AUTOGENERATED description for column `%s`', column),...
14+
'data', []);
15+
if isprop(DynamicTable, column)
16+
if isempty(DynamicTable.(column))
17+
DynamicTable.(column) = VecData;
18+
end
19+
VecData = DynamicTable.(column);
20+
elseif isKey(DynamicTable.vectordata, column)
21+
VecData = DynamicTable.vectordata.get(column);
22+
else
23+
DynamicTable.vectordata.set(column, VecData);
24+
end
25+
26+
if ~isempty(index)
27+
if isprop(DynamicTable, index)
28+
VecInd = DynamicTable.(index);
29+
else
30+
VecInd = DynamicTable.vectorindex.get(index);
31+
end
32+
33+
if isa(VecInd.data, 'types.untyped.DataPipe')
34+
if 0 == VecInd.data.dims
35+
raggedOffset = 0;
36+
else
37+
raggedOffset = VecInd.data.load(VecInd.data.dims);
38+
end
39+
else
40+
if isempty(VecInd.data)
41+
raggedOffset = 0;
42+
else
43+
raggedOffset = VecInd.data(end);
44+
end
45+
end
46+
47+
raggedValue = raggedOffset + size(data, 1);
48+
if isa(VecInd.data, 'types.untyped.DataPipe')
49+
VecInd.data.append(raggedValue);
50+
else
51+
VecInd.data = [VecInd.data; raggedValue];
52+
end
53+
end
54+
55+
if isa(VecData.data, 'types.untyped.DataPipe')
56+
VecData.data.append(data);
57+
else
58+
if ischar(data)
59+
data = {data};
60+
end
61+
VecData.data = [VecData.data; data];
62+
end
63+
end

‎+types/+util/+dynamictable/addRow.m‎

Lines changed: 23 additions & 212 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,28 @@
11
function addRow(DynamicTable, varargin)
2-
%ADDROW Given a dynamic table and a set of keyword arguments for the row,
3-
% add a single row to the dynamic table if possible.
2+
% ADDROW Given a dynamic table and a set of keyword arguments for the row,
3+
% add a single row to the dynamic table if using keywords, or multiple rows
4+
% if using a table.
5+
%
6+
% ADDROW(DT,table) append the MATLAB table to the DynamicTable
7+
%
8+
% ADDROW(DT,col1,val1,col2,val2,...,coln,valn) append a single row
9+
% to the DynamicTable
10+
%
11+
% ADDROW(DT,___,Name,Value) specify either 'id' or 'tablepath'
12+
%
413
% This function asserts the following:
514
% 1) DynamicTable is a valid dynamic table and has the correct
615
% properties.
7-
% 2) varargin is a set of keyword arguments (in MATLAB, this is a character
8-
% array indicating name and a value indicating the row value).
9-
% 3) The given keyword argument names match one of those ALREADY specified
16+
% 2) The given keyword argument names match one of those ALREADY specified
1017
% by the DynamicTable (that is, colnames MUST be filled out).
11-
% 4) If the dynamic table is non-empty, the types of the column value MUST
18+
% 3) If the dynamic table is non-empty, the types of the column value MUST
1219
% match the keyword value.
13-
% 5) All horizontal data must match the width of the rest of the rows.
20+
% 4) All horizontal data must match the width of the rest of the rows.
1421
% Variable length strings should use cell arrays each row.
15-
% 6) The type of the data cannot be a cell array of numeric values.
16-
% 7) Ragged arrays (that is, rows containing more than one sub-row) require
22+
% 5) The type of the data cannot be a cell array of numeric values if using
23+
% keyword arguments. For table appending mode, this is how ragged arrays
24+
% are represented.
25+
% 6) Ragged arrays (that is, rows containing more than one sub-row) require
1726
% an extra parameter called `tablepath` which indicates where in the NWB
1827
% file the table is.
1928

@@ -22,223 +31,25 @@ function addRow(DynamicTable, varargin)
2231
'MatNWB:DynamicTable:AddRow:NoColumns',...
2332
['The `colnames` property of the Dynamic Table needs to be populated with a cell array '...
2433
'of column names before being able to add row data.']);
25-
p = inputParser();
26-
p.KeepUnmatched = true;
27-
p.StructExpand = false;
28-
addParameter(p, 'tablepath', '', @(x)ischar(x)); % required for ragged arrays.
29-
addParameter(p, 'id', []); % `id` override but doesn't actually show up in `colnames`
30-
for i = 1:length(DynamicTable.colnames)
31-
addParameter(p, DynamicTable.colnames{i}, []);
32-
end
33-
parse(p, varargin{:});
34-
assert(isempty(fieldnames(p.Unmatched)),...
35-
'MatNWB:DynamicTable:AddRow:InvalidColumns',...
36-
'Invalid column name(s) { %s }', strjoin(fieldnames(p.Unmatched), ', '));
34+
assert(nargin > 1, 'MatNWB:DynamicTable:AddRow:NoData', 'Not enough arguments');
3735

3836
if isempty(DynamicTable.id)
3937
DynamicTable.id = types.hdmf_common.ElementIdentifiers();
4038
end
39+
4140
assert(~isa(DynamicTable.id.data, 'types.untyped.DataStub'),...
4241
'MatNWB:DynamicTable:AddRow:Uneditable',...
4342
['Cannot write to on-file Dynamic Tables without enabling data pipes. '...
4443
'If this was produced with pynwb, please enable chunking for this table.']);
45-
rowNames = fieldnames(p.Results);
46-
47-
% not using setDiff because we want to retain set order.
48-
rowNames(strcmp(rowNames, 'tablepath') | strcmp(rowNames, 'id')) = [];
49-
50-
missingColumns = setdiff(p.UsingDefaults, {'tablepath', 'id'});
51-
assert(isempty(missingColumns),...
52-
'MatNWB:DynamicTable:AddRow:MissingColumns',...
53-
'Missing columns { %s }', strjoin(missingColumns, ', '));
5444

55-
specifiesId = ~any(strcmp(p.UsingDefaults, 'id'));
56-
if specifiesId
57-
validateattributes(p.Results.id, {'numeric'}, {'scalar'});
58-
end
59-
60-
TypeMap = constructTypeMap(DynamicTable);
61-
for i = 1:length(rowNames)
62-
rn = rowNames{i};
63-
rv = p.Results.(rn);
64-
65-
if isKey(TypeMap, rn)
66-
rv = validateType(TypeMap(rn), rv);
67-
else
68-
assert(iscellstr(rv) || ~iscell(rv),...
69-
'MatNWB:DynamicTable:AddRow:InvalidCellArray',...
70-
'Cell arrays that are not cell strings are not allowed.');
71-
end
72-
73-
% instantiate vector index here because it's dependent on the table
74-
% fullpath.
75-
vecIndName = types.util.dynamictable.getIndex(DynamicTable, rn);
76-
if isempty(vecIndName) && size(rv, 1) > 1
77-
assert(~isempty(p.Results.tablepath),...
78-
'MatNWB:DynamicTable:AddRow:MissingTablePath',...
79-
['addRow cannot create ragged arrays without a full HDF5 path to the Dynamic Table. '...
80-
'Please either add the full expected HDF5 path under the keyword argument `tablepath` '...
81-
'or call addRow with row data only.']);
82-
vecIndName = [rn '_index']; % arbitrary convention of appending '_index' to data column names
83-
if endsWith(p.Results.tablepath, '/')
84-
tablePath = p.Results.tablepath;
85-
else
86-
tablePath = [p.Results.tablepath '/'];
87-
end
88-
vecTarget = types.untyped.ObjectView([tablePath rn]);
89-
oldDataHeight = 0;
90-
if isKey(DynamicTable.vectordata, rn) || isprop(DynamicTable, rn)
91-
if isprop(DynamicTable, rn)
92-
VecData = DynamicTable.(rn);
93-
else
94-
VecData = DynamicTable.vectordata.get(rn);
95-
end
96-
if isa(VecData.data, 'types.untyped.DataPipe')
97-
oldDataHeight = VecData.data.offset;
98-
else
99-
oldDataHeight = size(VecData.data, 1);
100-
end
101-
end
102-
103-
% we presume that if data already existed in the vectordata, then
104-
% it was never a ragged array and thus its elements corresponded
105-
% directly to each row index.
106-
VecIndex = types.hdmf_common.VectorIndex(...
107-
'target', vecTarget,...
108-
'data', [0:(oldDataHeight-1)] .'); %#ok<NBRAK>
109-
if isprop(DynamicTable, vecIndName)
110-
DynamicTable.(vecIndName) = VecIndex;
111-
else
112-
DynamicTable.vectorindex.set(vecIndName, VecIndex);
113-
end
114-
end
115-
appendData(DynamicTable, rn, rv, vecIndName);
116-
end
117-
118-
if specifiesId
119-
newId = p.Results.id;
120-
elseif isa(DynamicTable.id.data, 'types.untyped.DataPipe')
121-
newId = DynamicTable.id.data.offset;
122-
DynamicTable.id.data.append(DynamicTable.id.data.offset);
123-
else
124-
newId = length(DynamicTable.id.data);
125-
end
126-
127-
if isa(DynamicTable.id.data, 'types.untyped.DataPipe')
128-
DynamicTable.id.data.append(newId);
45+
if istable(varargin{1})
46+
types.util.dynamictable.addTableRow(DynamicTable, varargin{:});
12947
else
130-
DynamicTable.id.data = [DynamicTable.id.data; newId];
48+
types.util.dynamictable.addVarargRow(DynamicTable, varargin{:});
13149
end
13250
end
13351

134-
function rv = validateType(TypeStruct, rv)
135-
if strcmp(TypeStruct.type, 'cellstr')
136-
assert(iscellstr(rv) || (ischar(rv) && 1 == size(rv, 1)),...
137-
'MatNWB:DynamicTable:AddRow:InvalidType',...
138-
'Type of value must be a cell array of character vectors or a scalar character');
139-
if ischar(rv)
140-
rv = {rv};
141-
end
142-
else
143-
validateattributes(rv, {TypeStruct.type}, {'size', [NaN TypeStruct.dims(2:end)]});
144-
end
145-
end
146-
147-
function TypeMap = constructTypeMap(DynamicTable)
148-
TypeMap = containers.Map;
149-
if isempty(DynamicTable.id.data)...
150-
|| (isa(DynamicTable.id.data, 'types.untyped.DataPipe')...
151-
&& 0 == DynamicTable.id.data.offset)
152-
return;
153-
end
154-
TypeStruct = struct('type', '', 'dims', [0, 0]);
155-
for i = length(DynamicTable.colnames)
156-
colnm = DynamicTable.colnames{i};
157-
if isprop(DynamicTable, colnm)
158-
colVecData = DynamicTable.(colnm);
159-
else
160-
colVecData = DynamicTable.vectordata.get(colnm);
161-
end
162-
if isa(colVecData.data, 'types.untyped.DataPipe')
163-
colval = colVecData.data.load(1);
164-
else
165-
colval = colVecData.data(1);
166-
end
167-
168-
if iscellstr(colval)
169-
TypeStruct.type = 'cellstr';
170-
else
171-
TypeStruct.type = class(colval);
172-
end
173-
174-
if isa(colVecData.data, 'types.untyped.DataPipe')
175-
TypeStruct.dims = colVecData.data.internal.maxSize;
176-
else
177-
TypeStruct.dims = size(colVecData.data);
178-
end
179-
180-
TypeMap(colnm) = TypeStruct;
181-
end
182-
end
18352

184-
function appendData(DynamicTable, column, data, index)
185-
validateattributes(column, {'char'}, {'scalartext'});
186-
if nargin < 4
187-
% indicates an index column. Note we assume that the index name is correct.
188-
% Validation of this index name must occur upstream.
189-
index = '';
190-
end
19153

192-
% Don't set the data until after indices are updated.
193-
VecData = types.hdmf_common.VectorData(...
194-
'description', sprintf('AUTOGENERATED description for column `%s`', column),...
195-
'data', []);
196-
if isprop(DynamicTable, column)
197-
if isempty(DynamicTable.(column))
198-
DynamicTable.(column) = VecData;
199-
end
200-
VecData = DynamicTable.(column);
201-
elseif isKey(DynamicTable.vectordata, column)
202-
VecData = DynamicTable.vectordata.get(column);
203-
else
204-
DynamicTable.vectordata.set(column, VecData);
205-
end
20654

207-
if ~isempty(index)
208-
if isprop(DynamicTable, index)
209-
VecInd = DynamicTable.(index);
210-
else
211-
VecInd = DynamicTable.vectorindex.get(index);
212-
end
213-
214-
if isa(VecInd.data, 'types.untyped.DataPipe')
215-
if 0 == VecInd.data.dims
216-
raggedOffset = 0;
217-
else
218-
raggedOffset = VecInd.data.load(VecInd.data.dims);
219-
end
220-
else
221-
if isempty(VecInd.data)
222-
raggedOffset = 0;
223-
else
224-
raggedOffset = VecInd.data(end);
225-
end
226-
end
227-
228-
raggedValue = raggedOffset + size(data, 1);
229-
if isa(VecInd.data, 'types.untyped.DataPipe')
230-
VecInd.data.append(raggedValue);
231-
else
232-
VecInd.data = [VecInd.data; raggedValue];
233-
end
234-
end
23555

236-
if isa(VecData.data, 'types.untyped.DataPipe')
237-
VecData.data.append(data);
238-
else
239-
if ischar(data)
240-
data = {data};
241-
end
242-
VecData.data = [VecData.data; data];
243-
end
244-
end

0 commit comments

Comments
 (0)