splitNSxPauses: splits files according to available memory

.gitignore

@@ -0,0 +1,2 @@
+
+*.asv

NPMK/NEV Utilities/ExtractInitialRecordingSegmentSpikes.m

@@ -0,0 +1,22 @@
+function NEVData=ExtractInitialRecordingSegmentSpikes
+
+[fname,dname]=uigetfile({'*.nev;*.ns*','BR Formats';...
+    '*.*','All Files' },'Select data file');
+
+NSxInfo = openNSx('noread', [dname fname(1:end-3) 'ns6']);
+NEVData = openNEV('nosave',[dname fname(1:end-3) 'nev']);
+
+initialSectionTimeIdx=[find(NEVData.Data.Spikes.TimeStamp>=NSxInfo.MetaTags.Timestamp(1),1),...
+    find(NEVData.Data.Spikes.TimeStamp<=NSxInfo.MetaTags.Timestamp(2),1,'last')];
+
+NEVData.MetaTags.DataDuration=diff(NEVData.Data.Spikes.TimeStamp(initialSectionTimeIdx))+1;
+NEVData.MetaTags.DataDurationSec=NEVData.MetaTags.DataDuration/NEVData.MetaTags.SampleRes;
+
+disp(['initial recording segment lasted ' num2str(NEVData.MetaTags.DataDurationSec/60) 'mn']);
+
+NEVData.Data.Spikes.TimeStamp=NEVData.Data.Spikes.TimeStamp(initialSectionTimeIdx(1):initialSectionTimeIdx(2));
+NEVData.Data.Spikes.Electrode=NEVData.Data.Spikes.Electrode(initialSectionTimeIdx(1):initialSectionTimeIdx(2));
+NEVData.Data.Spikes.Unit=NEVData.Data.Spikes.Unit(initialSectionTimeIdx(1):initialSectionTimeIdx(2));
+NEVData.Data.Spikes.Waveform=NEVData.Data.Spikes.Waveform(:,initialSectionTimeIdx(1):initialSectionTimeIdx(2));
+
+saveNEV(NEVData, [dname fname(1:end-4) '_InitialRecordingSegment.nev'], 'noreport')

NPMK/NEV Utilities/saveNEV.m

@@ -105,25 +105,23 @@ function saveNEV(NEV, varargin)
 
 % Validating and opening the writable file
 if exist(FilePath)
-
-        if exist(FilePath)
-
-                disp('File already exists!');
-                OverwritePrompt = input('Would you like to overwrite? (Y/N)','s');
-                if strcmpi(OverwritePrompt,'y')
-                    Overwrite = 1;
-                    delete(FilePath);
-                else
-                    while exist(FilePath)
-                        ExistCount = ExistCount + 1;
-                        FilePath = fullfile(NEV.MetaTags.FilePath,[NEV.MetaTags.Filename '-Aligned-',num2str(ExistCount),NEV.MetaTags.FileExt]);
-                    end
-
-                end
-
+
+    disp('File already exists!');
+    OverwritePrompt = input('Would you like to overwrite? (Y/N)','s');
+    if strcmpi(OverwritePrompt,'y')
+        Overwrite = 1;
+        delete(FilePath);
+    else
+        while exist(FilePath)
+            ExistCount = ExistCount + 1;
+            FilePath = fullfile(NEV.MetaTags.FilePath,[NEV.MetaTags.Filename '-Aligned-',num2str(ExistCount),NEV.MetaTags.FileExt]);
         end
+
+    end
+
 end
 
+
 FileID = fopen(FilePath, 'w', 'ieee-le');
 
 if (FileID <= 0)
@@ -394,7 +392,7 @@ function saveNEV(NEV, varargin)
 
         fwrite(FileID, NEV.Data.Spikes.Waveform(:,IDX)','int16');
 
-
+        fwrite(FileID, zeros(BytesInPackets-(ftell(FileID)-Before),1),'uint8');
 
         %for Value = 1:SpikeLength
         %    fwrite(FileID, NEV.Data.Spikes.Waveform(Value,IDX),'int16');

NPMK/NSx Utilities/splitNSxPauses.m

@@ -1,18 +1,18 @@
-function splitNSxPauses(fileName)
+function segmentBytes=splitNSxPauses(fileName)
 
 % splitNSxPauses
 % 
 % Opens and splits an NSx file in smaller pieces, timewise.
 %
-% Use splitNSx(fileName)
+% Use splitNSxPauses(fileName)
 % 
 % All input arguments are optional. Input arguments can be in any order.
 %
 %   fileName:   File name of the file that needs to be split.
 %               DEFAULT: The user will be prompted to select a file.
 %
 %   Example 1: 
-%   splitNSx('C:\Datafolder\mydata.ns5');
+%   splitNSxPauses('C:\Datafolder\mydata.ns5');
 %
 %   In the example above, the file C:\Datafolder\mydata.ns5 will be opened.
 %   The loaded file will be split in samller files representing its paused 
@@ -29,7 +29,12 @@ function splitNSxPauses(fileName)
 % 1.0.0.0: August 31, 2016
 %   - Initial release.
 %   - Successor to separateNSxPaused running much more memory efficient.
-%
+% 1.1.0.0: Septembre 1, 2016 - VP
+%   - adjusts splitCount according to memory available, defines new segmentBytes
+%   - splits segmentBytes to work around loading stalls
+%   - loads segments as char, not doubles
+% 1.1.1.0: Septembre 2, 2016 - VP
+%   - adds segmentBytes output
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 %% Getting the file name
@@ -87,14 +92,36 @@ function splitNSxPauses(fileName)
     positionEOD = ftell(FID);
     % Calculating channelCount, data Length
     channelCount = typecast(basicHeader(311:314), 'uint32');
-    dataLength = positionEOD - positionEOE - 9;
+%     dataLength = positionEOD - positionEOE - 9;
     % Reading the number of packets
     fseek(FID, 28, 'bof');
-    numOfPackets = (dataLength)/(2*channelCount);
+%     numOfPackets = (dataLength)/(2*channelCount);
     % Calculating the number of splits
     splitCount = length(NSx.MetaTags.Timestamp);
     % Calculating the number of bytes in each segment
     segmentBytes = NSx.MetaTags.DataPoints * 2 * double(channelCount);
+    % see if available memory can accomodate that 
+    mem=memory;tooBigToLoad=segmentBytes/(0.5*mem.MemAvailableAllArrays)>1; %if > 50% available memory , to be on the safe side
+    if sum(tooBigToLoad) 
+        %then split segment(s)
+        splitIdx=find(tooBigToLoad);
+        newSegments=cell(length(segmentBytes),1);
+        for splitNum=1:length(segmentBytes)
+            if find(splitNum==splitIdx)
+            divideBy=ceil(segmentBytes(splitNum)/(0.25*mem.MemAvailableAllArrays)); %(0.5/2 bytes)
+            newSegments{splitNum}=zeros(1,divideBy);
+            newSegments{splitNum}(1:divideBy)=deal(floor(segmentBytes(splitNum)/divideBy));
+            if mod(segmentBytes(splitNum),divideBy)
+                newSegments{splitNum}(1)=newSegments{splitNum}(1)+mod(segmentBytes(splitNum),divideBy);
+            end
+            else
+                newSegments{splitNum}=segmentBytes(splitNum);
+            end
+        end
+        segmentBytes=[newSegments{:}];
+        splitCount=length(segmentBytes);
+    end
+
     % Reading the headers and the data header
     fseek(FID, 0, 'bof');
     fileHeader = fread(FID, positionEOE, 'char');
@@ -107,7 +134,16 @@ function splitNSxPauses(fileName)
         fprintf('\nReading segment %d... ', idx);
         % Reading the segment
         fseek(FID, 9, 'cof'); % Skipping the data header
-        dataSegment = fread(FID, segmentBytes(idx), 'char');
+        subdiv=ceil(segmentBytes(idx)/(4*10^8)); %need to subdivide, Matlab's loading of char into memory stalls for big chunks, for some reason
+        dataSegment=cell(subdiv,1);
+        for segbit=1:subdiv
+            if segbit==1 && mod(segmentBytes(idx),subdiv)>0
+                dataSegment{segbit} = fread(FID, floor(segmentBytes(idx)/subdiv)+mod(segmentBytes(idx),subdiv), '*char');
+            else
+                dataSegment{segbit} = fread(FID, segmentBytes(idx)/subdiv, '*char');
+            end
+        end
+%         dataSegment=vertcat(dataSegment{:});
         fprintf('Writing segment %d... ', idx);
         % Writing the segmented data into file
         fwrite(FIDw, fileHeader, 'char');
@@ -117,7 +153,7 @@ function splitNSxPauses(fileName)
             dataHeader(2:5) = 0;
         end
         fwrite(FIDw, dataHeader, 'char');
-        fwrite(FIDw, dataSegment, 'char');
+        fwrite(FIDw, vertcat(dataSegment{:}));
         % Clearing variables and closing file
         clear dataSegment;
         fclose(FIDw);

NPMK/NTrode Utilities/splitNEVNTrode.m

@@ -50,12 +50,7 @@
 splitCount = splitCount(splitCount>0);
 
 % Getting the file name
-
-if ~ismac
-    [fname, path] = getFile('*.nev', 'Choose an NEV file...');
-else
-    [fname, path] = getFile('*.nev', 'Choose an NEV file...');
-end
+[fname, path] = getFile('*.nev', 'Choose a NEV file...');
 
 if fname == 0
     disp('No file was selected.');

NPMK/openNSx.m

@@ -523,13 +523,14 @@
     NSx.MetaTags.Comment       = char(BasicHeader(23:278))';
     NSx.MetaTags.TimeRes       = double(typecast(BasicHeader(283:286), 'uint32'));
     NSx.MetaTags.SamplingFreq  = NSx.MetaTags.TimeRes / double(typecast(BasicHeader(279:282), 'uint32'));
-    t                          = double(typecast(BasicHeader(287:302), 'uint16'));
+    %     t                          = double(typecast(BasicHeader(287:302), 'uint16'));
+    t                          = dir(fileFullPath);
     ChannelCount               = double(typecast(BasicHeader(303:306), 'uint32'));
     NSx.MetaTags.ChannelCount  = ChannelCount;
     readSize                   = double(ChannelCount * ExtHeaderLength);
     ExtendedHeader             = fread(FID, readSize, '*uint8');
     if strcmpi(NSx.MetaTags.FileTypeID, 'NEURALCD')
-    	timeStampBytes = 4;
+        timeStampBytes = 4;
     elseif strcmpi(NSx.MetaTags.FileTypeID, 'BRSMPGRP')
         timeStampBytes = 8;
     end
@@ -577,8 +578,9 @@
 	end
 	clear ExtendedHeader;
 	%% Parsing and validating FileSpec and DateTime variables
-	NSx.MetaTags.DateTimeRaw = t.';
-	NSx.MetaTags.DateTime = datestr(datenum(t(1), t(2), t(4), t(5), t(6), t(7)));
+% 	NSx.MetaTags.DateTimeRaw = t.';
+% 	NSx.MetaTags.DateTime = datestr(datenum(t(1), t(2), t(4), t(5), t(6), t(7)));
+    NSx.MetaTags.DateTime      = t.date;
 	clear t;
 else
     disp('This version of openNSx can only read File Specs 2.1, 2.2, 2.3, and 3.0.');
@@ -597,9 +599,9 @@
 % Read Raw Header for saveNSx
 fseek(FID, 0, 'bof');
 NSx.RawData.Headers = fread(FID, f.EOexH, '*uint8');
-% if strcmpi(NSx.MetaTags.FileTypeID, 'NEURALCD')
-NSx.RawData.DataHeader = fread(FID, timeStampBytes+5, '*uint8');
-% end
+if strcmpi(NSx.MetaTags.FileTypeID, 'NEURALCD')
+    NSx.RawData.DataHeader = fread(FID, timeStampBytes+5, '*uint8');
+end
 fseek(FID, f.EOexH, 'bof');
 
 %% Reading all data headers and calculating all the file pointers for data