Merge pull request #327 from datajoint/external-storage

Add external storage, dj.config, and more

Author: Chris Turner

+dj/+config/load.m

@@ -0,0 +1,18 @@
+function load(fname)
+    % LOAD(fname)
+    %   Description:
+    %     Updates the setting from config file in JSON format.
+    %   Inputs:
+    %     fname[optional, default=dj.internal.Settings.LOCALFILE]: (string) Config file path.
+    %   Examples:
+    %     dj.config.load('/path/to/dj_local_conf.json')
+    %     dj.config.load
+    switch nargin
+        case 0
+            dj.internal.Settings.load();
+        case 1
+            dj.internal.Settings.load(fname);
+        otherwise
+        error('Exceeded 1 input limit.');
+    end
+end

+dj/+config/restore.m

@@ -0,0 +1,8 @@
+function out = restore()
+    % RESTORE()
+    %   Description:
+    %     Restores the configuration to initial default.
+    %   Examples:
+    %     dj.config.restore
+    out = dj.internal.Settings.restore();
+end

+dj/+config/save.m

@@ -0,0 +1,15 @@
+function save(fname)
+    % SAVE(fname)
+    %   Description:
+    %     Saves the settings in JSON format to the given file path.
+    %   Inputs:
+    %     fname[required]: (string) Config file path.
+    %   Examples:
+    %     dj.config.save('/path/to/dj_local_conf.json')
+    switch nargin
+        case 1
+            dj.internal.Settings.save(fname);
+        otherwise
+        error('Requires 1 input.');
+    end
+end

+dj/+config/saveGlobal.m

@@ -0,0 +1,8 @@
+function saveGlobal()
+    % SAVEGLOBAL()
+    %   Description:
+    %     Saves the settings in the global config file.
+    %   Examples:
+    %     dj.config.saveGlobal
+    dj.internal.Settings.saveGlobal();
+end 

+dj/+config/saveLocal.m

@@ -0,0 +1,8 @@
+function saveLocal()
+    % SAVELOCAL()
+    %   Description:
+    %     Saves the settings in the local config file.
+    %   Examples:
+    %     dj.config.saveLocal
+    dj.internal.Settings.saveLocal();
+end

+dj/+internal/AutoPopulate.m

@@ -111,12 +111,12 @@
             %
             % See also dj.internal.AutoPopulate/parpopulate
 
-            if ~dj.set('populateAncestors')
+            if ~dj.config('queryPopulate_ancestors')
                 rels = {self};
             else
                 % get all ancestors to be populated before self
                 assert(nargout==0, ...
-                    'parpopulate cannot return output when populateAncestors is true')
+                    'parpopulate cannot return output when queryPopulate_ancestors is true')
                 rels = cellfun(@feval, self.ancestors, 'uni', false);
                 rels = rels(cellfun(@(x) isa(x,'dj.internal.AutoPopulate'), rels));
             end
@@ -149,8 +149,10 @@ function parpopulate(self, varargin)
             %   key=null          : blob                  # structure containing the key
             %   error_message=""  : varchar(1023)         # error message returned if failed
             %   error_stack=null  : blob                  # error stack if failed
+            %   user=""           : varchar(255)          # database user
             %   host=""           : varchar(255)          # system hostname
             %   pid=0             : int unsigned          # system process id
+            %   connection_id=0   : bigint unsigned       # database connection id
             %   timestamp=CURRENT_TIMESTAMP : timestamp    # automatic timestamp
             %
             % A job is considered to be available when <package>.Jobs contains
@@ -171,7 +173,7 @@ function parpopulate(self, varargin)
             %
             % See also dj.internal.AutoPopulate/populate
 
-            if ~dj.set('populateAncestors')
+            if ~dj.config('queryPopulate_ancestors')
                 rels = {self};
             else
                 % get all ancestors to be populated before self
@@ -375,7 +377,7 @@ function cleanup(self, key)
                                 success = false;
                             end
                         end
-                        if ~success && dj.set('verbose')
+                        if ~success && strcmpi(dj.config('loglevel'), 'DEBUG')
                             fprintf('** %s: skipping already reserved\n', self.className)
                             disp(key)
                         end
@@ -390,8 +392,10 @@ function cleanup(self, key)
                     catch
                         [~,host] = system('hostname');
                     end
+                    jobKey.user = self.schema.conn.user;
                     jobKey.host = strtrim(host);
                     jobKey.pid = feature('getpid');
+                    jobKey.connection_id = self.schema.conn.serverId;
                 end
                 if ismember('error_key', self.jobs.header.names)
                     % for backward compatibility with versions prior to 2.6.3
@@ -420,8 +424,10 @@ function createJobTable(self)
             fprintf(f, 'key=null           : blob                     # structure containing the key\n');
             fprintf(f, 'error_message=""   : varchar(1023)            # error message returned if failed\n');
             fprintf(f, 'error_stack=null   : blob                     # error stack if failed\n');
+            fprintf(f, 'user=""            : varchar(255)             # database user\n');            
             fprintf(f, 'host=""            : varchar(255)             # system hostname\n');
             fprintf(f, 'pid=0              : int unsigned             # system process id\n');
+            fprintf(f, 'connection_id=0    : bigint unsigned          # database connection id\n');
             fprintf(f, 'timestamp=CURRENT_TIMESTAMP : timestamp       # automatic timestamp\n');
             fprintf(f, '%%}\n\n');
             fprintf(f, 'classdef Jobs < dj.Jobs\n');
@@ -433,8 +439,8 @@ function createJobTable(self)
         function populateSanityChecks(self)
             % Performs sanity checks that are common to populate,
             % parpopulate and batch_populate.
-            % To disable the sanity check: dj.set('populateCheck',false)
-            if dj.set('populateCheck')
+            % To disable the sanity check: dj.config('queryPopulate_check',false)
+            if dj.config('queryPopulate_check')
                 source = self.getKeySource;
                 abovePopRel = setdiff(self.primaryKey(1:min(end,length(source.primaryKey))), source.primaryKey);
                 if ~all(ismember(source.primaryKey, self.primaryKey))