enh: refactor sqilte database to speed up while loading large dataset

CommonData/.gitignore

@@ -1,3 +1 @@
-internalDbDefinition.h
-createIndexes.h
 jaspBase_*.h

CommonData/createIndexes.h

@@ -0,0 +1,4 @@
+R"for_c++_include(
+CREATE INDEX IF NOT EXISTS ColumnOrderIdx			ON Columns	(id, dataSet,	colIdx);
+CREATE INDEX IF NOT EXISTS LabelOrderPerColumnIdx	ON Labels	(id, columnId,	ordering);
+)for_c++_include"

CommonData/databaseinterface.h

@@ -243,14 +243,52 @@ int			dataSetInsert(							const std::string & dataFilePath = "", long dataFileT
 	std::mutex                              _loadMutex,
 											_dbCheckMutex;
 
-	static			std::string _wrap_sqlite3_column_text(sqlite3_stmt * stmt, int iCol);
+	static				std::string _wrap_sqlite3_column_text(sqlite3_stmt * stmt, int iCol);
 	static const	std::string _dbConstructionSql;
 	static const	std::string _dbIndexesSql;
 
 	static DatabaseInterface * _singleton;
 
 	friend class DataSetPackage;
 
+
+	//chunk
+	static const int CHUNK_SIZE = 1000;		// Number of rows contained in each chunks
+
+	// chunk serialization/deserialization
+	static std::vector<unsigned char>			serializeIntChunk(const intvec &ints,						size_t startRow,					size_t count);
+	static std::vector<unsigned char>			serializeDoubleChunk(const doublevec &dbls,			const stringvec &strs,		size_t startRow,	size_t count);
+	static void														deserializeIntChunk(const unsigned char *data,	size_t dataSize,					intvec &ints,			size_t startRow, size_t expectedCount);
+	static void														deserializeDoubleChunk(const unsigned char *data, size_t dataSize,				doublevec &dbls, stringvec &strs, size_t startRow, size_t expectedCount);
+
+	// read/write column chunks
+	void					writeColumnChunks		(	int columnId,		const intvec		&ints);
+	void					writeColumnChunks		(	int columnId,		const doublevec &dbls,	const stringvec &strs);
+	void					readColumnChunks		(	int columnId,		intvec &ints);
+	void					readColumnChunks		(	int columnId,		doublevec &dbls,				stringvec &strs);
+	void					deleteColumnChunks	(	int columnId);
+
+	// update single value
+	void					setIntChunkValue		(	int columnId,		size_t row,			int value													);
+	void					setDoubleChunkValue	(	int columnId,		size_t row,			double val, const std::string &str);
+
+	// adjust column row count
+	void					adjustIntColumnRowCount		(	int columnId, size_t oldRowCount, size_t newRowCount);
+	void					adjustDoubleColumnRowCount(	int columnId, size_t oldRowCount, size_t newRowCount);
+
+	// helper: get IDs and types of all columns in a dataset
+	std::vector<std::pair<int, bool>> getColumnsForDataSet(int dataSetId);
+
+	bool				readChunkBlob				(	int columnId,			int chunkId,				std::vector<unsigned char> &blob);
+	void				writeChunkBlob				(	int columnId,			int chunkId,				const std::vector<unsigned char> &blob);
+	void				growIntColumnChunks	(	int columnId,			size_t oldRowCount, size_t newRowCount);
+	void				shrinkIntColumnChunks(	int columnId,			size_t newRowCount);
+
+	//	migrate these DataSet_# to DataChunks table
+	bool				hasDataChunksTable();
+	void				migrateLegacyWideTableToDataChunks();
+
+
 };
 
 #endif // DATABASEINTERFACE_H

CommonData/internalDbDefinition.h

@@ -0,0 +1,79 @@
+R"for_c++_include(
+-- This file is automatically converted to an includable string at internalDbDefintion.h for inclusion
+
+CREATE TABLE IF NOT EXISTS DataChunks (
+    columnId   INTEGER,
+    chunkId    INTEGER,
+    data       BLOB,
+    PRIMARY KEY (columnId, chunkId)
+) WITHOUT ROWID; 
+
+CREATE TABLE DataSets ( 
+	id				INTEGER PRIMARY KEY, 
+	dataFilePath	TEXT, 
+	dataFileTimestamp INT DEFAULT 0,
+	description		TEXT,
+	databaseJson	TEXT, 
+	emptyValuesJson TEXT, 
+	revision		INT DEFAULT 0, 
+	dataFileSynch	INT,
+	showRSyntax		INT DEFAULT 0,
+	csvDelimiter	INT DEFAULT 0
+);
+
+CREATE TABLE Filters ( 
+	id				INTEGER PRIMARY KEY, 
+	dataSet			INT, 
+	rFilter			TEXT,
+	name			TEXT, 
+	generatedFilter TEXT, 
+	constructorJson TEXT, 
+	constructorR	TEXT, 
+	errorMsg		TEXT,
+	revision		INT DEFAULT 0, 
+
+	FOREIGN KEY(dataSet) REFERENCES DataSets(id)
+);
+
+CREATE TABLE Columns
+( 
+	id					INTEGER PRIMARY KEY, 
+	dataSet				INT, 
+	name				TEXT, 
+	title				TEXT, 
+	description			TEXT, 
+	columnType			TEXT,
+	computeFilter		TEXT DEFAULT "",
+	colIdx				INT, 
+	autoSortByValue		INT,
+	dropLevels			INT,
+	invalidated			INT		NULL,
+	hasLabels			INT		DEFAULT 0,
+	codeType			TEXT	NULL, 
+	rCode				TEXT	NULL, 
+	error				TEXT	NULL, 
+	constructorJson		TEXT	NULL, 
+	analysisID			INT		NULL, 
+	emptyValuesJson		TEXT	NULL,
+	revision			INT		DEFAULT 0, 
+
+	FOREIGN KEY(dataSet) REFERENCES DataSets(id)
+);
+
+CREATE TABLE Labels		
+( 
+	id					INTEGER PRIMARY KEY, 
+	columnId			INT, 
+	value				INT, 
+	ordering			INT, 
+	filterAllows		INT,
+	userAdded			INT DEFAULT 0,
+	label				TEXT, 
+	originalValueJson	TEXT, 
+	description			TEXT, 
+
+	FOREIGN KEY(columnId) REFERENCES Columns(id)
+);
+
+
+)for_c++_include"

CommonData/internalDbDefinition.sql

@@ -23,6 +23,13 @@ CREATE TABLE DataSets (
 	defaultInputFilter	INT NULL
 );
 
+CREATE TABLE IF NOT EXISTS DataChunks (
+	columnId	INTEGER,
+	chunkId		INTEGER,
+	data		BLOB,
+	PRIMARY KEY (columnId, chunkId)
+) WITHOUT ROWID;
+
 CREATE TABLE Filters ( 
 	id				INTEGER PRIMARY KEY, 
 	dataSet			INT, 

conanfile.py

@@ -13,7 +13,6 @@ class JaspConanConfig(ConanFile):
     options = {"syntax_interface_only": [True, False]}
     default_options = {
         "brotli*:shared": True,
-        "sqlite3*:max_column": 32767,
         "syntax_interface_only": False,
     }