Skip to content

Commit 8499eb5

Browse files
committed
Merge remote-tracking branch 'private/for-merging' into merge-private
2 parents eccce5d + daf7f9e commit 8499eb5

File tree

9 files changed

+175
-19
lines changed

9 files changed

+175
-19
lines changed

packages/client-app/src/flux/attributes/attribute-joined-data.es6

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,25 @@ Section: Database
3333
export default class AttributeJoinedData extends Attribute {
3434
static NullPlaceholder = NullPlaceholder;
3535

36-
constructor({modelKey, jsonKey, modelTable, queryable}) {
36+
constructor({modelKey, jsonKey, modelTable, queryable, serializeFn, deserializeFn}) {
3737
super({modelKey, jsonKey, queryable});
3838
this.modelTable = modelTable;
39+
this.serializeFn = serializeFn;
40+
this.deserializeFn = deserializeFn;
41+
}
42+
43+
serialize(thisValue, val) {
44+
if (this.serializeFn) {
45+
return this.serializeFn.call(thisValue, val);
46+
}
47+
return val;
48+
}
49+
50+
deserialize(thisValue, val) {
51+
if (this.deserializeFn) {
52+
return this.deserializeFn.call(thisValue, val);
53+
}
54+
return val;
3955
}
4056

4157
toJSON(val) {

packages/client-app/src/flux/models/message.es6

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import _ from 'underscore'
22
import moment from 'moment'
3+
import {MessageBodyUtils} from 'isomorphic-core'
34

45
import File from './file'
56
import Utils from './utils'
@@ -104,6 +105,20 @@ export default class Message extends ModelWithMetadata {
104105
body: Attributes.JoinedData({
105106
modelTable: 'MessageBody',
106107
modelKey: 'body',
108+
serializeFn: function serializeBody(val) {
109+
return MessageBodyUtils.writeBody({
110+
msgId: this.id,
111+
body: val,
112+
forceWrite: false,
113+
});
114+
},
115+
deserializeFn: function deserializeBody(val) {
116+
const result = MessageBodyUtils.tryReadBody(val);
117+
if (result) {
118+
return result;
119+
}
120+
return val;
121+
},
107122
}),
108123

109124
files: Attributes.Collection({

packages/client-app/src/flux/models/query.es6

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ export default class ModelQuery {
310310
if (value === AttributeJoinedData.NullPlaceholder) {
311311
value = null;
312312
}
313-
object[attr.modelKey] = value;
313+
object[attr.modelKey] = attr.deserialize(object, value);
314314
}
315315
return object;
316316
});

packages/client-app/src/flux/stores/database-writer.es6

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,9 @@ export default class DatabaseWriter {
322322

323323
joinedDataAttributes.forEach((attr) => {
324324
for (const model of models) {
325-
if (model[attr.modelKey] !== undefined) {
326-
promises.push(this._query(`REPLACE INTO \`${attr.modelTable}\` (\`id\`, \`value\`) VALUES (?, ?)`, [model.id, model[attr.modelKey]]));
325+
const value = model[attr.modelKey];
326+
if (value !== undefined) {
327+
promises.push(this._query(`REPLACE INTO \`${attr.modelTable}\` (\`id\`, \`value\`) VALUES (?, ?)`, [model.id, attr.serialize(model, value)]));
327328
}
328329
}
329330
});

packages/client-sync/src/message-processor/index.js

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,13 @@ class MessageProcessor {
233233

234234
const thread = await detectThread({db, messageValues});
235235
messageValues.threadId = thread.id;
236-
const createdMessage = await Message.create(messageValues);
236+
// The way that sequelize initializes objects doesn't guarantee that the
237+
// object will have a value for `id` before initializing the `body` field
238+
// (which we now depend on). By using `build` instead of `create`, we can
239+
// initialize an object with just the `id` field and then use `update` to
240+
// initialize the remaining fields and save the object to the database.
241+
const createdMessage = Message.build({id: messageValues.id});
242+
await createdMessage.update(messageValues);
237243

238244
if (messageValues.labels) {
239245
await createdMessage.addLabels(messageValues.labels)

packages/client-sync/src/models/message.js

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@ const {
33
ExponentialBackoffScheduler,
44
IMAPErrors,
55
IMAPConnectionPool,
6+
MessageBodyUtils,
67
} = require('isomorphic-core')
78
const {DatabaseTypes: {JSONArrayColumn}} = require('isomorphic-core');
89
const {Errors: {APIError}} = require('isomorphic-core')
910
const {Actions} = require('nylas-exports')
1011

1112
const MAX_IMAP_TIMEOUT_ERRORS = 5;
1213

13-
1414
function validateRecipientsPresent(message) {
1515
if (message.getRecipients().length === 0) {
1616
throw new APIError(`No recipients specified`, 400);
@@ -25,7 +25,23 @@ module.exports = (sequelize, Sequelize) => {
2525
headerMessageId: { type: Sequelize.STRING, allowNull: true },
2626
gMsgId: { type: Sequelize.STRING, allowNull: true },
2727
gThrId: { type: Sequelize.STRING, allowNull: true },
28-
body: Sequelize.TEXT,
28+
body: {
29+
type: Sequelize.TEXT,
30+
get: function getBody() {
31+
const val = this.getDataValue('body');
32+
const result = MessageBodyUtils.tryReadBody(val);
33+
if (result) {
34+
return result;
35+
}
36+
return val;
37+
},
38+
set: function setBody(val) {
39+
this.setDataValue('body', MessageBodyUtils.writeBody({
40+
msgId: this.id,
41+
body: val,
42+
}));
43+
},
44+
},
2945
subject: Sequelize.STRING(500),
3046
snippet: Sequelize.STRING(255),
3147
date: Sequelize.DATE,

packages/isomorphic-core/index.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ module.exports = {
1414
DatabaseTypes: require('./src/database-types'),
1515
IMAPConnection: require('./src/imap-connection').default,
1616
IMAPConnectionPool: require('./src/imap-connection-pool'),
17+
MessageBodyUtils: require('./src/message-body-utils'),
1718
SendmailClient: require('./src/sendmail-client'),
1819
DeltaStreamBuilder: require('./src/delta-stream-builder'),
1920
HookTransactionLog: require('./src/hook-transaction-log'),
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
const fs = require('fs')
2+
const mkdirp = require('mkdirp')
3+
const path = require('path')
4+
const zlib = require('zlib')
5+
6+
const MAX_PATH_DIRS = 5;
7+
const FILE_EXTENSION = 'nylasmail'
8+
9+
function baseMessagePath() {
10+
return path.join(process.env.NYLAS_HOME, 'messages');
11+
}
12+
13+
export function tryReadBody(val) {
14+
try {
15+
const parsed = JSON.parse(val);
16+
if (parsed && parsed.path && parsed.path.startsWith(baseMessagePath())) {
17+
if (parsed.compressed) {
18+
return zlib.gunzipSync(fs.readFileSync(parsed.path)).toString();
19+
}
20+
return fs.readFileSync(parsed.path, {encoding: 'utf8'});
21+
}
22+
} catch (err) {
23+
console.warn('Got error while trying to parse body path, assuming we need to migrate', err);
24+
}
25+
return null;
26+
}
27+
28+
export function pathForBodyFile(msgId) {
29+
const pathGroups = [];
30+
let remainingId = msgId;
31+
while (pathGroups.length < MAX_PATH_DIRS) {
32+
pathGroups.push(remainingId.substring(0, 2));
33+
remainingId = remainingId.substring(2);
34+
}
35+
const bodyPath = path.join(...pathGroups);
36+
return path.join(baseMessagePath, bodyPath, `${remainingId}.${FILE_EXTENSION}`);
37+
}
38+
39+
// NB: The return value of this function is what gets written into the database.
40+
export function writeBody({msgId, body, forceWrite = true} = {}) {
41+
const bodyPath = pathForBodyFile(msgId);
42+
const bodyDir = path.dirname(bodyPath);
43+
44+
const compressedBody = zlib.gzipSync(body);
45+
const dbEntry = {
46+
path: bodyPath,
47+
compressed: true,
48+
};
49+
50+
// It's possible that gzipping actually makes the body larger. If that's the
51+
// case then just write the uncompressed body instead.
52+
let bodyToWrite = compressedBody;
53+
if (compressedBody.length >= body.length) {
54+
dbEntry.compressed = false;
55+
bodyToWrite = body;
56+
}
57+
58+
const result = JSON.stringify(dbEntry);
59+
// If the JSON db entry would be longer than the body itself then just write
60+
// the body directly into the database.
61+
if (result.length > body.length) {
62+
return body;
63+
}
64+
65+
try {
66+
let exists;
67+
68+
// If we don't have to write to the file and it already exists then don't.
69+
if (!forceWrite) {
70+
exists = fs.existsSync(bodyPath);
71+
if (exists) {
72+
return result;
73+
}
74+
}
75+
76+
// We want to minimize the number of times we interact with the filesystem
77+
// since it can be slow.
78+
if (exists === undefined) {
79+
exists = fs.existsSync(bodyPath);
80+
}
81+
if (!exists) {
82+
mkdirp.sync(bodyDir);
83+
}
84+
85+
fs.writeFileSync(bodyPath, bodyToWrite);
86+
return result;
87+
} catch (err) {
88+
// If anything bad happens while trying to write to disk just store the
89+
// body in the database.
90+
return body;
91+
}
92+
}

scripts/postinstall.es6

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,16 @@ function copyErrorLoggerExtensions(privateDir) {
3232
fs.copySync(from, to);
3333
}
3434

35-
async function installPrivateResources() {
35+
function installClientSyncPackage() {
36+
console.log("\n---> Linking client-sync")
37+
// link client-sync
38+
const clientSyncDir = path.resolve(path.join('packages', 'client-sync'));
39+
const destination = path.resolve(path.join('packages', 'client-app', 'internal_packages', 'client-sync'));
40+
unlinkIfExistsSync(destination);
41+
fs.symlinkSync(clientSyncDir, destination, 'dir');
42+
}
43+
44+
function installPrivateResources() {
3645
console.log("\n---> Linking private plugins")
3746
const privateDir = path.resolve(path.join('packages', 'client-private-plugins'))
3847
if (!fs.existsSync(privateDir)) {
@@ -49,12 +58,6 @@ async function installPrivateResources() {
4958
unlinkIfExistsSync(to);
5059
fs.symlinkSync(from, to, 'dir');
5160
}
52-
53-
// link client-sync
54-
const clientSyncDir = path.resolve(path.join('packages', 'client-sync'));
55-
const destination = path.resolve(path.join('packages', 'client-app', 'internal_packages', 'client-sync'));
56-
unlinkIfExistsSync(destination);
57-
fs.symlinkSync(clientSyncDir, destination, 'dir');
5861
}
5962

6063
async function lernaBootstrap(installTarget) {
@@ -124,11 +127,16 @@ function linkJasmineConfigs() {
124127
console.log("\n---> Linking Jasmine configs");
125128
const linkToPackages = ['cloud-api', 'cloud-core', 'cloud-workers']
126129
const from = getJasmineConfigPath('isomorphic-core')
127-
128130
for (const packageName of linkToPackages) {
129-
const dir = getJasmineDir(packageName)
130-
if (!fs.existsSync(dir)) {
131-
fs.mkdirSync(dir)
131+
const packageDir = path.join('packages', packageName)
132+
if (!fs.existsSync(packageDir)) {
133+
console.log("\n---> No cloud packages to link. Moving on")
134+
return
135+
}
136+
137+
const jasmineDir = getJasmineDir(packageName)
138+
if (!fs.existsSync(jasmineDir)) {
139+
fs.mkdirSync(jasmineDir)
132140
}
133141
const to = getJasmineConfigPath(packageName)
134142
unlinkIfExistsSync(to)
@@ -161,7 +169,8 @@ async function main() {
161169
console.log(`\n---> Installing for target ${installTarget}`);
162170

163171
if ([TARGET_ALL, TARGET_CLIENT].includes(installTarget)) {
164-
await installPrivateResources()
172+
installPrivateResources()
173+
installClientSyncPackage()
165174
}
166175

167176
await lernaBootstrap(installTarget);

0 commit comments

Comments
 (0)