columnをUTF-16コード単位に変更

Author: takejohn

src/parser/streams/char-stream.ts

@@ -12,9 +12,9 @@ export class CharStream {
 	private address: number;
 	/** Unicode character */
 	private _char?: string;
-	/** zero-based number, based on Unicode code points */
+	/** zero-based number */
 	private line: number;
-	/** zero-based number, based on Unicode code points */
+	/** zero-based number, based on UTF-16 code unit */
 	private column: number;
 
 	constructor(source: string, opts?: { line?: number, column?: number }) {
@@ -64,7 +64,7 @@ export class CharStream {
 			this.line++;
 			this.column = 0;
 		} else {
-			this.column++;
+			this.column += this._char!.length;
 		}
 		this.incAddr();
 		this.moveNext();
@@ -82,9 +82,9 @@ export class CharStream {
 			const lastLineBreak = page.lastIndexOf('\n', this.address - 1);
 			const lineStart = lastLineBreak >= 0 ? lastLineBreak + 1 : 0;
 			const line = page.slice(lineStart, this.address);
-			this.column = [...line].length - 1;
+			this.column = line.length;
 		} else {
-			this.column--;
+			this.column -= this._char!.length;
 		}
 	}
 

test/parser.ts

@@ -45,7 +45,7 @@ describe('CharStream', () => {
 			assert.strictEqual('b', stream.char);
 			stream.prev();
 			assert.strictEqual('\n', stream.char);
-			assert.deepStrictEqual(stream.getPos(), { line: 1, column: 1 });
+			assert.deepStrictEqual(stream.getPos(), { line: 1, column: 2 });
 		});
 
 		test.concurrent('line breaks', async () => {
@@ -56,7 +56,7 @@ describe('CharStream', () => {
 			assert.strictEqual('c', stream.char);
 			stream.prev();
 			assert.strictEqual('\n', stream.char);
-			assert.deepStrictEqual(stream.getPos(), { line: 2, column: 0 });
+			assert.deepStrictEqual(stream.getPos(), { line: 2, column: 1 });
 		});
 
 		test.concurrent('CRは読み飛ばされる', async () => {
@@ -77,6 +77,26 @@ describe('CharStream', () => {
 			stream.prev();
 			assert.strictEqual('\ud83e\udd2f', stream.char);
 		});
+
+		test.concurrent('column is based on UTF-16 code unit', async () => {
+			const source = '\ud83e\udd2f!';
+			const stream = new CharStream(source);
+			stream.next();
+			stream.next();
+			stream.prev();
+			assert.strictEqual(stream.char, '!');
+			assert.deepStrictEqual(stream.getPos(), { line: 1, column: 3 });
+		});
+
+		test.concurrent('column is based on UTF-16 code unit, line break', async () => {
+			const source = '\ud83e\udd2f\n';
+			const stream = new CharStream(source);
+			stream.next();
+			stream.next();
+			stream.prev();
+			assert.strictEqual(stream.char, '\n');
+			assert.deepStrictEqual(stream.getPos(), { line: 1, column: 3 });
+		});
 	});
 
 	test.concurrent('eof', async () => {
@@ -122,6 +142,13 @@ describe('CharStream', () => {
 		stream.next();
 		assert.strictEqual(true, stream.eof);
 	});
+
+	test.concurrent('column is based on UTF-16 code unit', async () => {
+		const source = '\ud83e\udd2f';
+		const stream = new CharStream(source);
+		stream.next();
+		assert.deepStrictEqual(stream.getPos(), { line: 1, column: 3 });
+	});
 });
 
 describe('Scanner', () => {