v6.3.2

Fixed reference parsing bugs

Author: dijs

package.json

@@ -2,7 +2,7 @@
 	"name": "wikijs",
 	"description": "Mediawiki interface for Node and Web",
 	"author": "Richard van der Dys",
-	"version": "6.3.1",
+	"version": "6.3.2",
 	"keywords": [
 		"wiki",
 		"wikipedia",

src/page.js

@@ -241,6 +241,17 @@ export default function wikiPage(rawPageInfo, apiOptions) {
 		return null;
 	}
 
+	function findNodes(node, predicate, nodes) {
+		if (predicate(node)) {
+			nodes.push(node);
+		}
+		if (node.content.children) {
+			for (let child of node.content.children) {
+				findNodes(child, predicate, nodes);
+			}
+		}
+	}
+
 	/**
 	 * References from page
 	 * @example
@@ -257,14 +268,23 @@ export default function wikiPage(rawPageInfo, apiOptions) {
 			})
 			.then(ast => {
 				const links = [];
-				const refs = findNode(
+				const refs = [];
+				// There can be mulitple reference sections
+				findNodes(
 					ast,
-					node => isTag(node) && hasClass(node, 'references')
+					node =>
+						isTag(node) && hasName(node, 'ol') && hasClass(node, 'references'),
+					refs
 				);
-				if (refs) {
-					for (let ref of refs.content.children[0].content.children) {
+				for (let ref of refs) {
+					const items = ref.content.children.filter(
+						el => isTag(el) && hasName(el, 'li') && el.content.children
+					);
+					for (let item of items) {
+						// The reference was moved under a span under li
+						const span = item.content.children[2];
 						const cite = findNode(
-							ref,
+							span,
 							node => isTag(node) && hasName(node, 'cite')
 						);
 						if (cite) {

test/real.js

@@ -24,7 +24,7 @@ describe('Live tests', () => {
 		this.timeout(timeoutTime);
 		setTimeout(() => {
 			done();
-		}, 1000);
+		}, 100);
 	});
 
 	it.skip('should handle error response', function(done) {
@@ -396,23 +396,18 @@ describe('Live tests', () => {
 			});
 	});
 
-	// Not parsing...
-	it.skip('should return references in correct order', function() {
+	it('should return references in correct order', function() {
 		this.timeout(timeoutTime);
 		return wiki()
 			.page('Elon Musk')
 			.then(page => page.references())
 			.then(refs => {
 				refs[0].should.equal(
-					'https://www.forbes.com/sites/trulia/2013/11/01/billionaire-tesla-ceo-elon-musk-buys-home/'
+					'https://www.independent.co.uk/life-style/elon-musk-son-grimes-childcare-interview-a9638321.html'
 				);
-				refs[1].should.equal(
-					'https://web.archive.org/web/20150207033543/http://www.bloomberg.com/news/videos/b/6e27fcba-309d-494e-b87d-c73fb8bb1750'
+				refs[3].should.equal(
+					'https://www.fastcompany.com/1367866/tesla-lawsuit-drama-ends-five-company-founders-emerge'
 				);
-				refs[2].should.equal(
-					'https://www.bloomberg.com/news/videos/b/6e27fcba-309d-494e-b87d-c73fb8bb1750'
-				);
-				refs[3].should.equal('https://www.forbes.com/profile/elon-musk/');
 			});
 	});
 
@@ -437,8 +432,7 @@ describe('Live tests', () => {
 			});
 	});
 
-	// TODO: new html parsing is NOT working
-	it.skip('should parse external references', function() {
+	it('should parse external references', function() {
 		this.timeout(timeoutTime);
 		return wiki()
 			.page('Batman')
@@ -447,7 +441,7 @@ describe('Live tests', () => {
 				refs.should.containEql(
 					'http://www.behindthevoiceactors.com/characters/Batman/Batman/'
 				);
-				refs.length.should.equal(140);
+				refs.length.should.equal(143);
 			});
 	});
 

test/spec.js

@@ -104,7 +104,7 @@ describe('Page Methods', () => {
 		this.timeout(timeoutTime);
 		setTimeout(() => {
 			done();
-		}, 1000);
+		}, 100);
 	});
 
 	before(done => {