Untokenizer extended to handle X16 tokens as well.

Author: Foerster-H

src/main/java/com/sixtyfour/cbmnative/shell/MoSpeedCL.java

@@ -123,6 +123,7 @@ public static void main(String[] args) {
 
 		PlatformProvider platform = new Platform64();
 		String appendix = ".prg";
+		boolean multiByteTokens=false;
 		if (cmds.containsKey("platform")) {
 			String pl = cmds.get("platform");
 			if (pl.equalsIgnoreCase("c64")) {
@@ -135,6 +136,7 @@ public static void main(String[] args) {
 				platform = new PlatformX16();
 				Basic.registerExtension(new X16Extensions());
 				cfg.setNonDecimalNumbersAware(true);
+				multiByteTokens=true;
 				appendix = ".prg";
 			} else if (pl.equalsIgnoreCase("ps")) {
 				platform = new PlatformPs();
@@ -192,7 +194,7 @@ public static void main(String[] args) {
 				System.out.println("Looks like a PRG file, trying to convert it...");
 				byte[] data = Loader.loadBlob(srcFile);
 				UnTokenizer unto = new UnTokenizer();
-				src = unto.getText(data).toArray(new String[0]);
+				src = unto.getText(data, multiByteTokens).toArray(new String[0]);
 				System.out.println("PRG file converted into ASCII, proceeding!");
 				srcFile = srcFile.replace(".prg", ".bas");
 			} catch (Exception e) {

src/main/java/com/sixtyfour/parser/cbmnative/UnTokenizer.java

@@ -10,7 +10,7 @@
 
 /**
  * Converts files in .PRG format back into ASCII by replacing the BASIC tokens
- * and such. Supports C64 and VIC 20 for now, not the X16's new commands.
+ * and such. Supports C64 and VIC 20 BASIC V2 and the X16's extensions.
  * 
  * @author EgonOlsen71
  */
@@ -95,6 +95,26 @@ public class UnTokenizer {
 			this.put(201, "RIGHT$");
 			this.put(202, "MID$");
 			this.put(203, "GO");
+			
+			// X16-Extensions go here...this isn't really a good solution, it
+			// would be better to make the extension itself fill this, but
+			// I can't be bothered right now...
+			this.put(0xCE86, "SCREEN");
+			this.put(0xCE87, "PSET");
+			this.put(0xCE8A, "RECT");
+			this.put(0xCE88, "LINE");
+			this.put(0xCE89, "FRAME");
+			this.put(0xCE8B, "CHAR");
+			this.put(0xCE8C, "MOUSE");
+			this.put(0xCE8E, "MX");
+			this.put(0xCE8F, "MY");
+			this.put(0xCE90, "MB");
+			this.put(0xCE84, "VPOKE");
+			this.put(0xCE8D, "VPEEK");
+			this.put(0xCE81, "DOS");
+			this.put(0xCE80, "MON");
+			this.put(0xCE82, "OLD");
+			this.put(0xCE83, "GEOS");
 		}
 	};
 
@@ -108,7 +128,7 @@ public UnTokenizer() {
 	 * @param program the file in binary
 	 * @return the ascii version of it
 	 */
-	public List<String> getText(byte[] program) {
+	public List<String> getText(byte[] program, boolean multiByte) {
 		List<String> lines = new ArrayList<>();
 		StringBuilder line = new StringBuilder();
 
@@ -134,14 +154,18 @@ public List<String> getText(byte[] program) {
 						if (c == '"') {
 							inString = !inString;
 						}
-						if (b < 128 || inString || b > 203) {
+						if (b < 128 || inString || (b > 203 && (!multiByte || b!=206))) {
 							if (inString && isSpecialChar(c)) {
 								line.append(ControlCodes.getPlaceHolder(b));
 							} else {
 								line.append(convertChar(c));
 							}
 						} else {
-							String token = TOKENS.get(b);
+						    	String token = TOKENS.get(b);
+						    	if (multiByte && b==206) {
+						    	    int mb = b*256+(program[(addr++) - start] & 0xff);
+						    	    token=TOKENS.get(mb);
+						    	}
 							if (token == null) {
 								throw new RuntimeException("Unknown token: " + b);
 							}

src/main/java/com/sixtyfour/runner/VisualMospeed.java

@@ -394,7 +394,7 @@ private void loadProgram() {
 				Logger.log("Looks like a PRG file, trying to convert it...");
 				byte[] data = Loader.loadBlob(srcFile);
 				UnTokenizer unto = new UnTokenizer();
-				code = unto.getText(data).toArray(new String[0]);
+				code = unto.getText(data, true).toArray(new String[0]);
 				Logger.log("PRG file converted into ASCII, proceeding!");
 				srcFile = srcFile.replace(".prg", ".bas");
 			} catch (Exception e) {

src/test/java/com/sixtyfour/test/UnTokenizerTest.java

@@ -14,12 +14,13 @@ public class UnTokenizerTest {
 
 	public static void main(String[] args) {
 		testUnTokenizer();
+		testUnTokenizerX16();
 	}
 
 	public static void testUnTokenizer() {
 		byte[] data = Loader.loadBlob("src/test/resources/prg/demo.prg");
 		UnTokenizer unto = new UnTokenizer();
-		List<String> res = unto.getText(data);
+		List<String> res = unto.getText(data, false);
 		res.forEach(p -> System.out.println(p));
 		try (PrintWriter pw = new PrintWriter("src/test/resources/prg/demo.bas")) {
 			for (String line : res) {
@@ -29,5 +30,19 @@ public static void testUnTokenizer() {
 			throw new RuntimeException(e);
 		}
 	}
+	
+	public static void testUnTokenizerX16() {
+		byte[] data = Loader.loadBlob("src/test/resources/prg/cmds.prg");
+		UnTokenizer unto = new UnTokenizer();
+		List<String> res = unto.getText(data, true);
+		res.forEach(p -> System.out.println(p));
+		try (PrintWriter pw = new PrintWriter("src/test/resources/prg/cmds.bas")) {
+			for (String line : res) {
+				pw.println(line);
+			}
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
 
 }

src/test/resources/prg/cmds.bas

@@ -0,0 +1,15 @@
+5 a=2
+10 screen a: rem ce 86
+20 pset a,a,a: rem ce 87
+30 rect a,a,a,a,a: rem ce 8a
+40 line a,a,a,a,a: rem ce 88
+50 frame a,a,a,a,a: rem ce 89
+60 char a,a,a,"a": rem ce 8b
+70 mouse a: rem ce 8c
+80 a=mx:a=my:a=mb: rem ce 8e, ce 8f, ce 90
+90 vpoke a,a,a: rem ce 84
+100 a=vpeek(a,a): rem ce 8d
+110 dos: rem ce 81
+120 mon: rem ce 80
+130 old: rem ce 82
+140 geos: rem ce 83