change math.crc32 to the same algorithm as pkzip/zlib uses (ISO-HDLC). Add math.crc32_end_result(). Fix a parse error in profiler.py script.

Author: irmen

compiler/res/prog8lib/math.p8

@@ -536,11 +536,9 @@ log2_tab
             eor  cx16.r15H
             sta  cx16.r15H
             ldy  #8
--           lda  cx16.r15H
-            asl  cx16.r15L
+-           asl  cx16.r15L
             rol  cx16.r15H
-            and  #$80
-            beq  +
+            bcc  +
             lda  cx16.r15H
             eor  #$10
             sta  cx16.r15H
@@ -554,11 +552,9 @@ log2_tab
 ; orignal prog8 code was:
 ;        cx16.r15H ^= value
 ;        repeat 8 {
-;            if cx16.r15H & $80 !=0 {
-;                cx16.r15 <<=1
+;            cx16.r15<<=1
+;            if_cs
 ;                cx16.r15 ^= $1021
-;            } else
-;                cx16.r15<<=1
 ;        }
     }
 
@@ -568,7 +564,7 @@ log2_tab
     }
 
     sub crc32(uword data, uword length) {
-        ; Calculates the CRC-32 (POSIX) checksum of the buffer.
+        ; Calculates the CRC-32 (ISO-HDLC/PKZIP) checksum of the buffer.
         ; because prog8 doesn't have 32 bits integers, we have to split up the calculation over 2 words.
         ; result stored in cx16.r14 (low word) and cx16.r15 (high word)
         ; There are also "streaming" crc32_start/update/end routines below, that allow you to calculate crc32 for data that doesn't fit in a single memory block.
@@ -586,62 +582,75 @@ log2_tab
         ; start the "streaming" crc32
         ; note: tracks the crc32 checksum in cx16.r14 and cx16.r15!
         ;       if your code uses these, it must save/restore them before calling this routine
-        cx16.r14 = cx16.r15 = 0
+        cx16.r14 = cx16.r15 = $ffff
     }
 
-    asmsub crc32_update(ubyte value @A) {
+    sub crc32_update(ubyte value) {
         ; update the "streaming" crc32 with next byte value
         ; note: tracks the crc32 checksum in cx16.r14 and cx16.r15!
         ;       if your code uses these, it must save/restore them before calling this routine
+        ; implementation detail: see https://stackoverflow.com/a/75951866  , the zlib crc32 is the "reflected" variant
         %asm {{
-            eor  cx16.r15H
-            sta  cx16.r15H
+            eor  cx16.r14L
+            sta  cx16.r14L
             ldy  #8
--           lda  cx16.r15H
-            asl  cx16.r14L
-            rol  cx16.r14H
-            rol  cx16.r15L
-            rol  cx16.r15H
-            and  #$80
-            beq  +
+-           lsr  cx16.r15H
+            ror  cx16.r15L
+            ror  cx16.r14H
+            ror  cx16.r14L
+            bcc  +
             lda  cx16.r15H
-            eor  #$04
+            eor  #$ed
             sta  cx16.r15H
             lda  cx16.r15L
-            eor  #$c1
+            eor  #$b8
             sta  cx16.r15L
             lda  cx16.r14H
-            eor  #$1d
+            eor  #$83
             sta  cx16.r14H
             lda  cx16.r14L
-            eor  #$b7
+            eor  #$20
             sta  cx16.r14L
 +           dey
             bne  -
             rts
         }}
 ; original prog8 code:
-;        cx16.r15H ^= value
+;        cx16.r14L ^= value
 ;        repeat 8 {
-;            if cx16.r15H & $80 !=0 {
-;                cx16.r14 <<= 1
-;                rol(cx16.r15)
-;                cx16.r15 ^= $04c1
-;                cx16.r14 ^= $1db7
-;            }
-;            else {
-;                cx16.r14 <<= 1
-;                rol(cx16.r15)
+;            cx16.r15 >>= 1
+;            ror(cx16.r14)
+;            if_cs {
+;                cx16.r15 ^= $edb8
+;                cx16.r14 ^= $8320
 ;            }
 ;        }
-
     }
 
     sub crc32_end() {
         ; finalize the "streaming" crc32
         ; result stored in cx16.r14 (low word) and cx16.r15 (high word)
-        cx16.r15 ^= $ffff
-        cx16.r14 ^= $ffff
+        void crc32_end_result()
+    }
+
+    asmsub crc32_end_result() -> uword @R15, uword @R14 {
+        ; finalize the "streaming" crc32
+        ; returns the result value in cx16.r15 (high word) and r14 (low word)
+        %asm {{
+            lda  cx16.r15H
+            eor  #255
+            sta  cx16.r15H
+            lda  cx16.r15L
+            eor  #255
+            sta  cx16.r15L
+            lda  cx16.r14H
+            eor  #255
+            sta  cx16.r14H
+            lda  cx16.r14L
+            eor  #255
+            sta  cx16.r14L
+            rts
+        }}
     }
 
 

compiler/res/prog8lib/virtual/math.p8

@@ -339,10 +339,9 @@ math {
         ;       if your code uses that, it must save/restore it before calling this routine
         cx16.r15H ^= value
         repeat 8 {
-            if cx16.r15H & $80 !=0
-                cx16.r15 = (cx16.r15<<1)^$1021
-            else
-                cx16.r15<<=1
+            cx16.r15<<=1
+            if_cs
+                cx16.r15 ^= $1021
         }
     }
 
@@ -352,7 +351,7 @@ math {
     }
 
     sub crc32(uword data, uword length) {
-        ; Calculates the CRC-32 (POSIX) checksum of the buffer.
+        ; Calculates the CRC-32 (ISO-HDLC/PKZIP) checksum of the buffer.
         ; because prog8 doesn't have 32 bits integers, we have to split up the calculation over 2 words.
         ; result stored in cx16.r14 (low word) and cx16.r15 (high word)
         ; There are also "streaming" crc32_start/update/end routines below, that allow you to calculate crc32 for data that doesn't fit in a single memory block.
@@ -370,24 +369,21 @@ math {
         ; start the "streaming" crc32
         ; note: tracks the crc32 checksum in cx16.r14 and cx16.r15!
         ;       if your code uses these, it must save/restore them before calling this routine
-        cx16.r14 = cx16.r15 = 0
+        cx16.r14 = cx16.r15 = $ffff
     }
 
     sub crc32_update(ubyte value) {
         ; update the "streaming" crc32 with next byte value
         ; note: tracks the crc32 checksum in cx16.r14 and cx16.r15!
         ;       if your code uses these, it must save/restore them before calling this routine
-        cx16.r15H ^= value
+        ; implementation detail: see https://stackoverflow.com/a/75951866  , the zlib crc32 is the "reflected" variant
+        cx16.r14L ^= value
         repeat 8 {
-            if cx16.r15H & $80 !=0 {
-                cx16.r14 <<= 1
-                rol(cx16.r15)
-                cx16.r15 ^= $04c1
-                cx16.r14 ^= $1db7
-            }
-            else {
-                cx16.r14 <<= 1
-                rol(cx16.r15)
+            cx16.r15 >>= 1
+            ror(cx16.r14)
+            if_cs {
+                cx16.r15 ^= $edb8
+                cx16.r14 ^= $8320
             }
         }
     }
@@ -399,6 +395,7 @@ math {
         cx16.r14 ^= $ffff
     }
 
+    ; there's no crc32_end_result() here because IR cannot return multiple values yet
 
     sub lerp(ubyte v0, ubyte v1, ubyte t) -> ubyte {
         ; Linear interpolation (LERP)

docs/source/libraries.rst

@@ -725,16 +725,20 @@ but perhaps the provided ones can be of service too.
     "streaming" crc16 calculation routines, when the data doesn't fit in a single buffer.
     Tracks the crc16 checksum in cx16.r15! If your code uses that, it must save/restore it before calling this routine!
     Call the start() routine first, feed it bytes with the update() routine, finalize with calling the end() routine which returns the crc16 value.
+    Note: after calling the crc16_end() routine you must start over.
 
 ``crc32 (uword data, uword length)``
-    Calculates a CRC-32 (POSIX) checksum over the given data buffer.
+    Calculates a CRC-32 (ISO-HDLC/PKZIP) checksum over the given data buffer.
     The 32 bits result is stored in cx16.r14 (low word) and cx16.r15 (high word).
 
-``crc32_start() / crc32_update(ubyte value) / crc32_end()``
+``crc32_start() / crc32_update(ubyte value) / crc32_end() / crc32_end_result()``
     "streaming" crc32 calculation routines, when the data doesn't fit in a single buffer.
     Tracks the crc32 checksum in cx16.r14 and cx16.r15! If your code uses these, it must save/restore them before calling this routine!
     Call the start() routine first, feed it bytes with the update() routine, finalize with calling the end() routine.
     The 32 bits result is stored in cx16.r14 (low word) and cx16.r15 (high word).
+    Instead of the normal end() routine you can also call crc32_end_result() which finalizes the calculation,
+    and actually returns the high and low words of the 32 bits result value as two return word values.
+    Note: after calling the crc32_end() or crc32_end_result() routine you must start over.
 
 ``lerp(v0, v1, t)``
     Linear interpolation routine for unsigned byte values.

docs/source/technical.rst

@@ -232,14 +232,23 @@ Some notes and references into the compiler's source code modules:
 
 Run-time memory profiling with the X16 emulator
 -----------------------------------------------
+
+The compiler has the ``-dumpvars`` switch that will print a list of all variables and where they are placed into memory.
+This can be useful to track which variables end up in zeropage for instance. But it doesn't really show if the choices
+made are good, i.e. if the variables that are actually the most used in your program, are placed in zeropage.
+
+But there is a way to actually *measure* the behavior of your program as it runs on the X16.
+See it as a simple way of *profiling* your program to find the hotspots that maybe need optimizing:
+
 The X16 emulator has a ``-memorystats`` option that enables it to keep track of memory access count statistics,
 and write the accumulated counts to a file on exit.
-Prog8 includes a Python script ``profiler.py`` (find it in the "scripts" subdirectory of the source code distribution)
-that can cross-reference that file with an assembly listing produced by the compiler with the ``-asmlist`` option.
+Prog8 then provides a Python script ``profiler.py`` (find it in the "scripts" subdirectory of the source code distribution,
+or `online here <https://github.com/irmen/prog8/blob/master/scripts/profiler.py>`_).
+This script cross-references the memory stats file with an assembly listing of the program, produced by the Prog8 compiler with the ``-asmlist`` option.
 It then prints the top N lines in your (assembly) program source that perform the most reads and writes,
 which you can use to identify possible hot spots/bottlenecks/variables that should be better placed in zeropage etc.
-Note that the profiler just works with the number of accesses to memory locations, this is *not* the same
-as the most run-time (cpu instructions cycle times aren't taken into account at all).
+Note that the profiler simply works with the total number of accesses to memory locations.
+This is *not* the same as the most run-time (cpu instructions cycle times aren't taken into account at all)!
 Here is an example of the output it generates::
 
     $ scripts/profiler.py -n 10 cobramk3-gfx.list memstats.txt                                                                             ✔
@@ -274,4 +283,10 @@ Here is an example of the output it generates::
     $01e7 (1280140) : cpu stack
     $0264 (1258159) : unknown
 
-Apparently the most cpu activity while running this program is spent in a division routine.
+Apparently the most cpu activity while running this program is spent in a division routine which uses the 'remainder' and 'dividend' variables.
+As you can see, sometimes even actual assembly instructions end up in the tables above if they are in a routine that is executed very often (the 'stz' instructions in this example).
+The tool isn't powerful enough to see what routine the variables or instructions are part of, but it prints the line number in the assembly listing file so you can investigate that manually.
+
+You can see in the example above that the variables that are among the most used are neatly placed in zeropage already.
+If you see for instance a variable that is heavily used and that is *not* in zeropage, you
+could consider adding ``@zp`` to that variable's declaration to prioritize it to be put into zeropage.

docs/source/todo.rst

@@ -1,8 +1,6 @@
 TODO
 ====
 
-- write something in the docs about how to optimize your program on the x16 using the -dumpvars option,
-  the emulator's memory profiler + the profiler.py script to find hotspots for routines and variables that could be placed into zeropage
 - announce prog8 on the 6502.org site?
 
 ...
@@ -24,6 +22,7 @@ Future Things and Ideas
   But all library code written in asm uses .proc already..... (textual search/replace when writing the actual asm?)
   Once new codegen is written that is based on the IR, this point is mostly moot anyway as that will have its own dead code removal on the IR level.
 
+- Allow normal subroutines to return multiple values as well (just as asmsubs already can)
 - Change scoping rules for qualified symbols so that they don't always start from the root but behave like other programming languages (look in local scope first)
 - Fix missing cases where regular & has to return the start of the split array in memory whatever byte comes first. Search TODO("address of split word array")
 - something to reduce the need to use fully qualified names all the time. 'with' ?  Or 'using <prefix>'?

examples/test.p8

@@ -1,13 +1,31 @@
 %import textio
-%import diskio
+%import math
 
 %zeropage basicsafe
 %option no_sysinit
 
 
 main {
     sub start() {
-        cx16.r2 = diskio.get_loadaddress("test.prg")
-        txt.print_uwhex(cx16.r2, true)
+        str input = iso:"the quick brown fox jumps over the lazy dog"
+
+
+        txt.print_uwhex(math.crc16(input, len(input)), true)
+        txt.nl()
+
+        math.crc32(input, len(input))
+
+        txt.print_uwhex(cx16.r15, true)
+        txt.print_uwhex(cx16.r14, false)
+        txt.nl()
+
+        math.crc32_start()
+        for cx16.r0L in input
+            math.crc32_update(cx16.r0L)
+        uword hiw,low
+        hiw,low = math.crc32_end_result()
+        txt.print_uwhex(hiw, true)
+        txt.print_uwhex(low, false)
+        txt.nl()
     }
 }

scripts/profiler.py

@@ -7,6 +7,9 @@
 and prints out what assembly lines and variables were read from and written to the most.
 These may indicate hot paths or even bottlenecks in your program,
 and what variables in system ram might be better placed in Zeropage.
+
+Also see https://prog8.readthedocs.io/en/latest/technical.html#run-time-memory-profiling-with-the-x16-emulator
+for an example of how to use this tool together with the X16 emulator.
 """
 
 
@@ -26,7 +29,7 @@ def __init__(self, filename: str) -> None:
             if not line or line == '\n' or line[0] == ';':
                 continue
             if line[0] == '=':
-                value, symbol, _ = line.split(maxsplit=2)
+                value, symbol = line.split(maxsplit=2)[:2]
                 value = value[1:]
                 if value:
                     try: