duplicator_info_fm_decoder_from_g64.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Duplicator Info (DI) data decoder (FM encoding)\nThis notebook illustrates how to extract information from an FM sector as recorded in a G64 file.\n\nYou might want to replace the embedded data block with one from a different title and test the process."},{"metadata":{},"cell_type":"markdown","source":"## FM sector structure\n\nPre-index gap (It starts right after the first edge of the index hole signal):\n - 40 bytes filled with 0xff or 0x00\n - 6 bytes filled with 0x00\n\nIndex mark:\n - 1 byte index mark 0xfc = %11111100 (written with a clock sequence that, grouped on its own, results in a byte value of 0xd7 = %11010111)\n\nPost index gap:\n - 26 bytes filled with 0xff or 0x00\n - 6 bytes filled with 0x00\n\nID record (This starts the sector):\n - 1 byte ID address mark 0xfe = %11111110 (written with a clock sequence that, grouped on its own, results in a byte value of 0xc7 = %11000111)\n - 1 byte Cylinder number (0-based)\n - 1 byte Side number (0-based)\n - 1 byte Sector number (1-based)\n - 1 byte Sector length (log2(length of user data field)−7)\n - 2 bytes CRC\n\nID gap:\n - 11 bytes filled with 0xff or 0x00\n - 6 bytes filled with 0x00\n\nData record:\n - 1 byte data address mark 0xfb = %11111100 (written with a clock sequence that, grouped on its own, results in a byte value of 0xc7 = %11000111)\n - User data\n - 2 bytes CRC\n\nData gap:\n - 27 bytes filled with 0xff\n - 6 bytes filled with 0x00\n\nReference: \"The floppy user guide\" by Michael Haardt, Alain Knaff, David C. Niemi."},{"metadata":{},"cell_type":"markdown","source":"## Initialization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check versions\nimport sys\n\nprint(\"Python is:\", sys.version)\n\n# Import required libraries\nimport numpy as np","execution_count":1,"outputs":[{"output_type":"stream","text":"Python is: 3.7.8 | packaged by conda-forge | (default, Jul 23 2020, 03:54:19) \n[GCC 7.5.0]\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"## G64 data\nWe shall be working with partial data from track 36 of \"California Games\", taken from a G64 file. Only picking a block of the data allows us to speed up the decoding process, without loss of generality."},{"metadata":{"trusted":true},"cell_type":"code","source":"y = np.array([\n0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,\n0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,\n0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,\n0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,\n0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0xEA,0xFD,0x5D,0x5F,\n0x55,0x55,0x55,0x57,0x55,0x55,0x7D,0x55,0x75,0xFD,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,\n0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0x55,\n0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0xEA,0xDF,0x55,0x57,0x55,\n0x5D,0x55,0x5F,0x55,0x75,0x55,0x77,0x55,0x55,0x55,0x75,0x55,0x55,0xD5,0x55,0x5D,\n0x55,0x55,0xD5,0x55,0x5D,0x55,0x55,0x57,0x55,0xD5,0x7F,0x55,0xD5,0x5D,0x77,0x57,\n0x55,0x5D,0xD5,0x55,0x57,0x55,0x55,0x55,0x57,0x55,0x55,0x5F,0x77,0x5F,0x55,0x5F,\n0xD5,0x5D,0xF7,0x5F,0x55,0x5F,0x75,0x5F,0x55,0x75,0x57,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x75,0x5F,0x5F,0x7D,0x5F,\n0x75,0x5D,0x55,0x77,0x7F,0x75,0xFF,0x75,0xFD,0x75,0x75,0x75,0x77,0x77,0x5D,0x75,\n0xF7,0x75,0x57,0x77,0x75,0x5D,0x55,0x75,0xFD,0x75,0xFF,0x77,0x5D,0x75,0xF7,0x75,\n0x57,0x75,0xF5,0x55,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x55,0x55,0x55,0x55,0x55,\n0x55,0x55,0x55,0x5F,0x57,0x5F,0x5D,0x5F,0xD5,0x5D,0x55,0x5D,0xD5,0x5F,0x5F,0x5F,\n0x55,0x5F,0x55,0x7F,0x7F,0x55,0x55,0x55,0x75,0x55,0x55,0x7F,0x7F,0x55,0x55,0xD7,\n0x5D,0x55,0x57,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,\n0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,\n0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,0x55,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,\n0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x5D,0x55,0x55,\n0x55,0x55,0x55,0x55,0x55,0x55,0x55,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,\n0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,\n], dtype=np.uint8)","execution_count":2,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Convert to bits\nWe convert the data to a bit stream."},{"metadata":{"trusted":true},"cell_type":"code","source":"bitstream = np.unpackbits(y)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\".join(map(lambda b: str(b), bitstream))","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"'1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010111101010111111010101110101011111010101010101010101010101010101110101010101010101011111010101010101110101111111011111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111110101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101011110101011011111010101010101011101010101010111010101010101011111010101010111010101010101011101110101010101010101010101010111010101010101010101011101010101010101010111010101010101010101110101010101010101011101010101010101010101010111010101011101010101111111010101011101010101011101011101110101011101010101010111011101010101010101010101110101010101010101010101010101011101010101010101010101111101110111010111110101010101011111110101010101110111110111010111110101010101011111011101010101111101010101011101010101011101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101011101010101111101011111011111010101111101110101010111010101010101110111011111110111010111111111011101011111110101110101011101010111010101110111011101110101110101110101111101110111010101010111011101110111010101011101010101010111010111111101011101011111111101110111010111010111010111110111011101010101011101110101111101010101010101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010101010101010101010101010101010101010101010101010101010101010101011111010101110101111101011101010111111101010101011101010101010101110111010101010111110101111101011111010101010101111101010101011111110111111101010101010101010101010101110101010101010101010101111111011111110101010101010101110101110101110101010101010101110101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010111010101010101011101010101010101110101010101010101010101010101010101010101010101010101010101010101010101010111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"## Find data record positions\nFor FM, if we use the last two bytes of the ID gap and the first byte of the data record as search pattern, what we are looking for is the below:\n\n```\n1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 1 1 1  = clock sequence 0xff 0xff 0xc7\n 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 0 1 1 = data sequence  0x00 0x00 0xfb\n101010101010101010101010101010101111010101101111 = clock and data interleaved 0xaa 0xaa 0xaa 0xaa 0xf5 0x6f\n```\n\nWe can use this pattern in order to realign the stream."},{"metadata":{"trusted":true},"cell_type":"code","source":"searchseq = [1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,1,1,1,0,1,0,1,0,1,1,0,1,1,1,1]\nN = len(searchseq)\npossibles = np.where(bitstream == searchseq[0])[0]\n\nsyncpositions = []\nfor p in possibles:\n    check = bitstream[p:p+N]\n    if np.all(check == searchseq):\n        syncpositions.append(p)\n\nprint(syncpositions)","execution_count":5,"outputs":[{"output_type":"stream","text":"[951]\n","name":"stdout"},{"output_type":"stream","text":"/srv/conda/envs/notebook/lib/python3.7/site-packages/ipykernel_launcher.py:8: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.\n  \n","name":"stderr"}]},{"metadata":{},"cell_type":"markdown","source":"## Shift array to the data record\nWe are now shifting bits in the bitstream so that the latter is byte-aligned and we can better recognize the ID gap and data record."},{"metadata":{"trusted":true},"cell_type":"code","source":"b = np.roll(bitstream, -syncpositions[0])","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\".join(map(lambda b: format(b, \"02x\"), np.packbits(b)))","execution_count":7,"outputs":[{"output_type":"execute_result","execution_count":7,"data":{"text/plain":"'aaaaaaaaf56faaabaaaeaaafaabaaabbaaaaaabaaaaaeaaaaeaaaaeaaaaeaaaaabaaeabfaaeaaebbabaaaeeaaaabaaaaaaabaaaaafbbafaaafeaaefbafaaafbaafaabaabaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaabaafafbeafbaaeaabbbfbaffbafebabababbbbaebafbbaabbbbaaeaabafebaffbbaebafbbaabbafaaaaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaaaaaaaaaaaaaaaaafabafaeafeaaeaaaeeaafafafaaafaabfbfaaaaaabaaaaabfbfaaaaebaeaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaeaaaaaaaaaaaaaaaaaaffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffaaaaaaaaaaaaaaaaaaaaaaaaf57eaeafaaaaaaabaaaabeaabafeffffffffffffffffffffffffffffffffffffffffffaaaaaaaaaaaaaaaa'"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"## Remove clock bits and decode data"},{"metadata":{},"cell_type":"markdown","source":"Knowing that every other bit is a clock bit we can remove clock bits, so we are left with the data record."},{"metadata":{"trusted":true},"cell_type":"code","source":"c = b[1::2]","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\".join(map(lambda b: format(b, \"02x\"), np.packbits(c)))","execution_count":9,"outputs":[{"output_type":"execute_result","execution_count":9,"data":{"text/plain":"'0000fb01020304050004008020080200108708251028010001003530382d30343041202020202020202043363420574f4e4445524d4154204e4f524d414c0020202020202020202020202020202020202020202020202020202000000000313238202833303077000400770092010000000000000000000000000000000000000000000000202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202000000000fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff000000000000fe23000100604efffffffffffffffffffff00000000'"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"Let's now decode as ASCII. The string \"C64 WONDERMAT NORMAL\" is what stands out clearly now."},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\".join(map(lambda b: chr(b), np.packbits(c)))","execution_count":10,"outputs":[{"output_type":"execute_result","execution_count":10,"data":{"text/plain":"'\\x00\\x00û\\x01\\x02\\x03\\x04\\x05\\x00\\x04\\x00\\x80 \\x08\\x02\\x00\\x10\\x87\\x08%\\x10(\\x01\\x00\\x01\\x00508-040A        C64 WONDERMAT NORMAL\\x00                           \\x00\\x00\\x00\\x00128 (300w\\x00\\x04\\x00w\\x00\\x92\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00                                               \\x00\\x00\\x00\\x00ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿð\\x00\\x00\\x00\\x00\\x00\\x0fâ0\\x00\\x10\\x06\\x04ïÿÿÿÿÿÿÿÿÿÿ\\x00\\x00\\x00\\x00'"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"---"}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}