Merge pull request #3446 from trailofbits/update-magic-defs

Update libmagic pattern defs

Author: ESultanik

file

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: acorn,v 1.8 2021/04/26 15:56:00 christos Exp $
+# $File: acorn,v 1.9 2024/08/30 17:29:28 christos Exp $
 # acorn:  file(1) magic for files found on Acorn systems
 #
 
@@ -67,36 +67,3 @@
 >>8	byte	x	version %d,
 >>10	leshort	=1	1 pattern
 >>10	leshort	!1	%d patterns
-
-# From: Joerg Jenderek
-# URL: https://www.kyzer.me.uk/pack/xad/#PackDir
-# reference: https://www.kyzer.me.uk/pack/xad/xad_PackDir.lha/PackDir.c
-# GRR: line below is too general as it matches also "Git pack" in ./revision
-0	string	PACK\0
-# check for valid compression method 0-4
->5	ulelong	<5
-# https://www.riscosopen.org/wiki/documentation/show/Introduction%20To%20Filing%20Systems
-# To skip "Git pack" version 0 test for root directory object like
-# ADFS::RPC.$.websitezip.FONTFIX
->>9	string	>ADFS\  PackDir archive (RISC OS)
-# TrID labels above as "Acorn PackDir compressed Archive"
-# compression mode y (0 - 4) for GIF LZW with a maximum n bits
-# (y~n,0~12,1~13,2~14,3~15,4~16)
->>>5	ulelong+12 x	\b, LZW %u-bits compression
-# https://www.filebase.org.uk/filetypes
-# !Packdir compressed archive has three hexadecimal digits code 68E
-!:mime	application/x-acorn-68E
-!:ext	pkd/bin
-# null terminated root directory object like IDEFS::IDE-4.$.Apps.GRAPHICS.!XFMPdemo
->>>9	string	x	\b, root "%s"
-# load address 0xFFFtttdd, ttt is the object filetype and dddddddddd is time
->>>>&1	ulelong	x	\b, load address %#x
-# execution address 0xdddddddd dddddddddd is 40 bit unsigned centiseconds since 1.1.1900 UTC
->>>>&5	ulelong	x	\b, exec address %#x
-# attributes (bits: 0~owner read,1~owner write,3~no delete,4~public read,5~public write)
->>>>&9	ulelong	x	\b, attributes %#x 
-# number of entries in this directory. for root dir 0
-#>>>&13	ulelong	x	\b, entries %#x 
-# the entries start here with object name
->>>>&17	string	x	\b, 1st object "%s"
-

polyfile/magic.py

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: adventure,v 1.18 2019/04/19 00:42:27 christos Exp $
+# $File: adventure,v 1.19 2023/12/02 13:48:56 christos Exp $
 # adventure: file(1) magic for Adventure game files
 #
 # from Allen Garvin <earendil@faeryland.tamu-commerce.edu>
@@ -10,6 +10,8 @@
 # ALAN
 # I assume there are other, lower versions, but these are the only ones I
 # saw in the archive.
+#
+# FIXME: Conflicts with Microsoft x.out big-endian and PDP-11 binaries (./xenix)
 0	beshort	0x0206	ALAN game data
 >2	byte	<10	version 2.6%d
 

polyfile/magic_defs/acorn

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: algol68,v 1.6 2022/11/06 18:36:55 christos Exp $
+# $File: algol68,v 1.7 2024/08/27 18:50:56 christos Exp $
 # algol68:  file(1) magic for Algol 68 source
 #
 # URL: 		https://en.wikipedia.org/wiki/ALGOL_68
@@ -13,8 +13,6 @@
 >0	use		algol_68
 0	regex/1024	\bMODE[\t\ ]
 >0	use		algol_68
-0	regex/1024	\bMODE[\t\ ]
->0	use		algol_68
 0	regex/1024	\bREF[\t\ ]
 >0	use		algol_68
 0	regex/1024	\bFLEX[\t\ ]\*\\[

polyfile/magic_defs/adventure

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: amigaos,v 1.20 2021/09/20 00:42:19 christos Exp $
+# $File: amigaos,v 1.23 2026/02/05 18:49:06 christos Exp $
 # amigaos:  file(1) magic for AmigaOS binary formats:
 
 #
@@ -36,7 +36,6 @@
 0	string		COSO\0		Hippel-COSO Module sound file
 # Too simple (short, pure ASCII, deep), MPi
 #26	string		V.3		Brian Postma's Soundmon Module sound file v3
-#26	string		BPSM		Brian Postma's Soundmon Module sound file v3
 #26	string		V.2		Brian Postma's Soundmon Module sound file v2
 
 # The following are from: "Stefan A. Haubenthal" <polluks@web.de>
@@ -195,8 +194,7 @@
 0	string		LZX		LZX compressed archive (Amiga)
 
 # From: Przemek Kramarczyk <pkramarczyk@gmail.com>
-0	string 		.KEY		AmigaDOS script
-0	string 		.key		AmigaDOS script
+0	string/c	.key		AmigaDOS script
 
 # AMOS Basic file formats
 # https://www.exotica.org.uk/wiki/AMOS_file_formats
@@ -216,3 +214,7 @@
 >12	regex		.{8}			\b, type %s
 0	string		AmBs			AMOS Basic memory banks
 >4	beshort		x			\b, %d banks
+
+
+# https://github.com/alb42/Leu/blob/master/TCDReaderUnit.pas
+3	string		TURBOCALC		TurboCalc spreadsheet

polyfile/magic_defs/algol68

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------
-# $File: android,v 1.24 2023/02/20 16:51:59 christos Exp $
+# $File: android,v 1.26 2024/09/04 19:06:11 christos Exp $
 # Various android related magic entries
 #------------------------------------------------------------
 
@@ -9,11 +9,11 @@
 # Fixed to avoid regexec 17 errors on some dex files
 # From <diff@lookout.com> "Tim Strazzere"
 0	string	dex\n
->0	regex	dex\n[0-9]{2}\0	Dalvik dex file
->4	string	>000			version %s
+>0	regex	dex\n[0-9]{2}	Dalvik dex file
+>>4	string	>000			version %s
 0	string	dey\n
->0	regex	dey\n[0-9]{2}\0	Dalvik dex file (optimized for host)
->4	string	>000			version %s
+>0	regex	dey\n[0-9]{2}	Dalvik dex file (optimized for host)
+>>4	string	>000			version %s
 
 # Android bootimg format
 # From https://android.googlesource.com/\
@@ -223,15 +223,15 @@
 #            +/refs/heads/androidx-main/profileinstaller/profileinstaller/\
 #            src/main/java/androidx/profileinstaller/ProfileVersion.java
 0	string	pro\x00
->0	regex	pro\x000[0-9][0-9]\x00	Android ART profile
+>4	regex	0[0-9][0-9]	Android ART profile
 !:ext	prof
 >>4	string	001\x00	\b, version 001 N
 >>4	string	005\x00	\b, version 005 O
 >>4	string	009\x00	\b, version 009 O MR1
 >>4	string	010\x00	\b, version 010 P
 >>4	string	015\x00	\b, version 015 S
 0	string	prm\x00
->0	regex	prm\x000[0-9][0-9]\x00	Android ART profile metadata
+>0	regex	0[0-9][0-9]	Android ART profile metadata
 !:ext	profm
 >>4	string	001\x00	\b, version 001 N
 >>4	string	002\x00	\b, version 002

polyfile/magic_defs/amigaos

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: animation,v 1.93 2023/05/21 17:13:19 christos Exp $
+# $File: animation,v 1.101 2025/05/28 19:54:08 christos Exp $
 # animation:  file(1) magic for animation/movie formats
 #
 # animation formats
@@ -542,36 +542,39 @@
 >>2	byte&0xF0	!0xF0		MPEG ADTS, layer III, v1
 !:strength +20
 !:mime	audio/mpeg
->2	byte&0xF0	0x10		\b, 32 kbps
->2	byte&0xF0	0x20		\b, 40 kbps
->2	byte&0xF0	0x30		\b, 48 kbps
->2	byte&0xF0	0x40		\b, 56 kbps
->2	byte&0xF0	0x50		\b, 64 kbps
->2	byte&0xF0	0x60		\b, 80 kbps
->2	byte&0xF0	0x70		\b, 96 kbps
->2	byte&0xF0	0x80		\b, 112 kbps
->2	byte&0xF0	0x90		\b, 128 kbps
->2	byte&0xF0	0xA0		\b, 160 kbps
->2	byte&0xF0	0xB0		\b, 192 kbps
->2	byte&0xF0	0xC0		\b, 224 kbps
->2	byte&0xF0	0xD0		\b, 256 kbps
->2	byte&0xF0	0xE0		\b, 320 kbps
+>>>2	search/100	Xing		\b, variable bitrate
+>>>2	search/100	VBRI		\b, variable bitrate
+>>>2	default		x
+>>>>2	byte&0xF0	0x10		\b, 32 kbps
+>>>>2	byte&0xF0	0x20		\b, 40 kbps
+>>>>2	byte&0xF0	0x30		\b, 48 kbps
+>>>>2	byte&0xF0	0x40		\b, 56 kbps
+>>>>2	byte&0xF0	0x50		\b, 64 kbps
+>>>>2	byte&0xF0	0x60		\b, 80 kbps
+>>>>2	byte&0xF0	0x70		\b, 96 kbps
+>>>>2	byte&0xF0	0x80		\b, 112 kbps
+>>>>2	byte&0xF0	0x90		\b, 128 kbps
+>>>>2	byte&0xF0	0xA0		\b, 160 kbps
+>>>>2	byte&0xF0	0xB0		\b, 192 kbps
+>>>>2	byte&0xF0	0xC0		\b, 224 kbps
+>>>>2	byte&0xF0	0xD0		\b, 256 kbps
+>>>>2	byte&0xF0	0xE0		\b, 320 kbps
 # timing
->2	byte&0x0C	0x00		\b, 44.1 kHz
->2	byte&0x0C	0x04		\b, 48 kHz
->2	byte&0x0C	0x08		\b, 32 kHz
+>>>2	byte&0x0C	0x00		\b, 44.1 kHz
+>>>2	byte&0x0C	0x04		\b, 48 kHz
+>>>2	byte&0x0C	0x08		\b, 32 kHz
 # channels/options
->3	byte&0xC0	0x00		\b, Stereo
->3	byte&0xC0	0x40		\b, JntStereo
->3	byte&0xC0	0x80		\b, 2x Monaural
->3	byte&0xC0	0xC0		\b, Monaural
-#>1	byte		^0x01		\b, Data Verify
-#>2	byte		&0x02		\b, Packet Pad
-#>2	byte		&0x01		\b, Custom Flag
-#>3	byte		&0x08		\b, Copyrighted
-#>3	byte		&0x04		\b, Original Source
-#>3	byte&0x03	1		\b, NR: 50/15 ms
-#>3	byte&0x03	3		\b, NR: CCIT J.17
+>>>3	byte&0xC0	0x00		\b, Stereo
+>>>3	byte&0xC0	0x40		\b, JntStereo
+>>>3	byte&0xC0	0x80		\b, 2x Monaural
+>>>3	byte&0xC0	0xC0		\b, Monaural
+#>>>1	byte		^0x01		\b, Data Verify
+#>>>2	byte		&0x02		\b, Packet Pad
+#>>>2	byte		&0x01		\b, Custom Flag
+#>>>3	byte		&0x08		\b, Copyrighted
+#>>>3	byte		&0x04		\b, Original Source
+#>>>3	byte&0x03	1		\b, NR: 50/15 ms
+#>>>3	byte&0x03	3		\b, NR: CCIT J.17
 
 # MP2, M1A
 0       beshort&0xFFFE  0xFFFC         MPEG ADTS, layer II, v1
@@ -855,7 +858,7 @@
 # Live MPEG-4 audio streams (instead of RTP FlexMux)
 0       beshort&0xFFE0  0x56E0         MPEG-4 LOAS
 !:mime	audio/x-mp4a-latm
-#>1     beshort&0x1FFF  x              \b, %hu byte packet
+#>1     beshort&0x1FFF  x              \b, %u byte packet
 >3      byte&0xE0       0x40
 >>4     byte&0x3C       0x04           \b, single stream
 >>4     byte&0x3C       0x08           \b, 2 streams
@@ -927,24 +930,24 @@
 #
 # from Oskar Schirmer <schirmer@scara.com> Feb 3, 2001 (ISO 13818.1)
 # syncbyte      8 bit	0x47
-# error_ind     1 bit	-
-# payload_start 1 bit	1
-# priority      1 bit	-
-# PID          13 bit	0x0000
-# scrambling    2 bit	-
-# adaptfld_ctrl 2 bit	1 or 3
-# conti_count   4 bit	-
-0	belong&0xFF5FFF10	0x47400010
->188	byte			0x47		MPEG transport stream data
+# 188 bytes per packet
+0		byte			0x47
+>188	byte			0x47
+>>376	byte			0x47
+>>>564	byte			0x47
+>>>>752	byte			0x47		MPEG transport stream data
 !:mime  video/MP2T
 !:ext	ts
 
 # Blu-ray disc Audio-Video MPEG-2 transport stream
 # From: Alexandre Iooss <erdnaxe@crans.org>
 # URL: https://en.wikipedia.org/wiki/MPEG_transport_stream
 # Note: similar to ISO 13818.1 but with 4 extra bytes per packets
-4	belong&0xFF5FFF10	=0x47400010
->196	byte			=0x47		BDAV MPEG-2 Transport Stream (M2TS)
+4		byte			0x47
+>196	byte			0x47
+>>388	byte			0x47
+>>>580	byte			0x47
+>>>>772	byte			0x47		BDAV MPEG-2 Transport Stream (M2TS)
 !:mime	video/MP2T
 !:ext	m2ts/mts
 
@@ -1195,3 +1198,56 @@
 >30	lelong	x	\b, height: %d
 >34	lelong	x	\b, %d bit
 >38	lelong	x	\b, frames: %d
+
+# https://wiki.multimedia.cx/index.php/Duck_IVF
+0	string	DKIF	Duck IVF video file
+!:mime	video/x-ivf
+>4	leshort	>0	\b, version %d
+>8	string	x	\b, codec %s
+>12	leshort x	\b, %d
+>14	leshort x	\bx%d
+>24	lelong	>0	\b, %d frames
+
+
+# libplacebo cache file
+# https://libplacebo.org
+0	string	pl_cache	libplacebo cache
+>8	ulelong	x		\b, version %u
+>12	ulelong	=0		\b, empty
+>12	ulelong	=1		\b, 1 entry
+>12	ulelong	>1		\b, %u entries
+
+>4	byte	1		\b, version 3.00.00
+>4	byte	2		\b, version 3.03.00
+>4	byte	3		\b, version 4.00.00
+>4	byte	4		\b, version 4.02.00
+>4	byte	5		\b, version 5.00.00
+
+# live2d: file(1) magic for Live2D Cubism file formats
+# https://www.live2d.com/en/
+0	string/4	MOC3	Live2D Cubism MOC3
+>4	byte	0	\b, SDK version invalid/unknown (%d)
+>4	byte	1	\b, SDK version 3.0.00 - 3.2.07 (%d)
+>4	byte	2	\b, SDK version 3.3.00 - 3.3.03 (%d)
+>4	byte	3	\b, SDK version 4.0.00 - 4.1.05 (%d)
+>4	byte	4	\b, SDK version 4.2.00 - 4.2.02 (%d)
+>4	byte	5	\b, SDK version 5.0.00 (%d)
+>4	byte	>5	\b, SDK version unknown (%d)
+>5	byte	0	\b, little endian
+>5	byte	>0	\b, big endian
+!:mime	application/x-moc3-data
+!:ext	moc3
+
+0	string/4	CAFF	Live2D Cubism archive
+>4	ubyte	x	version %d
+>&0	ubyte	x	\b.%d
+>&1	ubyte	x	\b.%d
+>7	string/4	=----	\b, standard format
+>7	string/4	!----	\b, unknown format (%.4s)
+>11	ubyte	x	version %d
+>&0	ubyte	x	\b.%d
+>&1	ubyte	x	\b.%d
+>14	belong	=0	\b, no obfuscation
+>14	belong	!0	\b, XOR obfuscation key %d
+!:mime	application/x-cubism-archive
+!:ext	cmo3/can3

polyfile/magic_defs/android

@@ -1,18 +1,28 @@
 
 #------------------------------------------------------------------------------
-# $File: apache,v 1.1 2017/04/11 14:52:15 christos Exp $
+# $File: apache,v 1.3 2025/05/30 13:25:13 christos Exp $
 # apache: file(1) magic for Apache Big Data formats
 
 # Avro files
-0	string		Obj		Apache Avro
->3	byte		x		version %d
+0	string		Obj\001		Apache Avro, version 1
 
 # ORC files
 # Important information is in file footer, which we can't index to :(
 0	string		ORC		Apache ORC
 
-# Parquet files
-0	string		PAR1		Apache Parquet
+# Apache arrow file format
+# MIME: https://www.iana.org/assignments/media-types/application/vnd.apache.arrow.stream
+# Description: https://arrow.apache.org/docs/format/Columnar.html
+0	string		ARROW1		Apache Arrow columnar file
+!:mime application/vnd.apache.arrow.file
+!:ext arrow/feather
+
+# Apache parquet file format
+# MIME: https://www.iana.org/assignments/media-types/application/vnd.apache.parquet
+# Description: https://parquet.apache.org/docs/file-format/
+0	string		PAR1		Apache Parquet file
+!:mime application/vnd.apache.parquet
+!:ext parquet
 
 # Hive RC files
 0	string		RCF		Apache Hive RC file