From 16e3b8b3705ac35ae238ac8e6fb45e5e4e31d13e Mon Sep 17 00:00:00 2001 From: Erik Kristensen Date: Fri, 10 Jan 2025 08:41:54 -0700 Subject: [PATCH 01/64] fix(cast): support 24.04 in config --- .cast.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cast.yml b/.cast.yml index 82edbf35..3a0ae35a 100644 --- a/.cast.yml +++ b/.cast.yml @@ -21,10 +21,10 @@ manifest: deprecated: true replacement: desktop supported_os: - - id: ubuntu - release: 20.04 - id: ubuntu release: 22.04 + - id: ubuntu + release: 24.04 saltstack: pillars: sift_user_template: "{{ .User }}" From d03f76376c371939d70be3faf27e9df4867615a6 Mon Sep 17 00:00:00 2001 From: Erik Kristensen Date: Fri, 10 Jan 2025 08:42:10 -0700 Subject: [PATCH 02/64] fix(package/exfat): remove focal support --- sift/packages/exfat-extras.sls | 16 +++++++++------- sift/packages/exfat-extras_focal.sls | 10 ---------- sift/packages/exfat-extras_jammy.sls | 10 ---------- 3 files changed, 9 insertions(+), 27 deletions(-) delete mode 100644 sift/packages/exfat-extras_focal.sls delete mode 100644 sift/packages/exfat-extras_jammy.sls diff --git a/sift/packages/exfat-extras.sls b/sift/packages/exfat-extras.sls index a5b27222..dfb798c7 100644 --- a/sift/packages/exfat-extras.sls +++ b/sift/packages/exfat-extras.sls @@ -1,8 +1,10 @@ -include: - - sift.packages.exfat-extras_{{ grains['oscodename'] }} +# Name: exfat-utils +# Website: https://github.com/relan/exfat +# Description: Free exFAT File System Implementation +# Category: +# Author: Relan +# License: GNU General Public License v2 (https://github.com/relan/exfat/blob/master/COPYING) +# Notes: -sift-package-exfat-extras-distro: - test.nop: - - name: sift-package-exfat-extras-distro - - require: - - sls: sift.packages.exfat-extras_{{ grains['oscodename'] }} \ No newline at end of file +exfatprogs: + pkg.installed diff --git a/sift/packages/exfat-extras_focal.sls b/sift/packages/exfat-extras_focal.sls deleted file mode 100644 index a4bda9eb..00000000 --- a/sift/packages/exfat-extras_focal.sls +++ /dev/null @@ -1,10 +0,0 @@ -# Name: exfat-utils -# Website: https://github.com/relan/exfat -# Description: Free exFAT File System Implementation -# Category: -# Author: Relan -# License: GNU General Public License v2 (https://github.com/relan/exfat/blob/master/COPYING) -# Notes: - -exfat-utils: - pkg.installed diff --git a/sift/packages/exfat-extras_jammy.sls b/sift/packages/exfat-extras_jammy.sls deleted file mode 100644 index dfb798c7..00000000 --- a/sift/packages/exfat-extras_jammy.sls +++ /dev/null @@ -1,10 +0,0 @@ -# Name: exfat-utils -# Website: https://github.com/relan/exfat -# Description: Free exFAT File System Implementation -# Category: -# Author: Relan -# License: GNU General Public License v2 (https://github.com/relan/exfat/blob/master/COPYING) -# Notes: - -exfatprogs: - pkg.installed From 6b2a151a256fabaae6f4633de32908902b1de5b4 Mon Sep 17 00:00:00 2001 From: Erik Kristensen Date: Fri, 10 Jan 2025 15:35:00 -0700 Subject: [PATCH 03/64] fix(config): remove salt-minion, salt is onedir installed, not an issue --- sift/config/init.sls | 2 -- sift/config/salt-minion.sls | 4 ---- 2 files changed, 6 deletions(-) delete mode 100644 sift/config/salt-minion.sls diff --git a/sift/config/init.sls b/sift/config/init.sls index f7aa6f7a..1ae8ec86 100644 --- a/sift/config/init.sls +++ b/sift/config/init.sls @@ -3,7 +3,6 @@ include: - sift.config.user - sift.config.timezone - sift.config.folders - - sift.config.salt-minion - sift.config.samba - sift.config.tools @@ -15,7 +14,6 @@ sift-config: - sls: sift.config.user - sls: sift.config.timezone - sls: sift.config.folders - - sls: sift.config.salt-minion - sls: sift.config.samba - sls: sift.config.tools diff --git a/sift/config/salt-minion.sls b/sift/config/salt-minion.sls deleted file mode 100644 index 6de651d0..00000000 --- a/sift/config/salt-minion.sls +++ /dev/null @@ -1,4 +0,0 @@ -salt-minion: - service.dead: - - name: salt-minion - - enable: False From ee8a6b104d9b04303ec1517c41a3a37626eac552 Mon Sep 17 00:00:00 2001 From: Erik Kristensen Date: Fri, 10 Jan 2025 15:40:29 -0700 Subject: [PATCH 04/64] fix: tests --- .github/workflows/tests.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 038e0145..d961057d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,9 +4,12 @@ on: push: branches: - master + - main + - next pull_request: branches: - master + - main - next jobs: @@ -29,18 +32,18 @@ jobs: if: ${{ needs.changed_states.outputs.matrix != '[]' }} strategy: matrix: - salt: [3006, 3007] - os: [20.04, 22.04] + salt: [3007, 3006] + os: [22.04, 24.04] state: ${{ fromJson(needs.changed_states.outputs.matrix) }} include: - - os: 20.04 - code: focal + - os: 24.04 + code: noble - os: 22.04 code: jammy container: - image: docker://ghcr.io/ekristen/cast-tools/saltstack-tester:${{ matrix.code }}-${{ matrix.salt }} + image: docker://ghcr.io/ekristen/cast-tools/saltstack-tester:${{ matrix.os }}-${{ matrix.salt }} steps: - uses: actions/checkout@v4 - name: test-state run: | - salt-call -l info --file-root . --local --retcode-passthrough --state-output=mixed state.sls ${{ matrix.state }} pillar="{sift_user: root}" + salt-call --local -l info --file-root . --retcode-passthrough --state-output=mixed state.sls ${{ matrix.state }} pillar="{sift_user: root}" From 94a9de15f36249f52f8fe00435f8e8cf56e81b5c Mon Sep 17 00:00:00 2001 From: Erik Kristensen Date: Fri, 10 Jan 2025 15:47:19 -0700 Subject: [PATCH 05/64] fix(scripts/docker-compose): docker-compose@2.23.2 with ARM64 support --- sift/scripts/docker-compose.sls | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sift/scripts/docker-compose.sls b/sift/scripts/docker-compose.sls index c93802f9..6897c14e 100644 --- a/sift/scripts/docker-compose.sls +++ b/sift/scripts/docker-compose.sls @@ -1,10 +1,9 @@ -{%- set version = "2.15.1" -%} -{%- set hash = "bcfd9ea51dee4c19dccdfaeef0e7956ef68bf14f3d175933742061a7271ef0f5" -%} +{%- set version = "2.23.2" -%} sift-scripts-docker-compose: file.managed: - name: /usr/local/bin/docker-compose - source: https://github.com/docker/compose/releases/download/v{{ version }}/docker-compose-{{ grains['kernel'] }}-{{ grains['cpuarch'] }} - - source_hash: sha256={{ hash }} + - source_hash: https://github.com/docker/compose/releases/download/v{{ version }}/docker-compose-{{ grains['kernel'] }}-{{ grains['cpuarch'] }}.sha256 - mode: 755 - replace: True From d787e744235f2f3e3d979d698db3022d27f75908 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 12 Jan 2025 19:53:53 +0000 Subject: [PATCH 06/64] Update defang with virtualenv --- sift/python3-packages/defang.sls | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/sift/python3-packages/defang.sls b/sift/python3-packages/defang.sls index d1bf3c66..6cd8838d 100644 --- a/sift/python3-packages/defang.sls +++ b/sift/python3-packages/defang.sls @@ -1,11 +1,31 @@ # WEBSITE: https://github.com/HurricaneLabs/machinae # LICENSE: MIT include: - - sift.python3-packages.pip + - sift.packages.python3-virtualenv -sift-python3-packages-defang: +sift-python3-package-defang-venv: + virtualenv.managed: + - name: /opt/defang + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-defang: pip.installed: - - name: defang==0.5.2 - - bin_env: /usr/bin/python3 + - name: defang + - bin_env: /opt/defang/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-defang-venv + +sift-python3-package-defang-symlink: + file.symlink: + - name: /usr/local/bin/defang + - target: /opt/defang/bin/defang + - makedirs: False - require: - - sls: sift.python3-packages.pip + - pip: sift-python3-package-defang From 66cb7ba3fdd9ea720de4205e6af2fa6822996f5f Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 12 Jan 2025 22:32:28 +0000 Subject: [PATCH 07/64] Add header documentation --- sift/python3-packages/defang.sls | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sift/python3-packages/defang.sls b/sift/python3-packages/defang.sls index 6cd8838d..62237d61 100644 --- a/sift/python3-packages/defang.sls +++ b/sift/python3-packages/defang.sls @@ -1,5 +1,11 @@ -# WEBSITE: https://github.com/HurricaneLabs/machinae -# LICENSE: MIT +# Name: defang +# Website: https://bitbucket.org/johannestaas/defang/src/master/ +# Description: Defangs and refangs malicious URLs +# Category: +# Author: Johan Nestaas +# License: GNU General Public License v2+ (https://bitbucket.org/johannestaas/defang/src/master/LICENSE) +# Notes: + include: - sift.packages.python3-virtualenv From 7f42c0de027e317625ed26c58aabb781bfb6d60b Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Mon, 13 Jan 2025 22:06:08 +0000 Subject: [PATCH 08/64] Update analyzemft to use a Virtualenv, point to new repo using python 3 --- sift/python-packages/analyzemft.sls | 14 ---------- sift/python-packages/init.sls | 2 -- sift/python3-packages/analyzemft.sls | 41 ++++++++++++++++++++++++++++ sift/python3-packages/init.sls | 2 ++ 4 files changed, 43 insertions(+), 16 deletions(-) delete mode 100644 sift/python-packages/analyzemft.sls create mode 100644 sift/python3-packages/analyzemft.sls diff --git a/sift/python-packages/analyzemft.sls b/sift/python-packages/analyzemft.sls deleted file mode 100644 index 0f6cd22b..00000000 --- a/sift/python-packages/analyzemft.sls +++ /dev/null @@ -1,14 +0,0 @@ -{%- set commit="64c71d7c8905a119b7abdf9813e6ef5f11d3ccf1" -%} -include: - - sift.packages.git - - sift.packages.python3-pip - - sift.packages.python2-pip - -analyzemft: - pip.installed: - - name: git+https://github.com/dkovar/analyzeMFT.git@{{ commit }} - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.git - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/init.sls b/sift/python-packages/init.sls index 6bd0298e..09827f2f 100644 --- a/sift/python-packages/init.sls +++ b/sift/python-packages/init.sls @@ -1,5 +1,4 @@ include: - - sift.python-packages.analyzemft - sift.python-packages.appcompatprocessor - sift.python-packages.argparse - sift.python-packages.bitstring @@ -32,7 +31,6 @@ sift-python-packages: test.nop: - name: sift-python-packages - require: - - sls: sift.python-packages.analyzemft - sls: sift.python-packages.appcompatprocessor - sls: sift.python-packages.argparse - sls: sift.python-packages.bitstring diff --git a/sift/python3-packages/analyzemft.sls b/sift/python3-packages/analyzemft.sls new file mode 100644 index 00000000..33c7a477 --- /dev/null +++ b/sift/python3-packages/analyzemft.sls @@ -0,0 +1,41 @@ +# Name: analyzeMFT +# Website: https://github.com/rowingdude/analyzeMFT +# Description: NTFS MFT File Parser +# Category: +# Author: Benjamin Cance +# License: MIT License (https://github.com/rowingdude/analyzeMFT/blob/master/LICENSE.txt) +# Notes: analyzemft + +{% set commit = 'b1d0e6a0aa58d42000bfdb8e6588513bd62eaeab' %} + +include: + - sift.packages.python3-virtualenv + - sift.packages.git + +sift-python3-package-analyzemft-virtualenv: + virtualenv.managed: + - name: /opt/analyzemft + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-analyzemft: + pip.installed: + - name: git+https://github.com/rowingdude/analyzemft.git@{{ commit }} + - bin_env: /opt/analyzemft/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-analyzemft-virtualenv + - sls: sift.packages.git + +sift-python3-package-analyzemft-symlink: + file.symlink: + - name: /usr/local/bin/analyzemft + - target: /opt/analyzemft/bin/analyzemft + - makedirs: False + - require: + - pip: sift-python3-package-analyzemft diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index 4d8a3aff..b06aa9d9 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -1,4 +1,5 @@ include: + - sift.python3-packages.analyzemft - sift.python3-packages.python3-keyring - sift.python3-packages.pip - sift.python3-packages.python3-keyring @@ -31,6 +32,7 @@ sift-python3-packages: test.nop: - name: sift-python3-packages - require: + - sls: sift.python3-packages.analyzemft - sls: sift.python3-packages.python3-keyring - sls: sift.python3-packages.pip - sls: sift.python3-packages.python3-keyring From eab283964f7f2491f6448715d79f81c1115ccf3a Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Fri, 17 Jan 2025 14:59:41 +0000 Subject: [PATCH 09/64] Update bless for 24 --- sift/packages/bless.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sift/packages/bless.sls b/sift/packages/bless.sls index 32b71a94..ed566c56 100644 --- a/sift/packages/bless.sls +++ b/sift/packages/bless.sls @@ -5,6 +5,12 @@ # Author: Alexandros Frantzis # License: GNU General Public License v2.0 (https://github.com/afrantzis/bless/blob/master/COPYING) # Notes: bless +# TODO: fix when package is available +{% if grains['oscodename'] != 'noble' %} bless: pkg.installed +{% else %} +Bless is not available on Noble: + test.nop +{% endif %} From 8dca764e8d7cb8f2082cc0cceec310f5b1066c86 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Fri, 17 Jan 2025 18:30:39 +0000 Subject: [PATCH 10/64] Update cryptcat for 24 --- sift/packages/cryptcat.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sift/packages/cryptcat.sls b/sift/packages/cryptcat.sls index 8d40dc9f..a9223217 100644 --- a/sift/packages/cryptcat.sls +++ b/sift/packages/cryptcat.sls @@ -5,6 +5,12 @@ # Author: http://cryptcat.sourceforge.net/credits.php # License: GNU General Public License v2.0 # Notes: +# TODO: fix when package available +{% if grains['oscodename'] != 'noble' %} cryptcat: pkg.installed +{% else %} +Cryptcat is not available in Noble: + test.nop +{% endif %} From aea3700be561335aba72025ccdc764a5dda7230c Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 19 Jan 2025 21:42:46 +0000 Subject: [PATCH 11/64] Remove flasm to support 24 - not available on Jammy or Noble --- sift/packages/flasm.sls | 19 ------------------- sift/packages/init.sls | 2 -- 2 files changed, 21 deletions(-) delete mode 100644 sift/packages/flasm.sls diff --git a/sift/packages/flasm.sls b/sift/packages/flasm.sls deleted file mode 100644 index bb2c0f47..00000000 --- a/sift/packages/flasm.sls +++ /dev/null @@ -1,19 +0,0 @@ -# Name: flasm -# Website: https://www.nowrap.de/flasm.html -# Description: SWF dissassembler -# Category: -# Author: Igor Kogan -# License: BSD License (https://www.nowrap.de/flasm.html#useterms) -# Notes: flasm - -{% if grains['oscodename'] != "jammy" %} - -flasm: - pkg.installed - -{% else %} - -flasm-not-in-jammy: - test.nop - -{% endif %} diff --git a/sift/packages/init.sls b/sift/packages/init.sls index f72df38e..1edd39b3 100644 --- a/sift/packages/init.sls +++ b/sift/packages/init.sls @@ -40,7 +40,6 @@ include: - sift.packages.extundelete - sift.packages.fdupes - sift.packages.feh - - sift.packages.flasm - sift.packages.flex - sift.packages.foremost - sift.packages.g++ @@ -246,7 +245,6 @@ sift-packages: - sls: sift.packages.extundelete - sls: sift.packages.fdupes - sls: sift.packages.feh - - sls: sift.packages.flasm - sls: sift.packages.flex - sls: sift.packages.foremost - sls: sift.packages.g++ From cf8f491214ccf4e09912cb05a584ede03030d0c1 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 19 Jan 2025 21:57:29 +0000 Subject: [PATCH 12/64] Remove knocker to support 24 - not available on Jammy or Noble --- sift/packages/init.sls | 2 -- sift/packages/knocker.sls | 11 ----------- 2 files changed, 13 deletions(-) delete mode 100644 sift/packages/knocker.sls diff --git a/sift/packages/init.sls b/sift/packages/init.sls index f72df38e..55cc32a8 100644 --- a/sift/packages/init.sls +++ b/sift/packages/init.sls @@ -62,7 +62,6 @@ include: - sift.packages.ipython3 - sift.packages.jq - sift.packages.kdiff3 - - sift.packages.knocker - sift.packages.kpartx - sift.packages.lft - sift.packages.libafflib-dev @@ -268,7 +267,6 @@ sift-packages: - sls: sift.packages.ipython3 - sls: sift.packages.jq - sls: sift.packages.kdiff3 - - sls: sift.packages.knocker - sls: sift.packages.kpartx - sls: sift.packages.lft - sls: sift.packages.libafflib-dev diff --git a/sift/packages/knocker.sls b/sift/packages/knocker.sls deleted file mode 100644 index 76616e89..00000000 --- a/sift/packages/knocker.sls +++ /dev/null @@ -1,11 +0,0 @@ -{% if grains['oscodename'] != "jammy" %} - -knocker: - pkg.installed - -{% else %} - -knocker-not-in-jammy: - test.nop - -{% endif %} From 0d02512e9f535fc775d094ca9d0a37ad2a63e9ab Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 19 Jan 2025 22:11:58 +0000 Subject: [PATCH 13/64] Update libafflib for 24 --- sift/packages/libafflib-dev.sls | 10 +++++++++- sift/packages/libafflib.sls | 20 +++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/sift/packages/libafflib-dev.sls b/sift/packages/libafflib-dev.sls index 3254cb9f..f35364f7 100644 --- a/sift/packages/libafflib-dev.sls +++ b/sift/packages/libafflib-dev.sls @@ -1,2 +1,10 @@ +# Name: AFFLIBv3 +# Website: https://github.com/sshock/AFFLIBv3 +# Description: Development file for AFFLIB +# Category: +# Author: Simson L. Garfinkel / Phillip Hellewell et al (https://github.com/sshock/AFFLIBv3/blob/master/AUTHORS) +# License: Multiple Licenses (https://github.com/sshock/AFFLIBv3/blob/master/COPYING) +# Notes: + libafflib-dev: - pkg.installed \ No newline at end of file + pkg.installed diff --git a/sift/packages/libafflib.sls b/sift/packages/libafflib.sls index e6c810c5..e8304765 100644 --- a/sift/packages/libafflib.sls +++ b/sift/packages/libafflib.sls @@ -1,3 +1,21 @@ -libafflib: +# Name: AFFLIBv3 +# Website: https://github.com/sshock/AFFLIBv3 +# Description: AFF is an open and extensible file format to store disk images +# Category: +# Author: Simson L. Garfinkel / Phillip Hellewell et al (https://github.com/sshock/AFFLIBv3/blob/master/AUTHORS) +# License: Multiple Licenses (https://github.com/sshock/AFFLIBv3/blob/master/COPYING) +# Notes: + +{% if grains['oscodename'] == 'jammy' %} + +sift-package-libafflib0v5: pkg.installed: - name: libafflib0v5 + +{% elif grains['oscodename'] == 'noble' %} + +sift-package-libafflib0t64: + pkg.installed: + - name: libafflib0t64 + +{% endif %} From 775dd5d78103a87751f3637bf377191ec907aa9d Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 19 Jan 2025 22:23:36 +0000 Subject: [PATCH 14/64] Update libext2fs2 for 24 --- sift/packages/libext2fs2.sls | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/sift/packages/libext2fs2.sls b/sift/packages/libext2fs2.sls index bdef2318..462e10e3 100644 --- a/sift/packages/libext2fs2.sls +++ b/sift/packages/libext2fs2.sls @@ -1,2 +1,21 @@ -libext2fs2: - pkg.installed +# Name: libext2fs2 (e2fsprogs) +# Website: https://e2fsprogs.sourceforge.net/ +# Description: File system utilities for use with the ext2 file system +# Category: +# Author: Theodore Ts'o (https://thunk.org/tytso/) +# License: GNU General Public License v2 and GNU Library General Public License v2 (https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git/tree/NOTICE) +# Notes: + +{% if grains['oscodename'] == 'jammy' %} + +sift-package-libext2fs2: + pkg.installed: + - name: libext2fs2 + +{% elif grains['oscodename'] == 'noble' %} + +sift-package-libext2fs2t64: + pkg.installed: + - name: libext2fs2t64 + +{% endif %} From 9eb20231615ad4be6bda978e31c9cdbdee46382c Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 19 Jan 2025 22:37:00 +0000 Subject: [PATCH 15/64] Update libncurses for 24 --- sift/packages/libncurses.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sift/packages/libncurses.sls b/sift/packages/libncurses.sls index 334e93bb..bef9e47a 100644 --- a/sift/packages/libncurses.sls +++ b/sift/packages/libncurses.sls @@ -1,3 +1,3 @@ -libncurses: +sift-package-libncurses: pkg.installed: - - name: libncurses5-dev + - name: libncurses-dev From fa32b6c6cb3b784884baf70a627c78d0baea8f9f Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 19 Jan 2025 23:22:24 +0000 Subject: [PATCH 16/64] Update libpff states for 24 --- sift/packages/libpff-dev.sls | 13 +++++++++++-- sift/packages/libpff-tools.sls | 2 -- sift/packages/libpff.sls | 23 +++++++++++++++++++++-- sift/packages/pff-tools.sls | 13 +++++++++++-- sift/packages/python3-pypff.sls | 13 +++++++++++-- 5 files changed, 54 insertions(+), 10 deletions(-) delete mode 100644 sift/packages/libpff-tools.sls diff --git a/sift/packages/libpff-dev.sls b/sift/packages/libpff-dev.sls index 97b10e6e..3c632d66 100644 --- a/sift/packages/libpff-dev.sls +++ b/sift/packages/libpff-dev.sls @@ -1,2 +1,11 @@ -libpff-dev: - pkg.installed +# Name: libpff-dev +# Website: https://github.com/libyal/libpff +# Description: Development files for the libpff library +# Category: +# Author: Joachim Metz et al (https://github.com/libyal/libpff/blob/main/AUTHORS) +# License: GNU and GNU Lesser General Public License v3 (https://github.com/libyal/libpff/blob/main/COPYING) +# Notes: + +sift-package-libpff-dev: + pkg.installed: + - name: libpff-dev diff --git a/sift/packages/libpff-tools.sls b/sift/packages/libpff-tools.sls deleted file mode 100644 index 33533226..00000000 --- a/sift/packages/libpff-tools.sls +++ /dev/null @@ -1,2 +0,0 @@ -pff-tools: - pkg.installed diff --git a/sift/packages/libpff.sls b/sift/packages/libpff.sls index 22f0663b..65ecb374 100644 --- a/sift/packages/libpff.sls +++ b/sift/packages/libpff.sls @@ -1,2 +1,21 @@ -libpff1: - pkg.installed +# Name: libpff +# Website: https://github.com/libyal/libpff +# Description: Library to access the Personal and Offline File Folder formats (PST / OST) +# Category: +# Author: Joachim Metz et al (https://github.com/libyal/libpff/blob/main/AUTHORS) +# License: GNU and GNU Lesser General Public License v3 (https://github.com/libyal/libpff/blob/main/COPYING) +# Notes: + +{% if grains['oscodename'] == 'jammy' %} + +sift-package-libpff1: + pkg.installed: + - name: libpff1 + +{% elif grains['oscodename'] == 'noble' %} + +sift-package-libpff1t64: + pkg.installed: + - name: libpff1t64 + +{% endif %} diff --git a/sift/packages/pff-tools.sls b/sift/packages/pff-tools.sls index 33533226..6fde4395 100644 --- a/sift/packages/pff-tools.sls +++ b/sift/packages/pff-tools.sls @@ -1,2 +1,11 @@ -pff-tools: - pkg.installed +# Name: pff-tools +# Website: https://github.com/libyal/libpff +# Description: Library to access the Personal and Offline File Folder formats (PST / OST) +# Category: +# Author: Joachim Metz et al (https://github.com/libyal/libpff/blob/main/AUTHORS) +# License: GNU and GNU Lesser General Public License v3 (https://github.com/libyal/libpff/blob/main/COPYING) +# Notes: pffexport, pffinfo + +sift-package-pff-tools: + pkg.installed: + - name: pff-tools diff --git a/sift/packages/python3-pypff.sls b/sift/packages/python3-pypff.sls index 1a05a3cb..5a33910c 100644 --- a/sift/packages/python3-pypff.sls +++ b/sift/packages/python3-pypff.sls @@ -1,2 +1,11 @@ -python3-pypff: - pkg.installed +# Name: python3-pypff +# Website: https://github.com/libyal/libpff +# Description: Python3 bindings for the libpff library +# Category: +# Author: Joachim Metz et al (https://github.com/libyal/libpff/blob/main/AUTHORS) +# License: GNU and GNU Lesser General Public License v3 (https://github.com/libyal/libpff/blob/main/COPYING) +# Notes: + +sift-package-python3-pypff: + pkg.installed: + - name: python3-pypff From 60e55494aac1621465833f31c379cd73dc36e389 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Sun, 19 Jan 2025 23:45:13 +0000 Subject: [PATCH 17/64] Update netcat for 24 --- sift/packages/netcat.sls | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sift/packages/netcat.sls b/sift/packages/netcat.sls index 3d6058e1..9036fa74 100644 --- a/sift/packages/netcat.sls +++ b/sift/packages/netcat.sls @@ -1,2 +1,11 @@ -netcat: - pkg.installed +# Name: netcat-openbsd +# Website: https://github.com/openbsd/src/tree/master/usr.bin/nc +# Description: OpenBSD rewrite of the original netcat +# Category: +# Author: OpenBSD (see License for individual names) +# License: Multiple (https://git.launchpad.net/ubuntu/+source/netcat-openbsd/tree/debian/copyright) +# Notes: nc + +sift-package-netcat-openbsd: + pkg.installed: + - name: netcat-openbsd From e7ecd378e26b5ccaf5a755f6ec122ca571c5eb81 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Mon, 20 Jan 2025 01:26:23 +0000 Subject: [PATCH 18/64] Update libicu for 24 --- sift/packages/libicu.sls | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sift/packages/libicu.sls b/sift/packages/libicu.sls index 04ab6fae..6d8018e5 100644 --- a/sift/packages/libicu.sls +++ b/sift/packages/libicu.sls @@ -1,7 +1,13 @@ {% if grains['oscodename'] == 'focal' %} -libicu66: - pkg.installed +sift-package-libicu66: + pkg.installed: + - name: libicu66 {% elif grains['oscodename'] == 'jammy' %} -libicu70: - pkg.installed +sift-package-libicu70: + pkg.installed: + - name: libicu70 +{% elif grains['oscodename'] == 'noble' %} +sift-package-libicu74: + pkg.installed: + - name: libicu74 {% endif %} From c1e6d807229819eb646dce20a14acc026fd0ee0d Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Mon, 20 Jan 2025 01:31:27 +0000 Subject: [PATCH 19/64] Update PowerShell for 24 --- sift/packages/powershell.sls | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/sift/packages/powershell.sls b/sift/packages/powershell.sls index 089f41c2..e7bbffa3 100644 --- a/sift/packages/powershell.sls +++ b/sift/packages/powershell.sls @@ -1,24 +1,32 @@ +# Name: PowerShell +# Website: https://microsoft.com/powershell +# Description: Linux package for PowerShell +# Category: +# Author: Microsoft +# License: MIT License (https://github.com/PowerShell/PowerShell/blob/master/LICENSE.txt) +# Notes: + {# renovate: datasource=github-release-attachments depName=Powershell/Powershell #} -{%- set version = "7.4.1" -%} -{%- set hash = "625B7EE0B71147421723CB6022A41B5D8FC0D6E19DF25B1240008EE491BF6997" -%} +{%- set version = "7.4.6" -%} +{%- set hash = "79642721f0bc9baf07dafaab68ece1cbd822f86722492acf9b4031d41029a735" -%} {%- set filename = "powershell_" ~ version ~ "-1.deb_amd64.deb" -%} {%- set base_url = "https://github.com/Powershell/Powershell/releases/download/v" -%} include: - sift.packages.libicu -sift-powershell-source: +sift-package-powershell-source: file.managed: - name: /var/cache/sift/archives/{{ filename }} - source: "{{ base_url }}{{ version }}/{{ filename }}" - source_hash: sha256={{ hash }} - makedirs: True -sift-powershell: +sift-package-powershell: pkg.installed: - sources: - powershell: /var/cache/sift/archives/{{ filename }} - watch: - - file: sift-powershell-source + - file: sift-package-powershell-source - require: - sls: sift.packages.libicu From 3a27f04683ff3158143d51a4b0564355f56544ec Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Mon, 20 Jan 2025 01:55:05 +0000 Subject: [PATCH 20/64] Update qemu for 24 --- sift/packages/qemu.sls | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sift/packages/qemu.sls b/sift/packages/qemu.sls index ff7c687a..4da27bcd 100644 --- a/sift/packages/qemu.sls +++ b/sift/packages/qemu.sls @@ -1,2 +1,11 @@ -qemu: - pkg.installed \ No newline at end of file +# Name: qemu +# Website: https://www.qemu.org +# Description: A generic and open source machine emulator and virtualizer +# Category: +# Author: Multiple (https://gitlab.com/qemu-project/qemu/-/blob/master/MAINTAINERS) +# License: GNU General Public License v2 (https://gitlab.com/qemu-project/qemu/-/blob/master/LICENSE) +# Notes: + +sift-package-qemu-system: + pkg.installed: + - name: qemu-system From 308af62872a36bb78341b9e0d15caed97a8a6773 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Mon, 20 Jan 2025 13:34:20 +0000 Subject: [PATCH 21/64] Update ioc_writer for 24 --- sift/python-packages/init.sls | 2 -- sift/python-packages/ioc_writer.sls | 14 --------- sift/python3-packages/init.sls | 4 +-- sift/python3-packages/ioc-writer.sls | 47 ++++++++++++++++++++++++++++ sift/python3-packages/ioc_writer.sls | 13 -------- 5 files changed, 49 insertions(+), 31 deletions(-) delete mode 100644 sift/python-packages/ioc_writer.sls create mode 100644 sift/python3-packages/ioc-writer.sls delete mode 100644 sift/python3-packages/ioc_writer.sls diff --git a/sift/python-packages/init.sls b/sift/python-packages/init.sls index 09827f2f..170ef2bc 100644 --- a/sift/python-packages/init.sls +++ b/sift/python-packages/init.sls @@ -8,7 +8,6 @@ include: - sift.python-packages.docopt - sift.python-packages.geoip2 - sift.python-packages.indxparse - - sift.python-packages.ioc_writer - sift.python-packages.lxml - sift.python-packages.ntdsxtract - sift.python-packages.pefile @@ -40,7 +39,6 @@ sift-python-packages: - sls: sift.python-packages.docopt - sls: sift.python-packages.geoip2 - sls: sift.python-packages.indxparse - - sls: sift.python-packages.ioc_writer - sls: sift.python-packages.lxml - sls: sift.python-packages.ntdsxtract - sls: sift.python-packages.pefile diff --git a/sift/python-packages/ioc_writer.sls b/sift/python-packages/ioc_writer.sls deleted file mode 100644 index 076521eb..00000000 --- a/sift/python-packages/ioc_writer.sls +++ /dev/null @@ -1,14 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.python-packages.lxml - -sift-python-packages-ioc-writer: - pip.installed: - - name: ioc_writer - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip - - sls: sift.python-packages.lxml - diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index b06aa9d9..0b026012 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -7,7 +7,7 @@ include: - sift.python3-packages.bitstring - sift.python3-packages.colorama - sift.python3-packages.geoip2 - - sift.python3-packages.ioc_writer + - sift.python3-packages.ioc-writer ### - sift.python3-packages.imagemounter - sift.python3-packages.keyrings-alt - sift.python3-packages.lxml @@ -40,7 +40,7 @@ sift-python3-packages: - sls: sift.python3-packages.bitstring - sls: sift.python3-packages.colorama - sls: sift.python3-packages.geoip2 - - sls: sift.python3-packages.ioc_writer + - sls: sift.python3-packages.ioc-writer ### - sls: sift.python3-packages.imagemounter - sls: sift.python3-packages.keyrings-alt - sls: sift.python3-packages.lxml diff --git a/sift/python3-packages/ioc-writer.sls b/sift/python3-packages/ioc-writer.sls new file mode 100644 index 00000000..4b86d711 --- /dev/null +++ b/sift/python3-packages/ioc-writer.sls @@ -0,0 +1,47 @@ +# Name: ioc_writer +# Website: https://github.com/mandiant/ioc_writer/ +# Description: Tool to write and edit IOC objects +# Category: +# Author: William Gibb +# License: Apache License 2.0 (https://github.com/mandiant/ioc_writer/blob/master/LICENSE) +# Notes: iocdump, openioc_10_to_11, openioc_11_to_10 + +{% set files = ['iocdump','openioc_10_to_11','openioc_11_to_10'] %} + +include: + - sift.packages.python3-virtualenv + - sift.packages.libxml2-dev + - sift.packages.libxslt-dev + +sift-python3-package-ioc-writer-venv: + virtualenv.managed: + - name: /opt/ioc_writer + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - lxml + - yara-python + - require: + - sls: sift.packages.python3-virtualenv + - sls: sift.packages.libxml2-dev + - sls: sift.packages.libxslt-dev + +sift-python3-package-ioc-writer: + pip.installed: + - name: ioc_writer + - bin_env: /opt/ioc_writer/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-ioc-writer-venv + +{% for file in files %} +sift-python3-package-ioc-writer-symlink-{{ file }}: + file.symlink: + - name: /usr/local/bin/{{ file }} + - target: /opt/ioc_writer/bin/{{ file }} + - makedirs: False + - require: + - pip: sift-python3-package-ioc-writer +{% endfor %} diff --git a/sift/python3-packages/ioc_writer.sls b/sift/python3-packages/ioc_writer.sls deleted file mode 100644 index 34161993..00000000 --- a/sift/python3-packages/ioc_writer.sls +++ /dev/null @@ -1,13 +0,0 @@ -include: - - sift.python3-packages.pip - - sift.python3-packages.lxml - - sift.python3-packages.yara-python - -sift-python3-packages-ioc-writer: - pip.installed: - - name: ioc_writer - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip - - sls: sift.python3-packages.lxml - - sls: sift.python3-packages.yara-python From 4d8a4e76b21a5ca53c03603cc5b4c771bd38b265 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Mon, 20 Jan 2025 21:54:32 +0000 Subject: [PATCH 22/64] Add mac-apt and ensure support for 24 --- sift/packages/libbz2-dev.sls | 2 + sift/packages/zlib1g-dev.sls | 2 + sift/python3-packages/mac-apt.sls | 86 +++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 sift/packages/libbz2-dev.sls create mode 100644 sift/packages/zlib1g-dev.sls create mode 100644 sift/python3-packages/mac-apt.sls diff --git a/sift/packages/libbz2-dev.sls b/sift/packages/libbz2-dev.sls new file mode 100644 index 00000000..d33d2828 --- /dev/null +++ b/sift/packages/libbz2-dev.sls @@ -0,0 +1,2 @@ +libbz2-dev: + pkg.installed diff --git a/sift/packages/zlib1g-dev.sls b/sift/packages/zlib1g-dev.sls new file mode 100644 index 00000000..d5c9b853 --- /dev/null +++ b/sift/packages/zlib1g-dev.sls @@ -0,0 +1,2 @@ +zlib1g-dev: + pkg.installed diff --git a/sift/python3-packages/mac-apt.sls b/sift/python3-packages/mac-apt.sls new file mode 100644 index 00000000..9b6fdda8 --- /dev/null +++ b/sift/python3-packages/mac-apt.sls @@ -0,0 +1,86 @@ +# Name: mac_apt +# Website: https://github.com/ydkhatri/mac_apt +# Description: macOS and iOS Artifact Parsing Tool +# Category: +# Author: Yogesh Khatri +# License: MIT License (https://github.com/ydkhatri/mac_apt/blob/master/LICENSE.txt) +# Notes: mac_apt.py, mac_apt_artifact_only.py, mac_apt_mounted_sys_data.py, ios_apt.py, extract_apfs_fs.py + +{% set files = ['mac_apt.py','mac_apt_artifact_only.py','mac_apt_mounted_sys_data.py','ios_apt.py','extract_apfs_fs.py'] %} + +include: + - sift.packages.python3-virtualenv + - sift.packages.python3-dev + - sift.packages.libbz2-dev + - sift.packages.zlib1g-dev + - sift.packages.git + +sift-python3-package-mac-apt-venv: + virtualenv.managed: + - name: /opt/mac-apt + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - pybindgen==0.21.0 + - require: + - sls: sift.packages.python3-virtualenv + - sls: sift.packages.python3-dev + - sls: sift.packages.libbz2-dev + - sls: sift.packages.zlib1g-dev + +sift-python3-package-mac-apt-git: + git.latest: + - name: https://github.com/ydkhatri/mac_apt.git + - target: /opt/mac-apt/bin/mac_apt_git/ + - user: root + - rev: master + - force_clone: True + - force_reset: True + - require: + - sls: sift.packages.git + - virtualenv: sift-python3-package-mac-apt-venv + +sift-python3-package-mac-apt-requirements: + pip.installed: + - bin_env: /opt/mac-apt/bin/python3 + - requirements: /opt/mac-apt/bin/mac_apt_git/requirements.txt + - upgrade: False + - cwd: /opt/mac-apt/bin/mac_apt_git/ + - require: + - git: sift-python3-package-mac-apt-git + +{% for file in files %} + +sift-python3-package-mac-apt-chmod-{{ file }}: + file.managed: + - name: /opt/mac-apt/bin/mac_apt_git/{{ file }} + - mode: 755 + - require: + - pip: sift-python3-package-mac-apt-requirements + +sift-python3-package-mac-apt-prepend-{{ file }}: + file.prepend: + - name: /opt/mac-apt/bin/mac_apt_git/{{ file }} + - text: '#!/opt/mac-apt/bin/python3' + - watch: + - file: sift-python3-package-mac-apt-chmod-{{ file }} + +sift-python3-package-mac-apt-fix-crlf-{{ file }}: + file.replace: + - name: /opt/mac-apt/bin/mac_apt_git/{{ file }} + - pattern: '\r' + - repl: '' + - require: + - file: sift-python3-package-mac-apt-prepend-{{ file }} + +sift-python3-package-mac-apt-symlink-{{ file }}: + file.symlink: + - name: /usr/local/bin/{{ file }} + - target: /opt/mac-apt/bin/mac_apt_git/{{ file }} + - makedirs: False + - require: + - file: sift-python3-package-mac-apt-fix-crlf-{{ file }} + +{% endfor %} From bef1478519bfa56d40e32d94e0e43b2606d46455 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Tue, 21 Jan 2025 00:58:10 +0000 Subject: [PATCH 23/64] Update machinae for 24 --- sift/python3-packages/machinae.sls | 49 +++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/sift/python3-packages/machinae.sls b/sift/python3-packages/machinae.sls index 5203d7d2..d9ec90d0 100644 --- a/sift/python3-packages/machinae.sls +++ b/sift/python3-packages/machinae.sls @@ -1,13 +1,46 @@ -# WEBSITE: https://github.com/HurricaneLabs/machinae -# LICENSE: MIT +# Name: machinae +# Website: https://github.com/HurricaneLabs/machinae +# Description: Machinae Security Intelligence Collector +# Category: +# Author: Hurricane Labs +# License: MIT License (https://github.com/HurricaneLabs/machinae/blob/master/LICENSE.txt) +# Notes: + include: - - sift.python3-packages.pip - - sift.python3-packages.defang + - sift.packages.python3-virtualenv + +sift-python3-package-machinae-venv: + virtualenv.managed: + - name: /opt/machinae + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - defang + - require: + - sls: sift.packages.python3-virtualenv -sift-python3-packages-machinae: +sift-python3-package-machinae: pip.installed: - name: machinae - - bin_env: /usr/bin/python3 + - bin_env: /opt/machinae/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-machinae-venv + +sift-python3-package-machinae-symlink: + file.symlink: + - name: /usr/local/bin/machinae + - target: /opt/machinae/bin/machinae + - makedirs: False + - require: + - pip: sift-python3-package-machinae + +sift-python3-package-machine-config: + file.managed: + - name: /etc/machinae.yml + - source: https://raw.githubusercontent.com/HurricaneLabs/machinae/refs/heads/master/machinae.yml + - skip_verify: True - require: - - sls: sift.python3-packages.pip - - sls: sift.python3-packages.defang + - file: sift-python3-package-machinae-symlink From e0d6d4546f52efc58a9b0a40ef657235a971169e Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Tue, 21 Jan 2025 01:13:05 +0000 Subject: [PATCH 24/64] Update init for mac-apt --- sift/python3-packages/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index b06aa9d9..c10989d9 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -11,6 +11,7 @@ include: ### - sift.python3-packages.imagemounter - sift.python3-packages.keyrings-alt - sift.python3-packages.lxml + - sift.python3-packages.mac-apt - sift.python3-packages.machinae - sift.python3-packages.pefile - sift.python3-packages.pillow @@ -44,6 +45,7 @@ sift-python3-packages: ### - sls: sift.python3-packages.imagemounter - sls: sift.python3-packages.keyrings-alt - sls: sift.python3-packages.lxml + - sls: sift.python3-packages.mac-apt - sls: sift.python3-packages.machinae - sls: sift.python3-packages.pefile - sls: sift.python3-packages.pillow From 5ea15102102f985edac4fc92e598eb820dc605da Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Tue, 21 Jan 2025 02:06:25 +0000 Subject: [PATCH 25/64] Update state to have package names dependent on OS, but state title remains the same --- sift/packages/libafflib.sls | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/sift/packages/libafflib.sls b/sift/packages/libafflib.sls index e8304765..3a598dc9 100644 --- a/sift/packages/libafflib.sls +++ b/sift/packages/libafflib.sls @@ -7,15 +7,11 @@ # Notes: {% if grains['oscodename'] == 'jammy' %} - -sift-package-libafflib0v5: - pkg.installed: - - name: libafflib0v5 - + {% set package = 'libafflib0v5' %} {% elif grains['oscodename'] == 'noble' %} + {% set package = 'libafflib0t64' %} +{% endif %} -sift-package-libafflib0t64: +sift-package-libafflib: pkg.installed: - - name: libafflib0t64 - -{% endif %} + - name: {{ package }} From e9f94f21426d164f5d9de2a295deebb16b435c76 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 02:43:58 +0000 Subject: [PATCH 26/64] Update dotnet and zimmerman state --- sift/packages/dotnet.sls | 17 ++--------------- sift/scripts/zimmerman.sls | 23 ++++++++++++++--------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/sift/packages/dotnet.sls b/sift/packages/dotnet.sls index c39d5602..c9be360a 100644 --- a/sift/packages/dotnet.sls +++ b/sift/packages/dotnet.sls @@ -1,16 +1,3 @@ -{% if grains['oscodename'] == "focal" %} -include: - - sift.repos.microsoft - -dotnet6-install: +sift-package-dotnet8: pkg.installed: - - name: dotnet-sdk-6.0 - - require: - - sls: sift.repos.microsoft - -{% elif grains['oscodename'] == "jammy" %} -sift-package-dotnet6: - pkg.installed: - - name: dotnet-sdk-6.0 - -{% endif %} + - name: dotnet-sdk-8.0 diff --git a/sift/scripts/zimmerman.sls b/sift/scripts/zimmerman.sls index f6141c3d..543b9e0d 100644 --- a/sift/scripts/zimmerman.sls +++ b/sift/scripts/zimmerman.sls @@ -1,22 +1,21 @@ -{%- set user = salt['pillar.get']('sift_user', 'sansforensics') -%} -{%- set all_users = salt['user.list_users']() -%} -{%- if user == "root" -%} - {%- set home = "/root" -%} -{%- else -%} - {%- set home = "/home/" + user -%} -{%- endif -%} +# Name: Zimmerman Tools +# Website: https://ericzimmerman.github.io/#!index.md +# Description: A collection of Windows binaries to parse Windows artifacts, running with .NET +# Category: +# Author: Eric Zimmerman +# License: MIT License +# Notes: amcacheparser, appcompatcacheparser, bstrings, evtxecmd, iisgeolocate, jlecmd, lecmd, mftecmd, rbcmd, recentfilecacheparser, recmd, rla, sbecmd, sqlecmd, wxtcmd {% set tools = ['AmcacheParser','AppCompatCacheParser','bstrings','EvtxECmd','iisGeolocate','JLECmd','LECmd','MFTECmd','RBCmd','RecentFileCacheParser','RECmd','rla','SBECmd','SQLECmd','WxTCmd'] %} include: - sift.packages.dotnet - - sift.config.user.user {% for tool in tools %} download-{{ tool }}: file.managed: - name: /tmp/{{ tool }}.zip - - source: https://f001.backblazeb2.com/file/EricZimmermanTools/net6/{{ tool }}.zip + - source: https://download.ericzimmermanstools.com/net6/{{ tool }}.zip - skip_verify: True - makedirs: True @@ -42,4 +41,10 @@ extract-{{ tool }}: {% endif %} - mode: 755 - replace: True + +remove-{{ tool }}-zip: + file.absent: + - name: /tmp/{{ tool }}.zip + - require: + - file: {{ tool }}-wrapper {% endfor %} From 335d59a17f33c270c8487f65157e3780c05a93ae Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 02:45:29 +0000 Subject: [PATCH 27/64] Update libbz2-dev and zlib1g-dev --- sift/packages/libbz2-dev.sls | 5 +++-- sift/packages/zlib1g-dev.sls | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sift/packages/libbz2-dev.sls b/sift/packages/libbz2-dev.sls index d33d2828..af7d9fe5 100644 --- a/sift/packages/libbz2-dev.sls +++ b/sift/packages/libbz2-dev.sls @@ -1,2 +1,3 @@ -libbz2-dev: - pkg.installed +sift-package-libbz2-dev: + pkg.installed: + - name: libbz2-dev diff --git a/sift/packages/zlib1g-dev.sls b/sift/packages/zlib1g-dev.sls index d5c9b853..a89aedc1 100644 --- a/sift/packages/zlib1g-dev.sls +++ b/sift/packages/zlib1g-dev.sls @@ -1,2 +1,3 @@ -zlib1g-dev: - pkg.installed +sift-package-zlib1g-dev: + pkg.installed: + - name: zlib1g-dev From 12df4d8554da998d742e159a95da3d89108b8cfc Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 03:02:09 +0000 Subject: [PATCH 28/64] Fix the naming structure for libicu --- sift/packages/libicu.sls | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/sift/packages/libicu.sls b/sift/packages/libicu.sls index 6d8018e5..44e2a810 100644 --- a/sift/packages/libicu.sls +++ b/sift/packages/libicu.sls @@ -1,13 +1,8 @@ -{% if grains['oscodename'] == 'focal' %} -sift-package-libicu66: - pkg.installed: - - name: libicu66 -{% elif grains['oscodename'] == 'jammy' %} -sift-package-libicu70: - pkg.installed: - - name: libicu70 +{% if grains['oscodename'] == 'jammy' %} + {% set package = 'libicu70' %} {% elif grains['oscodename'] == 'noble' %} -sift-package-libicu74: - pkg.installed: - - name: libicu74 + {% set package = 'libicu74' %} {% endif %} +sift-package-libicu: + pkg.installed: + - name: {{ package }} From 1ee39629760bece821af03fa135be2523e6a6ef2 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 04:31:51 +0000 Subject: [PATCH 29/64] Add dotnet-backports repo and support for dotnet9 zimmerman tools --- sift/packages/dotnet.sls | 9 +++++++-- sift/repos/dotnet-backports.sls | 12 ++++++++++++ sift/repos/init.sls | 4 ++-- sift/scripts/zimmerman.sls | 2 +- 4 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 sift/repos/dotnet-backports.sls diff --git a/sift/packages/dotnet.sls b/sift/packages/dotnet.sls index c9be360a..bb697d54 100644 --- a/sift/packages/dotnet.sls +++ b/sift/packages/dotnet.sls @@ -1,3 +1,8 @@ -sift-package-dotnet8: +include: + - sift.repos.dotnet-backports + +sift-package-dotnet9: pkg.installed: - - name: dotnet-sdk-8.0 + - name: dotnet-sdk-9.0 + - require: + - sls: sift.repos.dotnet-backports diff --git a/sift/repos/dotnet-backports.sls b/sift/repos/dotnet-backports.sls new file mode 100644 index 00000000..bf33949d --- /dev/null +++ b/sift/repos/dotnet-backports.sls @@ -0,0 +1,12 @@ +include: + - sift.packages.software-properties-common + +sift-dotnet-backports-repo: + pkgrepo.managed: + - name: dotnet-backports + - ppa: dotnet/backports + - keyid: 45A3F127159BE9E5017811C62125B164E8E5D3FA + - keyserver: hkp://p80.pool.sks-keyservers.net:80 + - refresh: true + - require: + - sls: sift.packages.software-properties-common diff --git a/sift/repos/init.sls b/sift/repos/init.sls index bc079fbe..44e33226 100644 --- a/sift/repos/init.sls +++ b/sift/repos/init.sls @@ -5,7 +5,7 @@ include: - sift.repos.openjdk - sift.repos.ubuntu-multiverse - sift.repos.ubuntu-universe - + - sift.repos.dotnet-backports sift-repos: test.nop: @@ -17,4 +17,4 @@ sift-repos: - sls: sift.repos.openjdk - sls: sift.repos.ubuntu-multiverse - sls: sift.repos.ubuntu-universe - + - sls: sift.repos.dotnet-backports diff --git a/sift/scripts/zimmerman.sls b/sift/scripts/zimmerman.sls index 543b9e0d..05664277 100644 --- a/sift/scripts/zimmerman.sls +++ b/sift/scripts/zimmerman.sls @@ -15,7 +15,7 @@ include: download-{{ tool }}: file.managed: - name: /tmp/{{ tool }}.zip - - source: https://download.ericzimmermanstools.com/net6/{{ tool }}.zip + - source: https://download.ericzimmermanstools.com/net9/{{ tool }}.zip - skip_verify: True - makedirs: True From bfce77dd2da696d97bc4fefc6cde4d96262a542d Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 15:08:02 +0000 Subject: [PATCH 30/64] Fix package style for libpff --- sift/packages/libpff.sls | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sift/packages/libpff.sls b/sift/packages/libpff.sls index 65ecb374..0cca7323 100644 --- a/sift/packages/libpff.sls +++ b/sift/packages/libpff.sls @@ -7,15 +7,12 @@ # Notes: {% if grains['oscodename'] == 'jammy' %} - -sift-package-libpff1: - pkg.installed: - - name: libpff1 - + {% set package = 'libpff1' %} {% elif grains['oscodename'] == 'noble' %} + {% set package = 'libpff1t64' %} +{% endif %} -sift-package-libpff1t64: +sift-package-libpff1: pkg.installed: - - name: libpff1t64 + - name: {{ package }} -{% endif %} From 6b5e6c270a4e3b21d7498a7e4aa74000cb364fab Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 15:11:42 +0000 Subject: [PATCH 31/64] Update package style for libext2fs2 --- sift/packages/libext2fs2.sls | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sift/packages/libext2fs2.sls b/sift/packages/libext2fs2.sls index 462e10e3..73938ad3 100644 --- a/sift/packages/libext2fs2.sls +++ b/sift/packages/libext2fs2.sls @@ -7,15 +7,12 @@ # Notes: {% if grains['oscodename'] == 'jammy' %} - -sift-package-libext2fs2: - pkg.installed: - - name: libext2fs2 - + {% set package = 'libext2fs2' %} {% elif grains['oscodename'] == 'noble' %} + {% set package = 'libext2fs2t64' %} +{% endif %} -sift-package-libext2fs2t64: +sift-package-libext2fs2: pkg.installed: - - name: libext2fs2t64 + - name: {{ package }} -{% endif %} From 6c9f3bd8ac283bd4bdfbbd11bd80cb2aadb00f05 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 18:18:05 +0000 Subject: [PATCH 32/64] Update python-evtx for 24 --- sift/python3-packages/python-evtx.sls | 46 +++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/sift/python3-packages/python-evtx.sls b/sift/python3-packages/python-evtx.sls index cde132cc..501f0589 100644 --- a/sift/python3-packages/python-evtx.sls +++ b/sift/python3-packages/python-evtx.sls @@ -4,16 +4,50 @@ # Category: # Author: Willi Ballenthin # License: Apache License 2.0 (https://github.com/williballenthin/python-evtx/blob/master/LICENSE.TXT) -# Notes: evtx_dates.py, evtx_dump.py, evtx_dump_chunk_slack.py, evtx_dump_json.py, evtx_info.py +# Notes: evtx_dump.py, evtx_dump_chunk_slack.py, evtx_dump_json.py, evtx_eid_record_numbers.py, evtx_extract_record.py, evtx_filter_records.py, evtx_info.py, evtx_record_structure.py, evtx_structure.py, evtx_templates.py + +{% set files = ['evtx_dump.py','evtx_dump_chunk_slack.py','evtx_dump_json.py','evtx_eid_record_numbers.py','evtx_extract_record.py','evtx_filter_records.py','evtx_info.py','evtx_record_structure.py','evtx_structure.py','evtx_templates.py'] %} include: - - sift.python3-packages.pip + - sift.packages.python3-virtualenv - sift.packages.git -sift-python3-packages-python-evtx: +sift-python3-package-python-evtx-venv: + virtualenv.managed: + - name: /opt/python-evtx + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - xmltodict + - lxml + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-python-evtx: pip.installed: - name: git+https://github.com/williballenthin/python-evtx.git - - bin_env: /usr/bin/python3 + - bin_env: /opt/python-evtx/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-python-evtx-venv + +sift-python3-package-python-evtx-import-fix: + file.replace: + - name: /opt/python-evtx/bin/evtx_eid_record_numbers.py + - pattern: 'from filter_records' + - repl: 'from evtx_filter_records' + - count: 1 + - require: + - pip: sift-python3-package-python-evtx + +{% for file in files %} +sift-python3-package-python-evtx-symlink-{{ file }}: + file.symlink: + - name: /usr/local/bin/{{ file }} + - target: /opt/python-evtx/bin/{{ file }} + - makedirs: False - require: - - sls: sift.python3-packages.pip - - sls: sift.packages.git + - pip: sift-python3-package-python-evtx +{% endfor %} From 2f318da7677f21d6c575578894720d4fc6e8dbbf Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 18:21:05 +0000 Subject: [PATCH 33/64] Add git requirement --- sift/python3-packages/python-evtx.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/sift/python3-packages/python-evtx.sls b/sift/python3-packages/python-evtx.sls index 501f0589..63ada010 100644 --- a/sift/python3-packages/python-evtx.sls +++ b/sift/python3-packages/python-evtx.sls @@ -32,6 +32,7 @@ sift-python3-package-python-evtx: - upgrade: True - require: - virtualenv: sift-python3-package-python-evtx-venv + - sls: sift.packages.git sift-python3-package-python-evtx-import-fix: file.replace: From 3892075c4d842877cf9cd97a2c112baab52bd569 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Wed, 22 Jan 2025 22:46:19 +0000 Subject: [PATCH 34/64] Update amcache script for 24 --- sift/files/amcache/amcache.py | 226 ++++++++++++++++++++++ sift/python3-packages/init.sls | 2 - sift/python3-packages/python-registry.sls | 9 - sift/scripts/amcache.sls | 56 ++++-- 4 files changed, 266 insertions(+), 27 deletions(-) create mode 100644 sift/files/amcache/amcache.py delete mode 100644 sift/python3-packages/python-registry.sls diff --git a/sift/files/amcache/amcache.py b/sift/files/amcache/amcache.py new file mode 100644 index 00000000..a14f24f2 --- /dev/null +++ b/sift/files/amcache/amcache.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# This file is part of python-registry. +# +# Copyright 2015 Will Ballenthin +# while at Mandiant Exe +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import logging +import datetime +from collections import namedtuple + +import argparse +import csv +from Registry import Registry +from Registry.RegistryParse import parse_windows_timestamp as _parse_windows_timestamp + + +g_logger = logging.getLogger("amcache") +Field = namedtuple("Field", ["name", "getter"]) + + +def make_value_getter(value_name): + """ return a function that fetches the value from the registry key """ + def _value_getter(key): + try: + return key.value(value_name).value() + except Registry.RegistryValueNotFoundException: + return None + return _value_getter + + +def make_windows_timestamp_value_getter(value_name): + """ + return a function that fetches the value from the registry key + as a Windows timestamp. + """ + f = make_value_getter(value_name) + def _value_getter(key): + try: + return parse_windows_timestamp(f(key) or 0) + except ValueError: + return datetime.datetime.min + return _value_getter + + +def parse_unix_timestamp(qword): + return datetime.datetime.fromtimestamp(qword) + + +def parse_windows_timestamp(qword): + try: + return _parse_windows_timestamp(qword) + except ValueError: + return datetime.datetime.min + + +def make_unix_timestamp_value_getter(value_name): + """ + return a function that fetches the value from the registry key + as a UNIX timestamp. + """ + f = make_value_getter(value_name) + def _value_getter(key): + try: + return parse_unix_timestamp(f(key) or 0) + except ValueError: + return datetime.datetime.min + return _value_getter + + +UNIX_TIMESTAMP_ZERO = parse_unix_timestamp(0) +WINDOWS_TIMESTAMP_ZERO = parse_windows_timestamp(0) + + +# via: http://www.swiftforensics.com/2013/12/amcachehve-in-windows-8-goldmine-for.html +#Product Name UNICODE string +#============================================================================== +#0 Product Name UNICODE string +#1 Company Name UNICODE string +#2 File version number only UNICODE string +#3 Language code (1033 for en-US) DWORD +#4 SwitchBackContext QWORD +#5 File Version UNICODE string +#6 File Size (in bytes) DWORD +#7 PE Header field - SizeOfImage DWORD +#8 Hash of PE Header (unknown algorithm) UNICODE string +#9 PE Header field - Checksum DWORD +#a Unknown QWORD +#b Unknown QWORD +#c File Description UNICODE string +#d Unknown, maybe Major & Minor OS version DWORD +#f Linker (Compile time) Timestamp DWORD - Unix time +#10 Unknown DWORD +#11 Last Modified Timestamp FILETIME +#12 Created Timestamp FILETIME +#15 Full path to file UNICODE string +#16 Unknown DWORD +#17 Last Modified Timestamp 2 FILETIME +#100 Program ID UNICODE string +#101 SHA1 hash of file + + +# note: order here implicitly orders CSV column ordering cause I'm lazy +FIELDS = [ + Field("path", make_value_getter("15")), + Field("sha1", make_value_getter("101")), + Field("size", make_value_getter("6")), + Field("file_description", make_value_getter("c")), + Field("source_key_timestamp", lambda key: key.timestamp()), + Field("created_timestamp", make_windows_timestamp_value_getter("12")), + Field("modified_timestamp", make_windows_timestamp_value_getter("11")), + Field("modified_timestamp2", make_windows_timestamp_value_getter("17")), + Field("linker_timestamp", make_unix_timestamp_value_getter("f")), + Field("product", make_value_getter("0")), + Field("company", make_value_getter("1")), + Field("pe_sizeofimage", make_value_getter("7")), + Field("version_number", make_value_getter("2")), + Field("version", make_value_getter("5")), + Field("language", make_value_getter("3")), + Field("header_hash", make_value_getter("8")), + Field("pe_checksum", make_value_getter("9")), + Field("id", make_value_getter("100")), + Field("switchbackcontext", make_value_getter("4")), +] + + +ExecutionEntry = namedtuple("ExecutionEntry", map(lambda e: e.name, FIELDS)) + + +def parse_execution_entry(key): + return ExecutionEntry(**dict((e.name, e.getter(key)) for e in FIELDS)) + + + +class NotAnAmcacheHive(Exception): + pass + + +def parse_execution_entries(registry): + try: + volumes = registry.open("Root\\File") + except Registry.RegistryKeyNotFoundException: + raise NotAnAmcacheHive() + ret = [] + for volumekey in volumes.subkeys(): + for filekey in volumekey.subkeys(): + ret.append(parse_execution_entry(filekey)) + return ret + + +TimelineEntry = namedtuple("TimelineEntry", ["timestamp", "type", "entry"]) + + +def main(): + + parser = argparse.ArgumentParser( + description="Parse program execution entries from the Amcache.hve Registry hive") + parser.add_argument("registry_hive", type=str, + help="Path to the Amcache.hve hive to process") + parser.add_argument("-v", action="store_true", dest="verbose", + help="Enable verbose output") + parser.add_argument("-t", action="store_true", dest="do_timeline", + help="Output in simple timeline format") + + if len(sys.argv[1:]) == 0: + parser.print_help() + parser.exit() + + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + if sys.platform == "win32": + import os, msvcrt + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + + r = Registry.Registry(args.registry_hive) + + try: + ee = parse_execution_entries(r) + except NotAnAmcacheHive: + g_logger.error("doesn't appear to be an Amcache.hve hive") + return + + if args.do_timeline: + entries = [] + for e in ee: + for t in ["source_key_timestamp", "created_timestamp", "modified_timestamp", + "modified_timestamp2", "linker_timestamp"]: + ts = getattr(e, t) + if ts == UNIX_TIMESTAMP_ZERO: + continue + if ts == WINDOWS_TIMESTAMP_ZERO: + continue + if ts == datetime.datetime.min: + continue + + entries.append(TimelineEntry(ts, t, e)) + w = csv.writer(sys.stdout, delimiter="|", quotechar="\"", quoting=csv.QUOTE_MINIMAL) + w.writerow(["timestamp", "timestamp_type", "path", "sha1"]) + for e in sorted(entries, key=lambda e: e.timestamp): + w.writerow([e.timestamp, e.type, e.entry.path, e.entry.sha1]) + else: + w = csv.writer(sys.stdout, delimiter="|", quotechar="\"", quoting=csv.QUOTE_MINIMAL) + w.writerow(map(lambda e: e.name, FIELDS)) + for e in ee: + w.writerow(map(lambda i: getattr(e, i.name), FIELDS)) + + +if __name__ == "__main__": + main() diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index 73eaec8c..2ddcdcf1 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -19,7 +19,6 @@ include: - sift.python3-packages.python-dateutil - sift.python3-packages.python-evtx - sift.python3-packages.python-magic - - sift.python3-packages.python-registry - sift.python3-packages.setuptools - sift.python3-packages.setuptools-rust - sift.python3-packages.six @@ -53,7 +52,6 @@ sift-python3-packages: - sls: sift.python3-packages.python-dateutil - sls: sift.python3-packages.python-evtx - sls: sift.python3-packages.python-magic - - sls: sift.python3-packages.python-registry - sls: sift.python3-packages.setuptools - sls: sift.python3-packages.setuptools-rust - sls: sift.python3-packages.six diff --git a/sift/python3-packages/python-registry.sls b/sift/python3-packages/python-registry.sls deleted file mode 100644 index e8c3ab71..00000000 --- a/sift/python3-packages/python-registry.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-python-registry: - pip.installed: - - name: python-registry - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/scripts/amcache.sls b/sift/scripts/amcache.sls index 0d6b2764..f110f4cc 100644 --- a/sift/scripts/amcache.sls +++ b/sift/scripts/amcache.sls @@ -1,22 +1,46 @@ -# source=https://github.com/williballenthin/python-registry -# license=apache2 -# license_source=https://github.com/williballenthin/python-registry/blob/master/LICENSE.TXT +# Name: amcache +# Website: https://github.com/williballenthin/python-registry +# Description: Python script to parse amcache artifacts from the Amcache.hve registry file +# Category: +# Author: Willi Ballenthin +# License: Apache License v2 (https://github.com/williballenthin/python-registry/blob/master/LICENSE.TXT) +# Notes: amcache.py -{% set commit = "1a669eada6f7933798751e0cf482a9eb654c739b" -%} -{% set hash = "1065c23fdea1fde90e931bf5ccabc93b508bee0f6855a6ef2b3b9fd74495e279" -%} +include: + - sift.packages.python3-virtualenv -sift-scripts-amcache: - file.managed: - - name: /usr/local/bin/amcache.py - - source: https://raw.githubusercontent.com/williballenthin/python-registry/{{ commit }}/samples/amcache.py - - source_hash: sha256={{ hash }} - - mode: 755 +sift-python3-package-amcache-venv: + virtualenv.managed: + - name: /opt/amcache + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - python-registry + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-amcache: + file.recurse: + - name: /opt/amcache/bin/ + - source: salt://sift/files/amcache + - file_mode: 755 -sift-scripts-amcache-shebang: +sift-python3-package-amcache-shebang: file.replace: - - name: /usr/local/bin/amcache.py - - pattern: '#!/usr/bin/python' - - repl: '#!/usr/bin/env python2' + - name: /opt/amcache/bin/amcache.py + - pattern: '#!/usr/bin/env python3' + - repl: '#!/opt/amcache/bin/python3' - count: 1 - watch: - - file: sift-scripts-amcache + - file: sift-python3-package-amcache + +sift-python3-package-amcache-symlink: + file.symlink: + - name: /usr/local/bin/amcache.py + - target: /opt/amcache/bin/amcache.py + - makedirs: False + - force: True + - require: + - file: sift-python3-package-amcache From 93d630eb961c15e49b611f78007718b1586311a2 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Fri, 24 Jan 2025 15:44:31 +0000 Subject: [PATCH 35/64] Replace parseusn with usnparser, update for 24 --- sift/python3-packages/init.sls | 2 ++ sift/python3-packages/usnparser.sls | 37 +++++++++++++++++++++++++++++ sift/scripts/init.sls | 2 -- sift/scripts/parseusn.sls | 21 ---------------- 4 files changed, 39 insertions(+), 23 deletions(-) create mode 100644 sift/python3-packages/usnparser.sls delete mode 100644 sift/scripts/parseusn.sls diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index 73eaec8c..d69f5106 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -25,6 +25,7 @@ include: - sift.python3-packages.six - sift.python3-packages.stix-validator - sift.python3-packages.stix + - sift.python3-packages.usnparser - sift.python3-packages.virustotal-api - sift.python3-packages.wheel - sift.python3-packages.yara-python @@ -59,6 +60,7 @@ sift-python3-packages: - sls: sift.python3-packages.six - sls: sift.python3-packages.stix-validator - sls: sift.python3-packages.stix + - sls: sift.python3-packages.usnparser - sls: sift.python3-packages.virustotal-api - sls: sift.python3-packages.wheel - sls: sift.python3-packages.yara-python diff --git a/sift/python3-packages/usnparser.sls b/sift/python3-packages/usnparser.sls new file mode 100644 index 00000000..3d17908f --- /dev/null +++ b/sift/python3-packages/usnparser.sls @@ -0,0 +1,37 @@ +# Name: USN Journal Parser +# Website: https://github.com/digitalsleuth/USN-Journal-Parser +# Description: Python script to parse the NTFS USN Change Journal +# Category: +# Author: Adam Witt (PoorBillionaire) / Corey Forman (digitalsleuth) +# License: Apache License v2 (https://github.com/digitalsleuth/USN-Journal-Parser/blob/main/LICENSE) +# Notes: usnparser + +include: + - sift.packages.python3-virtualenv + +sift-python3-package-usnparser-venv: + virtualenv.managed: + - name: /opt/usnparser + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-usnparser: + pip.installed: + - name: git+https://github.com/digitalsleuth/USN-Journal-Parser.git + - bin_env: /opt/usnparser/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-usnparser-venv + +sift-python3-package-usnparser-symlink: + file.symlink: + - name: /usr/local/bin/usnparser + - target: /opt/usnparser/bin/usn.py + - force: True + - require: + - pip: sift-python3-package-usnparser diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index a29a0e68..3afea02b 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -12,7 +12,6 @@ include: - sift.scripts.keydet-tools - sift.scripts.packerid - sift.scripts.page-brute - - sift.scripts.parseusn - sift.scripts.pdf-tools - sift.scripts.pe-carver - sift.scripts.pescanner @@ -45,7 +44,6 @@ sift-scripts: - sls: sift.scripts.keydet-tools - sls: sift.scripts.packerid - sls: sift.scripts.page-brute - - sls: sift.scripts.parseusn - sls: sift.scripts.pdf-tools - sls: sift.scripts.pe-carver - sls: sift.scripts.pescanner diff --git a/sift/scripts/parseusn.sls b/sift/scripts/parseusn.sls deleted file mode 100644 index 8e63021e..00000000 --- a/sift/scripts/parseusn.sls +++ /dev/null @@ -1,21 +0,0 @@ -# source=https://github.com/superponible/DFIR/ -# license=MIT - -{% set commit = "ee681a07a0c32a5ccaea788cd7d012d19872f181" -%} -{% set hash = "sha256=4540eba4cdddcb0eab1bc21ccea6a6ab7c010936909bb233807dc9bf4189ab10" -%} - -sift-scripts-parseusn: - file.managed: - - name: /usr/local/bin/parseusn.py - - source: https://raw.githubusercontent.com/superponible/DFIR/{{ commit }}/parseusn.py - - source_hash: {{ hash }} - - mode: 755 - -sift-scripts-parseusn-shebang: - file.replace: - - name: /usr/local/bin/parseusn.py - - pattern: '#!/usr/bin/env python\n' - - repl: '#!/usr/bin/env python2\n' - - count: 1 - - watch: - - file: sift-scripts-parseusn From aa38b8e36f851cfe66a6165186582f17b4738751 Mon Sep 17 00:00:00 2001 From: digitalsleuth Date: Fri, 24 Jan 2025 17:16:25 +0000 Subject: [PATCH 36/64] Removed python 2 usnparser --- sift/python-packages/init.sls | 2 -- sift/python-packages/usnparser.sls | 11 ----------- 2 files changed, 13 deletions(-) delete mode 100644 sift/python-packages/usnparser.sls diff --git a/sift/python-packages/init.sls b/sift/python-packages/init.sls index 170ef2bc..dff39f96 100644 --- a/sift/python-packages/init.sls +++ b/sift/python-packages/init.sls @@ -21,7 +21,6 @@ include: - sift.python-packages.shellbags - sift.python-packages.six - sift.python-packages.unicodecsv - - sift.python-packages.usnparser - sift.python-packages.volatility - sift.python-packages.wheel - sift.python-packages.windowsprefetch @@ -52,7 +51,6 @@ sift-python-packages: - sls: sift.python-packages.shellbags - sls: sift.python-packages.six - sls: sift.python-packages.unicodecsv - - sls: sift.python-packages.usnparser - sls: sift.python-packages.volatility - sls: sift.python-packages.wheel - sls: sift.python-packages.windowsprefetch diff --git a/sift/python-packages/usnparser.sls b/sift/python-packages/usnparser.sls deleted file mode 100644 index 9fbe5ea7..00000000 --- a/sift/python-packages/usnparser.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-usnparser: - pip.installed: - - name: usnparser - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip From 5283960d2f8fe5f89814920ba454c0e1ffc0cb55 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:49:27 -0500 Subject: [PATCH 37/64] Update windowsprefetch for 24 (#164) --- sift/python-packages/init.sls | 2 -- sift/python-packages/windowsprefetch.sls | 11 ------- sift/python3-packages/windowsprefetch.sls | 37 +++++++++++++++++++++++ 3 files changed, 37 insertions(+), 13 deletions(-) delete mode 100644 sift/python-packages/windowsprefetch.sls create mode 100644 sift/python3-packages/windowsprefetch.sls diff --git a/sift/python-packages/init.sls b/sift/python-packages/init.sls index dff39f96..b7acdc9b 100644 --- a/sift/python-packages/init.sls +++ b/sift/python-packages/init.sls @@ -23,7 +23,6 @@ include: - sift.python-packages.unicodecsv - sift.python-packages.volatility - sift.python-packages.wheel - - sift.python-packages.windowsprefetch sift-python-packages: test.nop: @@ -53,4 +52,3 @@ sift-python-packages: - sls: sift.python-packages.unicodecsv - sls: sift.python-packages.volatility - sls: sift.python-packages.wheel - - sls: sift.python-packages.windowsprefetch diff --git a/sift/python-packages/windowsprefetch.sls b/sift/python-packages/windowsprefetch.sls deleted file mode 100644 index 02beb15d..00000000 --- a/sift/python-packages/windowsprefetch.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-windowsprefetch: - pip.installed: - - name: windowsprefetch - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python3-packages/windowsprefetch.sls b/sift/python3-packages/windowsprefetch.sls new file mode 100644 index 00000000..321961f6 --- /dev/null +++ b/sift/python3-packages/windowsprefetch.sls @@ -0,0 +1,37 @@ +# Name: windowsprefetch +# Website: https://github.com/PoorBillionaire/Windows-Prefetch-Parser +# Description: Windows Prefetch file parser +# Category: +# Author: Adam Witt (PoorBillionaire) +# License: Apache License v2 (https://github.com/PoorBillionaire/Windows-Prefetch-Parser/blob/master/LICENSE) +# Notes: prefetch.py + +include: + - sift.packages.python3-virtualenv + +sift-python3-package-windowsprefetch-venv: + virtualenv.managed: + - name: /opt/windowsprefetch + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-windowsprefetch: + pip.installed: + - name: windowsprefetch + - bin_env: /opt/windowsprefetch/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-windowsprefetch-venv + +sift-python3-package-windowsprefetch-symlink: + file.symlink: + - name: /usr/local/bin/prefetch.py + - target: /opt/windowsprefetch/bin/prefetch.py + - makedirs: False + - require: + - pip: sift-python3-package-windowsprefetch From 6cc73bd8fb8cbc0b82e25db277d5659efc0de1f5 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:51:29 -0500 Subject: [PATCH 38/64] Update usbdeviceforensics for 24 (#161) Co-authored-by: Erik Kristensen --- sift/python3-packages/init.sls | 2 + sift/python3-packages/usbdeviceforensics.sls | 43 ++++++++++++++++++++ sift/scripts/init.sls | 2 - sift/scripts/usbdeviceforensics.sls | 21 ---------- 4 files changed, 45 insertions(+), 23 deletions(-) create mode 100644 sift/python3-packages/usbdeviceforensics.sls delete mode 100644 sift/scripts/usbdeviceforensics.sls diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index f758f2a6..d55c6d8e 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -24,6 +24,7 @@ include: - sift.python3-packages.six - sift.python3-packages.stix-validator - sift.python3-packages.stix + - sift.python3-packages.usbdeviceforensics - sift.python3-packages.usnparser - sift.python3-packages.virustotal-api - sift.python3-packages.wheel @@ -58,6 +59,7 @@ sift-python3-packages: - sls: sift.python3-packages.six - sls: sift.python3-packages.stix-validator - sls: sift.python3-packages.stix + - sls: sift.python3-packages.usbdeviceforensics - sls: sift.python3-packages.usnparser - sls: sift.python3-packages.virustotal-api - sls: sift.python3-packages.wheel diff --git a/sift/python3-packages/usbdeviceforensics.sls b/sift/python3-packages/usbdeviceforensics.sls new file mode 100644 index 00000000..aaa25788 --- /dev/null +++ b/sift/python3-packages/usbdeviceforensics.sls @@ -0,0 +1,43 @@ +# Name: usbdeviceforensics +# Website: https://github.com/digitalsleuth/usbdeviceforensics +# Description: Python script to parse USB Device artifacts from Windows systems +# Category: +# Author: Mark Woan, Corey Forman (digitalsleuth) +# License: None +# Notes: usbdeviceforensics + +include: + - sift.packages.python3-virtualenv + - sift.packages.git + - sift.packages.python3-dev + +sift-python3-package-usbdeviceforensics-venv: + virtualenv.managed: + - name: /opt/usbdeviceforensics + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - python-registry + - enum34 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-usbdeviceforensics: + pip.installed: + - name: git+https://github.com/digitalsleuth/usbdeviceforensics.git + - bin_env: /opt/usbdeviceforensics/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-usbdeviceforensics-venv + - sls: sift.packages.git + - sls: sift.packages.python3-dev + +sift-python3-package-usbdeviceforensics-symlink: + file.symlink: + - name: /usr/local/bin/usbdeviceforensics + - target: /opt/usbdeviceforensics/bin/usbdeviceforensics + - force: True + - require: + - pip: sift-python3-package-usbdeviceforensics diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index 3afea02b..b89231c2 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -22,7 +22,6 @@ include: - sift.scripts.sorter - sift.scripts.sqlite_miner - sift.scripts.sqlparser - - sift.scripts.usbdeviceforensics - sift.scripts.virustotal-tools - sift.scripts.vshot - sift.scripts.zimmerman @@ -54,7 +53,6 @@ sift-scripts: - sls: sift.scripts.sorter - sls: sift.scripts.sqlite_miner - sls: sift.scripts.sqlparser - - sls: sift.scripts.usbdeviceforensics - sls: sift.scripts.virustotal-tools - sls: sift.scripts.vshot - sls: sift.scripts.zimmerman diff --git a/sift/scripts/usbdeviceforensics.sls b/sift/scripts/usbdeviceforensics.sls deleted file mode 100644 index 2aa04ae5..00000000 --- a/sift/scripts/usbdeviceforensics.sls +++ /dev/null @@ -1,21 +0,0 @@ -# source=https://github.com/woanware/usbdeviceforensics -# license=unknown - -{% set commit = "5a0705d5beca09eab2fd5a47a52240dbc0db5bc9" -%} -{% set hash = "sha256=cc643ae2ccd7b772f6d8a2abaa0e9dd33514c60328c5bc3b7d60bb69398b9637" -%} - -sift-scripts-usbdeviceforensics: - file.managed: - - name: /usr/local/bin/usbdeviceforensics.py - - source: https://raw.githubusercontent.com/woanware/usbdeviceforensics/{{ commit }}/usbdeviceforensics.py - - source_hash: {{ hash }} - - mode: 755 - -sift-scripts-usbdeviceforensics-shebang: - file.replace: - - name: /usr/local/bin/usbdeviceforensics.py - - pattern: '#!/usr/bin/python' - - repl: '#!/usr/bin/env python2' - - count: 1 - - watch: - - file: sift-scripts-usbdeviceforensics From 22cfcacf92c89a5e75f2ac35d4a76833c9eb6674 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:52:09 -0500 Subject: [PATCH 39/64] Update stix-validator for 24 (#159) Co-authored-by: Erik Kristensen --- sift/python3-packages/stix-validator.sls | 40 ++++++++++++++++++++---- sift/python3-packages/stix.sls | 11 ------- 2 files changed, 34 insertions(+), 17 deletions(-) delete mode 100644 sift/python3-packages/stix.sls diff --git a/sift/python3-packages/stix-validator.sls b/sift/python3-packages/stix-validator.sls index e9e8d559..d29b206e 100644 --- a/sift/python3-packages/stix-validator.sls +++ b/sift/python3-packages/stix-validator.sls @@ -1,11 +1,39 @@ +# Name: stix-validator +# Website: https://stixproject.github.io/ +# Description: Tool for using the Structured Threat Information eXpression language +# Category: +# Author: The MITRE Corporation +# License: BSD 3-Clause (https://github.com/STIXProject/stix-validator/blob/master/LICENSE.txt) +# Notes: stix-validator + include: - - sift.python3-packages.pip - - sift.python3-packages.stix + - sift.packages.python3-virtualenv + +sift-python3-package-stix-validator-venv: + virtualenv.managed: + - name: /opt/stix-validator + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - lxml + - stix + - require: + - sls: sift.packages.python3-virtualenv -sift-python3-packages-stix-validator: +sift-python3-package-stix-validator: pip.installed: - name: stix-validator - - bin_env: /usr/bin/python3 + - bin_env: /opt/stix-validator/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-stix-validator-venv + +sift-python3-package-stix-validator-symlink: + file.symlink: + - name: /usr/local/bin/stix-validator + - target: /opt/stix-validator/bin/stix-validator + - makedirs: False - require: - - sls: sift.python3-packages.pip - - sls: sift.python3-packages.stix + - pip: sift-python3-package-stix-validator diff --git a/sift/python3-packages/stix.sls b/sift/python3-packages/stix.sls deleted file mode 100644 index fd1c2f0e..00000000 --- a/sift/python3-packages/stix.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.python3-packages.pip - - sift.python3-packages.lxml - -sift-python3-packages-stix: - pip.installed: - - name: stix - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip - - sls: sift.python3-packages.lxml From 96d6b4e4d4c5146dc4845e4f40d161beda667a27 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:53:28 -0500 Subject: [PATCH 40/64] Update sqlite-carver (sqlparser) for 24 (#158) Co-authored-by: Erik Kristensen --- sift/python3-packages/init.sls | 2 ++ sift/python3-packages/sqlite-carver.sls | 37 +++++++++++++++++++++++++ sift/scripts/init.sls | 2 ++ sift/scripts/sqlparser.sls | 18 ------------ 4 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 sift/python3-packages/sqlite-carver.sls delete mode 100644 sift/scripts/sqlparser.sls diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index d55c6d8e..687ff93e 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -22,6 +22,7 @@ include: - sift.python3-packages.setuptools - sift.python3-packages.setuptools-rust - sift.python3-packages.six + - sift.python3-packages.sqlite-carver - sift.python3-packages.stix-validator - sift.python3-packages.stix - sift.python3-packages.usbdeviceforensics @@ -57,6 +58,7 @@ sift-python3-packages: - sls: sift.python3-packages.setuptools - sls: sift.python3-packages.setuptools-rust - sls: sift.python3-packages.six + - sls: sift.python3-packages.sqlite-carver - sls: sift.python3-packages.stix-validator - sls: sift.python3-packages.stix - sls: sift.python3-packages.usbdeviceforensics diff --git a/sift/python3-packages/sqlite-carver.sls b/sift/python3-packages/sqlite-carver.sls new file mode 100644 index 00000000..d086c8e7 --- /dev/null +++ b/sift/python3-packages/sqlite-carver.sls @@ -0,0 +1,37 @@ +# Name: sqlite-carver +# Website: https://github.com/digitalsleuth/sqlite-carver +# Description: Script to recover deleted entries in an SQLite database, rebuild of SQLite-Deleted-Records-Parser +# Category: +# Author: Mari DeGrazia and Corey Forman (digitalsleuth) +# License: GNU General Public License v3 (https://github.com/digitalsleuth/sqlite-carver/blob/main/LICENSE) +# Notes: sqlite-carver + +include: + - sift.packages.python3-virtualenv + +sift-python3-package-sqlite-carver-venv: + virtualenv.managed: + - name: /opt/sqlite-carver + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-sqlite-carver: + pip.installed: + - name: sqlite-carver + - bin_env: /opt/sqlite-carver/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-sqlite-carver-venv + +sift-python3-package-sqlite-carver-symlink: + file.symlink: + - name: /usr/local/bin/sqlite-carver + - target: /opt/sqlite-carver/bin/sqlite-carver + - makedirs: False + - require: + - pip: sift-python3-package-sqlite-carver diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index b89231c2..4386f63e 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -21,6 +21,7 @@ include: - sift.scripts.sift - sift.scripts.sorter - sift.scripts.sqlite_miner + - sift.scripts.usbdeviceforensics - sift.scripts.sqlparser - sift.scripts.virustotal-tools - sift.scripts.vshot @@ -52,6 +53,7 @@ sift-scripts: - sls: sift.scripts.sift - sls: sift.scripts.sorter - sls: sift.scripts.sqlite_miner + - sls: sift.scripts.usbdeviceforensics - sls: sift.scripts.sqlparser - sls: sift.scripts.virustotal-tools - sls: sift.scripts.vshot diff --git a/sift/scripts/sqlparser.sls b/sift/scripts/sqlparser.sls deleted file mode 100644 index e5b0e291..00000000 --- a/sift/scripts/sqlparser.sls +++ /dev/null @@ -1,18 +0,0 @@ -# source=https://github.com/mdegrazia/SQLite-Deleted-Records-Parser -# license=unknown - -{% set hash = "sha256=0bb28498141380821d5adc43cc3557ce6a96aeb8a33c414a48e3ccc2a1aad8c9" -%} - -sift-scripts-sqlparser: - file.managed: - - name: /usr/local/bin/sqlparser.py - - source: https://github.com/mdegrazia/SQLite-Deleted-Records-Parser/releases/download/v.1.1/sqlparse_v1.1.py - - source_hash: {{ hash }} - - mode: 755 - -sift-scripts-sqlparser-shebang: - file.prepend: - - name: /usr/local/bin/sqlparser.py - - text: '#!/usr/bin/env python2' - - watch: - - file: sift-scripts-sqlparser From c7a21b4acdc7ff090365f740079d9e97b7f93750 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:54:19 -0500 Subject: [PATCH 41/64] Update hindsight for 24 (#156) --- sift/python3-packages/hindsight.sls | 51 +++++++++++++++++++++++++ sift/python3-packages/init.sls | 4 +- sift/python3-packages/pyhindsight.sls | 55 --------------------------- 3 files changed, 53 insertions(+), 57 deletions(-) create mode 100644 sift/python3-packages/hindsight.sls delete mode 100644 sift/python3-packages/pyhindsight.sls diff --git a/sift/python3-packages/hindsight.sls b/sift/python3-packages/hindsight.sls new file mode 100644 index 00000000..d7f21a94 --- /dev/null +++ b/sift/python3-packages/hindsight.sls @@ -0,0 +1,51 @@ +# Name: hindsight +# Website: https://github.com/obsidianforensics/hindsight +# Description: Web browser forensics for Google Chrome / Chromium +# Category: +# Author: Ryan Benson (obsidianforensics) +# License: Apache License v2 (https://github.com/obsidianforensics/hindsight/blob/main/LICENSE.md) +# Notes: hindsight.py, hindsight_gui.py + +{% set files = ['hindsight.py','hindsight_gui.py'] %} + +include: + - sift.packages.python3-virtualenv + +sift-python3-package-pyhindsight-venv: + virtualenv.managed: + - name: /opt/pyhindsight + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - setuptools_rust + - keyrings.alt + - git+https://github.com/cclgroupltd/ccl_chromium_reader.git + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-pyhindsight: + pip.installed: + - name: pyhindsight + - bin_env: /opt/pyhindsight/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-pyhindsight-venv + +{% for file in files %} +sift-python3-package-pyhindsight-symlink-{{ file }}: + file.symlink: + - name: /usr/local/bin/{{ file }} + - target: /opt/pyhindsight/bin/{{ file }} + - makedirs: False + - require: + - pip: sift-python3-package-pyhindsight + +sift-python3-package-pyhindsight-chmod-{{ file }}: + file.managed: + - name: /opt/pyhindsight/bin/{{ file }} + - mode: 755 + - require: + - file: sift-python3-package-pyhindsight-symlink-{{ file }} +{% endfor %} diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index 687ff93e..ebb96172 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -7,6 +7,7 @@ include: - sift.python3-packages.bitstring - sift.python3-packages.colorama - sift.python3-packages.geoip2 + - sift.python3-packages.hindsight - sift.python3-packages.ioc-writer ### - sift.python3-packages.imagemounter - sift.python3-packages.keyrings-alt @@ -15,7 +16,6 @@ include: - sift.python3-packages.machinae - sift.python3-packages.pefile - sift.python3-packages.pillow - - sift.python3-packages.pyhindsight - sift.python3-packages.python-dateutil - sift.python3-packages.python-evtx - sift.python3-packages.python-magic @@ -43,6 +43,7 @@ sift-python3-packages: - sls: sift.python3-packages.bitstring - sls: sift.python3-packages.colorama - sls: sift.python3-packages.geoip2 + - sls: sift.python3-packages.hindsight - sls: sift.python3-packages.ioc-writer ### - sls: sift.python3-packages.imagemounter - sls: sift.python3-packages.keyrings-alt @@ -51,7 +52,6 @@ sift-python3-packages: - sls: sift.python3-packages.machinae - sls: sift.python3-packages.pefile - sls: sift.python3-packages.pillow - - sls: sift.python3-packages.pyhindsight - sls: sift.python3-packages.python-dateutil - sls: sift.python3-packages.python-evtx - sls: sift.python3-packages.python-magic diff --git a/sift/python3-packages/pyhindsight.sls b/sift/python3-packages/pyhindsight.sls deleted file mode 100644 index 3fe531af..00000000 --- a/sift/python3-packages/pyhindsight.sls +++ /dev/null @@ -1,55 +0,0 @@ -include: - - sift.python3-packages.pip - - sift.python3-packages.setuptools-rust - - sift.python3-packages.keyrings-alt - -sift-python3-packages-pyhindsight: - pip.installed: - - name: pyhindsight - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip - - sls: sift.python3-packages.setuptools-rust - - sls: sift.python3-packages.keyrings-alt - -sift-python3-packages-pyhindsight-encoding: - file.replace: - - name: /usr/local/bin/hindsight.py - - pattern: '\r' - - repl: '' - - require: - - pip: sift-python3-packages-pyhindsight - -sift-python3-packages-pyhindsight-chmod: - file.managed: - - name: /usr/local/bin/hindsight.py - - mode: 755 - - watch: - - file: sift-python3-packages-pyhindsight-encoding - -sift-python3-packages-pyhindsight-gui-encoding: - file.replace: - - name: /usr/local/bin/hindsight_gui.py - - pattern: '\r' - - repl: '' - - require: - - pip: sift-python3-packages-pyhindsight - -sift-python3-packages-pyhindsight-gui-prepend: - file.replace: - - name: /usr/local/bin/hindsight_gui.py - - pattern: '#!/usr/bin/env python3' - - repl: '#!/usr/bin/env python3' - - prepend_if_not_found: True - - count: 1 - - require: - - pip: sift-python3-packages-pyhindsight - -sift-python3-packages-pyhindsight-gui-chmod: - file.managed: - - name: /usr/local/bin/hindsight_gui.py - - mode: 755 - - watch: - - file: sift-python3-packages-pyhindsight-gui-prepend - - From e1b8f2cb337eaa6b97f0a6dd115071d2bab38b6c Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:54:51 -0500 Subject: [PATCH 42/64] Update pe-scanner for 24 (#155) --- sift/packages/python3-magic.sls | 3 ++ sift/python3-packages/init.sls | 2 ++ sift/python3-packages/pe-scanner.sls | 42 ++++++++++++++++++++++++++++ sift/scripts/init.sls | 2 -- sift/scripts/pescanner.sls | 34 ---------------------- 5 files changed, 47 insertions(+), 36 deletions(-) create mode 100644 sift/packages/python3-magic.sls create mode 100644 sift/python3-packages/pe-scanner.sls delete mode 100644 sift/scripts/pescanner.sls diff --git a/sift/packages/python3-magic.sls b/sift/packages/python3-magic.sls new file mode 100644 index 00000000..da279b2b --- /dev/null +++ b/sift/packages/python3-magic.sls @@ -0,0 +1,3 @@ +sift-package-python3-magic: + pkg.installed: + - name: python3-magic diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index ebb96172..535d0dc5 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -15,6 +15,7 @@ include: - sift.python3-packages.mac-apt - sift.python3-packages.machinae - sift.python3-packages.pefile + - sift.python3-packages.pe-scanner - sift.python3-packages.pillow - sift.python3-packages.python-dateutil - sift.python3-packages.python-evtx @@ -51,6 +52,7 @@ sift-python3-packages: - sls: sift.python3-packages.mac-apt - sls: sift.python3-packages.machinae - sls: sift.python3-packages.pefile + - sls: sift.python3-packages.pe-scanner - sls: sift.python3-packages.pillow - sls: sift.python3-packages.python-dateutil - sls: sift.python3-packages.python-evtx diff --git a/sift/python3-packages/pe-scanner.sls b/sift/python3-packages/pe-scanner.sls new file mode 100644 index 00000000..c91dfda4 --- /dev/null +++ b/sift/python3-packages/pe-scanner.sls @@ -0,0 +1,42 @@ +# Name: pe-scanner +# Website: https://github.com/digitalsleuth/pe-scanner +# Description: Python 3 rebuild of the original pescanner +# Category: +# Author: Michael Ligh, Glenn P. Edwards Jr., Corey Forman (digitalsleuth) +# License: GNU General Public License v3 (https://github.com/digitalsleuth/pe-scanner/blob/main/LICENSE) +# Notes: pe-scanner + +include: + - sift.packages.python3-virtualenv + - sift.packages.git + - sift.packages.python3-magic + +sift-python3-package-pe-scanner-venv: + virtualenv.managed: + - name: /opt/pe-scanner + - venv_bin: /usr/bin/virtualenv + - system_site_packages: True + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-pe-scanner: + pip.installed: + - name: git+https://github.com/digitalsleuth/pe-scanner.git + - bin_env: /opt/pe-scanner/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-pe-scanner-venv + - sls: sift.packages.git + - sls: sift.packages.python3-magic + +sift-python3-package-pe-scanner-symlink: + file.symlink: + - name: /usr/local/bin/pe-scanner + - target: /opt/pe-scanner/bin/pe-scanner + - makedirs: False + - require: + - pip: sift-python3-package-pe-scanner diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index 4386f63e..ab2c89d2 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -14,7 +14,6 @@ include: - sift.scripts.page-brute - sift.scripts.pdf-tools - sift.scripts.pe-carver - - sift.scripts.pescanner - sift.scripts.regripper - sift.scripts.screen-scale - sift.scripts.shim-cache-parser @@ -46,7 +45,6 @@ sift-scripts: - sls: sift.scripts.page-brute - sls: sift.scripts.pdf-tools - sls: sift.scripts.pe-carver - - sls: sift.scripts.pescanner - sls: sift.scripts.regripper - sls: sift.scripts.screen-scale - sls: sift.scripts.shim-cache-parser diff --git a/sift/scripts/pescanner.sls b/sift/scripts/pescanner.sls deleted file mode 100644 index eefbfb8a..00000000 --- a/sift/scripts/pescanner.sls +++ /dev/null @@ -1,34 +0,0 @@ -# source=https://github.com/hiddenillusion/AnalyzePE/ -# license=unknown - -{% set commit = "9c76ecbc3ac417bc07439c244f2d5ed19af06578" -%} -{% set hash = "sha256=0c4e2a8916df3de0bde67ef47543db6f6068b267fa2b665667a52bc6002e6529" -%} - -include: - - sift.packages.python2 - - sift.python-packages.pefile - - sift.python-packages.pydasm - - sift.python-packages.python-magic - - sift.python-packages.yara-python - -sift-scripts-pescanner: - file.managed: - - name: /usr/local/bin/pescanner.py - - source: https://raw.githubusercontent.com/hiddenillusion/AnalyzePE/{{ commit }}/pescanner.py - - source_hash: {{ hash }} - - mode: 755 - - require: - - sls: sift.packages.python2 - - sls: sift.python-packages.pefile - - sls: sift.python-packages.pydasm - - sls: sift.python-packages.python-magic - - sls: sift.python-packages.yara-python - -sift-scripts-pescanner-shebang: - file.replace: - - name: /usr/local/bin/pescanner.py - - pattern: '#!/usr/bin/env python\n' - - repl: '#!/usr/bin/env python2\n' - - count: 1 - - watch: - - file: sift-scripts-pescanner From 772df573b769150002047425493c038ef18a0aee Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:56:00 -0500 Subject: [PATCH 43/64] Update pe-carver for 24 (#154) Co-authored-by: Erik Kristensen --- sift/python3-packages/init.sls | 2 ++ sift/python3-packages/pe-carver.sls | 37 +++++++++++++++++++++++++++++ sift/scripts/init.sls | 2 ++ sift/scripts/pe-carver.sls | 26 -------------------- 4 files changed, 41 insertions(+), 26 deletions(-) create mode 100644 sift/python3-packages/pe-carver.sls delete mode 100644 sift/scripts/pe-carver.sls diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index 535d0dc5..975e8074 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -15,6 +15,7 @@ include: - sift.python3-packages.mac-apt - sift.python3-packages.machinae - sift.python3-packages.pefile + - sift.python3-packages.pe-carver - sift.python3-packages.pe-scanner - sift.python3-packages.pillow - sift.python3-packages.python-dateutil @@ -52,6 +53,7 @@ sift-python3-packages: - sls: sift.python3-packages.mac-apt - sls: sift.python3-packages.machinae - sls: sift.python3-packages.pefile + - sls: sift.python3-packages.pe-carver - sls: sift.python3-packages.pe-scanner - sls: sift.python3-packages.pillow - sls: sift.python3-packages.python-dateutil diff --git a/sift/python3-packages/pe-carver.sls b/sift/python3-packages/pe-carver.sls new file mode 100644 index 00000000..4b68498c --- /dev/null +++ b/sift/python3-packages/pe-carver.sls @@ -0,0 +1,37 @@ +# Name: pe-carver +# Website: https://github.com/digitalsleuth/pe-carver +# Description: Carves EXEs from given data files +# Category: +# Author: Brian Baskin (Rurik), Corey Forman (digitalsleuth) +# License: Apache License v2 (https://github.com/digitalsleuth/pe-carver/blob/main/LICENSE) +# Notes: pe-carver + +include: + - sift.packages.python3-virtualenv + +sift-python3-package-pe-carver-venv: + virtualenv.managed: + - name: /opt/pe-carver + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-pe-carver: + pip.installed: + - name: pe-carver + - bin_env: /opt/pe-carver/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-pe-carver-venv + +sift-python3-package-pe-carver-symlink: + file.symlink: + - name: /usr/local/bin/pe-carver + - target: /opt/pe-carver/bin/pe-carver + - makedirs: False + - require: + - pip: sift-python3-package-pe-carver diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index ab2c89d2..2a715c27 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -13,6 +13,7 @@ include: - sift.scripts.packerid - sift.scripts.page-brute - sift.scripts.pdf-tools + - sift.scripts.pescanner - sift.scripts.pe-carver - sift.scripts.regripper - sift.scripts.screen-scale @@ -44,6 +45,7 @@ sift-scripts: - sls: sift.scripts.packerid - sls: sift.scripts.page-brute - sls: sift.scripts.pdf-tools + - sls: sift.scripts.pescanner - sls: sift.scripts.pe-carver - sls: sift.scripts.regripper - sls: sift.scripts.screen-scale diff --git a/sift/scripts/pe-carver.sls b/sift/scripts/pe-carver.sls deleted file mode 100644 index f52a1ec4..00000000 --- a/sift/scripts/pe-carver.sls +++ /dev/null @@ -1,26 +0,0 @@ -# source=https://github.com/Rurik/PE_Carver -# license=unknown - -{% set commit = "9026cd2ca4bd0633f9898a93cb798cd19cffc8f6" -%} -{% set hash = "sha256=6b245decadde4652ff6d1e2b24f6496dd252bee4bf57e7c934fbb9c9f21df849" -%} - -include: - - sift.python-packages.bitstring - - sift.python-packages.pefile - -sift-scripts-pecarve: - file.managed: - - name: /usr/local/bin/pecarve.py - - source: https://raw.githubusercontent.com/Rurik/PE_Carver/{{ commit }}/pe_carve.py - - source_hash: {{ hash }} - - mode: 755 - - require: - - sls: sift.python-packages.bitstring - - sls: sift.python-packages.pefile - -sift-scripts-pecarve-shebang: - file.prepend: - - name: /usr/local/bin/pecarve.py - - text: '#!/usr/bin/env python2' - - watch: - - file: sift-scripts-pecarve From 9cd9f0c9e01b6ed05ac74b75edfc8f62c70de972 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:51:17 -0500 Subject: [PATCH 44/64] Update packerid for 24 (#150) --- sift/scripts/packerid.sls | 54 +++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/sift/scripts/packerid.sls b/sift/scripts/packerid.sls index 172fda40..94a66eda 100644 --- a/sift/scripts/packerid.sls +++ b/sift/scripts/packerid.sls @@ -1,30 +1,52 @@ -# source=https://github.com/sooshie/packerid -# license=Unknown +# Name: packerid +# Website: https://github.com/sooshie/packerid +# Description: Script to identify packed files +# Category: +# Author: Jim Clausing +# License: Unknown +# Notes: packerid -{% set commit = "7b2ee6ef57db903bf356fd342c8ca998abdb68cd" -%} -{% set hash = "sha256=be589d4cbe70ecdc3424a6da48d8fc24630d51a6ebf92e5328b36e39423eb038" -%} +{% set commit = "bc54e6d5204ebe83db8d87125d677035d9f456a7" -%} +{% set hash = "sha256=417830ccbf357e8e2b7d9cf47ee4a63a481151fc8cdf03c40b5538aecf96d15d" -%} include: - - sift.packages.python2 - - sift.python-packages.pefile - - sift.python-packages.capstone + - sift.packages.python3-virtualenv -sift-scripts-packerid: +sift-python3-package-packerid-venv: + virtualenv.managed: + - name: /opt/packerid + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - pefile + - capstone + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-packerid: file.managed: - - name: /usr/local/bin/packerid.py + - name: /opt/packerid/bin/packerid.py - source: https://raw.githubusercontent.com/sooshie/packerid/{{ commit }}/packerid.py - source_hash: {{ hash }} - mode: 755 - require: - - sls: sift.packages.python2 - - sls: sift.python-packages.pefile - - sls: sift.python-packages.capstone + - virtualenv: sift-python3-package-packerid-venv -sift-scripts-packerid-shebang: +sift-python3-package-packerid-shebang: file.replace: - - name: /usr/local/bin/packerid.py + - name: /opt/packerid/bin/packerid.py - pattern: '#!/usr/local/bin/python' - - repl: '#!/usr/bin/env python2' + - repl: '#!/opt/packerid/bin/python3' - count: 1 - watch: - - file: sift-scripts-packerid + - file: sift-python3-package-packerid + +sift-python3-package-packerid-symlink: + file.symlink: + - name: /usr/local/bin/packerid.py + - target: /opt/packerid/bin/packerid.py + - makedirs: False + - require: + - file: sift-python3-package-packerid From 55b223d84cfa9c4292369734dd12dc90fe373fb2 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:03:30 -0500 Subject: [PATCH 45/64] Remove sqlite_miner to support 24 (#170) --- sift/scripts/init.sls | 2 -- sift/scripts/sqlite_miner.sls | 41 ----------------------------------- 2 files changed, 43 deletions(-) delete mode 100644 sift/scripts/sqlite_miner.sls diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index 2a715c27..bf5ee5f3 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -20,7 +20,6 @@ include: - sift.scripts.shim-cache-parser - sift.scripts.sift - sift.scripts.sorter - - sift.scripts.sqlite_miner - sift.scripts.usbdeviceforensics - sift.scripts.sqlparser - sift.scripts.virustotal-tools @@ -52,7 +51,6 @@ sift-scripts: - sls: sift.scripts.shim-cache-parser - sls: sift.scripts.sift - sls: sift.scripts.sorter - - sls: sift.scripts.sqlite_miner - sls: sift.scripts.usbdeviceforensics - sls: sift.scripts.sqlparser - sls: sift.scripts.virustotal-tools diff --git a/sift/scripts/sqlite_miner.sls b/sift/scripts/sqlite_miner.sls deleted file mode 100644 index 42dac61b..00000000 --- a/sift/scripts/sqlite_miner.sls +++ /dev/null @@ -1,41 +0,0 @@ -# Name: sqlite_miner -# Website: https://github.com/threeplanetssoftware/sqlite_miner -# Description: A script to mine SQLite databases for hidden gems that might be overlooked -# Category: -# Author: Jon Baumann, Ciofeca Forensics -# License: GNU General Public License v3.0 (https://github.com/threeplanetssoftware/sqlite_miner/blob/master/LICENSE) -# Notes: sqlite_miner.pl, fun_stuff.pl - -{% set commit = "4220dae48a6e45c1316b153231dc6beef36f2f59" -%} -{% set hash_fun = "sha256=c2e887dc62cb8191e0333f95d2e0eee330f62a778abf394f2ae158be39e44590" -%} -{% set hash_miner = "sha256=0d4b380a27dd57380b581224b1258fbd5059b9314d59aa7ee2f260d352f82278" -%} - -include: - - sift.packages.perl - - sift.perl-packages.dbd-sqlite - -sift-scripts-sqlite-miner-funstuff: - file.managed: - - name: /usr/local/bin/fun_stuff.pl - - source: https://raw.githubusercontent.com/threeplanetssoftware/sqlite_miner/{{ commit }}/fun_stuff.pl - - source_hash: {{ hash_fun }} - - mode: 755 - - require: - - sls: sift.packages.perl - -sift-scripts-sqlite-miner: - file.managed: - - name: /usr/local/bin/sqlite_miner.pl - - source: https://raw.githubusercontent.com/threeplanetssoftware/sqlite_miner/master/sqlite_miner.pl - - source_hash: {{ hash_miner }} - - mode: 755 - - require: - - sls: sift.packages.perl - - sls: sift.perl-packages.dbd-sqlite - -sift-scripts-sqlite-miner-shebang: - file.prepend: - - name: /usr/local/bin/sqlite_miner.pl - - text: '#!/usr/bin/env perl' - - watch: - - file: sift-scripts-sqlite-miner From c80d46818bf3691901e9e17fbf494a0f2c9c636c Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:03:46 -0500 Subject: [PATCH 46/64] Update regripper for 24 (#169) --- sift/scripts/regripper.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sift/scripts/regripper.sls b/sift/scripts/regripper.sls index 61908396..86e4a7d0 100644 --- a/sift/scripts/regripper.sls +++ b/sift/scripts/regripper.sls @@ -71,8 +71,8 @@ sift-scripts-regripper-plugins-path-cleanup: sift-scripts-regripper-plugins-cleanup-2: file.replace: - name: /usr/share/regripper/rip.pl - - pattern: ': \(\$plugindir = File::Spec->catfile\("plugins"\)\);' - - repl: '#: ($plugindir = File::Spec->catfile("plugins"));' + - pattern: ': \(\$plugindir = File::Spec->catfile\(\$str, "plugins"\)\);' + - repl: '#: ($plugindir = File::Spec->catfile($str, "plugins"));' - count: 1 - prepend_if_not_found: False - require: From 153c83f4ad3d01da5b24ee4b230a9634f1235866 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:03:58 -0500 Subject: [PATCH 47/64] Remove plutil to support 24 (#168) --- sift/scripts/plutil.sls | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 sift/scripts/plutil.sls diff --git a/sift/scripts/plutil.sls b/sift/scripts/plutil.sls deleted file mode 100644 index 48f91af7..00000000 --- a/sift/scripts/plutil.sls +++ /dev/null @@ -1,35 +0,0 @@ -# source=https://github.com/HearthSim/extract-scripts -# license=unknown - -{% set commit = "b830f58fe53958c54def0ec66d0617f9cf5c60d9" -%} -{% set hash = "sha256=a1db4dccfa54a41361d11273dfdd88c59b4caca60f4d58e672cc867393a72166" -%} - -include: - - sift.packages.libencode-perl - -sift-scripts-plutil: - file.managed: - - name: /usr/local/src/scripts/plutil.pl - - source: https://raw.githubusercontent.com/HearthSim/extract-scripts/{{ commit }}/plutil.pl - - source_hash: {{ hash }} - - makedirs: True - - require: - - sls: sift.packages.libencode-perl - -sift-scripts-plutil-binary: - file.copy: - - name: /usr/local/bin/plutil.pl - - source: /usr/local/src/scripts/plutil.pl - - force: True - - mode: 755 - - watch: - - file: sift-scripts-plutil - -sift-scripts-plutil-shebang: - file.replace: - - name: /usr/local/bin/plutil.pl - - pattern: '#!/usr/bin/perl' - - repl: '#!/usr/bin/env perl' - - count: 1 - - watch: - - file: sift-scripts-plutil-binary From 31b61a42360dcdde6512667f6626a78b617a7d81 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:04:07 -0500 Subject: [PATCH 48/64] Removes dumbpig to support 24 (#167) --- sift/scripts/dumbpig.sls | 12 ------------ sift/scripts/init.sls | 2 -- 2 files changed, 14 deletions(-) delete mode 100644 sift/scripts/dumbpig.sls diff --git a/sift/scripts/dumbpig.sls b/sift/scripts/dumbpig.sls deleted file mode 100644 index 7e47685a..00000000 --- a/sift/scripts/dumbpig.sls +++ /dev/null @@ -1,12 +0,0 @@ -# source=https://github.com/leonward/dumbpig -# license=gpl - -{% set commit = "429a880e6fb8e1528e406bc962e23f16df5ca959" -%} -{% set hash = "cbb11d7a20556b8c645e71b0c3dc422b4e48c2f18dce719cf6504a4af516bf07" -%} - -sift-scripts-dumbpig: - file.managed: - - name: /usr/local/bin/dumbpig.pl - - source: https://raw.githubusercontent.com/leonward/dumbpig/{{ commit }}/dumbpig.pl - - source_hash: sha256={{ hash }} - - mode: 755 diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index bf5ee5f3..06ee3b8f 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -4,7 +4,6 @@ include: - sift.scripts.amcache - sift.scripts.cyberchef - sift.scripts.densityscout - - sift.scripts.dumbpig - sift.scripts.dump-mft-entry - sift.scripts.image-mounter - sift.scripts.java-idx-parser @@ -35,7 +34,6 @@ sift-scripts: - sls: sift.scripts.amcache - sls: sift.scripts.cyberchef - sls: sift.scripts.densityscout - - sls: sift.scripts.dumbpig - sls: sift.scripts.dump-mft-entry - sls: sift.scripts.image-mounter - sls: sift.scripts.java-idx-parser From ef6dc0810e7436c11b62401dcaa41188ade19aeb Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:04:17 -0500 Subject: [PATCH 49/64] Update docker compose for 24 (#166) --- sift/scripts/docker-compose.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sift/scripts/docker-compose.sls b/sift/scripts/docker-compose.sls index 6897c14e..554ffe48 100644 --- a/sift/scripts/docker-compose.sls +++ b/sift/scripts/docker-compose.sls @@ -1,4 +1,4 @@ -{%- set version = "2.23.2" -%} +{%- set version = "2.32.4" -%} sift-scripts-docker-compose: file.managed: From ee7e9017971e276f5bcde5d454fbadd996659a03 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:04:29 -0500 Subject: [PATCH 50/64] Remove afterglow to support 24 (#165) --- sift/scripts/afterglow.sls | 21 --------------------- sift/scripts/init.sls | 2 -- 2 files changed, 23 deletions(-) delete mode 100644 sift/scripts/afterglow.sls diff --git a/sift/scripts/afterglow.sls b/sift/scripts/afterglow.sls deleted file mode 100644 index e3aa20ec..00000000 --- a/sift/scripts/afterglow.sls +++ /dev/null @@ -1,21 +0,0 @@ -# license=gpl -# license_source=https://github.com/zrlram/afterglow/blob/master/afterglow.pl - -{% set commit = "91e7dd3f1f7fb9ab3b68fbe47b31997d8f073c1c" %} -{% set hash = "95900e17f696d4020efc9f52399996badc9974aa084e593200a1a54c1a523a3f" %} - -sift-scripts-afterglow: - file.managed: - - name: /usr/local/bin/afterglow.pl - - source: https://raw.githubusercontent.com/zrlram/afterglow/{{ commit }}/afterglow.pl - - source_hash: sha256={{ hash }} - - mode: 755 - -sift-scripts-afterglow-shebang: - file.replace: - - name: /usr/local/bin/afterglow.pl - - pattern: '#!/usr/bin/perl' - - repl: '#!/usr/bin/env perl' - - count: 1 - - watch: - - file: sift-scripts-afterglow diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index 06ee3b8f..e95f038e 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -1,6 +1,5 @@ include: - sift.scripts.4n6 - - sift.scripts.afterglow - sift.scripts.amcache - sift.scripts.cyberchef - sift.scripts.densityscout @@ -30,7 +29,6 @@ sift-scripts: - name: sift-scripts - require: - sls: sift.scripts.4n6 - - sls: sift.scripts.afterglow - sls: sift.scripts.amcache - sls: sift.scripts.cyberchef - sls: sift.scripts.densityscout From dfee82c26e145dca0909ae4914d10751858e93f3 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:08:17 -0500 Subject: [PATCH 51/64] Update exiftool to support 24 (#172) --- sift/include-server.sls | 2 - sift/perl-packages/cgi.sls | 2 +- sift/perl-packages/datecalc.sls | 2 +- sift/perl-packages/dbd-sqlite.sls | 2 +- sift/perl-packages/dbi.sls | 2 +- sift/perl-packages/exiftool.sls | 2 +- sift/perl-packages/init.sls | 18 -------- sift/perl-packages/json.sls | 2 +- sift/perl-packages/quicktable.sls | 2 +- sift/perl-packages/xpath.sls | 2 +- sift/scripts/exiftool.sls | 70 +++++++++++-------------------- sift/scripts/init.sls | 2 + 12 files changed, 35 insertions(+), 73 deletions(-) delete mode 100644 sift/perl-packages/init.sls diff --git a/sift/include-server.sls b/sift/include-server.sls index 2dbdbf43..c0677017 100644 --- a/sift/include-server.sls +++ b/sift/include-server.sls @@ -3,7 +3,6 @@ include: - sift.python3-packages - sift.python-packages - sift.packages - - sift.perl-packages - sift.scripts sift-server-include: @@ -14,5 +13,4 @@ sift-server-include: - sls: sift.python3-packages - sls: sift.python-packages - sls: sift.packages - - sls: sift.perl-packages - sls: sift.scripts diff --git a/sift/perl-packages/cgi.sls b/sift/perl-packages/cgi.sls index 537e77fb..3405865a 100644 --- a/sift/perl-packages/cgi.sls +++ b/sift/perl-packages/cgi.sls @@ -1,7 +1,7 @@ include: - sift.packages.perl -cgi: +sift-perl-package-libcgi-pm-perl: pkg.installed: - name: libcgi-pm-perl - require: diff --git a/sift/perl-packages/datecalc.sls b/sift/perl-packages/datecalc.sls index 09a4f6fd..664a789a 100644 --- a/sift/perl-packages/datecalc.sls +++ b/sift/perl-packages/datecalc.sls @@ -1,7 +1,7 @@ include: - sift.packages.perl -datecalc: +sift-perl-package-libdate-calc-perl: pkg.installed: - name: libdate-calc-perl - require: diff --git a/sift/perl-packages/dbd-sqlite.sls b/sift/perl-packages/dbd-sqlite.sls index f4077b12..a0babe90 100644 --- a/sift/perl-packages/dbd-sqlite.sls +++ b/sift/perl-packages/dbd-sqlite.sls @@ -2,7 +2,7 @@ include: - sift.packages.perl - sift.packages.build-essential -sift-perl-packages-dbd-sqlite: +sift-perl-package-dbd-sqlite: cmd.run: - name: cpanm --no-man-pages install DBD::SQLite - env: diff --git a/sift/perl-packages/dbi.sls b/sift/perl-packages/dbi.sls index 99721f10..7c3d2e83 100644 --- a/sift/perl-packages/dbi.sls +++ b/sift/perl-packages/dbi.sls @@ -1,7 +1,7 @@ include: - sift.packages.perl -dbi: +sift-perl-package-libdbi-perl: pkg.installed: - name: libdbi-perl - require: diff --git a/sift/perl-packages/exiftool.sls b/sift/perl-packages/exiftool.sls index 0b9bf0c1..4a80fd24 100644 --- a/sift/perl-packages/exiftool.sls +++ b/sift/perl-packages/exiftool.sls @@ -1,7 +1,7 @@ include: - sift.packages.perl -exiftool: +sift-perl-package-libimage-exiftool-perl: pkg.installed: - name: libimage-exiftool-perl - require: diff --git a/sift/perl-packages/init.sls b/sift/perl-packages/init.sls deleted file mode 100644 index 43b17998..00000000 --- a/sift/perl-packages/init.sls +++ /dev/null @@ -1,18 +0,0 @@ -include: - - sift.perl-packages.datecalc - - sift.perl-packages.dbi - - sift.perl-packages.exiftool - - sift.perl-packages.json - - sift.perl-packages.xpath - - sift.perl-packages.dbd-sqlite - -sift-perl-packages: - test.nop: - - name: sift-perl-packages - - require: - - sls: sift.perl-packages.datecalc - - sls: sift.perl-packages.dbi - - sls: sift.perl-packages.exiftool - - sls: sift.perl-packages.json - - sls: sift.perl-packages.xpath - - sls: sift.perl-packages.dbd-sqlite diff --git a/sift/perl-packages/json.sls b/sift/perl-packages/json.sls index 659e1de8..0a5d9836 100644 --- a/sift/perl-packages/json.sls +++ b/sift/perl-packages/json.sls @@ -1,7 +1,7 @@ include: - sift.packages.perl -json: +sift-perl-package-libjson-perl: pkg.installed: - name: libjson-perl - require: diff --git a/sift/perl-packages/quicktable.sls b/sift/perl-packages/quicktable.sls index e3768ac6..d5f30ff0 100644 --- a/sift/perl-packages/quicktable.sls +++ b/sift/perl-packages/quicktable.sls @@ -2,7 +2,7 @@ include: - sift.packages.perl - sift.packages.build-essential -sift-perl-packages-quicktable: +sift-perl-package-quicktable: cmd.run: - name: cpanm --no-man-pages install HTML::QuickTable - env: diff --git a/sift/perl-packages/xpath.sls b/sift/perl-packages/xpath.sls index 1a249e72..ed615bf0 100644 --- a/sift/perl-packages/xpath.sls +++ b/sift/perl-packages/xpath.sls @@ -1,7 +1,7 @@ include: - sift.packages.perl -xpath: +sift-perl-package-libxml-xpath-perl: pkg.installed: - name: libxml-xpath-perl - require: diff --git a/sift/scripts/exiftool.sls b/sift/scripts/exiftool.sls index a381209e..593fdbfb 100644 --- a/sift/scripts/exiftool.sls +++ b/sift/scripts/exiftool.sls @@ -1,16 +1,21 @@ -# source=https://owl.phy.queensu.ca/~phil/exiftool/ -# license=free +# Name: exiftool +# Website: https://exiftool.org +# Description: Platform-independent tool to read, write, and edit metadata information +# Category: +# Author: Phil Harvey +# License: Free +# Notes: exiftool -{% set exiftool_version = '10.60' -%} -{% set exiftool_sha256 = 'df0988f60e1a6c086799e1f2ecd419e8abbad4dfb5dfa66c6080c78a5cb7acfa' -%} +{% set exiftool_version = '13.16' -%} +{% set exiftool_sha256 = 'c4d12812ace44caea59173b75c47d97b32fc195dc4c0b561f305d847417839d1' -%} include: - - sift.packages.patch - + - sift.packages.build-essential + sift-exiftool-source: file.managed: - name: /var/cache/sift/archives/Image-ExifTool-{{ exiftool_version }}.tar.gz - - source: https://owl.phy.queensu.ca/~phil/exiftool/Image-ExifTool-{{ exiftool_version }}.tar.gz + - source: https://exiftool.org/Image-ExifTool-{{ exiftool_version }}.tar.gz - source_hash: sha256={{ exiftool_sha256 }} - makedirs: True @@ -22,44 +27,19 @@ sift-exiftool-extracted: - watch: - file: sift-exiftool-source -sift-exiftool-patch-file: - file.managed: - - name: /usr/local/src/exiftool-{{ exiftool_version }}/exiftool.patch - - contents: | - diff --git 1/exiftool 2/exiftool - index eeff10b..a7c1259 100755 - --- 1/exiftool - +++ 2/exiftool - @@ -18,7 +18,7 @@ my $version = '10.60'; - my $exeDir; - BEGIN { - # get exe directory - - $exeDir = ($0 =~ /(.*)[\\\/]/) ? $1 : '.'; - + $exeDir = "/usr/local/share/exiftool-$version/Image-ExifTool-$version"; - # add lib directory at start of include path - unshift @INC, "$exeDir/lib"; - # load or disable config file if specified - - watch: - - archive: sift-exiftool-extracted - -sift-exiftool-patch: - file.patch: - - name: /usr/local/src/exiftool-{{ exiftool_version }}/Image-ExifTool-{{ exiftool_version }}/exiftool - - source: /usr/local/src/exiftool-{{ exiftool_version }}/exiftool.patch - - hash: sha256=8790e165825aa7028d3a71ce656c876f8430d2505c6ca5aa058e74b16faee611 - - require: - - sls: sift.packages.patch - - file: sift-exiftool-patch-file - - watch: - - archive: sift-exiftool-extracted - -sift-exiftool-binary: - file.managed: - - name: /usr/local/bin/exiftool - - source: /usr/local/src/exiftool-{{ exiftool_version }}/Image-ExifTool-{{ exiftool_version }}/exiftool - - mode: 755 - - watch: - - file: sift-exiftool-patch +sift-exiftool-makefile: + cmd.run: + - name: perl Makefile.PL + - cwd: /usr/local/src/exiftool-{{ exiftool_version }}/Image-ExifTool-{{ exiftool_version }}/ + - include: + - sls: sift.packages.build-essential + +sift-exiftool-install: + cmd.run: + - name: make install + - cwd: /usr/local/src/exiftool-{{ exiftool_version }}/Image-ExifTool-{{ exiftool_version }}/ + - include: + - cmd: sift-exiftool-makefile diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index e95f038e..c55f8dea 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -4,6 +4,7 @@ include: - sift.scripts.cyberchef - sift.scripts.densityscout - sift.scripts.dump-mft-entry + - sift.scripts.exiftool - sift.scripts.image-mounter - sift.scripts.java-idx-parser - sift.scripts.jobparser @@ -33,6 +34,7 @@ sift-scripts: - sls: sift.scripts.cyberchef - sls: sift.scripts.densityscout - sls: sift.scripts.dump-mft-entry + - sls: sift.scripts.exiftool - sls: sift.scripts.image-mounter - sls: sift.scripts.java-idx-parser - sls: sift.scripts.jobparser From 2c8364f8efcbeb0c73822b17e87bbe559a6b64ba Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:16:09 -0500 Subject: [PATCH 52/64] Update pdf-tools for 24 (#152) --- sift/files/pdf-tools/mPDF.py | 659 ++++++- sift/files/pdf-tools/make-pdf-embedded.py | 26 +- sift/files/pdf-tools/make-pdf-helloworld.py | 68 +- sift/files/pdf-tools/make-pdf-javascript.py | 27 +- sift/files/pdf-tools/pdf-parser.py | 785 +++++++- sift/files/pdf-tools/pdfid.ini | 2 + sift/files/pdf-tools/pdfid.py | 230 ++- sift/files/pdf-tools/pdftool.py | 1727 +++++++++++++++++ sift/files/pdf-tools/plugin_embeddedfile.py | 2 +- .../files/pdf-tools/plugin_nameobfuscation.py | 2 +- sift/files/pdf-tools/plugin_triage.py | 45 +- sift/scripts/pdf-tools.sls | 44 +- 12 files changed, 3369 insertions(+), 248 deletions(-) create mode 100644 sift/files/pdf-tools/pdfid.ini create mode 100644 sift/files/pdf-tools/pdftool.py diff --git a/sift/files/pdf-tools/mPDF.py b/sift/files/pdf-tools/mPDF.py index 836931f2..f749d3f2 100644 --- a/sift/files/pdf-tools/mPDF.py +++ b/sift/files/pdf-tools/mPDF.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#! /usr/bin/env python3 # module with simple class to build PDF documents with basic PDF elements # Source code put in public domain by Didier Stevens, no Copyright @@ -6,7 +6,7 @@ # Use at your own risk # # History: -# +# # 2008/05/18: continue # 2008/05/19: continue # 2008/05/28: stream2 @@ -16,37 +16,102 @@ # 2011/03/03: Added support for info in trailer and xrefAndTrailer # 2011/07/01: V0.1.4: Added support for filters i and I; added support for Python 3 # 2012/02/25: fixed printing \n for filters i and I +# 2013/04/03: V0.2.0: Added cNameObfuscation; filter j and *; cFuzzer +# 2013/04/05: added docstrings +# 2013/04/11: added SetReference +# 2013/04/14: V0.2.1: added return value to stream method +# 2013/04/20: V0.2.2: added version parameter to header function +# 2014/09/25: V0.2.3: added comment method +# 2014/10/15: V0.2.4: added CObjectStream +# 2017/04/16: V0.2.5: added support for filter i## # Todo: # - add support for extra filters to stream2 __author__ = 'Didier Stevens' -__version__ = '0.1.4' -__date__ = '2012/02/25' +__version__ = '0.2.5' +__date__ = '2017/04/16' import sys import zlib import platform +import random +import re +import struct + +def ReadBinaryFile(name): + """Read a binary file and return the content, return None if error occured + """ + + try: + fBinary = open(name, 'rb') + except: + return None + try: + content = fBinary.read() + except: + return None + finally: + fBinary.close() + return content -def SplitByLength(input, length): +def ParseFilters(definition): + filters = [] + number = '' + for character in definition + ' ': + if character.isdigit(): + number += character + else: + if number != '': + filters.append(number) + number = '' + filters.append(character) result = [] - while len(input) > length: - result.append(input[0:length] + '\n') - input = input[length:] - result.append(input + '>') + filters = filters[:-1] + while filters != []: + token = filters[0] + filters = filters[1:] + if token.lower() == 'i': + if filters != [] and filters[0].isdigit(): + result.append((token, int(filters[0]))) + filters = filters[1:] + else: + result.append((token, 512)) + else: + result.append((token, None)) return result +def IsLastFilterI(filters): + if filters == []: + return False + return filters[-1][0].lower() == 'i' + class cPDF: + """ + Class to create a PDF file + """ def __init__(self, filename): + """ + class instantiation arguments: + + filename is the name of the PDF file to be created + """ self.filename = filename self.indirectObjects = {} - + self.objstms = [] + def appendString(self, str): + """ + Internal helper function + """ fPDF = open(self.filename, 'a') fPDF.write(str) fPDF.close() def appendBinary(self, str): + """ + Internal helper function + """ fPDF = open(self.filename, 'ab') if sys.version_info[0] == 2: fPDF.write(str) @@ -55,90 +120,202 @@ def appendBinary(self, str): fPDF.close() def filesize(self): + """ + Internal helper function + """ fPDF = open(self.filename, 'rb') fPDF.seek(0, 2) size = fPDF.tell() fPDF.close() return size - + def IsWindows(self): + """ + Internal helper function + """ return platform.system() in ('Windows', 'Microsoft') - - def header(self): + + def header(self, version='1.1'): + """ + Method to create a PDF header (%PDF-1.1) and output it + to the PDF file. + + By default, the version is 1.1, but can be specified with + the version argument. + """ fPDF = open(self.filename, 'w') - fPDF.write("%PDF-1.1\n") + fPDF.write('%%PDF-%s\n' % version) fPDF.close() - + def binary(self): + """ + Method to create a comment (%\\xD0\\xD0\\xD0\\xD0) and output it + to the PDF file. + Use this after the header to indicate a PDF file has binary + (not printable) content. + """ self.appendString("%\xD0\xD0\xD0\xD0\n") + def comment(self, comment): + """ + Method to create a comment and output it to the PDF file. + """ + self.appendString('%' + comment + '\n') + def indirectobject(self, index, version, io): + """ + Method to create an indirect object and output it to the PDF file. + + index is the index number of the object. + + version is the version number of the object. Use 0 by convention. + + io is the content of the indirect object. + """ self.appendString("\n") self.indirectObjects[index] = self.filesize() self.appendString("%d %d obj\n%s\nendobj\n" % (index, version, io)) def stream(self, index, version, streamdata, dictionary="<< /Length %d >>"): + """ + Method to create an indirect object with a stream and output it + to the PDF file. + + index is the index number of the object. + + version is the version number of the object. Use 0 by convention. + + streamdata is the stream that will be put inside the object + without any modifications. + + dictionary is the PDF dictionary to be put before the stream. + By default this is << /Length %d >>. If you provide a dictionary, + you must include /Length %d. + + The return value is the file position of the stream data. + + Use this method when you want to provide the stream yourself. + """ self.appendString("\n") self.indirectObjects[index] = self.filesize() self.appendString(("%d %d obj\n" + dictionary + "\nstream\n") % (index, version, len(streamdata))) + position = self.filesize() self.appendBinary(streamdata) self.appendString("\nendstream\nendobj\n") - def Data2HexStr(self, data): + return position + + def Data2HexStr(self, data, whitespace=0): + """ + Internal helper function + """ hex = '' if sys.version_info[0] == 2: for b in data: - hex += "%02x" % ord(b) + hex += "%02x%s" % (ord(b), ' ' * random.randint(0, whitespace)) else: for b in data: - hex += "%02x" % b + hex += "%02x%s" % (b, ' ' * random.randint(0, whitespace)) return hex - def stream2(self, index, version, streamdata, entries="", filters=""): - """ - * h ASCIIHexDecode - * H AHx - * i like ASCIIHexDecode but with 512 long lines - * I like AHx but with 512 long lines - * ASCII85Decode - * LZWDecode - * f FlateDecode - * F Fl - * RunLengthDecode - * CCITTFaxDecode - * JBIG2Decode - * DCTDecode - * JPXDecode - * Crypt - """ - + def stream2(self, index, version, streamdata, entries="", filters="", fuzzer=None): + """ + Method to create an indirect object with a stream and + output it to the PDF file. + + index is the index number of the object. + + version is the version number of the object. Use 0 by convention. + + streamdata is the stream that will be put inside the object + modified according to the filters. + + entries is a string with a list of entries to be put inside + the PDF dictionary. Empty string by default. + + filters is a string with the encoding filters to be applied. + Each filter is represented by a letter, and filters are applied + from left to right. + For example, "hf" will apply the ASCIIHexDecode encoding filter and + then the FlateDecode encoding filter. For more details regarding + filters, see below. + Empty string by default. + + fuzzer is a fuzzer object to be used by the fuzzer filter (*). + If no object is provided, a default instance of class cFuzzer + is used. + + Use this method when you want the stream to be encoded. + + Implemented filters: + h ASCIIHexDecode + H AHx + i like ASCIIHexDecode but with 512 character long lines (default) + add number to speficy length of line, example: i80 for 80 characters + I like AHx but with 512 character long lines (default) + add number to speficy length of line, example: I80 for 80 characters + j like ASCIIHexDecode but with random whitespace + J like AHx but with random whitespace + f FlateDecode + F Fl + + Special filters (these are applied but not added to /Filters): + * for fuzzing + + Not implemented filters: + ASCII85Decode + LZWDecode + RunLengthDecode + CCITTFaxDecode + JBIG2Decode + DCTDecode + JPXDecode + Crypt + """ + + if fuzzer == None: + oFuzzer = cFuzzer() + else: + oFuzzer = fuzzer encodeddata = streamdata filter = [] + filters = ParseFilters(filters) for i in filters: - if i.lower() == "h": + if i[0].lower() == 'h': encodeddata = self.Data2HexStr(encodeddata) + '>' - if i == "h": + if i[0] == 'h': filter.insert(0, "/ASCIIHexDecode") else: filter.insert(0, "/AHx") - elif i.lower() == "i": - encodeddata = ''.join(SplitByLength(self.Data2HexStr(encodeddata), 512)) - if i == "i": + elif i[0].lower() == "i": + encodeddata = ''.join(self.SplitByLength(self.Data2HexStr(encodeddata), i[1])) + if i[0] == "i": filter.insert(0, "/ASCIIHexDecode") else: filter.insert(0, "/AHx") - elif i.lower() == "f": + elif i[0].lower() == "j": + encodeddata = self.Data2HexStr(encodeddata, 2) + '>' + if i[0] == "j": + filter.insert(0, "/ASCIIHexDecode") + else: + filter.insert(0, "/AHx") + elif i[0].lower() == "f": encodeddata = zlib.compress(encodeddata) - if i == "f": + if i[0] == "f": filter.insert(0, "/FlateDecode") else: filter.insert(0, "/Fl") + elif i[0] == "*": + encodeddata = oFuzzer.Fuzz(encodeddata) else: print("Error") return self.appendString("\n") self.indirectObjects[index] = self.filesize() - self.appendString("%d %d obj\n<<\n /Length %d\n" % (index, version, len(encodeddata))) + length = len(encodeddata) + if IsLastFilterI(filters) and self.IsWindows(): + length += encodeddata.count('\n') + self.appendString("%d %d obj\n<<\n /Length %d\n" % (index, version, length)) if len(filter) == 1: self.appendString(" /Filter %s\n" % filter[0]) if len(filter) > 1: @@ -146,45 +323,425 @@ def stream2(self, index, version, streamdata, entries="", filters=""): if entries != "": self.appendString(" %s\n" % entries) self.appendString(">>\nstream\n") - if filters[-1].lower() == 'i': + if IsLastFilterI(filters): self.appendString(encodeddata) else: self.appendBinary(encodeddata) self.appendString("\nendstream\nendobj\n") def xref(self): + """ + Method to create an xref table and output it to the PDF file. + + Returns the file position of the xref table and the size of the + xref table in a list. + """ self.appendString("\n") startxref = self.filesize() - max = 0 + maximumIndexValue = 0 for i in self.indirectObjects.keys(): - if i > max: - max = i - self.appendString("xref\n0 %d\n" % (max+1)) + if i > maximumIndexValue: + maximumIndexValue = i + self.appendString("xref\n0 %d\n" % (maximumIndexValue+1)) if self.IsWindows(): eol = '\n' else: eol = ' \n' - for i in range(0, max+1): + for i in range(0, maximumIndexValue+1): if i in self.indirectObjects: self.appendString("%010d %05d n%s" % (self.indirectObjects[i], 0, eol)) else: self.appendString("0000000000 65535 f%s" % eol) - return (startxref, (max+1)) + return (startxref, (maximumIndexValue+1)) def trailer(self, startxref, size, root, info=None): + """ + Method to create a trailer and output it to the PDF file. + + startxref is the file position of the xref table (this value is + returned by the xref method) + + size is the size of the xref table (this value is + returned by the xref method) + + root is a string with a reference to the root object (/Root). + Example: "1 0 R" + + info is a string with a reference to the info object (/Info). + This argument is optional. + Example: "9 0 R" + """ if info == None: self.appendString("trailer\n<<\n /Size %d\n /Root %s\n>>\nstartxref\n%d\n%%%%EOF\n" % (size, root, startxref)) else: self.appendString("trailer\n<<\n /Size %d\n /Root %s\n /Info %s\n>>\nstartxref\n%d\n%%%%EOF\n" % (size, root, info, startxref)) def xrefAndTrailer(self, root, info=None): + """ + Method to create an xref table together with a trailer and + output it to the PDF file. + + root is a string with a reference to the root object (/Root). + Example: "1 0 R" + + info is a string with a reference to the info object (/Info). + This argument is optional. + Example: "9 0 R" + """ xrefdata = self.xref() self.trailer(xrefdata[0], xrefdata[1], root, info) def template1(self): + """ + Method to create 5 indirect objects that form a template for + the start of a PDF file. + """ self.indirectobject(1, 0, "<<\n /Type /Catalog\n /Outlines 2 0 R\n /Pages 3 0 R\n>>") self.indirectobject(2, 0, "<<\n /Type /Outlines\n /Count 0\n>>") self.indirectobject(3, 0, "<<\n /Type /Pages\n /Kids [4 0 R]\n /Count 1\n>>") self.indirectobject(4, 0, "<<\n /Type /Page\n /Parent 3 0 R\n /MediaBox [0 0 612 792]\n /Contents 5 0 R\n /Resources <<\n /ProcSet [/PDF /Text]\n /Font << /F1 6 0 R >>\n >>\n>>") self.indirectobject(6, 0, "<<\n /Type /Font\n /Subtype /Type1\n /Name /F1\n /BaseFont /Helvetica\n /Encoding /MacRomanEncoding\n>>") + def MatchDictionary(self, string): + """ + Internal helper function + """ + status = 0 + level = 0 + result = '' + for c in string: + result += c + if status == 0 and c == '<': + status = 1 + elif status == 1: + if c == '<': + level += 1 + status = 0 + elif status == 0 and c == '>': + status = 2 + elif status == 2: + if c == '>': + level -= 1 + if level == 0: + return result + status = 0 + return None + + def originalIncrementalUpdate(self, pdffilename): + """ + Method to start an incremental update of an existing PDF file. + + pdffilename is the name of the PDF file to be used for the + incremental update. + + This methods returns the dictionary of the root object, + the dictionary of the trailer and the file position of the + xrf table found in the existing PDF file. These 3 values are + returned in a list. + + Use this method to start an incremental update. + """ + original = ReadBinaryFile(pdffilename) + fPDF = open(self.filename, 'wb') + if sys.version_info[0] == 2: + fPDF.write(original) + else: + fPDF.write(bytes(original, 'ascii')) + fPDF.close() + startxrefs = re.findall(r'startxref\s+(\d+)', original) + if startxrefs == []: + return None, None, None + oMatch = re.search(r'trailer\s+', original[int(startxrefs[-1]):]) + if oMatch == None: + return None, None, None + positionDictionaryTrailer = oMatch.end() + int(startxrefs[-1]) + dictionaryTrailer = self.MatchDictionary(original[positionDictionaryTrailer:]) + if dictionaryTrailer == None: + return None, None, None + oDictionaryTrailer = cDictionary(dictionaryTrailer) + idRoot = oDictionaryTrailer.GetID('Root') + if idRoot == None: + return None, None, None + oMatch = re.search(r'\s+%d\s+0\s+obj\s+' % idRoot, original) + if oMatch == None: + return None, None, None + dictionaryRoot = self.MatchDictionary(original[oMatch.end():]) + if dictionaryRoot == None: + return None, None, None + oDictionaryRoot = cDictionary(dictionaryRoot) + return oDictionaryTrailer, oDictionaryRoot, int(startxrefs[-1]) + + def xrefIncrementalAndTrailer(self, dictionaryTrailer): + """ + Method to create an xref table together with a trailer for + an incremental update and output it to the PDF file. + + dictionaryTrailer is a (modified) dictionary returned by method + originalIncrementalUpdate. + + Use this method to terminate an incremental update. + """ + if self.IsWindows(): + eol = '\n' + else: + eol = ' \n' + + self.appendString("\n") + startxref = self.filesize() + self.appendString("xref\n0 1\n") + self.appendString("0000000000 65535 f%s" % eol) + for i in self.indirectObjects.keys(): + self.appendString("%d 1\n" % i) + self.appendString("%010d %05d n%s" % (self.indirectObjects[i], 0, eol)) + self.appendString("trailer\n%s\nstartxref\n%d\n%%%%EOF\n" % (dictionaryTrailer, startxref)) + return startxref + + def SplitByLength(self, input, length): + """ + Internal helper function + """ + result = [] + while len(input) > length: + result.append(input[0:length] + '\n') + input = input[length:] + result.append(input + '>') + return result + + def objstm(self, oObjectStream): + """ + Method to add an object stream to the PDF file. + + oObjectStream is an instantiated object of class cObjectStream. + """ + self.stream2(oObjectStream.index, oObjectStream.version, oObjectStream.getStream(), oObjectStream.getDictionaryEntries(), oObjectStream.filters) + self.objstms.append(oObjectStream) + + def xrefobjAndTrailer(self, index, version, root): + """ + Method to create an xref object together with a trailer and + output it to the PDF file. + + index is the index number of the xref object. + + version is the version number of the xref object. Use 0 by convention. + + root is a string with a reference to the root object (/Root). + Example: "1 0 R" + """ + maximumIndexValue = max(index, max(self.indirectObjects.keys())) + dObjects = {} + for objstm in self.objstms: + for indexIter in objstm.objects: + dObjects[indexIter] = objstm + maximumIndexValue = max(maximumIndexValue, max(dObjects.keys())) + + self.appendString('\n') + self.indirectObjects[index] = self.filesize() + + xrefFormat = '>BII' + xrefStream = '' + for iter in range(maximumIndexValue + 1): + if iter in self.indirectObjects.keys(): + xrefStream += struct.pack(xrefFormat, 1, self.indirectObjects[iter], 0) + elif iter in dObjects.keys(): + xrefStream += struct.pack(xrefFormat, 2, dObjects[iter].index, dObjects[iter].objects.index(iter)) + else: + xrefStream += struct.pack(xrefFormat, 0, 0, 0) + + formatSizes = ' '.join([str(size) for size in map(struct.calcsize, [c for c in xrefFormat]) if size != 0]) + self.appendString(('%d %d obj\n<< /Type /XRef /Length %d /W [%s] /Root %s /Size %d >>\nstream\n') % (index, version, len(xrefStream), formatSizes, root, maximumIndexValue + 1)) + self.appendBinary(xrefStream) + self.appendString('\nendstream\nendobj\n') + + self.appendString('\nstartxref\n%d\n%%%%EOF\n' % self.indirectObjects[index]) + +class cNameObfuscation: + """ + Class to implement random PDF name obfuscation + Example: /Page becomes /P#61ge + """ + + def __init__(self, probability=0.5, characters=1): + """ + class instantiation arguments: + + probability is a number between 0.0 and 1.0. It indicates + the probability a name gets obfuscated. 0.0 means a name will + never be obfuscated, 1.0 means a name will always be obfuscated. + default 0.5 + + characters is the number of characters in the name to obfuscated + by replacing them with the hex-equivalent (#??); default 1 + """ + self.probability = probability + self.characters = characters + + def IsNameCharacter(self, c): + """ + Internal helper function + """ + return c.lower() >= 'a' and c.lower() <= 'z' or c >= '0' and c <= '9' + + def ObfuscateName(self, name): + """ + Internal helper function + """ + if random.random() < self.probability: + if self.characters >= len(name): + population = range(len(name)) + else: + population = random.sample(range(len(name)), self.characters) + for iIndex in population: + name[iIndex] = '#%02X' % ord(name[iIndex]) + return '/' + ''.join(name) + + def Obfuscate(self, str): + """ + Use this method to randomly obfuscate the names found in the + provided string according to the instantiated class parameters. + The return value is the string with obfuscated names. + """ + result = '' + foundName = False + for c in str: + if not foundName and c == '/': + foundName = True + name = [] + elif foundName: + if self.IsNameCharacter(c): + name.append(c) + else: + result += self.ObfuscateName(name) + result += c + foundName = False + name = [] + else: + result += c + if foundName: + result += self.ObfuscateName(name) + return result + +class cFuzzer: + """ + Class to implement a simple fuzzer + """ + + def __init__(self, count=10, minimum=1, maximum=10, character='A'): + """ + class instantiation arguments: + + count is the number of fuzzed sequences (i.e. overwritten bytes) + produced by the fuzzer; default 10 + + minimum is the minimum length of a fuzzed sequence; default 1 + + maximum is the maximum length of a fuzzed sequence; default 10 + + character is the character used to generate the + fuzzed sequences; default 'A' + """ + self.count = count + self.minimum = minimum + self.maximum = maximum + self.character = character + + def Fuzz(self, str): + """ + Use this method to fuzz a string according to the + instantiated class parameters. + The return value is the fuzzed string. + """ + exploded = [c for c in str] + for count in range(self.count): + size = random.randint(self.minimum, self.maximum) + position = random.randint(0, len(str) - size) + for iIter in range(size): + exploded[position + iIter] = self.character + return ''.join(exploded) + +class cDictionary: + """ + Helper class to get and set values in PDF dictionaries + """ + + def __init__(self, string): + self.dictionary = string + + def GetID(self, name): + result = re.findall(r'/' + name + r'\s+(\d+)\s+0\s+[rR]', self.dictionary) + if result == []: + return None + return int(result[0]) + + def GetNumber(self, name): + result = re.findall(r'/' + name + r'\s+(\d+)', self.dictionary) + if result == []: + return None + return int(result[0]) + + def SetNumber(self, name, value): + oMatch = re.search(r'/' + name + r'\s+(\d+)', self.dictionary) + if oMatch == None: + self.Insert(name, str(value)) + else: + self.dictionary = self.dictionary[0:oMatch.start()] + '/' + name + ' ' + str(value) + self.dictionary[oMatch.end():] + + def Insert(self, name, value): + self.dictionary = self.dictionary[0:2] + '/' + name + ' ' + value + self.dictionary[2:] + + def SetReference(self, name, value): + oMatch = re.search(r'/' + name + r'\s+(\d+)\s+(\d+)\s+R', self.dictionary) + if oMatch == None: + oMatch = re.search(r'/' + name + r'\s*\[[^\[\]]+\]', self.dictionary) + if oMatch == None: + self.Insert(name, str(value)) + else: + self.dictionary = self.dictionary[0:oMatch.start()] + '/' + name + ' ' + str(value) + self.dictionary[oMatch.end():] + +class cObjectStream: + """ + Class to create an object stream (/ObjStm) + """ + + def __init__(self, index, version, filters=''): + """ + class instantiation arguments: + + index is the index number of the /ObjStm object. + + version is the version number of the /ObjStm object. Use 0 by convention. + + filters is a string with the encoding filters to be applied (see method stream2) + """ + self.index = index + self.version = version + self.filters = filters + self.indices = '' + self.ios = '' + self.objects = [] + + def indirectobject(self, index, io): + """ + Method to add an indirect object to the object stream. + + index is the index number of the object. + + io is the content of the indirect object. + """ + if self.indices != '': + self.indices += ' ' + self.indices += '%d %d' % (index, len(self.ios)) + self.ios += io + self.objects.append(index) + + def getDictionaryEntries(self): + """ + Internal helper function + """ + return '/Type /ObjStm\n /N %d\n /First %d' % (len(self.objects), len(self.indices)) + + def getStream(self): + """ + Internal helper function + """ + return self.indices + self.ios diff --git a/sift/files/pdf-tools/make-pdf-embedded.py b/sift/files/pdf-tools/make-pdf-embedded.py index 1a080d13..71ff6c4e 100644 --- a/sift/files/pdf-tools/make-pdf-embedded.py +++ b/sift/files/pdf-tools/make-pdf-embedded.py @@ -1,9 +1,9 @@ -#!/usr/bin/python +#!/usr/bin/env python3 __description__ = 'tool to create a PDF document with an embedded file' __author__ = 'Didier Stevens' -__version__ = '0.5.0' -__date__ = '2011/07/01' +__version__ = '0.5.1' +__date__ = '2017/04/23' """ Source code put in public domain by Didier Stevens, no Copyright @@ -19,13 +19,28 @@ 2008/11/09: V0.3, added autostart and button 2009/06/15: V0.4.0: added stego 2011/07/01: V0.5.0: added support for Python 3 - + 2017/04/23: V0.5.1: added option -n + Todo: """ import mPDF import optparse +# CIC: Call If Callable +def CIC(expression): + if callable(expression): + return expression() + else: + return expression + +# IFF: IF Function +def IFF(expression, valueTrue, valueFalse): + if expression: + return CIC(valueTrue) + else: + return CIC(valueFalse) + def ReadBinaryFile(name): """Read a binary file and return the content, return None if error occured """ @@ -93,6 +108,7 @@ def Main(): oParser.add_option('-b', '--button', action='store_true', default=False, help='add a "button" to launch the embedded file') oParser.add_option('-s', '--stego', action='store_true', default=False, help='"hide" the embedded file by replacing /EmbeddedFiles with /Embeddedfiles') oParser.add_option('-m', '--message', default='', help='text to display in the PDF document') + oParser.add_option('-n', '--name', default='', help='filename to use in the PDF objects (by default same as file-to-embed name)') (options, args) = oParser.parse_args() if len(args) != 2: @@ -111,7 +127,7 @@ def Main(): if embeddedFileContent == None: print('Error opening/reading file %s' % embeddedFileName) else: - CreatePDFWithEmbeddedFile(pdfFileName, embeddedFileName, embeddedFileContent, options.filters, options.nobinary, options.autoopen, options.button, options.stego, options.message) + CreatePDFWithEmbeddedFile(pdfFileName, IFF(options.name == '', embeddedFileName, options.name), embeddedFileContent, options.filters, options.nobinary, options.autoopen, options.button, options.stego, options.message) if __name__ == '__main__': Main() diff --git a/sift/files/pdf-tools/make-pdf-helloworld.py b/sift/files/pdf-tools/make-pdf-helloworld.py index c191c1c6..65a7fcd5 100644 --- a/sift/files/pdf-tools/make-pdf-helloworld.py +++ b/sift/files/pdf-tools/make-pdf-helloworld.py @@ -1,34 +1,34 @@ -#!/usr/bin/env python2 -#20080518 -#20080519 - -import mPDF -import time -import zlib -import sys - -if len(sys.argv) != 2: - print "Usage: make-pdf-helloworld pdf-file" - print " " - print " Source code put in the public domain by Didier Stevens, no Copyright" - print " Use at your own risk" - print " https://DidierStevens.com" - -else: - pdffile = sys.argv[1] - - oPDF = mPDF.cPDF(pdffile) - - oPDF.header() - - oPDF.template1() - - #oPDF.stream(5, 0, "BT /F1 24 Tf 100 700 Td (Hello World) Tj ET") - oPDF.stream(5, 0, """BT /F1 12 Tf 100 700 Td 15 TL -(Hello World) Tj -(Second Line) ' -(Third Line) ' -ET -100 712 100 -100 re S""") - - oPDF.xrefAndTrailer("1 0 R") +#!/usr/bin/env python3 +#20080518 +#20080519 + +import mPDF +import time +import zlib +import sys + +if len(sys.argv) != 2: + print("Usage: make-pdf-helloworld pdf-file") + print(" ") + print(" Source code put in the public domain by Didier Stevens, no Copyright") + print(" Use at your own risk") + print(" https://DidierStevens.com") + +else: + pdffile = sys.argv[1] + + oPDF = mPDF.cPDF(pdffile) + + oPDF.header() + + oPDF.template1() + + #oPDF.stream(5, 0, "BT /F1 24 Tf 100 700 Td (Hello World) Tj ET") + oPDF.stream(5, 0, """BT /F1 12 Tf 100 700 Td 15 TL +(Hello World) Tj +(Second Line) ' +(Third Line) ' +ET +100 712 100 -100 re S""") + + oPDF.xrefAndTrailer("1 0 R") diff --git a/sift/files/pdf-tools/make-pdf-javascript.py b/sift/files/pdf-tools/make-pdf-javascript.py index f22c6ecc..1a1a0287 100644 --- a/sift/files/pdf-tools/make-pdf-javascript.py +++ b/sift/files/pdf-tools/make-pdf-javascript.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # V0.1 2008/05/23 # make-pdf-javascript, use it to create a PDF document with embedded JavaScript that will execute automatically when the document is opened @@ -8,7 +8,7 @@ # Use at your own risk # # History: -# +# # 2008/05/29: continue # 2008/11/09: cleanup for release @@ -26,24 +26,21 @@ def Main(): if len(args) != 1: parser.print_help() - print '' - print ' make-pdf-javascript, use it to create a PDF document with embedded JavaScript that will execute automatically when the document is opened' - print ' Source code put in the public domain by Didier Stevens, no Copyright' - print ' Use at your own risk' - print ' https://DidierStevens.com' - + print('') + print(' make-pdf-javascript, use it to create a PDF document with embedded JavaScript that will execute automatically when the document is opened') + print(' Source code put in the public domain by Didier Stevens, no Copyright') + print(' Use at your own risk') + print(' https://DidierStevens.com') + else: oPDF = mPDF.cPDF(args[0]) - oPDF.header() - oPDF.indirectobject(1, 0, '<<\n /Type /Catalog\n /Outlines 2 0 R\n /Pages 3 0 R\n /OpenAction 7 0 R\n>>') oPDF.indirectobject(2, 0, '<<\n /Type /Outlines\n /Count 0\n>>') oPDF.indirectobject(3, 0, '<<\n /Type /Pages\n /Kids [4 0 R]\n /Count 1\n>>') oPDF.indirectobject(4, 0, '<<\n /Type /Page\n /Parent 3 0 R\n /MediaBox [0 0 612 792]\n /Contents 5 0 R\n /Resources <<\n /ProcSet [/PDF /Text]\n /Font << /F1 6 0 R >>\n >>\n>>') oPDF.stream(5, 0, 'BT /F1 12 Tf 100 700 Td 15 TL (JavaScript example) Tj ET') oPDF.indirectobject(6, 0, '<<\n /Type /Font\n /Subtype /Type1\n /Name /F1\n /BaseFont /Helvetica\n /Encoding /MacRomanEncoding\n>>') - if options.javascript == None and options.javascriptfile == None: javascript = """app.alert({cMsg: 'Hello from PDF JavaScript', cTitle: 'Testing PDF JavaScript', nIcon: 3});""" elif options.javascript != None: @@ -52,19 +49,19 @@ def Main(): try: fileJavasScript = open(options.javascriptfile, 'rb') except: - print "error opening file %s" % options.javascriptfile + print("error opening file %s" % options.javascriptfile) return try: javascript = fileJavasScript.read() except: - print "error reading file %s" % options.javascriptfile + print("error reading file %s" % options.javascriptfile) return finally: fileJavasScript.close() - + oPDF.indirectobject(7, 0, '<<\n /Type /Action\n /S /JavaScript\n /JS (%s)\n>>' % javascript) - + oPDF.xrefAndTrailer('1 0 R') if __name__ == '__main__': diff --git a/sift/files/pdf-tools/pdf-parser.py b/sift/files/pdf-tools/pdf-parser.py index 190b6928..b4083288 100644 --- a/sift/files/pdf-tools/pdf-parser.py +++ b/sift/files/pdf-tools/pdf-parser.py @@ -1,11 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/env python3 __description__ = 'pdf-parser, use it to parse a PDF document' __author__ = 'Didier Stevens' -__version__ = '0.6.0' -__date__ = '2015/01/11' +__version__ = '0.7.10' +__date__ = '2024/10/26' __minimum_python_version__ = (2, 5, 1) -__maximum_python_version__ = (3, 4, 2) +__maximum_python_version__ = (3, 11, 1) """ Source code put in public domain by Didier Stevens, no Copyright @@ -49,10 +49,37 @@ 2014/12/09: cleanup, refactoring 2014/12/13: Python 3 fixes 2015/01/11: Added support for multiple YARA rule files; added request to search in trailer + 2015/01/31: V0.6.1 Added optionyarastrings + 2015/02/09: Added decoders + 2015/04/05: V0.6.2 Added generateembedded + 2015/04/06: fixed bug reported by Kurt for stream produced by Ghostscript where endstream is not preceded by whitespace; fixed prettyprint bug + 2015/04/24: V0.6.3 when option dump's filename is -, content is dumped to stdout + 2015/08/12: V0.6.4 option hash now also calculates hashes of streams when selecting or searching objects; and displays hexasciidump first line + 2016/07/27: V0.6.5 bugfix whitespace 0x00 0x0C after stream 0x0D 0x0A reported by @mr_me + 2016/11/20: V0.6.6 added workaround zlib errors FlateDecode + 2016/12/17: V0.6.7 added option -k + 2017/01/07: V0.6.8 changed cPDFParseDictionary to handle strings () with % character + 2017/10/28: fixed bug + 2017/10/29: added # support for option -y + 2018/06/29: V0.6.9 added option --overridingfilters + 2018/10/20: added keywords to statistics + 2019/02/22: V0.7.0 added option -O --objstm to parse the stream of /ObjStm objects, inspired by a contributor wishing anonymity + 2019/03/01: V0.7.1 added ContainsName for correct keyword statistics (-a) + 2019/04/12: V0.7.2 Python 2.6.6 compatibility fix + 2019/07/30: bug fixes (including fixes Josef Hinteregger) + 2019/09/26: V0.7.3 added multiple id selection to option -o; added man page (-m); added environment variable PDFPARSER_OPTIONS; bug fixes + 2019/11/05: V0.7.4 fixed plugin path when compiled with pyinstaller, replaced eval with int + 2021/07/03: V0.7.5 bug fixes; fixed ASCII85Decode Python 3 bug thanks to R Primus + 2021/11/23: V0.7.6 Python 3 bug fixes + 2022/05/24: bug fixes + 2022/11/09: V0.7.7 added support for environment variable DSS_DEFAULT_HASH_ALGORITHMS + 2023/01/03: V0.7.8 added unreferenced objects to statistics + 2024/03/21: V0.7.9 added option jsonoutput; added verbose YARA rules + 2024/10/25: V0.7.10 /ObjStm fix (x9090 PR) + 2024/10/26: added pyzipper support Todo: - handle printf todo - - fix PrettyPrint - support for JS hex string EC61C64349DB8D88AF0523C4C06E0F4D.pdf.vir """ @@ -63,21 +90,28 @@ import binascii import hashlib import sys -import zipfile import time import os +import textwrap +import json if sys.version_info[0] >= 3: from io import StringIO import urllib.request urllib23 = urllib.request + import configparser as ConfigParser else: from cStringIO import StringIO import urllib2 urllib23 = urllib2 + import ConfigParser try: import yara except: pass +try: + import pyzipper as zipfile +except ImportError: + import zipfile CHAR_WHITESPACE = 1 CHAR_DELIMITER = 2 @@ -95,13 +129,67 @@ PDF_ELEMENT_STARTXREF = 5 PDF_ELEMENT_MALFORMED = 6 +dumplinelength = 16 + +def PrintManual(): + manual = ''' +Manual: + +This manual is a work in progress. + +There is a free PDF analysis book: +https://blog.didierstevens.com/2010/09/26/free-malicious-pdf-analysis-e-book/ + +Option -o is used to select objects by id. Provide a single id or multiple ids separated by a comma (,). + +When environment variable PDFPARSER_OPTIONS is defined, the options it defines are added implicitely to the command line arguments. +Use this to define options you want included with each use of pdf-parser.py. +Like option -O, to parse stream objects (/ObjStm). +By defining PDFPARSER_OPTIONS=-O, pdf-parser will always parse stream objects (when found). +PS: this feature is experimental. + +Option -H calculates the MD5 hash by default. +This can be changed by setting environment variable DSS_DEFAULT_HASH_ALGORITHMS. +Like this: set DSS_DEFAULT_HASH_ALGORITHMS=sha256 + +Option --jsonoutput produces JSON output with the stream content of all objects with streams. Options -f and --overridingfilters apply. +For example, if option -f is used, the JSON output contains the filtered streams, otherwise the JSON output contains the unfiltered streams. + +''' + for line in manual.split('\n'): + print(textwrap.fill(line)) + #Convert 2 Bytes If Python 3 def C2BIP3(string): if sys.version_info[0] > 2: - return bytes([ord(x) for x in string]) + if type(string) == bytes: + return string + else: + return bytes([ord(x) for x in string]) else: return string +#Convert 2 String If Python 3 +def C2SIP3(bytes): + if sys.version_info[0] > 2: + return ''.join([chr(byte) for byte in bytes]) + else: + return bytes + +# CIC: Call If Callable +def CIC(expression): + if callable(expression): + return expression() + else: + return expression + +# IFF: IF Function +def IFF(expression, valueTrue, valueFalse): + if expression: + return CIC(valueTrue) + else: + return CIC(valueFalse) + def Timestamp(epoch=None): if epoch == None: localTime = time.localtime() @@ -119,10 +207,18 @@ def CopyWithoutWhiteSpace(content): def Obj2Str(content): return ''.join(map(lambda x: repr(x[1])[1:-1], CopyWithoutWhiteSpace(content))) +def CreateZipFileObject(arg1, arg2): + if 'AESZipFile' in dir(zipfile): + return zipfile.AESZipFile(arg1, arg2) + else: + return zipfile.ZipFile(arg1, arg2) + class cPDFDocument: def __init__(self, file): self.file = file - if file.lower().startswith('http://') or file.lower().startswith('https://'): + if type(file) != str: + self.infile = file + elif file.lower().startswith('http://') or file.lower().startswith('https://'): try: if sys.hexversion >= 0x020601F0: self.infile = urllib23.urlopen(file, timeout=5) @@ -134,7 +230,7 @@ def __init__(self, file): sys.exit() elif file.lower().endswith('.zip'): try: - self.zipfile = zipfile.ZipFile(file, 'r') + self.zipfile = CreateZipFileObject(file, 'r') self.infile = self.zipfile.open(self.zipfile.infolist()[0], 'r', C2BIP3('infected')) except: print('Error opening file %s' % file) @@ -251,16 +347,25 @@ def TokenIgnoreWhiteSpace(self): token = self.Token() return token + def Tokens(self): + tokens = [] + token = self.Token() + while token != None: + tokens.append(token) + token = self.Token() + return tokens + def unget(self, byte): self.ungetted.append(byte) class cPDFParser: - def __init__(self, file, verbose=False, extract=None): + def __init__(self, file, verbose=False, extract=None, objstm=None): self.context = CONTEXT_NONE self.content = [] self.oPDFTokenizer = cPDFTokenizer(file) self.verbose = verbose self.extract = extract + self.objstm = objstm def GetObject(self): while True: @@ -300,7 +405,7 @@ def GetObject(self): else: if self.context == CONTEXT_OBJ: if self.token[1] == 'endobj': - self.oPDFElementIndirectObject = cPDFElementIndirectObject(self.objectId, self.objectVersion, self.content) + self.oPDFElementIndirectObject = cPDFElementIndirectObject(self.objectId, self.objectVersion, self.content, self.objstm) self.context = CONTEXT_NONE self.content = [] return self.oPDFElementIndirectObject @@ -330,8 +435,8 @@ def GetObject(self): if IsNumeric(self.token2[1]): self.token3 = self.oPDFTokenizer.TokenIgnoreWhiteSpace() if self.token3[1] == 'obj': - self.objectId = eval(self.token[1]) - self.objectVersion = eval(self.token2[1]) + self.objectId = int(self.token[1], 10) + self.objectVersion = int(self.token2[1], 10) self.context = CONTEXT_OBJ else: self.oPDFTokenizer.unget(self.token3) @@ -351,7 +456,7 @@ def GetObject(self): elif self.token[1] == 'startxref': self.token2 = self.oPDFTokenizer.TokenIgnoreWhiteSpace() if self.token2 and IsNumeric(self.token2[1]): - return cPDFElementStartxref(eval(self.token2[1])) + return cPDFElementStartxref(int(self.token2[1], 10)) else: self.oPDFTokenizer.unget(self.token2) if self.verbose: @@ -402,11 +507,31 @@ def IIf(expr, truepart, falsepart): return falsepart class cPDFElementIndirectObject: - def __init__(self, id, version, content): + def __init__(self, id, version, content, objstm=None): self.type = PDF_ELEMENT_INDIRECT_OBJECT self.id = id self.version = version self.content = content + self.objstm = objstm + #fix stream for Ghostscript bug reported by Kurt + if self.ContainsStream(): + position = len(self.content) - 1 + if position < 0: + return + while self.content[position][0] == CHAR_WHITESPACE and position >= 0: + position -= 1 + if position < 0: + return + if self.content[position][1].endswith('endstream\n'): + self.content = self.content[0:position] + [(self.content[position][0], self.content[position][1][:-len('endstream\n')])] + [(CHAR_REGULAR, 'endstream')] + self.content[position+1:] + return + if self.content[position][0] != CHAR_REGULAR: + return + if self.content[position][1] == 'endstream': + return + if not self.content[position][1].endswith('endstream'): + return + self.content = self.content[0:position] + [(self.content[position][0], self.content[position][1][:-len('endstream')])] + [(self.content[position][0], 'endstream')] + self.content[position+1:] def GetType(self): content = CopyWithoutWhiteSpace(self.content) @@ -449,12 +574,22 @@ def Contains(self, keyword): data += Canonicalize(self.content[i][1]) return data.upper().find(keyword.upper()) != -1 - def StreamContains(self, keyword, filter, casesensitive, regex): + def ContainsName(self, keyword): + for token in self.content: + if token[1] == 'stream': + return False + if token[0] == CHAR_DELIMITER and EqualCanonical(token[1], keyword): + return True + return False + + def StreamContains(self, keyword, filter, casesensitive, regex, overridingfilters): if not self.ContainsStream(): return False - streamData = self.Stream(filter) + streamData = self.Stream(filter, overridingfilters) if filter and streamData == 'No filters': - streamData = self.Stream(False) + streamData = self.Stream(False, overridingfilters) + if isinstance(streamData, bytes): + keyword = keyword.encode() if regex: return re.search(keyword, streamData, IIf(casesensitive, 0, re.I)) elif casesensitive: @@ -462,7 +597,7 @@ def StreamContains(self, keyword, filter, casesensitive, regex): else: return keyword.lower() in streamData.lower() - def Stream(self, filter=True): + def Stream(self, filter=True, overridingfilters=''): state = 'start' countDirectories = 0 data = '' @@ -492,13 +627,24 @@ def Stream(self, filter=True): if self.content[i][0] == CHAR_REGULAR and self.content[i][1] == 'stream': state = 'stream-whitespace' elif state == 'stream-whitespace': - if self.content[i][0] != CHAR_WHITESPACE: + if self.content[i][0] == CHAR_WHITESPACE: + whitespace = self.content[i][1] + if whitespace.startswith('\x0D\x0A') and len(whitespace) > 2: + data += whitespace[2:] + elif whitespace.startswith('\x0A') and len(whitespace) > 1: + data += whitespace[1:] + else: data += self.content[i][1] state = 'stream-concat' elif state == 'stream-concat': if self.content[i][0] == CHAR_REGULAR and self.content[i][1] == 'endstream': if filter: - return self.Decompress(data, filters) + if overridingfilters == '': + return self.Decompress(data, filters) + elif overridingfilters == 'raw': + return data + else: + return self.Decompress(data, overridingfilters.split(' ')) else: return data else: @@ -546,19 +692,30 @@ def Decompress(self, data, filters): else: return data - def StreamYARAMatch(self, rules, filter): + def StreamYARAMatch(self, rules, decoders, decoderoptions, filter, overridingfilters): if not self.ContainsStream(): return None - streamData = self.Stream(filter) + streamData = self.Stream(filter, overridingfilters) if filter and streamData == 'No filters': - streamData = self.Stream(False) - return rules.match(data=streamData) -# return rules.match(data=streamData, callback=mycallback) - -def mycallback(data): - print(data['rule']) - yara.CALLBACK_CONTINUE - + streamData = self.Stream(False, overridingfilters) + + oDecoders = [cIdentity(streamData, None)] + for cDecoder in decoders: + try: + oDecoder = cDecoder(streamData, decoderoptions) + oDecoders.append(oDecoder) + except Exception as e: + print('Error instantiating decoder: %s' % cDecoder.name) + raise e + results = [] + for oDecoder in oDecoders: + while oDecoder.Available(): + yaraResults = rules.match(data=oDecoder.Decode()) + if yaraResults != []: + results.append([oDecoder.Name(), yaraResults]) + + return results + class cPDFElementStartxref: def __init__(self, index): self.type = PDF_ELEMENT_STARTXREF @@ -586,7 +743,7 @@ def __init__(self, content, nocanonicalizedoutput): dataTrimmed = TrimLWhiteSpace(TrimRWhiteSpace(self.content)) if dataTrimmed == []: self.parsed = None - elif self.isOpenDictionary(dataTrimmed[0]) and self.isCloseDictionary(dataTrimmed[-1]): + elif self.isOpenDictionary(dataTrimmed[0]) and (self.isCloseDictionary(dataTrimmed[-1]) or self.couldBeCloseDictionary(dataTrimmed[-1])): self.parsed = self.ParseDictionary(dataTrimmed)[0] else: self.parsed = None @@ -597,6 +754,9 @@ def isOpenDictionary(self, token): def isCloseDictionary(self, token): return token[0] == CHAR_DELIMITER and token[1] == '>>' + def couldBeCloseDictionary(self, token): + return token[0] == CHAR_DELIMITER and token[1].rstrip().endswith('>>') + def ParseDictionary(self, tokens): state = 0 # start dictionary = [] @@ -634,6 +794,28 @@ def ParseDictionary(self, tokens): dictionary.append((key, value)) value = [] state = 1 + elif value == [] and tokens[0][1] == '(': + value.append(tokens[0][1]) + elif value != [] and value[0] == '(' and tokens[0][1] != ')': + if tokens[0][1][0] == '%': + tokens = [tokens[0]] + cPDFTokenizer(StringIO(tokens[0][1][1:])).Tokens() + tokens[1:] + value.append('%') + else: + value.append(tokens[0][1]) + elif value != [] and value[0] == '(' and tokens[0][1] == ')': + value.append(tokens[0][1]) + balanced = 0 + for item in value: + if item == '(': + balanced += 1 + elif item == ')': + balanced -= 1 + if balanced < 0 and self.verbose: + print('todo 11: ' + repr(value)) + if balanced < 1: + dictionary.append((key, value)) + value = [] + state = 1 elif value != [] and tokens[0][1][0] == '/': dictionary.append((key, value)) key = ConditionalCanonicalize(tokens[0][1], self.nocanonicalizedoutput) @@ -642,25 +824,33 @@ def ParseDictionary(self, tokens): else: value.append(ConditionalCanonicalize(tokens[0][1], self.nocanonicalizedoutput)) tokens = tokens[1:] + return None, tokens def Retrieve(self): return self.parsed + def PrettyPrintSubElement(self, prefix, e): + if e[1] == []: + print('%s %s' % (prefix, e[0])) + elif type(e[1][0]) == type(''): + if len(e[1]) == 3 and IsNumeric(e[1][0]) and e[1][1] == '0' and e[1][2] == 'R': + joiner = ' ' + else: + joiner = '' + value = joiner.join(e[1]).strip() + reprValue = repr(value) + if "'" + value + "'" != reprValue: + value = reprValue + print('%s %s %s' % (prefix, e[0], value)) + else: + print('%s %s' % (prefix, e[0])) + self.PrettyPrintSub(prefix + ' ', e[1]) + def PrettyPrintSub(self, prefix, dictionary): if dictionary != None: print('%s<<' % prefix) for e in dictionary: - if e[1] == []: - print('%s %s' % (prefix, e[0])) - elif type(e[1][0]) == type(''): - value = ''.join(e[1]).strip() - reprValue = repr(value) - if "'" + value + "'" != reprValue: - value = reprValue - print('%s %s %s' % (prefix, e[0], value)) - else: - print('%s %s' % (prefix, e[0])) - self.PrettyPrintSub(prefix + ' ', e[1]) + self.PrettyPrintSubElement(prefix, e) print('%s>>' % prefix) def PrettyPrint(self, prefix): @@ -672,17 +862,60 @@ def Get(self, select): return value return None + def GetNestedSub(self, dictionary, select): + for key, value in dictionary: + if key == select: + return self.PrettyPrintSubElement('', [select, value]) + if type(value) == type([]) and len(value) > 0 and type(value[0]) == type((None,)): + result = self.GetNestedSub(value, select) + if result !=None: + return self.PrettyPrintSubElement('', [select, result]) + return None + + def GetNested(self, select): + return self.GetNestedSub(self.parsed, select) + def FormatOutput(data, raw): if raw: if type(data) == type([]): return ''.join(map(lambda x: x[1], data)) else: return data + elif sys.version_info[0] > 2: + return ascii(data) else: return repr(data) +#Fix for http://bugs.python.org/issue11395 +def StdoutWriteChunked(data): + if sys.version_info[0] > 2: + sys.stdout.buffer.write(data) + else: + while data != '': + sys.stdout.write(data[0:10000]) + try: + sys.stdout.flush() + except IOError: + return + data = data[10000:] + +def IfWIN32SetBinary(io): + if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(io.fileno(), os.O_BINARY) + def PrintOutputObject(object, options): + if options.dump == '-': + filtered = object.Stream(options.filter == True, options.overridingfilters) + if filtered == []: + filtered = '' + IfWIN32SetBinary(sys.stdout) + StdoutWriteChunked(filtered) + return + print('obj %d %d' % (object.id, object.version)) + if object.objstm != None: + print(' Containing /ObjStm: %d %d' % object.objstm) print(' Type: %s' % ConditionalCanonicalize(object.GetType(), options.nocanonicalizedoutput)) print(' Referencing: %s' % ', '.join(map(lambda x: '%s %s %s' % x, object.GetReferences()))) dataPrecedingStream = object.ContainsStream() @@ -692,6 +925,16 @@ def PrintOutputObject(object, options): if options.debug: print(' %s' % FormatOutput(dataPrecedingStream, options.raw)) oPDFParseDictionary = cPDFParseDictionary(dataPrecedingStream, options.nocanonicalizedoutput) + if options.hash: + streamContent = object.Stream(False, options.overridingfilters) + print(' unfiltered') + print(' len: %6d md5: %s' % (len(streamContent), hashlib.md5(streamContent).hexdigest())) + print(' %s' % HexAsciiDumpLine(streamContent)) + streamContent = object.Stream(True, options.overridingfilters) + print(' filtered') + print(' len: %6d md5: %s' % (len(streamContent), hashlib.md5(streamContent).hexdigest())) + print(' %s' % HexAsciiDumpLine(streamContent)) + streamContent = None else: if options.debug or options.raw: print(' %s' % FormatOutput(object.content, options.raw)) @@ -700,14 +943,14 @@ def PrintOutputObject(object, options): oPDFParseDictionary.PrettyPrint(' ') print('') if options.filter and not options.dump: - filtered = object.Stream() + filtered = object.Stream(overridingfilters=options.overridingfilters) if filtered == []: print(' %s' % FormatOutput(object.content, options.raw)) else: print(' %s' % FormatOutput(filtered, options.raw)) if options.content: if object.ContainsStream(): - stream = object.Stream(False) + stream = object.Stream(False, options.overridingfilters) if stream != []: print(' %s' % FormatOutput(stream, options.raw)) else: @@ -715,7 +958,7 @@ def PrintOutputObject(object, options): if options.dump: - filtered = object.Stream(options.filter == True) + filtered = object.Stream(options.filter == True, options.overridingfilters) if filtered == []: filtered = '' try: @@ -766,7 +1009,7 @@ def ConditionalCanonicalize(sIn, nocanonicalizedoutput): def ASCII85Decode(data): import struct n = b = 0 - out = '' + out = b'' for c in data: if '!' <= c and c <= 'u': n += 1 @@ -776,7 +1019,7 @@ def ASCII85Decode(data): n = b = 0 elif c == 'z': assert n == 0 - out += '\0\0\0\0' + out += b'\0\0\0\0' elif c == '~': if n: for _ in range(5-n): @@ -788,8 +1031,26 @@ def ASCII85Decode(data): def ASCIIHexDecode(data): return binascii.unhexlify(''.join([c for c in data if c not in ' \t\n\r']).rstrip('>')) +# if inflating fails, we try to inflate byte per byte (sample 4da299d6e52bbb79c0ac00bad6a1d51d4d5fe42965a8d94e88a359e5277117e2) def FlateDecode(data): - return zlib.decompress(C2BIP3(data)) + try: + return zlib.decompress(C2BIP3(data)) + except: + if len(data) <= 10: + raise + oDecompress = zlib.decompressobj() + oStringIO = StringIO() + count = 0 + for byte in C2BIP3(data): + try: + oStringIO.write(oDecompress.decompress(byte)) + count += 1 + except: + break + if len(data) - count <= 2: + return oStringIO.getvalue() + else: + raise def RunLengthDecode(data): f = StringIO(data) @@ -892,13 +1153,17 @@ def run(self): def LZWDecode(data): return ''.join(LZWDecoder(StringIO(data)).run()) -def PrintGenerateObject(object, options): +def PrintGenerateObject(object, options, newId=None): + if newId == None: + objectId = object.id + else: + objectId = newId dataPrecedingStream = object.ContainsStream() if dataPrecedingStream: if options.filter: - decompressed = object.Stream(True) + decompressed = object.Stream(True, options.overridingfilters) if decompressed == 'No filters' or decompressed.startswith('Unsupported filter: '): - print(' oPDF.stream(%d, %d, %s, %s)' % (object.id, object.version, repr(object.Stream(False).rstrip()), repr(re.sub('/Length\s+\d+', '/Length %d', FormatOutput(dataPrecedingStream, True)).strip()))) + print(' oPDF.stream(%d, %d, %s, %s)' % (objectId, object.version, repr(object.Stream(False, options.overridingfilters).rstrip()), repr(re.sub('/Length\s+\d+', '/Length %d', FormatOutput(dataPrecedingStream, True)).strip()))) else: dictionary = FormatOutput(dataPrecedingStream, True) dictionary = re.sub(r'/Length\s+\d+', '', dictionary) @@ -907,11 +1172,11 @@ def PrintGenerateObject(object, options): dictionary = re.sub(r'^\s*<<', '', dictionary) dictionary = re.sub(r'>>\s*$', '', dictionary) dictionary = dictionary.strip() - print(" oPDF.stream2(%d, %d, %s, %s, 'f')" % (object.id, object.version, repr(decompressed.rstrip()), repr(dictionary))) + print(" oPDF.stream2(%d, %d, %s, %s, 'f')" % (objectId, object.version, repr(decompressed.rstrip()), repr(dictionary))) else: - print(' oPDF.stream(%d, %d, %s, %s)' % (object.id, object.version, repr(object.Stream(False).rstrip()), repr(re.sub('/Length\s+\d+', '/Length %d', FormatOutput(dataPrecedingStream, True)).strip()))) + print(' oPDF.stream(%d, %d, %s, %s)' % (objectId, object.version, repr(object.Stream(False, options.overridingfilters).rstrip()), repr(re.sub('/Length\s+\d+', '/Length %d', FormatOutput(dataPrecedingStream, True)).strip()))) else: - print(' oPDF.indirectobject(%d, %d, %s)' % (object.id, object.version, repr(FormatOutput(object.content, True).strip()))) + print(' oPDF.indirectobject(%d, %d, %s)' % (objectId, object.version, repr(FormatOutput(object.content, True).strip()))) def PrintObject(object, options): if options.generate: @@ -941,31 +1206,243 @@ def ProcessAt(argument): else: return [argument] -def YARACompile(fileordirname): - dFilepaths = {} - if os.path.isdir(fileordirname): - for root, dirs, files in os.walk(fileordirname): - for file in files: - filename = os.path.join(root, file) +def YARACompile(ruledata): + if ruledata.startswith('#'): + if ruledata.startswith('#h#'): + rule = binascii.a2b_hex(ruledata[3:]) + elif ruledata.startswith('#b#'): + rule = binascii.a2b_base64(ruledata[3:]) + elif ruledata.startswith('#s#'): + rule = 'rule string {strings: $a = "%s" ascii wide nocase condition: $a}' % ruledata[3:] + elif ruledata.startswith('#q#'): + rule = ruledata[3:].replace("'", '"') + elif ruledata.startswith('#x#'): + rule = 'rule hexadecimal {strings: $a = { %s } condition: $a}' % ruledata[3:] + elif ruledata.startswith('#r#'): + rule = 'rule regex {strings: $a = /%s/ ascii wide nocase condition: $a}' % ruledata[3:] + else: + rule = ruledata[1:] + return yara.compile(source=rule), rule + else: + dFilepaths = {} + if os.path.isdir(ruledata): + for root, dirs, files in os.walk(ruledata): + for file in files: + filename = os.path.join(root, file) + dFilepaths[filename] = filename + else: + for filename in ProcessAt(ruledata): dFilepaths[filename] = filename + return yara.compile(filepaths=dFilepaths), ','.join(dFilepaths.values()) + +def AddDecoder(cClass): + global decoders + + decoders.append(cClass) + +class cDecoderParent(): + pass + +def GetScriptPath(): + if getattr(sys, 'frozen', False): + return os.path.dirname(sys.executable) + else: + return os.path.dirname(sys.argv[0]) + +def LoadDecoders(decoders, verbose): + if decoders == '': + return + scriptPath = GetScriptPath() + for decoder in sum(map(ProcessAt, decoders.split(',')), []): + try: + if not decoder.lower().endswith('.py'): + decoder += '.py' + if os.path.dirname(decoder) == '': + if not os.path.exists(decoder): + scriptDecoder = os.path.join(scriptPath, decoder) + if os.path.exists(scriptDecoder): + decoder = scriptDecoder + exec(open(decoder, 'r').read(), globals(), globals()) + except Exception as e: + print('Error loading decoder: %s' % decoder) + if verbose: + raise e + +class cIdentity(cDecoderParent): + name = 'Identity function decoder' + + def __init__(self, stream, options): + self.stream = stream + self.options = options + self.available = True + + def Available(self): + return self.available + + def Decode(self): + self.available = False + return self.stream + + def Name(self): + return '' + +def DecodeFunction(decoders, options, stream): + if decoders == []: + return stream + return decoders[0](stream, options.decoderoptions).Decode() + +class cDumpStream(): + def __init__(self): + self.text = '' + + def Addline(self, line): + if line != '': + self.text += line + '\n' + + def Content(self): + return self.text + +def HexDump(data): + oDumpStream = cDumpStream() + hexDump = '' + for i, b in enumerate(data): + if i % dumplinelength == 0 and hexDump != '': + oDumpStream.Addline(hexDump) + hexDump = '' + hexDump += IFF(hexDump == '', '', ' ') + '%02X' % ord(b) + oDumpStream.Addline(hexDump) + return oDumpStream.Content() + +def CombineHexAscii(hexDump, asciiDump): + if hexDump == '': + return '' + return hexDump + ' ' + (' ' * (3 * (dumplinelength - len(asciiDump)))) + asciiDump + +def HexAsciiDump(data): + oDumpStream = cDumpStream() + hexDump = '' + asciiDump = '' + for i, b in enumerate(data): + if i % dumplinelength == 0: + if hexDump != '': + oDumpStream.Addline(CombineHexAscii(hexDump, asciiDump)) + hexDump = '%08X:' % i + asciiDump = '' + hexDump+= ' %02X' % ord(b) + asciiDump += IFF(ord(b) >= 32, b, '.') + oDumpStream.Addline(CombineHexAscii(hexDump, asciiDump)) + return oDumpStream.Content() + +def HexAsciiDumpLine(data): + return HexAsciiDump(data[0:16])[10:-1] + +def ParseINIFile(): + oConfigParser = ConfigParser.ConfigParser(allow_no_value=True) + oConfigParser.optionxform = str + oConfigParser.read(os.path.join(GetScriptPath(), 'pdfid.ini')) + keywords = [] + if oConfigParser.has_section('keywords'): + for key, value in oConfigParser.items('keywords'): + if not key in keywords: + keywords.append(key) + return keywords + +def MatchObjectID(id, selection): + return str(id) in selection.split(',') + +def GetArguments(): + arguments = sys.argv[1:] + envvar = os.getenv('PDFPARSER_OPTIONS') + if envvar == None: + return arguments + return envvar.split(' ') + arguments + +class cHashCRC32(): + def __init__(self): + self.crc32 = None + + def update(self, data): + self.crc32 = zlib.crc32(data) + + def hexdigest(self): + return '%08x' % (self.crc32 & 0xffffffff) + +class cHashChecksum8(): + def __init__(self): + self.sum = 0 + + def update(self, data): + if sys.version_info[0] >= 3: + self.sum += sum(data) + else: + self.sum += sum(map(ord, data)) + + def hexdigest(self): + return '%08x' % (self.sum) + +dSpecialHashes = {'crc32': cHashCRC32, 'checksum8': cHashChecksum8} + +def GetHashObjects(algorithms): + global dSpecialHashes + + dHashes = {} + + if algorithms == '': + algorithms = os.getenv('DSS_DEFAULT_HASH_ALGORITHMS', 'md5') + if ',' in algorithms: + hashes = algorithms.split(',') else: - for filename in ProcessAt(fileordirname): - dFilepaths[filename] = filename - return yara.compile(filepaths=dFilepaths) + hashes = algorithms.split(';') + for name in hashes: + if not name in dSpecialHashes.keys() and not name in hashlib.algorithms_available: + print('Error: unknown hash algorithm: %s' % name) + print('Available hash algorithms: ' + ' '.join([name for name in list(hashlib.algorithms_available)] + list(dSpecialHashes.keys()))) + return [], {} + elif name in dSpecialHashes.keys(): + dHashes[name] = dSpecialHashes[name]() + else: + dHashes[name] = hashlib.new(name) + + return hashes, dHashes + +def CalculateChosenHash(data): + hashes, dHashes = GetHashObjects('') + dHashes[hashes[0]].update(data) + return dHashes[hashes[0]].hexdigest(), hashes[0] + +class cMyJSONOutput(): + + def __init__(self): + self.items = [] + self.counter = 1 + + def AddIdItem(self, id, name, data): + self.items.append({'id': id, 'name': name, 'content': binascii.b2a_base64(data).strip(b'\n').decode()}) + + def AddItem(self, name, data): + self.AddIdItem(self.counter, name, data) + self.counter += 1 + + def GetJSON(self): + return json.dumps({'version': 2, 'id': 'didierstevens.com', 'type': 'content', 'fields': ['id', 'name', 'content'], 'items': self.items}) def Main(): """pdf-parser, use it to parse a PDF document """ + global decoders + oParser = optparse.OptionParser(usage='usage: %prog [options] pdf-file|zip-file|url\n' + __description__, version='%prog ' + __version__) + oParser.add_option('-m', '--man', action='store_true', default=False, help='Print manual') oParser.add_option('-s', '--search', help='string to search in indirect objects (except streams)') oParser.add_option('-f', '--filter', action='store_true', default=False, help='pass stream object through filters (FlateDecode, ASCIIHexDecode, ASCII85Decode, LZWDecode and RunLengthDecode only)') - oParser.add_option('-o', '--object', help='id of indirect object to select (version independent)') + oParser.add_option('-o', '--object', help='id(s) of indirect object(s) to select, use comma (,) to separate ids (version independent)') oParser.add_option('-r', '--reference', help='id of indirect object being referenced (version independent)') oParser.add_option('-e', '--elements', help='type of elements to select (cxtsi)') oParser.add_option('-w', '--raw', action='store_true', default=False, help='raw output for data and filters') oParser.add_option('-a', '--stats', action='store_true', default=False, help='display stats for pdf document') oParser.add_option('-t', '--type', help='type of indirect object to select') + oParser.add_option('-O', '--objstm', action='store_true', default=False, help='parse stream of /ObjStm objects') oParser.add_option('-v', '--verbose', action='store_true', default=False, help='display malformed PDF elements') oParser.add_option('-x', '--extract', help='filename to extract malformed content to') oParser.add_option('-H', '--hash', action='store_true', default=False, help='display hash of objects') @@ -977,9 +1454,21 @@ def Main(): oParser.add_option('--unfiltered', action='store_true', default=False, help='search in unfiltered streams') oParser.add_option('--casesensitive', action='store_true', default=False, help='case sensitive search in streams') oParser.add_option('--regex', action='store_true', default=False, help='use regex to search in streams') + oParser.add_option('--overridingfilters', type=str, default='', help='override filters with given filters (use raw for the raw stream content)') oParser.add_option('-g', '--generate', action='store_true', default=False, help='generate a Python program that creates the parsed PDF file') + oParser.add_option('--generateembedded', type=int, default=0, help='generate a Python program that embeds the selected indirect object as a file') oParser.add_option('-y', '--yara', help='YARA rule (or directory or @file) to check streams (can be used with option --unfiltered)') - (options, args) = oParser.parse_args() + oParser.add_option('--yarastrings', action='store_true', default=False, help='Print YARA strings') + oParser.add_option('--decoders', type=str, default='', help='decoders to load (separate decoders with a comma , ; @file supported)') + oParser.add_option('--decoderoptions', type=str, default='', help='options for the decoder') + oParser.add_option('-k', '--key', help='key to search in dictionaries') + oParser.add_option('-j', '--jsonoutput', action='store_true', default=False, help='produce json output') + (options, args) = oParser.parse_args(GetArguments()) + + if options.man: + oParser.print_help() + PrintManual() + return 0 if len(args) != 1: oParser.print_help() @@ -990,6 +1479,9 @@ def Main(): print(' https://DidierStevens.com') else: + decoders = [] + LoadDecoders(options.decoders, True) + oPDFParser = cPDFParser(args[0], options.verbose, options.extract) cntComment = 0 cntXref = 0 @@ -997,6 +1489,19 @@ def Main(): cntStartXref = 0 cntIndirectObject = 0 dicObjectTypes = {} + objectsAll = set() + objectsReferenced = set() + objectsWithStream = [] + keywords = ['/JS', '/JavaScript', '/AA', '/OpenAction', '/AcroForm', '/RichMedia', '/Launch', '/EmbeddedFile', '/XFA', '/URI'] + for extrakeyword in ParseINIFile(): + if not extrakeyword in keywords: + keywords.append(extrakeyword) + +# dKeywords = {keyword: [] for keyword in keywords} +# Done for compatibility with 2.6.6 + dKeywords = {} + for keyword in keywords: + dKeywords[keyword] = [] selectComment = False selectXref = False @@ -1020,12 +1525,12 @@ def Main(): return else: selectIndirectObject = True - if not options.search and not options.object and not options.reference and not options.type and not options.searchstream: + if not options.search and not options.object and not options.reference and not options.type and not options.searchstream and not options.key: selectComment = True selectXref = True selectTrailer = True selectStartXref = True - if options.search: + if options.search or options.key or options.reference: selectTrailer = True if options.type == '-': @@ -1033,7 +1538,7 @@ def Main(): else: optionsType = options.type - if options.generate: + if options.generate or options.generateembedded != 0: savedRoot = ['1', '0', 'R'] print('#!/usr/bin/python') print('') @@ -1057,14 +1562,55 @@ def Main(): print(' return') print(' oPDF = mPDF.cPDF(sys.argv[1])') + if options.generateembedded != 0: + print(" oPDF.header('1.1')") + print(r" oPDF.comment('\xd0\xd0\xd0\xd0')") + print(r" oPDF.indirectobject(1, 0, '<<\r\n /Type /Catalog\r\n /Outlines 2 0 R\r\n /Pages 3 0 R\r\n /Names << /EmbeddedFiles << /Names [(test.bin) 7 0 R] >> >>\r\n>>')") + print(r" oPDF.indirectobject(2, 0, '<<\r\n /Type /Outlines\r\n /Count 0\r\n>>')") + print(r" oPDF.indirectobject(3, 0, '<<\r\n /Type /Pages\r\n /Kids [4 0 R]\r\n /Count 1\r\n>>')") + print(r" oPDF.indirectobject(4, 0, '<<\r\n /Type /Page\r\n /Parent 3 0 R\r\n /MediaBox [0 0 612 792]\r\n /Contents 5 0 R\r\n /Resources <<\r\n /ProcSet [/PDF /Text]\r\n /Font << /F1 6 0 R >>\r\n >>\r\n>>')") + print(r" oPDF.stream(5, 0, 'BT /F1 12 Tf 70 700 Td 15 TL (This PDF document embeds file test.bin) Tj ET', '<< /Length %d >>')") + print(r" oPDF.indirectobject(6, 0, '<<\r\n /Type /Font\r\n /Subtype /Type1\r\n /Name /F1\r\n /BaseFont /Helvetica\r\n /Encoding /MacRomanEncoding\r\n>>')") + print(r" oPDF.indirectobject(7, 0, '<<\r\n /Type /Filespec\r\n /F (test.bin)\r\n /EF << /F 8 0 R >>\r\n>>')") + if options.yara != None: if not 'yara' in sys.modules: print('Error: option yara requires the YARA Python module.') return - rules = YARACompile(options.yara) + rules, rulesVerbose = YARACompile(options.yara) + if options.verbose: + print(rulesVerbose) + oPDFParserOBJSTM = None + oMyJSONOutput = cMyJSONOutput() while True: - object = oPDFParser.GetObject() + if oPDFParserOBJSTM == None: + object = oPDFParser.GetObject() + else: + object = oPDFParserOBJSTM.GetObject() + if object == None: + oPDFParserOBJSTM = None + object = oPDFParser.GetObject() + if options.objstm and hasattr(object, 'GetType') and EqualCanonical(object.GetType(), '/ObjStm') and object.ContainsStream(): + # parsing objects inside an /ObjStm object by extracting & parsing the stream content to create a synthesized PDF document, that is then parsed by cPDFParser + oPDFParseDictionary = cPDFParseDictionary(object.ContainsStream(), options.nocanonicalizedoutput) + numberOfObjects = int(oPDFParseDictionary.Get('/N')[0]) + offsetFirstObject = int(oPDFParseDictionary.Get('/First')[0]) + indexes = list(map(int, C2SIP3(object.Stream())[:offsetFirstObject].strip().replace('\n', ' ').split(' '))) + if len(indexes) % 2 != 0 or len(indexes) / 2 != numberOfObjects: + raise Exception('Error in index of /ObjStm stream') + streamObject = C2SIP3(object.Stream()[offsetFirstObject:]) + synthesizedPDF = '' + while len(indexes) > 0: + objectNumber = indexes[0] + offset = indexes[1] + indexes = indexes[2:] + if len(indexes) >= 2: + offsetNextObject = indexes[1] + else: + offsetNextObject = len(streamObject) + synthesizedPDF += '%d 0 obj\n%s\nendobj\n' % (objectNumber, streamObject[offset:offsetNextObject]) + oPDFParserOBJSTM = cPDFParser(StringIO(synthesizedPDF), options.verbose, options.extract, (object.id, object.version)) if object != None: if options.stats: if object.type == PDF_ELEMENT_COMMENT: @@ -1073,6 +1619,10 @@ def Main(): cntXref += 1 elif object.type == PDF_ELEMENT_TRAILER: cntTrailer += 1 + oPDFParseDictionary = cPDFParseDictionary(object.content[1:], options.nocanonicalizedoutput) + for keyTrailer, valueTrailer in oPDFParseDictionary.parsed: + if len(valueTrailer) == 3 and valueTrailer[2] == 'R' and IsNumeric(valueTrailer[0]) and IsNumeric(valueTrailer[1]): + objectsReferenced.add(tuple(valueTrailer)) elif object.type == PDF_ELEMENT_STARTXREF: cntStartXref += 1 elif object.type == PDF_ELEMENT_INDIRECT_OBJECT: @@ -1082,6 +1632,21 @@ def Main(): dicObjectTypes[type1] = [object.id] else: dicObjectTypes[type1].append(object.id) + for keyword in dKeywords.keys(): + if object.ContainsName(keyword): + dKeywords[keyword].append(object.id) + if object.ContainsStream(): + objectsWithStream.append(object.id) + for reference in object.GetReferences(): + objectsReferenced.add(reference) + objectsAll.add((str(object.id), str(object.version), 'R')) + elif options.jsonoutput: + if object.type == PDF_ELEMENT_INDIRECT_OBJECT: + if object.ContainsStream(): + filtered = object.Stream(options.filter == True, options.overridingfilters) + if filtered == []: + filtered = '' + oMyJSONOutput.AddItem('obj %s %s' % (object.id, object.version), C2BIP3(filtered)) else: if object.type == PDF_ELEMENT_COMMENT and selectComment: if options.generate: @@ -1090,11 +1655,11 @@ def Main(): print(" oPDF.header('%s')" % comment[4:]) elif comment != '%EOF': print(' oPDF.comment(%s)' % repr(comment)) - elif options.yara == None: + elif options.yara == None and options.generateembedded == 0: print('PDF Comment %s' % FormatOutput(object.comment, options.raw)) print('') elif object.type == PDF_ELEMENT_XREF and selectXref: - if not options.generate and options.yara == None: + if not options.generate and options.yara == None and options.generateembedded == 0: if options.debug: print('xref %s' % FormatOutput(object.content, options.raw)) else: @@ -1106,24 +1671,43 @@ def Main(): result = oPDFParseDictionary.Get('/Root') if result != None: savedRoot = result - elif options.yara == None: - if not options.search or options.search and object.Contains(options.search): + elif options.yara == None and options.generateembedded == 0: + if not options.search and not options.key and not options.reference or options.search and object.Contains(options.search): if oPDFParseDictionary == None: print('trailer %s' % FormatOutput(object.content, options.raw)) else: print('trailer') oPDFParseDictionary.PrettyPrint(' ') print('') + elif options.key: + if oPDFParseDictionary.parsed != None: + result = oPDFParseDictionary.GetNested(options.key) + if result != None: + print(result) + elif options.reference: + for key, value in oPDFParseDictionary.Retrieve(): + if value == [str(options.reference), '0', 'R']: + print('trailer') + oPDFParseDictionary.PrettyPrint(' ') elif object.type == PDF_ELEMENT_STARTXREF and selectStartXref: - if not options.generate and options.yara == None: + if not options.generate and options.yara == None and options.generateembedded == 0: print('startxref %d' % object.index) print('') elif object.type == PDF_ELEMENT_INDIRECT_OBJECT and selectIndirectObject: if options.search: if object.Contains(options.search): PrintObject(object, options) + elif options.key: + contentDictionary = object.ContainsStream() + if not contentDictionary: + contentDictionary = object.content[1:] + oPDFParseDictionary = cPDFParseDictionary(contentDictionary, options.nocanonicalizedoutput) + if oPDFParseDictionary.parsed != None: + result = oPDFParseDictionary.GetNested(options.key) + if result != None: + print(result) elif options.object: - if object.id == eval(options.object): + if MatchObjectID(object.id, options.object): PrintObject(object, options) elif options.reference: if object.References(options.reference): @@ -1134,17 +1718,27 @@ def Main(): elif options.hash: print('obj %d %d' % (object.id, object.version)) rawContent = FormatOutput(object.content, True) - print(' len: %d md5: %s' % (len(rawContent), hashlib.md5(rawContent).hexdigest())) + hashHexdigest, hashAlgo = CalculateChosenHash(rawContent.encode('latin')) + print(' len: %d %s: %s' % (len(rawContent), hashAlgo, hashHexdigest)) print('') elif options.searchstream: - if object.StreamContains(options.searchstream, not options.unfiltered, options.casesensitive, options.regex): + if object.StreamContains(options.searchstream, not options.unfiltered, options.casesensitive, options.regex, options.overridingfilters): PrintObject(object, options) elif options.yara != None: - results = object.StreamYARAMatch(rules, not options.unfiltered) + results = object.StreamYARAMatch(rules, decoders, options.decoderoptions, not options.unfiltered, options.overridingfilters) if results != None and results != []: for result in results: - print('YARA rule: %s (%s)' % (result.rule, result.namespace)) - PrintObject(object, options) + for yaraResult in result[1]: + print('YARA rule%s: %s (%s)' % (IFF(result[0] == '', '', ' (stream decoder: %s)' % result[0]), yaraResult.rule, yaraResult.namespace)) + if options.yarastrings: + for stringdata in yaraResult.strings: + print('%06x %s:' % (stringdata[0], stringdata[1])) + print(' %s' % binascii.hexlify(C2BIP3(stringdata[2]))) + print(' %s' % repr(stringdata[2])) + PrintObject(object, options) + elif options.generateembedded != 0: + if object.id == options.generateembedded: + PrintGenerateObject(object, options, 8) else: PrintObject(object, options) elif object.type == PDF_ELEMENT_MALFORMED: @@ -1166,18 +1760,33 @@ def Main(): print('Trailer: %s' % cntTrailer) print('StartXref: %s' % cntStartXref) print('Indirect object: %s' % cntIndirectObject) - names = dicObjectTypes.keys() - names.sort() - for key in names: + print('Indirect objects with a stream: %s' % ', '.join([str(id) for id in objectsWithStream])) + objectsUnreferenced = objectsAll - objectsReferenced + for key in sorted(dicObjectTypes.keys()): print(' %s %d: %s' % (key, len(dicObjectTypes[key]), ', '.join(map(lambda x: '%d' % x, dicObjectTypes[key])))) - - if options.generate: + if len(objectsUnreferenced) > 0: + print('Unreferenced indirect objects: %s' % ', '.join([' '.join(reference) for reference in sorted(objectsUnreferenced, key=lambda a: int(a[0]))])) + if '/ObjStm' in dicObjectTypes: + objectsUnreferencedMinusObjStm = set() + for unreferencedObject in objectsUnreferenced: + if not int(unreferencedObject[0]) in dicObjectTypes['/ObjStm']: + objectsUnreferencedMinusObjStm.add(unreferencedObject) + print('Unreferenced indirect objects without /ObjStm objects: %s' % ', '.join([' '.join(reference) for reference in sorted(objectsUnreferencedMinusObjStm, key=lambda a: int(a[0]))])) + if sum(map(len, dKeywords.values())) > 0: + print('Search keywords:') + for keyword in keywords: + if len(dKeywords[keyword]) > 0: + print(' %s %d: %s' % (keyword, len(dKeywords[keyword]), ', '.join(map(lambda x: '%d' % x, dKeywords[keyword])))) + + if options.jsonoutput: + print(oMyJSONOutput.GetJSON()) + + if options.generate or options.generateembedded != 0: print(" oPDF.xrefAndTrailer('%s')" % ' '.join(savedRoot)) print('') print("if __name__ == '__main__':") print(' Main()') - def TestPythonVersion(enforceMaximumVersion=False, enforceMinimumVersion=False): if sys.version_info[0:3] > __maximum_python_version__: if enforceMaximumVersion: diff --git a/sift/files/pdf-tools/pdfid.ini b/sift/files/pdf-tools/pdfid.ini new file mode 100644 index 00000000..8ae6c6f5 --- /dev/null +++ b/sift/files/pdf-tools/pdfid.ini @@ -0,0 +1,2 @@ +[keywords] +/URI diff --git a/sift/files/pdf-tools/pdfid.py b/sift/files/pdf-tools/pdfid.py index 95c5b766..22ac85a9 100644 --- a/sift/files/pdf-tools/pdfid.py +++ b/sift/files/pdf-tools/pdfid.py @@ -1,9 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 __description__ = 'Tool to test a PDF file' __author__ = 'Didier Stevens' -__version__ = '0.2.1' -__date__ = '2014/10/18' +__version__ = '0.2.9' +__date__ = '2024/10/26' """ @@ -45,6 +45,19 @@ 2014/09/30: added CSV header 2014/10/16: V0.2.1: added output when plugin & file not pdf 2014/10/18: some fixes for Python 3 + 2015/08/12: V0.2.2: added option pluginoptions + 2015/08/13: added plugin Instructions method + 2016/04/12: added option literal + 2017/10/29: added pdfid.ini support + 2017/11/05: V0.2.3: added option -n + 2018/01/03: V0.2.4: bugfix entropy calculation for PDFs without streams; sample 28cb208d976466b295ee879d2d233c8a https://twitter.com/DubinRan/status/947783629123416069 + 2018/01/15: bugfix ConfigParser privately reported + 2018/01/29: bugfix oPDFEOF.cntCharsAfterLastEOF when no %%EOF + 2018/07/05: V0.2.5 introduced cExpandFilenameArguments; renamed option literal to literalfilenames + 2019/09/30: V0.2.6 color bugfix, thanks to Leo + 2019/11/05: V0.2.7 fixed plugin path when compiled with pyinstaller + 2020/11/21: V0.2.8 added data argument to PDFiD function + 2024/10/26: V0.2.9 added pyzipper support Todo: - update XML example (entropy, EOF) @@ -61,15 +74,25 @@ import os.path import sys import json -import zipfile import collections import glob +import fnmatch +if sys.version_info[0] >= 3: + import urllib.request as urllib23 +else: + import urllib2 as urllib23 +if sys.version_info[0] >= 3: + import configparser as ConfigParser +else: + import ConfigParser +if sys.version_info[0] >= 3: + from io import BytesIO as DataIO +else: + from cStringIO import StringIO as DataIO try: - import urllib2 - urllib23 = urllib2 -except: - import urllib.request - urllib23 = urllib.request + import pyzipper as zipfile +except ImportError: + import zipfile #Convert 2 Bytes If Python 3 def C2BIP3(string): @@ -78,10 +101,18 @@ def C2BIP3(string): else: return string +def CreateZipFileObject(arg1, arg2): + if 'AESZipFile' in dir(zipfile): + return zipfile.AESZipFile(arg1, arg2) + else: + return zipfile.ZipFile(arg1, arg2) + class cBinaryFile: - def __init__(self, file): + def __init__(self, file, data=None): self.file = file - if file == '': + if data != None: + self.infile = DataIO(data) + elif file == '': self.infile = sys.stdin elif file.lower().startswith('http://') or file.lower().startswith('https://'): try: @@ -95,7 +126,7 @@ def __init__(self, file): sys.exit() elif file.lower().endswith('.zip'): try: - self.zipfile = zipfile.ZipFile(file, 'r') + self.zipfile = CreateZipFileObject(file, 'r') self.infile = self.zipfile.open(self.zipfile.infolist()[0], 'r', C2BIP3('infected')) except: print('Error opening file %s' % file) @@ -230,11 +261,14 @@ def removeInsideStream(self, byte): self.streamBucket[byte] -= 1 def calc(self): - self.nonStreamBucket = map(operator.sub, self.allBucket, self.streamBucket) + self.nonStreamBucket = list(map(operator.sub, self.allBucket, self.streamBucket)) allCount = sum(self.allBucket) streamCount = sum(self.streamBucket) nonStreamCount = sum(self.nonStreamBucket) - return (allCount, sum(map(lambda x: fEntropy(x, allCount), self.allBucket)), streamCount, sum(map(lambda x: fEntropy(x, streamCount), self.streamBucket)), nonStreamCount, sum(map(lambda x: fEntropy(x, nonStreamCount), self.nonStreamBucket))) + if streamCount == 0: + return (allCount, sum(map(lambda x: fEntropy(x, allCount), self.allBucket)), streamCount, None, nonStreamCount, sum(map(lambda x: fEntropy(x, nonStreamCount), self.nonStreamBucket))) + else: + return (allCount, sum(map(lambda x: fEntropy(x, allCount), self.allBucket)), streamCount, sum(map(lambda x: fEntropy(x, streamCount), self.streamBucket)), nonStreamCount, sum(map(lambda x: fEntropy(x, nonStreamCount), self.nonStreamBucket))) class cPDFEOF: def __init__(self): @@ -338,7 +372,7 @@ def __init__(self): self.count = 0 def Check(self, lastName, word): - if (lastName == '/Colors' and word.isdigit() and int(word) > 2^24): # decided to alert when the number of colors is expressed with more than 3 bytes + if (lastName == '/Colors' and word.isdigit() and int(word) > 2**24): # decided to alert when the number of colors is expressed with more than 3 bytes self.count += 1 def XMLAddAttribute(xmlDoc, name, value=None): @@ -346,8 +380,26 @@ def XMLAddAttribute(xmlDoc, name, value=None): xmlDoc.documentElement.setAttributeNode(att) if value != None: att.nodeValue = value + return att + +def GetScriptPath(): + if getattr(sys, 'frozen', False): + return os.path.dirname(sys.executable) + else: + return os.path.dirname(sys.argv[0]) -def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False): +def ParseINIFile(): + oConfigParser = ConfigParser.ConfigParser(allow_no_value=True) + oConfigParser.optionxform = str + oConfigParser.read(os.path.join(GetScriptPath(), 'pdfid.ini')) + keywords = [] + if oConfigParser.has_section('keywords'): + for key, value in oConfigParser.items('keywords'): + if not key in keywords: + keywords.append(key) + return keywords + +def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False, data=None): """Example of XML output: @@ -377,7 +429,7 @@ def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False): hexcode = False lastName = '' insideStream = False - keywords = ('obj', + keywords = ['obj', 'endobj', 'stream', 'endstream', @@ -397,9 +449,12 @@ def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False): '/Launch', '/EmbeddedFile', '/XFA', - ) + ] words = {} dates = [] + for extrakeyword in ParseINIFile(): + if not extrakeyword in keywords: + keywords.append(extrakeyword) for keyword in keywords: words[keyword] = [0, 0] slash = '' @@ -416,7 +471,7 @@ def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False): try: attIsPDF = xmlDoc.createAttribute('IsPDF') xmlDoc.documentElement.setAttributeNode(attIsPDF) - oBinaryFile = cBinaryFile(file) + oBinaryFile = cBinaryFile(file, data) if extraData: oPDFDate = cPDFDate() oEntropy = cEntropy() @@ -532,7 +587,10 @@ def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False): (countAll, entropyAll , countStream, entropyStream, countNonStream, entropyNonStream) = oEntropy.calc() attEntropyAll.nodeValue = '%f' % entropyAll attCountAll.nodeValue = '%d' % countAll - attEntropyStream.nodeValue = '%f' % entropyStream + if entropyStream == None: + attEntropyStream.nodeValue = 'N/A ' + else: + attEntropyStream.nodeValue = '%f' % entropyStream attCountStream.nodeValue = '%d' % countStream attEntropyNonStream.nodeValue = '%f' % entropyNonStream attCountNonStream.nodeValue = '%d' % countNonStream @@ -549,7 +607,10 @@ def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False): xmlDoc.documentElement.setAttributeNode(attCountCharsAfterLastEOF) if oPDFEOF != None: attCountEOF.nodeValue = '%d' % oPDFEOF.cntEOFs - attCountCharsAfterLastEOF.nodeValue = '%d' % oPDFEOF.cntCharsAfterLastEOF + if oPDFEOF.cntEOFs > 0: + attCountCharsAfterLastEOF.nodeValue = '%d' % oPDFEOF.cntCharsAfterLastEOF + else: + attCountCharsAfterLastEOF.nodeValue = '' else: attCountEOF.nodeValue = '' attCountCharsAfterLastEOF.nodeValue = '' @@ -608,7 +669,7 @@ def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False): eleDate.setAttributeNode(att) return xmlDoc -def PDFiD2String(xmlDoc, force): +def PDFiD2String(xmlDoc, nozero, force): result = 'PDFiD %s %s\n' % (xmlDoc.documentElement.getAttribute('Version'), xmlDoc.documentElement.getAttribute('Filename')) if xmlDoc.documentElement.getAttribute('ErrorOccured') == 'True': return result + '***Error occured***\n%s\n' % xmlDoc.documentElement.getAttribute('ErrorMessage') @@ -616,10 +677,11 @@ def PDFiD2String(xmlDoc, force): return result + ' Not a PDF document\n' result += ' PDF Header: %s\n' % xmlDoc.documentElement.getAttribute('Header') for node in xmlDoc.documentElement.getElementsByTagName('Keywords')[0].childNodes: - result += ' %-16s %7d' % (node.getAttribute('Name'), int(node.getAttribute('Count'))) - if int(node.getAttribute('HexcodeCount')) > 0: - result += '(%d)' % int(node.getAttribute('HexcodeCount')) - result += '\n' + if not nozero or nozero and int(node.getAttribute('Count')) > 0: + result += ' %-16s %7d' % (node.getAttribute('Name'), int(node.getAttribute('Count'))) + if int(node.getAttribute('HexcodeCount')) > 0: + result += '(%d)' % int(node.getAttribute('HexcodeCount')) + result += '\n' if xmlDoc.documentElement.getAttribute('CountEOF') != '': result += ' %-16s %7d\n' % ('%%EOF', int(xmlDoc.documentElement.getAttribute('CountEOF'))) if xmlDoc.documentElement.getAttribute('CountCharsAfterLastEOF') != '': @@ -703,7 +765,7 @@ def MakeCSVLine(fields, separator=';', quote='"'): def ProcessFile(filename, options, plugins): xmlDoc = PDFiD(filename, options.all, options.extra, options.disarm, options.force) if plugins == [] and options.select == '': - Print(PDFiD2String(xmlDoc, options.force), options) + Print(PDFiD2String(xmlDoc, options.nozero, options.force), options) return oPDFiD = cPDFiD(xmlDoc, options.force) @@ -721,12 +783,12 @@ def ProcessFile(filename, options, plugins): if options.csv: Print(filename, options) else: - Print(PDFiD2String(xmlDoc, options.force), options) + Print(PDFiD2String(xmlDoc, options.nozero, options.force), options) else: for cPlugin in plugins: if not cPlugin.onlyValidPDF or not oPDFiD.errorOccured and oPDFiD.isPDF: try: - oPlugin = cPlugin(oPDFiD) + oPlugin = cPlugin(oPDFiD, options.pluginoptions) except Exception as e: Print('Error instantiating plugin: %s' % cPlugin.name, options) if options.verbose: @@ -746,8 +808,12 @@ def ProcessFile(filename, options, plugins): Print(MakeCSVLine((('%s', filename), ('%s', cPlugin.name), ('%.02f', score))), options) else: if score >= options.minimumscore: - Print(PDFiD2String(xmlDoc, options.force), options) - Print('%s score: %.02f' % (cPlugin.name, score), options) + Print(PDFiD2String(xmlDoc, options.nozero, options.force), options) + Print('%s score: %.02f' % (cPlugin.name, score), options) + try: + Print('%s instructions: %s' % (cPlugin.name, oPlugin.Instructions(score)), options) + except AttributeError: + pass else: if options.csv: if oPDFiD.errorOccured: @@ -755,7 +821,7 @@ def ProcessFile(filename, options, plugins): if not oPDFiD.isPDF: Print(MakeCSVLine((('%s', filename), ('%s', cPlugin.name), ('%s', 'Not a PDF document'))), options) else: - Print(PDFiD2String(xmlDoc, options.force), options) + Print(PDFiD2String(xmlDoc, options.nozero, options.force), options) def Scan(directory, options, plugins): @@ -842,8 +908,93 @@ def AddPlugin(cClass): plugins.append(cClass) -def ExpandFilenameArguments(filenames): - return list(collections.OrderedDict.fromkeys(sum(map(glob.glob, sum(map(ProcessAt, filenames), [])), []))) +class cExpandFilenameArguments(): + def __init__(self, filenames, literalfilenames=False, recursedir=False, checkfilenames=False, expressionprefix=None): + self.containsUnixShellStyleWildcards = False + self.warning = False + self.message = '' + self.filenameexpressions = [] + self.expressionprefix = expressionprefix + self.literalfilenames = literalfilenames + + expression = '' + if len(filenames) == 0: + self.filenameexpressions = [['', '']] + elif literalfilenames: + self.filenameexpressions = [[filename, ''] for filename in filenames] + elif recursedir: + for dirwildcard in filenames: + if expressionprefix != None and dirwildcard.startswith(expressionprefix): + expression = dirwildcard[len(expressionprefix):] + else: + if dirwildcard.startswith('@'): + for filename in ProcessAt(dirwildcard): + self.filenameexpressions.append([filename, expression]) + elif os.path.isfile(dirwildcard): + self.filenameexpressions.append([dirwildcard, expression]) + else: + if os.path.isdir(dirwildcard): + dirname = dirwildcard + basename = '*' + else: + dirname, basename = os.path.split(dirwildcard) + if dirname == '': + dirname = '.' + for path, dirs, files in os.walk(dirname): + for filename in fnmatch.filter(files, basename): + self.filenameexpressions.append([os.path.join(path, filename), expression]) + else: + for filename in list(collections.OrderedDict.fromkeys(sum(map(self.Glob, sum(map(ProcessAt, filenames), [])), []))): + if expressionprefix != None and filename.startswith(expressionprefix): + expression = filename[len(expressionprefix):] + else: + self.filenameexpressions.append([filename, expression]) + self.warning = self.containsUnixShellStyleWildcards and len(self.filenameexpressions) == 0 + if self.warning: + self.message = "Your filename argument(s) contain Unix shell-style wildcards, but no files were matched.\nCheck your wildcard patterns or use option literalfilenames if you don't want wildcard pattern matching." + return + if self.filenameexpressions == [] and expression != '': + self.filenameexpressions = [['', expression]] + if checkfilenames: + self.CheckIfFilesAreValid() + + def Glob(self, filename): + if not ('?' in filename or '*' in filename or ('[' in filename and ']' in filename)): + return [filename] + self.containsUnixShellStyleWildcards = True + return glob.glob(filename) + + def CheckIfFilesAreValid(self): + valid = [] + doesnotexist = [] + isnotafile = [] + for filename, expression in self.filenameexpressions: + hashfile = False + try: + hashfile = FilenameCheckHash(filename, self.literalfilenames)[0] == FCH_DATA + except: + pass + if filename == '' or hashfile: + valid.append([filename, expression]) + elif not os.path.exists(filename): + doesnotexist.append(filename) + elif not os.path.isfile(filename): + isnotafile.append(filename) + else: + valid.append([filename, expression]) + self.filenameexpressions = valid + if len(doesnotexist) > 0: + self.warning = True + self.message += 'The following files do not exist and will be skipped: ' + ' '.join(doesnotexist) + '\n' + if len(isnotafile) > 0: + self.warning = True + self.message += 'The following files are not regular files and will be skipped: ' + ' '.join(isnotafile) + '\n' + + def Filenames(self): + if self.expressionprefix == None: + return [filename for filename, expression in self.filenameexpressions] + else: + return self.filenameexpressions class cPluginParent(): onlyValidPDF = True @@ -851,7 +1002,7 @@ class cPluginParent(): def LoadPlugins(plugins, verbose): if plugins == '': return - scriptPath = os.path.dirname(sys.argv[0]) + scriptPath = GetScriptPath() for plugin in sum(map(ProcessAt, plugins.split(',')), []): try: if not plugin.lower().endswith('.py'): @@ -907,7 +1058,11 @@ def Main(): oParser.add_option('-m', '--minimumscore', type=float, default=0.0, help='minimum score for plugin results output') oParser.add_option('-v', '--verbose', action='store_true', default=False, help='verbose (will also raise catched exceptions)') oParser.add_option('-S', '--select', type=str, default='', help='selection expression') + oParser.add_option('-n', '--nozero', action='store_true', default=False, help='supress output for counts equal to zero') oParser.add_option('-o', '--output', type=str, default='', help='output to log file') + oParser.add_option('--pluginoptions', type=str, default='', help='options for the plugin') + oParser.add_option('-l', '--literalfilenames', action='store_true', default=False, help='take filenames literally, no wildcard matching') + oParser.add_option('--recursedir', action='store_true', default=False, help='Recurse directories (wildcards and here files (@...) allowed)') (options, args) = oParser.parse_args() if len(args) == 0: @@ -920,7 +1075,10 @@ def Main(): filenames = [''] else: try: - filenames = ExpandFilenameArguments(args) + oExpandFilenameArguments = cExpandFilenameArguments(args, options.literalfilenames, options.recursedir, False) + filenames = oExpandFilenameArguments.Filenames() + if oExpandFilenameArguments.warning: + print(oExpandFilenameArguments.message) except Exception as e: print(e) return diff --git a/sift/files/pdf-tools/pdftool.py b/sift/files/pdf-tools/pdftool.py new file mode 100644 index 00000000..8421f2e0 --- /dev/null +++ b/sift/files/pdf-tools/pdftool.py @@ -0,0 +1,1727 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +__description__ = 'Tool to process PDFs' +__author__ = 'Didier Stevens' +__version__ = '0.0.1' +__date__ = '2021/01/08' + +""" +Source code put in the public domain by Didier Stevens, no Copyright +https://DidierStevens.com +Use at your own risk + +History: + 2020/08/22: start + 2020/08/22: continue + 2020/10/21: Python 3 fix in cBinaryFile + 2021/01/06: man page + 2021/01/07: DataIO + 2021/01/08: sync with template; man + +Todo: + +""" + +import optparse +import sys +import os +import zipfile +import binascii +import random +import gzip +import collections +import glob +import textwrap +import re +import struct +import string +import math +import fnmatch +import json +import time +import hashlib +import csv +if sys.version_info[0] >= 3: + from io import BytesIO as DataIO +else: + from cStringIO import StringIO as DataIO +if sys.version_info[0] >= 3: + from io import StringIO +else: + from cStringIO import StringIO + +def PrintManual(): + manual = r''' +Manual: + +pdftool.py is a tool to process PDFs. + +For the moment, it has one command: iu (incremental updates). + +Command iu can be used to analyze PDFs with incremental updates. + +When a PDF without incremental updates is analyzed by this tool, one or two "versions" will be listed. +Each "version" is prefixed with a number (starting from 1) allowing selection of an update for further processing. + +For a non-linearized PDF without incremental updates, only one "version" will be listed: + +pdftool.py iu hello.pdf + +File: hello.pdf +1: objects= 6 length= 859 difference= 859 MD5= 8433a21e2ab8d9cb6655dec30aea1c2a + +For a linearized PDF without incremental updates, two "versions" will be listed: + +pdftool.py iu hello-linearized.pdf + +File: hello-linearized.pdf +1: objects= 1 (Linearized) length= 500 difference= 500 MD5= f8a1feff9f47b0bfacaacad1303ab1e2 +2: objects= 7 length= 1367 difference= 867 MD5= e0331685b60664a10f405325473541c5 + +Notice "(Linearized)" for the first "version": this indicates that this is a linearized PDF. + +For a non-linearized PDF with incremental updates, at least two "versions" will be listed: + +pdftool.py iu pdf-puzzle.pdf + +File: pdf-puzzle.pdf +1: objects= 6 length= 933 difference= 933 MD5= 489a96a2621f9abe13156b22afca5fcf +2: objects= 1 length= 1243 difference= 310 MD5= dabe6cb9c5fe3d213a08fb75f8d33ac4 + + +And for a linearized PDF with incremental updates, at least three "versions" will be listed: + +pdftool.py iu data.pdf.zip + +File: data.pdf.zip (extracted) +1: objects= 1 (Linearized) length= 672 difference= 672 MD5= 9c818e7a9ecd864e3bae97f5c3fa0816 +2: objects= 35 length= 8535 difference= 7863 MD5= e0e44d544ac922c3d64c7408944bf60b +3: objects= 6 length= 14241 difference= 5706 MD5= 4943ccbe72c8ba2e81e5bc030b730f69 +4: objects= 5 length= 19639 difference= 5398 MD5= be45f57e2056745d6da0569b5f154ac2 +5: objects= 5 length= 25042 difference= 5403 MD5= 69953f5809e74cad3f3e63323f990cae +6: objects= 6 length= 30658 difference= 5616 MD5= 1a8e5242f21727959683fa8cc7aa94ad + +"Versions" can be selected with option -s. + +For example: + +pdftool.py -s 1 iu pdf-puzzle.pdf + +00000000: 25 50 44 46 2D 31 2E 31 0D 0A 0D 0A 31 20 30 20 %PDF-1.1....1 0 +00000010: 6F 62 6A 0D 0A 3C 3C 0D 0A 20 2F 54 79 70 65 20 obj..<<.. /Type +00000020: 2F 43 61 74 61 6C 6F 67 0D 0A 20 2F 4F 75 74 6C /Catalog.. /Outl +00000030: 69 6E 65 73 20 32 20 30 20 52 0D 0A 20 2F 50 61 ines 2 0 R.. /Pa +00000040: 67 65 73 20 33 20 30 20 52 0D 0A 3E 3E 0D 0A 65 ges 3 0 R..>>..e +... +00000360: 30 20 6E 0D 0A 74 72 61 69 6C 65 72 0D 0A 3C 3C 0 n..trailer..<< +00000370: 0D 0A 20 2F 53 69 7A 65 20 37 0D 0A 20 2F 52 6F .. /Size 7.. /Ro +00000380: 6F 74 20 31 20 30 20 52 0D 0A 3E 3E 0D 0A 73 74 ot 1 0 R..>>..st +00000390: 61 72 74 78 72 65 66 0D 0A 37 31 38 0D 0A 25 25 artxref..718..%% +000003A0: 45 4F 46 0D 0A EOF.. + +By default, a selected version is dumped as an hexadecimal & ascii dump. +This output format can be changed to pure hexadecimal (-x), binary (-d) or run-length compressed hexadecimal & ascii dump (-A). +The default hexadecimal & ascii dump is -a. + +When a version is selected, the complete PDF for the version is produced. If the desired output is the difference between the selected version and the previous version, suffix d (delta) must be used. +In this example, -s 2d selects the delta between version 1 and 2: + +pdftool.py -s 2d iu pdf-puzzle.pdf + +00000000: 0D 0A 35 20 30 20 6F 62 6A 0D 0A 3C 3C 0D 0A 20 ..5 0 obj..<<.. +00000010: 2F 4C 65 6E 67 74 68 20 38 39 0D 0A 20 2F 46 69 /Length 89.. /Fi +00000020: 6C 74 65 72 20 2F 41 53 43 49 49 38 35 44 65 63 lter /ASCII85Dec +00000030: 6F 64 65 0D 0A 3E 3E 0D 0A 73 74 72 65 61 6D 0D ode..>>..stream. +00000040: 0A 36 3C 23 27 5C 37 50 51 23 40 31 61 23 62 30 .6<#'\7PQ#@1a#b0 +00000050: 2B 3E 47 51 28 2B 3F 28 75 2E 2B 42 32 6B 6F 2D +>GQ(+?(u.+B2ko- +00000060: 72 61 6B 6B 2B 45 31 62 31 46 29 59 66 35 40 3C rakk+E1b1F)Yf5@< +00000070: 36 21 26 42 6C 62 44 21 3D 42 4A 5B 2D 3D 42 4A 6!&BlbD!=BJ[-=BJ +00000080: 5B 2D 3D 42 4A 5B 2D 3D 42 4A 5B 2D 3D 42 49 21 [-=BJ[-=BJ[-=BI! +00000090: 70 3C 2C 2A 4F 45 3B 75 7E 3E 0D 0A 65 6E 64 73 p<,*OE;u~>..ends +000000A0: 74 72 65 61 6D 0D 0A 65 6E 64 6F 62 6A 0D 0A 0D tream..endobj... +000000B0: 0A 78 72 65 66 0D 0A 30 20 31 0D 0A 30 30 30 30 .xref..0 1..0000 +000000C0: 30 30 30 30 30 30 20 36 35 35 33 35 20 66 0D 0A 000000 65535 f.. +000000D0: 35 20 31 0D 0A 30 30 30 30 30 30 30 39 33 35 20 5 1..0000000935 +000000E0: 30 30 30 30 30 20 6E 0D 0A 74 72 61 69 6C 65 72 00000 n..trailer +000000F0: 0D 0A 3C 3C 0D 0A 20 2F 53 69 7A 65 20 37 0D 0A ..<<.. /Size 7.. +00000100: 20 2F 52 6F 6F 74 20 31 20 30 20 52 0D 0A 20 2F /Root 1 0 R.. / +00000110: 50 72 65 76 20 37 31 38 0D 0A 3E 3E 0D 0A 73 74 Prev 718..>>..st +00000120: 61 72 74 78 72 65 66 0D 0A 31 31 31 30 0D 0A 25 artxref..1110..% +00000130: 25 45 4F 46 0D 0A %EOF.. + + + +Output can also be directed to a file using option -o. + + +This tool is very versatile when it comes to handling files. This will be explained now. + +This tool reads files in binary mode. It can read files from disk, from standard input (stdin) and from "generated" files via the command line. +It can also partially read files (this is done with the cut operator). + +If no file arguments are provided to this tool, it will read data from standard input (stdin). This way, this tool can be used in a piped chain of commands, like this: + +oledump.py -s 4 -d sample.doc.vir | tool.py + +When one or more file arguments are provided to this tool, it will read the files and process the content. +How the files are read, depends on the type of file arguments that are provided. File arguments that start with character @ or # have special meaning, and will be explained later. + +If a file argument does not start with @ or #, it is considered to be a file on disk and the content will be read from disk. +If the file is not a compressed file, the binary content of the file is read from disk for processing. +Compressed files are solely recognized based on their extension: .zip and .gz. +If a file argument with extension .gz is provided, the tool will decompress the gzip file in memory and process the decompressed content. No checks are made to ensure that the file with extension .gz is an actual gzip compressed file. +If a file argument with extension .zip is provided and it contains a single file, the tool will extract the file from the ZIP file in memory and process the decompressed content. No checks are made to ensure that the file with extension .zip is an actual ZIP compressed file. +Password protected ZIP files can be processed too. The tool uses password 'infected' (without quotes) as default password. A different password can be provided using option --password. + +Example: + +tool.py sample.zip + +To prevent the tool from decompressing .zip or .gz files, but to process the compressed file itself, use option --noextraction. + +File arguments that start with character @ ("here files"), are read as text files that contain file arguments (one per line) to be processed. +For example, we take a text file with filename list.txt and following content: + +sample-1.bin +sample-5.bin +sample-7.bin + +When using this file (list.txt) in the following command: + +tool.py @list.txt + +the tool will process the following files: sample-1.bin, sample-5.bin and sample-7.bin. +A single @ character as filename is a here file read from stdin. + +Wildcards are supported too. The classic *, ? and [] wildcard characters are supported. For example, use the following command to process all .exe and .dll files in the Windows directory: + +tool.py C:\Windows\*.exe C:\Windows\*.dll + +To prevent the tool from processing file arguments with wildcard characters or special initial characters (@ and #) differently, but to process them as normal files, use option --literalfilenames. + +The content of folders can be processed too: use option --recursedir and provide folder names as argument. Wildcards and here files (for folder names) can be used too. + +File arguments that start with character # have special meaning. These are not processed as actual files on disk (except when option --literalfilenames is used), but as file arguments that specify how to "generate" the file content. + +File arguments that start with #, #h#, #b# or #e# are used to "generate" the file content. +Arguments that start with #c# are not file arguments, but cut operators (explained later). +Arguments that start with #f# are not file arguments, but flags (explained later). + +Generating the file content with a # file argument means that the file content is not read from disk, but generated in memory based on the characteristics provided via the file argument. + +When a file argument starts with # (and not with #h#, #b#, #e# or #c#), all characters that follow the # character specify the content of the generated file. +For example, file argument #ABCDE specifies a file containing exactly 5 bytes: ASCII characters A, B, C, D and E. +Thus the following command: + +tool.py #ABCDE + +will make the tool process data with binary content ABCDE. #ABCDE is not an actual file written on disk, but it is a notational convention to provide data via the command line. + +Since this notation can not be used to specify all possible byte values, hexadecimal encoding (#h#) and BASE64 encoding (#b#) notation is supported too. +For example, #h#4142434445 is an hexadecimal notation that generates data ABCDE. Hexadecimal notation allows the generation of non-printable characters for example, like NULL bytes: #h#00 +File argument #b#QUJDREU= is another example, this time BASE64 notation, that generates data ABCDE. + +File arguments that start with #e# are a notational convention to use expressions to generate data. An expression is a single function/string or the concatenation of several functions/strings (using character + as concatenation operator). +Strings can be characters enclosed by single quotes ('example') or hexadecimal strings prefixed by 0x (0xBEEF). +4 functions are available: random, loremipsum, repeat and chr. + +Function random takes exactly one argument: an integer (with value 1 or more). Integers can be specified using decimal notation or hexadecimal notation (prefix 0x). +The random function generates a sequence of bytes with a random value (between 0 and 255), the argument specifies how many bytes need to be generated. Remark that the random number generator that is used is just the Python random number generator, not a cryptographic random number generator. + +Example: + +tool.py #e#random(100) + +will make the tool process data consisting of a sequence of 100 random bytes. + +Function loremipsum takes exactly one argument: an integer (with value 1 or more). +The loremipsum function generates "lorem ipsum" text (fake latin), the argument specifies the number of sentences to generate. + +Example: #e#loremipsum(2) generates this text: +Ipsum commodo proin pulvinar hac vel nunc dignissim neque eget odio erat magna lorem urna cursus fusce facilisis porttitor congue eleifend taciti. Turpis duis suscipit facilisi tristique dictum praesent natoque sem mi egestas venenatis per dui sit sodales est condimentum habitasse ipsum phasellus non bibendum hendrerit. + +Function chr takes one argument or two arguments. +chr with one argument takes an integer between 0 and 255, and generates a single byte with the value specified by the integer. +chr with two arguments takes two integers between 0 and 255, and generates a byte sequence with the values specified by the integers. +For example #e#chr(0x41,0x45) generates data ABCDE. + +Function repeat takes two arguments: an integer (with value 1 or more) and a byte sequence. This byte sequence can be a quoted string of characters (single quotes), like 'ABCDE' or an hexadecimal string prefixed with 0x, like 0x4142434445. +The repeat function will create a sequence of bytes consisting of the provided byte sequence (the second argument) repeated as many times as specified by the first argument. +For example, #e#repeat(3, 'AB') generates byte sequence ABABAB. + +When more than one function needs to be used, the byte sequences generated by the functions can be concatenated with the + operator. +For example, #e#repeat(10,0xFF)+random(100) will generate a byte sequence of 10 FF bytes followed by 100 random bytes. + +File arguments that start with #p# are a notational convention to pack a Python expression to generate data (using Python module struct). +The string after #p# must contain 2 expressions separated by a # character, like #p#I#123456. +The first expression (I in this example) is the format string for the Python struct.pack function, and the second expression (123456 in this example) is a Python expression that needs to be packed by struct.pack. +In this example, format string I represents an unsigned, 32-bit, little-endian integer, and thus #p#I#123456 generates byte sequence 40E20100 (hexadecimal). + +The cut argument (or cut operator) allows for the partial selection of the content of a file. This argument starts with #c# followed by a "cut-expression". Use this expression to "cut out" part of the content. +The cut-argument must be put in front of a file argument, like in this example: + +tool.py #c#0:100l data.bin + +With these arguments, tool.py will only process the first 100 bytes (0:100l) of file data.bin. + +A cut argument is applied to all file arguments that follow it. Example: + +tool.py #c#0:100l data-1.bin data-2.bin + +With these arguments, tool.py will only process the first 100 bytes (0:100l) of file data-1.bin and the first 100 bytes file data-2.bin. + +More than one cut argument can be used, like in this example: + +tool.py #c#0:100l data-1.bin #c#0:200l data-2.bin + +With these arguments, tool.py will only process the first 100 bytes (0:100l) of file data-1.bin and the first 200 bytes (0:200l) of file data-2.bin. + +A cut-expression is composed of 2 terms separated by a colon (:), like this: +termA:termB +termA and termB can be: +- nothing (an empty string) +- a positive decimal number; example: 10 +- an hexadecimal number (to be preceded by 0x); example: 0x10 +- a case sensitive ASCII string to search for (surrounded by square brackets and single quotes); example: ['MZ'] +- a case sensitive UNICODE string to search for (surrounded by square brackets and single quotes prefixed with u); example: [u'User'] +- an hexadecimal string to search for (surrounded by square brackets); example: [d0cf11e0] +If termA is nothing, then the cut section of bytes starts with the byte at position 0. +If termA is a number, then the cut section of bytes starts with the byte at the position given by the number (first byte has index 0). +If termA is a string to search for, then the cut section of bytes starts with the byte at the position where the string is first found. If the string is not found, the cut is empty (0 bytes). +If termB is nothing, then the cut section of bytes ends with the last byte. +If termB is a number, then the cut section of bytes ends with the byte at the position given by the number (first byte has index 0). +When termB is a number, it can have suffix letter l. This indicates that the number is a length (number of bytes), and not a position. +termB can also be a negative number (decimal or hexademical): in that case the position is counted from the end of the file. For example, :-5 selects the complete file except the last 5 bytes. +If termB is a string to search for, then the cut section of bytes ends with the last byte at the position where the string is first found. If the string is not found, the cut is empty (0 bytes). +No checks are made to assure that the position specified by termA is lower than the position specified by termB. This is left up to the user. +Search string expressions (ASCII, UNICODE and hexadecimal) can be followed by an instance (a number equal to 1 or greater) to indicate which instance needs to be taken. For example, ['ABC']2 will search for the second instance of string 'ABC'. If this instance is not found, then nothing is selected. +Search string expressions (ASCII, UNICODE and hexadecimal) can be followed by an offset (+ or - a number) to add (or substract) an offset to the found instance. This number can be a decimal or hexadecimal (prefix 0x) value. For example, ['ABC']+3 will search for the first instance of string 'ABC' and then select the bytes after ABC (+ 3). +Finally, search string expressions (ASCII, UNICODE and hexadecimal) can be followed by an instance and an offset. +Examples: +This cut-expression can be used to dump the first 256 bytes of a PE file located inside the file content: ['MZ']:0x100l +This cut-expression can be used to dump the OLE file located inside the file content: [d0cf11e0]: + +A flag argument starts with #f# and is passed on for all files that are provided after the flag argument. It can be used to change the behavior of the tool for certain files. +Example: + +tool.py data-1.bin #f#-l data-2.bin + +data-2.bin will be processed differently (using flag option -l) than file data-1.bin. + +With option --jsoninput, the tool will parse the output produced by another tool using option --jsonoutput. +Example: +zipdump.py --jsonoutput Book1.xlsm | file-magic.py --jsoninput +[Content_Types].xml XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +_rels/.rels XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +xl/_rels/workbook.xml.rels XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +xl/workbook.xml XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +xl/drawings/drawing1.xml XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +xl/worksheets/_rels/sheet1.xml.rels XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +xl/theme/theme1.xml XML 1.0 document, UTF-8 Unicode text, with very long lines, with CRLF line terminators +xl/styles.xml XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +xl/worksheets/sheet1.xml XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +xl/vbaProject.bin Composite Document File V2 Document, Cannot read section info +xl/drawings/vmlDrawing1.vml ASCII text, with CRLF line terminators +docProps/app.xml XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators +xl/ctrlProps/ctrlProp1.xml XML 1.0 document, ASCII text, with CRLF line terminators +docProps/core.xml XML 1.0 document, ASCII text, with very long lines, with CRLF line terminators + +In this example, zipdump is used to produce JSON data with the content of each file contained inside file Book1.xlsm (a ZIP container), which is then consumed by file-magic.py to identify (libmagic) the type of each file. + +With option --ignoreprocessingerrors, the tool will continue processing the next file when an error occurs while processing the current file. Files that can not be opened will always be skipped to move to the next file. + +Option --logfile direct the tool to create a logfile, and option --logcomment can be used to add a comment to the log file. The log file will contain metadata and a list of processed files, it does not contain processing results. +It is best to use this option when option --ignoreprocessingerrors is used, to have a record of file processing errors. + +The lines are written to standard output, except when option -o is used. When option -o is used, the lines are written to the filename specified by option -o. +Filenames used with option -o starting with # have special meaning. +#c#example.txt will write output both to the console (stdout) and file example.txt. +#g# will write output to a file with a filename generated by the tool like this: toolname-date-time.txt. +#g#KEYWORD will write output to a file with a filename generated by the tool like this: toolname-KEYWORD-date-time.txt. +Use #p#filename to display execution progress. +To process several files while creating seperate output files for each input file, use -o #s#%f%.result *. +This will create output files with the name of the inputfile and extension .result. +There are several variables available when creating separate output files: + %f% is the full filename (with directory if present) + %b% is the base name: the filename without directory + %d% is the directory + %r% is the root: the filename without extension + %ru% is the root made unique by appending a counter (if necessary) + %e% is the extension +#h# is like the head command: only the first 10 lines will be outputed. +#t# is like the tail command: only the last 10 lines will be outputed. +Most options can be combined, like #ps# for example. +#l# is used for literal filenames: if the output filename has to start with # (#example.txt for example), use filename #l##example.txt for example. + +''' + for line in manual.split('\n'): + print(textwrap.fill(line, 79)) + +DEFAULT_SEPARATOR = ',' +QUOTE = '"' + +def PrintError(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +#Convert 2 Bytes If Python 3 +def C2BIP3(string): + if sys.version_info[0] > 2: + return bytes([ord(x) for x in string]) + else: + return string + +#Convert 2 Integer If Python 2 +def C2IIP2(data): + if sys.version_info[0] > 2: + return data + else: + return ord(data) + +# CIC: Call If Callable +def CIC(expression): + if callable(expression): + return expression() + else: + return expression + +# IFF: IF Function +def IFF(expression, valueTrue, valueFalse): + if expression: + return CIC(valueTrue) + else: + return CIC(valueFalse) + +#-BEGINCODE cBinaryFile------------------------------------------------------------------------------ +#import random +#import binascii +#import zipfile +#import gzip +#import sys +#if sys.version_info[0] >= 3: +# from io import BytesIO as DataIO +#else: +# from cStringIO import StringIO as DataIO + +def LoremIpsumSentence(minimum, maximum): + words = ['lorem', 'ipsum', 'dolor', 'sit', 'amet', 'consectetur', 'adipiscing', 'elit', 'etiam', 'tortor', 'metus', 'cursus', 'sed', 'sollicitudin', 'ac', 'sagittis', 'eget', 'massa', 'praesent', 'sem', 'fermentum', 'dignissim', 'in', 'vel', 'augue', 'scelerisque', 'auctor', 'libero', 'nam', 'a', 'gravida', 'odio', 'duis', 'vestibulum', 'vulputate', 'quam', 'nec', 'cras', 'nibh', 'feugiat', 'ut', 'vitae', 'ornare', 'justo', 'orci', 'varius', 'natoque', 'penatibus', 'et', 'magnis', 'dis', 'parturient', 'montes', 'nascetur', 'ridiculus', 'mus', 'curabitur', 'nisl', 'egestas', 'urna', 'iaculis', 'lectus', 'maecenas', 'ultrices', 'velit', 'eu', 'porta', 'hac', 'habitasse', 'platea', 'dictumst', 'integer', 'id', 'commodo', 'mauris', 'interdum', 'malesuada', 'fames', 'ante', 'primis', 'faucibus', 'accumsan', 'pharetra', 'aliquam', 'nunc', 'at', 'est', 'non', 'leo', 'nulla', 'sodales', 'porttitor', 'facilisis', 'aenean', 'condimentum', 'rutrum', 'facilisi', 'tincidunt', 'laoreet', 'ultricies', 'neque', 'diam', 'euismod', 'consequat', 'tempor', 'elementum', 'lobortis', 'erat', 'ligula', 'risus', 'donec', 'phasellus', 'quisque', 'vivamus', 'pellentesque', 'tristique', 'venenatis', 'purus', 'mi', 'dictum', 'posuere', 'fringilla', 'quis', 'magna', 'pretium', 'felis', 'pulvinar', 'lacinia', 'proin', 'viverra', 'lacus', 'suscipit', 'aliquet', 'dui', 'molestie', 'dapibus', 'mollis', 'suspendisse', 'sapien', 'blandit', 'morbi', 'tellus', 'enim', 'maximus', 'semper', 'arcu', 'bibendum', 'convallis', 'hendrerit', 'imperdiet', 'finibus', 'fusce', 'congue', 'ullamcorper', 'placerat', 'nullam', 'eros', 'habitant', 'senectus', 'netus', 'turpis', 'luctus', 'volutpat', 'rhoncus', 'mattis', 'nisi', 'ex', 'tempus', 'eleifend', 'vehicula', 'class', 'aptent', 'taciti', 'sociosqu', 'ad', 'litora', 'torquent', 'per', 'conubia', 'nostra', 'inceptos', 'himenaeos'] + sample = random.sample(words, random.randint(minimum, maximum)) + sample[0] = sample[0].capitalize() + return ' '.join(sample) + '.' + +def LoremIpsum(sentences): + return ' '.join([LoremIpsumSentence(15, 30) for i in range(sentences)]) + +STATE_START = 0 +STATE_IDENTIFIER = 1 +STATE_STRING = 2 +STATE_SPECIAL_CHAR = 3 +STATE_ERROR = 4 + +FUNCTIONNAME_REPEAT = 'repeat' +FUNCTIONNAME_RANDOM = 'random' +FUNCTIONNAME_CHR = 'chr' +FUNCTIONNAME_LOREMIPSUM = 'loremipsum' + +def Tokenize(expression): + result = [] + token = '' + state = STATE_START + while expression != '': + char = expression[0] + expression = expression[1:] + if char == "'": + if state == STATE_START: + state = STATE_STRING + elif state == STATE_IDENTIFIER: + result.append([STATE_IDENTIFIER, token]) + state = STATE_STRING + token = '' + elif state == STATE_STRING: + result.append([STATE_STRING, token]) + state = STATE_START + token = '' + elif char >= '0' and char <= '9' or char.lower() >= 'a' and char.lower() <= 'z': + if state == STATE_START: + token = char + state = STATE_IDENTIFIER + else: + token += char + elif char == ' ': + if state == STATE_IDENTIFIER: + result.append([STATE_IDENTIFIER, token]) + token = '' + state = STATE_START + elif state == STATE_STRING: + token += char + else: + if state == STATE_IDENTIFIER: + result.append([STATE_IDENTIFIER, token]) + token = '' + state = STATE_START + result.append([STATE_SPECIAL_CHAR, char]) + elif state == STATE_STRING: + token += char + else: + result.append([STATE_SPECIAL_CHAR, char]) + token = '' + if state == STATE_IDENTIFIER: + result.append([state, token]) + elif state == STATE_STRING: + result = [[STATE_ERROR, 'Error: string not closed', token]] + return result + +def ParseFunction(tokens): + if len(tokens) == 0: + print('Parsing error') + return None, tokens + if tokens[0][0] == STATE_STRING or tokens[0][0] == STATE_IDENTIFIER and tokens[0][1].startswith('0x'): + return [[FUNCTIONNAME_REPEAT, [[STATE_IDENTIFIER, '1'], tokens[0]]], tokens[1:]] + if tokens[0][0] != STATE_IDENTIFIER: + print('Parsing error') + return None, tokens + function = tokens[0][1] + tokens = tokens[1:] + if len(tokens) == 0: + print('Parsing error') + return None, tokens + if tokens[0][0] != STATE_SPECIAL_CHAR or tokens[0][1] != '(': + print('Parsing error') + return None, tokens + tokens = tokens[1:] + if len(tokens) == 0: + print('Parsing error') + return None, tokens + arguments = [] + while True: + if tokens[0][0] != STATE_IDENTIFIER and tokens[0][0] != STATE_STRING: + print('Parsing error') + return None, tokens + arguments.append(tokens[0]) + tokens = tokens[1:] + if len(tokens) == 0: + print('Parsing error') + return None, tokens + if tokens[0][0] != STATE_SPECIAL_CHAR or (tokens[0][1] != ',' and tokens[0][1] != ')'): + print('Parsing error') + return None, tokens + if tokens[0][0] == STATE_SPECIAL_CHAR and tokens[0][1] == ')': + tokens = tokens[1:] + break + tokens = tokens[1:] + if len(tokens) == 0: + print('Parsing error') + return None, tokens + return [[function, arguments], tokens] + +def Parse(expression): + tokens = Tokenize(expression) + if len(tokens) == 0: + print('Parsing error') + return None + if tokens[0][0] == STATE_ERROR: + print(tokens[0][1]) + print(tokens[0][2]) + print(expression) + return None + functioncalls = [] + while True: + functioncall, tokens = ParseFunction(tokens) + if functioncall == None: + return None + functioncalls.append(functioncall) + if len(tokens) == 0: + return functioncalls + if tokens[0][0] != STATE_SPECIAL_CHAR or tokens[0][1] != '+': + print('Parsing error') + return None + tokens = tokens[1:] + +def InterpretInteger(token): + if token[0] != STATE_IDENTIFIER: + return None + try: + return int(token[1]) + except: + return None + +def Hex2Bytes(hexadecimal): + if len(hexadecimal) % 2 == 1: + hexadecimal = '0' + hexadecimal + try: + return binascii.a2b_hex(hexadecimal) + except: + return None + +def InterpretHexInteger(token): + if token[0] != STATE_IDENTIFIER: + return None + if not token[1].startswith('0x'): + return None + bytes = Hex2Bytes(token[1][2:]) + if bytes == None: + return None + integer = 0 + for byte in bytes: + integer = integer * 0x100 + C2IIP2(byte) + return integer + +def InterpretNumber(token): + number = InterpretInteger(token) + if number == None: + return InterpretHexInteger(token) + else: + return number + +def InterpretBytes(token): + if token[0] == STATE_STRING: + return token[1] + if token[0] != STATE_IDENTIFIER: + return None + if not token[1].startswith('0x'): + return None + return Hex2Bytes(token[1][2:]) + +def CheckFunction(functionname, arguments, countarguments, maxcountarguments=None): + if maxcountarguments == None: + if countarguments == 0 and len(arguments) != 0: + print('Error: function %s takes no arguments, %d are given' % (functionname, len(arguments))) + return True + if countarguments == 1 and len(arguments) != 1: + print('Error: function %s takes 1 argument, %d are given' % (functionname, len(arguments))) + return True + if countarguments != len(arguments): + print('Error: function %s takes %d arguments, %d are given' % (functionname, countarguments, len(arguments))) + return True + else: + if len(arguments) < countarguments or len(arguments) > maxcountarguments: + print('Error: function %s takes between %d and %d arguments, %d are given' % (functionname, countarguments, maxcountarguments, len(arguments))) + return True + return False + +def CheckNumber(argument, minimum=None, maximum=None): + number = InterpretNumber(argument) + if number == None: + print('Error: argument should be a number: %s' % argument[1]) + return None + if minimum != None and number < minimum: + print('Error: argument should be minimum %d: %d' % (minimum, number)) + return None + if maximum != None and number > maximum: + print('Error: argument should be maximum %d: %d' % (maximum, number)) + return None + return number + +def Interpret(expression): + functioncalls = Parse(expression) + if functioncalls == None: + return None + decoded = '' + for functioncall in functioncalls: + functionname, arguments = functioncall + if functionname == FUNCTIONNAME_REPEAT: + if CheckFunction(functionname, arguments, 2): + return None + number = CheckNumber(arguments[0], minimum=1) + if number == None: + return None + bytes = InterpretBytes(arguments[1]) + if bytes == None: + print('Error: argument should be a byte sequence: %s' % arguments[1][1]) + return None + decoded += number * bytes + elif functionname == FUNCTIONNAME_RANDOM: + if CheckFunction(functionname, arguments, 1): + return None + number = CheckNumber(arguments[0], minimum=1) + if number == None: + return None + decoded += ''.join([chr(random.randint(0, 255)) for x in range(number)]) + elif functionname == FUNCTIONNAME_LOREMIPSUM: + if CheckFunction(functionname, arguments, 1): + return None + number = CheckNumber(arguments[0], minimum=1) + if number == None: + return None + decoded += LoremIpsum(number) + elif functionname == FUNCTIONNAME_CHR: + if CheckFunction(functionname, arguments, 1, 2): + return None + number = CheckNumber(arguments[0], minimum=0, maximum=255) + if number == None: + return None + if len(arguments) == 1: + decoded += chr(number) + else: + number2 = CheckNumber(arguments[1], minimum=0, maximum=255) + if number2 == None: + return None + if number < number2: + decoded += ''.join([chr(n) for n in range(number, number2 + 1)]) + else: + decoded += ''.join([chr(n) for n in range(number, number2 - 1, -1)]) + else: + print('Error: unknown function: %s' % functionname) + return None + return decoded + +def ParsePackExpression(data): + try: + packFormat, pythonExpression = data.split('#', 1) + data = struct.pack(packFormat, int(pythonExpression)) + return data + except: + return None + +FCH_FILENAME = 0 +FCH_DATA = 1 +FCH_ERROR = 2 + +def FilenameCheckHash(filename, literalfilename): + if literalfilename: + return FCH_FILENAME, filename + elif filename.startswith('#h#'): + result = Hex2Bytes(filename[3:].replace(' ', '')) + if result == None: + return FCH_ERROR, 'hexadecimal' + else: + return FCH_DATA, result + elif filename.startswith('#b#'): + try: + return FCH_DATA, binascii.a2b_base64(filename[3:]) + except: + return FCH_ERROR, 'base64' + elif filename.startswith('#e#'): + result = Interpret(filename[3:]) + if result == None: + return FCH_ERROR, 'expression' + else: + return FCH_DATA, C2BIP3(result) + elif filename.startswith('#p#'): + result = ParsePackExpression(filename[3:]) + if result == None: + return FCH_ERROR, 'pack' + else: + return FCH_DATA, result + elif filename.startswith('#'): + return FCH_DATA, C2BIP3(filename[1:]) + else: + return FCH_FILENAME, filename + +def AnalyzeFileError(filename): + PrintError('Error opening file %s' % filename) + PrintError(sys.exc_info()[1]) + try: + if not os.path.exists(filename): + PrintError('The file does not exist') + elif os.path.isdir(filename): + PrintError('The file is a directory') + elif not os.path.isfile(filename): + PrintError('The file is not a regular file') + except: + pass + +class cBinaryFile: + def __init__(self, filename, zippassword='infected', noextraction=False, literalfilename=False): + self.filename = filename + self.zippassword = zippassword + self.noextraction = noextraction + self.literalfilename = literalfilename + self.oZipfile = None + self.extracted = False + self.fIn = None + + fch, data = FilenameCheckHash(self.filename, self.literalfilename) + if fch == FCH_ERROR: + line = 'Error %s parsing filename: %s' % (data, self.filename) + raise Exception(line) + + try: + if self.filename == '': + if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + self.fIn = sys.stdin + elif fch == FCH_DATA: + self.fIn = DataIO(data) + elif not self.noextraction and self.filename.lower().endswith('.zip'): + self.oZipfile = zipfile.ZipFile(self.filename, 'r') + if len(self.oZipfile.infolist()) == 1: + self.fIn = self.oZipfile.open(self.oZipfile.infolist()[0], 'r', self.zippassword) + self.extracted = True + else: + self.oZipfile.close() + self.oZipfile = None + self.fIn = open(self.filename, 'rb') + elif not self.noextraction and self.filename.lower().endswith('.gz'): + self.fIn = gzip.GzipFile(self.filename, 'rb') + self.extracted = True + else: + self.fIn = open(self.filename, 'rb') + except: + AnalyzeFileError(self.filename) + raise + + def close(self): + if self.fIn != sys.stdin and self.fIn != None: + self.fIn.close() + if self.oZipfile != None: + self.oZipfile.close() + + def read(self, size=None): + try: + fRead = self.fIn.buffer + except: + fRead = self.fIn + if size == None: + return fRead.read() + else: + return fRead.read(size) + + def Data(self): + data = self.read() + self.close() + return data + +#-ENDCODE cBinaryFile-------------------------------------------------------------------------------- + +def File2Strings(filename): + try: + if filename == '': + f = sys.stdin + else: + f = open(filename, 'r') + except: + return None + try: + return map(lambda line:line.rstrip('\n'), f.readlines()) + except: + return None + finally: + if f != sys.stdin: + f.close() + +def File2String(filename): + try: + f = open(filename, 'rb') + except: + return None + try: + return f.read() + except: + return None + finally: + f.close() + +def ProcessAt(argument): + if argument.startswith('@'): + strings = File2Strings(argument[1:]) + if strings == None: + raise Exception('Error reading %s' % argument) + else: + return strings + else: + return [argument] + +def Glob(filename): + filenames = glob.glob(filename) + if len(filenames) == 0: + return [filename] + else: + return filenames + +class cExpandFilenameArguments(): + def __init__(self, filenames, literalfilenames=False, recursedir=False, checkfilenames=False, expressionprefix=None, flagprefix=None): + self.containsUnixShellStyleWildcards = False + self.warning = False + self.message = '' + self.filenameexpressionsflags = [] + self.expressionprefix = expressionprefix + self.flagprefix = flagprefix + self.literalfilenames = literalfilenames + + expression = '' + flag = '' + if len(filenames) == 0: + self.filenameexpressionsflags = [['', '', '']] + elif literalfilenames: + self.filenameexpressionsflags = [[filename, '', ''] for filename in filenames] + elif recursedir: + for dirwildcard in filenames: + if expressionprefix != None and dirwildcard.startswith(expressionprefix): + expression = dirwildcard[len(expressionprefix):] + elif flagprefix != None and dirwildcard.startswith(flagprefix): + flag = dirwildcard[len(flagprefix):] + else: + if dirwildcard.startswith('@'): + for filename in ProcessAt(dirwildcard): + self.filenameexpressionsflags.append([filename, expression, flag]) + elif os.path.isfile(dirwildcard): + self.filenameexpressionsflags.append([dirwildcard, expression, flag]) + else: + if os.path.isdir(dirwildcard): + dirname = dirwildcard + basename = '*' + else: + dirname, basename = os.path.split(dirwildcard) + if dirname == '': + dirname = '.' + for path, dirs, files in os.walk(dirname): + for filename in fnmatch.filter(files, basename): + self.filenameexpressionsflags.append([os.path.join(path, filename), expression, flag]) + else: + for filename in list(collections.OrderedDict.fromkeys(sum(map(self.Glob, sum(map(ProcessAt, filenames), [])), []))): + if expressionprefix != None and filename.startswith(expressionprefix): + expression = filename[len(expressionprefix):] + elif flagprefix != None and filename.startswith(flagprefix): + flag = filename[len(flagprefix):] + else: + self.filenameexpressionsflags.append([filename, expression, flag]) + self.warning = self.containsUnixShellStyleWildcards and len(self.filenameexpressionsflags) == 0 + if self.warning: + self.message = "Your filename argument(s) contain Unix shell-style wildcards, but no files were matched.\nCheck your wildcard patterns or use option literalfilenames if you don't want wildcard pattern matching." + return + if self.filenameexpressionsflags == [] and (expression != '' or flag != ''): + self.filenameexpressionsflags = [['', expression, flag]] + if checkfilenames: + self.CheckIfFilesAreValid() + + def Glob(self, filename): + if not ('?' in filename or '*' in filename or ('[' in filename and ']' in filename)): + return [filename] + self.containsUnixShellStyleWildcards = True + return glob.glob(filename) + + def CheckIfFilesAreValid(self): + valid = [] + doesnotexist = [] + isnotafile = [] + for filename, expression, flag in self.filenameexpressionsflags: + hashfile = False + try: + hashfile = FilenameCheckHash(filename, self.literalfilenames)[0] == FCH_DATA + except: + pass + if filename == '' or hashfile: + valid.append([filename, expression, flag]) + elif not os.path.exists(filename): + doesnotexist.append(filename) + elif not os.path.isfile(filename): + isnotafile.append(filename) + else: + valid.append([filename, expression, flag]) + self.filenameexpressionsflags = valid + if len(doesnotexist) > 0: + self.warning = True + self.message += 'The following files do not exist and will be skipped: ' + ' '.join(doesnotexist) + '\n' + if len(isnotafile) > 0: + self.warning = True + self.message += 'The following files are not regular files and will be skipped: ' + ' '.join(isnotafile) + '\n' + + def Filenames(self): + if self.expressionprefix == None: + return [filename for filename, expression, flag in self.filenameexpressionsflags] + else: + return self.filenameexpressionsflags + +def CheckJSON(stringJSON): + try: + object = json.loads(stringJSON) + except: + print('Error parsing JSON') + print(sys.exc_info()[1]) + return None + if not isinstance(object, dict): + print('Error JSON is not a dictionary') + return None + if not 'version' in object: + print('Error JSON dictionary has no version') + return None + if object['version'] != 2: + print('Error JSON dictionary has wrong version') + return None + if not 'id' in object: + print('Error JSON dictionary has no id') + return None + if object['id'] != 'didierstevens.com': + print('Error JSON dictionary has wrong id') + return None + if not 'type' in object: + print('Error JSON dictionary has no type') + return None + if object['type'] != 'content': + print('Error JSON dictionary has wrong type') + return None + if not 'fields' in object: + print('Error JSON dictionary has no fields') + return None + if not 'name' in object['fields']: + print('Error JSON dictionary has no name field') + return None + if not 'content' in object['fields']: + print('Error JSON dictionary has no content field') + return None + if not 'items' in object: + print('Error JSON dictionary has no items') + return None + for item in object['items']: + item['content'] = binascii.a2b_base64(item['content']) + return object['items'] + +CUTTERM_NOTHING = 0 +CUTTERM_POSITION = 1 +CUTTERM_FIND = 2 +CUTTERM_LENGTH = 3 + +def Replace(string, dReplacements): + if string in dReplacements: + return dReplacements[string] + else: + return string + +def ParseInteger(argument): + sign = 1 + if argument.startswith('+'): + argument = argument[1:] + elif argument.startswith('-'): + argument = argument[1:] + sign = -1 + if argument.startswith('0x'): + return sign * int(argument[2:], 16) + else: + return sign * int(argument) + +def ParseCutTerm(argument): + if argument == '': + return CUTTERM_NOTHING, None, '' + oMatch = re.match(r'\-?0x([0-9a-f]+)', argument, re.I) + if oMatch == None: + oMatch = re.match(r'\-?(\d+)', argument) + else: + value = int(oMatch.group(1), 16) + if argument.startswith('-'): + value = -value + return CUTTERM_POSITION, value, argument[len(oMatch.group(0)):] + if oMatch == None: + oMatch = re.match(r'\[([0-9a-f]+)\](\d+)?([+-](?:0x[0-9a-f]+|\d+))?', argument, re.I) + else: + value = int(oMatch.group(1)) + if argument.startswith('-'): + value = -value + return CUTTERM_POSITION, value, argument[len(oMatch.group(0)):] + if oMatch == None: + oMatch = re.match(r"\[u?\'(.+?)\'\](\d+)?([+-](?:0x[0-9a-f]+|\d+))?", argument) + else: + if len(oMatch.group(1)) % 2 == 1: + raise Exception("Uneven length hexadecimal string") + else: + return CUTTERM_FIND, (binascii.a2b_hex(oMatch.group(1)), int(Replace(oMatch.group(2), {None: '1'})), ParseInteger(Replace(oMatch.group(3), {None: '0'}))), argument[len(oMatch.group(0)):] + if oMatch == None: + return None, None, argument + else: + if argument.startswith("[u'"): + # convert ascii to unicode 16 byte sequence + searchtext = oMatch.group(1).decode('unicode_escape').encode('utf16')[2:] + else: + searchtext = oMatch.group(1) + return CUTTERM_FIND, (searchtext, int(Replace(oMatch.group(2), {None: '1'})), ParseInteger(Replace(oMatch.group(3), {None: '0'}))), argument[len(oMatch.group(0)):] + +def ParseCutArgument(argument): + type, value, remainder = ParseCutTerm(argument.strip()) + if type == CUTTERM_NOTHING: + return CUTTERM_NOTHING, None, CUTTERM_NOTHING, None + elif type == None: + if remainder.startswith(':'): + typeLeft = CUTTERM_NOTHING + valueLeft = None + remainder = remainder[1:] + else: + return None, None, None, None + else: + typeLeft = type + valueLeft = value + if typeLeft == CUTTERM_POSITION and valueLeft < 0: + return None, None, None, None + if typeLeft == CUTTERM_FIND and valueLeft[1] == 0: + return None, None, None, None + if remainder.startswith(':'): + remainder = remainder[1:] + else: + return None, None, None, None + type, value, remainder = ParseCutTerm(remainder) + if type == CUTTERM_POSITION and remainder == 'l': + return typeLeft, valueLeft, CUTTERM_LENGTH, value + elif type == None or remainder != '': + return None, None, None, None + elif type == CUTTERM_FIND and value[1] == 0: + return None, None, None, None + else: + return typeLeft, valueLeft, type, value + +def Find(data, value, nth, startposition=-1): + position = startposition + while nth > 0: + position = data.find(value, position + 1) + if position == -1: + return -1 + nth -= 1 + return position + +def CutData(stream, cutArgument): + if cutArgument == '': + return [stream, None, None] + + typeLeft, valueLeft, typeRight, valueRight = ParseCutArgument(cutArgument) + + if typeLeft == None: + return [stream, None, None] + + if typeLeft == CUTTERM_NOTHING: + positionBegin = 0 + elif typeLeft == CUTTERM_POSITION: + positionBegin = valueLeft + elif typeLeft == CUTTERM_FIND: + positionBegin = Find(stream, valueLeft[0], valueLeft[1]) + if positionBegin == -1: + return ['', None, None] + positionBegin += valueLeft[2] + else: + raise Exception("Unknown value typeLeft") + + if typeRight == CUTTERM_NOTHING: + positionEnd = len(stream) + elif typeRight == CUTTERM_POSITION and valueRight < 0: + positionEnd = len(stream) + valueRight + elif typeRight == CUTTERM_POSITION: + positionEnd = valueRight + 1 + elif typeRight == CUTTERM_LENGTH: + positionEnd = positionBegin + valueRight + elif typeRight == CUTTERM_FIND: + positionEnd = Find(stream, valueRight[0], valueRight[1], positionBegin) + if positionEnd == -1: + return ['', None, None] + else: + positionEnd += len(valueRight[0]) + positionEnd += valueRight[2] + else: + raise Exception("Unknown value typeRight") + + return [stream[positionBegin:positionEnd], positionBegin, positionEnd] + +#-BEGINCODE cDump------------------------------------------------------------------------------------ +#import binascii +#import sys +#if sys.version_info[0] >= 3: +# from io import StringIO +#else: +# from cStringIO import StringIO + +class cDump(): + def __init__(self, data, prefix='', offset=0, dumplinelength=16): + self.data = data + self.prefix = prefix + self.offset = offset + self.dumplinelength = dumplinelength + + def HexDump(self): + oDumpStream = self.cDumpStream(self.prefix) + hexDump = '' + for i, b in enumerate(self.data): + if i % self.dumplinelength == 0 and hexDump != '': + oDumpStream.Addline(hexDump) + hexDump = '' + hexDump += IFF(hexDump == '', '', ' ') + '%02X' % self.C2IIP2(b) + oDumpStream.Addline(hexDump) + return oDumpStream.Content() + + def CombineHexAscii(self, hexDump, asciiDump): + if hexDump == '': + return '' + countSpaces = 3 * (self.dumplinelength - len(asciiDump)) + if len(asciiDump) <= self.dumplinelength / 2: + countSpaces += 1 + return hexDump + ' ' + (' ' * countSpaces) + asciiDump + + def HexAsciiDump(self, rle=False): + oDumpStream = self.cDumpStream(self.prefix) + position = '' + hexDump = '' + asciiDump = '' + previousLine = None + countRLE = 0 + for i, b in enumerate(self.data): + b = self.C2IIP2(b) + if i % self.dumplinelength == 0: + if hexDump != '': + line = self.CombineHexAscii(hexDump, asciiDump) + if not rle or line != previousLine: + if countRLE > 0: + oDumpStream.Addline('* %d 0x%02x' % (countRLE, countRLE * self.dumplinelength)) + oDumpStream.Addline(position + line) + countRLE = 0 + else: + countRLE += 1 + previousLine = line + position = '%08X:' % (i + self.offset) + hexDump = '' + asciiDump = '' + if i % self.dumplinelength == self.dumplinelength / 2: + hexDump += ' ' + hexDump += ' %02X' % b + asciiDump += IFF(b >= 32 and b < 127, chr(b), '.') + if countRLE > 0: + oDumpStream.Addline('* %d 0x%02x' % (countRLE, countRLE * self.dumplinelength)) + oDumpStream.Addline(self.CombineHexAscii(position + hexDump, asciiDump)) + return oDumpStream.Content() + + def Base64Dump(self, nowhitespace=False): + encoded = binascii.b2a_base64(self.data).decode().strip() + if nowhitespace: + return encoded + oDumpStream = self.cDumpStream(self.prefix) + length = 64 + for i in range(0, len(encoded), length): + oDumpStream.Addline(encoded[0+i:length+i]) + return oDumpStream.Content() + + class cDumpStream(): + def __init__(self, prefix=''): + self.oStringIO = StringIO() + self.prefix = prefix + + def Addline(self, line): + if line != '': + self.oStringIO.write(self.prefix + line + '\n') + + def Content(self): + return self.oStringIO.getvalue() + + @staticmethod + def C2IIP2(data): + if sys.version_info[0] > 2: + return data + else: + return ord(data) +#-ENDCODE cDump-------------------------------------------------------------------------------------- + +def IfWIN32SetBinary(io): + if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(io.fileno(), os.O_BINARY) + +#Fix for http://bugs.python.org/issue11395 +def StdoutWriteChunked(data): + if sys.version_info[0] > 2: + if isinstance(data, str): + sys.stdout.write(data) + else: + sys.stdout.buffer.write(data) + else: + while data != '': + sys.stdout.write(data[0:10000]) + try: + sys.stdout.flush() + except IOError: + return + data = data[10000:] + +class cVariables(): + def __init__(self, variablesstring='', separator=DEFAULT_SEPARATOR): + self.dVariables = {} + if variablesstring == '': + return + for variable in variablesstring.split(separator): + name, value = VariableNameValue(variable) + self.dVariables[name] = value + + def SetVariable(self, name, value): + self.dVariables[name] = value + + def Instantiate(self, astring): + for key, value in self.dVariables.items(): + astring = astring.replace('%' + key + '%', value) + return astring + +class cOutput(): + def __init__(self, filenameOption=None, binary=False): + self.starttime = time.time() + self.filenameOption = filenameOption + self.separateFiles = False + self.progress = False + self.console = False + self.head = False + self.headCounter = 0 + self.tail = False + self.tailQueue = [] + self.fOut = None + self.oCsvWriter = None + self.rootFilenames = {} + self.binary = binary + if self.binary: + self.fileoptions = 'wb' + else: + self.fileoptions = 'w' + if self.filenameOption: + if self.ParseHash(self.filenameOption): + if not self.separateFiles and self.filename != '': + self.fOut = open(self.filename, self.fileoptions) + elif self.filenameOption != '': + self.fOut = open(self.filenameOption, self.fileoptions) + + def ParseHash(self, option): + if option.startswith('#'): + position = self.filenameOption.find('#', 1) + if position > 1: + switches = self.filenameOption[1:position] + self.filename = self.filenameOption[position + 1:] + for switch in switches: + if switch == 's': + self.separateFiles = True + elif switch == 'p': + self.progress = True + elif switch == 'c': + self.console = True + elif switch == 'l': + pass + elif switch == 'g': + if self.filename != '': + extra = self.filename + '-' + else: + extra = '' + self.filename = '%s-%s%s.txt' % (os.path.splitext(os.path.basename(sys.argv[0]))[0], extra, self.FormatTime()) + elif switch == 'h': + self.head = True + elif switch == 't': + self.tail = True + else: + return False + return True + return False + + @staticmethod + def FormatTime(epoch=None): + if epoch == None: + epoch = time.time() + return '%04d%02d%02d-%02d%02d%02d' % time.localtime(epoch)[0:6] + + def RootUnique(self, root): + if not root in self.rootFilenames: + self.rootFilenames[root] = None + return root + iter = 1 + while True: + newroot = '%s_%04d' % (root, iter) + if not newroot in self.rootFilenames: + self.rootFilenames[newroot] = None + return newroot + iter += 1 + + def LineSub(self, line, eol): + if self.fOut == None or self.console: + try: + print(line, end=eol) + except UnicodeEncodeError: + encoding = sys.stdout.encoding + print(line.encode(encoding, errors='backslashreplace').decode(encoding), end=eol) +# sys.stdout.flush() + if self.fOut != None: + self.fOut.write(line + '\n') + self.fOut.flush() + + def Line(self, line, eol='\n'): + if self.head: + if self.headCounter < 10: + self.LineSub(line, eol) + elif self.tail: + self.tailQueue = self.tailQueue[-9:] + [[line, eol]] + self.headCounter += 1 + elif self.tail: + self.tailQueue = self.tailQueue[-9:] + [[line, eol]] + else: + self.LineSub(line, eol) + + def LineTimestamped(self, line): + self.Line('%s: %s' % (self.FormatTime(), line)) + + def WriteBinary(self, data): + if self.fOut != None: + self.fOut.write(data) + self.fOut.flush() + else: + IfWIN32SetBinary(sys.stdout) + StdoutWriteChunked(data) + + def CSVWriteRow(self, row): + if self.oCsvWriter == None: + self.StringIOCSV = StringIO() +# self.oCsvWriter = csv.writer(self.fOut) + self.oCsvWriter = csv.writer(self.StringIOCSV) + self.oCsvWriter.writerow(row) + self.Line(self.StringIOCSV.getvalue(), '') + self.StringIOCSV.truncate(0) + self.StringIOCSV.seek(0) + + def Filename(self, filename, index, total): + self.separateFilename = filename + if self.progress: + if index == 0: + eta = '' + else: + seconds = int(float((time.time() - self.starttime) / float(index)) * float(total - index)) + eta = 'estimation %d seconds left, finished %s ' % (seconds, self.FormatTime(time.time() + seconds)) + PrintError('%d/%d %s%s' % (index + 1, total, eta, self.separateFilename)) + if self.separateFiles and self.filename != '': + oFilenameVariables = cVariables() + oFilenameVariables.SetVariable('f', self.separateFilename) + basename = os.path.basename(self.separateFilename) + oFilenameVariables.SetVariable('b', basename) + oFilenameVariables.SetVariable('d', os.path.dirname(self.separateFilename)) + root, extension = os.path.splitext(basename) + oFilenameVariables.SetVariable('r', root) + oFilenameVariables.SetVariable('ru', self.RootUnique(root)) + oFilenameVariables.SetVariable('e', extension) + + self.Close() + self.fOut = open(oFilenameVariables.Instantiate(self.filename), self.fileoptions) + + def Close(self): + if self.head and self.tail and len(self.tailQueue) > 0: + self.LineSub('...', '\n') + + for line, eol in self.tailQueue: + self.LineSub(line, eol) + + self.headCounter = 0 + self.tailQueue = [] + + if self.fOut != None: + self.fOut.close() + self.fOut = None + +def ToString(value): + if isinstance(value, str): + return value + else: + return str(value) + +def Quote(value, separator, quote): + value = ToString(value) + if len(value) > 1 and value[0] == quote and value[-1] == quote: + return value + if separator in value or value == '': + return quote + value + quote + else: + return value + +def MakeCSVLine(row, separator, quote): + return separator.join([Quote(value, separator, quote) for value in row]) + +class cLogfile(): + def __init__(self, keyword, comment): + self.starttime = time.time() + self.errors = 0 + if keyword == '': + self.oOutput = None + else: + self.oOutput = cOutput('%s-%s-%s.log' % (os.path.splitext(os.path.basename(sys.argv[0]))[0], keyword, self.FormatTime())) + self.Line('Start') + self.Line('UTC', '%04d%02d%02d-%02d%02d%02d' % time.gmtime(time.time())[0:6]) + self.Line('Comment', comment) + self.Line('Args', repr(sys.argv)) + self.Line('Version', __version__) + self.Line('Python', repr(sys.version_info)) + self.Line('Platform', sys.platform) + self.Line('CWD', repr(os.getcwd())) + + @staticmethod + def FormatTime(epoch=None): + if epoch == None: + epoch = time.time() + return '%04d%02d%02d-%02d%02d%02d' % time.localtime(epoch)[0:6] + + def Line(self, *line): + if self.oOutput != None: + self.oOutput.Line(MakeCSVLine((self.FormatTime(), ) + line, DEFAULT_SEPARATOR, QUOTE)) + + def LineError(self, *line): + self.Line('Error', *line) + self.errors += 1 + + def Close(self): + if self.oOutput != None: + self.Line('Finish', '%d error(s)' % self.errors, '%d second(s)' % (time.time() - self.starttime)) + self.oOutput.Close() + +def CalculateByteStatistics(dPrevalence=None, data=None): + averageConsecutiveByteDifference = None + if dPrevalence == None: + dPrevalence = {iter: 0 for iter in range(0x100)} + sumDifferences = 0.0 + previous = None + if len(data) > 1: + for byte in data: + byte = C2IIP2(byte) + dPrevalence[byte] += 1 + if previous != None: + sumDifferences += abs(byte - previous) + previous = byte + averageConsecutiveByteDifference = sumDifferences /float(len(data)-1) + sumValues = sum(dPrevalence.values()) + countNullByte = dPrevalence[0] + countControlBytes = 0 + countWhitespaceBytes = 0 + countUniqueBytes = 0 + for iter in range(1, 0x21): + if chr(iter) in string.whitespace: + countWhitespaceBytes += dPrevalence[iter] + else: + countControlBytes += dPrevalence[iter] + countControlBytes += dPrevalence[0x7F] + countPrintableBytes = 0 + for iter in range(0x21, 0x7F): + countPrintableBytes += dPrevalence[iter] + countHighBytes = 0 + for iter in range(0x80, 0x100): + countHighBytes += dPrevalence[iter] + countHexadecimalBytes = 0 + countBASE64Bytes = 0 + for iter in range(0x30, 0x3A): + countHexadecimalBytes += dPrevalence[iter] + countBASE64Bytes += dPrevalence[iter] + for iter in range(0x41, 0x47): + countHexadecimalBytes += dPrevalence[iter] + for iter in range(0x61, 0x67): + countHexadecimalBytes += dPrevalence[iter] + for iter in range(0x41, 0x5B): + countBASE64Bytes += dPrevalence[iter] + for iter in range(0x61, 0x7B): + countBASE64Bytes += dPrevalence[iter] + countBASE64Bytes += dPrevalence[ord('+')] + dPrevalence[ord('/')] + dPrevalence[ord('=')] + entropy = 0.0 + for iter in range(0x100): + if dPrevalence[iter] > 0: + prevalence = float(dPrevalence[iter]) / float(sumValues) + entropy += - prevalence * math.log(prevalence, 2) + countUniqueBytes += 1 + return sumValues, entropy, countUniqueBytes, countNullByte, countControlBytes, countWhitespaceBytes, countPrintableBytes, countHighBytes, countHexadecimalBytes, countBASE64Bytes, averageConsecutiveByteDifference + +def Unpack(format, data): + size = struct.calcsize(format) + result = list(struct.unpack(format, data[:size])) + result.append(data[size:]) + return result + +def InstantiateCOutput(options): + filenameOption = None + if options.output != '': + filenameOption = options.output + return cOutput(filenameOption, binary=options.dump) + +def WhitespaceOnly(data): + for byte in data: + if not byte in [9, 10, 11, 12, 13, 32]: + return False + return True + +def PDFIncrementalUpdatesSub(data, oOutput, options): + accumulate = DataIO(b'') + token = b'' + data += b'\x00' + dCounters = {} + versions = [] + for iter in range(len(data)): + byte = data[iter:iter+1] + if token == b'': + if byte == b'%' or byte == b'/' or byte >= b'a' and byte <= b'z' or byte >= b'A' and byte <= b'Z': + token += byte + else: + accumulate.write(byte) + elif token[0] == ord(b'%') and byte == b'%': + token += byte + elif byte >= b'a' and byte <= b'z' or byte >= b'A' and byte <= b'Z': + token += byte + elif token == b'%%EOF' and byte in [b'\x0a', b'\x0d']: + token += byte + elif token == b'%%EOF\x0d' and byte == b'\x0a': + token += byte + else: + accumulate.write(token) + if token[:5] == b'%%EOF': + dCounters[b'%%EOF'] = 1 + if len(versions) == 0: + offset = 0 + else: + offset = len(versions[-1][1]) + versions.append([dCounters, accumulate.getvalue(), accumulate.getvalue()[offset:]]) + dCounters = {} + else: + if token in [b'obj', b'endobj', b'/Linearized']: + dCounters[token] = dCounters.get(token, 0) + 1 + token = b'' + accumulate.write(byte) + data = data[:-1] + accumulate = accumulate.getvalue()[:-1] + if len(versions[-1][1]) != len(accumulate): + offset = len(versions[-1][1]) + versions.append([dCounters, accumulate, accumulate[offset:]]) + newVersions = [] + for index, version in enumerate(versions): + index += 1 + if b'obj' in version[0] and b'endobj' in version[0] and version[0][b'obj'] == version[0][b'endobj']: + info = 'objects= %d' % version[0][b'obj'] + if b'/Linearized' in version[0]: + info += ' (Linearized)' + else: + info = repr(version[0]) + if version[0] == {}: + info = 'no objects' + if WhitespaceOnly(version[2]): + info = 'whitespace' + newVersions.append([index, info] + version) + return newVersions + +def PDFIncrementalUpdates(data, oOutput, options): + newVersions = PDFIncrementalUpdatesSub(data, oOutput, options) + if options.select == '': + for version in newVersions: + oOutput.Line('%d: %s length= %d difference= %d MD5= %s' % (version[0], version[1], len(version[3]), len(version[4]), hashlib.md5(version[3]).hexdigest())) + else: + if options.select.endswith('d'): + indexData = 4 + index = int(options.select[:-1]) + else: + indexData = 3 + index = int(options.select) + for version in newVersions: + if index == version[0]: + if options.dump: + oOutput.WriteBinary(version[indexData]) + elif options.hexdump: + oOutput.Line(cDump(version[indexData]).HexDump()) + elif options.asciidumprle: + oOutput.Line(cDump(version[indexData]).HexAsciiDump(True)) + else: + oOutput.Line(cDump(version[indexData]).HexAsciiDump()) + +def ProcessBinaryFile(command, filename, content, cutexpression, flag, oOutput, oLogfile, options, oParserFlag): + if content == None: + try: + oBinaryFile = cBinaryFile(filename, C2BIP3(options.password), options.noextraction, options.literalfilenames) + except: + oLogfile.LineError('Opening file %s %s' % (filename, repr(sys.exc_info()[1]))) + return + oLogfile.Line('Success', 'Opening file %s' % filename) + try: + data = oBinaryFile.read() + except: + oLogfile.LineError('Reading file %s %s' % (filename, repr(sys.exc_info()[1]))) + return + data = CutData(data, cutexpression)[0] + oBinaryFile.close() + else: + data = content + + (flagoptions, flagargs) = oParserFlag.parse_args(flag.split(' ')) + + try: + # ----- Put your data processing code here ----- + if options.select == '': + oOutput.Line('File: %s%s' % (filename, IFF(oBinaryFile.extracted, ' (extracted)', ''))) + PDFIncrementalUpdates(data, oOutput, options) + # ---------------------------------------------- + except: + oLogfile.LineError('Processing file %s %s' % (filename, repr(sys.exc_info()[1]))) + if not options.ignoreprocessingerrors: + raise + +# data = CutData(cBinaryFile(filename, C2BIP3(options.password), options.noextraction, options.literalfilenames).Data(), cutexpression)[0] + +def ProcessBinaryFiles(command, filenames, oLogfile, options, oParserFlag): + oOutput = InstantiateCOutput(options) + index = 0 + if options.jsoninput: + items = CheckJSON(sys.stdin.read()) + if items == None: + return + for item in items: + oOutput.Filename(item['name'], index, len(items)) + index += 1 + ProcessBinaryFile(command, item['name'], item['content'], '', '', oOutput, oLogfile, options, oParserFlag) + else: + for filename, cutexpression, flag in filenames: + oOutput.Filename(filename, index, len(filenames)) + index += 1 + ProcessBinaryFile(command, filename, None, cutexpression, flag, oOutput, oLogfile, options, oParserFlag) + +def Main(): + moredesc = ''' + +Source code put in the public domain by Didier Stevens, no Copyright +Use at your own risk +https://DidierStevens.com''' + + oParserFlag = optparse.OptionParser(usage='\nFlag arguments start with #f#:') + oParserFlag.add_option('-l', '--length', action='store_true', default=False, help='Print length of files') + + oParser = optparse.OptionParser(usage='usage: %prog [options] command [[@]file|cut-expression|flag-expression ...]\n' + __description__ + moredesc, version='%prog ' + __version__, epilog='This tool also accepts flag arguments (#f#), read the man page (-m) for more info.') + oParser.add_option('-m', '--man', action='store_true', default=False, help='Print manual') + oParser.add_option('-o', '--output', type=str, default='', help='Output to file (# supported)') + oParser.add_option('-s', '--select', default='', help='select item nr for dumping (a for all)') + oParser.add_option('-d', '--dump', action='store_true', default=False, help='perform dump') + oParser.add_option('-x', '--hexdump', action='store_true', default=False, help='perform hex dump') + oParser.add_option('-a', '--asciidump', action='store_true', default=False, help='perform ascii dump') + oParser.add_option('-A', '--asciidumprle', action='store_true', default=False, help='perform ascii dump with RLE') + oParser.add_option('-p', '--password', default='infected', help='The ZIP password to be used (default infected)') + oParser.add_option('-n', '--noextraction', action='store_true', default=False, help='Do not extract from archive file') + oParser.add_option('-l', '--literalfilenames', action='store_true', default=False, help='Do not interpret filenames') + oParser.add_option('-r', '--recursedir', action='store_true', default=False, help='Recurse directories (wildcards and here files (@...) allowed)') + oParser.add_option('--checkfilenames', action='store_true', default=False, help='Perform check if files exist prior to file processing') + oParser.add_option('-j', '--jsoninput', action='store_true', default=False, help='Consume JSON from stdin') + oParser.add_option('--logfile', type=str, default='', help='Create logfile with given keyword') + oParser.add_option('--logcomment', type=str, default='', help='A string with comments to be included in the log file') + oParser.add_option('--ignoreprocessingerrors', action='store_true', default=False, help='Ignore errors during file processing') + (options, args) = oParser.parse_args() + + if options.man: + oParser.print_help() + oParserFlag.print_help() + PrintManual() + return + + if len(args) == 0: + oParser.print_help() + return + + command = args[0] + args = args[1:] + + commands = ['iu'] + + if not command in commands: + print('Error: unknown command: %s' % command) + print('Available commands: %s' % ' '.join(commands)) + oParser.print_help() + return + + if len(args) != 0 and options.jsoninput: + print('Error: option -j can not be used with files') + return + + oLogfile = cLogfile(options.logfile, options.logcomment) + oExpandFilenameArguments = cExpandFilenameArguments(args, options.literalfilenames, options.recursedir, options.checkfilenames, '#c#', '#f#') + oLogfile.Line('FilesCount', str(len(oExpandFilenameArguments.Filenames()))) + oLogfile.Line('Files', repr(oExpandFilenameArguments.Filenames())) + if oExpandFilenameArguments.warning: + PrintError('\nWarning:') + PrintError(oExpandFilenameArguments.message) + oLogfile.Line('Warning', repr(oExpandFilenameArguments.message)) + + ProcessBinaryFiles(command, oExpandFilenameArguments.Filenames(), oLogfile, options, oParserFlag) + + if oLogfile.errors > 0: + PrintError('Number of errors: %d' % oLogfile.errors) + oLogfile.Close() + +if __name__ == '__main__': + Main() diff --git a/sift/files/pdf-tools/plugin_embeddedfile.py b/sift/files/pdf-tools/plugin_embeddedfile.py index 2d68bcbb..2a78936c 100644 --- a/sift/files/pdf-tools/plugin_embeddedfile.py +++ b/sift/files/pdf-tools/plugin_embeddedfile.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #2014/10/13 diff --git a/sift/files/pdf-tools/plugin_nameobfuscation.py b/sift/files/pdf-tools/plugin_nameobfuscation.py index e116da2e..913b0e81 100644 --- a/sift/files/pdf-tools/plugin_nameobfuscation.py +++ b/sift/files/pdf-tools/plugin_nameobfuscation.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #2013/11/04 #2013/11/08 diff --git a/sift/files/pdf-tools/plugin_triage.py b/sift/files/pdf-tools/plugin_triage.py index 8245c6f5..36c8cffd 100644 --- a/sift/files/pdf-tools/plugin_triage.py +++ b/sift/files/pdf-tools/plugin_triage.py @@ -1,22 +1,51 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #2014/09/30 +#2015/08/12 added options; changed scoring: /ObjStm 0.75; obj/endobj or stream/endstream discrepancy: 0.50 +#2015/08/13 added instructions +#2017/10/29 added /URI class cPDFiDTriage(cPluginParent): -# onlyValidPDF = True + onlyValidPDF = False name = 'Triage plugin' - def __init__(self, oPDFiD): + def __init__(self, oPDFiD, options): + self.options = options self.oPDFiD = oPDFiD def Score(self): - for keyword in ('/ObjStm', '/JS', '/JavaScript', '/AA', '/OpenAction', '/AcroForm', '/JBIG2Decode', '/RichMedia', '/Launch', '/EmbeddedFile', '/XFA', '/Colors > 2^24'): + for keyword in ('/JS', '/JavaScript', '/AA', '/OpenAction', '/AcroForm', '/JBIG2Decode', '/RichMedia', '/Launch', '/EmbeddedFile', '/XFA', '/Colors > 2^24'): if keyword in self.oPDFiD.keywords and self.oPDFiD.keywords[keyword].count > 0: return 1.0 - if self.oPDFiD.keywords['obj'].count != self.oPDFiD.keywords['endobj'].count: - return 1.0 - if self.oPDFiD.keywords['stream'].count != self.oPDFiD.keywords['endstream'].count: - return 1.0 + if self.options != '--io': + for keyword in ('/ObjStm', ): + if keyword in self.oPDFiD.keywords and self.oPDFiD.keywords[keyword].count > 0: + return 0.75 + for keyword in ('/URI', ): + if keyword in self.oPDFiD.keywords and self.oPDFiD.keywords[keyword].count > 0: + return 0.6 + if self.oPDFiD.keywords['obj'].count != self.oPDFiD.keywords['endobj'].count: + return 0.5 + if self.oPDFiD.keywords['stream'].count != self.oPDFiD.keywords['endstream'].count: + return 0.5 return 0.0 + def Instructions(self, score): + if score == 1.0: + return 'Sample is likely malicious and requires further analysis' + + if score == 0.75: + return '/ObjStm detected, analyze sample with pdfid-objstm.bat' + + if score == 0.5: + return 'Sample is likely not malicious but requires further analysis' + + if score == 0.6: + return 'Sample is likely not malicious but could contain phishing or payload URL' + + if score == 0.0: + return 'Sample is likely not malicious, unless you suspect this is used in a targeted/sophisticated attack' + + return '' + AddPlugin(cPDFiDTriage) diff --git a/sift/scripts/pdf-tools.sls b/sift/scripts/pdf-tools.sls index 83a7ea11..643a9ab9 100644 --- a/sift/scripts/pdf-tools.sls +++ b/sift/scripts/pdf-tools.sls @@ -1,13 +1,39 @@ -sift-scripts-pdf-tools: +# Name: pdf-tools +# Website: https://blog.didierstevens.com/programs/pdf-tools/ +# Description: A collection of python scripts used for parsing PDF's +# Category: +# Author: Didier Stevens +# License: Free, Public Domain +# Notes: make-pdf-embedded.py, make-pdf-helloworld.py, make-pdf-javascript.py, pdfid.py, pdf-parser.py, pdftool.py + +{% set files = ['make-pdf-embedded.py','make-pdf-helloworld.py','make-pdf-javascript.py','mPDF.py','pdfid.py','pdf-parser.py','pdftool.py','plugin_embeddedfile.py','plugin_nameobfuscation.py','plugin_triage.py'] %} + +include: + - sift.packages.python3-virtualenv + +sift-python3-package-pdf-tools-venv: + virtualenv.managed: + - name: /opt/pdf-tools + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-pdf-tools: file.recurse: - - name: /usr/local/bin + - name: /opt/pdf-tools/bin/ - source: salt://sift/files/pdf-tools - file_mode: 755 - -sift-scripts-pdf-tools-helloworld-shebang: - file.prepend: - - name: /usr/local/bin/make-pdf-helloworld.py - - text: '#!/usr/bin/env python2' - - watch: - - file: sift-scripts-pdf-tools +{% for file in files %} +sift-python3-package-pdf-tools-symlink-{{ file }}: + file.symlink: + - name: /usr/local/bin/{{ file }} + - target: /opt/pdf-tools/bin/{{ file }} + - makedirs: False + - require: + - file: sift-python3-package-pdf-tools +{% endfor %} From ace7aea5390d91be7919ea4a151c8d0b7b413712 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:16:22 -0500 Subject: [PATCH 53/64] Update page-brute for 24 (#151) * Update page-brute for 24 * Update page-brute to be a python3 package * Update page-brute to be a python3 package --- sift/files/page-brute/README.md | 131 ------- sift/files/page-brute/default_signatures.yar | 347 ------------------- sift/files/page-brute/page_brute-BETA.py | 227 ------------ sift/python3-packages/init.sls | 2 + sift/python3-packages/page-brute.sls | 38 ++ sift/scripts/init.sls | 4 +- sift/scripts/page-brute.sls | 15 - 7 files changed, 42 insertions(+), 722 deletions(-) delete mode 100644 sift/files/page-brute/README.md delete mode 100644 sift/files/page-brute/default_signatures.yar delete mode 100755 sift/files/page-brute/page_brute-BETA.py create mode 100644 sift/python3-packages/page-brute.sls delete mode 100644 sift/scripts/page-brute.sls diff --git a/sift/files/page-brute/README.md b/sift/files/page-brute/README.md deleted file mode 100644 index d7426770..00000000 --- a/sift/files/page-brute/README.md +++ /dev/null @@ -1,131 +0,0 @@ -page_brute (beta!) -========== - -**page_brute.py** is a digital forensic tool purposed to analyze and categorize individual paged memory frames from Windows Page Files by appying YARA-based signatures to fix-sized blocks of pagefile.sys. - -***This tool can be used to:*** - * Disambiguate evidence within pagefile.sys by logically grouping blocks/pages into categories based on YARA rulesets of forensic artifacts that follow a pattern/convention. - * Identify page files that contain remanants of popular cleartext protocols such as HTTP/FTP, etc to identify network activities. - * Identify potential attacker activities based on popular command syntaxes used during internal propagations. - * Identify evidence of active malware infections based on YARA signatures for known malware. - * Isolate page files that contain signatures/magic values for popular file formats for more precise file carving. - -##NOTICE: -This tool is currently in beta! This utility and its signature set is subject to change in the near future! For suggestions - email the author via github. - -##Requires: - * yara & yara-python: http://code.google.com/p/yara-project/downloads/list - * default_signatures.yar (see above) - -##How does it work? -1. Given block size, page_brute.py reads in pagefile in fixed-sized blocks (default, 4096 bytes) -2. For each block, page_brute decides if the block is null - if null, the block is skipped. -3. If block is not null, the block is applied against compiled yara signatures (defined in -r/--rules argument). - * If -r/--rules not provided, page_brute.py will read from the default ruleset: default_signatures.yar - * Custom rules stored in a folder can also be provided as an argument to -r/--rules (must end in .yar) -4. If a block matches a YARA signature, the raw block will be stored in the corresponding output directory. - * -o/--scanname defines output folder that raw blocks will be saved. - * If no output is specified, a default folder is created in pwd: PAGE_BRUTE-YYYY-MM-DD-HH:MM:SS-RESULTS -5. Blocks are labeled by their logical page ID beginning at 0. - * To determine offset, multiply pageID by the page size. - -***NOTE:*** if a page file matches against multiple signatures, the corresponding page file will be copied to each rule directory. - -##How do I write signatures? -YARA is a powerful engine that allows you to match groups of strings,binary sequences,and regular expressions with user-defined boolean conditions against pretty much anything. - -To learn more about writing YARA rules, please see the informative user guide here: http://yara-project.googlecode.com/files/YARA%20User%27s%20Manual%201.6.pdf - -##Current Signatures: - * FTP - * HTTP requests/responses - * IRC - * Administrative/Hidden Share Abuse - * Remote system syntaxes - * HTML - * Javascript - * CMD Shell (this might suck) - * SMTP Message Headers - -##Usage: -From the help page: -``` -usage: page_brute-BETA.py [-h] [-f FILE] [-p SIZE] [-o SCANNAME] [-i] - [-r RULEFILE] - -Checks pages in pagefiles for YARA-based rule matches. Useful to identify -forensic artifacts within Windows-based page files and characterize blocks -based on regular expressions. - -optional arguments: - -h, --help show this help message and exit - -r RULEFILE, --rules RULEFILE - File/directory containing YARA signatures (must end - with .yar) - - -f FILE, --file FILE Pagefile or any chunk/block-based binary file - -p SIZE, --size SIZE Size of chunk/block in bytes (Default 4096) - -o SCANNAME, --scanname SCANNAME - Descriptor of the scan session - used for output - directory - -i, --invert Given scan options, match all blocks that DO NOT match - a ruleset -``` -###In Action: -``` -root@system:~/Desktop/page/page_brute# ./page_brute-BETA.py --file=pagefile.sys -[+] - PAGE_BRUTE processing file: pagefile.sys -[+] - Ruleset Compilation Successful. -[+] - PAGE_BRUTE running with the following options: - [-] - FILE: pagefile.sys - [-] - PAGE_SIZE: 4096 - [-] - RULES TYPE: DEFAULT - [-] - RULE LOCATION: default_signatures.yar - [-] - INVERSION SCAN: False - [-] - WORKING DIR: PAGE_BRUTE-2013-10-27-01:09:33-RESULTS - ================= - - [!] FLAGGED BLOCK 56: cmdshell - [!] FLAGGED BLOCK 87: cmdshell - [!] FLAGGED BLOCK 1222: webartifact_html - [!] FLAGGED BLOCK 1454: webartifact_html - [!] FLAGGED BLOCK 1782: webartifact_html - [!] FLAGGED BLOCK 2200: webartifact_html - [!] FLAGGED BLOCK 3781: webartifact_html - -root@system:~/Desktop/page/page_brute# ls -lR PAGE_BRUTE-2013-10-27-01\:09\:33-RESULTS/ -PAGE_BRUTE-2013-10-27-01:09:33-RESULTS/: -total 8 -drwxr-xr-x 2 root root 4096 Oct 27 01:09 cmdshell -drwxr-xr-x 2 root root 4096 Oct 27 01:09 webartifact_html - -PAGE_BRUTE-2013-10-27-01:09:33-RESULTS/cmdshell: -total 8 --rw-r--r-- 1 root root 4096 Oct 27 01:09 118.page --rw-r--r-- 1 root root 4096 Oct 27 01:09 77.page - -PAGE_BRUTE-2013-10-27-01:09:33-RESULTS/webartifact_html: -total 20 --rw-r--r-- 1 root root 4096 Oct 27 01:09 1330.page --rw-r--r-- 1 root root 4096 Oct 27 01:09 1445.page - -root@system:~/Desktop/page/page_brute/PAGE_BRUTE-2013-10-27-01:20:28-RESULTS/webartifact_html# xxd 24606.page -0000000: 613e 3c2f 7464 3e0d 0a20 2020 2020 2020 a>.. -0000010: 2020 203c 2f74 723e 0d0a 0d0a 2020 2020 .... -0000020: 2020 2020 2020 3c74 7220 6964 3d22 446f .. -0000060: 3c74 643e 3c69 6d67 2069 643d 226e 6f74 Not recomm -00000b0: 656e 6465 6420 6963 6f6e 2220 636c 6173 ended icon< -00000d0: 2f74 643e 0d0a 2020 2020 2020 2020 2020 /td>.. -00000e0: 2020 3c74 6420 7374 796c 653d 2270 6164 =24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - yara-python + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-page-brute: + pip.installed: + - name: page-brute + - bin_env: /opt/page-brute/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-page-brute-venv + +sift-python3-package-page-brute-symlink: + file.symlink: + - name: /usr/local/bin/page-brute + - target: /opt/page-brute/bin/page-brute + - makedirs: False + - require: + - pip: sift-python3-package-page-brute diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index c55f8dea..105e1985 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -10,7 +10,7 @@ include: - sift.scripts.jobparser - sift.scripts.keydet-tools - sift.scripts.packerid - - sift.scripts.page-brute + - sift.scripts.parseusn - sift.scripts.pdf-tools - sift.scripts.pescanner - sift.scripts.pe-carver @@ -40,7 +40,7 @@ sift-scripts: - sls: sift.scripts.jobparser - sls: sift.scripts.keydet-tools - sls: sift.scripts.packerid - - sls: sift.scripts.page-brute + - sls: sift.scripts.parseusn - sls: sift.scripts.pdf-tools - sls: sift.scripts.pescanner - sls: sift.scripts.pe-carver diff --git a/sift/scripts/page-brute.sls b/sift/scripts/page-brute.sls deleted file mode 100644 index 273d349b..00000000 --- a/sift/scripts/page-brute.sls +++ /dev/null @@ -1,15 +0,0 @@ -sift-scripts-page-brute: - file.recurse: - - name: /usr/local/bin - - source: salt://sift/files/page-brute - - file_mode: 755 - - include_pat: '*.py' - -sift-scripts-page-brute-shebang: - file.replace: - - name: /usr/local/bin/page_brute-BETA.py - - pattern: '#!/usr/bin/python\n' - - repl: '#!/usr/bin/env python2\n' - - count: 1 - - watch: - - file: sift-scripts-page-brute From d5d175031898afeffd61b4d535f7f0b30860462b Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:20:20 -0500 Subject: [PATCH 54/64] Update java-idx-parser for 24 (#146) * Update java-idx-parser for 24 * Update java-idx-parser to be pip installable --- sift/python3-packages/init.sls | 2 ++ sift/python3-packages/java-idx-parser.sls | 38 +++++++++++++++++++++++ sift/scripts/init.sls | 2 -- sift/scripts/java-idx-parser.sls | 20 ------------ 4 files changed, 40 insertions(+), 22 deletions(-) create mode 100644 sift/python3-packages/java-idx-parser.sls delete mode 100644 sift/scripts/java-idx-parser.sls diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index 53c795b1..0a8ae230 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -10,6 +10,7 @@ include: - sift.python3-packages.hindsight - sift.python3-packages.ioc-writer ### - sift.python3-packages.imagemounter + - sift.python3-packages.java-idx-parser - sift.python3-packages.keyrings-alt - sift.python3-packages.lxml - sift.python3-packages.mac-apt @@ -49,6 +50,7 @@ sift-python3-packages: - sls: sift.python3-packages.hindsight - sls: sift.python3-packages.ioc-writer ### - sls: sift.python3-packages.imagemounter + - sls: sift.python3-packages.java-idx-parser - sls: sift.python3-packages.keyrings-alt - sls: sift.python3-packages.lxml - sls: sift.python3-packages.mac-apt diff --git a/sift/python3-packages/java-idx-parser.sls b/sift/python3-packages/java-idx-parser.sls new file mode 100644 index 00000000..cd8a62ee --- /dev/null +++ b/sift/python3-packages/java-idx-parser.sls @@ -0,0 +1,38 @@ +# Name: Java_IDX_Parser +# Website: https://github.com/digitalsleuth/Java_IDX_Parser +# Description: Parses Java Cache IDX files +# Category: +# Author: Brian Baskin and Corey Forman (digitalsleuth) +# License: Apache License 2.0 (https://github.com/digitalsleuth/Java_IDX_Parser/blob/master/LICENSE) +# Notes: idx-parser + +include: + - sift.packages.python3-virtualenv + +sift-python3-package-java-idx-parser-venv: + virtualenv.managed: + - name: /opt/java-idx-parser + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-java-idx-parser: + pip.installed: + - name: git+https://github.com/digitalsleuth/Java_IDX_Parser.git + - bin_env: /opt/java-idx-parser/bin/python3 + - replace: True + - mode: 755 + - require: + - virtualenv: sift-python3-package-java-idx-parser-venv + +sift-python3-package-java-idx-parser-symlink: + file.symlink: + - name: /usr/local/bin/idx-parser + - target: /opt/java-idx-parser/bin/idx-parser + - makedirs: False + - require: + - pip: sift-python3-package-java-idx-parser diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index 105e1985..84302a80 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -6,7 +6,6 @@ include: - sift.scripts.dump-mft-entry - sift.scripts.exiftool - sift.scripts.image-mounter - - sift.scripts.java-idx-parser - sift.scripts.jobparser - sift.scripts.keydet-tools - sift.scripts.packerid @@ -36,7 +35,6 @@ sift-scripts: - sls: sift.scripts.dump-mft-entry - sls: sift.scripts.exiftool - sls: sift.scripts.image-mounter - - sls: sift.scripts.java-idx-parser - sls: sift.scripts.jobparser - sls: sift.scripts.keydet-tools - sls: sift.scripts.packerid diff --git a/sift/scripts/java-idx-parser.sls b/sift/scripts/java-idx-parser.sls deleted file mode 100644 index 73fdf18c..00000000 --- a/sift/scripts/java-idx-parser.sls +++ /dev/null @@ -1,20 +0,0 @@ -# source=https://github.com/Rurik/Java_IDX_Parser -# license=apache2.0 -# license_source=https://github.com/Rurik/Java_IDX_Parser#copyright-and-license - -{% set commit = "f9b7a3aeb66a86e891e28d5e762483dff5e15851" -%} -{% set hash = "sha256=963d5f38b93016f147295ab6871dcba326c9315ea9402652745ae6290b594f45" -%} - -sift-scripts-java-idx-parser: - file.managed: - - name: /usr/local/bin/idx_parser.py - - source: https://raw.githubusercontent.com/Rurik/Java_IDX_Parser/{{ commit }}/idx_parser.py - - source_hash: {{ hash }} - - mode: 755 - -sift-scripts-java-idx-parser-shebang: - file.prepend: - - name: /usr/local/bin/idx_parser.py - - text: '#!/usr/bin/env python2' - - watch: - - file: sift-scripts-java-idx-parser From e178a8a6599698c9e487b17e8f02eb1649192e7b Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:21:50 -0500 Subject: [PATCH 55/64] Updates to imagemounter (#128) * Update imagemounter with venv * Update imagemounter for Noble * Update state names --- sift/packages/avfs.sls | 2 +- sift/packages/disktype.sls | 2 +- sift/packages/ewf-tools.sls | 10 ++++ sift/packages/init.sls | 12 ++--- sift/packages/libbde-tools.sls | 8 --- sift/packages/libbde-utils.sls | 3 ++ sift/packages/libewf-dev.sls | 7 +-- sift/packages/libewf-python3.sls | 2 +- sift/packages/libewf.sls | 8 --- sift/packages/libewf2.sls | 3 ++ sift/packages/libvshadow-tools.sls | 9 +--- sift/packages/mtd-utils.sls | 3 ++ sift/packages/ntfs-3g.sls | 2 +- sift/packages/python3-pytsk3.sls | 13 ----- sift/packages/python3-tsk.sls | 9 ++++ sift/packages/sleuthkit.sls | 7 --- sift/packages/squashfs-tools.sls | 3 ++ sift/packages/xmount.sls | 10 +++- sift/python3-packages/imagemounter.sls | 70 +++++++++++++++++++------- 19 files changed, 105 insertions(+), 78 deletions(-) create mode 100644 sift/packages/ewf-tools.sls delete mode 100644 sift/packages/libbde-tools.sls create mode 100644 sift/packages/libbde-utils.sls delete mode 100644 sift/packages/libewf.sls create mode 100644 sift/packages/libewf2.sls create mode 100644 sift/packages/mtd-utils.sls delete mode 100644 sift/packages/python3-pytsk3.sls create mode 100644 sift/packages/python3-tsk.sls create mode 100644 sift/packages/squashfs-tools.sls diff --git a/sift/packages/avfs.sls b/sift/packages/avfs.sls index a22e26be..165ab19b 100644 --- a/sift/packages/avfs.sls +++ b/sift/packages/avfs.sls @@ -1,3 +1,3 @@ -sift-packages-avfs: +sift-package-avfs: pkg.installed: - name: avfs diff --git a/sift/packages/disktype.sls b/sift/packages/disktype.sls index 8cf07bec..a4e59d54 100644 --- a/sift/packages/disktype.sls +++ b/sift/packages/disktype.sls @@ -1,3 +1,3 @@ -sift-packages-disktype: +sift-package-disktype: pkg.installed: - name: disktype diff --git a/sift/packages/ewf-tools.sls b/sift/packages/ewf-tools.sls new file mode 100644 index 00000000..503e2b8b --- /dev/null +++ b/sift/packages/ewf-tools.sls @@ -0,0 +1,10 @@ +include: + - sift.packages.libewf2 + - sift.packages.libewf-dev + +sift-package-ewf-tools: + pkg.installed: + - name: ewf-tools + - require: + - sls: sift.packages.libewf2 + - sls: sift.packages.libewf-dev diff --git a/sift/packages/init.sls b/sift/packages/init.sls index 262ae175..28a5467f 100644 --- a/sift/packages/init.sls +++ b/sift/packages/init.sls @@ -67,7 +67,7 @@ include: - sift.packages.libafflib - sift.packages.libbcprov-java - sift.packages.libbde - - sift.packages.libbde-tools + - sift.packages.libbde-utils - sift.packages.libcommons-lang3-java - sift.packages.libdatetime-perl - sift.packages.libesedb @@ -76,7 +76,7 @@ include: - sift.packages.libevt-tools - sift.packages.libevtx - sift.packages.libevtx-tools - - sift.packages.libewf + - sift.packages.libewf2 - sift.packages.libewf-dev - sift.packages.libewf-python3 - sift.packages.libewf-tools @@ -147,7 +147,7 @@ include: - sift.packages.python3-pefile - sift.packages.python3-pip - sift.packages.python3-pypff - - sift.packages.python3-pytsk3 + - sift.packages.python3-tsk - sift.packages.python3-pyqt5 - sift.packages.python3-redis - sift.packages.python3-tk @@ -271,7 +271,7 @@ sift-packages: - sls: sift.packages.libafflib - sls: sift.packages.libbcprov-java - sls: sift.packages.libbde - - sls: sift.packages.libbde-tools + - sls: sift.packages.libbde-utils - sls: sift.packages.libcommons-lang3-java - sls: sift.packages.libdatetime-perl - sls: sift.packages.libesedb @@ -280,7 +280,7 @@ sift-packages: - sls: sift.packages.libevt-tools - sls: sift.packages.libevtx - sls: sift.packages.libevtx-tools - - sls: sift.packages.libewf + - sls: sift.packages.libewf2 - sls: sift.packages.libewf-dev - sls: sift.packages.libewf-python3 - sls: sift.packages.libewf-tools @@ -351,7 +351,7 @@ sift-packages: - sls: sift.packages.python3-pefile - sls: sift.packages.python3-pip - sls: sift.packages.python3-pypff - - sls: sift.packages.python3-pytsk3 + - sls: sift.packages.python3-tsk - sls: sift.packages.python3-pyqt5 - sls: sift.packages.python3-redis - sls: sift.packages.python3-tk diff --git a/sift/packages/libbde-tools.sls b/sift/packages/libbde-tools.sls deleted file mode 100644 index 1411328a..00000000 --- a/sift/packages/libbde-tools.sls +++ /dev/null @@ -1,8 +0,0 @@ -include: - - sift.repos.gift - -libbde-tools: - pkg.installed: - - name: libbde-tools - - require: - - sls: sift.repos.gift diff --git a/sift/packages/libbde-utils.sls b/sift/packages/libbde-utils.sls new file mode 100644 index 00000000..0c116bfb --- /dev/null +++ b/sift/packages/libbde-utils.sls @@ -0,0 +1,3 @@ +sift-package-libbde-utils: + pkg.installed: + - name: libbde-utils diff --git a/sift/packages/libewf-dev.sls b/sift/packages/libewf-dev.sls index a91fb0a4..fa718d9e 100644 --- a/sift/packages/libewf-dev.sls +++ b/sift/packages/libewf-dev.sls @@ -1,8 +1,3 @@ -include: - - sift.repos.gift - -libewf-dev: +sift-package-libewf-dev: pkg.installed: - name: libewf-dev - - require: - - sls: sift.repos.gift diff --git a/sift/packages/libewf-python3.sls b/sift/packages/libewf-python3.sls index 106e6a2c..8f45467b 100644 --- a/sift/packages/libewf-python3.sls +++ b/sift/packages/libewf-python3.sls @@ -1,7 +1,7 @@ include: - sift.repos.gift -libewf-python: +sift-package-libewf-python3: pkg.installed: - name: libewf-python3 - require: diff --git a/sift/packages/libewf.sls b/sift/packages/libewf.sls deleted file mode 100644 index d20c6783..00000000 --- a/sift/packages/libewf.sls +++ /dev/null @@ -1,8 +0,0 @@ -include: - - sift.repos.gift - -libewf: - pkg.installed: - - name: libewf - - require: - - sls: sift.repos.gift diff --git a/sift/packages/libewf2.sls b/sift/packages/libewf2.sls new file mode 100644 index 00000000..5573ec79 --- /dev/null +++ b/sift/packages/libewf2.sls @@ -0,0 +1,3 @@ +sift-package-libewf2: + pkg.installed: + - name: libewf2 diff --git a/sift/packages/libvshadow-tools.sls b/sift/packages/libvshadow-tools.sls index 80cf329d..00b5ad3a 100644 --- a/sift/packages/libvshadow-tools.sls +++ b/sift/packages/libvshadow-tools.sls @@ -1,8 +1,3 @@ -include: - - sift.repos.gift - -sift-package-libvshadow-tools: +sift-package-libvshadow-utils: pkg.installed: - - name: libvshadow-tools - - require: - - sls: sift.repos.gift + - name: libvshadow-utils diff --git a/sift/packages/mtd-utils.sls b/sift/packages/mtd-utils.sls new file mode 100644 index 00000000..612a3f5c --- /dev/null +++ b/sift/packages/mtd-utils.sls @@ -0,0 +1,3 @@ +sift-package-mtd-utils: + pkg.installed: + - name: mtd-utils diff --git a/sift/packages/ntfs-3g.sls b/sift/packages/ntfs-3g.sls index 40d004cb..1ef605ce 100644 --- a/sift/packages/ntfs-3g.sls +++ b/sift/packages/ntfs-3g.sls @@ -1,3 +1,3 @@ -sift-packages-ntfs-3g: +sift-package-ntfs-3g: pkg.installed: - name: ntfs-3g diff --git a/sift/packages/python3-pytsk3.sls b/sift/packages/python3-pytsk3.sls deleted file mode 100644 index 14e827ee..00000000 --- a/sift/packages/python3-pytsk3.sls +++ /dev/null @@ -1,13 +0,0 @@ -include: - - sift.repos.gift - -sift-package-removed-pytsk3: - pkg.removed: - - name: pytsk3 - -sift-package-python-pytsk3: - pkg.installed: - - name: python3-pytsk3 - - required: - - pkg: sift-package-removed-pytsk3 - - sls: sift.repos.gift diff --git a/sift/packages/python3-tsk.sls b/sift/packages/python3-tsk.sls new file mode 100644 index 00000000..7d367cbc --- /dev/null +++ b/sift/packages/python3-tsk.sls @@ -0,0 +1,9 @@ +sift-package-removed-pytsk3: + pkg.removed: + - name: pytsk3 + +sift-package-python3-tsk: + pkg.installed: + - name: python3-tsk + - require: + - pkg: sift-package-removed-pytsk3 diff --git a/sift/packages/sleuthkit.sls b/sift/packages/sleuthkit.sls index 07d07f16..453e46d0 100644 --- a/sift/packages/sleuthkit.sls +++ b/sift/packages/sleuthkit.sls @@ -1,10 +1,3 @@ -include: - - sift.repos.gift - - sift.repos.sift - sift-package-sleuthkit: pkg.latest: - name: sleuthkit - - require: - - sls: sift.repos.sift - - sls: sift.repos.gift diff --git a/sift/packages/squashfs-tools.sls b/sift/packages/squashfs-tools.sls new file mode 100644 index 00000000..6c772506 --- /dev/null +++ b/sift/packages/squashfs-tools.sls @@ -0,0 +1,3 @@ +sift-package-squashfs-tools: + pkg.installed: + - name: squashfs-tools diff --git a/sift/packages/xmount.sls b/sift/packages/xmount.sls index 1457c5a4..eac9ba90 100644 --- a/sift/packages/xmount.sls +++ b/sift/packages/xmount.sls @@ -1,8 +1,16 @@ +{% if grains['oscodename'] != "noble" %} + include: - sift.repos.ubuntu-universe -sift-packages-xmount: +sift-package-xmount: pkg.latest: - name: xmount - require: - sls: sift.repos.ubuntu-universe + +{% else %} +sift-package-xmount-noble: + pkg.latest: + - name: xmount +{% endif %} diff --git a/sift/python3-packages/imagemounter.sls b/sift/python3-packages/imagemounter.sls index 107c9656..57d24708 100644 --- a/sift/python3-packages/imagemounter.sls +++ b/sift/python3-packages/imagemounter.sls @@ -1,16 +1,21 @@ +# Name: imagemounter +# Website: https://github.com/ralphje/imagemounter +# Description: Command line utility and Python package to mount/unmount forensic disk images +# Category: +# Author: Ralph Broenink +# License: MIT License (https://github.com/ralphje/imagemounter/blob/master/LICENSE) +# Notes: imount + include: - - sift.python3-packages.pip - - sift.python3-packages.python-magic + - sift.packages.python3-virtualenv - sift.packages.afflib-tools - sift.packages.avfs - sift.packages.disktype - - sift.packages.libbde-tools - - sift.packages.libewf - - sift.packages.libewf-dev - - sift.packages.libewf-tools + - sift.packages.libbde-utils + - sift.packages.ewf-tools - sift.packages.libvshadow-tools - sift.packages.ntfs-3g - - sift.packages.python3-pytsk3 + - sift.packages.python3-tsk - sift.packages.qemu-utils - sift.packages.sleuthkit - sift.packages.testdisk @@ -18,24 +23,33 @@ include: - sift.packages.xfsprogs - sift.packages.xmount - sift.packages.libguestfs-tools + - sift.packages.mtd-utils + - sift.packages.squashfs-tools + - sift.packages.git + - sift.packages.build-essential + - sift.packages.python3-dev -sift-python3-packages-imagemounter: - pip.installed: - - name: imagemounter - - bin_env: /usr/bin/python3 +sift-python3-package-imagemounter-venv: + virtualenv.managed: + - name: /opt/imagemounter + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - python-magic>=0.4.27 + - pytsk3>=20231007 - require: - - sls: sift.python3-packages.pip - - sls: sift.python3-packages.python-magic + - sls: sift.packages.python3-virtualenv + - sls: sift.packages.build-essential - sls: sift.packages.afflib-tools - sls: sift.packages.avfs - sls: sift.packages.disktype - - sls: sift.packages.libbde-tools - - sls: sift.packages.libewf - - sls: sift.packages.libewf-dev - - sls: sift.packages.libewf-tools + - sls: sift.packages.libbde-utils + - sls: sift.packages.ewf-tools - sls: sift.packages.libvshadow-tools - sls: sift.packages.ntfs-3g - - sls: sift.packages.python3-pytsk3 + - sls: sift.packages.python3-tsk - sls: sift.packages.qemu-utils - sls: sift.packages.sleuthkit - sls: sift.packages.testdisk @@ -43,3 +57,23 @@ sift-python3-packages-imagemounter: - sls: sift.packages.xfsprogs - sls: sift.packages.xmount - sls: sift.packages.libguestfs-tools + - sls: sift.packages.mtd-utils + - sls: sift.packages.squashfs-tools + - sls: sift.packages.git + - sls: sift.packages.python3-dev + +sift-python3-package-imagemounter: + pip.installed: + - name: imagemounter + - bin_env: /opt/imagemounter/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-imagemounter-venv + +sift-python3-package-imagemounter-symlink: + file.symlink: + - name: /usr/local/bin/imount + - target: /opt/imagemounter/bin/imount + - makedirs: False + - require: + - pip: sift-python3-package-imagemounter From e2a46bc9ea068bc88eaeac1ac36a7208ba4f147d Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:22:14 -0500 Subject: [PATCH 56/64] Update 4n6 scripts (#126) --- sift/scripts/4n6.sls | 131 ++++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 70 deletions(-) diff --git a/sift/scripts/4n6.sls b/sift/scripts/4n6.sls index b635d382..1bf638f1 100644 --- a/sift/scripts/4n6.sls +++ b/sift/scripts/4n6.sls @@ -1,3 +1,11 @@ +# Name: 4n6-scripts +# Website: https://github.com/digitalsleuth/4n6-scripts +# Description: A collection of forensics scripts by Cheeky4N6Monkey, updated for Python 3 +# Category: +# Author: Cheeky4N6Monkey / Corey Forman (digitalsleuth) +# License: GNU General Public License v3+ (https://github.com/digitalsleuth/4n6-scripts/blob/master/README.md) +# Notes: + {% set files = [('Android', ['fbmsg-extractor.py','imgcache-parse-mod.py','imgcache-parse.py','print_apk_perms.py','wwf-chat-parser.py']), ('Ford', ['sync3-unisearch.py','sync3-unisearch2kml.py']), ('Google_Takeout_Records', ['gRecordsActivity_ijson_date.py']), @@ -8,16 +16,9 @@ ('utilities', ['chunkymonkey.py','dextract.def','dextract.py','google-ei-time.py','msoffice-pic-extractor.py','parse_garmin56LM.py','plist2db.py','s2-cellid2latlong.py','s2-latlong2cellid.py','sqlite-base64-decode.py','sqlite-blob-dumper.py','sqlite-parser.pl','squirrelgripper-README.txt','squirrelgripper.pl','timediff32.pl']) ] %} -{% set noshebang = ['sqlite-base64-decode.py','sqlite-blob-dumper.py','wp8-sha256-pin-finder.py'] %} -{% set fixshebangpy2 = ['fbmsg-extractor.py','imgcache-parse-mod.py','imgcache-parse.py','print_apk_perms.py','wwf-chat-parser.py','WP8_AppPerms.py','wp8-1-callhistory.py','wp8-1-contacts.py','wp8-1-mms-filesort.py','wp8-1-mms.py','wp8-1-sms.py','wp8-callhistory.py','wp8-contacts.py','wp8-fb-msg.py','wp8-sms.py','chunkymonkey.py','dextract.py','google-ei-time.py','msoffice-pic-extractor.py','s2-cellid2latlong.py','s2-latlong2cellid.py'] %} -{% set fixshebangpy3 = ['sync3-unisearch.py','sync3-unisearch2kml.py','gLocationHistoryActivity.py','gRecordsActivity_ijson_date.py','java-hashcode.py','samsung_gallery3d_filesysmon_parser_v11.py','samsung_gallery3d_log_parser_v10.py','samsung_gallery3d_log_parser_v11.py','samsung_gallery3d_trash_parser_v10.py'] %} - include: - - sift.python3-packages.ijson - - sift.python-packages.s2sphere + - sift.packages.python3-virtualenv - sift.packages.git - - sift.packages.python2 - - sift.packages.python3 - sift.perl-packages.exiftool - sift.perl-packages.cgi - sift.perl-packages.xpath @@ -26,87 +27,77 @@ include: - sift.perl-packages.dbi - sift.perl-packages.datecalc -sift-scripts-4n6-git: +sift-python3-package-4n6-scripts-venv: + virtualenv.managed: + - name: /opt/4n6-scripts + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - ijson + - s2sphere + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-4n6-scripts-git: git.latest: - - name: https://github.com/cheeky4n6monkey/4n6-scripts.git + - name: https://github.com/digitalsleuth/4n6-scripts.git - target: /usr/local/src/4n6-scripts - user: root - - rev: f57a5301b317a9842c0d43853595161843086923 + - rev: master - force_clone: True - force_reset: True - require: - sls: sift.packages.git - - sls: sift.packages.python2 - - sls: sift.packages.python3 {% for folder, file_list in files %} -{% for file in file_list %} -sift-scripts-4n6-{{ file }}: - file.copy: - - name: /usr/local/bin/{{ file }} - - source: /usr/local/src/4n6-scripts/{{ folder }}/{{ file }} - - force: True - - mode: 755 - - watch: - - git: sift-scripts-4n6-git -{% endfor %} -{% endfor %} +{% for file in file_list %} +{% if file.split('.')[1] == "py" %} -sift-scripts-4n6-gLocationHistoryActivity: +sift-python3-package-4n6-scripts-copy-{{ file }}: file.copy: - - name: /usr/local/bin/gLocationHistoryActivity.py - - source: '/usr/local/src/4n6-scripts/Google_Takeout_Location_History/# gLocationHistoryActivity.py' + - name: /opt/4n6-scripts/bin/{{ file }} + - source: /usr/local/src/4n6-scripts/{{ folder }}/{{ file }} - force: True - mode: 755 - - watch: - - git: sift-scripts-4n6-git + - require: + - git: sift-python3-package-4n6-scripts-git -{% for file in fixshebangpy2 %} -sift-scripts-4n6-python2-{{ file }}: +sift-python3-package-4n6-scripts-shebang-{{ file }}: file.replace: - - name: /usr/local/bin/{{ file }} - - pattern: '#! /usr/bin/env python\n' - - repl: '#! /usr/bin/env python2\n' + - name: /opt/4n6-scripts/bin/{{ file }} + - pattern: '#! /usr/bin/env python3' + - repl: '#!/opt/4n6-scripts/bin/python3' - count: 1 - - watch: - - git: sift-scripts-4n6-git -{% endfor %} + - require: + - file: sift-python3-package-4n6-scripts-copy-{{ file }} -{% for file in fixshebangpy3 %} -sift-scripts-4n6-python3-{{ file }}: - file.replace: +sift-python3-package-4n6-scripts-symlink-{{ file }}: + file.symlink: - name: /usr/local/bin/{{ file }} - - pattern: '#! /usr/bin/env python\n' - - repl: '#! /usr/bin/env python3\n' - - count: 1 - - watch: - - git: sift-scripts-4n6-git -{% endfor %} + - target: /opt/4n6-scripts/bin/{{ file }} + - makedirs: False + - require: + - file: sift-python3-package-4n6-scripts-shebang-{{ file }} -{% for file in fixshebangpy3 %} -sift-scripts-4n6-python3-CRLF{{ file }}: - file.replace: - - name: /usr/local/bin/{{ file }} - - pattern: '#! /usr/bin/env python\r' - - repl: '#! /usr/bin/env python3\n' - - count: 1 - - watch: - - git: sift-scripts-4n6-git -{% endfor %} +sift-python3-package-4n6-scripts-remove-{{ file }}-bak: + file.absent: + - name: /opt/4n6-scripts/bin/{{ file }}.bak + - require: + - file: sift-python3-package-4n6-scripts-symlink-{{ file }} -{%- for file in noshebang %} -sift-scripts-4n6-add-shebang-{{ file }}: - file.prepend: - - name: /usr/local/bin/{{ file }} - - text: '#!/usr/bin/env python2' - - watch: - - git: sift-scripts-4n6-git -{% endfor %} +{% else %} -sift-scripts-4n6-plistdb2py-shebang: - file.prepend: - - name: /usr/local/bin/plist2db.py - - text: '#!/usr/bin/env python3' - - watch: - - git: sift-scripts-4n6-git +sift-python3-package-4n6-scripts-copy-{{ file }}: + file.copy: + - name: /usr/local/bin/{{ file }} + - source: /usr/local/src/4n6-scripts/{{ folder }}/{{ file }} + - force: True + - mode: 755 + - require: + - git: sift-python3-package-4n6-scripts-git +{% endif %} +{% endfor %} +{% endfor %} From 4a7ea250dc00a5a106bff3fa43f1ac3a8859b496 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:08:40 -0500 Subject: [PATCH 57/64] Indxparse 24 (#144) * Update indxparse for 24 * Add the commands to the Notes section of the header --- sift/packages/libgtk-3-dev.sls | 3 ++ sift/packages/python3-wxgtk4.sls | 3 ++ sift/python-packages/indxparse.sls | 41 ---------------- sift/python-packages/init.sls | 4 +- sift/python3-packages/indxparse.sls | 72 +++++++++++++++++++++++++++++ sift/python3-packages/init.sls | 2 + 6 files changed, 82 insertions(+), 43 deletions(-) create mode 100644 sift/packages/libgtk-3-dev.sls create mode 100644 sift/packages/python3-wxgtk4.sls delete mode 100644 sift/python-packages/indxparse.sls create mode 100644 sift/python3-packages/indxparse.sls diff --git a/sift/packages/libgtk-3-dev.sls b/sift/packages/libgtk-3-dev.sls new file mode 100644 index 00000000..2a66b755 --- /dev/null +++ b/sift/packages/libgtk-3-dev.sls @@ -0,0 +1,3 @@ +sift-package-libgtk-3-dev: + pkg.installed: + - name: libgtk-3-dev diff --git a/sift/packages/python3-wxgtk4.sls b/sift/packages/python3-wxgtk4.sls new file mode 100644 index 00000000..ff1c9191 --- /dev/null +++ b/sift/packages/python3-wxgtk4.sls @@ -0,0 +1,3 @@ +sift-package-python3-wxgtk4: + pkg.installed: + - name: python3-wxgtk4.0 diff --git a/sift/python-packages/indxparse.sls b/sift/python-packages/indxparse.sls deleted file mode 100644 index b02c1f69..00000000 --- a/sift/python-packages/indxparse.sls +++ /dev/null @@ -1,41 +0,0 @@ -{% if grains['oscodename'] != "jammy" %} - -{%- set user = salt['pillar.get']('sift_user', 'sansforensics') -%} -{%- set commit = "ca08236b0f70798cb6f89785820c9b82ee0c66ff" -%} - -include: - - sift.packages.git - - sift.packages.g++ - - sift.packages.libfuse-dev - - sift.packages.pkg-config - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.packages.python2-dev - - sift.packages.python-wxgtk3 - -sift-python-packages-indxparse: - pip.installed: - - name: git+https://github.com/williballenthin/INDXParse.git@{{ commit }} - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.git - - sls: sift.packages.g++ - - sls: sift.packages.pkg-config - - sls: sift.packages.python2-pip - - sls: sift.packages.python2-dev - - sls: sift.packages.libfuse-dev - - sls: sift.packages.python-wxgtk3 - -sift-python-packages-indxparse-shebang: - file.prepend: - - name: /usr/local/bin/INDXParse.py - - text: '#!/usr/bin/env python2' - - watch: - - pip: sift-python-packages-indxparse - -{% else %} -python-wxgtk3-not-in-jammy: - test.nop - -{% endif %} diff --git a/sift/python-packages/init.sls b/sift/python-packages/init.sls index b7acdc9b..7a1bba55 100644 --- a/sift/python-packages/init.sls +++ b/sift/python-packages/init.sls @@ -7,7 +7,7 @@ include: - sift.python-packages.distorm3 - sift.python-packages.docopt - sift.python-packages.geoip2 - - sift.python-packages.indxparse + - sift.python-packages.ioc_writer - sift.python-packages.lxml - sift.python-packages.ntdsxtract - sift.python-packages.pefile @@ -36,7 +36,7 @@ sift-python-packages: - sls: sift.python-packages.distorm3 - sls: sift.python-packages.docopt - sls: sift.python-packages.geoip2 - - sls: sift.python-packages.indxparse + - sls: sift.python-packages.ioc_writer - sls: sift.python-packages.lxml - sls: sift.python-packages.ntdsxtract - sls: sift.python-packages.pefile diff --git a/sift/python3-packages/indxparse.sls b/sift/python3-packages/indxparse.sls new file mode 100644 index 00000000..547288fb --- /dev/null +++ b/sift/python3-packages/indxparse.sls @@ -0,0 +1,72 @@ +# Name: indxparse +# Website: https://github.com/williballenthin/INDXParse +# Description: Tool suite for inspecting NTFS artifacts +# Category: +# Author: Willi Ballenthin +# License: Apache License 2.0 (https://github.com/williballenthin/INDXParse/blob/master/LICENSE) +# Notes: INDXParse.py, MFTINDX.py, MFTView.py, SDS_get_index.py, extract_mft_record_slack.py, get_file_info.py, list_mft.py, fuse-mft.py, tree_mft.py + +{% if grains['oscodename'] == 'jammy' %} + {% set py_ver = '3.10' %} +{% elif grains['oscodename'] == 'noble' %} + {% set py_ver = '3.12' %} +{% endif %} +{%- set commit = "038e8ec836cf23600124db74b40757b7184c08c5" -%} +{% set files = ['INDXParse.py','MFTINDX.py','MFTView.py','SDS_get_index.py','extract_mft_record_slack.py','get_file_info.py','list_mft.py','fuse-mft.py','tree_mft.py'] %} + +include: + - sift.packages.python3-virtualenv + - sift.packages.git + - sift.packages.python3-dev + - sift.packages.libgtk-3-dev + - sift.packages.pkg-config + - sift.packages.build-essential + - sift.packages.libfuse-dev + - sift.packages.python3-wxgtk4 + +sift-python3-package-indxparse-venv: + virtualenv.managed: + - name: /opt/indxparse + - venv_bin: /usr/bin/virtualenv + - system_site_packages: True + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - importlib_metadata>=8.0.0 + - fusepy + - require: + - sls: sift.packages.python3-virtualenv + - sls: sift.packages.python3-dev + - sls: sift.packages.libgtk-3-dev + - sls: sift.packages.pkg-config + - sls: sift.packages.build-essential + - sls: sift.packages.libfuse-dev + - sls: sift.packages.python3-wxgtk4 + +sift-python3-package-indxparse: + pip.installed: + - name: git+https://github.com/williballenthin/INDXParse.git@{{ commit }} + - bin_env: /opt/indxparse/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-indxparse-venv + - sls: sift.packages.git + +{% for file in files %} +sift-python3-package-indxparse-symlink-{{ file }}: + file.symlink: + - name: /usr/local/bin/{{ file }} + - target: /opt/indxparse/bin/{{ file }} + - makedirs: False + - require: + - pip: sift-python3-package-indxparse +{% endfor %} + +sift-python3-package-indxparse-rename-fuse-mft: + file.managed: + - name: /opt/indxparse/lib/python{{ py_ver }}/site-packages/indxparse/fuse_mft.py + - source: /opt/indxparse/lib/python{{ py_ver }}/site-packages/indxparse/fuse-mft.py + - force: True + - require: + - pip: sift-python3-package-indxparse diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index 0a8ae230..e847da8d 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -10,6 +10,7 @@ include: - sift.python3-packages.hindsight - sift.python3-packages.ioc-writer ### - sift.python3-packages.imagemounter + - sift.python3-packages.indxparse - sift.python3-packages.java-idx-parser - sift.python3-packages.keyrings-alt - sift.python3-packages.lxml @@ -50,6 +51,7 @@ sift-python3-packages: - sls: sift.python3-packages.hindsight - sls: sift.python3-packages.ioc-writer ### - sls: sift.python3-packages.imagemounter + - sls: sift.python3-packages.indxparse - sls: sift.python3-packages.java-idx-parser - sls: sift.python3-packages.keyrings-alt - sls: sift.python3-packages.lxml From 0fbb103f260dd6e2c3a81317b0cacb9c98d5fe32 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 19 Feb 2025 09:58:45 -0500 Subject: [PATCH 58/64] Fix an issue with imagemounter installation (#177) --- sift/packages/file.sls | 3 +++ sift/packages/init.sls | 4 ---- sift/packages/libbde-tools.sls | 8 ++++++++ sift/packages/libbde-utils.sls | 3 --- sift/packages/libbde.sls | 2 +- sift/packages/libewf-dev.sls | 5 +++++ sift/packages/libewf-tools.sls | 2 +- sift/packages/libewf2.sls | 9 +++++++-- sift/packages/libvhdi.sls | 8 ++++++++ sift/packages/libvshadow-tools.sls | 9 +++++++-- sift/packages/lvm2.sls | 3 +++ sift/packages/mdadm.sls | 3 +++ sift/packages/python3-pytsk3.sls | 8 ++++++++ sift/packages/python3-tsk.sls | 4 ++++ sift/python3-packages/imagemounter.sls | 24 ++++++++++++++---------- sift/python3-packages/init.sls | 16 ++++++++-------- 16 files changed, 80 insertions(+), 31 deletions(-) create mode 100644 sift/packages/file.sls create mode 100644 sift/packages/libbde-tools.sls delete mode 100644 sift/packages/libbde-utils.sls create mode 100644 sift/packages/libvhdi.sls create mode 100644 sift/packages/lvm2.sls create mode 100644 sift/packages/mdadm.sls create mode 100644 sift/packages/python3-pytsk3.sls diff --git a/sift/packages/file.sls b/sift/packages/file.sls new file mode 100644 index 00000000..b7769b39 --- /dev/null +++ b/sift/packages/file.sls @@ -0,0 +1,3 @@ +sift-package-file: + pkg.installed: + - name: file diff --git a/sift/packages/init.sls b/sift/packages/init.sls index 28a5467f..5c112234 100644 --- a/sift/packages/init.sls +++ b/sift/packages/init.sls @@ -67,7 +67,6 @@ include: - sift.packages.libafflib - sift.packages.libbcprov-java - sift.packages.libbde - - sift.packages.libbde-utils - sift.packages.libcommons-lang3-java - sift.packages.libdatetime-perl - sift.packages.libesedb @@ -147,7 +146,6 @@ include: - sift.packages.python3-pefile - sift.packages.python3-pip - sift.packages.python3-pypff - - sift.packages.python3-tsk - sift.packages.python3-pyqt5 - sift.packages.python3-redis - sift.packages.python3-tk @@ -271,7 +269,6 @@ sift-packages: - sls: sift.packages.libafflib - sls: sift.packages.libbcprov-java - sls: sift.packages.libbde - - sls: sift.packages.libbde-utils - sls: sift.packages.libcommons-lang3-java - sls: sift.packages.libdatetime-perl - sls: sift.packages.libesedb @@ -351,7 +348,6 @@ sift-packages: - sls: sift.packages.python3-pefile - sls: sift.packages.python3-pip - sls: sift.packages.python3-pypff - - sls: sift.packages.python3-tsk - sls: sift.packages.python3-pyqt5 - sls: sift.packages.python3-redis - sls: sift.packages.python3-tk diff --git a/sift/packages/libbde-tools.sls b/sift/packages/libbde-tools.sls new file mode 100644 index 00000000..add31ad2 --- /dev/null +++ b/sift/packages/libbde-tools.sls @@ -0,0 +1,8 @@ +include: + - sift.repos.gift + +sift-package-libbde-tools: + pkg.installed: + - name: libbde-tools + - require: + - sls: sift.repos.gift diff --git a/sift/packages/libbde-utils.sls b/sift/packages/libbde-utils.sls deleted file mode 100644 index 0c116bfb..00000000 --- a/sift/packages/libbde-utils.sls +++ /dev/null @@ -1,3 +0,0 @@ -sift-package-libbde-utils: - pkg.installed: - - name: libbde-utils diff --git a/sift/packages/libbde.sls b/sift/packages/libbde.sls index 583bd476..d50c3e49 100644 --- a/sift/packages/libbde.sls +++ b/sift/packages/libbde.sls @@ -1,7 +1,7 @@ include: - sift.repos.gift -libbde: +sift-package-libbde: pkg.installed: - name: libbde - require: diff --git a/sift/packages/libewf-dev.sls b/sift/packages/libewf-dev.sls index fa718d9e..eb073c5a 100644 --- a/sift/packages/libewf-dev.sls +++ b/sift/packages/libewf-dev.sls @@ -1,3 +1,8 @@ +include: + - sift.repos.gift + sift-package-libewf-dev: pkg.installed: - name: libewf-dev + - require: + - sls: sift.repos.gift diff --git a/sift/packages/libewf-tools.sls b/sift/packages/libewf-tools.sls index 19a6deab..a7e6bb38 100644 --- a/sift/packages/libewf-tools.sls +++ b/sift/packages/libewf-tools.sls @@ -1,7 +1,7 @@ include: - sift.repos.gift -libewf-tools: +sift-package-libewf-tools: pkg.installed: - name: libewf-tools - require: diff --git a/sift/packages/libewf2.sls b/sift/packages/libewf2.sls index 5573ec79..b6785e44 100644 --- a/sift/packages/libewf2.sls +++ b/sift/packages/libewf2.sls @@ -1,3 +1,8 @@ -sift-package-libewf2: +include: + - sift.repos.gift + +sift-package-libewf: pkg.installed: - - name: libewf2 + - name: libewf + - require: + - sls: sift.repos.gift diff --git a/sift/packages/libvhdi.sls b/sift/packages/libvhdi.sls new file mode 100644 index 00000000..bec95e54 --- /dev/null +++ b/sift/packages/libvhdi.sls @@ -0,0 +1,8 @@ +include: + - sift.repos.gift + +sift-package-libvhdi: + pkg.installed: + - name: libvhdi + - require: + - sls: sift.repos.gift diff --git a/sift/packages/libvshadow-tools.sls b/sift/packages/libvshadow-tools.sls index 00b5ad3a..80cf329d 100644 --- a/sift/packages/libvshadow-tools.sls +++ b/sift/packages/libvshadow-tools.sls @@ -1,3 +1,8 @@ -sift-package-libvshadow-utils: +include: + - sift.repos.gift + +sift-package-libvshadow-tools: pkg.installed: - - name: libvshadow-utils + - name: libvshadow-tools + - require: + - sls: sift.repos.gift diff --git a/sift/packages/lvm2.sls b/sift/packages/lvm2.sls new file mode 100644 index 00000000..a1d5ad5c --- /dev/null +++ b/sift/packages/lvm2.sls @@ -0,0 +1,3 @@ +sift-package-lvm2: + pkg.installed: + - name: lvm2 diff --git a/sift/packages/mdadm.sls b/sift/packages/mdadm.sls new file mode 100644 index 00000000..4c632d65 --- /dev/null +++ b/sift/packages/mdadm.sls @@ -0,0 +1,3 @@ +sift-package-mdadm: + pkg.installed: + - name: mdadm diff --git a/sift/packages/python3-pytsk3.sls b/sift/packages/python3-pytsk3.sls new file mode 100644 index 00000000..aaeeb2b5 --- /dev/null +++ b/sift/packages/python3-pytsk3.sls @@ -0,0 +1,8 @@ +include: + - sift.repos.gift + +sift-package-python3-pytsk3: + pkg.installed: + - name: python3-pytsk3 + - require: + - sls: sift.repos.gift diff --git a/sift/packages/python3-tsk.sls b/sift/packages/python3-tsk.sls index 7d367cbc..ed44b968 100644 --- a/sift/packages/python3-tsk.sls +++ b/sift/packages/python3-tsk.sls @@ -1,3 +1,6 @@ +include: + - sift.repos.gift + sift-package-removed-pytsk3: pkg.removed: - name: pytsk3 @@ -7,3 +10,4 @@ sift-package-python3-tsk: - name: python3-tsk - require: - pkg: sift-package-removed-pytsk3 + - sls: sift.repos.gift diff --git a/sift/python3-packages/imagemounter.sls b/sift/python3-packages/imagemounter.sls index 57d24708..4225997f 100644 --- a/sift/python3-packages/imagemounter.sls +++ b/sift/python3-packages/imagemounter.sls @@ -11,23 +11,25 @@ include: - sift.packages.afflib-tools - sift.packages.avfs - sift.packages.disktype - - sift.packages.libbde-utils - - sift.packages.ewf-tools + - sift.packages.file + - sift.packages.libbde + - sift.packages.libbde-tools + - sift.packages.libewf-tools - sift.packages.libvshadow-tools + - sift.packages.lvm2 + - sift.packages.mdadm - sift.packages.ntfs-3g - - sift.packages.python3-tsk - sift.packages.qemu-utils - sift.packages.sleuthkit - sift.packages.testdisk - sift.packages.vmfs-tools - sift.packages.xfsprogs - - sift.packages.xmount - - sift.packages.libguestfs-tools - sift.packages.mtd-utils - sift.packages.squashfs-tools - sift.packages.git - sift.packages.build-essential - sift.packages.python3-dev + - sift.packages.python3-pytsk3 sift-python3-package-imagemounter-venv: virtualenv.managed: @@ -45,22 +47,24 @@ sift-python3-package-imagemounter-venv: - sls: sift.packages.afflib-tools - sls: sift.packages.avfs - sls: sift.packages.disktype - - sls: sift.packages.libbde-utils - - sls: sift.packages.ewf-tools + - sls: sift.packages.file + - sls: sift.packages.libbde + - sls: sift.packages.libbde-tools + - sls: sift.packages.libewf-tools - sls: sift.packages.libvshadow-tools + - sls: sift.packages.lvm2 + - sls: sift.packages.mdadm - sls: sift.packages.ntfs-3g - - sls: sift.packages.python3-tsk - sls: sift.packages.qemu-utils - sls: sift.packages.sleuthkit - sls: sift.packages.testdisk - sls: sift.packages.vmfs-tools - sls: sift.packages.xfsprogs - - sls: sift.packages.xmount - - sls: sift.packages.libguestfs-tools - sls: sift.packages.mtd-utils - sls: sift.packages.squashfs-tools - sls: sift.packages.git - sls: sift.packages.python3-dev + - sls: sift.packages.python3-pytsk3 sift-python3-package-imagemounter: pip.installed: diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index e847da8d..9e06ce75 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -9,7 +9,7 @@ include: - sift.python3-packages.geoip2 - sift.python3-packages.hindsight - sift.python3-packages.ioc-writer -### - sift.python3-packages.imagemounter + - sift.python3-packages.imagemounter - sift.python3-packages.indxparse - sift.python3-packages.java-idx-parser - sift.python3-packages.keyrings-alt @@ -24,9 +24,6 @@ include: - sift.python3-packages.python-dateutil - sift.python3-packages.python-evtx - sift.python3-packages.python-magic - - sift.python3-packages.setuptools - - sift.python3-packages.setuptools-rust - - sift.python3-packages.six - sift.python3-packages.sqlite-carver - sift.python3-packages.stix-validator - sift.python3-packages.stix @@ -35,6 +32,9 @@ include: - sift.python3-packages.virustotal-api - sift.python3-packages.wheel - sift.python3-packages.yara-python + - sift.python3-packages.setuptools + - sift.python3-packages.setuptools-rust + - sift.python3-packages.six sift-python3-packages: test.nop: @@ -50,7 +50,7 @@ sift-python3-packages: - sls: sift.python3-packages.geoip2 - sls: sift.python3-packages.hindsight - sls: sift.python3-packages.ioc-writer -### - sls: sift.python3-packages.imagemounter + - sls: sift.python3-packages.imagemounter - sls: sift.python3-packages.indxparse - sls: sift.python3-packages.java-idx-parser - sls: sift.python3-packages.keyrings-alt @@ -65,9 +65,6 @@ sift-python3-packages: - sls: sift.python3-packages.python-dateutil - sls: sift.python3-packages.python-evtx - sls: sift.python3-packages.python-magic - - sls: sift.python3-packages.setuptools - - sls: sift.python3-packages.setuptools-rust - - sls: sift.python3-packages.six - sls: sift.python3-packages.sqlite-carver - sls: sift.python3-packages.stix-validator - sls: sift.python3-packages.stix @@ -76,3 +73,6 @@ sift-python3-packages: - sls: sift.python3-packages.virustotal-api - sls: sift.python3-packages.wheel - sls: sift.python3-packages.yara-python + - sls: sift.python3-packages.setuptools + - sls: sift.python3-packages.setuptools-rust + - sls: sift.python3-packages.six From 7c9ce8370fca57c458c4892325f2c661969afb6f Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 19 Feb 2025 09:59:00 -0500 Subject: [PATCH 59/64] Add Linux at package (#178) --- sift/packages/at.sls | 3 +++ sift/packages/init.sls | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 sift/packages/at.sls diff --git a/sift/packages/at.sls b/sift/packages/at.sls new file mode 100644 index 00000000..d250009d --- /dev/null +++ b/sift/packages/at.sls @@ -0,0 +1,3 @@ +sift-package-at: + pkg.installed: + - name: at diff --git a/sift/packages/init.sls b/sift/packages/init.sls index 5c112234..d5a25f0a 100644 --- a/sift/packages/init.sls +++ b/sift/packages/init.sls @@ -5,6 +5,7 @@ include: - sift.packages.aircrack-ng - sift.packages.apache2 - sift.packages.arp-scan + - sift.packages.at - sift.packages.autopsy - sift.packages.aws-cli - sift.packages.avfs @@ -207,6 +208,7 @@ sift-packages: - sls: sift.packages.aircrack-ng - sls: sift.packages.apache2 - sls: sift.packages.arp-scan + - sls: sift.packages.at - sls: sift.packages.autopsy - sls: sift.packages.aws-cli - sls: sift.packages.avfs From 94b5575d62f30e6f4cb18675a325e7c3f7061cdf Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Wed, 19 Feb 2025 09:59:48 -0500 Subject: [PATCH 60/64] Cleanup of Python 2 states for 24 (#175) --- sift/python-packages/appcompatprocessor.sls | 19 ------- sift/python-packages/argparse.sls | 11 ---- sift/python-packages/bitstring.sls | 11 ---- sift/python-packages/capstone.sls | 16 ------ sift/python-packages/geoip2.sls | 11 ---- sift/python-packages/init.sls | 46 +-------------- .../{docopt.sls => ioc_writer.sls} | 6 +- sift/python-packages/ntdsxtract.sls | 13 ----- sift/python-packages/poster.sls | 11 ---- sift/python-packages/python-dateutil.sls | 10 ---- sift/python-packages/python-magic.sls | 11 ---- sift/python-packages/python-registry.sls | 11 ---- sift/python-packages/s2sphere.sls | 11 ---- sift/python-packages/shellbags.sls | 15 ----- sift/python-packages/six.sls | 11 ---- sift/python-packages/unicodecsv.sls | 11 ---- sift/scripts/image-mounter.sls | 22 -------- sift/scripts/init.sls | 10 ---- sift/scripts/jobparser.sls | 21 ------- sift/scripts/shim-cache-parser.sls | 25 --------- sift/scripts/virustotal-tools.sls | 56 ------------------- sift/scripts/vshot.sls | 35 ------------ 22 files changed, 6 insertions(+), 387 deletions(-) delete mode 100644 sift/python-packages/appcompatprocessor.sls delete mode 100644 sift/python-packages/argparse.sls delete mode 100644 sift/python-packages/bitstring.sls delete mode 100644 sift/python-packages/capstone.sls delete mode 100644 sift/python-packages/geoip2.sls rename sift/python-packages/{docopt.sls => ioc_writer.sls} (60%) delete mode 100644 sift/python-packages/ntdsxtract.sls delete mode 100644 sift/python-packages/poster.sls delete mode 100644 sift/python-packages/python-dateutil.sls delete mode 100644 sift/python-packages/python-magic.sls delete mode 100644 sift/python-packages/python-registry.sls delete mode 100644 sift/python-packages/s2sphere.sls delete mode 100644 sift/python-packages/shellbags.sls delete mode 100644 sift/python-packages/six.sls delete mode 100644 sift/python-packages/unicodecsv.sls delete mode 100644 sift/scripts/image-mounter.sls delete mode 100644 sift/scripts/jobparser.sls delete mode 100644 sift/scripts/shim-cache-parser.sls delete mode 100644 sift/scripts/virustotal-tools.sls delete mode 100644 sift/scripts/vshot.sls diff --git a/sift/python-packages/appcompatprocessor.sls b/sift/python-packages/appcompatprocessor.sls deleted file mode 100644 index b81a040c..00000000 --- a/sift/python-packages/appcompatprocessor.sls +++ /dev/null @@ -1,19 +0,0 @@ -{%- set commit="6c847937c5a836e2ce2fe2b915f213c345a3c389" -%} - -include: - - sift.packages.git - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.packages.libregf - - sift.packages.python2-dev - -appcompatprocessor: - pip.installed: - - name: git+https://github.com/mbevilacqua/appcompatprocessor.git@{{ commit }} - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.git - - sls: sift.packages.python2-pip - - sls: sift.packages.libregf - - sls: sift.packages.python2-dev diff --git a/sift/python-packages/argparse.sls b/sift/python-packages/argparse.sls deleted file mode 100644 index b5bd02d0..00000000 --- a/sift/python-packages/argparse.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-argparse: - pip.installed: - - name: argparse - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/bitstring.sls b/sift/python-packages/bitstring.sls deleted file mode 100644 index 878d939b..00000000 --- a/sift/python-packages/bitstring.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-bitstring: - pip.installed: - - name: bitstring - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/capstone.sls b/sift/python-packages/capstone.sls deleted file mode 100644 index 6ead9242..00000000 --- a/sift/python-packages/capstone.sls +++ /dev/null @@ -1,16 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-pkg-remove: - pkg.removed: - - name: python-capstone - -sift-python-packages-capstone: - pip.installed: - - name: capstone - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip - - pkg: sift-python-packages-pkg-remove diff --git a/sift/python-packages/geoip2.sls b/sift/python-packages/geoip2.sls deleted file mode 100644 index 501d06ed..00000000 --- a/sift/python-packages/geoip2.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-geoip2: - pip.installed: - - name: geoip2 - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/init.sls b/sift/python-packages/init.sls index 7a1bba55..631d7a9a 100644 --- a/sift/python-packages/init.sls +++ b/sift/python-packages/init.sls @@ -1,54 +1,12 @@ include: - - sift.python-packages.appcompatprocessor - - sift.python-packages.argparse - - sift.python-packages.bitstring - - sift.python-packages.colorama - - sift.python-packages.construct - - sift.python-packages.distorm3 - - sift.python-packages.docopt - - sift.python-packages.geoip2 - - sift.python-packages.ioc_writer - - sift.python-packages.lxml - - sift.python-packages.ntdsxtract - - sift.python-packages.pefile - - sift.python-packages.pillow - - sift.python-packages.poster - - sift.python-packages.pysocks - - sift.python-packages.python-dateutil - - sift.python-packages.python-magic - - sift.python-packages.python-registry - sift.python-packages.setuptools - - sift.python-packages.shellbags - - sift.python-packages.six - - sift.python-packages.unicodecsv - - sift.python-packages.volatility - sift.python-packages.wheel + - sift.python-packages.volatility sift-python-packages: test.nop: - name: sift-python-packages - require: - - sls: sift.python-packages.appcompatprocessor - - sls: sift.python-packages.argparse - - sls: sift.python-packages.bitstring - - sls: sift.python-packages.colorama - - sls: sift.python-packages.construct - - sls: sift.python-packages.distorm3 - - sls: sift.python-packages.docopt - - sls: sift.python-packages.geoip2 - - sls: sift.python-packages.ioc_writer - - sls: sift.python-packages.lxml - - sls: sift.python-packages.ntdsxtract - - sls: sift.python-packages.pefile - - sls: sift.python-packages.pillow - - sls: sift.python-packages.poster - - sls: sift.python-packages.pysocks - - sls: sift.python-packages.python-dateutil - - sls: sift.python-packages.python-magic - - sls: sift.python-packages.python-registry - sls: sift.python-packages.setuptools - - sls: sift.python-packages.shellbags - - sls: sift.python-packages.six - - sls: sift.python-packages.unicodecsv - - sls: sift.python-packages.volatility - sls: sift.python-packages.wheel + - sls: sift.python-packages.volatility diff --git a/sift/python-packages/docopt.sls b/sift/python-packages/ioc_writer.sls similarity index 60% rename from sift/python-packages/docopt.sls rename to sift/python-packages/ioc_writer.sls index 745b1cc3..decb3b3d 100644 --- a/sift/python-packages/docopt.sls +++ b/sift/python-packages/ioc_writer.sls @@ -1,11 +1,13 @@ include: - sift.packages.python3-pip - sift.packages.python2-pip + - sift.python-packages.lxml -sift-python-packages-docopt: +sift-python-packages-ioc-writer: pip.installed: - - name: docopt + - name: ioc_writer - bin_env: /usr/bin/python2 - upgrade: True - require: - sls: sift.packages.python2-pip + - sls: sift.python-packages.lxml diff --git a/sift/python-packages/ntdsxtract.sls b/sift/python-packages/ntdsxtract.sls deleted file mode 100644 index df847052..00000000 --- a/sift/python-packages/ntdsxtract.sls +++ /dev/null @@ -1,13 +0,0 @@ -include: - - sift.packages.git - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-ntdsxtract: - pip.installed: - - name: git+https://github.com/csababarta/ntdsxtract.git@7fa1c8c28cbbf97a42bef40f20009dba85e4c25f - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.git - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/poster.sls b/sift/python-packages/poster.sls deleted file mode 100644 index edb35706..00000000 --- a/sift/python-packages/poster.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-poster: - pip.installed: - - name: poster - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/python-dateutil.sls b/sift/python-packages/python-dateutil.sls deleted file mode 100644 index aec61491..00000000 --- a/sift/python-packages/python-dateutil.sls +++ /dev/null @@ -1,10 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-python-dateutil: - pip.installed: - - name: python-dateutil >= 2.4.2 - - bin_env: /usr/bin/python2 - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/python-magic.sls b/sift/python-packages/python-magic.sls deleted file mode 100644 index 01bf4af0..00000000 --- a/sift/python-packages/python-magic.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-python-magic: - pip.installed: - - name: python-magic - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/python-registry.sls b/sift/python-packages/python-registry.sls deleted file mode 100644 index a68ed743..00000000 --- a/sift/python-packages/python-registry.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-python-registry: - pip.installed: - - name: python-registry - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/s2sphere.sls b/sift/python-packages/s2sphere.sls deleted file mode 100644 index b1f3ba7a..00000000 --- a/sift/python-packages/s2sphere.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-s2sphere: - pip.installed: - - name: s2sphere - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/shellbags.sls b/sift/python-packages/shellbags.sls deleted file mode 100644 index dea62f42..00000000 --- a/sift/python-packages/shellbags.sls +++ /dev/null @@ -1,15 +0,0 @@ -{%- set user = salt['pillar.get']('sift_user', 'sansforensics') -%} - -include: - - sift.packages.git - - sift.packages.python3-pip - - sift.packages.python2-pip - -shellbags: - pip.installed: - - name: git+https://github.com/williballenthin/shellbags.git@fee76eb - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.git - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/six.sls b/sift/python-packages/six.sls deleted file mode 100644 index 1a095ac8..00000000 --- a/sift/python-packages/six.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-six: - pip.installed: - - name: six - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/unicodecsv.sls b/sift/python-packages/unicodecsv.sls deleted file mode 100644 index 686df4ac..00000000 --- a/sift/python-packages/unicodecsv.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-unicodecsv: - pip.installed: - - name: unicodecsv - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/scripts/image-mounter.sls b/sift/scripts/image-mounter.sls deleted file mode 100644 index 20709190..00000000 --- a/sift/scripts/image-mounter.sls +++ /dev/null @@ -1,22 +0,0 @@ -# source=https://github.com/kevthehermit/Scripts -# license=unknown - -{% set commit = "28b3e08a5ad16576ffe487691376f3e2a2bc0cf5" -%} -{% set hash = "sha256=7e810482b5aa58f8085a7a03be266c113530145306c73c75ba9956ba83e39151" -%} - -sift-scripts-image-mounter: - file.managed: - - name: /usr/local/bin/imageMounter.py - - source: https://raw.githubusercontent.com/kevthehermit/Scripts/{{ commit }}/imageMounter.py - - source_hash: {{ hash }} - - mode: 755 - -sift-scripts-image-mounter-shebang: - file.replace: - - name: /usr/local/bin/imageMounter.py - - pattern: '#!/usr/bin/env python\n' - - repl: '#!/usr/bin/env python2\n' - - count: 1 - - prepend_if_not_found: False - - watch: - - file: sift-scripts-image-mounter diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index 84302a80..ab51362d 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -5,8 +5,6 @@ include: - sift.scripts.densityscout - sift.scripts.dump-mft-entry - sift.scripts.exiftool - - sift.scripts.image-mounter - - sift.scripts.jobparser - sift.scripts.keydet-tools - sift.scripts.packerid - sift.scripts.parseusn @@ -15,13 +13,10 @@ include: - sift.scripts.pe-carver - sift.scripts.regripper - sift.scripts.screen-scale - - sift.scripts.shim-cache-parser - sift.scripts.sift - sift.scripts.sorter - sift.scripts.usbdeviceforensics - sift.scripts.sqlparser - - sift.scripts.virustotal-tools - - sift.scripts.vshot - sift.scripts.zimmerman sift-scripts: @@ -34,8 +29,6 @@ sift-scripts: - sls: sift.scripts.densityscout - sls: sift.scripts.dump-mft-entry - sls: sift.scripts.exiftool - - sls: sift.scripts.image-mounter - - sls: sift.scripts.jobparser - sls: sift.scripts.keydet-tools - sls: sift.scripts.packerid - sls: sift.scripts.parseusn @@ -44,11 +37,8 @@ sift-scripts: - sls: sift.scripts.pe-carver - sls: sift.scripts.regripper - sls: sift.scripts.screen-scale - - sls: sift.scripts.shim-cache-parser - sls: sift.scripts.sift - sls: sift.scripts.sorter - sls: sift.scripts.usbdeviceforensics - sls: sift.scripts.sqlparser - - sls: sift.scripts.virustotal-tools - - sls: sift.scripts.vshot - sls: sift.scripts.zimmerman diff --git a/sift/scripts/jobparser.sls b/sift/scripts/jobparser.sls deleted file mode 100644 index a15b7026..00000000 --- a/sift/scripts/jobparser.sls +++ /dev/null @@ -1,21 +0,0 @@ -# source=https://github.com/gleeda/misc-scripts -# license=gplv2 - -{% set commit = "03a0d9126359c6b4b0b508062d3422bea9b69036" -%} -{% set hash = "sha256=a6869e7f0f2f360681ff67a67b65c627b0084ebec25d7a9bb44abe8a1cdfb467" -%} - -sift-scripts-jobparser: - file.managed: - - name: /usr/local/bin/jobparser.py - - source: https://raw.githubusercontent.com/gleeda/misc-scripts/{{ commit }}/misc_python/jobparser.py - - source_hash: {{ hash }} - - mode: 755 - -sift-scripts-jobparser-python: - file.replace: - - name: /usr/local/bin/jobparser.py - - pattern: '#!/usr/bin/env python\n' - - repl: '#!/usr/bin/env python2\n' - - count: 1 - - watch: - - file: sift-scripts-jobparser diff --git a/sift/scripts/shim-cache-parser.sls b/sift/scripts/shim-cache-parser.sls deleted file mode 100644 index bc712ab6..00000000 --- a/sift/scripts/shim-cache-parser.sls +++ /dev/null @@ -1,25 +0,0 @@ -# source=https://github.com/mandiant/ShimCacheParser -# license=apache2 -# license_source=https://github.com/mandiant/ShimCacheParser/blob/master/LICENSE - -{% set commit = "d7c517af9f3b09b810c5859ee52a6540f3b25855" -%} -{% set shasum = "sha256=61e75e485c0efc862e7b1c7746a493ca944afcf3e96512fb864706089f89d9aa" -%} - -include: - - sift.python-packages.python-registry - -sift-scripts-shim-cache-parser: - file.managed: - - name: /usr/local/bin/ShimCacheParser.py - - source: https://raw.githubusercontent.com/mandiant/ShimCacheParser/{{ commit }}/ShimCacheParser.py - - source_hash: {{ shasum }} - - mode: 755 - - require: - - sls: sift.python-packages.python-registry - -sift-scripts-shim-cache-parser-shebang: - file.prepend: - - name: /usr/local/bin/ShimCacheParser.py - - text: '#!/usr/bin/env python2' - - watch: - - file: sift-scripts-shim-cache-parser diff --git a/sift/scripts/virustotal-tools.sls b/sift/scripts/virustotal-tools.sls deleted file mode 100644 index 0e2cef0c..00000000 --- a/sift/scripts/virustotal-tools.sls +++ /dev/null @@ -1,56 +0,0 @@ -# source=https://blog.didierstevens.com/programs/virustotal-tools/ -# license=unknown - -include: - - sift.python-packages.poster - -sift-scripts-virustotal-search-archive: - archive.extracted: - - name: /usr/local/src/virustotal-search-v0.1.4 - - source: https://didierstevens.com/files/software/virustotal-search_V0_1_4.zip - - source_hash: sha256=8c033b3c46767590c54c191aeedc0162b3b8ccde0d7b75841a6552ca9de76044 - - enforce_toplevel: False - -sift-scripts-virustotal-search-script: - file.managed: - - name: /usr/local/bin/virustotal-search.py - - source: /usr/local/src/virustotal-search-v0.1.4/virustotal-search.py - - mode: 755 - - watch: - - archive: sift-scripts-virustotal-search-archive - -sift-scripts-virustotal-submit-archive: - archive.extracted: - - name: /usr/local/src/virustotal-submit-v0.0.3 - - source: https://didierstevens.com/files/software/virustotal-submit_V0_0_3.zip - - source_hash: sha256=37cce3e8469de097912cb23bac6b909c9c7f5a5cee09c9279d32bdb9d6e23bcc - - enforce_toplevel: False - -sift-scripts-virustotal-submit-script: - file.managed: - - name: /usr/local/bin/virustotal-submit.py - - source: /usr/local/src/virustotal-submit-v0.0.3/virustotal-submit.py - - mode: 755 - - watch: - - archive: sift-scripts-virustotal-submit-archive - - require: - - sls: sift.python-packages.poster - -sift-scripts-virustotal-search-shebang: - file.replace: - - name: /usr/local/bin/virustotal-search.py - - pattern: '#!/usr/bin/env python\n' - - repl: '#!/usr/bin/env python2\n' - - count: 1 - - watch: - - file: sift-scripts-virustotal-search-script - -sift-scripts-virustotal-submit-shebang: - file.replace: - - name: /usr/local/bin/virustotal-submit.py - - pattern: '#!/usr/bin/env python\n' - - repl: '#!/usr/bin/env python2\n' - - count: 1 - - watch: - - file: sift-scripts-virustotal-submit-script - diff --git a/sift/scripts/vshot.sls b/sift/scripts/vshot.sls deleted file mode 100644 index acd11fa8..00000000 --- a/sift/scripts/vshot.sls +++ /dev/null @@ -1,35 +0,0 @@ -{% set commit = "62d8ae4ed1ca276f2a1ffe251e1750d10538ae52" -%} -{% set hash = "sha256=590fb825df2d17f2e83fcbf1a578f39d8c7bd38017d85edfb250c0fb92db8b3a" -%} - -include: - - sift.packages.bulk-extractor - - sift.python-packages.volatility - -sift-scripts-vshot: - file.managed: - - name: /usr/local/bin/vshot - - source: https://raw.githubusercontent.com/CrowdStrike/Forensics/{{ commit }}/vshot - - source_hash: {{ hash }} - - mode: 755 - - require: - - sls: sift.python-packages.volatility - - sls: sift.packages.bulk-extractor - -sift-scripts-vshot-config-volatility: - file.replace: - - name: /usr/local/bin/vshot - - pattern: 'volpath="/usr/bin/volatility"' - - repl: 'volpath="/usr/local/bin/vol.py"' - - count: 1 - - watch: - - file: sift-scripts-vshot - -sift-scripts-vshot-config-bulk-extractor: - file.replace: - - name: /usr/local/bin/vshot - - pattern: 'bulkpath="/usr/local/bin/bulk_extractor"' - - repl: 'bulkpath="/usr/bin/bulk_extractor"' - - count: 1 - - watch: - - file: sift-scripts-vshot - From a7c5ee361132ba55dea9cc34a95a62dd5aa74e1e Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Tue, 25 Feb 2025 13:31:23 -0500 Subject: [PATCH 61/64] Update Volatility to Volatility 3 for 24 (#163) --- sift/config/symlinks.sls | 7 - sift/files/volatility/__init__.py | 0 sift/files/volatility/pstotal.py | 227 -------------------------- sift/files/volatility/sqlite_help.py | 185 --------------------- sift/python-packages/dpapick.sls | 18 -- sift/python-packages/pycoin.sls | 13 -- sift/python-packages/pycrypto.sls | 14 -- sift/python-packages/simplejson.sls | 13 -- sift/python-packages/volatility.sls | 120 -------------- sift/python-packages/yara-python.sls | 13 -- sift/python3-packages/volatility3.sls | 42 +++++ sift/scripts/init.sls | 2 - 12 files changed, 42 insertions(+), 612 deletions(-) delete mode 100644 sift/files/volatility/__init__.py delete mode 100644 sift/files/volatility/pstotal.py delete mode 100755 sift/files/volatility/sqlite_help.py delete mode 100644 sift/python-packages/dpapick.sls delete mode 100644 sift/python-packages/pycoin.sls delete mode 100644 sift/python-packages/pycrypto.sls delete mode 100644 sift/python-packages/simplejson.sls delete mode 100644 sift/python-packages/volatility.sls delete mode 100644 sift/python-packages/yara-python.sls create mode 100644 sift/python3-packages/volatility3.sls diff --git a/sift/config/symlinks.sls b/sift/config/symlinks.sls index f1f4809b..792ddecf 100644 --- a/sift/config/symlinks.sls +++ b/sift/config/symlinks.sls @@ -1,13 +1,6 @@ include: - - sift.python-packages.volatility - sift.scripts.regripper -/usr/bin/vol.py: - file.symlink: - - target: /usr/bin/vol - - require: - - sls: sift.python-packages.volatility - /usr/bin/mactime: file.symlink: - target: /usr/local/bin/mactime-sleuthkit diff --git a/sift/files/volatility/__init__.py b/sift/files/volatility/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sift/files/volatility/pstotal.py b/sift/files/volatility/pstotal.py deleted file mode 100644 index 18fde0bc..00000000 --- a/sift/files/volatility/pstotal.py +++ /dev/null @@ -1,227 +0,0 @@ -# pstotal -# Copyright (C) 2014 Sue Stirrup -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or (at -# your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -''' -Rewrite of and enhancements to the SANS(tm) Institute's text based pstotal plugin -based on Jesse Kornblum's original for Volatility 2.0. - -@author: Sue Stirrup -@license: GNU General Public License 2.0 or later -@contact: info@sans.org -@organization: The SANS(tm) Institute -Amendments + enhancements - * Default behaviour to display complete list of processes (process scan) - - Interesting column added to show processes hidden from pslist. - * Command line option to display only processes hidden from process list (original behaviour) - * Graphical visualisation option using Graphviz and .dot format via the command line added: - - Command line option to display process command (graphical representation only) - - Command line option to display process path name (graphical representation only) - - Processes from prior boot rendered in light blue (with exit time before current boot or not available - - Processes from prior boot rendered in medium blue (with exit time after current boot - - Processes from current boot but hidden from pslist rendered in red - - Suspected pid reuse rendered in yellow -''' - -import volatility.plugins.filescan as filescan -import volatility.plugins.common as common -import volatility.utils as utils -import volatility.obj as obj -import volatility.win32.tasks as tasks -import pdb -import re - -class pstotal(common.AbstractWindowsCommand): - ''' Combination of pslist,psscan & pstree --output=dot gives graphical representation ''' - - def __init__(self, config,*args, **kwargs): - common.AbstractWindowsCommand.__init__(self, config, *args, **kwargs) - config.add_option('SHORT', short_option = 'S', default = False, help = 'Interesting processes only', action = 'store_true') - config.add_option('CMD', short_option = 'c', default = False, help = 'Display process command line. All {} removed', action = 'store_true') - config.add_option('PATH', short_option = 'P', default = False, help = 'Display process image path', action = 'store_true') - - def render_text(self, outfd, data): - processes = data[0] - interest = data[1] - outfd.write("Offset (P) Name PID PPID PDB Time created Time exited Interesting \n" + \ - "----------- ---------------- ------ ------ ----------- ---------------------------- ---------------------------- ----------- \n") - for eprocess in processes: - if interest[processes[eprocess].obj_offset] == 1: - interesting = 'TRUE' - else: - interesting = ' ' - outfd.write("0x{0:09x} {1:16} {2:6} {3:6} 0x{4:09x} {5:28} {6:28} {7:7}\n".format( - processes[eprocess].obj_offset, - processes[eprocess].ImageFileName, - processes[eprocess].UniqueProcessId, - processes[eprocess].InheritedFromUniqueProcessId, - processes[eprocess].Pcb.DirectoryTableBase, - processes[eprocess].CreateTime or '', - processes[eprocess].ExitTime or '', interesting)) - - def render_dot(self, outfd, data): - objects = set() - links = set() - proc_seen = set() - procs_to_check = set() - proc_times = {} - processes = data[0] - filling = data[1] - cmdline = data[2] - pathname = data[3] - smssTime = ' ' - - # Obtain boot time - for proc in processes: - proc_name = processes[proc].ImageFileName - ppid = processes[proc].InheritedFromUniqueProcessId - processp = "%s" % (processes[proc].UniqueProcessId) - proc_times[processp] = processes[proc].CreateTime - if proc_name.find("System") == 0 and processes[proc].CreateTime: - smssTime = processes[proc].CreateTime - elif proc_name.find("smss.exe") == 0 and ppid == 4: - smssTime = processes[proc].CreateTime - - for eprocess in processes: - proc_offset = processes[eprocess].obj_offset - parentp = "%s" % (processes.get(eprocess).InheritedFromUniqueProcessId) - label = "{0} | offset (P)\\n0x{1:08x} | {2} | ".format(processes[eprocess].UniqueProcessId, - proc_offset, - processes[eprocess].ImageFileName) - # Display process command line option - if self._config.CMD : - try: - if not processes[eprocess].CreateTime < smssTime: - s = "%s" % (cmdline[proc_offset]) - s = s.replace('"', '') - s = s.replace('\\', '\\\\') - pos = s.find("csrss.exe") - if pos > 0: - pos = pos + 9 - s = s[:pos] + "\\n (Run pstree to get command parameters)" - pos = s.find("conhost.exe") - if pos > 0: - pos = pos + 11 - s = s[:pos] + "\\n (Run pstree to get command parameters)" - label += "command:\\n{0} | ".format(s or 'not available') - label = label.replace('{', '') - label = label.replace('}', '') - except KeyError: - pass - # Display process path option - if self._config.PATH : - try: - if not processes[eprocess].CreateTime < smssTime: - s = "%s" % (pathname[proc_offset]) - s = s.replace('"', '') - s = s.replace('\\', '\\\\') - pos = s.find("csrss.exe") - if pos > 0: - pos = pos + 9 - s = s[:pos] + "\\n (Run pstree to get command parameters)" - pos = s.find("conhost.exe") - if pos > 0: - pos = pos + 11 - s = s[:pos] + "\\n (Run pstree to get command parameters)" - label += "path:\\n{0} | ".format(s or 'not available') - except KeyError: - pass - label += "created:\\n{0} |".format(processes[eprocess].CreateTime or 'not available') - # Identify processes that have exited - if processes[eprocess].ExitTime: - label += "exited:\\n{0}".format(processes[eprocess].ExitTime) - options = ' style = "filled" fillcolor = "lightgray" ' - else: - label += "running" - options = '' - # Identify processes that are 'hidden' and relate to the current boot - if filling[proc_offset] == 1: - options = ' style = "filled" fillcolor = "red" ' - # Identify processes that are 'hidden' and relate to the previous boot - if processes[eprocess].CreateTime < smssTime and processes[eprocess].UniqueProcessId != 4: - options = ' style = "filled" fillcolor = "lightblue" ' - if not processes[eprocess].ExitTime: - label = label[:-7] - label += "not available\\nprior boot" - # Exit time is after current boot time - elif processes[eprocess].ExitTime > smssTime: - options = ' style = "filled" fillcolor = "darkblue" ' - label = "{" + label + "}" - # Sometimes windows creates duplicate process blocks - one in the doubly linked list and one scraped. We need to see both - pid = "%s" % (processes[eprocess].UniqueProcessId) - - if pid in proc_seen: - objects.add('pid{0}a [label="{1}" shape="record" {2}];\n'.format(processes[eprocess].UniqueProcessId, - label, options)) - links.add("pid{0} -> pid{1}a [];\n".format(processes[eprocess].InheritedFromUniqueProcessId, - processes[eprocess].UniqueProcessId)) - else: - proc_seen.add(pid) - if parentp in proc_times and (processes.get(eprocess).CreateTime < proc_times[parentp]): - links.add("pid{0}r -> pid{1} [];\n".format(processes[eprocess].InheritedFromUniqueProcessId, processes[eprocess].UniqueProcessId)) - parent = "%sr" % processes[eprocess].InheritedFromUniqueProcessId - if not parent in proc_seen: - proc_seen.add(parent) - objects.add('pid{0} [label="pid{1}" shape="oval" style = "filled" fillcolor = "yellow" ];\n'.format(parent, parentp)) - else: - links.add("pid{0} -> pid{1} [];\n".format(processes[eprocess].InheritedFromUniqueProcessId, - processes[eprocess].UniqueProcessId)) - objects.add('pid{0} [label="{1}" shape="record" {2}];\n'.format(processes[eprocess].UniqueProcessId, - label, options)) - - ## Now write the dot file - outfd.write("digraph processtree { \ngraph [rankdir = \"TB\"];\n") - for link in links: - outfd.write(link) - - for item in objects: - outfd.write(item) - outfd.write("}") - - def calculate(self): - eproc = {} - found = {} - cmdline = {} - pathname = {} - - # Brute force search for eproc blocks in pool memory - address_space = utils.load_as(self._config) - for eprocess in filescan.PSScan(self._config).calculate(): - eproc[eprocess.obj_offset] = eprocess - found[eprocess.obj_offset] = 1 - - # Walking the active process list. - # Remove any tasks we find here from the brute force search if the --short option is set. - # Anything left is something which was hidden/terminated/of interest. - address_space = utils.load_as(self._config) - for task in tasks.pslist(address_space): - phys = address_space.vtop(task.obj_offset) - if phys in eproc: - if self._config.SHORT : - del eproc[phys] - del found[phys] - else: - found[phys] = 0 - - # Grab command line and parameters - peb = task.Peb - if peb: - cmdline[phys] = peb.ProcessParameters.CommandLine - pathname[phys] = peb.ProcessParameters.ImagePathName - - ret = [eproc, found, cmdline, pathname] - - return ret diff --git a/sift/files/volatility/sqlite_help.py b/sift/files/volatility/sqlite_help.py deleted file mode 100755 index 23238ce5..00000000 --- a/sift/files/volatility/sqlite_help.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright (C) 2014 Dave Lassalle (@superponible) -# Donated under Volatility Foundation, Inc. Individual Contributor Licensing Agreement -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or (at -# your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# - -""" -@author: Dave Lassalle (@superponible) -@license: GNU General Public License 2.0 or later -@contact: dave@superponible.com -""" -# Helper functions for working with a SQLite database - -import struct -import datetime -import math - -FORWARD = 1 -BACKWARD = -1 - -def unix_time(dt): - epoch = datetime.datetime.utcfromtimestamp(0) - delta = dt - epoch - return int(delta.total_seconds()) - -def get_wintime_from_msec(msec): - """ Convert windows epoch time in microseconds to a date string """ - seconds, msec= divmod(msec, 1000000) - days, seconds = divmod(seconds, 86400) - if days > 160000 or days < 140000: - days = 0 - seconds = 0 - msec = 0 - return datetime.datetime(1601, 1, 1) + datetime.timedelta(days, seconds, msec) - -def get_nixtime_from_sec(sec): - """ Convert unix epoch time in seconds to a date string """ - return get_nixtime_from_msec(sec*1000000) - -def get_nixtime_from_msec(msec): - """ Convert unix epoch time in microseconds to a date string """ - seconds, msec= divmod(msec, 1000000) - days, seconds = divmod(seconds, 86400) - if days > 20000 or days < 9000: - days = 0 - seconds = 0 - msec = 0 - return datetime.datetime(1970, 1, 1) + datetime.timedelta(days, seconds, msec) - -def varint_type_to_length(varint): - """ Return the number of bytes used by a varint type """ - if varint == 5: - return (6, "") - elif varint == 6 or varint == 7: - return (8, "") - elif varint == 8: - return (0,0) - elif varint == 9: - return (0,1) - else: - return (varint, "") - -def ones_comp(bin_str): - """ Return the ones complement of a string of 0s and 1s """ - output = "" - for i in bin_str: - if i == '0': - output += '1' - if i == '1': - output += '0' - return output - -def find_varint(buff, start, direct): - """ varint are 1-9 bytes, big-endian. The most sig bit is not used, which is why 128 is subtracted - in the for loops below. - See: http://www.evolane.com/support/manuals/shared/manuals/tcltk/sqlite/fileformat.html#varint_format""" - buff_len = len(buff) - varint_len = 1 - varint_buff = "" - begin = 0 - # at start index and going backwards, so only 1 byte available - if direct == BACKWARD and start == 0: - begin = 0 - # going backwards - elif direct == BACKWARD: - # set stopping point, lowest possible is start of the buffer - if start >= 9: - stop = start - 9 - else: - stop = 0 - for i in range(start, stop, direct): - if ord(buff[i-1]) < 128: - break - if i > stop + 1: - varint_len += 1 - begin = start - varint_len + 1 - # going forwards - else: - # set a stopping point, maximum length of 9 bytes - if start + 9 > buff_len: - stop = buff_len - else: - stop = start + 9 - begin = start - for i in range(start, stop, direct): - if ord(buff[i]) < 128: - break - if i < stop-1: - varint_len += 1 - # num_buff contains the varint that was extracted - num_buff = buff[begin:begin+varint_len] - - if num_buff == "": - return (-1, 0) - return (varint_to_int(num_buff), varint_len) - -def varint_to_int(buff): - """ convert a varint to an integer """ - - bin_str = "" - varint_len = len(buff) - # convert each byte to a binary string, keeping 7 bytes, unless the buffer is 9 bytes and - # and we are grabbing the last byte, then keep all 8 - for i in range(0,varint_len): - if i == 8 and varint_len == 9: - bin_str += bin(ord(buff[i]))[2:].zfill(8) - else: - bin_str += bin(ord(buff[i]))[2:].zfill(8)[1:] - - if len(bin_str) == 64 and bin_str[0] == '1': - # negative numbers use all 64 bits and will start with a 1. - # take the ones complement, add 1, then put a negative sign in front - sub_bin_str = ones_comp(bin_str) - value = -(int(sub_bin_str, 2) + 1) - else: - value = int(bin_str, 2) - - return value - -def varint_to_blob_length(l): - """ Blob field lengths are doubled and 12 is added so that they are even and at least 12 """ - if l == 0: - return 0 - else: - return (l - 12) / 2 - -def varint_to_text_length(l): - """ Text field lengths are doubled and 13 is added so that they are odd and at least 13 """ - if l == 0: - return 0 - else: - return (l - 13) / 2 - -def sql_unpack(buff): - """ Convert SQL integer bytes into decimal integer """ - size = len(buff) - value = "" - if size == 1: - value = struct.unpack(">b", buff)[0] - elif size == 2: - value = struct.unpack(">h", buff)[0] - elif size == 3: - tmp = "\x00" + buff - value = struct.unpack(">l", tmp)[0] - elif size == 4: - value = struct.unpack(">l", buff)[0] - elif size == 6: - tmp = "\x00\x00" + buff - value = struct.unpack(">q", tmp)[0] - elif size == 8: - value = struct.unpack(">q", buff)[0] - return value - diff --git a/sift/python-packages/dpapick.sls b/sift/python-packages/dpapick.sls deleted file mode 100644 index 88d209ce..00000000 --- a/sift/python-packages/dpapick.sls +++ /dev/null @@ -1,18 +0,0 @@ -# Note: not included in init.sls, only required by volatility - -include: - - sift.packages.libssl-dev - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.python-packages.m2crypto - -dpapick: - pip.installed: - - name: dpapick - - upgrade: True - - bin_env: /usr/bin/python2 - #- install_options: --upgrade-strategy=only-if-needed - - require: - - sls: sift.packages.libssl-dev - - sls: sift.packages.python2-pip - - sls: sift.python-packages.m2crypto diff --git a/sift/python-packages/pycoin.sls b/sift/python-packages/pycoin.sls deleted file mode 100644 index 100e745b..00000000 --- a/sift/python-packages/pycoin.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Note: not included in init.sls, only required by python-volatility - -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-pycoin: - pip.installed: - - name: pycoin - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/pycrypto.sls b/sift/python-packages/pycrypto.sls deleted file mode 100644 index f5db146c..00000000 --- a/sift/python-packages/pycrypto.sls +++ /dev/null @@ -1,14 +0,0 @@ -# This package is only required for volatility, so it is not included in the init.sls - -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-pycrypto: - pip.installed: - - name: pycrypto - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip - diff --git a/sift/python-packages/simplejson.sls b/sift/python-packages/simplejson.sls deleted file mode 100644 index 8681f5a3..00000000 --- a/sift/python-packages/simplejson.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Note: not included in init.sls, only required by python-volatility - -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-simplejson: - pip.installed: - - name: simplejson - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/volatility.sls b/sift/python-packages/volatility.sls deleted file mode 100644 index 97f27649..00000000 --- a/sift/python-packages/volatility.sls +++ /dev/null @@ -1,120 +0,0 @@ -{%- set remove_plugins = ["malprocfind.py","idxparser.py","chromehistory.py","mimikatz.py","openioc_scan.py","pstotal.py","firefoxhistory.py","autoruns.py","malfinddeep.py","prefetch.py","ssdeepscan.py","uninstallinfo.py","trustrecords.py","usnparser.py","apihooksdeep.py","editbox.py","javarat.py"] -%} - -# Name: Volatility Framework -# Website: https://github.com/volatilityfoundation/volatility -# Description: Memory forensics tool and framework -# Category: Perform Memory Forensics -# Author: https://github.com/volatilityfoundation/volatility/blob/2.6.1/AUTHORS.txt -# License: GNU General Public License (GPL) v2: https://github.com/volatilityfoundation/volatility/blob/2.6.1/LICENSE.txt -# Notes: Use vol.py to invoke this version of Volatility. To eliminate conflicts among command-line options for Volatility plugins, the following `yarascan` options have been changed: `-Y` became `-U` and `-C` became `-c`. - -include: - - sift.packages.git - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.python-packages.colorama - - sift.python-packages.construct - - sift.python-packages.dpapick - - sift.python-packages.distorm3 - - sift.python-packages.ioc_writer - - sift.python-packages.lxml - - sift.python-packages.openpyxl - - sift.python-packages.pefile - - sift.python-packages.pillow - - sift.python-packages.pycoin - - sift.python-packages.pycrypto - - sift.python-packages.pysocks - - sift.python-packages.requests - - sift.python-packages.simplejson - - sift.python-packages.yara-python - -sift-python-packages-volatility: - pip.installed: - - name: git+https://github.com/volatilityfoundation/volatility.git@master - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.git - - sls: sift.packages.python2-pip - -sift-python-volatility-community-plugins: - git.latest: - - name: https://github.com/sans-dfir/volatility-plugins-community.git - # Note: This path changed to /usr/local/lib starting in 18.04 - - target: /usr/local/lib/python2.7/dist-packages/volatility/plugins/community - - user: root - - branch: master - - force_fetch: True - - force_checkout: True - - force_clone: True - - force_reset: True - - require: - - sls: sift.packages.git - - pip: sift-python-packages-volatility - - sls: sift.python-packages.colorama - - sls: sift.python-packages.construct - - sls: sift.python-packages.dpapick - - sls: sift.python-packages.distorm3 - - sls: sift.python-packages.ioc_writer - - sls: sift.python-packages.lxml - - sls: sift.python-packages.openpyxl - - sls: sift.python-packages.pefile - - sls: sift.python-packages.pillow - - sls: sift.python-packages.pycoin - - sls: sift.python-packages.pycrypto - - sls: sift.python-packages.pysocks - - sls: sift.python-packages.requests - - sls: sift.python-packages.simplejson - - sls: sift.python-packages.yara-python - -sift-python-volatility-sift-plugins: - file.recurse: - # Note: This path changed to /usr/local/lib starting in 18.04 - - name: /usr/local/lib/python2.7/dist-packages/volatility/plugins/sift/ - - source: salt://sift/files/volatility - - makedirs: True - - file_mode: 644 - - include_pat: '*.py' - - watch: - - git: sift-python-volatility-community-plugins - - pip: sift-python-packages-volatility - -{% for plugin in remove_plugins -%} -sift-python-volatility-plugins-{{ plugin }}-absent: - file.absent: - # Note: This path changed to /usr/local/lib starting in 18.04 - - name: /usr/local/lib/python2.7/dist-packages/volatility/plugins/{{ plugin }} - - watch: - - git: sift-python-volatility-community-plugins - - pip: sift-python-packages-volatility -{% endfor -%} - -sift-python-volatility-mimikatz-plugin-update: - file.managed: - - name: /usr/local/lib/python2.7/dist-packages/volatility/plugins/community/FrancescoPicasso/mimikatz.py - - source: https://github.com/RealityNet/hotoloti/raw/master/volatility/mimikatz.py - - source_hash: sha256=75e2e6d3b09daffad83211ba0215ed3f204623b8c37c2a2950665b88a3d2ce86 - - mode: 644 - - watch: - - git: sift-python-volatility-community-plugins - - pip: sift-python-packages-volatility - -sift-python-packages-volatility-malfind-yarascan-options1: - file.replace: - - name: /usr/local/lib/python2.7/dist-packages/volatility/plugins/malware/malfind.py - - pattern: short_option = 'C' - - repl: short_option = 'c' - - prepend_if_not_found: False - - count: 1 - - require: - - git: sift-python-volatility-community-plugins - -sift-python-packages-volatility-malfind-yarascan-options2: - file.replace: - - name: /usr/local/lib/python2.7/dist-packages/volatility/plugins/malware/malfind.py - - pattern: short_option = 'Y' - - repl: short_option = 'U' - - prepend_if_not_found: False - - count: 1 - - require: - - file: sift-python-packages-volatility-malfind-yarascan-options1 diff --git a/sift/python-packages/yara-python.sls b/sift/python-packages/yara-python.sls deleted file mode 100644 index 5fec01d0..00000000 --- a/sift/python-packages/yara-python.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Note: not included in init.sls, only required by python-volatility - -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-yara-python: - pip.installed: - - name: yara-python - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python3-packages/volatility3.sls b/sift/python3-packages/volatility3.sls new file mode 100644 index 00000000..f4922dad --- /dev/null +++ b/sift/python3-packages/volatility3.sls @@ -0,0 +1,42 @@ +# Name: Volatility3 +# Website: https://github.com/volatilityfoundation/volatility3 +# Description: Memory analysis toolset +# Category: +# Author: Volatility Foundation +# License: Volatility Software License (https://www.volatilityfoundation.org/license/vsl-v1.0) +# Notes: vol, volshell + +{% set files = ['vol','volshell'] %} + +include: + - sift.packages.python3-virtualenv + +sift-python3-package-volatility3-venv: + virtualenv.managed: + - name: /opt/volatility3 + - venv_bin: /usr/bin/virtualenv + - pip_pkgs: + - pip>=24.1.3 + - setuptools>=70.0.0 + - wheel>=0.38.4 + - require: + - sls: sift.packages.python3-virtualenv + +sift-python3-package-volatility3: + pip.installed: + - name: volatility3 + - bin_env: /opt/volatility3/bin/python3 + - upgrade: True + - require: + - virtualenv: sift-python3-package-volatility3-venv + +{% for file in files %} +sift-python3-package-volatility3-symlink-{{ file }}: + file.symlink: + - name: /usr/local/bin/{{ file }} + - target: /opt/volatility3/bin/{{ file }} + - force: True + - makedirs: False + - require: + - pip: sift-python3-package-volatility3 +{% endfor %} diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index ab51362d..df300c86 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -16,7 +16,6 @@ include: - sift.scripts.sift - sift.scripts.sorter - sift.scripts.usbdeviceforensics - - sift.scripts.sqlparser - sift.scripts.zimmerman sift-scripts: @@ -40,5 +39,4 @@ sift-scripts: - sls: sift.scripts.sift - sls: sift.scripts.sorter - sls: sift.scripts.usbdeviceforensics - - sls: sift.scripts.sqlparser - sls: sift.scripts.zimmerman From 5f8e03cd36f1414b6f3a57981222f0dd09bdcc91 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Sun, 2 Mar 2025 19:28:11 -0500 Subject: [PATCH 62/64] Remove and re-organize remaining python3 states for 24 (#173) --- sift/packages/init.sls | 6 ++ sift/packages/python3-keyrings-alt.sls | 8 +++ sift/packages/python3-pip.sls | 2 +- sift/packages/python3-setuptools-rust.sls | 8 +++ sift/packages/python3-setuptools.sls | 8 +++ sift/packages/python3-wheel.sls | 8 +++ sift/packages/python3-yara.sls | 7 ++- sift/python3-packages/argparse.sls | 9 --- sift/python3-packages/bitstring.sls | 9 --- sift/python3-packages/colorama.sls | 9 --- sift/python3-packages/geoip2.sls | 9 --- sift/python3-packages/ijson.sls | 9 --- sift/python3-packages/init.sls | 40 +------------ sift/python3-packages/keyrings-alt.sls | 9 --- sift/python3-packages/lxml.sls | 13 ---- sift/python3-packages/pefile.sls | 10 ---- sift/python3-packages/pillow.sls | 9 --- sift/python3-packages/pip.sls | 9 --- sift/python3-packages/python-dateutil.sls | 9 --- sift/python3-packages/python-magic.sls | 9 --- sift/python3-packages/setuptools-rust.sls | 9 --- sift/python3-packages/setuptools.sls | 9 --- sift/python3-packages/six.sls | 9 --- sift/python3-packages/upgrade.sls | 72 ----------------------- sift/python3-packages/virustotal-api.sls | 9 --- sift/python3-packages/wheel.sls | 9 --- sift/python3-packages/yara-python.sls | 9 --- sift/scripts/init.sls | 8 --- 28 files changed, 46 insertions(+), 288 deletions(-) create mode 100644 sift/packages/python3-keyrings-alt.sls create mode 100644 sift/packages/python3-setuptools-rust.sls create mode 100644 sift/packages/python3-setuptools.sls create mode 100644 sift/packages/python3-wheel.sls delete mode 100644 sift/python3-packages/argparse.sls delete mode 100644 sift/python3-packages/bitstring.sls delete mode 100644 sift/python3-packages/colorama.sls delete mode 100644 sift/python3-packages/geoip2.sls delete mode 100644 sift/python3-packages/ijson.sls delete mode 100644 sift/python3-packages/keyrings-alt.sls delete mode 100644 sift/python3-packages/lxml.sls delete mode 100644 sift/python3-packages/pefile.sls delete mode 100644 sift/python3-packages/pillow.sls delete mode 100644 sift/python3-packages/pip.sls delete mode 100644 sift/python3-packages/python-dateutil.sls delete mode 100644 sift/python3-packages/python-magic.sls delete mode 100644 sift/python3-packages/setuptools-rust.sls delete mode 100644 sift/python3-packages/setuptools.sls delete mode 100644 sift/python3-packages/six.sls delete mode 100644 sift/python3-packages/upgrade.sls delete mode 100644 sift/python3-packages/virustotal-api.sls delete mode 100644 sift/python3-packages/wheel.sls delete mode 100644 sift/python3-packages/yara-python.sls diff --git a/sift/packages/init.sls b/sift/packages/init.sls index d5a25f0a..d07a4290 100644 --- a/sift/packages/init.sls +++ b/sift/packages/init.sls @@ -149,8 +149,11 @@ include: - sift.packages.python3-pypff - sift.packages.python3-pyqt5 - sift.packages.python3-redis + - sift.packages.python3-setuptools + - sift.packages.python3-setuptools-rust - sift.packages.python3-tk - sift.packages.python3-virtualenv + - sift.packages.python3-wheel - sift.packages.python3-xlsxwriter - sift.packages.python3-yara - sift.packages.pst-utils @@ -352,8 +355,11 @@ sift-packages: - sls: sift.packages.python3-pypff - sls: sift.packages.python3-pyqt5 - sls: sift.packages.python3-redis + - sls: sift.packages.python3-setuptools + - sls: sift.packages.python3-setuptools-rust - sls: sift.packages.python3-tk - sls: sift.packages.python3-virtualenv + - sls: sift.packages.python3-wheel - sls: sift.packages.python3-xlsxwriter - sls: sift.packages.python3-yara - sls: sift.packages.pst-utils diff --git a/sift/packages/python3-keyrings-alt.sls b/sift/packages/python3-keyrings-alt.sls new file mode 100644 index 00000000..6f2da774 --- /dev/null +++ b/sift/packages/python3-keyrings-alt.sls @@ -0,0 +1,8 @@ +include: + - sift.packages.python3 + +sift-package-python3-keyrings-alt: + pkg.installed: + - name: python3-keyrings.alt + - require: + - sls: sift.packages.python3 diff --git a/sift/packages/python3-pip.sls b/sift/packages/python3-pip.sls index e21530de..02fd397c 100644 --- a/sift/packages/python3-pip.sls +++ b/sift/packages/python3-pip.sls @@ -1,7 +1,7 @@ include: - sift.packages.python3 -sift-pacakge-python3-pip: +sift-package-python3-pip: pkg.installed: - name: python3-pip - require: diff --git a/sift/packages/python3-setuptools-rust.sls b/sift/packages/python3-setuptools-rust.sls new file mode 100644 index 00000000..b8b7f027 --- /dev/null +++ b/sift/packages/python3-setuptools-rust.sls @@ -0,0 +1,8 @@ +include: + - sift.packages.python3 + +sift-package-python3-setuptools-rust: + pkg.installed: + - name: python3-setuptools-rust + - require: + - sls: sift.packages.python3 diff --git a/sift/packages/python3-setuptools.sls b/sift/packages/python3-setuptools.sls new file mode 100644 index 00000000..7a6ca722 --- /dev/null +++ b/sift/packages/python3-setuptools.sls @@ -0,0 +1,8 @@ +include: + - sift.packages.python3 + +sift-package-python3-setuptools: + pkg.installed: + - name: python3-setuptools + - require: + - sls: sift.packages.python3 diff --git a/sift/packages/python3-wheel.sls b/sift/packages/python3-wheel.sls new file mode 100644 index 00000000..e7e71adc --- /dev/null +++ b/sift/packages/python3-wheel.sls @@ -0,0 +1,8 @@ +include: + - sift.packages.python3 + +sift-package-python3-wheel: + pkg.installed: + - name: python3-wheel + - require: + - sls: sift.packages.python3 diff --git a/sift/packages/python3-yara.sls b/sift/packages/python3-yara.sls index 5dd0efaf..cf78da09 100644 --- a/sift/packages/python3-yara.sls +++ b/sift/packages/python3-yara.sls @@ -1,3 +1,8 @@ +include: + - sift.packages.python3 + sift-package-python3-yara: pkg.installed: - - name: python3-yara \ No newline at end of file + - name: python3-yara + - require: + - sls: sift.packages.python3 diff --git a/sift/python3-packages/argparse.sls b/sift/python3-packages/argparse.sls deleted file mode 100644 index c3226407..00000000 --- a/sift/python3-packages/argparse.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-argparse: - pip.installed: - - name: argparse - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/bitstring.sls b/sift/python3-packages/bitstring.sls deleted file mode 100644 index 3f0eb9b9..00000000 --- a/sift/python3-packages/bitstring.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-bitstring: - pip.installed: - - name: bitstring - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/colorama.sls b/sift/python3-packages/colorama.sls deleted file mode 100644 index 57033ee4..00000000 --- a/sift/python3-packages/colorama.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-colorama: - pip.installed: - - name: colorama - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/geoip2.sls b/sift/python3-packages/geoip2.sls deleted file mode 100644 index cfb7349d..00000000 --- a/sift/python3-packages/geoip2.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-geoip2: - pip.installed: - - name: geoip2 - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/ijson.sls b/sift/python3-packages/ijson.sls deleted file mode 100644 index 75ad193c..00000000 --- a/sift/python3-packages/ijson.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-ijson: - pip.installed: - - name: ijson - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/init.sls b/sift/python3-packages/init.sls index 9e06ce75..48b57450 100644 --- a/sift/python3-packages/init.sls +++ b/sift/python3-packages/init.sls @@ -1,40 +1,21 @@ include: - sift.python3-packages.analyzemft - sift.python3-packages.python3-keyring - - sift.python3-packages.pip - - sift.python3-packages.python3-keyring - - sift.python3-packages.argparse - - sift.python3-packages.bitstring - - sift.python3-packages.colorama - - sift.python3-packages.geoip2 - sift.python3-packages.hindsight - sift.python3-packages.ioc-writer - sift.python3-packages.imagemounter - sift.python3-packages.indxparse - sift.python3-packages.java-idx-parser - - sift.python3-packages.keyrings-alt - - sift.python3-packages.lxml - sift.python3-packages.mac-apt - sift.python3-packages.machinae - sift.python3-packages.page-brute - - sift.python3-packages.pefile - sift.python3-packages.pe-carver - sift.python3-packages.pe-scanner - - sift.python3-packages.pillow - - sift.python3-packages.python-dateutil - sift.python3-packages.python-evtx - - sift.python3-packages.python-magic - sift.python3-packages.sqlite-carver - sift.python3-packages.stix-validator - - sift.python3-packages.stix - sift.python3-packages.usbdeviceforensics - sift.python3-packages.usnparser - - sift.python3-packages.virustotal-api - - sift.python3-packages.wheel - - sift.python3-packages.yara-python - - sift.python3-packages.setuptools - - sift.python3-packages.setuptools-rust - - sift.python3-packages.six sift-python3-packages: test.nop: @@ -42,37 +23,18 @@ sift-python3-packages: - require: - sls: sift.python3-packages.analyzemft - sls: sift.python3-packages.python3-keyring - - sls: sift.python3-packages.pip - - sls: sift.python3-packages.python3-keyring - - sls: sift.python3-packages.argparse - - sls: sift.python3-packages.bitstring - - sls: sift.python3-packages.colorama - - sls: sift.python3-packages.geoip2 - sls: sift.python3-packages.hindsight - sls: sift.python3-packages.ioc-writer - sls: sift.python3-packages.imagemounter - sls: sift.python3-packages.indxparse - sls: sift.python3-packages.java-idx-parser - - sls: sift.python3-packages.keyrings-alt - - sls: sift.python3-packages.lxml - sls: sift.python3-packages.mac-apt - sls: sift.python3-packages.machinae - sls: sift.python3-packages.page-brute - - sls: sift.python3-packages.pefile - sls: sift.python3-packages.pe-carver - sls: sift.python3-packages.pe-scanner - - sls: sift.python3-packages.pillow - - sls: sift.python3-packages.python-dateutil - sls: sift.python3-packages.python-evtx - - sls: sift.python3-packages.python-magic - sls: sift.python3-packages.sqlite-carver - sls: sift.python3-packages.stix-validator - - sls: sift.python3-packages.stix - sls: sift.python3-packages.usbdeviceforensics - - sls: sift.python3-packages.usnparser - - sls: sift.python3-packages.virustotal-api - - sls: sift.python3-packages.wheel - - sls: sift.python3-packages.yara-python - - sls: sift.python3-packages.setuptools - - sls: sift.python3-packages.setuptools-rust - - sls: sift.python3-packages.six + - sls: sift.python3-packages.usnparser \ No newline at end of file diff --git a/sift/python3-packages/keyrings-alt.sls b/sift/python3-packages/keyrings-alt.sls deleted file mode 100644 index 6761c66e..00000000 --- a/sift/python3-packages/keyrings-alt.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-keyrings-alt: - pip.installed: - - name: keyrings.alt - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/lxml.sls b/sift/python3-packages/lxml.sls deleted file mode 100644 index 98bd67f5..00000000 --- a/sift/python3-packages/lxml.sls +++ /dev/null @@ -1,13 +0,0 @@ -include: - - sift.python3-packages.pip - - sift.packages.libxml2-dev - - sift.packages.libxslt-dev - -sift-python3-packages-lxml: - pip.installed: - - name: lxml - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip - - sls: sift.packages.libxml2-dev - - sls: sift.packages.libxslt-dev diff --git a/sift/python3-packages/pefile.sls b/sift/python3-packages/pefile.sls deleted file mode 100644 index 2602c403..00000000 --- a/sift/python3-packages/pefile.sls +++ /dev/null @@ -1,10 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-pefile: - pip.installed: - - name: pefile - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip - diff --git a/sift/python3-packages/pillow.sls b/sift/python3-packages/pillow.sls deleted file mode 100644 index 5c5cfbe7..00000000 --- a/sift/python3-packages/pillow.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-pillow: - pip.installed: - - name: pillow - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/pip.sls b/sift/python3-packages/pip.sls deleted file mode 100644 index 1f059aeb..00000000 --- a/sift/python3-packages/pip.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.packages.python3-pip - -sift-python3-packages-pip: - pip.installed: - - name: pip==21.0.1 - - bin_env: /usr/bin/python3 - - require: - - sls: sift.packages.python3-pip diff --git a/sift/python3-packages/python-dateutil.sls b/sift/python3-packages/python-dateutil.sls deleted file mode 100644 index b347a7d9..00000000 --- a/sift/python3-packages/python-dateutil.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-python-dateutil: - pip.installed: - - name: python-dateutil - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/python-magic.sls b/sift/python3-packages/python-magic.sls deleted file mode 100644 index 2c88fffd..00000000 --- a/sift/python3-packages/python-magic.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-python-magic: - pip.installed: - - name: python-magic - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/setuptools-rust.sls b/sift/python3-packages/setuptools-rust.sls deleted file mode 100644 index 439c1625..00000000 --- a/sift/python3-packages/setuptools-rust.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-setuptools-rust: - pip.installed: - - name: setuptools_rust - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/setuptools.sls b/sift/python3-packages/setuptools.sls deleted file mode 100644 index 51837150..00000000 --- a/sift/python3-packages/setuptools.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-setuptools: - pip.installed: - - name: 'setuptools<66.0.0' - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/six.sls b/sift/python3-packages/six.sls deleted file mode 100644 index 43f7c2a2..00000000 --- a/sift/python3-packages/six.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-six: - pip.installed: - - name: six - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/upgrade.sls b/sift/python3-packages/upgrade.sls deleted file mode 100644 index ae8a4459..00000000 --- a/sift/python3-packages/upgrade.sls +++ /dev/null @@ -1,72 +0,0 @@ -argparse.sls: - upgrade: True -bitstring.sls: - upgrade: True -colorama.sls: - upgrade: True -geoip2.sls: - upgrade: True -ioc_writer.sls: - upgrade: True -lxml.sls: - upgrade: True -pefile.sls: - upgrade: True -pillow.sls: - upgrade: True -pyhindsight.sls: - upgrade: True -python-dateutil.sls: - upgrade: True -python-evtx.sls: - upgrade: True -python-magic.sls: - upgrade: True -python-registry.sls: - upgrade: True -setuptools.sls: - upgrade: True -six.sls: - upgrade: True -stix-validator.sls: - upgrade: True -stix.sls: - upgrade: True -virustotal-api.sls: - upgrade: True -wheel.sls: - upgrade: True -yara-python.sls: - upgrade: True - -include: - - sift.python3-packages.pip - - sift.python3-packages.argparse - - sift.python3-packages.bitstring - - sift.python3-packages.colorama - - sift.python3-packages.geoip2 - - sift.python3-packages.ioc_writer - - sift.python3-packages.lxml - - sift.python3-packages.pefile - - sift.python3-packages.pillow - - sift.python3-packages.pyhindsight - - sift.python3-packages.python-dateutil - - sift.python3-packages.python-evtx - - sift.python3-packages.python-magic - - sift.python3-packages.python-registry - - sift.python3-packages.setuptools - - sift.python3-packages.setuptools-rust - - sift.python3-packages.six - - sift.python3-packages.stix-validator - - sift.python3-packages.stix - - sift.python3-packages.virustotal-api - - sift.python3-packages.wheel - - sift.python3-packages.yara-python - -sift-python3-packages-upgrade: - cmd.run: - - name: /usr/bin/python3 -m pip install --upgrade argparse bitstring colorama geoip2 ioc_writer lxml pefile pillow pyhindsight python-dateutil python-evtx python-magic python-registry setuptools setuptools_rust six stix-validator stix virustotal-api wheel yara-python pip - - require: - - sls: sift.python3-packages.pip - - sls: sift.python3-packages.argparse - - sls: sift.python3-packages.bitstring - - sls: sift.python3-packages.colorama - - sls: sift.python3-packages.geoip2 - - sls: sift.python3-packages.ioc_writer - - sls: sift.python3-packages.lxml - - sls: sift.python3-packages.pefile - - sls: sift.python3-packages.pillow - - sls: sift.python3-packages.pyhindsight - - sls: sift.python3-packages.python-dateutil - - sls: sift.python3-packages.python-evtx - - sls: sift.python3-packages.python-magic - - sls: sift.python3-packages.python-registry - - sls: sift.python3-packages.setuptools - - sls: sift.python3-packages.setuptools-rust - - sls: sift.python3-packages.six - - sls: sift.python3-packages.stix-validator - - sls: sift.python3-packages.stix - - sls: sift.python3-packages.virustotal-api - - sls: sift.python3-packages.wheel - - sls: sift.python3-packages.yara-python - diff --git a/sift/python3-packages/virustotal-api.sls b/sift/python3-packages/virustotal-api.sls deleted file mode 100644 index 580cc85e..00000000 --- a/sift/python3-packages/virustotal-api.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-virustotal-api: - pip.installed: - - name: virustotal-api - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/wheel.sls b/sift/python3-packages/wheel.sls deleted file mode 100644 index fac0f6a3..00000000 --- a/sift/python3-packages/wheel.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-wheel: - pip.installed: - - name: wheel - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/python3-packages/yara-python.sls b/sift/python3-packages/yara-python.sls deleted file mode 100644 index 382cdedc..00000000 --- a/sift/python3-packages/yara-python.sls +++ /dev/null @@ -1,9 +0,0 @@ -include: - - sift.python3-packages.pip - -sift-python3-packages-yara-python: - pip.installed: - - name: yara-python - - bin_env: /usr/bin/python3 - - require: - - sls: sift.python3-packages.pip diff --git a/sift/scripts/init.sls b/sift/scripts/init.sls index df300c86..4f3b6d63 100644 --- a/sift/scripts/init.sls +++ b/sift/scripts/init.sls @@ -7,15 +7,11 @@ include: - sift.scripts.exiftool - sift.scripts.keydet-tools - sift.scripts.packerid - - sift.scripts.parseusn - sift.scripts.pdf-tools - - sift.scripts.pescanner - - sift.scripts.pe-carver - sift.scripts.regripper - sift.scripts.screen-scale - sift.scripts.sift - sift.scripts.sorter - - sift.scripts.usbdeviceforensics - sift.scripts.zimmerman sift-scripts: @@ -30,13 +26,9 @@ sift-scripts: - sls: sift.scripts.exiftool - sls: sift.scripts.keydet-tools - sls: sift.scripts.packerid - - sls: sift.scripts.parseusn - sls: sift.scripts.pdf-tools - - sls: sift.scripts.pescanner - - sls: sift.scripts.pe-carver - sls: sift.scripts.regripper - sls: sift.scripts.screen-scale - sls: sift.scripts.sift - sls: sift.scripts.sorter - - sls: sift.scripts.usbdeviceforensics - sls: sift.scripts.zimmerman From 4d4313d8feb452dae86866878b8322ca17aca302 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Sun, 2 Mar 2025 19:28:22 -0500 Subject: [PATCH 63/64] Remove remaining python2 references (#179) --- sift/include-server.sls | 2 -- sift/packages/python-wxgtk3.sls | 8 -------- sift/packages/python2-dev.sls | 3 --- sift/packages/python2-pip.sls | 18 ------------------ sift/packages/python2.sls | 3 --- sift/python-packages/colorama.sls | 11 ----------- sift/python-packages/construct.sls | 11 ----------- sift/python-packages/distorm3.sls | 12 ------------ sift/python-packages/init.sls | 12 ------------ sift/python-packages/ioc_writer.sls | 13 ------------- sift/python-packages/lxml.sls | 14 -------------- sift/python-packages/m2crypto.sls | 17 ----------------- sift/python-packages/openpyxl.sls | 10 ---------- sift/python-packages/pefile.sls | 11 ----------- sift/python-packages/pillow.sls | 11 ----------- sift/python-packages/pydasm.sls | 14 -------------- sift/python-packages/pysocks.sls | 10 ---------- sift/python-packages/requests.sls | 11 ----------- sift/python-packages/setuptools.sls | 11 ----------- sift/python-packages/wheel.sls | 11 ----------- sift/tests/python-apt.sls | 13 ------------- 21 files changed, 226 deletions(-) delete mode 100644 sift/packages/python-wxgtk3.sls delete mode 100644 sift/packages/python2-dev.sls delete mode 100644 sift/packages/python2-pip.sls delete mode 100644 sift/packages/python2.sls delete mode 100644 sift/python-packages/colorama.sls delete mode 100644 sift/python-packages/construct.sls delete mode 100644 sift/python-packages/distorm3.sls delete mode 100644 sift/python-packages/init.sls delete mode 100644 sift/python-packages/ioc_writer.sls delete mode 100644 sift/python-packages/lxml.sls delete mode 100644 sift/python-packages/m2crypto.sls delete mode 100644 sift/python-packages/openpyxl.sls delete mode 100644 sift/python-packages/pefile.sls delete mode 100644 sift/python-packages/pillow.sls delete mode 100644 sift/python-packages/pydasm.sls delete mode 100644 sift/python-packages/pysocks.sls delete mode 100644 sift/python-packages/requests.sls delete mode 100644 sift/python-packages/setuptools.sls delete mode 100644 sift/python-packages/wheel.sls delete mode 100644 sift/tests/python-apt.sls diff --git a/sift/include-server.sls b/sift/include-server.sls index c0677017..76156043 100644 --- a/sift/include-server.sls +++ b/sift/include-server.sls @@ -1,7 +1,6 @@ include: - sift.repos - sift.python3-packages - - sift.python-packages - sift.packages - sift.scripts @@ -11,6 +10,5 @@ sift-server-include: - require: - sls: sift.repos - sls: sift.python3-packages - - sls: sift.python-packages - sls: sift.packages - sls: sift.scripts diff --git a/sift/packages/python-wxgtk3.sls b/sift/packages/python-wxgtk3.sls deleted file mode 100644 index bae4bd18..00000000 --- a/sift/packages/python-wxgtk3.sls +++ /dev/null @@ -1,8 +0,0 @@ -include: - - sift.packages.python2 - -sift-package-python-wxgtk3: - pkg.installed: - - name: python-wxgtk3.0 - - require: - - sls: sift.packages.python2 diff --git a/sift/packages/python2-dev.sls b/sift/packages/python2-dev.sls deleted file mode 100644 index 8fbd8c57..00000000 --- a/sift/packages/python2-dev.sls +++ /dev/null @@ -1,3 +0,0 @@ -sift-packages-python2-dev: - pkg.installed: - - name: python2-dev diff --git a/sift/packages/python2-pip.sls b/sift/packages/python2-pip.sls deleted file mode 100644 index c4c4118b..00000000 --- a/sift/packages/python2-pip.sls +++ /dev/null @@ -1,18 +0,0 @@ -include: - - sift.packages.python2 - - sift.packages.curl - -sift-package-python2-pip-install-script: - cmd.run: - - name: curl -o /tmp/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py - - unless: which pip2 - - require: - - sls: sift.packages.python2 - - sls: sift.packages.curl - -sift-package-python2-pip-install: - cmd.run: - - name: python2 /tmp/get-pip.py - - unless: which pip2 - - require: - - cmd: sift-package-python2-pip-install-script diff --git a/sift/packages/python2.sls b/sift/packages/python2.sls deleted file mode 100644 index ccbd59d8..00000000 --- a/sift/packages/python2.sls +++ /dev/null @@ -1,3 +0,0 @@ -sift-package-python2: - pkg.installed: - - name: python2 diff --git a/sift/python-packages/colorama.sls b/sift/python-packages/colorama.sls deleted file mode 100644 index e1acfec8..00000000 --- a/sift/python-packages/colorama.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-colorama: - pip.installed: - - name: colorama - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/construct.sls b/sift/python-packages/construct.sls deleted file mode 100644 index 008885d5..00000000 --- a/sift/python-packages/construct.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-construct: - pip.installed: - - name: construct == 2.10.54 - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/distorm3.sls b/sift/python-packages/distorm3.sls deleted file mode 100644 index 7a819d96..00000000 --- a/sift/python-packages/distorm3.sls +++ /dev/null @@ -1,12 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.packages.python2-dev - -sift-python-packages-distorm3: - pip.installed: - - name: distorm3 == 3.4.4 - - bin_env: /usr/bin/python2 - - require: - - sls: sift.packages.python2-pip - - sls: sift.packages.python2-dev diff --git a/sift/python-packages/init.sls b/sift/python-packages/init.sls deleted file mode 100644 index 631d7a9a..00000000 --- a/sift/python-packages/init.sls +++ /dev/null @@ -1,12 +0,0 @@ -include: - - sift.python-packages.setuptools - - sift.python-packages.wheel - - sift.python-packages.volatility - -sift-python-packages: - test.nop: - - name: sift-python-packages - - require: - - sls: sift.python-packages.setuptools - - sls: sift.python-packages.wheel - - sls: sift.python-packages.volatility diff --git a/sift/python-packages/ioc_writer.sls b/sift/python-packages/ioc_writer.sls deleted file mode 100644 index decb3b3d..00000000 --- a/sift/python-packages/ioc_writer.sls +++ /dev/null @@ -1,13 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.python-packages.lxml - -sift-python-packages-ioc-writer: - pip.installed: - - name: ioc_writer - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip - - sls: sift.python-packages.lxml diff --git a/sift/python-packages/lxml.sls b/sift/python-packages/lxml.sls deleted file mode 100644 index 34077869..00000000 --- a/sift/python-packages/lxml.sls +++ /dev/null @@ -1,14 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.packages.libxml2-dev - - sift.packages.libxslt-dev - -sift-python-packages-lxml: - pip.installed: - - name: lxml - - bin_env: /usr/bin/python2 - - require: - - sls: sift.packages.python2-pip - - sls: sift.packages.libxml2-dev - - sls: sift.packages.libxslt-dev diff --git a/sift/python-packages/m2crypto.sls b/sift/python-packages/m2crypto.sls deleted file mode 100644 index a005bb83..00000000 --- a/sift/python-packages/m2crypto.sls +++ /dev/null @@ -1,17 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - - sift.packages.swig - - sift.packages.python2-dev - - sift.packages.libssl-dev - -sift-python-packages-m2crypto: - pip.installed: - - name: m2crypto==0.40.1 - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip - - sls: sift.packages.swig - - sls: sift.packages.python2-dev - - sls: sift.packages.libssl-dev diff --git a/sift/python-packages/openpyxl.sls b/sift/python-packages/openpyxl.sls deleted file mode 100644 index 6bb8d982..00000000 --- a/sift/python-packages/openpyxl.sls +++ /dev/null @@ -1,10 +0,0 @@ -include: - - sift.packages.python2-pip - - sift.packages.python3-pip - -openpyxl==2.1.2: - pip.installed: - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/pefile.sls b/sift/python-packages/pefile.sls deleted file mode 100644 index 570aa8c9..00000000 --- a/sift/python-packages/pefile.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-pefile: - pip.installed: - - name: pefile - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/pillow.sls b/sift/python-packages/pillow.sls deleted file mode 100644 index bf036117..00000000 --- a/sift/python-packages/pillow.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python2-pip - - sift.packages.python3-pip - -sift-python-packages-pillow: - pip.installed: - - name: pillow - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/pydasm.sls b/sift/python-packages/pydasm.sls deleted file mode 100644 index 179f2660..00000000 --- a/sift/python-packages/pydasm.sls +++ /dev/null @@ -1,14 +0,0 @@ -include: - - sift.packages.git - - sift.packages.python3-pip - - sift.packages.python2-pip - -pydasm: - pip.installed: - - name: git+https://github.com/jtpereyda/libdasm.git@68d61b1#egg=version_subpkg&subdirectory=pydasm - - bin_env: /usr/bin/python2 -# - editable: git+https://github.com/jtpereyda/libdasm.git@68d61b1#egg=version_subpkg&subdirectory=pydasm - - upgrade: True - - require: - - sls: sift.packages.git - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/pysocks.sls b/sift/python-packages/pysocks.sls deleted file mode 100644 index e7b588cc..00000000 --- a/sift/python-packages/pysocks.sls +++ /dev/null @@ -1,10 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-pysocks: - pip.installed: - - name: pysocks - - bin_env: /usr/bin/python2 - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/requests.sls b/sift/python-packages/requests.sls deleted file mode 100644 index 74913c29..00000000 --- a/sift/python-packages/requests.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-requests: - pip.installed: - - name: requests - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/setuptools.sls b/sift/python-packages/setuptools.sls deleted file mode 100644 index 51fb55df..00000000 --- a/sift/python-packages/setuptools.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-setuptools: - pip.installed: - - name: setuptools - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/python-packages/wheel.sls b/sift/python-packages/wheel.sls deleted file mode 100644 index 815bac2c..00000000 --- a/sift/python-packages/wheel.sls +++ /dev/null @@ -1,11 +0,0 @@ -include: - - sift.packages.python3-pip - - sift.packages.python2-pip - -sift-python-packages-wheel: - pip.installed: - - name: wheel - - bin_env: /usr/bin/python2 - - upgrade: True - - require: - - sls: sift.packages.python2-pip diff --git a/sift/tests/python-apt.sls b/sift/tests/python-apt.sls deleted file mode 100644 index d0b7a946..00000000 --- a/sift/tests/python-apt.sls +++ /dev/null @@ -1,13 +0,0 @@ -include: - - sift.python3-packages - - sift.python-packages - - sift.packages - -sift-tests-python-with-apt: - test.nop: - - name: sift-tests-python-with-apt - - require: - - sls: sift.python3-packages - - sls: sift.python-packages - - sls: sift.packages - From 1ff51e9f8d1f69a616d7c3432cfc1d28f201cda8 Mon Sep 17 00:00:00 2001 From: Digital Sleuth <62841822+digitalsleuth@users.noreply.github.com> Date: Mon, 3 Mar 2025 15:24:55 -0500 Subject: [PATCH 64/64] Update INDXParse and scripts venv packages (#181) --- sift/python3-packages/indxparse.sls | 1 + sift/scripts/4n6.sls | 1 + 2 files changed, 2 insertions(+) diff --git a/sift/python3-packages/indxparse.sls b/sift/python3-packages/indxparse.sls index 547288fb..5de38625 100644 --- a/sift/python3-packages/indxparse.sls +++ b/sift/python3-packages/indxparse.sls @@ -34,6 +34,7 @@ sift-python3-package-indxparse-venv: - setuptools>=70.0.0 - wheel>=0.38.4 - importlib_metadata>=8.0.0 + - packaging>=22.0.0 - fusepy - require: - sls: sift.packages.python3-virtualenv diff --git a/sift/scripts/4n6.sls b/sift/scripts/4n6.sls index 1bf638f1..5c177536 100644 --- a/sift/scripts/4n6.sls +++ b/sift/scripts/4n6.sls @@ -35,6 +35,7 @@ sift-python3-package-4n6-scripts-venv: - pip>=24.1.3 - setuptools>=70.0.0 - wheel>=0.38.4 + - packaging>=22.0.0 - ijson - s2sphere - require: