diff -Nru plaso-20181219/appveyor.yml plaso-20190131/appveyor.yml --- plaso-20181219/appveyor.yml 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/appveyor.yml 2019-01-31 19:09:43.000000000 +0000 @@ -40,7 +40,7 @@ - cmd: if [%TARGET%]==[windows_python27] ( mkdir dependencies && set PYTHONPATH=..\l2tdevtools && - "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track dev PyYAML XlsxWriter artifacts bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py funcsigs future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lz4 mock pbr pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python ) + "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track dev PyYAML XlsxWriter artifacts bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py funcsigs future idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lz4 mock pbr pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python ) - cmd: if [%TARGET%]==[windows_python36] ( mkdir dependencies && set PYTHONPATH=..\l2tdevtools && diff -Nru plaso-20181219/config/dpkg/changelog plaso-20190131/config/dpkg/changelog --- plaso-20181219/config/dpkg/changelog 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/dpkg/changelog 2019-01-31 19:09:43.000000000 +0000 @@ -1,5 +1,5 @@ -plaso (20181219-1) unstable; urgency=low +plaso (20190131-1) unstable; urgency=low * Auto-generated - -- Log2Timeline maintainers Wed, 19 Dec 2018 20:00:23 +0100 \ No newline at end of file + -- Log2Timeline maintainers Thu, 31 Jan 2019 06:58:43 +0100 \ No newline at end of file diff -Nru plaso-20181219/config/dpkg/control plaso-20190131/config/dpkg/control --- plaso-20181219/config/dpkg/control 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/dpkg/control 2019-01-31 19:09:43.000000000 +0000 @@ -18,7 +18,9 @@ Package: python-plaso Architecture: all -Depends: plaso-data, libbde-python (>= 20140531), libesedb-python (>= 20150409), libevt-python (>= 20120410), libevtx-python (>= 20141112), libewf-python (>= 20131210), libfsapfs-python (>= 20181205), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20180117), libfwsi-python (>= 20150606), liblnk-python (>= 20150830), libmsiecf-python (>= 20150314), libolecf-python (>= 20151223), libqcow-python (>= 20131204), libregf-python (>= 20150315), libscca-python (>= 20161031), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-artifacts (>= 20170818), python-backports.lzma, python-bencode, python-biplist (>= 1.0.3), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), python-crypto (>= 2.6), python-dateutil (>= 1.5), python-dfdatetime (>= 20180704), python-dfvfs (>= 20181209), python-dfwinreg (>= 20180712), python-dtfabric (>= 20181128), python-efilter (>= 1.5), python-elasticsearch (>= 6.0), python-elasticsearch5 (>= 5.4.0), python-future (>= 0.16.0), python-hachoir-core (>= 1.3.3), python-hachoir-metadata (>= 1.3.3), python-hachoir-parser (>= 1.3.4), python-idna (>= 2.5), python-lz4 (>= 0.10.0), python-pefile (>= 2018.8.8), python-psutil (>= 5.4.3), python-pyparsing (>= 2.0.3), python-pysqlite2, python-pytsk3 (>= 20160721), python-requests (>= 2.18.0), python-six (>= 1.1.0), python-tz, python-urllib3 (>= 1.21.1), python-xlsxwriter (>= 0.9.3), python-yaml (>= 3.10), python-yara (>= 3.4.0), python-zmq (>= 2.1.11), ${python:Depends}, ${misc:Depends} +Depends: plaso-data, libbde-python (>= 20140531), libesedb-python (>= 20150409), libevt-python (>= 20120410), libevtx-python (>= 20141112), libewf-python (>= 20131210), libfsapfs-python (>= 20181205), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20180117), libfwsi-python (>= 20150606), liblnk-python (>= 20150830), libmsiecf-python (>= 20150314), libolecf-python (>= 20151223), libqcow-python (>= 20131204), libregf-python (>= 20150315), libscca-python (>= 20161031), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-artifacts (>= 20170818), python-backports.lzma, python-bencode, python-biplist (>= 1.0.3), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), python-crypto (>= 2.6), python-dateutil (>= 1.5), python-dfdatetime (>= 20180704), python-dfvfs (>= 20181209), python-dfwinreg (>= 20180712), python-dtfabric (>= 20181128), python-efilter (>= 1.5), python-elasticsearch (>= 6.0), python-elasticsearch5 (>= 5.4.0), python-future (>= 0.16.0), python-idna (>= 2.5), python-lz4 (>= 0.10.0), python-pefile (>= 2018.8.8), python-psutil (>= 5.4.3), python-pyparsing (>= 2.3.0), python-pysqlite2, python-pytsk3 (>= 20160721), python-requests (>= 2.18.0), python-six (>= 1.1.0), python-tz, python-urllib3 (>= 1.21.1), python-xlsxwriter (>= 0.9.3), python-yaml (>= 3.10), python-yara (>= 3.4.0), python-zmq (>= 2.1.11), ${python:Depends}, ${misc:Depends} +Conflicts: plaso +Replaces: plaso Description: Python 2 module of plaso (log2timeline) Plaso (log2timeline) is a framework to create super timelines. Its purpose is to extract timestamps from various files found on typical @@ -26,7 +28,7 @@ Package: python3-plaso Architecture: all -Depends: plaso-data, libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20120410), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20151130), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20161031), libsigscan-python3 (>= 20150627), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20170818), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-chardet (>= 2.0.1), python3-crypto (>= 2.6), python3-dateutil (>= 1.5), python3-dfdatetime (>= 20180704), python3-dfvfs (>= 20181209), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20181128), python3-efilter (>= 1.5), python3-elasticsearch (>= 6.0), python3-elasticsearch5 (>= 5.4.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.0.3), python3-pytsk3 (>= 20160721), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends} +Depends: plaso-data, libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20120410), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20151130), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20161031), libsigscan-python3 (>= 20150627), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20170818), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-chardet (>= 2.0.1), python3-crypto (>= 2.6), python3-dateutil (>= 1.5), python3-dfdatetime (>= 20180704), python3-dfvfs (>= 20181209), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20181128), python3-efilter (>= 1.5), python3-elasticsearch (>= 6.0), python3-elasticsearch5 (>= 5.4.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends} Description: Python 3 module of plaso (log2timeline) Plaso (log2timeline) is a framework to create super timelines. Its purpose is to extract timestamps from various files found on typical @@ -34,7 +36,7 @@ Package: plaso-tools Architecture: all -Depends: python-plaso, python (>= 2.7~), ${python:Depends}, ${misc:Depends} +Depends: python-plaso, ${python:Depends}, ${misc:Depends} Description: Tools of plaso (log2timeline) Plaso (log2timeline) is a framework to create super timelines. Its purpose is to extract timestamps from various files found on typical diff -Nru plaso-20181219/config/jenkins/greendale/dean_mac-browserhistory.ini plaso-20190131/config/jenkins/greendale/dean_mac-browserhistory.ini --- plaso-20181219/config/jenkins/greendale/dean_mac-browserhistory.ini 1970-01-01 00:00:00.000000000 +0000 +++ plaso-20190131/config/jenkins/greendale/dean_mac-browserhistory.ini 2019-01-31 19:09:43.000000000 +0000 @@ -0,0 +1,8 @@ +[dean_mac-browserhistory] +case=extract_and_output +source=dean_mac.dd +extract_options=--partitions all --artifact_filters BrowserHistory +logging_options=--debug --log-file=plaso-out/%command%.log.gz +output_file=dean.csv +output_format=dynamic +profiling_options=--profilers=memory,parsers,processing,serializers,storage,task_queue,tasks --profiling_directory=plaso-out/profiling diff -Nru plaso-20181219/config/linux/gift_copr_install.sh plaso-20190131/config/linux/gift_copr_install.sh --- plaso-20181219/config/linux/gift_copr_install.sh 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/linux/gift_copr_install.sh 2019-01-31 19:09:43.000000000 +0000 @@ -8,60 +8,57 @@ # Dependencies for running plaso, alphabetized, one per line. # This should not include packages only required for testing or development. -PYTHON2_DEPENDENCIES="PyYAML - libbde-python - libesedb-python - libevt-python - libevtx-python - libewf-python +PYTHON2_DEPENDENCIES="libbde-python2 + libesedb-python2 + libevt-python2 + libevtx-python2 + libewf-python2 libfsapfs-python2 - libfsntfs-python - libfvde-python - libfwnt-python - libfwsi-python - liblnk-python - libmsiecf-python - libolecf-python - libqcow-python - libregf-python - libscca-python - libsigscan-python - libsmdev-python - libsmraw-python - libvhdi-python - libvmdk-python - libvshadow-python - libvslvm-python - python-XlsxWriter - python-artifacts - python-bencode + libfsntfs-python2 + libfvde-python2 + libfwnt-python2 + libfwsi-python2 + liblnk-python2 + libmsiecf-python2 + libolecf-python2 + libqcow-python2 + libregf-python2 + libscca-python2 + libsigscan-python2 + libsmdev-python2 + libsmraw-python2 + libvhdi-python2 + libvmdk-python2 + libvshadow-python2 + libvslvm-python2 python-biplist python-chardet - python-crypto - python-dfdatetime python-dfvfs python-dfwinreg - python-dtfabric python-efilter python-elasticsearch python-elasticsearch5 - python-hachoir-core - python-hachoir-metadata - python-hachoir-parser - python-pefile python-psutil - python-pysqlite - python-pytsk3 - python-requests - python-six + python2-XlsxWriter + python2-artifacts python2-backports-lzma + python2-bencode python2-certifi + python2-crypto python2-dateutil + python2-dfdatetime + python2-dtfabric python2-future python2-idna python2-lz4 + python2-pefile python2-pyparsing + python2-pysqlite + python2-pytsk3 python2-pytz + python2-pyyaml + python2-requests + python2-six python2-urllib3 python2-yara python2-zmq"; @@ -69,8 +66,7 @@ # Additional dependencies for running tests, alphabetized, one per line. TEST_DEPENDENCIES="python2-funcsigs python2-mock - python2-pbr - python2-six"; + python2-pbr"; # Additional dependencies for development, alphabetized, one per line. DEVELOPMENT_DEPENDENCIES="pylint @@ -78,51 +74,51 @@ # Additional dependencies for debugging, alphabetized, one per line. DEBUG_DEPENDENCIES="libbde-debuginfo - libbde-python-debuginfo + libbde-python2-debuginfo libesedb-debuginfo - libesedb-python-debuginfo + libesedb-python2-debuginfo libevt-debuginfo - libevt-python-debuginfo + libevt-python2-debuginfo libevtx-debuginfo - libevtx-python-debuginfo + libevtx-python2-debuginfo libewf-debuginfo - libewf-python-debuginfo + libewf-python2-debuginfo libfsapfs-debuginfo libfsapfs-python2-debuginfo libfsntfs-debuginfo - libfsntfs-python-debuginfo + libfsntfs-python2-debuginfo libfvde-debuginfo - libfvde-python-debuginfo + libfvde-python2-debuginfo libfwnt-debuginfo - libfwnt-python-debuginfo + libfwnt-python2-debuginfo libfwsi-debuginfo - libfwsi-python-debuginfo + libfwsi-python2-debuginfo liblnk-debuginfo - liblnk-python-debuginfo + liblnk-python2-debuginfo libmsiecf-debuginfo - libmsiecf-python-debuginfo + libmsiecf-python2-debuginfo libolecf-debuginfo - libolecf-python-debuginfo + libolecf-python2-debuginfo libqcow-debuginfo - libqcow-python-debuginfo + libqcow-python2-debuginfo libregf-debuginfo - libregf-python-debuginfo + libregf-python2-debuginfo libscca-debuginfo - libscca-python-debuginfo + libscca-python2-debuginfo libsigscan-debuginfo - libsigscan-python-debuginfo + libsigscan-python2-debuginfo libsmdev-debuginfo - libsmdev-python-debuginfo + libsmdev-python2-debuginfo libsmraw-debuginfo - libsmraw-python-debuginfo + libsmraw-python2-debuginfo libvhdi-debuginfo - libvhdi-python-debuginfo + libvhdi-python2-debuginfo libvmdk-debuginfo - libvmdk-python-debuginfo + libvmdk-python2-debuginfo libvshadow-debuginfo - libvshadow-python-debuginfo + libvshadow-python2-debuginfo libvslvm-debuginfo - libvslvm-python-debuginfo + libvslvm-python2-debuginfo python-guppy"; sudo dnf install dnf-plugins-core diff -Nru plaso-20181219/config/linux/gift_ppa_install.sh plaso-20190131/config/linux/gift_ppa_install.sh --- plaso-20181219/config/linux/gift_ppa_install.sh 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/linux/gift_ppa_install.sh 2019-01-31 19:09:43.000000000 +0000 @@ -47,9 +47,6 @@ python-elasticsearch python-elasticsearch5 python-future - python-hachoir-core - python-hachoir-metadata - python-hachoir-parser python-idna python-lz4 python-pefile diff -Nru plaso-20181219/config/macos/install.sh plaso-20190131/config/macos/install.sh --- plaso-20181219/config/macos/install.sh 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/macos/install.sh 2019-01-31 19:09:43.000000000 +0000 @@ -8,7 +8,7 @@ EXIT_SUCCESS=0; EXIT_FAILURE=1; -DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite python-lz4 pytsk3 pytz pyzmq requests six urllib3 yara-python"; +DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py future idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite python-lz4 pytsk3 pytz pyzmq requests six urllib3 yara-python"; SCRIPT_NAME=$(basename $0); DEPENDENCIES_ONLY=0; diff -Nru plaso-20181219/config/macos/make_dist.sh plaso-20190131/config/macos/make_dist.sh --- plaso-20181219/config/macos/make_dist.sh 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/macos/make_dist.sh 2019-01-31 19:09:43.000000000 +0000 @@ -8,7 +8,7 @@ EXIT_SUCCESS=0; EXIT_FAILURE=1; -DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite python-lz4 pytsk3 pytz pyzmq requests six urllib3 yara-python"; +DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py future idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite python-lz4 pytsk3 pytz pyzmq requests six urllib3 yara-python"; MACOS_VERSION=`sw_vers -productVersion | awk -F '.' '{print $1 "." $2}'`; diff -Nru plaso-20181219/config/macos/uninstall.sh plaso-20190131/config/macos/uninstall.sh --- plaso-20181219/config/macos/uninstall.sh 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/macos/uninstall.sh 2019-01-31 19:09:43.000000000 +0000 @@ -8,7 +8,7 @@ EXIT_SUCCESS=0; EXIT_FAILURE=1; -DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite python-lz4 pytsk3 pytz pyzmq requests six urllib3 yara-python"; +DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py future idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite python-lz4 pytsk3 pytz pyzmq requests six urllib3 yara-python"; SCRIPT_NAME=`basename $0`; DEPENDENCIES_ONLY=0; @@ -61,23 +61,25 @@ then echo "Uninstalling plaso."; + # Note that the paths returned by pkgutil do not have a leading /. INSTALLED_FILES=`/usr/sbin/pkgutil --files ${PACKAGE_IDENTIFIER} --only-files`; for INSTALLED_FILE in ${INSTALLED_FILES}; do - if test -f ${INSTALLED_FILE}; + if test -f /${INSTALLED_FILE}; then - rm -f ${INSTALLED_FILE}; + rm -f /${INSTALLED_FILE}; fi done + # Note that the paths returned by pkgutil do not have a leading /. INSTALLED_FILES=`/usr/sbin/pkgutil --files ${PACKAGE_IDENTIFIER} --only-dirs | sort -r`; for INSTALLED_FILE in ${INSTALLED_FILES}; do - if test -d ${INSTALLED_FILE}; + if test -d /${INSTALLED_FILE}; then - rmdir ${INSTALLED_FILE} 2> /dev/null; + rmdir /${INSTALLED_FILE} 2> /dev/null; fi done @@ -95,23 +97,25 @@ continue fi + # Note that the paths returned by pkgutil do not have a leading /. INSTALLED_FILES=`/usr/sbin/pkgutil --files ${PACKAGE_IDENTIFIER} --only-files`; for INSTALLED_FILE in ${INSTALLED_FILES}; do - if test -f ${INSTALLED_FILE}; + if test -f /${INSTALLED_FILE}; then - rm -f ${INSTALLED_FILE}; + rm -f /${INSTALLED_FILE}; fi done + # Note that the paths returned by pkgutil do not have a leading /. INSTALLED_FILES=`/usr/sbin/pkgutil --files ${PACKAGE_IDENTIFIER} --only-dirs | sort -r`; for INSTALLED_FILE in ${INSTALLED_FILES}; do - if test -d ${INSTALLED_FILE}; + if test -d /${INSTALLED_FILE}; then - rmdir ${INSTALLED_FILE} 2> /dev/null; + rmdir /${INSTALLED_FILE} 2> /dev/null; fi done diff -Nru plaso-20181219/config/pylint/spelling-private-dict plaso-20190131/config/pylint/spelling-private-dict --- plaso-20181219/config/pylint/spelling-private-dict 1970-01-01 00:00:00.000000000 +0000 +++ plaso-20190131/config/pylint/spelling-private-dict 2019-01-31 19:09:43.000000000 +0000 @@ -0,0 +1,690 @@ +abc +abspath +abstractclassmethod +abstractmethod +abu +accessors +ack +að +aires +algjörlega +alsonotevil +amcache +analysisplugin +analysisreport +analytics +analyzerresult +apache +apenwarr +apfsvolumesystem +appdata +appfirewall +argparse +args +argumentparser +artifactdefinition +artifactdefinitionsregistry +ary +ascii +ast +atime +aus +autodetected +autofill +autologin +automaticdestinations +autoupdater +avc +awk +backupset +badzipfile +baja +baseanalyzer +basehasher +basename +bataar +bde +bencode +bencoded +bing +biplist +bkup +bla +bluetooth +bodyfile +bom +bool +boolean +branco +bsm +bt +buenos +cacheaddress +ccleaner +cen +charset +cifs +cleaninfo +cli +clientgroup +cliinputreader +clioutputwriter +cmd +cmp +codecs +codepage +collectfromfilesystem +companyname +compatiblity +concatting +conf +conffile +config +configparser +connecteddevicesplatform +crankd +createdtd +credentialconfiguration +criticality +cron +crtime +csv +ctime +cuiaba +currentcontrolset +currentversion +customdestinations +cwd +czip +dat +datastream +datatypefabric +datatypemap +datatypemapcontext +datatypes +datetime +datetimevalue +datetimevalues +dateutil +dayfirst +dbs +de +debian +dec +dedent +deduplicated +deduplication +definfo +defseqnumber +deleteinfo +dequeue +deregister +deregisters +deregistration +desc +deserialized +deserializes +designator +desynchronized +dev +devicecache +devicemacs +dfdatetime +dfvfs +dfwinreg +dhabi +dictreader +dicts +dll +dls +dmg +dns +dockerforwindows +docstring +docstrings +docx +domainname +dottysql +dpkg +dst +dsv +dtfabric +dtime +dvfvs +dword +edb +efilter +eg +ekaterinburg +elasticsearch +elgoog +elif +endian +enqueue +entrytype +enum +environmentvariableartifact +eof +esedbcache +eventdata +eventextractorworker +eventlog +eventobject +eventsource +eventtag +eventtags +evt +evtx +ewf +exe +expander +expanders +extractionerror +extractionworker +failsafe +favorited +fbi +fgrep +filedescription +fileentry +fileguard +fileio +filename +filenames +filepath +filesize +filestat +filesystem +filesystems +filesystem's +filesystemsearcher +filesytem +filetime +fileversion +filterobject +findspec +findspecs +findutils +fle +formatter +formatters +fpi +friended +frozenset +fs +fse +fsevents +fseventsd +ftw +fullname +geo +getattr +getentries +getpass +gid +gmail +greenqloud +grey +groupid +grr +grr's +gtb +gugle +guid +gz +gzip +habben +hasattr +hashanalysis +hasher +hashers +hfs +hfsx +hh +hhmmss +hoc +homedir +hong +hostid +hostname +html +http +httpd +httponly +https +hve +hwm +icmpcode +icmptype +identd +idx +ie +iecompat +iecompatua +iis +impfunctions +imphash +importeddlls +ímynd +ing +ini +init +inittab +inode +inputreader +instantiation +ios +ip +ipc +ipp +iter +iterable +iterables +january +jayawardenepura +joachim +jquery +json +kaliningrad +kamchatsky +keychain +keychaindatabasecolumn +keyname +kik +kml +kodi +kolkata +kristinn +kuala +kwarg +kwargs +kyiv +langar +languagecode +lanka +lastinquiryupdate +lastmodifiedts +lcid +lfu +lifecycle +linkedin +linkerts +listable +lnk +localappdata +localtime +lockdown +loction +logline +logname +logon +lowercaseattributefilterimp +lowercased +lowercases +lpstr +lru +lsb +lumpur +macaddr +macos +mactime +magadan +mame +manaus +mappingerror +matchobject +mathplotlib +matplotlib +maxsize +metadata +mft +microsec +migwiz +milli +mixin +mmssms +mnt +moc +moresby +mru +msg +msi +msie +msiecf +msipackagecode +msiproductcode +mtf +mtime +multi +multiline +mutators +mystorage +myvalue +namd +namedtuple +namespace +nano +netwerk +nis +nofiles +nondeterministic +notevil +ns +nsrlsvr +ntdomain +nubmer +nuku'alofa +numimpfunctions +numpy +objectfilter +od +og +ola +olecf +öllu +orphanfiles +os +osx +outputwriter +overridable +packagecode +params +parseresults +parsers +parser's +passwd +passwordpolicyoptions +pathname +pathspec +pathspecs +paz +pdf +pe +pefile +pennarun +petersburg +petropavlovsk +pfilter +pid +pids +pinfo +pirelli +plaintext +plas +plaso +plaso's +plist +plistevent +plists +pls +popcontest +popen +posix +ppid +pre +prefetch +prepend +prepended +preprocess +preprocessed +preprocesses +preprocessfail +preprocessing +preprocessor +preprocessors +preprocesspluginsmanager +presetdefinition +productcode +productname +profiler +profilers +programfiles +programid +protobuf +psort +psteal +psutil +pubmatic +py +pyesedb +pyevt +pyevtx +pyfsntfs +pyfwsi +pymsiecf +pyolecf +pyparsing +pyparsing's +pyregf +pysigscan +pysqlite +pytz +quarfwd +queueempty +queueing +raðar +rc +readfp +readline +readlines +recurse +recv +recycler +refactor +refactoring +referer +reformats +regexes +regf +regvalue +reimplement +renable +repl +req +resync +retweeted +rnbt +rp +rpc +rubiconproject +runtime +rus +safna +santa +sav +sc +scalability +scancontext +scanid +schemas +scrollback +scrollbacks +sdf +sdist +securityd +seedtime +seeked +selinux +sem +sendall +serializable +serializer +serializers +sessionize +setuptools +shellbags +síar +sid +sitename +sizehint +sjálfvirk +sleuthkit +smb +sms +snd +sophos +sourcescancontext +sourcescannercontext +sourcescannode +specfic +spelt +sqlite +sqlitecache +sqlitedatabase +src +sri +ss +ssid +ssl +startswith +startup +stderr +stdin +stdout +storagefile +storagewriter +str +strftime +structs +subclasses +subclassing +subdirectories +subexpressions +subkey +subkeys +subprocess +substatus +substring +substrings +superclass's +symantec +symlink +sys +syslog +systemd +systemroot +systemtime +tc +tcp +tcpack +tcpflags +tcpsyn +tcpwin +tempfile +testdir +textfile +textwrap +tgz +tha +thismykey +timeelements +timelib +timeline +timemachine +timerange +timesketch +timeslice +timestamp +timestamped +timestamps +timezones +tln +tmp +todo +totalling +traceback +trackpad +trigproc +tsk +tskvolumesystem +tuple +tuples +txt +tyre +tyres +tz +tzif +tzinfo +udaterui +udp +ui +uid +ulaan +ulaanbaatar +unary +uncheck +unchecking +unhandled +unicode +uninstallkey +unix +unparsed +unpickleable +unserialized +unvendor +uri +url +urls +usbstor +usec +useraccountartifact +username +usnjrnl +usr +utc +utcoffset +utmp +utmpx +uuid +vbin +ver +versa +vfsstat +vhd +virustype +vlc +vshadowvolumesystem +vss +wal +webpage +whitespace +whitespaces +wifi +wildcard +wildcards +windir +windowspathresolver +windowsservice +windwos +winlnk +winlogon +winregistry +winregistrykey +winregistrysearcher +winregistryvalue +winxp +wpnidm +wrongplugin +ws +wtmp +www +xchat +xchatlog +xchatscrollback +xlsxwriter +xml +xp +xz +yaml +yandex +yara +yo'd +yy +yyyy +zealand +zeromq +zipfile +zmq +zmqerror +zsh +þessu diff -Nru plaso-20181219/config/travis/install.sh plaso-20190131/config/travis/install.sh --- plaso-20181219/config/travis/install.sh 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/travis/install.sh 2019-01-31 19:09:43.000000000 +0000 @@ -5,11 +5,11 @@ # This file is generated by l2tdevtools update-dependencies.py any dependency # related changes should be made in dependencies.ini. -L2TBINARIES_DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite python-lz4 pytsk3 pytz pyzmq requests six urllib3 yara-python"; +L2TBINARIES_DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py elasticsearch5-py future idna libbde libesedb libevt libevtx libewf libfsapfs libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite python-lz4 pytsk3 pytz pyzmq requests six urllib3 yara-python"; L2TBINARIES_TEST_DEPENDENCIES="funcsigs mock pbr six"; -PYTHON2_DEPENDENCIES="libbde-python libesedb-python libevt-python libevtx-python libewf-python libfsapfs-python libfsntfs-python libfvde-python libfwnt-python libfwsi-python liblnk-python libmsiecf-python libolecf-python libqcow-python libregf-python libscca-python libsigscan-python libsmdev-python libsmraw-python libvhdi-python libvmdk-python libvshadow-python libvslvm-python python-artifacts python-backports.lzma python-bencode python-biplist python-certifi python-chardet python-crypto python-dateutil python-dfdatetime python-dfvfs python-dfwinreg python-dtfabric python-efilter python-elasticsearch python-elasticsearch5 python-future python-hachoir-core python-hachoir-metadata python-hachoir-parser python-idna python-lz4 python-pefile python-psutil python-pyparsing python-pysqlite2 python-pytsk3 python-requests python-six python-tz python-urllib3 python-xlsxwriter python-yaml python-yara python-zmq"; +PYTHON2_DEPENDENCIES="libbde-python libesedb-python libevt-python libevtx-python libewf-python libfsapfs-python libfsntfs-python libfvde-python libfwnt-python libfwsi-python liblnk-python libmsiecf-python libolecf-python libqcow-python libregf-python libscca-python libsigscan-python libsmdev-python libsmraw-python libvhdi-python libvmdk-python libvshadow-python libvslvm-python python-artifacts python-backports.lzma python-bencode python-biplist python-certifi python-chardet python-crypto python-dateutil python-dfdatetime python-dfvfs python-dfwinreg python-dtfabric python-efilter python-elasticsearch python-elasticsearch5 python-future python-idna python-lz4 python-pefile python-psutil python-pyparsing python-pysqlite2 python-pytsk3 python-requests python-six python-tz python-urllib3 python-xlsxwriter python-yaml python-yara python-zmq"; PYTHON2_TEST_DEPENDENCIES="python-coverage python-funcsigs python-mock python-pbr python-six python-tox"; @@ -51,6 +51,8 @@ if test ${TRAVIS_PYTHON_VERSION} = "2.7"; then sudo add-apt-repository ppa:gift/pylint2 -y; + else + sudo add-apt-repository ppa:gift/pylint3 -y; fi fi diff -Nru plaso-20181219/config/windows/make_release.ps1 plaso-20190131/config/windows/make_release.ps1 --- plaso-20181219/config/windows/make_release.ps1 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/config/windows/make_release.ps1 2019-01-31 19:09:43.000000000 +0000 @@ -56,18 +56,6 @@ $Version = & Invoke-Expression -Command "git describe --tags --abbrev=0" -# Remove support for hachoir which is GPLv2 and cannot be distributed -# in binary form. Leave the formatter because it does not link in the -# hachoir code. -Get-Content "plaso\parsers\__init__.py" | %{$_ -replace "from plaso.parsers import hachoir", "pass"} | Set-Content "plaso\parsers\__init__.py.patched" -mv -Force plaso\parsers\__init__.py.patched plaso\parsers\__init__.py - -Get-Content "plaso\parsers\presets.py" | %{$_ -replace "'hachoir', ", ""} | Set-Content "plaso\parsers\presets.py.patched" -mv -Force plaso\parsers\presets.py.patched plaso\parsers\presets.py - -Get-Content "plaso\dependencies.py" | Select-String -pattern 'hachoir_' -notmatch | Set-Content "plaso\dependencies.py.patched" -mv -Force plaso\dependencies.py.patched plaso\dependencies.py - # Build the binaries for each tool If (Test-Path "dist") { @@ -89,7 +77,7 @@ Exit 1 } -$Arguments = "--hidden-import artifacts --hidden-import requests --onedir tools\log2timeline.py" +$Arguments = "--hidden-import artifacts --hidden-import future --hidden-import requests --onedir tools\log2timeline.py" If ( $Python -ne "" ) { @@ -134,7 +122,7 @@ Exit 1 } -$Arguments = "--hidden-import artifacts --hidden-import requests --onedir tools\psteal.py" +$Arguments = "--hidden-import artifacts --hidden-import future --hidden-import requests --onedir tools\psteal.py" If ( $Python -ne "" ) { @@ -179,7 +167,7 @@ cp "dist\l2tdevtools\data\licenses\LICENSE.$($d)" dist\plaso\licenses } -rm -Force dist\plaso\licenses\LICENSE.hachoir-* +# Remove debug, test and yet unused dependencies. rm -Force dist\plaso\licenses\LICENSE.guppy rm -Force dist\plaso\licenses\LICENSE.libexe rm -Force dist\plaso\licenses\LICENSE.libwrc diff -Nru plaso-20181219/data/presets.yaml plaso-20190131/data/presets.yaml --- plaso-20181219/data/presets.yaml 1970-01-01 00:00:00.000000000 +0000 +++ plaso-20190131/data/presets.yaml 2019-01-31 19:09:43.000000000 +0000 @@ -0,0 +1,164 @@ +# Parser and parser plugin presets. + +name: android +description: Preset for Android. +parsers: +- android_app_usage +- chrome_cache +- filestat +- sqlite/android_calls +- sqlite/android_sms +- sqlite/android_webview +- sqlite/android_webviewcache +- sqlite/chrome_27_history +- sqlite/chrome_8_history +- sqlite/chrome_cookies +- sqlite/skype +--- +name: linux +description: Preset for Linux. +operating_systems: +- {family: Linux} +parsers: +- bash_history +- bencode +- czip/oxml +- dockerjson +- dpkg +- filestat +- gdrive_synclog +- java_idx +- olecf +- pls_recall +- popularity_contest +- selinux +- sqlite/google_drive +- sqlite/skype +- sqlite/zeitgeist +- syslog +- systemd_journal +- utmp +- webhist +- xchatlog +- xchatscrollback +- zsh_extended_history +--- +name: macos +description: Preset for MacOS. +operating_systems: +- {family: MacOS} +parsers: +- asl_log +- bash_history +- bencode +- bsm_log +- cups_ipp +- czip/oxml +- filestat +- fseventsd +- gdrive_synclog +- java_idx +- mac_appfirewall_log +- mac_keychain +- mac_securityd +- macwifi +- olecf +- plist +- sqlite/appusage +- sqlite/google_drive +- sqlite/imessage +- sqlite/ls_quarantine +- sqlite/mac_document_versions +- sqlite/mackeeper_cache +- sqlite/skype +- syslog +- utmpx +- webhist +- zsh_extended_history +--- +name: webhist +description: Preset for Web history. +parsers: +- binary_cookies +- chrome_cache +- chrome_preferences +- esedb/msie_webcache +- firefox_cache +- java_idx +- msiecf +- opera_global +- opera_typed_history +- plist/safari_history +- sqlite/chrome_27_history +- sqlite/chrome_8_history +- sqlite/chrome_autofill +- sqlite/chrome_cookies +- sqlite/chrome_extension_activity +- sqlite/firefox_cookies +- sqlite/firefox_downloads +- sqlite/firefox_history +--- +name: win_gen +description: Preset for generic Windows, intended as template for other Windows presets. +parsers: +- bencode +- czip/oxml +- esedb +- filestat +- gdrive_synclog +- java_idx +- lnk +- mcafee_protection +- olecf +- pe +- prefetch +- sccm +- skydrive_log +- skydrive_log_old +- sqlite/google_drive +- sqlite/skype +- symantec_scanlog +- usnjrnl +- webhist +- winfirewall +- winjob +- winreg +--- +name: win7 +description: Preset for generic Windows, intended for Windows Vista and later. +operating_systems: +- {family: Windows NT} +parsers: +- amcache +- custom_destinations +- esedb/file_history +- olecf/olecf_automatic_destinations +- recycle_bin +- winevtx +- win_gen +--- +name: win7_slow +description: Preset for Windows 7 and later including parsers that require more processing time. +parsers: +- hachoir +- mft +- win7 +--- +name: winxp +description: Preset for Windows 2000, Windows XP and Windows 2003. +operating_systems: +- {product: Windows 2000} +- {product: Windows XP} +- {product: Windows 2003} +parsers: +- recycle_bin_info2 +- rplog +- win_gen +- winevt +--- +name: winxp_slow +description: Preset for Windows XP and 2003 including parsers that require more processing time. +parsers: +- hachoir +- mft +- winxp diff -Nru plaso-20181219/data/tag_macos.txt plaso-20190131/data/tag_macos.txt --- plaso-20181219/data/tag_macos.txt 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/data/tag_macos.txt 2019-01-31 19:09:43.000000000 +0000 @@ -18,4 +18,4 @@ data_type is 'plist:key' and plugin is 'plist_airport' document_print - (data_type is 'metadata:hachoir' OR data_type is 'metadata:OLECF') AND timestamp_desc contains 'Printed' + data_type is 'metadata:OLECF' AND timestamp_desc contains 'Printed' diff -Nru plaso-20181219/data/tag_windows.txt plaso-20190131/data/tag_windows.txt --- plaso-20181219/data/tag_windows.txt 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/data/tag_windows.txt 2019-01-31 19:09:43.000000000 +0000 @@ -1,7 +1,7 @@ application_execution - data_type is 'windows:prefetch' + data_type is 'windows:prefetch:execution' data_type is 'windows:lnk:link' and filename contains 'Recent' and (local_path contains '.exe' or network_path contains '.exe' or relative_path contains '.exe') - data_type is 'windows:registry:key_value' AND (plugin contains 'userassist' or plugin contains 'mru') AND regvalue.__all__ contains '.exe' + data_type is 'windows:registry:key_value' AND (parser contains 'userassist' or parser contains 'mru') AND regvalue contains '.exe' data_type is 'windows:evtx:record' and strings contains 'user mode service' and strings contains 'demand start' data_type is 'fs:stat' and filename contains 'Windows/Tasks/At' data_type is 'windows:tasks:job' @@ -21,7 +21,7 @@ data_type is 'windows:evtx:record' and source_name is 'Microsoft-Windows-Application-Experience' and event_identifier is 908 document_open - data_type is 'windows:registry:key_value' AND plugin contains 'mru' AND regvalue.__all__ notcontains '.exe' AND timestamp > 0 + data_type is 'windows:registry:key_value' AND parser contains 'mru' AND regvalue notcontains '.exe' AND timestamp > 0 login_failed data_type is 'windows:evtx:record' and source_name is 'Microsoft-Windows-Security-Auditing' and event_identifier is 4625 @@ -81,12 +81,12 @@ data_type is 'windows:evtx:record' and source_name is 'Microsoft-Windows-Kernel-Power' and event_identifier is 42 autorun - data_type is 'windows:registry:key_value' and plugin contains 'Run' - data_type is 'windows:registry:key_value' AND (plugin contains 'run' or plugin contains 'lfu') AND (regvalue.__all__ contains '.exe' OR regvalue.__all__ contains '.dll') + data_type is 'windows:registry:key_value' and parser contains 'Run' + data_type is 'windows:registry:key_value' AND (parser contains 'run' or parser contains 'lfu') AND (regvalue contains '.exe' OR regvalue contains '.dll') file_download data_type is 'chrome:history:file_downloaded' timestamp_desc is 'File Downloaded' document_print - (data_type is 'metadata:hachoir' OR data_type is 'olecf:summary_info') AND timestamp_desc contains 'Printed' + data_type is 'olecf:summary_info' AND timestamp_desc contains 'Printed' diff -Nru plaso-20181219/debian/changelog plaso-20190131/debian/changelog --- plaso-20181219/debian/changelog 2019-01-16 10:38:09.000000000 +0000 +++ plaso-20190131/debian/changelog 2019-02-09 22:53:49.000000000 +0000 @@ -1,3 +1,10 @@ +plaso (20190131-1) unstable; urgency=medium + + * New upstream version 20190131 + * Update bencode from bencode.py, remove python-bittorrent dependency + + -- Hilko Bengen Sat, 09 Feb 2019 23:53:49 +0100 + plaso (20181219-1) unstable; urgency=medium [ Raphaël Hertzog ] diff -Nru plaso-20181219/debian/control plaso-20190131/debian/control --- plaso-20181219/debian/control 2019-01-16 10:32:47.000000000 +0000 +++ plaso-20190131/debian/control 2019-02-09 22:53:27.000000000 +0000 @@ -8,7 +8,6 @@ python-xlsxwriter (>= 0.9.3), python-artifacts (>= 20170818), python-lzma, - python-bittorrent, python-biplist (>= 1.0.3), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), @@ -70,7 +69,6 @@ python-xlsxwriter (>= 0.9.3), python-artifacts (>= 20170818), python-lzma, - python-bittorrent, python-biplist (>= 1.0.3), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), diff -Nru plaso-20181219/debian/patches/0001-Add-vendored-bencode-1.0.patch plaso-20190131/debian/patches/0001-Add-vendored-bencode-1.0.patch --- plaso-20181219/debian/patches/0001-Add-vendored-bencode-1.0.patch 2019-01-16 10:32:57.000000000 +0000 +++ plaso-20190131/debian/patches/0001-Add-vendored-bencode-1.0.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,143 +0,0 @@ -From: Hilko Bengen -Date: Tue, 15 Jan 2019 10:56:54 +0100 -Subject: Add vendored bencode 1.0 - ---- - plaso/parsers/bencode.py | 128 +++++++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 128 insertions(+) - create mode 100644 plaso/parsers/bencode.py - -diff --git a/plaso/parsers/bencode.py b/plaso/parsers/bencode.py -new file mode 100644 -index 0000000..5023db5 ---- /dev/null -+++ b/plaso/parsers/bencode.py -@@ -0,0 +1,128 @@ -+# The contents of this file are subject to the BitTorrent Open Source License -+# Version 1.1 (the License). You may not copy or use this file, in either -+# source code or executable form, except in compliance with the License. You -+# may obtain a copy of the License at http://www.bittorrent.com/license/. -+# -+# Software distributed under the License is distributed on an AS IS basis, -+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -+# for the specific language governing rights and limitations under the -+# License. -+ -+# Written by Petru Paler -+ -+def decode_int(x, f): -+ f += 1 -+ newf = x.index('e', f) -+ n = int(x[f:newf]) -+ if x[f] == '-': -+ if x[f + 1] == '0': -+ raise ValueError -+ elif x[f] == '0' and newf != f+1: -+ raise ValueError -+ return (n, newf+1) -+ -+def decode_string(x, f): -+ colon = x.index(':', f) -+ n = int(x[f:colon]) -+ if x[f] == '0' and colon != f+1: -+ raise ValueError -+ colon += 1 -+ return (x[colon:colon+n], colon+n) -+ -+def decode_list(x, f): -+ r, f = [], f+1 -+ while x[f] != 'e': -+ v, f = decode_func[x[f]](x, f) -+ r.append(v) -+ return (r, f + 1) -+ -+def decode_dict(x, f): -+ r, f = {}, f+1 -+ while x[f] != 'e': -+ k, f = decode_string(x, f) -+ r[k], f = decode_func[x[f]](x, f) -+ return (r, f + 1) -+ -+decode_func = {} -+decode_func['l'] = decode_list -+decode_func['d'] = decode_dict -+decode_func['i'] = decode_int -+decode_func['0'] = decode_string -+decode_func['1'] = decode_string -+decode_func['2'] = decode_string -+decode_func['3'] = decode_string -+decode_func['4'] = decode_string -+decode_func['5'] = decode_string -+decode_func['6'] = decode_string -+decode_func['7'] = decode_string -+decode_func['8'] = decode_string -+decode_func['9'] = decode_string -+ -+def bdecode(x): -+ try: -+ r, l = decode_func[x[0]](x, 0) -+ except (IndexError, KeyError, ValueError): -+ raise "not a valid bencoded string" -+ if l != len(x): -+ raise "invalid bencoded value (data after valid prefix)" -+ return r -+ -+from types import StringType, IntType, LongType, DictType, ListType, TupleType -+ -+ -+class Bencached(object): -+ -+ __slots__ = ['bencoded'] -+ -+ def __init__(self, s): -+ self.bencoded = s -+ -+def encode_bencached(x,r): -+ r.append(x.bencoded) -+ -+def encode_int(x, r): -+ r.extend(('i', str(x), 'e')) -+ -+def encode_bool(x, r): -+ if x: -+ encode_int(1, r) -+ else: -+ encode_int(0, r) -+ -+def encode_string(x, r): -+ r.extend((str(len(x)), ':', x)) -+ -+def encode_list(x, r): -+ r.append('l') -+ for i in x: -+ encode_func[type(i)](i, r) -+ r.append('e') -+ -+def encode_dict(x,r): -+ r.append('d') -+ ilist = x.items() -+ ilist.sort() -+ for k, v in ilist: -+ r.extend((str(len(k)), ':', k)) -+ encode_func[type(v)](v, r) -+ r.append('e') -+ -+encode_func = {} -+encode_func[Bencached] = encode_bencached -+encode_func[IntType] = encode_int -+encode_func[LongType] = encode_int -+encode_func[StringType] = encode_string -+encode_func[ListType] = encode_list -+encode_func[TupleType] = encode_list -+encode_func[DictType] = encode_dict -+ -+try: -+ from types import BooleanType -+ encode_func[BooleanType] = encode_bool -+except ImportError: -+ pass -+ -+def bencode(x): -+ r = [] -+ encode_func[type(x)](x, r) -+ return ''.join(r) diff -Nru plaso-20181219/debian/patches/0001-Add-vendored-bencode-2.0.patch plaso-20190131/debian/patches/0001-Add-vendored-bencode-2.0.patch --- plaso-20181219/debian/patches/0001-Add-vendored-bencode-2.0.patch 1970-01-01 00:00:00.000000000 +0000 +++ plaso-20190131/debian/patches/0001-Add-vendored-bencode-2.0.patch 2019-02-09 22:50:28.000000000 +0000 @@ -0,0 +1,349 @@ +From: Hilko Bengen +Date: Sat, 9 Feb 2019 00:48:12 +0100 +Subject: Add vendored bencode 2.0 + +--- + plaso/parsers/bencode.py | 320 ++++++++++++++++++++++++++++++++++++++++ + plaso/parsers/bencode_parser.py | 2 +- + 2 files changed, 321 insertions(+), 1 deletion(-) + create mode 100644 plaso/parsers/bencode.py + +diff --git a/plaso/parsers/bencode.py b/plaso/parsers/bencode.py +new file mode 100644 +index 0000000..d29ae1c +--- /dev/null ++++ b/plaso/parsers/bencode.py +@@ -0,0 +1,320 @@ ++# The contents of this file are subject to the BitTorrent Open Source License ++# Version 1.1 (the License). You may not copy or use this file, in either ++# source code or executable form, except in compliance with the License. You ++# may obtain a copy of the License at http://www.bittorrent.com/license/. ++# ++# Software distributed under the License is distributed on an AS IS basis, ++# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License ++# for the specific language governing rights and limitations under the ++# License. ++ ++# Written by Petru Paler ++ ++"""bencode.py - bencode encoder + decoder.""" ++ ++from collections import deque ++import sys ++ ++try: ++ from collections import OrderedDict ++except ImportError: ++ OrderedDict = None ++ ++try: ++ import pathlib ++except ImportError: ++ pathlib = None ++ ++__all__ = ( ++ 'bencode', ++ 'bdecode', ++ 'bread', ++ 'bwrite', ++ 'encode', ++ 'decode' ++) ++ ++ ++def decode_int(x, f): ++ f += 1 ++ newf = x.index(b'e', f) ++ n = int(x[f:newf]) ++ ++ if x[f:f + 1] == b'-': ++ if x[f + 1:f + 2] == b'0': ++ raise ValueError ++ elif x[f:f + 1] == b'0' and newf != f + 1: ++ raise ValueError ++ ++ return n, newf + 1 ++ ++ ++def decode_string(x, f, try_decode_utf8=True, force_decode_utf8=False): ++ """Decode torrent bencoded 'string' in x starting at f. ++ ++ An attempt is made to convert the string to a python string from utf-8. ++ However, both string and non-string binary data is intermixed in the ++ torrent bencoding standard. So we have to guess whether the byte ++ sequence is a string or just binary data. We make this guess by trying ++ to decode (from utf-8), and if that fails, assuming it is binary data. ++ There are some instances where the data SHOULD be a string though. ++ You can check enforce this by setting force_decode_utf8 to True. If the ++ decoding from utf-8 fails, an UnidcodeDecodeError is raised. Similarly, ++ if you know it should not be a string, you can skip the decoding ++ attempt by setting try_decode_utf8=False. ++ """ ++ colon = x.index(b':', f) ++ n = int(x[f:colon]) ++ ++ if x[f:f + 1] == b'0' and colon != f + 1: ++ raise ValueError ++ ++ colon += 1 ++ s = x[colon:colon + n] ++ ++ if try_decode_utf8: ++ try: ++ return s.decode('utf-8'), colon + n ++ except UnicodeDecodeError: ++ if force_decode_utf8: ++ raise ++ ++ return bytes(s), colon + n ++ ++ ++def decode_list(x, f): ++ r, f = [], f + 1 ++ ++ while x[f:f + 1] != b'e': ++ v, f = decode_func[x[f:f + 1]](x, f) ++ r.append(v) ++ ++ return r, f + 1 ++ ++ ++def decode_dict_py26(x, f): ++ r, f = {}, f + 1 ++ ++ while x[f] != 'e': ++ k, f = decode_string(x, f) ++ r[k], f = decode_func[x[f]](x, f) ++ ++ return r, f + 1 ++ ++ ++def decode_dict(x, f, force_sort=True): ++ """Decode bencoded data to an OrderedDict. ++ ++ The BitTorrent standard states that: ++ Keys must be strings and appear in sorted order (sorted as raw ++ strings, not alphanumerics) ++ - http://www.bittorrent.org/beps/bep_0003.html ++ ++ Therefore, this function will force the keys to be strings (decoded ++ from utf-8), and by default the keys are (re)sorted after reading. ++ Set force_sort to False to keep the order of the dictionary as ++ represented in x, as many other encoders and decoders do not force this ++ property. ++ """ ++ r, f = OrderedDict(), f + 1 ++ ++ while x[f:f + 1] != b'e': ++ k, f = decode_string(x, f, force_decode_utf8=True) ++ r[k], f = decode_func[x[f:f + 1]](x, f) ++ ++ if force_sort: ++ r = OrderedDict(sorted(r.items())) ++ ++ return r, f + 1 ++ ++ ++# noinspection PyDictCreation ++decode_func = {} ++decode_func[b'l'] = decode_list ++decode_func[b'i'] = decode_int ++decode_func[b'0'] = decode_string ++decode_func[b'1'] = decode_string ++decode_func[b'2'] = decode_string ++decode_func[b'3'] = decode_string ++decode_func[b'4'] = decode_string ++decode_func[b'5'] = decode_string ++decode_func[b'6'] = decode_string ++decode_func[b'7'] = decode_string ++decode_func[b'8'] = decode_string ++decode_func[b'9'] = decode_string ++ ++if sys.version_info[0] == 2 and sys.version_info[1] == 6: ++ decode_func[b'd'] = decode_dict_py26 ++else: ++ decode_func[b'd'] = decode_dict ++ ++ ++def bdecode(value): ++ """ ++ Decode bencode formatted string ``value``. ++ ++ :param value: Bencode formatted string ++ :type value: str ++ ++ :return: Decoded value ++ :rtype: object ++ """ ++ try: ++ r, l = decode_func[value[0:1]](value, 0) ++ except (IndexError, KeyError, TypeError, ValueError): ++ raise "not a valid bencoded string" ++ ++ if l != len(value): ++ raise "invalid bencoded value (data after valid prefix)" ++ ++ return r ++ ++ ++class Bencached(object): ++ __slots__ = ['bencoded'] ++ ++ def __init__(self, s): ++ self.bencoded = s ++ ++ ++def encode_bencached(x, r): ++ r.append(x.bencoded) ++ ++ ++def encode_int(x, r): ++ r.extend((b'i', str(x).encode('utf-8'), b'e')) ++ ++ ++def encode_bool(x, r): ++ if x: ++ encode_int(1, r) ++ else: ++ encode_int(0, r) ++ ++ ++def encode_bytes(x, r): ++ r.extend((str(len(x)).encode('utf-8'), b':', x)) ++ ++ ++def encode_string(x, r): ++ try: ++ s = x.encode('utf-8') ++ except UnicodeDecodeError: ++ return encode_bytes(x, r) ++ ++ r.extend((str(len(s)).encode('utf-8'), b':', s)) ++ ++ ++def encode_list(x, r): ++ r.append(b'l') ++ ++ for i in x: ++ encode_func[type(i)](i, r) ++ ++ r.append(b'e') ++ ++ ++def encode_dict(x, r): ++ r.append(b'd') ++ ilist = list(x.items()) ++ ilist.sort() ++ ++ for k, v in ilist: ++ k = k.encode('utf-8') ++ r.extend((str(len(k)).encode('utf-8'), b':', k)) ++ encode_func[type(v)](v, r) ++ ++ r.append(b'e') ++ ++ ++# noinspection PyDictCreation ++encode_func = {} ++encode_func[Bencached] = encode_bencached ++ ++if sys.version_info[0] == 2: ++ from types import DictType, IntType, ListType, LongType, StringType, TupleType, UnicodeType ++ ++ encode_func[DictType] = encode_dict ++ encode_func[IntType] = encode_int ++ encode_func[ListType] = encode_list ++ encode_func[LongType] = encode_int ++ encode_func[StringType] = encode_string ++ encode_func[TupleType] = encode_list ++ encode_func[UnicodeType] = encode_string ++ ++ if OrderedDict is not None: ++ encode_func[OrderedDict] = encode_dict ++ ++ try: ++ from types import BooleanType ++ ++ encode_func[BooleanType] = encode_bool ++ except ImportError: ++ pass ++else: ++ encode_func[OrderedDict] = encode_dict ++ encode_func[bool] = encode_bool ++ encode_func[dict] = encode_dict ++ encode_func[int] = encode_int ++ encode_func[list] = encode_list ++ encode_func[str] = encode_string ++ encode_func[tuple] = encode_list ++ encode_func[bytes] = encode_bytes ++ ++ ++def bencode(value): ++ """ ++ Encode ``value`` into the bencode format. ++ ++ :param value: Value ++ :type value: object ++ ++ :return: Bencode formatted string ++ :rtype: str ++ """ ++ r = deque() # makes more sense for something with lots of appends ++ ++ # Encode provided value ++ encode_func[type(value)](value, r) ++ ++ # Join parts ++ return b''.join(r) ++ ++ ++# Method proxies (for compatibility with other libraries) ++decode = bdecode ++encode = bencode ++ ++ ++def bread(fd): ++ """Return bdecoded data from filename, file, or file-like object. ++ ++ if fd is a bytes/string or pathlib.Path-like object, it is opened and ++ read, otherwise .read() is used. if read() not available, exception ++ raised. ++ """ ++ if isinstance(fd, (bytes, str)): ++ with open(fd, 'rb') as fd: ++ return bdecode(fd.read()) ++ elif pathlib is not None and isinstance(fd, (pathlib.Path, pathlib.PurePath)): ++ with open(str(fd), 'rb') as fd: ++ return bdecode(fd.read()) ++ else: ++ return bdecode(fd.read()) ++ ++ ++def bwrite(data, fd): ++ """Write data in bencoded form to filename, file, or file-like object. ++ ++ if fd is bytes/string or pathlib.Path-like object, it is opened and ++ written to, otherwise .write() is used. if write() is not available, ++ exception raised. ++ """ ++ if isinstance(fd, (bytes, str)): ++ with open(fd, 'wb') as fd: ++ fd.write(bencode(data)) ++ elif pathlib is not None and isinstance(fd, (pathlib.Path, pathlib.PurePath)): ++ with open(str(fd), 'wb') as fd: ++ fd.write(bencode(data)) ++ else: ++ fd.write(bencode(data)) +diff --git a/plaso/parsers/bencode_parser.py b/plaso/parsers/bencode_parser.py +index 6c933ec..18f977b 100644 +--- a/plaso/parsers/bencode_parser.py ++++ b/plaso/parsers/bencode_parser.py +@@ -10,7 +10,7 @@ from __future__ import unicode_literals + import re + import os + +-import bencode ++from bencode import bencode + + from plaso.lib import errors + from plaso.parsers import interface diff -Nru plaso-20181219/debian/patches/series plaso-20190131/debian/patches/series --- plaso-20181219/debian/patches/series 2019-01-16 10:32:57.000000000 +0000 +++ plaso-20190131/debian/patches/series 2019-02-09 22:50:11.000000000 +0000 @@ -1 +1 @@ -0001-Add-vendored-bencode-1.0.patch +0001-Add-vendored-bencode-2.0.patch diff -Nru plaso-20181219/dependencies.ini plaso-20190131/dependencies.ini --- plaso-20181219/dependencies.ini 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/dependencies.ini 2019-01-31 19:09:43.000000000 +0000 @@ -1,14 +1,14 @@ [artifacts] dpkg_name: python-artifacts minimum_version: 20170818 -rpm_name: python-artifacts +rpm_name: python2-artifacts version_property: __version__ # The bencode module does not appear to have version information. [bencode] dpkg_name: python-bencode pypi_name: bencode.py -rpm_name: python-bencode +rpm_name: python2-bencode [biplist] dpkg_name: python-biplist @@ -32,7 +32,7 @@ minimum_version: 2.6 l2tbinaries_name: pycrypto pypi_name: pycrypto -rpm_name: python-crypto +rpm_name: python2-crypto version_property: __version__ [dateutil] @@ -45,7 +45,7 @@ [dfdatetime] dpkg_name: python-dfdatetime minimum_version: 20180704 -rpm_name: python-dfdatetime +rpm_name: python2-dfdatetime version_property: __version__ [dfvfs] @@ -63,7 +63,7 @@ [dtfabric] dpkg_name: python-dtfabric minimum_version: 20181128 -rpm_name: python-dtfabric +rpm_name: python2-dtfabric version_property: __version__ # TODO: determine the version of Efilter. @@ -95,36 +95,6 @@ rpm_name: python2-future version_property: __version__ -[hachoir_core] -dpkg_name: python-hachoir-core -is_optional: true -l2tbinaries_name: hachoir-core -minimum_version: 1.3.3 -pypi_name: hachoir-core -python2_only: true -rpm_name: python-hachoir-core -version_property: __version__ - -[hachoir_metadata] -dpkg_name: python-hachoir-metadata -is_optional: true -l2tbinaries_name: hachoir-metadata -minimum_version: 1.3.3 -pypi_name: hachoir-metadata -python2_only: true -rpm_name: python-hachoir-metadata -version_property: __version__ - -[hachoir_parser] -dpkg_name: python-hachoir-parser -is_optional: true -l2tbinaries_name: hachoir-parser -minimum_version: 1.3.4 -pypi_name: hachoir-parser -python2_only: true -rpm_name: python-hachoir-parser -version_property: __version__ - [idna] dpkg_name: python-idna minimum_version: 2.5 @@ -150,7 +120,7 @@ [pefile] dpkg_name: python-pefile minimum_version: 2018.8.8 -rpm_name: python-pefile +rpm_name: python2-pefile version_property: __version__ [psutil] @@ -164,7 +134,7 @@ l2tbinaries_name: libbde minimum_version: 20140531 pypi_name: libbde-python -rpm_name: libbde-python +rpm_name: libbde-python2 version_property: get_version() [pyesedb] @@ -172,7 +142,7 @@ l2tbinaries_name: libesedb minimum_version: 20150409 pypi_name: libesedb-python -rpm_name: libesedb-python +rpm_name: libesedb-python2 version_property: get_version() [pyevt] @@ -180,7 +150,7 @@ l2tbinaries_name: libevt minimum_version: 20120410 pypi_name: libevt-python -rpm_name: libevt-python +rpm_name: libevt-python2 version_property: get_version() [pyevtx] @@ -188,7 +158,7 @@ l2tbinaries_name: libevtx minimum_version: 20141112 pypi_name: libevtx-python -rpm_name: libevtx-python +rpm_name: libevtx-python2 version_property: get_version() [pyewf] @@ -196,7 +166,7 @@ l2tbinaries_name: libewf minimum_version: 20131210 pypi_name: libewf-python -rpm_name: libewf-python +rpm_name: libewf-python2 version_property: get_version() [pyfsapfs] @@ -212,7 +182,7 @@ l2tbinaries_name: libfsntfs minimum_version: 20151130 pypi_name: libfsntfs-python -rpm_name: libfsntfs-python +rpm_name: libfsntfs-python2 version_property: get_version() [pyfvde] @@ -220,7 +190,7 @@ l2tbinaries_name: libfvde minimum_version: 20160719 pypi_name: libfvde-python -rpm_name: libfvde-python +rpm_name: libfvde-python2 version_property: get_version() [pyfwnt] @@ -228,7 +198,7 @@ l2tbinaries_name: libfwnt minimum_version: 20180117 pypi_name: libfwnt-python -rpm_name: libfwnt-python +rpm_name: libfwnt-python2 version_property: get_version() [pyfwsi] @@ -236,7 +206,7 @@ l2tbinaries_name: libfwsi minimum_version: 20150606 pypi_name: libfwsi-python -rpm_name: libfwsi-python +rpm_name: libfwsi-python2 version_property: get_version() [pylnk] @@ -244,7 +214,7 @@ l2tbinaries_name: liblnk minimum_version: 20150830 pypi_name: liblnk-python -rpm_name: liblnk-python +rpm_name: liblnk-python2 version_property: get_version() [pymsiecf] @@ -252,7 +222,7 @@ l2tbinaries_name: libmsiecf minimum_version: 20150314 pypi_name: libmsiecf-python -rpm_name: libmsiecf-python +rpm_name: libmsiecf-python2 version_property: get_version() [pyolecf] @@ -260,12 +230,12 @@ l2tbinaries_name: libolecf minimum_version: 20151223 pypi_name: libolecf-python -rpm_name: libolecf-python +rpm_name: libolecf-python2 version_property: get_version() [pyparsing] dpkg_name: python-pyparsing -minimum_version: 2.0.3 +minimum_version: 2.3.0 rpm_name: python2-pyparsing version_property: __version__ @@ -274,7 +244,7 @@ l2tbinaries_name: libqcow minimum_version: 20131204 pypi_name: libqcow-python -rpm_name: libqcow-python +rpm_name: libqcow-python2 version_property: get_version() [pyregf] @@ -282,7 +252,7 @@ l2tbinaries_name: libregf minimum_version: 20150315 pypi_name: libregf-python -rpm_name: libregf-python +rpm_name: libregf-python2 version_property: get_version() [pyscca] @@ -290,7 +260,7 @@ l2tbinaries_name: libscca minimum_version: 20161031 pypi_name: libscca-python -rpm_name: libscca-python +rpm_name: libscca-python2 version_property: get_version() [pysigscan] @@ -298,7 +268,7 @@ l2tbinaries_name: libsigscan minimum_version: 20150627 pypi_name: libsigscan-python -rpm_name: libsigscan-python +rpm_name: libsigscan-python2 version_property: get_version() [pysmdev] @@ -306,7 +276,7 @@ l2tbinaries_name: libsmdev minimum_version: 20140529 pypi_name: libsmdev-python -rpm_name: libsmdev-python +rpm_name: libsmdev-python2 version_property: get_version() [pysmraw] @@ -314,13 +284,13 @@ l2tbinaries_name: libsmraw minimum_version: 20140612 pypi_name: libsmraw-python -rpm_name: libsmraw-python +rpm_name: libsmraw-python2 version_property: get_version() [pytsk3] dpkg_name: python-pytsk3 minimum_version: 20160721 -rpm_name: python-pytsk3 +rpm_name: python2-pytsk3 version_property: get_version() [pyvhdi] @@ -328,7 +298,7 @@ l2tbinaries_name: libvhdi minimum_version: 20131210 pypi_name: libvhdi-python -rpm_name: libvhdi-python +rpm_name: libvhdi-python2 version_property: get_version() [pyvmdk] @@ -336,7 +306,7 @@ l2tbinaries_name: libvmdk minimum_version: 20140421 pypi_name: libvmdk-python -rpm_name: libvmdk-python +rpm_name: libvmdk-python2 version_property: get_version() [pyvshadow] @@ -344,7 +314,7 @@ l2tbinaries_name: libvshadow minimum_version: 20160109 pypi_name: libvshadow-python -rpm_name: libvshadow-python +rpm_name: libvshadow-python2 version_property: get_version() [pyvslvm] @@ -352,7 +322,7 @@ l2tbinaries_name: libvslvm minimum_version: 20160109 pypi_name: libvslvm-python -rpm_name: libvslvm-python +rpm_name: libvslvm-python2 version_property: get_version() # TODO: determine the version of pytz. @@ -365,13 +335,13 @@ [requests] dpkg_name: python-requests minimum_version: 2.18.0 -rpm_name: python-requests +rpm_name: python2-requests version_property: __version__ [six] dpkg_name: python-six minimum_version: 1.1.0 -rpm_name: python-six +rpm_name: python2-six version_property: __version__ [sqlite3] @@ -379,7 +349,7 @@ l2tbinaries_name: pysqlite pypi_name: pysqlite python2_only: true -rpm_name: python-pysqlite +rpm_name: python2-pysqlite version_property: __version__ [urllib3] @@ -393,7 +363,7 @@ l2tbinaries_name: XlsxWriter minimum_version: 0.9.3 pypi_name: XlsxWriter -rpm_name: python-XlsxWriter +rpm_name: python2-XlsxWriter version_property: __version__ [yaml] @@ -401,7 +371,7 @@ l2tbinaries_name: PyYAML minimum_version: 3.10 pypi_name: PyYAML -rpm_name: PyYAML +rpm_name: python2-pyyaml version_property: __version__ [yara] diff -Nru plaso-20181219/docs/conf.py plaso-20190131/docs/conf.py --- plaso-20181219/docs/conf.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/conf.py 2019-01-31 19:09:43.000000000 +0000 @@ -87,17 +87,15 @@ # to explicitly mock each submodule. # TODO: Find a better way to do this ADDITIONAL_MODULES = set([ - 'artifacts.knowledge_base', 'dateutil.parser', 'dtfabric.runtime', 'dfvfs.analyzer', - 'dfvfs.credentials', 'dfvfs.file_io', 'dfvfs.helpers', 'dfvfs.lib', - 'dfvfs.path', 'dfvfs.resolver', 'dfvfs.serializer', + 'artifacts.knowledge_base', 'dateutil.parser', 'dtfabric.runtime', + 'dfvfs.analyzer', 'dfvfs.credentials', 'dfvfs.file_io', 'dfvfs.helpers', + 'dfvfs.lib', 'dfvfs.path', 'dfvfs.resolver', 'dfvfs.serializer', 'dfvfs.serializer.json_serializer', 'dfvfs.vfs', 'dfvfs.volume', 'dfwinreg.definitions', 'efilter.protocols', 'elasticsearch', - 'elasticsearch.exceptions', 'flask', 'hachoir_core', - 'hachoir_core.config', 'hachoir_parser', 'hachoir_metadata', 'MySQLdb', - 'pyelasticsearch', 'timesketch', 'timesketch.lib', - 'timesketch.lib.datastores', 'timesketch.lib.datastores.elastic', - 'timesketch.models', 'timesketch.models.sketch', - 'timesketch.models.user', 'lz4.block']) + 'elasticsearch.exceptions', 'flask', 'MySQLdb', 'pyelasticsearch', + 'timesketch', 'timesketch.lib', 'timesketch.lib.datastores', + 'timesketch.lib.datastores.elastic', 'timesketch.models', + 'timesketch.models.sketch', 'timesketch.models.user', 'lz4.block']) modules_to_mock = set(modules_to_mock).union(ADDITIONAL_MODULES) # Readthedocs has it's own install of chardet, requests and urllib3, so remove @@ -379,7 +377,7 @@ """ if isinstance(node, nodes.reference) and 'refuri' in node: reference_uri = node['refuri'] - if reference_uri.endswith('.md'): + if reference_uri.endswith('.md') and not reference_uri.startswith('http'): reference_uri = reference_uri[:-3] + '.html' node['refuri'] = reference_uri else: diff -Nru plaso-20181219/docs/sources/developer/Dependencies-Fedora-Core.md plaso-20190131/docs/sources/developer/Dependencies-Fedora-Core.md --- plaso-20181219/docs/sources/developer/Dependencies-Fedora-Core.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/developer/Dependencies-Fedora-Core.md 2019-01-31 19:09:43.000000000 +0000 @@ -109,60 +109,6 @@ sudo rpm -ivh dfvfs-20140219/dist/dfvfs-20140219-1.noarch.rpm ``` -### IPython - -By default Fedora 20 comes with IPython 0.13.2. Plaso requires version 1.2.1 or later. - -**TODO: describe** - -### Hachoir - -Download the latest source package from: https://bitbucket.org/haypo/hachoir/wiki/Install/source - -You'll need: - -* hachoir-core-1.3.3.tar.gz -* hachoir-parser-1.3.4.tar.gz -* hachoir-metadata-1.3.3.tar.gz - -To build rpm files run the following command from the build root directory: -``` -tar xfv hachoir-core-1.3.3.tar.gz -cd hachoir-core-1.3.3 -python setup.py build bdist_rpm -cd .. -``` - -To install the required rpm files run: -``` -sudo rpm -ivh hachoir-core-1.3.3/dist/hachoir-core-1.3.3-1.noarch.rpm -``` - -To build rpm files run the following command from the build root directory: -``` -tar xfv hachoir-parser-1.3.4.tar.gz -cd hachoir-parser-1.3.4 -python setup.py build bdist_rpm -cd .. -``` - -To install the required rpm files run: -``` -sudo rpm -ivh hachoir-parser-1.3.4/dist/hachoir-parser-1.3.4-1.noarch.rpm -``` -To build rpm files run the following command from the build root directory: -``` -tar xfv hachoir-metadata-1.3.3.tar.gz -cd hachoir-metadata-1.3.3 -python setup.py build bdist_rpm -cd .. -``` - -To install the required rpm files run: -``` -sudo rpm -ivh hachoir-metadata-1.3.3/dist/hachoir-metadata-1.3.3-1.noarch.rpm -``` - ### libyal The following instructions apply to the following dependencies: diff -Nru plaso-20181219/docs/sources/developer/Dependencies-MacOS.md plaso-20190131/docs/sources/developer/Dependencies-MacOS.md --- plaso-20181219/docs/sources/developer/Dependencies-MacOS.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/developer/Dependencies-MacOS.md 2019-01-31 19:09:43.000000000 +0000 @@ -125,23 +125,6 @@ sudo installer -target / -pkg python-dfvfs-20140219.pkg ``` -### IPython -Download the latest source package from: https://github.com/ipython/ipython/releases - -To build pkg files run the following command from the build root directory: -``` -tar xfvz ipython-1.2.1.tar.gz -cd ipython-1.2.1 -python setup.py install --root=$PWD/tmp --install-data=/usr/local -pkgbuild --root tmp --identifier org.github.ipython.ipython --version 1.2.1 --ownership recommended ../ipython-1.2.1.pkg -cd .. -``` - -To install the required pkg files run: -``` -sudo installer -target / -pkg ipython-1.2.1.pkg -``` - #### gnureadline Download the latest source package from: https://pypi.python.org/pypi/gnureadline @@ -158,57 +141,6 @@ sudo installer -target / -pkg gnureadline-6.3.3.pkg ``` -### Hachoir -Download the latest source package from: https://bitbucket.org/haypo/hachoir/wiki/Install/source - -You'll need: - -* hachoir-core-1.3.3.tar.gz -* hachoir-parser-1.3.4.tar.gz -* hachoir-metadata-1.3.3.tar.gz - -To build pkg files run the following command from the build root directory: -``` -tar xfvz hachoir-core-1.3.3.tar.gz -cd hachoir-core-1.3.3 -python setup.py install --root=$PWD/tmp --install-data=/usr/local -pkgbuild --root tmp --identifier org.bitbucket.hachoir.core --version 1.3.3 --ownership recommended ../python-hachoir-core-1.3.3.pkg -cd .. -``` - -To install the required pkg files run: -``` -sudo installer -target / -pkg python-hachoir-core-1.3.3.pkg -``` - -To build pkg files run the following command from the build root directory: -``` -tar xfvz hachoir-parser-1.3.4.tar.gz -cd hachoir-parser-1.3.4 -python setup.py install --root=$PWD/tmp --install-data=/usr/local -pkgbuild --root tmp --identifier org.bitbucket.hachoir.parser --version 1.3.4 --ownership recommended ../python-hachoir-parser-1.3.4.pkg -cd .. -``` - -To install the required pkg files run: -``` -sudo installer -target / -pkg python-hachoir-parser-1.3.4.pkg -``` - -To build pkg files run the following command from the build root directory: -``` -tar xfvz hachoir-metadata-1.3.3.tar.gz -cd hachoir-metadata-1.3.3 -python setup.py install --root=$PWD/tmp --install-data=/usr/local -pkgbuild --root tmp --identifier org.bitbucket.hachoir.metadata --version 1.3.3 --ownership recommended ../python-hachoir-metadata-1.3.3.pkg -cd .. -``` - -To install the required pkg files run: -``` -sudo installer -target / -pkg python-hachoir-metadata-1.3.3.pkg -``` - ### libyal The following instructions apply to the following dependencies: diff -Nru plaso-20181219/docs/sources/developer/Dependencies.md plaso-20190131/docs/sources/developer/Dependencies.md --- plaso-20181219/docs/sources/developer/Dependencies.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/developer/Dependencies.md 2019-01-31 19:09:43.000000000 +0000 @@ -182,13 +182,13 @@ ``` set PYTHONPATH=. -C:\Python27\python.exe tools\build.py BUILD_TARGET +C:\Python27\python.exe tools\build.py --preset plaso BUILD_TARGET ``` On other platforms run: ``` -PYTHONPATH=. python tools/build.py BUILD_TARGET +PYTHONPATH=. python tools/build.py --preset plaso BUILD_TARGET ``` Where `BUILD_TARGET` is the build target for your configuration. If you are unable to find the proper build target we do not recommend using this installation method. @@ -200,4 +200,4 @@ * Dependencies-Fedora-Core * Dependencies-Mac-OS-X * Dependencies---Ubuntu -* Dependencies-Windows \ No newline at end of file +* Dependencies-Windows diff -Nru plaso-20181219/docs/sources/developer/Dependencies---Ubuntu.md plaso-20190131/docs/sources/developer/Dependencies---Ubuntu.md --- plaso-20181219/docs/sources/developer/Dependencies---Ubuntu.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/developer/Dependencies---Ubuntu.md 2019-01-31 19:09:43.000000000 +0000 @@ -128,20 +128,6 @@ sudo dpkg -i python-dfvfs_20140219-1_all.deb ``` -### Hachoir -To install hachoir run: -```bash -sudo apt-get install python-hachoir-core python-hachoir-metadata python-hachoir-parser -``` - -### IPython -To install IPython run: -```bash -sudo apt-get install ipython -``` - -For information on how to build IPython from source see: http://ipython.org/ipython-doc/stable/install/install.html - ### libyal The following instructions apply to the following dependencies: diff -Nru plaso-20181219/docs/sources/developer/Dependencies-Windows.md plaso-20190131/docs/sources/developer/Dependencies-Windows.md --- plaso-20181219/docs/sources/developer/Dependencies-Windows.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/developer/Dependencies-Windows.md 2019-01-31 19:09:43.000000000 +0000 @@ -22,7 +22,7 @@ **Note that when making MSI packages, make sure the remove the previous versions before installing the newer version.** -Alternative installation methods like installing directly from source, using easy_install or pip are [not recommended](https://stackoverflow.com/questions/3220404/why-use-pip-over-easy-install) because when not maintained correctly they can mess up your setup more easily than using MSIs. E.g. easy_installer and pip do not always remove older versions, e.g. when upgrading IPython 0.13 to 1.1, though Python distutil generated MSI packages don't detect and remove previous versions either it is less likely you'll end up with multiple different versions of the same package installed side-by-side. +Alternative installation methods like installing directly from source, using easy_install or pip are [not recommended](https://stackoverflow.com/questions/3220404/why-use-pip-over-easy-install) because when not maintained correctly they can mess up your setup more easily than using MSIs. E.g. easy_installer and pip do not always remove older versions, though Python distutil generated MSI packages don't detect and remove previous versions either it is less likely you'll end up with multiple different versions of the same package installed side-by-side. If you run into problems building, installing or running the dependencies first check: [Troubleshooting](Troubleshooting-Windows.md). @@ -75,9 +75,6 @@ dfvfs | https://github.com/log2timeline/dfvfs/releases | | dfwinreg | https://github.com/log2timeline/dfwinreg/releases | | google-apputils | https://pypi.python.org/pypi/google-apputils | | -hachoir-core | https://bitbucket.org/haypo/hachoir/wiki/Install/source | | -hachoir-parser | https://bitbucket.org/haypo/hachoir/wiki/Install/source | | -hachoir-metadata | https://bitbucket.org/haypo/hachoir/wiki/Install/source | | pefile | https://github.com/erocarrera/pefile/releases | | psutil | https://pypi.python.org/pypi/psutil | | PyParsing | http://sourceforge.net/projects/pyparsing/files/ | 2.0.3 or later 2.x version | diff -Nru plaso-20181219/docs/sources/developer/How-to-write-an-analysis-plugin.md plaso-20190131/docs/sources/developer/How-to-write-an-analysis-plugin.md --- plaso-20181219/docs/sources/developer/How-to-write-an-analysis-plugin.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/developer/How-to-write-an-analysis-plugin.md 2019-01-31 19:09:43.000000000 +0000 @@ -3,7 +3,7 @@ ## Create file and class * Plugin file in plaso/analysis/ * Create an empty subclass of plaso.analysis.interface.AnalysisPlugin - * Register it with the analysis pluging by calling AnalysisPluginManager.RegisterPlugin + * Register it with the analysis plugin by calling AnalysisPluginManager.RegisterPlugin * Test file in tests/analysis/ * Create an empty subclass of tests.analysis.test_lib.AnalysisPluginTestCase diff -Nru plaso-20181219/docs/sources/developer/Licenses-dependencies.md plaso-20190131/docs/sources/developer/Licenses-dependencies.md --- plaso-20181219/docs/sources/developer/Licenses-dependencies.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/developer/Licenses-dependencies.md 2019-01-31 19:09:43.000000000 +0000 @@ -41,7 +41,6 @@ --- | --- [bencode](https://pypi.python.org/pypi/bencode) | [BitTorrent Open Source License](http://web.archive.org/web/20080213154112/http://www.bittorrent.com/bittorrent-open-source-license) [biplist](docs/sources/developer/Licenses-dependencies.md) | [BSD 3-clause license](http://opensource.org/licenses/BSD-3-Clause) -[Hachoir](https://bitbucket.org/haypo/hachoir) | [GNU General Public License 2](http://www.gnu.org/licenses/gpl-2.0.html)
**Not integrated in the binary build.** [Libesedb and Python-bindings](https://github.com/libyal/libesedb/) | [GNU Lesser General Public License 3](http://www.gnu.org/licenses/lgpl.html) [Libevt and Python-bindings](https://github.com/libyal/libevt/) | [GNU Lesser General Public License 3](http://www.gnu.org/licenses/lgpl.html) [Libevtx and Python-bindings](https://github.com/libyal/libevtx/) | [GNU Lesser General Public License 3](http://www.gnu.org/licenses/lgpl.html) @@ -63,12 +62,6 @@ [Pyelasticsearch](https://github.com/rhec/pyelasticsearch/) | [BSD 3-clause license](http://opensource.org/licenses/BSD-3-Clause) [XlsxWriter](https://github.com/jmcnamara/XlsxWriter) | [BSD 2-clause license](http://opensource.org/licenses/BSD-2-Clause) -### Tool dependencies - -Dependency | License ---- | --- -[IPython](http://ipython.org/)
PyReadline is considered part of IPython | [BSD 3-clause license](http://opensource.org/licenses/BSD-3-Clause) - ### Windows packaged version dependencies Dependency | License diff -Nru plaso-20181219/docs/sources/user/Fedora-Core-Packaged-Release.md plaso-20190131/docs/sources/user/Fedora-Core-Packaged-Release.md --- plaso-20181219/docs/sources/user/Fedora-Core-Packaged-Release.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/user/Fedora-Core-Packaged-Release.md 2019-01-31 19:09:43.000000000 +0000 @@ -1,6 +1,6 @@ # Fedora Core Packaged Release -### Fedora Core 26, 27 and 28 +### Fedora Core 27 and 28 **Note that other versions of Fedora Core are not supported at this time.** @@ -19,5 +19,5 @@ Install plaso, tools and dependencies: ``` -sudo apt-get install python-plaso plaso-tools +sudo dnf install python-plaso plaso-tools ``` diff -Nru plaso-20181219/docs/sources/user/Parsers-and-plugins.md plaso-20190131/docs/sources/user/Parsers-and-plugins.md --- plaso-20181219/docs/sources/user/Parsers-and-plugins.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/user/Parsers-and-plugins.md 2019-01-31 19:09:43.000000000 +0000 @@ -2,8 +2,11 @@ Name | Description --- | --- -android_app_usage | Parser for the Android usage-history.xml file. +amcache | Parser for Amcache Registry entries. +android_app_usage | Parser for Android usage-history.xml files. +apache_access | Apache access Parser asl_log | Parser for ASL log files. +bash | Parser for Bash history files bencode | Parser for bencoded files. binary_cookies | Parser for Safari Binary Cookie files. bsm_log | Parser for BSM log files. @@ -11,25 +14,28 @@ chrome_preferences | Parser for Chrome Preferences files. cups_ipp | Parser for CUPS IPP files. custom_destinations | Parser for *.customDestinations-ms files. +czip | Parser for compound ZIP files. +dockerjson | Parser for JSON Docker files. +dpkg | Parser for Debian dpkg.log files. esedb | Parser for Extensible Storage Engine (ESE) database files. filestat | Parser for file system stat information. -firefox_cache | Parser for Firefox Cache files. -firefox_old_cache | Parser for Firefox Cache files. -hachoir | Parser that wraps Hachoir. +firefox_cache | Parser for Firefox Cache version 1 files (Firefox 31 or earlier). +firefox_cache2 | Parser for Firefox Cache version 2 files (Firefox 32 or later). +fsevents | Parser for fseventsd files. +gdrive_synclog | Parser for Google Drive Sync log files. java_idx | Parser for Java WebStart Cache IDX files. lnk | Parser for Windows Shortcut (LNK) files. mac_appfirewall_log | Parser for appfirewall.log files. -mac_keychain | Parser for Mac OS X Keychain files. -mac_securityd | Parser for Mac OS X securityd log files. -mactime | Parser for SleuthKit's mactime bodyfiles. -macwifi | Parser for Mac OS X wifi.log files. +mac_keychain | Parser for MacOS Keychain files. +mac_securityd | Parser for MacOS securityd log files. +mactime | Parser for SleuthKit version 3 bodyfiles. +macwifi | Parser for MacOS wifi.log files. mcafee_protection | Parser for McAfee AV Access Protection log files. +mft | Parser for NTFS $MFT metadata files. msiecf | Parser for MSIE Cache Files (MSIECF) also known as index.dat. olecf | Parser for OLE Compound Files (OLECF). -openxml | Parser for OpenXML (OXML) files. opera_global | Parser for Opera global_history.dat files. opera_typed_history | Parser for Opera typed_history.xml files. -pcap | Parser for PCAP files. pe | Parser for Portable Executable (PE) files. plist | Parser for binary and text plist files. pls_recall | Parser for PL/SQL Recall files. @@ -38,14 +44,21 @@ recycle_bin | Parser for Windows $Recycle.Bin $I files. recycle_bin_info2 | Parser for Windows Recycler INFO2 files. rplog | Parser for Windows Restore Point (rp.log) files. -selinux | Parser for SELinux audit log files. +santa | Santa Parser +sccm | Parser for SCCM logs files. +selinux | Parser for SELinux audit.log files. skydrive_log | Parser for OneDrive (or SkyDrive) log files. -skydrive_log_error | Parser for OneDrive (or SkyDrive) error log files. +skydrive_log_old | Parser for OneDrive (or SkyDrive) old log files. +sophos_av | Parser for Anti-Virus log (SAV.txt) files. sqlite | Parser for SQLite database files. symantec_scanlog | Parser for Symantec Anti-Virus log files. -syslog | Parser for syslog files. -utmp | Parser for Linux/Unix UTMP files. -utmpx | Parser for UTMPX files. +syslog | Syslog Parser +systemd_journal | Parser for Systemd Journal files. +trendmicro_url | Parser for Trend Micro Office Web Reputation log files. +trendmicro_vd | Parser for Trend Micro Office Scan Virus Detection log files. +usnjrnl | Parser for NTFS USN change journal ($UsnJrnl). +utmp | Parser for Linux libc6 utmp files. +utmpx | Parser for Mac OS X 10.5 utmpx files. winevt | Parser for Windows EventLog (EVT) files. winevtx | Parser for Windows XML EventLog (EVTX) files. winfirewall | Parser for Windows Firewall Log files. @@ -54,6 +67,7 @@ winreg | Parser for Windows NT Registry (REGF) files. xchatlog | Parser for XChat log files. xchatscrollback | Parser for XChat scrollback log files. +zsh_extended_history | Parser for ZSH extended history files ### Parser plugins: bencode @@ -62,12 +76,19 @@ bencode_transmission | Parser for Transmission bencoded files. bencode_utorrent | Parser for uTorrent bencoded files. +### Parser plugins: czip + +Name | Description +--- | --- +oxml | Parser for OpenXML (OXML) files. + ### Parser plugins: esedb Name | Description --- | --- -esedb_file_history | Parser for File History ESE database files. +file_history | Parser for File History ESE database files. msie_webcache | Parser for MSIE WebCache ESE database files. +srum | Parser for System Resource Usage Monitor (SRUM) ESE database files. ### Parser plugins: olecf @@ -87,8 +108,8 @@ ipod_device | Parser for iPod, iPad and iPhone plist files. macosx_bluetooth | Parser for Bluetooth plist files. macosx_install_history | Parser for installation history plist files. -macuser | Parser for Mac OS X user plist files. -maxos_software_update | Parser for Mac OS X software update plist files. +macuser | Parser for MacOS user plist files. +maxos_software_update | Parser for MacOS software update plist files. plist_default | Parser for plist files. safari_history | Parser for Safari history plist files. spotlight | Parser for Spotlight plist files. @@ -101,20 +122,42 @@ --- | --- android_calls | Parser for Android calls SQLite database files. android_sms | Parser for Android text messages SQLite database files. -appusage | Parser for Mac OS X application usage SQLite database files. +android_webview | Parser for Android WebView databases +android_webviewcache | Parser for Android WebViewCache databases +appusage | Parser for MacOS application usage SQLite database files. +chrome_27_history | Parser for Google Chrome 27 and up history SQLite database files. +chrome_8_history | Parser for Google Chrome 8 - 25 history SQLite database files. +chrome_autofill | Parser for Chrome autofill SQLite database files. chrome_cookies | Parser for Chrome cookies SQLite database files. chrome_extension_activity | Parser for Chrome extension activity SQLite database files. -chrome_history | Parser for Chrome history SQLite database files. firefox_cookies | Parser for Firefox cookies SQLite database files. firefox_downloads | Parser for Firefox downloads SQLite database files. firefox_history | Parser for Firefox history SQLite database files. google_drive | Parser for Google Drive SQLite database files. +hangouts_messages | Parser for Google Hangouts Messages SQLite database files. +imessage | Parser for the iMessage and SMS SQLite databases on OSX and iOS. +kik_messenger | Parser for iOS Kik messenger SQLite database files. +kodi | Parser for Kodi MyVideos.db files. ls_quarantine | Parser for LS quarantine events SQLite database files. mac_document_versions | Parser for document revisions SQLite database files. +mac_notificationcenter | Parser for the Notification Center SQLite database mackeeper_cache | Parser for MacKeeper Cache SQLite database files. +safari_history | Parser for Safari history SQLite database files. skype | Parser for Skype SQLite database files. +tango_android_profile | Parser for Tango on Android profile database. +tango_android_tc | Parser for Tango on Android tc database. +twitter_android | Parser for Twitter on android database +twitter_ios | Parser for Twitter on iOS 8+ database +windows_timeline | Parser for the Windows Timeline SQLite database zeitgeist | Parser for Zeitgeist activity SQLite database files. +### Parser plugins: syslog + +Name | Description +--- | --- +cron | Parser for syslog cron messages. +ssh | Parser for SSH syslog entries. + ### Parser plugins: winreg Name | Description @@ -133,14 +176,14 @@ mrulistex_string_and_shell_item | Parser for Most Recently Used (MRU) Registry data. mrulistex_string_and_shell_item_list | Parser for Most Recently Used (MRU) Registry data. msie_zone | Parser for Internet Explorer zone settings Registry data. -msie_zone_software | Parser for Internet Explorer zone settings Registry data. mstsc_rdp | Parser for Terminal Server Client Connection Registry data. mstsc_rdp_mru | Parser for Terminal Server Client MRU Registry data. +network_drives | Parser for Network Registry data. +networks | Parser for NetworkList data. userassist | Parser for User Assist Registry data. windows_boot_execute | Parser for Boot Execution Registry data. windows_boot_verify | Parser for Boot Verification Registry data. windows_run | Parser for run and run once Registry data. -windows_run_software | Parser for run and run once Registry data. windows_sam_users | Parser for SAM Users and Names Registry keys. windows_services | Parser for services and drivers Registry data. windows_shutdown | Parser for ShutdownTime Registry value. @@ -150,6 +193,7 @@ windows_usb_devices | Parser for USB device Registry entries. windows_usbstor_devices | Parser for USB Plug And Play Manager USBStor Registry Key. windows_version | Parser for Windows version Registry data. +winlogon | Parser for winlogon Registry data. winrar_mru | Parser for WinRAR History Registry data. winreg_default | Parser for Registry data. @@ -157,14 +201,13 @@ Name | Parsers and plugins --- | --- -android | android_app_usage, android_calls, android_sms -linux | bencode, filestat, google_drive, java_idx, olecf, openxml, pls_recall, popularity_contest, selinux, skype, syslog, utmp, webhist, xchatlog, xchatscrollback, zeitgeist -macosx | appusage, asl_log, bencode, bsm_log, cups_ipp, filestat, google_drive, java_idx, ls_quarantine, mac_appfirewall_log, mac_document_versions, mac_keychain, mac_securityd, mackeeper_cache, macwifi, olecf, openxml, plist, skype, utmpx, webhist -macosx_slow | macosx, syslog -webhist | binary_cookies, chrome_cache, chrome_cookies, chrome_extension_activity, chrome_history, chrome_preferences, firefox_cache, firefox_cookies, firefox_downloads, firefox_history, java_idx, msie_webcache, msiecf, opera_global, opera_typed_history, safari_history -win7 | recycle_bin, custom_destinations, esedb_file_history, olecf_automatic_destinations, win_gen, winevtx -win7_slow | hachoir, win7 -win_gen | bencode, esedb, filestat, google_drive, java_idx, lnk, mcafee_protection, olecf, openxml, pe, prefetch, skydrive_log_error, skydrive_log, skype, symantec_scanlog, webhist, winfirewall, winjob, winreg -winxp | recycle_bin_info2, win_gen, winevt -winxp_slow | hachoir, winxp +android | android_app_usage, filestat, chrome_cache, sqlite/android_calls, sqlite/android_sms, sqlite/android_webview, sqlite/android_webviewcache, sqlite/chrome_cookies, sqlite/chrome_8_history, sqlite/chrome_27_history, sqlite/skype +linux | bash_history, bencode, dockerjson, dpkg, filestat, sqlite/google_drive, gdrive_synclog, java_idx, olecf, czip/oxml, pls_recall, popularity_contest, selinux, sqlite/skype, syslog, systemd_journal, utmp, webhist, xchatlog, xchatscrollback, sqlite/zeitgeist, zsh_extended_history +macos | sqlite/appusage, asl_log, bash_history, bencode, bsm_log, cups_ipp, filestat, fseventsd, sqlite/google_drive, gdrive_synclog, sqlite/imessage, java_idx, sqlite/ls_quarantine, mac_appfirewall_log, sqlite/mac_document_versions, mac_keychain, mac_securityd, sqlite/mackeeper_cache, macwifi, olecf, czip/oxml, plist, sqlite/skype, syslog, utmpx, webhist, zsh_extended_history +webhist | binary_cookies, chrome_cache, sqlite/chrome_autofill, sqlite/chrome_cookies, sqlite/chrome_extension_activity, sqlite/chrome_8_history, sqlite/chrome_27_history, chrome_preferences, firefox_cache, sqlite/firefox_cookies, sqlite/firefox_downloads, sqlite/firefox_history, java_idx, esedb/msie_webcache, msiecf, opera_global, opera_typed_history, plist/safari_history +win7 | recycle_bin, custom_destinations, esedb/file_history, olecf/olecf_automatic_destinations, win_gen, winevtx, amcache +win7_slow | mft, win7 +win_gen | bencode, esedb, filestat, sqlite/google_drive, gdrive_synclog, java_idx, lnk, mcafee_protection, olecf, czip/oxml, pe, prefetch, sccm, skydrive_log, skydrive_log_old, sqlite/skype, symantec_scanlog, usnjrnl, webhist, winfirewall, winjob, winreg +winxp | recycle_bin_info2, rplog, win_gen, winevt +winxp_slow | mft, winxp diff -Nru plaso-20181219/docs/sources/user/Ubuntu-Packaged-Release.md plaso-20190131/docs/sources/user/Ubuntu-Packaged-Release.md --- plaso-20181219/docs/sources/user/Ubuntu-Packaged-Release.md 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/docs/sources/user/Ubuntu-Packaged-Release.md 2019-01-31 19:09:43.000000000 +0000 @@ -8,27 +8,16 @@ sudo apt-get install python-plaso plaso-tools ``` -### Ubuntu 14.04 and 16.04 LTS +### Ubuntu 14.04, 16.04 and 18.04 LTS -To install plaso from the GIFT Personal Package Archive (PPA) you'll need to have Ubuntu universe enabled: +To install plaso from the GIFT Personal Package Archive (PPA) add the [GIFT PPA](https://launchpad.net/~gift): ``` -sudo add-apt-repository universe -sudo apt-get update -``` - -Not necessary but we recommend that your Ubuntu installation is up to date: - -``` -sudo apt-get upgrade -``` - -Add the [GIFT PPA](https://launchpad.net/~gift): -``` sudo add-apt-repository ppa:gift/stable ``` Update and install plaso: + ``` sudo apt-get update sudo apt-get install python-plaso plaso-tools diff -Nru plaso-20181219/plaso/analysis/virustotal.py plaso-20190131/plaso/analysis/virustotal.py --- plaso-20181219/plaso/analysis/virustotal.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/analysis/virustotal.py 2019-01-31 19:09:43.000000000 +0000 @@ -150,18 +150,21 @@ response_code = hash_information['response_code'] if response_code == self._VIRUSTOTAL_NOT_PRESENT_RESPONSE_CODE: return ['virustotal_not_present'] - elif response_code == self._VIRUSTOTAL_PRESENT_RESPONSE_CODE: + + if response_code == self._VIRUSTOTAL_PRESENT_RESPONSE_CODE: positives = hash_information['positives'] if positives > 0: return ['virustotal_detections_{0:d}'.format(positives)] + return ['virsutotal_no_detections'] - elif response_code == self._VIRUSTOTAL_ANALYSIS_PENDING_RESPONSE_CODE: + + if response_code == self._VIRUSTOTAL_ANALYSIS_PENDING_RESPONSE_CODE: return ['virustotal_analysis_pending'] - else: - logger.error( - 'VirusTotal returned unknown response code {0!s}'.format( - response_code)) - return ['virustotal_unknown_response_code_{0:d}'.format(response_code)] + + logger.error( + 'VirusTotal returned unknown response code {0!s}'.format( + response_code)) + return ['virustotal_unknown_response_code_{0:d}'.format(response_code)] def SetAPIKey(self, api_key): """Sets the VirusTotal API key to use in queries. diff -Nru plaso-20181219/plaso/analysis/windows_services.py plaso-20190131/plaso/analysis/windows_services.py --- plaso-20181219/plaso/analysis/windows_services.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/analysis/windows_services.py 2019-01-31 19:09:43.000000000 +0000 @@ -263,9 +263,8 @@ event (EventObject): event to examine. """ # TODO: Handle event log entries here also (ie, event id 4697). - if getattr(event, 'data_type', None) != 'windows:registry:service': - return - else: + event_data_type = getattr(event, 'data_type', '') + if event_data_type == 'windows:registry:service': # Create and store the service. service = WindowsService.FromEvent(event) self._service_collection.AddService(service) diff -Nru plaso-20181219/plaso/cli/extraction_tool.py plaso-20190131/plaso/cli/extraction_tool.py --- plaso-20181219/plaso/cli/extraction_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/extraction_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -73,6 +73,10 @@ Returns: ProcessingConfiguration: processing configuration. + + Raises: + BadConfigOption: if more than 1 parser and parser plugins preset + was found for the detected operating system. """ # TODO: pass preferred_encoding. configuration = configurations.ProcessingConfiguration() @@ -103,16 +107,19 @@ 'operating_system_product') operating_system_version = knowledge_base.GetValue( 'operating_system_version') - parser_filter_expression = ( - parsers_manager.ParsersManager.GetPresetForOperatingSystem( + preset_definitions = ( + parsers_manager.ParsersManager.GetPresetsForOperatingSystem( operating_system, operating_system_product, operating_system_version)) - if parser_filter_expression: - logger.info('Parser filter expression changed to: {0:s}'.format( - parser_filter_expression)) - - configuration.parser_filter_expression = parser_filter_expression + if preset_definitions: + preset_names = [ + preset_definition.name for preset_definition in preset_definitions] + filter_expression = ','.join(preset_names) + + logger.info('Parser filter expression set to: {0:s}'.format( + filter_expression)) + configuration.parser_filter_expression = filter_expression return configuration @@ -138,7 +145,7 @@ self._buffer_size = int(self._buffer_size, 10) except ValueError: raise errors.BadConfigOption( - 'Invalid buffer size: {0:s}.'.format(self._buffer_size)) + 'Invalid buffer size: {0!s}.'.format(self._buffer_size)) self._queue_size = self.ParseNumericOption(options, 'queue_size') diff -Nru plaso-20181219/plaso/cli/helpers/analysis_plugins.py plaso-20190131/plaso/cli/helpers/analysis_plugins.py --- plaso-20181219/plaso/cli/helpers/analysis_plugins.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/helpers/analysis_plugins.py 2019-01-31 19:09:43.000000000 +0000 @@ -41,7 +41,7 @@ if '--analysis' in arguments: argument_index = arguments.index('--analysis') + 1 - if argument_index > 0 and argument_index < len(arguments): + if 0 < argument_index < len(arguments): names = [name.strip() for name in arguments[argument_index].split(',')] else: names = None diff -Nru plaso-20181219/plaso/cli/helpers/artifact_filters.py plaso-20190131/plaso/cli/helpers/artifact_filters.py --- plaso-20181219/plaso/cli/helpers/artifact_filters.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/helpers/artifact_filters.py 2019-01-31 19:09:43.000000000 +0000 @@ -30,8 +30,8 @@ """ argument_group.add_argument( '--artifact_filters', '--artifact-filters', - dest='artifact_filters', type=str, default=None, - action='store', help=( + dest='artifact_filter_string', type=str, default=None, + metavar='ARTIFACT_FILTERS', action='store', help=( 'Names of forensic artifact definitions, provided on the command ' 'command line (comma separated). Forensic artifacts are stored ' 'in .yaml files that are directly pulled from the artifact ' @@ -69,7 +69,7 @@ raise errors.BadConfigObject( 'Configuration object is not an instance of CLITool') - artifact_filters = cls._ParseStringOption(options, 'artifact_filters') + artifact_filters = cls._ParseStringOption(options, 'artifact_filter_string') artifact_filters_file = cls._ParseStringOption( options, 'artifact_filters_file') filter_file = cls._ParseStringOption(options, 'file_filter') diff -Nru plaso-20181219/plaso/cli/helpers/data_location.py plaso-20190131/plaso/cli/helpers/data_location.py --- plaso-20181219/plaso/cli/helpers/data_location.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/helpers/data_location.py 2019-01-31 19:09:43.000000000 +0000 @@ -69,6 +69,7 @@ data_location_egg = os.path.join(data_location, 'share', 'plaso') data_location_source = os.path.join(data_location, 'data') + data_location = None if os.path.exists(data_location_egg) and os.path.isfile(os.path.join( data_location_egg, 'plaso-data.README')): data_location = data_location_egg @@ -76,7 +77,7 @@ data_location_source, 'plaso-data.README')): data_location = data_location_source - if not os.path.exists(data_location): + if not data_location or not os.path.exists(data_location): data_location = os.path.join(sys.prefix, 'share', 'plaso') if not os.path.exists(data_location): data_location = os.path.join(sys.prefix, 'local', 'share', 'plaso') @@ -87,7 +88,7 @@ if not os.path.exists(data_location): data_location = os.path.join('/usr', 'local', 'share', 'plaso') - if not os.path.exists(data_location) and os.path.isfile(os.path.join( + if not os.path.exists(data_location) or not os.path.isfile(os.path.join( data_location, 'plaso-data.README')): data_location = None diff -Nru plaso-20181219/plaso/cli/helpers/filter_file.py plaso-20190131/plaso/cli/helpers/filter_file.py --- plaso-20181219/plaso/cli/helpers/filter_file.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/helpers/filter_file.py 2019-01-31 19:09:43.000000000 +0000 @@ -47,6 +47,7 @@ Raises: BadConfigObject: when the configuration object is of the wrong type. + BadConfigOption: if the collection file does not exist. """ if not isinstance(configuration_object, tools.CLITool): raise errors.BadConfigObject( diff -Nru plaso-20181219/plaso/cli/helpers/output_modules.py plaso-20190131/plaso/cli/helpers/output_modules.py --- plaso-20181219/plaso/cli/helpers/output_modules.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/helpers/output_modules.py 2019-01-31 19:09:43.000000000 +0000 @@ -52,7 +52,7 @@ elif '--output-format' in arguments: argument_index = arguments.index('--output-format') + 1 - if argument_index > 0 and argument_index < len(arguments): + if 0 < argument_index < len(arguments): names = [name.strip() for name in arguments[argument_index].split(',')] else: names = ['dynamic'] diff -Nru plaso-20181219/plaso/cli/image_export_tool.py plaso-20190131/plaso/cli/image_export_tool.py --- plaso-20181219/plaso/cli/image_export_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/image_export_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -443,6 +443,8 @@ Raises: IOError: if the format specification file could not be read from the specified data location. + OSError: if the format specification file could not be read from + the specified data location. ValueError: if no data location was specified. """ if not signature_identifiers: diff -Nru plaso-20181219/plaso/cli/log2timeline_tool.py plaso-20190131/plaso/cli/log2timeline_tool.py --- plaso-20181219/plaso/cli/log2timeline_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/log2timeline_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -4,6 +4,7 @@ from __future__ import unicode_literals import argparse +import os import sys import time import textwrap @@ -293,6 +294,13 @@ helpers_manager.ArgumentHelperManager.ParseOptions( options, self, names=['data_location']) + presets_file = os.path.join(self._data_location, 'presets.yaml') + if not os.path.isfile(presets_file): + raise errors.BadConfigOption( + 'No such parser presets file: {0:s}.'.format(presets_file)) + + parsers_manager.ParsersManager.ReadPresetsFromFile(presets_file) + # Check the list options first otherwise required options will raise. argument_helper_names = ['hashers', 'parsers', 'profiling'] helpers_manager.ArgumentHelperManager.ParseOptions( @@ -360,7 +368,7 @@ Raises: BadConfigOption: if the storage file path is invalid or the storage - format not supported. + format not supported or an invalid filter was specified. SourceScannerError: if the source scanner could not find a supported file system. UserAbort: if the user initiated an abort. @@ -420,10 +428,14 @@ self._SetExtractionParsersAndPlugins(configuration, session) self._SetExtractionPreferredTimeZone(extraction_engine.knowledge_base) - filter_find_specs = engine.BaseEngine.BuildFilterFindSpecs( - self._artifact_definitions_path, self._custom_artifacts_path, - extraction_engine.knowledge_base, self._artifact_filters, - self._filter_file) + try: + filter_find_specs = engine.BaseEngine.BuildFilterFindSpecs( + self._artifact_definitions_path, self._custom_artifacts_path, + extraction_engine.knowledge_base, self._artifact_filters, + self._filter_file) + except errors.InvalidFilter as exception: + raise errors.BadConfigOption( + 'Unable to build filter specification: {0!s}'.format(exception)) processing_status = None if single_process_mode: diff -Nru plaso-20181219/plaso/cli/pinfo_tool.py plaso-20190131/plaso/cli/pinfo_tool.py --- plaso-20181219/plaso/cli/pinfo_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/pinfo_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -358,7 +358,10 @@ # bytes. if isinstance(preferred_encoding, py2to3.BYTES_TYPE): preferred_encoding = preferred_encoding.decode('utf-8') - artifact_filters = session.artifact_filters or 'N/A' + if session.artifact_filters: + artifact_filters_string = ', '.join(session.artifact_filters) + else: + artifact_filters_string = 'N/A' filter_file = session.filter_file or 'N/A' title = 'Session: {0!s}'.format(session_identifier) @@ -374,7 +377,7 @@ table_view.AddRow(['Enabled parser and plugins', enabled_parser_names]) table_view.AddRow(['Preferred encoding', preferred_encoding]) table_view.AddRow(['Debug mode', session.debug_mode]) - table_view.AddRow(['Artifact filters', artifact_filters]) + table_view.AddRow(['Artifact filters', artifact_filters_string]) table_view.AddRow(['Filter file', filter_file]) table_view.Write(self._output_writer) diff -Nru plaso-20181219/plaso/cli/psort_tool.py plaso-20190131/plaso/cli/psort_tool.py --- plaso-20181219/plaso/cli/psort_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/psort_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -36,7 +36,7 @@ from plaso.multi_processing import psort from plaso.storage import factory as storage_factory -import pytz +import pytz # pylint: disable=wrong-import-order class PsortTool( @@ -162,15 +162,17 @@ """ # Get a list of all available plugins. analysis_plugin_info = self._analysis_manager.GetAllPluginInformation() - analysis_plugin_names = set([ - name.lower() for name, _, _ in analysis_plugin_info]) + # Use set-comprehension to create a set of the analysis plugin names. + analysis_plugin_names = { + name.lower() for name, _, _ in analysis_plugin_info} analysis_plugins = self.ParseStringOption(options, 'analysis_plugins') if not analysis_plugins: return - requested_plugin_names = set([ - name.strip().lower() for name in analysis_plugins.split(',')]) + # Use set-comprehension to create a set of the requested plugin names. + requested_plugin_names = { + name.strip().lower() for name in analysis_plugins.split(',')} # Check to see if we are trying to load plugins that do not exist. difference = requested_plugin_names.difference(analysis_plugin_names) diff -Nru plaso-20181219/plaso/cli/psteal_tool.py plaso-20190131/plaso/cli/psteal_tool.py --- plaso-20181219/plaso/cli/psteal_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/psteal_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -446,6 +446,13 @@ helpers_manager.ArgumentHelperManager.ParseOptions( options, self, names=['data_location']) + presets_file = os.path.join(self._data_location, 'presets.yaml') + if not os.path.isfile(presets_file): + raise errors.BadConfigOption( + 'No such parser presets file: {0:s}.'.format(presets_file)) + + parsers_manager.ParsersManager.ReadPresetsFromFile(presets_file) + # The output modules options are dependent on the preferred language # and preferred time zone options. self._ParseTimezoneOption(options) diff -Nru plaso-20181219/plaso/cli/status_view.py plaso-20190131/plaso/cli/status_view.py --- plaso-20181219/plaso/cli/status_view.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/status_view.py 2019-01-31 19:09:43.000000000 +0000 @@ -159,7 +159,7 @@ used_memory_1024 /= 1024 magnitude_1024 += 1 - if magnitude_1024 > 0 and magnitude_1024 <= 7: + if 0 < magnitude_1024 <= 7: return '{0:.1f} {1:s}'.format( used_memory_1024, self._UNITS_1024[magnitude_1024]) @@ -285,11 +285,12 @@ """Retrieves the analysis status update callback function. Returns: - function: status update callback function or None. + function: status update callback function or None if not available. """ if self._mode == self.MODE_LINEAR: return self._PrintAnalysisStatusUpdateLinear - elif self._mode == self.MODE_WINDOW: + + if self._mode == self.MODE_WINDOW: return self._PrintAnalysisStatusUpdateWindow return None @@ -298,11 +299,12 @@ """Retrieves the extraction status update callback function. Returns: - function: status update callback function or None. + function: status update callback function or None if not available. """ if self._mode == self.MODE_LINEAR: return self._PrintExtractionStatusUpdateLinear - elif self._mode == self.MODE_WINDOW: + + if self._mode == self.MODE_WINDOW: return self._PrintExtractionStatusUpdateWindow return None @@ -320,8 +322,9 @@ 'Source type\t: {0:s}\n'.format(self._source_type)) if self._artifact_filters: + artifacts_string = ', '.join(self._artifact_filters) self._output_writer.Write('Artifact filters\t: {0:s}\n'.format( - self._artifact_filters)) + artifacts_string)) if self._filter_file: self._output_writer.Write('Filter file\t: {0:s}\n'.format( self._filter_file)) @@ -411,8 +414,8 @@ Args: source_path (str): path of the source. source_type (str): source type. - artifact_filters (Optional[str]): names of artifact definitions to use as - filters. + artifact_filters (Optional[list[str]]): names of artifact definitions to + use as filters. filter_file (Optional[str]): filter file. """ self._artifact_filters = artifact_filters diff -Nru plaso-20181219/plaso/cli/tool_options.py plaso-20190131/plaso/cli/tool_options.py --- plaso-20181219/plaso/cli/tool_options.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/cli/tool_options.py 2019-01-31 19:09:43.000000000 +0000 @@ -17,7 +17,6 @@ from plaso.output import manager as output_manager from plaso.output import mediator as output_mediator from plaso.parsers import manager as parsers_manager -from plaso.parsers import presets as parsers_presets from plaso.winnt import language_ids @@ -237,8 +236,11 @@ str: parsers names corresponding to the preset """ parser_presets_information = [] - for preset_name, parser_names in sorted(parsers_presets.CATEGORIES.items()): - parser_presets_information.append((preset_name, ', '.join(parser_names))) + for preset_definition in parsers_manager.ParsersManager.GetPresets(): + preset_information_tuple = ( + preset_definition.name, ', '.join(preset_definition.parsers)) + # TODO: refactor to pass PresetDefinition. + parser_presets_information.append(preset_information_tuple) return parser_presets_information diff -Nru plaso-20181219/plaso/containers/artifacts.py plaso-20190131/plaso/containers/artifacts.py --- plaso-20181219/plaso/containers/artifacts.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/containers/artifacts.py 2019-01-31 19:09:43.000000000 +0000 @@ -5,6 +5,7 @@ from plaso.containers import interface from plaso.containers import manager +from plaso.lib import definitions class ArtifactAttributeContainer(interface.AttributeContainer): @@ -66,6 +67,153 @@ self.schema = schema +class OperatingSystemArtifact(ArtifactAttributeContainer): + """Operating system artifact attribute container. + + Attributes: + family (str): operating system family name, such as "Linux", "MacOS" + or "Windows", defined in definitions.OPERATING_SYSTEM_FAMILIES. + This value is used to programmatically link a parser preset to an + operating system and therefore must be one of predefined values. + name (str): operating system name, such as "macOS Mojave" or "Windows XP". + This value is used to programmatically link a parser preset to an + operating system and therefore must be one of predefined values. + product (str): product information, such as "macOS Mojave" or "Windows + Professional XP". This value is typically obtained from the source data. + version (str): version, such as "10.14.1" or "5.1". This value is typically + obtained from the source data. + """ + CONTAINER_TYPE = 'operating_system' + + _FAMILY_AND_VERSION_PER_NAME = { + 'Windows 2000': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (5, 0)), + 'Windows 2003': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (5, 2)), + 'Windows 2003 R2': ( + definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (5, 2)), + 'Windows 2008': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (6, 0)), + 'Windows 2008 R2': ( + definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (6, 1)), + 'Windows 2012': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (6, 2)), + 'Windows 2012 R2': ( + definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (6, 3)), + 'Windows 2016': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (10, 0)), + 'Windows 2019': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (10, 0)), + 'Windows 7': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (6, 1)), + 'Windows 8': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (6, 2)), + 'Windows 8.1': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (6, 3)), + 'Windows 10': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (10, 0)), + 'Windows Vista': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (6, 0)), + 'Windows XP': (definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, (5, 1))} + + def __init__(self, family=None, product=None, version=None): + """Initializes an operating system artifact. + + Args: + family (Optional[str]): operating system family name, such as "Linux", + "MacOS" or "Windows", defined in definitions.OPERATING_SYSTEM_FAMILIES. + This value is used to programmatically link a parser preset to an + operating system and therefore must be one of predefined values. + product (Optional[str]): product information, such as "macOS Mojave" or + "Windows Professional XP". This value is typically obtained from the + source data. + version (Optional[str]): version, such as "10.14.1" or "5.1". This value + is typically obtained from the source data. + """ + super(OperatingSystemArtifact, self).__init__() + self.family = family + self.name = None + self.product = product + self.version = version + + if product: + self.name = self._GetNameFromProduct() + + @property + def version_tuple(self): + """tuple[int]: version tuple or None if version is not set or invalid.""" + try: + return tuple([int(digit, 10) for digit in self.version.split('.')]) + except (AttributeError, TypeError, ValueError): + return None + + def _GetNameFromProduct(self): + """Determines the predefined operating system name from the product. + + Returns: + str: operating system name, such as "macOS Mojave" or "Windows XP" or + None if the name cannot be determined. This value is used to + programmatically link a parser preset to an operating system and + therefore must be one of predefined values. + """ + product = self.product or '' + product = product.split(' ') + product_lower_case = [segment.lower() for segment in product] + number_of_segments = len(product) + + if 'windows' in product_lower_case: + segment_index = product_lower_case.index('windows') + 1 + if product_lower_case[segment_index] == 'server': + segment_index += 1 + + # Check if the version has a suffix. + suffix_segment_index = segment_index + 1 + if (suffix_segment_index < number_of_segments and + product_lower_case[suffix_segment_index] == 'r2'): + return 'Windows {0:s} R2'.format(product[segment_index]) + + return 'Windows {0:s}'.format(product[segment_index]) + + return None + + def IsEquivalent(self, other): + """Determines if 2 operating system artifacts are equivalent. + + This function compares the operating systems based in order of: + * name derived from product + * family and version + * family + + Args: + other (OperatingSystemArtifact): operating system artifact attribute + container to compare with. + + Returns: + bool: True if the operating systems are considered equivalent, False if + the most specific criteria do no match, or no criteria are available. + """ + if self.name and other.name: + return self.name == other.name + + if self.name: + self_family, self_version_tuple = self._FAMILY_AND_VERSION_PER_NAME.get( + self.name) + return ( + self_family == other.family and + self_version_tuple == other.version_tuple) + + if self.family and self.version: + if other.name: + other_family, other_version_tuple = ( + self._FAMILY_AND_VERSION_PER_NAME.get(other.name)) + else: + other_family = other.family + other_version_tuple = other.version_tuple + + return ( + self.family == other_family and + self.version_tuple == other_version_tuple) + + if self.family: + if other.name: + other_family, _ = self._FAMILY_AND_VERSION_PER_NAME.get(other.name) + else: + other_family = other.family + + return self.family == other_family + + return False + + class SystemConfigurationArtifact(ArtifactAttributeContainer): """System configuration artifact attribute container. diff -Nru plaso-20181219/plaso/containers/interface.py plaso-20190131/plaso/containers/interface.py --- plaso-20181219/plaso/containers/interface.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/containers/interface.py 2019-01-31 19:09:43.000000000 +0000 @@ -3,10 +3,10 @@ from __future__ import unicode_literals -from plaso.lib import py2to3 - from efilter.protocols import structured +from plaso.lib import py2to3 + class AttributeContainerIdentifier(object): """The attribute container identifier. diff -Nru plaso-20181219/plaso/dependencies.py plaso-20190131/plaso/dependencies.py --- plaso-20181219/plaso/dependencies.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/dependencies.py 2019-01-31 19:09:43.000000000 +0000 @@ -34,9 +34,6 @@ 'elasticsearch': ('__versionstr__', '6.0', None, False), 'elasticsearch5': ('__versionstr__', '5.4.0', None, True), 'future': ('__version__', '0.16.0', None, True), - 'hachoir_core': ('__version__', '1.3.3', None, False), - 'hachoir_metadata': ('__version__', '1.3.3', None, False), - 'hachoir_parser': ('__version__', '1.3.4', None, False), 'idna': ('', '2.5', None, True), 'lz4': ('', '0.10.0', None, False), 'lzma': ('__version__', '', None, False), @@ -55,7 +52,7 @@ 'pylnk': ('get_version()', '20150830', None, True), 'pymsiecf': ('get_version()', '20150314', None, True), 'pyolecf': ('get_version()', '20151223', None, True), - 'pyparsing': ('__version__', '2.0.3', None, True), + 'pyparsing': ('__version__', '2.3.0', None, True), 'pyqcow': ('get_version()', '20131204', None, True), 'pyregf': ('get_version()', '20150315', None, True), 'pyscca': ('get_version()', '20161031', None, True), diff -Nru plaso-20181219/plaso/engine/artifact_filters.py plaso-20190131/plaso/engine/artifact_filters.py --- plaso-20181219/plaso/engine/artifact_filters.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/engine/artifact_filters.py 2019-01-31 19:09:43.000000000 +0000 @@ -44,7 +44,6 @@ self._artifacts = artifact_filters self._artifacts_registry = artifacts_registry self._knowledge_base = knowledge_base - self._find_specs_per_source_type = defaultdict(list) @staticmethod def CheckKeyCompatibility(key_path): @@ -75,68 +74,116 @@ environment_variables (Optional[list[EnvironmentVariableArtifact]]): environment variables. """ + find_specs = [] for name in self._artifacts: definition = self._artifacts_registry.GetDefinitionByName(name) if not definition: continue - for source in definition.sources: - if source.type_indicator == artifact_types.TYPE_INDICATOR_FILE: - # TODO: move source.paths iteration into - # BuildFindSpecsFromFileArtifact. - for path_entry in set(source.paths): - find_specs = self.BuildFindSpecsFromFileArtifact( - path_entry, source.separator, environment_variables, - self._knowledge_base.user_accounts) - artifact_group = self._find_specs_per_source_type[ - artifact_types.TYPE_INDICATOR_FILE] - artifact_group.extend(find_specs) - - elif (source.type_indicator == - artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY): - # TODO: move source.keys iteration into - # BuildFindSpecsFromRegistryArtifact. - for key_path in set(source.keys): - if self.CheckKeyCompatibility(key_path): - find_specs = self.BuildFindSpecsFromRegistryArtifact(key_path) - artifact_group = self._find_specs_per_source_type[ - artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY] - artifact_group.extend(find_specs) - - elif (source.type_indicator == - artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_VALUE): - # TODO: Handle Registry Values Once Supported in dfwinreg. - # https://github.com/log2timeline/dfwinreg/issues/98 - logger.warning(( - 'Windows Registry values are not supported, extracting key: ' - '"{0!s}"').format(source.key_value_pairs)) + artifact_find_specs = self._BuildFindSpecsFromArtifact( + definition, environment_variables) + find_specs.extend(artifact_find_specs) + + find_specs_per_source_type = defaultdict(list) + for find_spec in find_specs: + if isinstance(find_spec, registry_searcher.FindSpec): + artifact_list = find_specs_per_source_type[ + artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY] + artifact_list.append(find_spec) + continue - # TODO: move source.key_value_pairs iteration into - # BuildFindSpecsFromRegistryArtifact. - for key_path in set([ - key_value['key'] for key_value in source.key_value_pairs]): - if self.CheckKeyCompatibility(key_path): - find_specs = self.BuildFindSpecsFromRegistryArtifact(key_path) - artifact_group = self._find_specs_per_source_type[ - artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY] - artifact_group.extend(find_specs) - - elif (source.type_indicator == - artifact_types.TYPE_INDICATOR_ARTIFACT_GROUP): - self._artifacts.remove(name) - for name_entry in set(source.names): - self._artifacts.append(name_entry) - self.BuildFindSpecs(environment_variables=environment_variables) + if isinstance(find_spec, file_system_searcher.FindSpec): + artifact_list = find_specs_per_source_type[ + artifact_types.TYPE_INDICATOR_FILE] + artifact_list.append(find_spec) + continue - else: - logger.warning( - 'Unsupported artifact definition source type: "{0:s}"'.format( - source.type_indicator)) + logger.warning('Unknown find specification type: {0:s}'.format( + type(find_spec))) self._knowledge_base.SetValue( - self.KNOWLEDGE_BASE_VALUE, self._find_specs_per_source_type) + self.KNOWLEDGE_BASE_VALUE, find_specs_per_source_type) + + def _BuildFindSpecsFromArtifact(self, definition, environment_variables): + """Builds find specifications from an artifact definition. + + Args: + definition (artifacts.ArtifactDefinition): artifact definition. + environment_variables (list[EnvironmentVariableArtifact]): + environment variables. + + Returns: + list[dfwinreg.FindSpec|dfvfs.FindSpec]: find specifications. + """ + find_specs = [] + for source in definition.sources: + if source.type_indicator == artifact_types.TYPE_INDICATOR_FILE: + for path_entry in set(source.paths): + specifications = self._BuildFindSpecsFromFileSourcePath( + path_entry, source.separator, environment_variables, + self._knowledge_base.user_accounts) + find_specs.extend(specifications) + + elif (source.type_indicator == + artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY): + for key_path in set(source.keys): + if self.CheckKeyCompatibility(key_path): + specifications = self._BuildFindSpecsFromRegistrySourceKey(key_path) + find_specs.extend(specifications) + + elif (source.type_indicator == + artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_VALUE): + # TODO: Handle Registry Values Once Supported in dfwinreg. + # https://github.com/log2timeline/dfwinreg/issues/98 + + # Use set-comprehension to create a set of the source key paths. + key_paths = { + key_value['key'] for key_value in source.key_value_pairs} + key_paths_string = ', '.join(key_paths) + + logger.warning(( + 'Windows Registry values are not supported, extracting keys: ' + '"{0!s}"').format(key_paths_string)) + + for key_path in key_paths: + if self.CheckKeyCompatibility(key_path): + specifications = self._BuildFindSpecsFromRegistrySourceKey(key_path) + find_specs.extend(specifications) + + elif (source.type_indicator == + artifact_types.TYPE_INDICATOR_ARTIFACT_GROUP): + for name in source.names: + specifications = self._BuildFindSpecsFromGroupName( + name, environment_variables) + find_specs.extend(specifications) + + else: + logger.warning( + 'Unsupported artifact definition source type: "{0:s}"'.format( + source.type_indicator)) + + return find_specs + + def _BuildFindSpecsFromGroupName(self, group_name, environment_variables): + """Builds find specifications from a artifact group name. + + Args: + group_name (str): artifact group name. + environment_variables (list[str]): environment variable attributes used to + dynamically populate environment variables in file and registry + artifacts. + + Returns: + list[dfwinreg.FindSpec|dfvfs.FindSpec]: find specifications or None if no + artifact with the given name can be retrieved. + """ + definition = self._artifacts_registry.GetDefinitionByName(group_name) + if not definition: + return None + return self._BuildFindSpecsFromArtifact(definition, environment_variables) + - def BuildFindSpecsFromFileArtifact( + def _BuildFindSpecsFromFileSourcePath( self, source_path, path_separator, environment_variables, user_accounts): """Builds find specifications from a file source type. @@ -191,24 +238,24 @@ return find_specs - def BuildFindSpecsFromRegistryArtifact(self, source_key_path): + def _BuildFindSpecsFromRegistrySourceKey(self, key_path): """Build find specifications from a Windows Registry source type. Args: - source_key_path (str): Windows Registry key path defined by the source. + key_path (str): Windows Registry key path defined by the source. Returns: list[dfwinreg.FindSpec]: find specifications for the Windows Registry source type. """ find_specs = [] - for key_path in path_helper.PathHelper.ExpandRecursiveGlobs( - source_key_path, '\\'): - if '%%' in key_path: - logger.error('Unable to expand key path: "{0:s}"'.format(key_path)) + for key_path_glob in path_helper.PathHelper.ExpandRecursiveGlobs( + key_path, '\\'): + if '%%' in key_path_glob: + logger.error('Unable to expand key path: "{0:s}"'.format(key_path_glob)) continue - find_spec = registry_searcher.FindSpec(key_path_glob=key_path) + find_spec = registry_searcher.FindSpec(key_path_glob=key_path_glob) find_specs.append(find_spec) return find_specs diff -Nru plaso-20181219/plaso/engine/engine.py plaso-20190131/plaso/engine/engine.py --- plaso-20181219/plaso/engine/engine.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/engine/engine.py 2019-01-31 19:09:43.000000000 +0000 @@ -60,7 +60,7 @@ Returns: str: operating system for example "Windows". This should be one of - the values in definitions.OPERATING_SYSTEMS. + the values in definitions.OPERATING_SYSTEM_FAMILIES. """ find_specs = [ file_system_searcher.FindSpec( @@ -90,15 +90,15 @@ '\\winnt\\system32', '/winnt35/system32', '\\winnt35\\system32', '\\wtsrv\\system32', '/wtsrv/system32']) - operating_system = definitions.OPERATING_SYSTEM_UNKNOWN + operating_system = definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN if windows_locations.intersection(set(locations)): - operating_system = definitions.OPERATING_SYSTEM_WINDOWS + operating_system = definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT elif '/system/library' in locations: - operating_system = definitions.OPERATING_SYSTEM_MACOS + operating_system = definitions.OPERATING_SYSTEM_FAMILY_MACOS elif '/etc' in locations: - operating_system = definitions.OPERATING_SYSTEM_LINUX + operating_system = definitions.OPERATING_SYSTEM_FAMILY_LINUX return operating_system @@ -264,7 +264,7 @@ file_system, mount_point) operating_system = self._DetermineOperatingSystem(searcher) - if operating_system != definitions.OPERATING_SYSTEM_UNKNOWN: + if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN: preprocess_manager.PreprocessPluginsManager.RunPlugins( artifacts_registry_object, file_system, mount_point, self.knowledge_base) @@ -302,13 +302,13 @@ artifact_filter_names (Optional[list[str]]): names of artifact definitions that are used for filtering file system and Windows Registry key paths. - filter_file_path (Optional[str]): Path of filter file. + filter_file_path (Optional[str]): path of filter file. Returns: list[dfvfs.FindSpec]: find specifications for the file source type. Raises: - RuntimeError: if no valid FindSpecs are built. + InvalidFilter: if no valid FindSpecs are built. """ environment_variables = knowledge_base_object.GetEnvironmentVariables() find_specs = None @@ -329,7 +329,7 @@ environment_variables=environment_variables) if (artifact_filter_names or filter_file_path) and not find_specs: - raise RuntimeError( + raise errors.InvalidFilter( 'Error processing filters, no valid specifications built.') return find_specs diff -Nru plaso-20181219/plaso/engine/extractors.py plaso-20190131/plaso/engine/extractors.py --- plaso-20181219/plaso/engine/extractors.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/engine/extractors.py 2019-01-31 19:09:43.000000000 +0000 @@ -44,7 +44,7 @@ can have the value of: * An exact match of a list of parsers, or a preset (see - plaso/parsers/presets.py for a full list of available presets). + data/presets.yaml for the list of predefined presets). * A name of a single parser (case insensitive), e.g. msiecf. * A glob name for a single parser, e.g. '*msie*' (case insensitive). """ @@ -114,7 +114,7 @@ can have the value of: * An exact match of a list of parsers, or a preset (see - plaso/parsers/presets.py for a full list of available presets). + data/presets.yaml for the list of predefined presets). * A name of a single parser (case insensitive), e.g. msiecf. * A glob name for a single parser, e.g. '*msie*' (case insensitive). """ @@ -281,10 +281,11 @@ parse_result = self._ParseFileEntryWithParser( parser_mediator, parser, file_entry, file_object=file_object) + if parse_result == self._PARSE_RESULT_FAILURE: return self._PARSE_RESULT_FAILURE - elif parse_result == self._PARSE_RESULT_SUCCESS: + if parse_result == self._PARSE_RESULT_SUCCESS: parse_results = self._PARSE_RESULT_SUCCESS return parse_results diff -Nru plaso-20181219/plaso/engine/path_helper.py plaso-20190131/plaso/engine/path_helper.py --- plaso-20181219/plaso/engine/path_helper.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/engine/path_helper.py 2019-01-31 19:09:43.000000000 +0000 @@ -98,7 +98,7 @@ Expands the GRR artifacts path variable "%%users.homedir%%". Args: - path (str): Windows path with environment variables. + path (str): path with environment variables. path_separator (str): path segment separator. user_accounts (list[UserAccountArtifact]): user accounts. @@ -120,7 +120,7 @@ user_paths = [] for user_account in user_accounts: user_path = user_account.user_directory - # Prevent concatting 2 path segment separators. + # Prevent concatenating two consecuative path segment separators. if user_path[-1] == path_separator and path[0] == path_separator: user_path = user_path[:-1] diff -Nru plaso-20181219/plaso/engine/process_info.py plaso-20190131/plaso/engine/process_info.py --- plaso-20181219/plaso/engine/process_info.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/engine/process_info.py 2019-01-31 19:09:43.000000000 +0000 @@ -17,6 +17,7 @@ Raises: IOError: If the process identified by the PID does not exist. + OSError: If the process identified by the PID does not exist. """ if not psutil.pid_exists(pid): raise IOError('Process with PID: {0:d} does not exist'.format(pid)) diff -Nru plaso-20181219/plaso/engine/profilers.py plaso-20190131/plaso/engine/profilers.py --- plaso-20181219/plaso/engine/profilers.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/engine/profilers.py 2019-01-31 19:09:43.000000000 +0000 @@ -45,10 +45,10 @@ """Shared functionality for sample file-based profilers.""" - # This constant must be overriden by subclasses. + # This constant must be overridden by subclasses. _FILENAME_PREFIX = None - # This constant must be overriden by subclasses. + # This constant must be overridden by subclasses. _FILE_HEADER = None def __init__(self, identifier, configuration): diff -Nru plaso-20181219/plaso/engine/worker.py plaso-20190131/plaso/engine/worker.py --- plaso-20181219/plaso/engine/worker.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/engine/worker.py 2019-01-31 19:09:43.000000000 +0000 @@ -86,7 +86,7 @@ can have the value of: * An exact match of a list of parsers, or a preset (see - plaso/parsers/presets.py for a full list of available presets). + data/presets.yaml for the list of predefined presets). * A name of a single parser (case insensitive), e.g. msiecf. * A glob name for a single parser, e.g. '*msie*' (case insensitive). """ diff -Nru plaso-20181219/plaso/formatters/chrome_preferences.py plaso-20190131/plaso/formatters/chrome_preferences.py --- plaso-20181219/plaso/formatters/chrome_preferences.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/formatters/chrome_preferences.py 2019-01-31 19:09:43.000000000 +0000 @@ -108,7 +108,7 @@ if primary_url == '': subject = 'local file' - elif primary_url == secondary_url or secondary_url == '*': + elif secondary_url in (primary_url, '*'): subject = primary_url elif secondary_url == '': diff -Nru plaso-20181219/plaso/formatters/hachoir.py plaso-20190131/plaso/formatters/hachoir.py --- plaso-20181219/plaso/formatters/hachoir.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/formatters/hachoir.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -"""The Hachoir event formatter.""" - -from __future__ import unicode_literals - -from plaso.formatters import interface -from plaso.formatters import manager -from plaso.lib import errors - - -class HachoirFormatter(interface.EventFormatter): - """Formatter for a Hachoir event.""" - - DATA_TYPE = 'metadata:hachoir' - FORMAT_STRING = '{data}' - - SOURCE_LONG = 'Hachoir Metadata' - SOURCE_SHORT = 'META' - - # pylint: disable=unused-argument - def GetMessages(self, formatter_mediator, event): - """Determines the formatted message strings for an event object. - - Args: - formatter_mediator (FormatterMediator): mediates the interactions - between formatters and other components, such as storage and Windows - EventLog resources. - event (EventObject): event. - - Returns: - tuple(str, str): formatted message string and short message string. - - Raises: - WrongFormatter: if the event object cannot be formatted by the formatter. - """ - if self.DATA_TYPE != event.data_type: - raise errors.WrongFormatter('Unsupported data type: {0:s}.'.format( - event.data_type)) - - event_values = event.CopyToDict() - - string_parts = [] - metadata = event_values.get('metadata', None) - if metadata: - for key, value in sorted(metadata.items()): - string_parts.append('{0:s}: {1:s}'.format(key, value)) - event_values['data'] = ' '.join(string_parts) - - return self._FormatMessages( - self.FORMAT_STRING, self.FORMAT_STRING_SHORT, event_values) - - -manager.FormattersManager.RegisterFormatter(HachoirFormatter) diff -Nru plaso-20181219/plaso/formatters/__init__.py plaso-20190131/plaso/formatters/__init__.py --- plaso-20181219/plaso/formatters/__init__.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/formatters/__init__.py 2019-01-31 19:09:43.000000000 +0000 @@ -34,7 +34,6 @@ from plaso.formatters import ganalytics from plaso.formatters import gdrive from plaso.formatters import gdrive_synclog -from plaso.formatters import hachoir from plaso.formatters import hangouts_messages from plaso.formatters import iis from plaso.formatters import imessage diff -Nru plaso-20181219/plaso/formatters/winevt.py plaso-20190131/plaso/formatters/winevt.py --- plaso-20181219/plaso/formatters/winevt.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/formatters/winevt.py 2019-01-31 19:09:43.000000000 +0000 @@ -57,7 +57,7 @@ Returns: str: description of the event type. """ - if event_type >= 0 and event_type < len(self._EVENT_TYPES): + if 0 <= event_type < len(self._EVENT_TYPES): return self._EVENT_TYPES[event_type] return 'Unknown {0:d}'.format(event_type) @@ -70,7 +70,7 @@ Returns: str: description of the event severity. """ - if severity >= 0 and severity < len(self._SEVERITY): + if 0 <= severity < len(self._SEVERITY): return self._SEVERITY[severity] return 'Unknown {0:d}'.format(severity) diff -Nru plaso-20181219/plaso/formatters/winevt_rc.py plaso-20190131/plaso/formatters/winevt_rc.py --- plaso-20181219/plaso/formatters/winevt_rc.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/formatters/winevt_rc.py 2019-01-31 19:09:43.000000000 +0000 @@ -202,7 +202,7 @@ if number_of_values == 0: return None - elif number_of_values == 1: + if number_of_values == 1: values = values_list[0] return values['event_log_provider_key'] @@ -238,7 +238,7 @@ if number_of_values == 0: return None - elif number_of_values == 1: + if number_of_values == 1: return values[0]['message_string'] raise RuntimeError('More than one value found in database.') @@ -355,7 +355,7 @@ if number_of_values == 0: return None - elif number_of_values == 1: + if number_of_values == 1: return values[0]['value'] raise RuntimeError('More than one value found in database.') diff -Nru plaso-20181219/plaso/__init__.py plaso-20190131/plaso/__init__.py --- plaso-20181219/plaso/__init__.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/__init__.py 2019-01-31 19:09:43.000000000 +0000 @@ -8,4 +8,4 @@ from __future__ import unicode_literals -__version__ = '20181219' +__version__ = '20190131' diff -Nru plaso-20181219/plaso/lib/definitions.py plaso-20190131/plaso/lib/definitions.py --- plaso-20181219/plaso/lib/definitions.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/lib/definitions.py 2019-01-31 19:09:43.000000000 +0000 @@ -21,16 +21,18 @@ FAILURE_MODE_TERMINATED = 'terminated' FAILURE_MODE_TIME_OUT = 'time_out' -OPERATING_SYSTEM_LINUX = 'Linux' -OPERATING_SYSTEM_MACOS = 'MacOS' -OPERATING_SYSTEM_UNKNOWN = 'Unknown' -OPERATING_SYSTEM_WINDOWS = 'Windows' +OPERATING_SYSTEM_FAMILY_LINUX = 'Linux' +OPERATING_SYSTEM_FAMILY_MACOS = 'MacOS' +OPERATING_SYSTEM_FAMILY_UNKNOWN = 'Unknown' +OPERATING_SYSTEM_FAMILY_WINDOWS_9x = 'Windows 9x/Me' +OPERATING_SYSTEM_FAMILY_WINDOWS_NT = 'Windows NT' -OPERATING_SYSTEMS = frozenset([ - OPERATING_SYSTEM_LINUX, - OPERATING_SYSTEM_MACOS, - OPERATING_SYSTEM_UNKNOWN, - OPERATING_SYSTEM_WINDOWS]) +OPERATING_SYSTEM_FAMILIES = frozenset([ + OPERATING_SYSTEM_FAMILY_LINUX, + OPERATING_SYSTEM_FAMILY_MACOS, + OPERATING_SYSTEM_FAMILY_UNKNOWN, + OPERATING_SYSTEM_FAMILY_WINDOWS_9x, + OPERATING_SYSTEM_FAMILY_WINDOWS_NT]) PROCESSING_STATUS_ABORTED = 'aborted' PROCESSING_STATUS_ANALYZING = 'analyzing' diff -Nru plaso-20181219/plaso/lib/errors.py plaso-20190131/plaso/lib/errors.py --- plaso-20181219/plaso/lib/errors.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/lib/errors.py 2019-01-31 19:09:43.000000000 +0000 @@ -20,12 +20,16 @@ """Class that defines errors encountered connecting to a service.""" -class HeapFull(Error): - """Class that implements a heap full exception.""" - class InvalidEvent(Error): """Error indicating an event is malformed.""" +class InvalidFilter(Error): + """Error indicating an invalid filter was specified.""" + +class MalformedPresetError(Error): + """Raised when a parser preset definition is malformed.""" + + class MalformedQueryError(Error): """Raised when an objectfilter query is malformed.""" diff -Nru plaso-20181219/plaso/lib/lexer.py plaso-20190131/plaso/lib/lexer.py --- plaso-20181219/plaso/lib/lexer.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/lib/lexer.py 2019-01-31 19:09:43.000000000 +0000 @@ -238,21 +238,21 @@ return 'Expression: ({0:s}) ({1:s}) {2:s}'.format( self.attribute, self.operator, self.args) - def AddArg(self, arg): - """Adds a new arg to this expression. + def AddArg(self, argument): + """Adds a new argument to this expression. Args: - arg: The argument to add (string). + argument (str): argument to add. Returns: - True if this arg is the last arg, False otherwise. + True if the argument is the last argument, False otherwise. Raises: - ParseError: If there are too many args. + ParseError: If there are too many arguments. """ - self.args.append(arg) + self.args.append(argument) if len(self.args) > self.number_of_args: - raise errors.ParseError('Too many args for this expression.') + raise errors.ParseError('Too many arguments for this expression.') elif len(self.args) == self.number_of_args: return True @@ -315,9 +315,9 @@ def Compile(self, filter_implementation): """Compile the binary expression into a filter object.""" operator = self.operator.lower() - if operator == 'and' or operator == '&&': + if operator in ('and', '&&'): method = 'AndFilter' - elif operator == 'or' or operator == '||': + elif operator in ('or', '||'): method = 'OrFilter' else: raise errors.ParseError( @@ -431,8 +431,9 @@ if self.state == 'ATTRIBUTE': return self.StoreAttribute(string=self.string) - elif self.state == 'ARG_LIST': + if self.state == 'ARG_LIST': return self.InsertArg(string=self.string) + return None def StoreAttribute(self, string='', **unused_kwargs): @@ -453,7 +454,7 @@ self.current_expression.SetOperator(string) def InsertArg(self, string='', **unused_kwargs): - """Insert an arg to the current expression.""" + """Insert an argument to the current expression.""" logging.debug('Storing Argument {0:s}'.format(string)) # This expression is complete diff -Nru plaso-20181219/plaso/lib/objectfilter.py plaso-20190131/plaso/lib/objectfilter.py --- plaso-20181219/plaso/lib/objectfilter.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/lib/objectfilter.py 2019-01-31 19:09:43.000000000 +0000 @@ -171,8 +171,8 @@ return filter(self.Matches, objects) def __str__(self): - return '{0:s}({1:s})'.format( - self.__class__.__name__, ', '.join([str(arg) for arg in self.args])) + return '{0:s}({1:s})'.format(self.__class__.__name__, ', '.join([ + str(argument) for argument in self.args])) class AndFilter(Filter): @@ -262,10 +262,9 @@ try: if self.Operation(val, self.right_operand): return True - else: - continue - except (ValueError, TypeError): - continue + except (TypeError, ValueError): + pass + return False def Matches(self, obj): @@ -640,8 +639,8 @@ def Compile(self, filter_implementation): """Compile the expression.""" arguments = [self.attribute] - for arg in self.args: - arguments.append(arg.Compile(filter_implementation)) + for argument in self.args: + arguments.append(argument.Compile(filter_implementation)) expander = filter_implementation.FILTERS['ValueExpander'] context_cls = filter_implementation.FILTERS['Context'] return context_cls(arguments=arguments, @@ -652,9 +651,9 @@ def Compile(self, filter_implementation): """Compile the binary expression into a filter object.""" operator = self.operator.lower() - if operator == 'and' or operator == '&&': + if operator in ('and', '&&'): method = 'AndFilter' - elif operator == 'or' or operator == '||': + elif operator in ('or', '||'): method = 'OrFilter' else: raise errors.ParseError( @@ -771,7 +770,7 @@ 'Unable to perform a negative match, issuing a positive one.') def InsertArg(self, string='', **unused_kwargs): - """Insert an arg to the current expression.""" + """Insert an argument to the current expression.""" # Note that "string" is not necessarily of type string. logging.debug('Storing argument: {0!s}'.format(string)) @@ -815,7 +814,7 @@ if self.state == 'ATTRIBUTE': return self.StoreAttribute(string=self.string) - elif self.state == 'ARG': + if self.state == 'ARG': return self.InsertArg(string=self.string) return None diff -Nru plaso-20181219/plaso/lib/pfilter.py plaso-20190131/plaso/lib/pfilter.py --- plaso-20181219/plaso/lib/pfilter.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/lib/pfilter.py 2019-01-31 19:09:43.000000000 +0000 @@ -103,8 +103,9 @@ result, _ = formatters_manager.FormattersManager.GetMessageStrings( formatter_mediator, event_object) except KeyError as exception: - logging.warning('Unable to correctly assemble event: {0:s}'.format( - exception)) + logging.warning( + 'Unable to correctly assemble event with error: {0!s}'.format( + exception)) return result @@ -118,8 +119,9 @@ source_short, source_long = ( formatters_manager.FormattersManager.GetSourceStrings(event_object)) except KeyError as exception: - logging.warning('Unable to correctly assemble event: {0:s}'.format( - exception)) + logging.warning( + 'Unable to correctly assemble event with error: {0!s}'.format( + exception)) return source_short, source_long @@ -194,16 +196,16 @@ # timestamps in a "human readable" format. if self.attribute == 'timestamp': args = [] - for arg in self.args: - args.append(DateCompareObject(arg)) + for argument in self.args: + args.append(DateCompareObject(argument)) self.args = args - for arg in self.args: - if isinstance(arg, DateCompareObject): + for argument in self.args: + if isinstance(argument, DateCompareObject): if 'Less' in str(operator): - TimeRangeCache.SetUpperTimestamp(arg.data) + TimeRangeCache.SetUpperTimestamp(argument.data) else: - TimeRangeCache.SetLowerTimestamp(arg.data) + TimeRangeCache.SetLowerTimestamp(argument.data) arguments.extend(self.args) expander = filter_implementation.FILTERS['ValueExpander'] ops = operator(arguments=arguments, value_expander=expander) diff -Nru plaso-20181219/plaso/lib/plist.py plaso-20190131/plaso/lib/plist.py --- plaso-20181219/plaso/lib/plist.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/lib/plist.py 2019-01-31 19:09:43.000000000 +0000 @@ -60,6 +60,7 @@ Raises: IOError: if the plist file-like object cannot be read. + OSError: if the plist file-like object cannot be read. """ try: self.root_key = biplist.readPlist(file_object) diff -Nru plaso-20181219/plaso/multi_processing/engine.py plaso-20190131/plaso/multi_processing/engine.py --- plaso-20181219/plaso/multi_processing/engine.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/multi_processing/engine.py 2019-01-31 19:09:43.000000000 +0000 @@ -291,9 +291,10 @@ process (MultiProcessBaseProcess): process. Raises: + IOError: if the RPC client cannot connect to the server. KeyError: if the process is not registered with the engine or if the process is already being monitored. - IOError: if the RPC client cannot connect to the server. + OSError: if the RPC client cannot connect to the server. ValueError: if the process is missing. """ if process is None: diff -Nru plaso-20181219/plaso/multi_processing/multi_process_queue.py plaso-20190131/plaso/multi_processing/multi_process_queue.py --- plaso-20181219/plaso/multi_processing/multi_process_queue.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/multi_processing/multi_process_queue.py 2019-01-31 19:09:43.000000000 +0000 @@ -57,7 +57,7 @@ def Open(self): """Opens the queue.""" - pass + pass # pylint: disable=unnecessary-pass def Close(self, abort=False): """Closes the queue. diff -Nru plaso-20181219/plaso/multi_processing/task_engine.py plaso-20190131/plaso/multi_processing/task_engine.py --- plaso-20181219/plaso/multi_processing/task_engine.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/multi_processing/task_engine.py 2019-01-31 19:09:43.000000000 +0000 @@ -41,14 +41,22 @@ self._heap = [] self._maximum_number_of_items = maximum_number_of_items + def IsFull(self): + """Determines if the heap is full. + + Returns: + bool: True if the heap is full, False otherwise. + """ + return len(self._heap) >= self._maximum_number_of_items + def PopEventSource(self): """Pops an event source from the heap. Returns: - EventSource: event source or None on error. + EventSource: an event source or None on if no event source is available. """ try: - _, event_source = heapq.heappop(self._heap) + _, _, event_source = heapq.heappop(self._heap) except IndexError: return None @@ -60,10 +68,6 @@ Args: event_source (EventSource): event source. - - Raises: - HeapFull: if the heap contains the maximum number of items. Note that - this exception is raised after the item is added to the heap. """ if event_source.file_entry_type == ( dfvfs_definitions.FILE_ENTRY_TYPE_DIRECTORY): @@ -71,12 +75,9 @@ else: weight = 100 - heap_values = (weight, event_source) + heap_values = (weight, time.time(), event_source) heapq.heappush(self._heap, heap_values) - if len(self._heap) >= self._maximum_number_of_items: - raise errors.HeapFull() - class TaskMultiProcessEngine(engine.MultiProcessEngine): """Class that defines the task multi-process engine. @@ -167,9 +168,8 @@ self._processing_profiler.StopTiming('get_event_source') while event_source: - try: - event_source_heap.PushEventSource(event_source) - except errors.HeapFull: + event_source_heap.PushEventSource(event_source) + if event_source_heap.IsFull(): break if self._processing_profiler: @@ -451,7 +451,8 @@ self._MergeTaskStorage(storage_writer) - self._FillEventSourceHeap(storage_writer, event_source_heap) + if not event_source_heap.IsFull(): + self._FillEventSourceHeap(storage_writer, event_source_heap) if not task and not event_source: event_source = event_source_heap.PopEventSource() diff -Nru plaso-20181219/plaso/output/interface.py plaso-20190131/plaso/output/interface.py --- plaso-20181219/plaso/output/interface.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/output/interface.py 2019-01-31 19:09:43.000000000 +0000 @@ -46,7 +46,7 @@ def Close(self): """Closes the output.""" - pass + pass # pylint: disable=unnecessary-pass def GetMissingArguments(self): """Retrieves arguments required by the module that have not been specified. @@ -59,7 +59,7 @@ def Open(self): """Opens the output.""" - pass + pass # pylint: disable=unnecessary-pass def WriteEvent(self, event): """Writes the event to the output. @@ -97,7 +97,7 @@ Can be used for post-processing or output after an individual event has been written, such as writing closing XML tags, etc. """ - pass + pass # pylint: disable=unnecessary-pass def WriteEventMACBGroup(self, event_macb_group): """Writes an event MACB group to the output. @@ -123,7 +123,7 @@ Can be used for pre-processing or output before an individual event has been written, such as writing opening XML tags, etc. """ - pass + pass # pylint: disable=unnecessary-pass def WriteFooter(self): """Writes the footer to the output. @@ -131,7 +131,7 @@ Can be used for post-processing or output after the last event is written, such as writing a file footer. """ - pass + pass # pylint: disable=unnecessary-pass def WriteHeader(self): """Writes the header to the output. @@ -139,7 +139,7 @@ Can be used for pre-processing or output before the first event is written, such as writing a file header. """ - pass + pass # pylint: disable=unnecessary-pass class LinearOutputModule(OutputModule): diff -Nru plaso-20181219/plaso/output/mysql_4n6time.py plaso-20190131/plaso/output/mysql_4n6time.py --- plaso-20181219/plaso/output/mysql_4n6time.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/output/mysql_4n6time.py 2019-01-31 19:09:43.000000000 +0000 @@ -139,8 +139,8 @@ self._cursor.execute('DELETE FROM l2t_{0:s}s'.format(field)) for name, frequency in iter(values.items()): self._cursor.execute(( - 'INSERT INTO l2t_{0:s}s ({1:s}s, frequency) ' - 'VALUES("{2:s}", {3:d}) ').format(field, field, name, frequency)) + 'INSERT INTO l2t_{0:s}s ({0:s}s, frequency) ' + 'VALUES("{1:s}", {2:d}) ').format(field, name, frequency)) self._cursor.execute('DELETE FROM l2t_tags') for tag in self._GetTags(): @@ -161,8 +161,11 @@ """Connects to the database and creates the required tables. Raises: - IOError: If Unable to insert into database. - ValueError: If no database name given. + IOError: if the 4n6time tables cannot be created or data not inserted + in the database. + OSError: if the 4n6time tables cannot be created or data not inserted + in the database. + ValueError: if no database name given. """ if not self._dbname: raise ValueError('Missing database name.') diff -Nru plaso-20181219/plaso/output/null.py plaso-20190131/plaso/output/null.py --- plaso-20181219/plaso/output/null.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/output/null.py 2019-01-31 19:09:43.000000000 +0000 @@ -22,7 +22,7 @@ Args: event (EventObject): event. """ - pass + pass # pylint: disable=unnecessary-pass manager.OutputManager.RegisterOutput(NullOutputModule) diff -Nru plaso-20181219/plaso/output/sqlite_4n6time.py plaso-20190131/plaso/output/sqlite_4n6time.py --- plaso-20181219/plaso/output/sqlite_4n6time.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/output/sqlite_4n6time.py 2019-01-31 19:09:43.000000000 +0000 @@ -128,8 +128,8 @@ self._cursor.execute('DELETE FROM l2t_{0:s}s'.format(field)) for name, frequency in iter(values.items()): self._cursor.execute(( - 'INSERT INTO l2t_{0:s}s ({1:s}s, frequency) ' - 'VALUES("{2:s}", {3:d}) ').format(field, field, name, frequency)) + 'INSERT INTO l2t_{0:s}s ({0:s}s, frequency) ' + 'VALUES("{1:s}", {2:d}) ').format(field, name, frequency)) self._cursor.execute('DELETE FROM l2t_tags') for tag in self._ListTags(): self._cursor.execute('INSERT INTO l2t_tags (tag) VALUES (?)', [tag]) @@ -149,6 +149,7 @@ Raises: IOError: if the specified output file already exists. + OSError: if the specified output file already exists. ValueError: if the filename is not set. """ if not self._filename: diff -Nru plaso-20181219/plaso/output/xlsx.py plaso-20190131/plaso/output/xlsx.py --- plaso-20181219/plaso/output/xlsx.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/output/xlsx.py 2019-01-31 19:09:43.000000000 +0000 @@ -17,7 +17,7 @@ from plaso.output import interface from plaso.output import manager -import pytz +import pytz # pylint: disable=wrong-import-order class XLSXOutputModule(interface.OutputModule): @@ -111,6 +111,7 @@ Raises: IOError: if the specified output file already exists. + OSError: if the specified output file already exists. ValueError: if the filename is not set. """ if not self._filename: diff -Nru plaso-20181219/plaso/parsers/cups_ipp.py plaso-20190131/plaso/parsers/cups_ipp.py --- plaso-20181219/plaso/parsers/cups_ipp.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/cups_ipp.py 2019-01-31 19:09:43.000000000 +0000 @@ -35,7 +35,7 @@ from plaso.parsers import manager -# TODO: RFC Pendings types: resolution, dateTime, rangeOfInteger. +# TODO: RFC pending types: resolution, dateTime, rangeOfInteger. # "dateTime" is not used by Mac OS, instead it uses integer types. # TODO: Only tested against CUPS IPP MacOS. diff -Nru plaso-20181219/plaso/parsers/dsv_parser.py plaso-20190131/plaso/parsers/dsv_parser.py --- plaso-20181219/plaso/parsers/dsv_parser.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/dsv_parser.py 2019-01-31 19:09:43.000000000 +0000 @@ -152,10 +152,7 @@ file_object, end_of_line=self._end_of_line) # If we specifically define a number of lines we should skip, do that here. for _ in range(0, self.NUMBER_OF_HEADER_LINES): - try: - line_reader.readline(self._maximum_line_length) - except UnicodeDecodeError: - raise + line_reader.readline(self._maximum_line_length) return line_reader def _HasExpectedLineLength(self, file_object): diff -Nru plaso-20181219/plaso/parsers/esedb_plugins/interface.py plaso-20190131/plaso/parsers/esedb_plugins/interface.py --- plaso-20181219/plaso/parsers/esedb_plugins/interface.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/esedb_plugins/interface.py 2019-01-31 19:09:43.000000000 +0000 @@ -193,26 +193,26 @@ if column_type == pyesedb.column_types.NULL: return None - elif column_type == pyesedb.column_types.BOOLEAN: + if column_type == pyesedb.column_types.BOOLEAN: # TODO: implement raise ValueError('Boolean value support not implemented yet.') - elif column_type in self.INTEGER_COLUMN_TYPES: + if column_type in self.INTEGER_COLUMN_TYPES: if long_value: raise ValueError('Long integer value not supported.') return record.get_value_data_as_integer(value_entry) - elif column_type in self.FLOATING_POINT_COLUMN_TYPES: + if column_type in self.FLOATING_POINT_COLUMN_TYPES: if long_value: raise ValueError('Long floating point value not supported.') return record.get_value_data_as_floating_point(value_entry) - elif column_type in self.STRING_COLUMN_TYPES: + if column_type in self.STRING_COLUMN_TYPES: if long_value: return long_value.get_data_as_string() return record.get_value_data_as_string(value_entry) - elif column_type == pyesedb.column_types.GUID: + if column_type == pyesedb.column_types.GUID: # TODO: implement raise ValueError('GUID value support not implemented yet.') diff -Nru plaso-20181219/plaso/parsers/firefox_cache.py plaso-20190131/plaso/parsers/firefox_cache.py --- plaso-20181219/plaso/parsers/firefox_cache.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/firefox_cache.py 2019-01-31 19:09:43.000000000 +0000 @@ -253,6 +253,7 @@ Raises: IOError: if the cache record header cannot be validated. + OSError: if the cache record header cannot be validated. ParseError: if the cache record header cannot be parsed. """ file_offset = file_object.get_offset() diff -Nru plaso-20181219/plaso/parsers/hachoir.py plaso-20190131/plaso/parsers/hachoir.py --- plaso-20181219/plaso/parsers/hachoir.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/hachoir.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -"""Parser that uses Hachoir to extract metadata.""" - -from __future__ import unicode_literals - -# TODO: Add a unit test for this parser. - -import datetime - -# pylint: disable=import-error,wrong-import-position -import hachoir_core.config - -# This is necessary to do PRIOR to loading up other parts of hachoir -# framework, otherwise console does not work and other "weird" behavior -# is observed. -hachoir_core.config.unicode_stdout = False -hachoir_core.config.quiet = True - -import hachoir_core -import hachoir_parser -import hachoir_metadata - -from plaso.containers import events -from plaso.containers import time_events -from plaso.lib import errors -from plaso.lib import timelib -from plaso.parsers import interface -from plaso.parsers import manager - - -class HachoirEventData(events.EventData): - """Hachoir event data. - - Attributes: - metadata (dict[str, object]): hachoir metadata. - """ - - DATA_TYPE = 'metadata:hachoir' - - def __init__(self): - """Initializes event data.""" - super(HachoirEventData, self).__init__(data_type=self.DATA_TYPE) - self.metadata = {} - - -class HachoirParser(interface.FileObjectParser): - """Parser that uses Hachoir.""" - - NAME = 'hachoir' - DESCRIPTION = 'Parser that wraps Hachoir.' - - def ParseFileObject(self, parser_mediator, file_object): - """Parses a file-like object using Hachoir. - - Args: - parser_mediator (ParserMediator): mediates interactions between parsers - and other components, such as storage and dfvfs. - file_object (dfvfs.FileIO): a file-like object. - - Raises: - UnableToParseFile: when the file cannot be parsed. - """ - file_name = parser_mediator.GetDisplayName() - - try: - fstream = hachoir_core.stream.InputIOStream(file_object, None, tags=[]) - except hachoir_core.error.HachoirError as exception: - raise errors.UnableToParseFile( - '[{0:s}] unable to parse file {1:s}: {2:s}'.format( - self.NAME, file_name, exception)) - - if not fstream: - raise errors.UnableToParseFile( - '[{0:s}] unable to parse file {1:s}: {2:s}'.format( - self.NAME, file_name, 'Not fstream')) - - try: - doc_parser = hachoir_parser.guessParser(fstream) - except hachoir_core.error.HachoirError as exception: - raise errors.UnableToParseFile( - '[{0:s}] unable to parse file {1:s}: {2:s}'.format( - self.NAME, file_name, exception)) - - if not doc_parser: - raise errors.UnableToParseFile( - '[{0:s}] unable to parse file {1:s}: {2:s}'.format( - self.NAME, file_name, 'Not parser')) - - try: - metadata = hachoir_metadata.extractMetadata(doc_parser) - except (AssertionError, AttributeError) as exception: - raise errors.UnableToParseFile( - '[{0:s}] unable to parse file {1:s}: {2:s}'.format( - self.NAME, file_name, exception)) - - try: - metatext = metadata.exportPlaintext(human=False) - except AttributeError as exception: - raise errors.UnableToParseFile( - '[{0:s}] unable to parse file {1:s}: {2:s}'.format( - self.NAME, file_name, exception)) - - if not metatext: - raise errors.UnableToParseFile( - '[{0:s}] unable to parse file {1:s}: No metadata'.format( - self.NAME, file_name)) - - attributes = {} - extracted_events = [] - for meta in metatext: - if not meta.startswith('-'): - continue - - if len(meta) < 3: - continue - - key, _, value = meta[2:].partition(': ') - - key2, _, value2 = value.partition(': ') - if key2 == 'LastPrinted' and value2 != 'False': - date_object = timelib.Timestamp.FromTimeString( - value2, timezone=parser_mediator.timezone) - if isinstance(date_object, datetime.datetime): - extracted_events.append((date_object, key2)) - - try: - date = metadata.get(key) - if isinstance(date, datetime.datetime): - extracted_events.append((date, key)) - except ValueError: - pass - - if key in attributes: - if isinstance(attributes.get(key), list): - attributes[key].append(value) - else: - old_value = attributes.get(key) - attributes[key] = [old_value, value] - else: - attributes[key] = value - - if not extracted_events: - raise errors.UnableToParseFile( - '[{0:s}] unable to parse file {1:s}: {2:s}'.format( - self.NAME, file_name, 'No events discovered')) - - event_data = HachoirEventData() - event_data.metadata = attributes - - for datetime_value, usage in extracted_events: - event = time_events.PythonDatetimeEvent(datetime_value, usage) - parser_mediator.ProduceEventWithEventData(event, event_data) - - -manager.ParsersManager.RegisterParser(HachoirParser) diff -Nru plaso-20181219/plaso/parsers/__init__.py plaso-20190131/plaso/parsers/__init__.py --- plaso-20181219/plaso/parsers/__init__.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/__init__.py 2019-01-31 19:09:43.000000000 +0000 @@ -20,12 +20,6 @@ from plaso.parsers import firefox_cache from plaso.parsers import fseventsd from plaso.parsers import gdrive_synclog - -try: - from plaso.parsers import hachoir -except ImportError: - pass - from plaso.parsers import iis from plaso.parsers import java_idx from plaso.parsers import mac_appfirewall diff -Nru plaso-20181219/plaso/parsers/manager.py plaso-20190131/plaso/parsers/manager.py --- plaso-20181219/plaso/parsers/manager.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/manager.py 2019-01-31 19:09:43.000000000 +0000 @@ -5,7 +5,7 @@ import pysigscan -from plaso.lib import definitions +from plaso.containers import artifacts from plaso.lib import specification from plaso.parsers import logger from plaso.parsers import presets @@ -15,6 +15,7 @@ """The parsers and plugins manager.""" _parser_classes = {} + _presets = presets.ParserPresetsManager() @classmethod def _GetParserFilters(cls, parser_filter_expression): @@ -41,7 +42,7 @@ includes = {} excludes = {} - preset_categories = presets.CATEGORIES.keys() + preset_names = cls._presets.GetNames() for parser_filter in parser_filter_expression.split(','): parser_filter = parser_filter.strip() @@ -55,7 +56,7 @@ active_dict = includes parser_filter = parser_filter.lower() - if parser_filter in preset_categories: + if parser_filter in preset_names: for parser_in_category in cls._GetParsersFromPresetCategory( parser_filter): parser, _, plugin = parser_in_category.partition('/') @@ -82,12 +83,15 @@ Returns: list[str]: parser names in alphabetical order. """ - if category not in presets.CATEGORIES: + preset_definition = cls._presets.GetPresetByName(category) + if preset_definition is None: return [] + preset_names = cls._presets.GetNames() parser_names = set() - for element_name in presets.CATEGORIES.get(category): - if element_name in presets.CATEGORIES: + + for element_name in preset_definition.parsers: + if element_name in preset_names: category_parser_names = cls._GetParsersFromPresetCategory(element_name) parser_names.update(category_parser_names) else: @@ -402,67 +406,48 @@ return parsers_information @classmethod - def GetPresetForOperatingSystem( + def GetPresetsForOperatingSystem( cls, operating_system, operating_system_product, operating_system_version): - """Determines the preset for a specific operating system. + """Determines the presets for a specific operating system. Args: operating_system (str): operating system for example "Windows". This - should be one of the values in definitions.OPERATING_SYSTEMS. + should be one of the values in definitions.OPERATING_SYSTEM_FAMILIES. operating_system_product (str): operating system product for example "Windows XP" as determined by preprocessing. operating_system_version (str): operating system version for example "5.1" as determined by preprocessing. Returns: - str: parser filter preset, where None represents all parsers and plugins. + list[PresetDefinition]: preset definitions, where an empty list + represents all parsers and parser plugins (no preset). """ - # TODO: Make this more sane. Currently we are only checking against - # one possible version of Windows, and then making the assumption if - # that is not correct we default to Windows 7. Same thing with other - # OS's, no assumption or checks are really made there. - # Also this is done by default, and no way for the user to turn off - # this behavior, need to add a parameter to the CLI tools that takes - # care of overwriting this behavior. - - if operating_system == definitions.OPERATING_SYSTEM_LINUX: - return 'linux' - - if operating_system == definitions.OPERATING_SYSTEM_MACOS: - return 'macos' - - if operating_system_product: - operating_system_product = operating_system_product.lower() - else: - operating_system_product = '' - - if operating_system_version: - operating_system_version = operating_system_version.split('.') - else: - operating_system_version = ['0', '0'] - - # Windows NT 5 (2000, XP and 2003). - if ('windows' in operating_system_product and - operating_system_version[0] == '5'): - return 'winxp' - - # TODO: Improve this detection, this should be more 'intelligent', since - # there are quite a lot of versions out there that would benefit from - # loading up the set of 'winxp' parsers. - if ('windows xp' in operating_system_product or - 'windows server 2000' in operating_system_product or - 'windows server 2003' in operating_system_product): - return 'winxp' - - # Fallback for other Windows versions. - if 'windows' in operating_system_product: - return 'win7' + operating_system = artifacts.OperatingSystemArtifact( + family=operating_system, product=operating_system_product, + version=operating_system_version) - if operating_system == definitions.OPERATING_SYSTEM_WINDOWS: - return 'win7' + return cls._presets.GetPresetsByOperatingSystem(operating_system) - return None + @classmethod + def GetPresets(cls): + """Retrieves the preset definitions. + + Returns: + generator[PresetDefinition]: preset definition generator in alphabetical + order by name. + """ + return cls._presets.GetPresets() + + @classmethod + def ReadPresetsFromFile(cls, path): + """Reads parser and parser plugin presets from a file. + + Args: + path (str): path of file that contains the the parser and parser plugin + presets configuration. + """ + cls._presets.ReadFromFile(path) @classmethod def RegisterParser(cls, parser_class): diff -Nru plaso-20181219/plaso/parsers/olecf_plugins/summary.py plaso-20190131/plaso/parsers/olecf_plugins/summary.py --- plaso-20181219/plaso/parsers/olecf_plugins/summary.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/olecf_plugins/summary.py 2019-01-31 19:09:43.000000000 +0000 @@ -62,10 +62,10 @@ if property_value.type == pyolecf.value_types.BOOLEAN: return property_value.data_as_boolean - elif property_value.type in self._INTEGER_TYPES: + if property_value.type in self._INTEGER_TYPES: return property_value.data_as_integer - elif property_value.type in self._STRING_TYPES: + if property_value.type in self._STRING_TYPES: return property_value.data_as_string try: diff -Nru plaso-20181219/plaso/parsers/plist.py plaso-20190131/plaso/parsers/plist.py --- plaso-20181219/plaso/parsers/plist.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/plist.py 2019-01-31 19:09:43.000000000 +0000 @@ -91,11 +91,7 @@ raise errors.UnableToParseFile( 'File size: {0:d} bytes is larger than 50 MB.'.format(file_size)) - try: - top_level_object = self.GetTopLevel(file_object) - except errors.UnableToParseFile: - raise - + top_level_object = self.GetTopLevel(file_object) if not top_level_object: raise errors.UnableToParseFile( 'Unable to parse: {0:s} skipping.'.format(filename)) diff -Nru plaso-20181219/plaso/parsers/presets.py plaso-20190131/plaso/parsers/presets.py --- plaso-20181219/plaso/parsers/presets.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/presets.py 2019-01-31 19:09:43.000000000 +0000 @@ -1,46 +1,198 @@ # -*- coding: utf-8 -*- -"""The parser preset categories.""" +"""The parser and parser plugin presets.""" from __future__ import unicode_literals -CATEGORIES = { - 'win_gen': [ - 'bencode', 'esedb', 'filestat', 'sqlite/google_drive', 'gdrive_synclog', - 'java_idx', 'lnk', 'mcafee_protection', 'olecf', 'czip/oxml', 'pe', - 'prefetch', 'sccm', 'skydrive_log', 'skydrive_log_old', 'sqlite/skype', - 'symantec_scanlog', 'usnjrnl', 'webhist', 'winfirewall', 'winjob', - 'winreg'], - 'winxp': ['recycle_bin_info2', 'rplog', 'win_gen', 'winevt'], - 'winxp_slow': ['hachoir', 'mft', 'winxp'], - 'win7': [ - 'recycle_bin', 'custom_destinations', 'esedb/file_history', - 'olecf/olecf_automatic_destinations', 'win_gen', 'winevtx', 'amcache'], - 'win7_slow': ['hachoir', 'mft', 'win7'], - 'webhist': [ - 'binary_cookies', 'chrome_cache', 'sqlite/chrome_autofill', - 'sqlite/chrome_cookies', 'sqlite/chrome_extension_activity', - 'sqlite/chrome_8_history', 'sqlite/chrome_27_history', - 'chrome_preferences', 'firefox_cache', 'sqlite/firefox_cookies', - 'sqlite/firefox_downloads', 'sqlite/firefox_history', 'java_idx', - 'esedb/msie_webcache', 'msiecf', 'opera_global', - 'opera_typed_history', 'plist/safari_history'], - 'linux': [ - 'bash_history', 'bencode', 'dockerjson', 'dpkg', 'filestat', - 'sqlite/google_drive', 'gdrive_synclog', 'java_idx', 'olecf', - 'czip/oxml', 'pls_recall', 'popularity_contest', 'selinux', - 'sqlite/skype', 'syslog', 'systemd_journal', 'utmp', 'webhist', - 'xchatlog', 'xchatscrollback', 'sqlite/zeitgeist', - 'zsh_extended_history'], - 'macos': [ - 'sqlite/appusage', 'asl_log', 'bash_history', 'bencode', 'bsm_log', - 'cups_ipp', 'filestat', 'fseventsd', 'sqlite/google_drive', - 'gdrive_synclog', 'sqlite/imessage', 'java_idx', 'sqlite/ls_quarantine', - 'mac_appfirewall_log', 'sqlite/mac_document_versions', 'mac_keychain', - 'mac_securityd', 'sqlite/mackeeper_cache', 'macwifi', 'olecf', - 'czip/oxml', 'plist', 'sqlite/skype', 'syslog', 'utmpx', 'webhist', - 'zsh_extended_history'], - 'android': [ - 'android_app_usage', 'filestat', 'chrome_cache', - 'sqlite/android_calls', 'sqlite/android_sms', 'sqlite/android_webview', - 'sqlite/android_webviewcache', 'sqlite/chrome_cookies', - 'sqlite/chrome_8_history', 'sqlite/chrome_27_history', 'sqlite/skype']} +import yaml + +from plaso.containers import artifacts +from plaso.lib import errors + + +class ParserPreset(object): + """Parser and parser plugin preset. + + Attributes: + name (str): name of the preset. + operating_systems (list[OperatingSystemArtifact]): operating system + artifact attribute containers, that specify to which operating + systems the preset applies. + parsers (list[str]): names of parser and parser plugins. + """ + + def __init__(self, name, parsers): + """Initializes a parser and parser plugin preset. + + Attributes: + name (str): name of the preset. + parsers (list[str]): names of parser and parser plugins. + """ + super(ParserPreset, self).__init__() + self.name = name + self.operating_systems = [] + self.parsers = parsers + + +class ParserPresetsManager(object): + """The parsers and plugin presets manager.""" + + def __init__(self): + """Initializes a parser and parser plugin presets manager.""" + super(ParserPresetsManager, self).__init__() + self._definitions = {} + + def _ReadOperatingSystemArtifactValues(self, operating_system_values): + """Reads an operating system artifact from a dictionary. + + Args: + operating_system_values (dict[str, object]): operating system values. + + Returns: + OperatingSystemArtifact: an operating system artifact attribute container. + + Raises: + MalformedPresetError: if the format of the operating system values are + not set or incorrect. + """ + if not operating_system_values: + raise errors.MalformedPresetError('Missing operating system values.') + + family = operating_system_values.get('family', None) + product = operating_system_values.get('product', None) + version = operating_system_values.get('version', None) + + if not family and not product: + raise errors.MalformedPresetError( + 'Invalid operating system missing family and product.') + + return artifacts.OperatingSystemArtifact( + family=family, product=product, version=version) + + def _ReadParserPresetValues(self, preset_definition_values): + """Reads a parser preset from a dictionary. + + Args: + preset_definition_values (dict[str, object]): preset definition values. + + Returns: + ParserPreset: a parser preset. + + Raises: + MalformedPresetError: if the format of the preset definition is not set + or incorrect, or the preset of a specific operating system has already + been set. + """ + if not preset_definition_values: + raise errors.MalformedPresetError('Missing preset definition values.') + + name = preset_definition_values.get('name', None) + if not name: + raise errors.MalformedPresetError( + 'Invalid preset definition missing name.') + + parsers = preset_definition_values.get('parsers', None) + if not parsers: + raise errors.MalformedPresetError( + 'Invalid preset definition missing parsers.') + + parser_preset = ParserPreset(name, parsers) + + for operating_system_values in preset_definition_values.get( + 'operating_systems', []): + operating_system = self._ReadOperatingSystemArtifactValues( + operating_system_values) + parser_preset.operating_systems.append(operating_system) + + return parser_preset + + def _ReadPresetsFromFileObject(self, file_object): + """Reads parser and parser plugin presets from a file-like object. + + Args: + file_object (file): file-like object containing the parser and parser + plugin presets definitions. + + Yields: + ParserPreset: a parser preset. + + Raises: + MalformedPresetError: if one or more plugin preset definitions are + malformed. + """ + yaml_generator = yaml.safe_load_all(file_object) + + last_preset_definition = None + for yaml_definition in yaml_generator: + try: + preset_definition = self._ReadParserPresetValues(yaml_definition) + except errors.MalformedPresetError as exception: + error_location = 'At start' + if last_preset_definition: + error_location = 'After: {0:s}'.format(last_preset_definition.name) + + raise errors.MalformedPresetError( + '{0:s} {1!s}'.format(error_location, exception)) + + yield preset_definition + last_preset_definition = preset_definition + + def GetNames(self): + """Retrieves the preset names. + + Returns: + list[str]: preset names in alphabetical order. + """ + return sorted(self._definitions.keys()) + + def GetPresetByName(self, name): + """Retrieves a specific preset definition by name. + + Args: + name (str): name of the preset. + + Returns: + ParserPreset: a parser preset or None if not available. + """ + name = name.lower() + return self._definitions.get(name, None) + + def GetPresetsByOperatingSystem(self, operating_system): + """Retrieves preset definitions for a specific operating system. + + Args: + operating_system (OperatingSystemArtifact): an operating system artifact + attribute container. + + Returns: + list[PresetDefinition]: preset definition that correspond with the + operating system. + """ + preset_definitions = [] + for preset_definition in self._definitions.values(): + for preset_operating_system in preset_definition.operating_systems: + if preset_operating_system.IsEquivalent(operating_system): + preset_definitions.append(preset_definition) + + return preset_definitions + + def GetPresets(self): + """Retrieves the preset definitions. + + Yields: + ParserPreset: parser presets in alphabetical order by name. + """ + for _, parser_preset in sorted(self._definitions.items()): + yield parser_preset + + def ReadFromFile(self, path): + """Reads parser and parser plugin presets from a file. + + Args: + path (str): path of file that contains the the parser and parser plugin + presets configuration. + """ + self._definitions = {} + + with open(path, 'r') as file_object: + for preset_definition in self._ReadPresetsFromFileObject(file_object): + self._definitions[preset_definition.name] = preset_definition diff -Nru plaso-20181219/plaso/parsers/skydrivelog.py plaso-20190131/plaso/parsers/skydrivelog.py --- plaso-20181219/plaso/parsers/skydrivelog.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/skydrivelog.py 2019-01-31 19:09:43.000000000 +0000 @@ -22,7 +22,7 @@ Attributes: detail (str): details. log_level (str): log level. - module (str): name of the module that generated the log messsage. + module (str): name of the module that generated the log message. source_code (str): source file and line number that generated the log message. """ diff -Nru plaso-20181219/plaso/parsers/sqlite_plugins/twitter_ios.py plaso-20190131/plaso/parsers/sqlite_plugins/twitter_ios.py --- plaso-20181219/plaso/parsers/sqlite_plugins/twitter_ios.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/sqlite_plugins/twitter_ios.py 2019-01-31 19:09:43.000000000 +0000 @@ -56,7 +56,7 @@ favorited (int): value to mark status as favorite by the account. name (str): user's profile name. retweet_count (str): number of times the status message has been retweeted. - text (str): content of the status messsage. + text (str): content of the status message. user_id (int): user unique identifier. """ diff -Nru plaso-20181219/plaso/parsers/sqlite.py plaso-20190131/plaso/parsers/sqlite.py --- plaso-20181219/plaso/parsers/sqlite.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/sqlite.py 2019-01-31 19:09:43.000000000 +0000 @@ -212,6 +212,7 @@ Raises: IOError: if the file-like object cannot be read. + OSError: if the file-like object cannot be read. sqlite3.DatabaseError: if the database cannot be parsed. ValueError: if the file-like object is missing. """ diff -Nru plaso-20181219/plaso/parsers/text_parser.py plaso-20190131/plaso/parsers/text_parser.py --- plaso-20181219/plaso/parsers/text_parser.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/text_parser.py 2019-01-31 19:09:43.000000000 +0000 @@ -155,16 +155,9 @@ # Numbers. INTEGER = pyparsing.Word(pyparsing.nums).setParseAction(PyParseIntCast) - IPV4_OCTET = pyparsing.Word(pyparsing.nums, min=1, max=3).setParseAction( - PyParseIntCast, PyParseRangeCheck(0, 255)) - IPV4_ADDRESS = (IPV4_OCTET + ('.' + IPV4_OCTET) * 3).setParseAction( - PyParseJoinList) - - # TODO: Fix the IPv6 address specification to be more accurate (8 :, correct - # size, etc). - IPV6_ADDRESS = pyparsing.Word(':' + pyparsing.hexnums).setParseAction( - PyParseJoinList) + IPV4_ADDRESS = pyparsing.pyparsing_common.ipv4_address + IPV6_ADDRESS = pyparsing.pyparsing_common.ipv6_address IP_ADDRESS = (IPV4_ADDRESS | IPV6_ADDRESS) # TODO: deprecate and remove, use THREE_LETTERS instead. diff -Nru plaso-20181219/plaso/parsers/winreg_plugins/appcompatcache.py plaso-20190131/plaso/parsers/winreg_plugins/appcompatcache.py --- plaso-20181219/plaso/parsers/winreg_plugins/appcompatcache.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/winreg_plugins/appcompatcache.py 2019-01-31 19:09:43.000000000 +0000 @@ -153,7 +153,7 @@ # TODO: determine which format version is used (2003 or Vista). return self._FORMAT_TYPE_2003 - elif format_type == self._FORMAT_TYPE_8: + if format_type == self._FORMAT_TYPE_8: cached_entry_signature = value_data[signature:signature + 4] if cached_entry_signature in ( self._CACHED_ENTRY_SIGNATURE_8_0, self._CACHED_ENTRY_SIGNATURE_8_1): diff -Nru plaso-20181219/plaso/parsers/winreg_plugins/interface.py plaso-20190131/plaso/parsers/winreg_plugins/interface.py --- plaso-20181219/plaso/parsers/winreg_plugins/interface.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/parsers/winreg_plugins/interface.py 2019-01-31 19:09:43.000000000 +0000 @@ -119,9 +119,7 @@ # TODO: check if control_set is in bounds. return control_set is not None - return ( - key_path == self._key_path_upper or - key_path == self._wow64_key_path_upper) + return key_path in (self._key_path_upper, self._wow64_key_path_upper) class WindowsRegistryKeyPathPrefixFilter(BaseWindowsRegistryKeyFilter): diff -Nru plaso-20181219/plaso/preprocessors/manager.py plaso-20190131/plaso/preprocessors/manager.py --- plaso-20181219/plaso/preprocessors/manager.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/preprocessors/manager.py 2019-01-31 19:09:43.000000000 +0000 @@ -37,7 +37,7 @@ """Create a Windows path resolver and sets the environment variables. Args: - file_system (dfvfs.FileSytem): file system. + file_system (dfvfs.FileSystem): file system. mount_point (dfvfs.PathSpec): mount point path specification. environment_variables (list[EnvironmentVariableArtifact]): environment variables. diff -Nru plaso-20181219/plaso/serializer/json_serializer.py plaso-20190131/plaso/serializer/json_serializer.py --- plaso-20181219/plaso/serializer/json_serializer.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/serializer/json_serializer.py 2019-01-31 19:09:43.000000000 +0000 @@ -193,13 +193,13 @@ if class_type == 'bytes': return binascii.a2b_qp(json_dict['stream']) - elif class_type == 'tuple': + if class_type == 'tuple': return tuple(cls._ConvertListToObject(json_dict['values'])) - elif class_type == 'collections.Counter': + if class_type == 'collections.Counter': return cls._ConvertDictToCollectionsCounter(json_dict) - elif class_type == 'AttributeContainer': + if class_type == 'AttributeContainer': # Use __container_type__ to indicate the attribute container type. container_type = json_dict.get('__container_type__', None) diff -Nru plaso-20181219/plaso/storage/fake/writer.py plaso-20190131/plaso/storage/fake/writer.py --- plaso-20181219/plaso/storage/fake/writer.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/storage/fake/writer.py 2019-01-31 19:09:43.000000000 +0000 @@ -69,6 +69,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._is_open: raise IOError('Unable to write to closed storage writer.') @@ -100,6 +101,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -115,6 +117,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -132,6 +135,8 @@ Raises: IOError: when the storage writer is closed or if the event data identifier type is not supported. + OSError: when the storage writer is closed or + if the event data identifier type is not supported. """ self._RaiseIfNotWritable() @@ -156,6 +161,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -173,6 +179,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -189,6 +196,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -213,6 +221,7 @@ Raises: IOError: if the task storage already exists. + OSError: if the task storage already exists. """ if task.identifier in self._task_storage_writers: raise IOError('Storage writer for task: {0:s} already exists.'.format( @@ -228,6 +237,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -299,6 +309,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._is_open: raise IOError('Unable to read from closed storage writer.') @@ -319,6 +330,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._is_open: raise IOError('Unable to read from closed storage writer.') @@ -342,6 +354,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._is_open: raise IOError('Unable to read from closed storage writer.') @@ -371,6 +384,7 @@ Raises: IOError: if the task storage does not exist. + OSError: if the task storage does not exist. """ if task.identifier not in self._task_storage_writers: raise IOError('Storage writer for task: {0:s} does not exist.'.format( @@ -381,6 +395,7 @@ Raises: IOError: if the storage writer is already opened. + OSError: if the storage writer is already opened. """ if self._is_open: raise IOError('Storage writer already opened.') @@ -398,6 +413,7 @@ Raises: IOError: if the task storage does not exist. + OSError: if the task storage does not exist. """ if task.identifier not in self._task_storage_writers: raise IOError('Storage writer for task: {0:s} does not exist.'.format( @@ -418,6 +434,8 @@ Raises: IOError: if the storage type does not support writing preprocessing information or when the storage writer is closed. + OSError: if the storage type does not support writing preprocessing + information or when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -434,6 +452,7 @@ Raises: IOError: if the task storage does not exist. + OSError: if the task storage does not exist. """ if task.identifier not in self._task_storage_writers: raise IOError('Storage writer for task: {0:s} does not exist.'.format( @@ -447,7 +466,7 @@ Args: serializers_profiler (SerializersProfiler): serializers profiler. """ - pass + pass # pylint: disable=unnecessary-pass def SetStorageProfiler(self, storage_profiler): """Sets the storage profiler. @@ -455,9 +474,9 @@ Args: storage_profiler (StorageProfiler): storage profiler. """ - pass + pass # pylint: disable=unnecessary-pass -# pylint: disable=unused-argument + # pylint: disable=unused-argument def WritePreprocessingInformation(self, knowledge_base): """Writes preprocessing information. @@ -468,6 +487,8 @@ Raises: IOError: if the storage type does not support writing preprocessing information or when the storage writer is closed. + OSError: if the storage type does not support writing preprocessing + information or when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -485,6 +506,8 @@ Raises: IOError: if the storage type does not support writing a session completion or when the storage writer is closed. + OSError: if the storage type does not support writing a session + completion or when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -500,6 +523,8 @@ Raises: IOError: if the storage type does not support writing a session start or when the storage writer is closed. + OSError: if the storage type does not support writing a session + start or when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -517,6 +542,8 @@ Raises: IOError: if the storage type does not support writing a task completion or when the storage writer is closed. + OSError: if the storage type does not support writing a task + completion or when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -532,6 +559,8 @@ Raises: IOError: if the storage type does not support writing a task start or when the storage writer is closed. + OSError: if the storage type does not support writing a task + start or when the storage writer is closed. """ self._RaiseIfNotWritable() diff -Nru plaso-20181219/plaso/storage/interface.py plaso-20190131/plaso/storage/interface.py --- plaso-20181219/plaso/storage/interface.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/storage/interface.py 2019-01-31 19:09:43.000000000 +0000 @@ -89,6 +89,7 @@ self.data_size += len(serialized_data) self.next_sequence_number += 1 + # pylint: disable=redundant-returns-doc,redundant-yields-doc class BaseStore(object): """Storage interface.""" @@ -359,6 +360,7 @@ Raises: IOError: if the serialized data cannot be decoded. + OSError: if the serialized data cannot be decoded. """ if not serialized_data: return None @@ -431,6 +433,7 @@ Raises: IOError: if the attribute container cannot be serialized. + OSError: if the attribute container cannot be serialized. """ if self._serializers_profiler: self._serializers_profiler.StartTiming( @@ -458,6 +461,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ if not self._is_open: raise IOError('Unable to write to closed storage file.') @@ -521,6 +525,7 @@ Raises: IOError: if the serialized data cannot be decoded. + OSError: if the serialized data cannot be decoded. """ if not serialized_data: return None @@ -1179,6 +1184,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._storage_file: raise IOError('Unable to write to closed storage writer.') @@ -1191,6 +1197,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1209,6 +1216,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1223,6 +1231,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1239,6 +1248,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1252,6 +1262,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1266,6 +1277,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1290,6 +1302,7 @@ Raises: IOError: if the storage type is not supported or + OSError: if the storage type is not supported or if the temporary path for the task storage does not exist. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: @@ -1313,6 +1326,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1332,6 +1346,7 @@ Raises: IOError: if the storage type is not supported. + OSError: if the storage type is not supported. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') @@ -1358,6 +1373,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ return self._storage_file.GetEvents() @@ -1391,6 +1407,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._storage_file: raise IOError('Unable to read from closed storage writer.') @@ -1411,6 +1428,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._storage_file: raise IOError('Unable to read from closed storage writer.') @@ -1430,6 +1448,8 @@ Raises: IOError: if the storage type is not supported or if the temporary path for the task storage does not exist. + OSError: if the storage type is not supported or + if the temporary path for the task storage does not exist. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') @@ -1456,6 +1476,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._storage_file: raise IOError('Unable to read from closed storage writer.') @@ -1474,6 +1495,8 @@ Raises: IOError: if the storage type is not supported or if the storage file cannot be renamed. + OSError: if the storage type is not supported or + if the storage file cannot be renamed. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') @@ -1493,6 +1516,7 @@ Raises: IOError: if the storage writer is already opened. + OSError: if the storage writer is already opened. """ if self._storage_file: raise IOError('Storage writer already opened.') @@ -1523,6 +1547,8 @@ Raises: IOError: if the storage type is not supported or if the storage file cannot be renamed. + OSError: if the storage type is not supported or + if the storage file cannot be renamed. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') @@ -1552,6 +1578,7 @@ Raises: IOError: when the storage writer is closed. + OSError: when the storage writer is closed. """ if not self._storage_file: raise IOError('Unable to read from closed storage writer.') @@ -1567,6 +1594,8 @@ Raises: IOError: if the storage type is not supported or if the storage file cannot be removed. + OSError: if the storage type is not supported or + if the storage file cannot be removed. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') @@ -1614,6 +1643,10 @@ if the storage type is not supported or if the temporary path for the task storage does not exist or if the temporary path for the task storage doe not refers to a file. + OSError: if the storage file cannot be opened or + if the storage type is not supported or + if the temporary path for the task storage does not exist or + if the temporary path for the task storage doe not refers to a file. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') @@ -1634,6 +1667,8 @@ Raises: IOError: if the storage type is not supported or if the temporary path for the task storage already exists. + OSError: if the storage type is not supported or + if the temporary path for the task storage already exists. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') @@ -1662,6 +1697,7 @@ Raises: IOError: if the storage type is not supported. + OSError: if the storage type is not supported. """ if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') @@ -1697,6 +1733,8 @@ Raises: IOError: if the storage type does not support writing preprocessing information or when the storage writer is closed. + OSError: if the storage type does not support writing preprocessing + information or when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1714,6 +1752,8 @@ Raises: IOError: if the storage type is not supported or when the storage writer is closed. + OSError: if the storage type is not supported or + when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1730,6 +1770,8 @@ Raises: IOError: if the storage type is not supported or when the storage writer is closed. + OSError: if the storage type is not supported or + when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1748,6 +1790,8 @@ Raises: IOError: if the storage type is not supported or when the storage writer is closed. + OSError: if the storage type is not supported or + when the storage writer is closed. """ self._RaiseIfNotWritable() @@ -1764,6 +1808,8 @@ Raises: IOError: if the storage type is not supported or when the storage writer is closed. + OSError: if the storage type is not supported or + when the storage writer is closed. """ self._RaiseIfNotWritable() diff -Nru plaso-20181219/plaso/storage/sqlite/merge_reader.py plaso-20190131/plaso/storage/sqlite/merge_reader.py --- plaso-20181219/plaso/storage/sqlite/merge_reader.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/storage/sqlite/merge_reader.py 2019-01-31 19:09:43.000000000 +0000 @@ -228,7 +228,11 @@ RuntimeError: if the add method for the active attribute container type is missing. OSError: if the task storage file cannot be deleted. + ValueError: if the maximum number of containers is a negative value. """ + if maximum_number_of_containers < 0: + raise ValueError('Invalid maximum number of containers') + if not self._cursor: self._Open() self._ReadStorageMetadata() @@ -239,11 +243,11 @@ if not self._active_cursor: self._PrepareForNextContainerType() - if maximum_number_of_containers > 0: + if maximum_number_of_containers == 0: + rows = self._active_cursor.fetchall() + else: number_of_rows = maximum_number_of_containers - number_of_containers rows = self._active_cursor.fetchmany(size=number_of_rows) - else: - rows = self._active_cursor.fetchall() if not rows: self._active_cursor = None @@ -277,7 +281,7 @@ number_of_containers += 1 - if (maximum_number_of_containers > 0 and + if (maximum_number_of_containers != 0 and number_of_containers >= maximum_number_of_containers): return False diff -Nru plaso-20181219/plaso/storage/sqlite/sqlite_file.py plaso-20190131/plaso/storage/sqlite/sqlite_file.py --- plaso-20181219/plaso/storage/sqlite/sqlite_file.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/plaso/storage/sqlite/sqlite_file.py 2019-01-31 19:09:43.000000000 +0000 @@ -136,6 +136,7 @@ Raises: IOError: if the attribute container cannot be serialized. + OSError: if the attribute container cannot be serialized. """ container_list = self._GetSerializedAttributeContainerList(container_type) @@ -158,6 +159,7 @@ Raises: IOError: if the event cannot be serialized. + OSError: if the event cannot be serialized. """ identifier = identifiers.SQLTableIdentifier( self._CONTAINER_TYPE_EVENT, @@ -180,6 +182,7 @@ Raises: IOError: if the format version or the serializer format is not supported. + OSError: if the format version or the serializer format is not supported. """ format_version = metadata_values.get('format_version', None) @@ -260,6 +263,7 @@ Raises: IOError: when there is an error querying the storage file. + OSError: when there is an error querying the storage file. """ sequence_number = index + 1 query = 'SELECT _data FROM {0:s} WHERE rowid = {1:d}'.format( @@ -321,6 +325,7 @@ Raises: IOError: when there is an error querying the storage file. + OSError: when there is an error querying the storage file. """ query = 'SELECT _identifier, _data FROM {0:s}'.format(container_type) if filter_expression: @@ -529,6 +534,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ self._RaiseIfNotWritable() @@ -542,6 +548,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ self._RaiseIfNotWritable() @@ -556,6 +563,8 @@ Raises: IOError: when the storage file is closed or read-only or if the event data identifier type is not supported. + OSError: when the storage file is closed or read-only or + if the event data identifier type is not supported. """ self._RaiseIfNotWritable() @@ -579,6 +588,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ self._RaiseIfNotWritable() @@ -592,6 +602,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ self._RaiseIfNotWritable() @@ -607,6 +618,8 @@ Raises: IOError: when the storage file is closed or read-only or if the event identifier type is not supported. + OSError: when the storage file is closed or read-only or + if the event identifier type is not supported. """ self._RaiseIfNotWritable() @@ -628,6 +641,8 @@ Raises: IOError: when the storage file is closed or read-only or if the event tags cannot be serialized. + OSError: when the storage file is closed or read-only or + if the event tags cannot be serialized. """ self._RaiseIfNotWritable() @@ -670,6 +685,7 @@ Raises: IOError: if the storage file is already closed. + OSError: if the storage file is already closed. """ if not self._is_open: raise IOError('Storage file already closed.') @@ -835,6 +851,9 @@ IOError: if a stream is missing or there is a mismatch in session identifiers between the session start and completion attribute containers. + OSError: if a stream is missing or there is a mismatch in session + identifiers between the session start and completion attribute + containers. """ session_start_generator = self._GetAttributeContainers( self._CONTAINER_TYPE_SESSION_START) @@ -842,8 +861,8 @@ self._CONTAINER_TYPE_SESSION_COMPLETION) for session_index in range(0, self._last_session): - session_start = next(session_start_generator) - session_completion = next(session_completion_generator) + session_start = next(session_start_generator) # pylint: disable=stop-iteration-return + session_completion = next(session_completion_generator) # pylint: disable=stop-iteration-return session = sessions.Session() session.CopyAttributesFromSessionStart(session_start) @@ -932,6 +951,8 @@ Raises: IOError: if the storage file is already opened or if the database cannot be connected. + OSError: if the storage file is already opened or if the database + cannot be connected. ValueError: if path is missing. """ if self._is_open: @@ -1030,6 +1051,8 @@ Raises: IOError: if the storage type does not support writing preprocess information or the storage file is closed or read-only. + OSError: if the storage type does not support writing preprocess + information or the storage file is closed or read-only. """ self._RaiseIfNotWritable() @@ -1048,6 +1071,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ self._RaiseIfNotWritable() @@ -1061,6 +1085,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ self._RaiseIfNotWritable() @@ -1074,6 +1099,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ self._RaiseIfNotWritable() @@ -1087,6 +1113,7 @@ Raises: IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. """ self._RaiseIfNotWritable() diff -Nru plaso-20181219/.pylintrc plaso-20190131/.pylintrc --- plaso-20181219/.pylintrc 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/.pylintrc 2019-01-31 19:09:43.000000000 +0000 @@ -1,4 +1,4 @@ -# Pylint 1.7.x - 1.9.x configuration file +# Pylint 2.1.x - 2.2.x configuration file # # This file is generated by l2tdevtools update-dependencies.py, any dependency # related changes should be made in dependencies.ini. @@ -56,23 +56,23 @@ # --disable=W" # disable= - duplicate-code, - parameter-unpacking, - raw-checker-failed, + assignment-from-none, bad-inline-option, - locally-disabled, - locally-enabled, - file-ignored, - suppressed-message, - useless-suppression, deprecated-pragma, - no-absolute-import, - missing-param-doc, - metaclass-assignment, + duplicate-code, eq-without-hash, + file-ignored, fixme, + locally-disabled, + locally-enabled, logging-format-interpolation, + metaclass-assignment, + missing-param-doc, + no-absolute-import, no-self-use, + parameter-unpacking, + raw-checker-failed, + suppressed-message, too-few-public-methods, too-many-ancestors, too-many-boolean-expressions, @@ -84,7 +84,9 @@ too-many-public-methods, too-many-return-statements, too-many-statements, - unsubscriptable-object + unsubscriptable-object, + useless-object-inheritance, + useless-suppression # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff -Nru plaso-20181219/requirements.txt plaso-20190131/requirements.txt --- plaso-20181219/requirements.txt 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/requirements.txt 2019-01-31 19:09:43.000000000 +0000 @@ -15,9 +15,6 @@ elasticsearch >= 6.0 elasticsearch5 >= 5.4.0 future >= 0.16.0 -hachoir-core >= 1.3.3 ; python_version < '3.0' -hachoir-metadata >= 1.3.3 ; python_version < '3.0' -hachoir-parser >= 1.3.4 ; python_version < '3.0' idna >= 2.5 libbde-python >= 20140531 libesedb-python >= 20150409 @@ -46,7 +43,7 @@ pefile >= 2018.8.8 psutil >= 5.4.3 pycrypto >= 2.6 -pyparsing >= 2.0.3 +pyparsing >= 2.3.0 python-dateutil >= 1.5 pytsk3 >= 20160721 pytz diff -Nru plaso-20181219/setup.cfg plaso-20190131/setup.cfg --- plaso-20181219/setup.cfg 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/setup.cfg 2019-01-31 19:09:43.000000000 +0000 @@ -16,61 +16,58 @@ AUTHORS LICENSE README -build_requires = python-setuptools -requires = PyYAML >= 3.10 - libbde-python >= 20140531 - libesedb-python >= 20150409 - libevt-python >= 20120410 - libevtx-python >= 20141112 - libewf-python >= 20131210 +build_requires = python2-setuptools +requires = libbde-python2 >= 20140531 + libesedb-python2 >= 20150409 + libevt-python2 >= 20120410 + libevtx-python2 >= 20141112 + libewf-python2 >= 20131210 libfsapfs-python2 >= 20181205 - libfsntfs-python >= 20151130 - libfvde-python >= 20160719 - libfwnt-python >= 20180117 - libfwsi-python >= 20150606 - liblnk-python >= 20150830 - libmsiecf-python >= 20150314 - libolecf-python >= 20151223 - libqcow-python >= 20131204 - libregf-python >= 20150315 - libscca-python >= 20161031 - libsigscan-python >= 20150627 - libsmdev-python >= 20140529 - libsmraw-python >= 20140612 - libvhdi-python >= 20131210 - libvmdk-python >= 20140421 - libvshadow-python >= 20160109 - libvslvm-python >= 20160109 - python-XlsxWriter >= 0.9.3 - python-artifacts >= 20170818 - python-bencode + libfsntfs-python2 >= 20151130 + libfvde-python2 >= 20160719 + libfwnt-python2 >= 20180117 + libfwsi-python2 >= 20150606 + liblnk-python2 >= 20150830 + libmsiecf-python2 >= 20150314 + libolecf-python2 >= 20151223 + libqcow-python2 >= 20131204 + libregf-python2 >= 20150315 + libscca-python2 >= 20161031 + libsigscan-python2 >= 20150627 + libsmdev-python2 >= 20140529 + libsmraw-python2 >= 20140612 + libvhdi-python2 >= 20131210 + libvmdk-python2 >= 20140421 + libvshadow-python2 >= 20160109 + libvslvm-python2 >= 20160109 python-biplist >= 1.0.3 python-chardet >= 2.0.1 - python-crypto >= 2.6 - python-dfdatetime >= 20180704 python-dfvfs >= 20181209 python-dfwinreg >= 20180712 - python-dtfabric >= 20181128 python-efilter >= 1.5 python-elasticsearch >= 6.0 python-elasticsearch5 >= 5.4.0 - python-hachoir-core >= 1.3.3 - python-hachoir-metadata >= 1.3.3 - python-hachoir-parser >= 1.3.4 - python-pefile >= 2018.8.8 python-psutil >= 5.4.3 - python-pysqlite - python-pytsk3 >= 20160721 - python-requests >= 2.18.0 - python-six >= 1.1.0 + python2-XlsxWriter >= 0.9.3 + python2-artifacts >= 20170818 python2-backports-lzma + python2-bencode python2-certifi >= 2016.9.26 + python2-crypto >= 2.6 python2-dateutil >= 1.5 + python2-dfdatetime >= 20180704 + python2-dtfabric >= 20181128 python2-future >= 0.16.0 python2-idna >= 2.5 python2-lz4 >= 0.10.0 - python2-pyparsing >= 2.0.3 + python2-pefile >= 2018.8.8 + python2-pyparsing >= 2.3.0 + python2-pysqlite + python2-pytsk3 >= 20160721 python2-pytz + python2-pyyaml >= 3.10 + python2-requests >= 2.18.0 + python2-six >= 1.1.0 python2-urllib3 >= 1.21.1 python2-yara >= 3.4.0 python2-zmq >= 2.1.11 diff -Nru plaso-20181219/setup.py plaso-20190131/setup.py --- plaso-20181219/setup.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/setup.py 2019-01-31 19:09:43.000000000 +0000 @@ -29,12 +29,6 @@ except ImportError: from distutils.command.sdist import sdist -# Change PYTHONPATH to include plaso. -sys.path.insert(0, '.') - -import plaso # pylint: disable=wrong-import-position - - version_tuple = (sys.version_info[0], sys.version_info[1]) if version_tuple[0] not in (2, 3): print('Unsupported Python version: {0:s}.'.format(sys.version)) @@ -52,6 +46,11 @@ 'required.').format(sys.version)) sys.exit(1) +# Change PYTHONPATH to include plaso so that we can get the version. +sys.path.insert(0, '.') + +import plaso # pylint: disable=wrong-import-position + if not bdist_msi: BdistMSICommand = None @@ -87,45 +86,98 @@ spec_file = bdist_rpm._make_spec_file(self) if sys.version_info[0] < 3: - python_package = 'python' + python_package = 'python2' else: python_package = 'python3' description = [] - summary = '' + requires = '' + summary = '' # pylint: disable=unused-variable in_description = False python_spec_file = [] for line in iter(spec_file): if line.startswith('Summary: '): - summary = line + summary = line[9:] elif line.startswith('BuildRequires: '): - line = 'BuildRequires: {0:s}-setuptools'.format(python_package) + line = 'BuildRequires: {0:s}-setuptools, {0:s}-devel'.format( + python_package) elif line.startswith('Requires: '): + requires = line[10:] if python_package == 'python3': - line = line.replace('python', 'python3') + requires = requires.replace('python-', 'python3-') + requires = requires.replace('python2-', 'python3-') + continue elif line.startswith('%description'): in_description = True + elif line.startswith('python setup.py build'): + if python_package == 'python3': + line = '%py3_build' + else: + line = '%py2_build' + + elif line.startswith('python setup.py install'): + if python_package == 'python3': + line = '%py3_install' + else: + line = '%py2_install' + elif line.startswith('%files'): - # Cannot use %{_libdir} here since it can expand to "lib64". + python_spec_file.extend([ + '%package -n %{name}-tools', + 'Requires: python-plaso', + 'Summary: Tools for plaso (log2timeline)', + '', + '%description -n %{name}-tools']) + + python_spec_file.extend(description) + lines = [ - '%files -n {0:s}-%{{name}}'.format(python_package), + '%files -n %{name}-data', '%defattr(644,root,root,755)', - '%doc ACKNOWLEDGEMENTS AUTHORS LICENSE README', - '%{_prefix}/bin/*.py', - '%{_prefix}/lib/python*/site-packages/**/*.py', - '%{_prefix}/lib/python*/site-packages/**/*.yaml', - '%{_prefix}/lib/python*/site-packages/plaso*.egg-info/*', - '%{_prefix}/share/plaso/*', + '%license LICENSE', + '%doc ACKNOWLEDGEMENTS AUTHORS README', + '%{_datadir}/%{name}/*', '', - '%exclude %{_prefix}/share/doc/*', - '%exclude %{_prefix}/lib/python*/site-packages/**/*.pyc', - '%exclude %{_prefix}/lib/python*/site-packages/**/*.pyo', - '%exclude %{_prefix}/lib/python*/site-packages/**/__pycache__/*'] + '%files -n {0:s}-%{{name}}'.format(python_package), + '%defattr(644,root,root,755)', + '%license LICENSE', + '%doc ACKNOWLEDGEMENTS AUTHORS README'] + + if python_package == 'python3': + lines.extend([ + '%{python3_sitelib}/plaso/*.py', + '%{python3_sitelib}/plaso/*/*.py', + '%{python3_sitelib}/plaso/*/*.yaml', + '%{python3_sitelib}/plaso/*/*/*.py', + '%{python3_sitelib}/plaso/*/*/*.yaml', + '%{python3_sitelib}/plaso*.egg-info/*', + '', + '%exclude %{_prefix}/share/doc/*', + '%exclude %{python3_sitelib}/plaso/__pycache__/*', + '%exclude %{python3_sitelib}/plaso/*/__pycache__/*', + '%exclude %{python3_sitelib}/plaso/*/*/__pycache__/*']) + + else: + lines.extend([ + '%{python2_sitelib}/plaso/*.py', + '%{python2_sitelib}/plaso/*/*.py', + '%{python2_sitelib}/plaso/*/*.yaml', + '%{python2_sitelib}/plaso/*/*/*.py', + '%{python2_sitelib}/plaso/*/*/*.yaml', + '%{python2_sitelib}/plaso*.egg-info/*', + '', + '%exclude %{_prefix}/share/doc/*', + '%exclude %{python2_sitelib}/plaso/*.pyc', + '%exclude %{python2_sitelib}/plaso/*.pyo', + '%exclude %{python2_sitelib}/plaso/*/*.pyc', + '%exclude %{python2_sitelib}/plaso/*/*.pyo', + '%exclude %{python2_sitelib}/plaso/*/*/*.pyc', + '%exclude %{python2_sitelib}/plaso/*/*/*.pyo']) python_spec_file.extend(lines) break @@ -133,12 +185,30 @@ elif line.startswith('%prep'): in_description = False + python_spec_file.extend([ + '%package -n %{name}-data', + 'Summary: Data files for plaso (log2timeline)', + '', + '%description -n %{name}-data']) + + python_spec_file.extend(description) + python_spec_file.append( '%package -n {0:s}-%{{name}}'.format(python_package)) - python_spec_file.append('{0:s}'.format(summary)) - python_spec_file.append('') - python_spec_file.append( - '%description -n {0:s}-%{{name}}'.format(python_package)) + if python_package == 'python2': + python_spec_file.extend([ + 'Obsoletes: python-plaso < %{version}', + 'Provides: python-plaso = %{version}']) + python_summary = 'Python 2 module of plaso (log2timeline)' + else: + python_summary = 'Python 3 module of plaso (log2timeline)' + + python_spec_file.extend([ + 'Requires: plaso-data {0:s}'.format(requires), + 'Summary: {0:s}'.format(python_summary), + '', + '%description -n {0:s}-%{{name}}'.format(python_package)]) + python_spec_file.extend(description) elif in_description: @@ -150,6 +220,11 @@ python_spec_file.append(line) + python_spec_file.extend([ + '', + '%files -n %{name}-tools', + '%{_bindir}/*.py']) + return python_spec_file @@ -160,7 +235,7 @@ if not encoding: encoding = locale.getpreferredencoding() - # Make sure the default encoding is set correctly otherwise + # Make sure the default encoding is set correctly otherwise on Python 2 # setup.py sdist will fail to include filenames with Unicode characters. reload(sys) # pylint: disable=undefined-variable @@ -169,7 +244,8 @@ # Unicode in the description will break python-setuptools, hence # "Plaso Langar Að Safna Öllu" was removed. -plaso_description = 'Super timeline all the things' +plaso_description = ( + 'Super timeline all the things.') plaso_long_description = ( 'Plaso (log2timeline) is a framework to create super timelines. Its ' @@ -198,7 +274,7 @@ packages=find_packages('.', exclude=[ 'docs', 'tests', 'tests.*', 'tools', 'utils']), package_dir={ - 'plaso': 'plaso', + 'plaso': 'plaso' }, include_package_data=True, package_data={ @@ -206,7 +282,7 @@ 'plaso.parsers.esedb_plugins': ['*.yaml'], 'plaso.parsers.olecf_plugins': ['*.yaml'], 'plaso.parsers.plist_plugins': ['*.yaml'], - 'plaso.parsers.winreg_plugins': ['*.yaml'], + 'plaso.parsers.winreg_plugins': ['*.yaml'] }, zip_safe=False, scripts=glob.glob(os.path.join('tools', '[a-z]*.py')), diff -Nru plaso-20181219/test_data/artifacts/artifacts_filters.yaml plaso-20190131/test_data/artifacts/artifacts_filters.yaml --- plaso-20181219/test_data/artifacts/artifacts_filters.yaml 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/test_data/artifacts/artifacts_filters.yaml 2019-01-31 19:09:43.000000000 +0000 @@ -11,6 +11,17 @@ labels: [System] supported_os: [Windows] --- +name: TestGroupExtract +doc: Test Group +sources: +- type: ARTIFACT_GROUP + attributes: + names: + - 'TestFiles' + - 'TestFiles2' +labels: [System] +supported_os: [Windows] +--- name: TestFiles doc: Test Doc sources: diff -Nru plaso-20181219/test_data/presets.yaml plaso-20190131/test_data/presets.yaml --- plaso-20181219/test_data/presets.yaml 1970-01-01 00:00:00.000000000 +0000 +++ plaso-20190131/test_data/presets.yaml 2019-01-31 19:09:43.000000000 +0000 @@ -0,0 +1,150 @@ +# Parser and parser plugin presets. + +name: android +description: Preset for Android. +parsers: +- android_app_usage +- chrome_cache +- filestat +- sqlite/android_calls +- sqlite/android_sms +- sqlite/android_webview +- sqlite/android_webviewcache +- sqlite/chrome_27_history +- sqlite/chrome_8_history +- sqlite/chrome_cookies +- sqlite/skype +--- +name: linux +description: Preset for Linux. +operating_systems: +- {family: Linux} +parsers: +- bash_history +- bencode +- czip/oxml +- dockerjson +- dpkg +- filestat +- gdrive_synclog +- java_idx +- olecf +- pls_recall +- popularity_contest +- selinux +- sqlite/google_drive +- sqlite/skype +- sqlite/zeitgeist +- syslog +- systemd_journal +- utmp +- webhist +- xchatlog +- xchatscrollback +- zsh_extended_history +--- +name: macos +description: Preset for MacOS. +operating_systems: +- {family: MacOS} +parsers: +- asl_log +- bash_history +- bencode +- bsm_log +- cups_ipp +- czip/oxml +- filestat +- fseventsd +- gdrive_synclog +- java_idx +- mac_appfirewall_log +- mac_keychain +- mac_securityd +- macwifi +- olecf +- plist +- sqlite/appusage +- sqlite/google_drive +- sqlite/imessage +- sqlite/ls_quarantine +- sqlite/mac_document_versions +- sqlite/mackeeper_cache +- sqlite/skype +- syslog +- utmpx +- webhist +- zsh_extended_history +--- +name: webhist +description: Preset for Web history. +parsers: +- binary_cookies +- chrome_cache +- chrome_preferences +- esedb/msie_webcache +- firefox_cache +- java_idx +- msiecf +- opera_global +- opera_typed_history +- plist/safari_history +- sqlite/chrome_27_history +- sqlite/chrome_8_history +- sqlite/chrome_autofill +- sqlite/chrome_cookies +- sqlite/chrome_extension_activity +- sqlite/firefox_cookies +- sqlite/firefox_downloads +- sqlite/firefox_history +--- +name: win_gen +description: Preset for generic Windows, intended as template for other Windows presets. +parsers: +- bencode +- czip/oxml +- esedb +- filestat +- gdrive_synclog +- java_idx +- lnk +- mcafee_protection +- olecf +- pe +- prefetch +- sccm +- skydrive_log +- skydrive_log_old +- sqlite/google_drive +- sqlite/skype +- symantec_scanlog +- usnjrnl +- webhist +- winfirewall +- winjob +- winreg +--- +name: win7 +description: Preset for generic Windows, intended for Windows Vista and later. +operating_systems: +- {family: Windows NT} +parsers: +- amcache +- custom_destinations +- esedb/file_history +- olecf/olecf_automatic_destinations +- recycle_bin +- winevtx +- win_gen +--- +name: winxp +description: Preset for Windows 2000, Windows XP and Windows 2003. +operating_systems: +- {product: Windows 2000} +- {product: Windows XP} +- {product: Windows 2003} +parsers: +- recycle_bin_info2 +- rplog +- win_gen +- winevt diff -Nru plaso-20181219/tests/analysis/chrome_extension.py plaso-20190131/tests/analysis/chrome_extension.py --- plaso-20181219/tests/analysis/chrome_extension.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/analysis/chrome_extension.py 2019-01-31 19:09:43.000000000 +0000 @@ -27,12 +27,12 @@ extension_identifier (str): Chrome extension identifier. Returns: - str: page content or None. + str: page content or None if not available. """ chrome_web_store_file = shared_test_lib.GetTestFilePath([ 'chrome_extensions', extension_identifier]) if not os.path.exists(chrome_web_store_file): - return + return None with open(chrome_web_store_file, 'rb') as file_object: page_content = file_object.read() diff -Nru plaso-20181219/tests/analysis/manager.py plaso-20190131/tests/analysis/manager.py --- plaso-20181219/tests/analysis/manager.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/analysis/manager.py 2019-01-31 19:09:43.000000000 +0000 @@ -68,8 +68,9 @@ """Tests the GetPlugins function.""" manager.AnalysisPluginManager.RegisterPlugin(TestAnalysisPlugin) - plugin_set = set([name for name, _ in list( - manager.AnalysisPluginManager.GetPlugins())]) + # Use set-comprehension to create a set of the analysis plugin names. + plugin_set = {name for name, _ in list( + manager.AnalysisPluginManager.GetPlugins())} self.assertTrue('test_plugin' in plugin_set) manager.AnalysisPluginManager.DeregisterPlugin(TestAnalysisPlugin) diff -Nru plaso-20181219/tests/analysis/nsrlsvr.py plaso-20190131/tests/analysis/nsrlsvr.py --- plaso-20181219/tests/analysis/nsrlsvr.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/analysis/nsrlsvr.py 2019-01-31 19:09:43.000000000 +0000 @@ -51,7 +51,7 @@ def close(self): """Mocks the socket.close method""" - pass + pass # pylint: disable=unnecessary-pass class NsrlSvrTest(test_lib.AnalysisPluginTestCase): diff -Nru plaso-20181219/tests/analysis/virustotal.py plaso-20190131/tests/analysis/virustotal.py --- plaso-20181219/tests/analysis/virustotal.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/analysis/virustotal.py 2019-01-31 19:09:43.000000000 +0000 @@ -64,13 +64,15 @@ # pylint: disable=protected-access self.assertEqual( url, virustotal.VirusTotalAnalyzer._VIRUSTOTAL_API_REPORT_URL) - if params['resource'] == self._EVENT_1_HASH: - response = MockResponse() - response['resource'] = self._EVENT_1_HASH - response['response_code'] = 1 - response['positives'] = 10 - return response - self.fail('Unexpected parameters to request.get()') + + if params['resource'] != self._EVENT_1_HASH: + self.fail('Unexpected parameters to request.get()') + + response = MockResponse() + response['resource'] = self._EVENT_1_HASH + response['response_code'] = 1 + response['positives'] = 10 + return response def _CreateTestEventObject(self, event_dictionary): """Create a test event with a set of attributes. diff -Nru plaso-20181219/tests/cli/helpers/artifact_filters.py plaso-20190131/tests/cli/helpers/artifact_filters.py --- plaso-20181219/tests/cli/helpers/artifact_filters.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/cli/helpers/artifact_filters.py 2019-01-31 19:09:43.000000000 +0000 @@ -64,7 +64,7 @@ def testParseOptions(self): """Tests the ParseOptions function.""" options = cli_test_lib.TestOptions() - options.artifact_filters = 'TestFiles, TestFiles2' + options.artifact_filter_string = 'TestFiles, TestFiles2' expected_output = ['TestFiles', 'TestFiles2'] test_tool = tools.CLITool() @@ -82,7 +82,7 @@ expected_output = ['TestFiles', 'TestFiles2', 'TestFiles3'] - options.artifact_filters = None + options.artifact_filter_string = None artifact_filters.ArtifactFiltersArgumentsHelper.ParseOptions( options, test_tool) diff -Nru plaso-20181219/tests/cli/helpers/timesketch_output.py plaso-20190131/tests/cli/helpers/timesketch_output.py --- plaso-20181219/tests/cli/helpers/timesketch_output.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/cli/helpers/timesketch_output.py 2019-01-31 19:09:43.000000000 +0000 @@ -28,7 +28,7 @@ # We deliberately don't call super here to prevent the call to # timesketch.create_app() raising an exception. # pylint: disable=super-init-not-called - pass + pass # pylint: disable=unnecessary-pass class TimesketchOutputArgumentsHelperTest( diff -Nru plaso-20181219/tests/cli/image_export_tool.py plaso-20190131/tests/cli/image_export_tool.py --- plaso-20181219/tests/cli/image_export_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/cli/image_export_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -16,7 +16,7 @@ from plaso.lib import errors from tests import test_lib as shared_test_lib -from tests.cli import test_lib as test_lib +from tests.cli import test_lib class ImageExportToolTest(test_lib.CLIToolTestCase): @@ -410,7 +410,7 @@ options.artifact_definitions_path = self._GetTestFilePath(['artifacts']) options.image = self._GetTestFilePath(['image.qcow2']) options.quiet = True - options.artifact_filters = 'TestFilesImageExport' + options.artifact_filter_string = 'TestFilesImageExport' with shared_test_lib.TempDirectory() as temp_directory: options.path = temp_directory @@ -439,7 +439,7 @@ options.artifact_definitions_path = self._GetTestFilePath(['artifacts']) options.image = self._GetTestFilePath(['image.qcow2']) options.quiet = True - options.artifact_filters = 'TestGroupExport' + options.artifact_filter_string = 'TestGroupExport' with shared_test_lib.TempDirectory() as temp_directory: options.path = temp_directory diff -Nru plaso-20181219/tests/cli/log2timeline_tool.py plaso-20190131/tests/cli/log2timeline_tool.py --- plaso-20181219/tests/cli/log2timeline_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/cli/log2timeline_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -137,6 +137,8 @@ def testGetPluginData(self): """Tests the _GetPluginData function.""" test_tool = log2timeline_tool.Log2TimelineTool() + test_tool._data_location = self._GetTestFilePath([]) + plugin_info = test_tool._GetPluginData() self.assertIn('Hashers', plugin_info) diff -Nru plaso-20181219/tests/cli/psort_tool.py plaso-20190131/tests/cli/psort_tool.py --- plaso-20181219/tests/cli/psort_tool.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/cli/psort_tool.py 2019-01-31 19:09:43.000000000 +0000 @@ -47,7 +47,7 @@ @classmethod def AddArguments(cls, argument_group): """Mock the add argument section.""" - pass + pass # pylint: disable=unnecessary-pass # pylint: disable=arguments-differ @classmethod diff -Nru plaso-20181219/tests/cli/tool_options.py plaso-20190131/tests/cli/tool_options.py --- plaso-20181219/tests/cli/tool_options.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/cli/tool_options.py 2019-01-31 19:09:43.000000000 +0000 @@ -9,6 +9,7 @@ from plaso.cli import tool_options from plaso.cli import tools from plaso.output import manager as output_manager +from plaso.parsers import manager as parsers_manager from tests import test_lib as shared_test_lib from tests.cli import test_lib @@ -185,6 +186,7 @@ """Tool to test the parsers options.""" +@shared_test_lib.skipUnlessHasTestFile(['presets.yaml']) class ParsersOptionsTest(test_lib.CLIToolTestCase): """Tests for the parsers options.""" @@ -194,6 +196,9 @@ """Tests the _GetParserPresetsInformation function.""" test_tool = TestToolWithParsersOptions() + presets_file = self._GetTestFilePath(['presets.yaml']) + parsers_manager.ParsersManager.ReadPresetsFromFile(presets_file) + parser_presets_information = test_tool._GetParserPresetsInformation() self.assertGreaterEqual(len(parser_presets_information), 1) @@ -205,6 +210,9 @@ output_writer = test_lib.TestOutputWriter(encoding='utf-8') test_tool = TestToolWithParsersOptions(output_writer=output_writer) + presets_file = self._GetTestFilePath(['presets.yaml']) + parsers_manager.ParsersManager.ReadPresetsFromFile(presets_file) + test_tool.ListParsersAndPlugins() output = output_writer.ReadOutput() diff -Nru plaso-20181219/tests/containers/artifacts.py plaso-20190131/tests/containers/artifacts.py --- plaso-20181219/tests/containers/artifacts.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/containers/artifacts.py 2019-01-31 19:09:43.000000000 +0000 @@ -7,6 +7,7 @@ import unittest from plaso.containers import artifacts +from plaso.lib import definitions from tests import test_lib as shared_test_lib @@ -14,86 +15,130 @@ class EnvironmentVariableArtifactTest(shared_test_lib.BaseTestCase): """Tests for the environment variable artifact.""" - # TODO: replace by GetAttributeNames test - def testCopyToDict(self): - """Tests the CopyToDict function.""" - attribute_container = artifacts.EnvironmentVariableArtifact( - case_sensitive=False, name='SystemRoot', value='C:\\Windows') - - self.assertEqual(attribute_container.name, 'SystemRoot') - - expected_dict = { - 'case_sensitive': False, - 'name': 'SystemRoot', - 'value': 'C:\\Windows'} + def testGetAttributeNames(self): + """Tests the GetAttributeNames function.""" + attribute_container = artifacts.EnvironmentVariableArtifact() - test_dict = attribute_container.CopyToDict() + expected_attribute_names = ['case_sensitive', 'name', 'value'] - self.assertEqual(test_dict, expected_dict) + attribute_names = sorted(attribute_container.GetAttributeNames()) + self.assertEqual(attribute_names, expected_attribute_names) class HostnameArtifactTest(shared_test_lib.BaseTestCase): """Tests for the hostname artifact.""" - # TODO: replace by GetAttributeNames test - def testCopyToDict(self): - """Tests the CopyToDict function.""" - attribute_container = artifacts.HostnameArtifact(name='mydomain.com') + def testGetAttributeNames(self): + """Tests the GetAttributeNames function.""" + attribute_container = artifacts.HostnameArtifact() - self.assertEqual(attribute_container.name, 'mydomain.com') + expected_attribute_names = ['name', 'schema'] - expected_dict = { - 'name': 'mydomain.com', - 'schema': 'DNS'} + attribute_names = sorted(attribute_container.GetAttributeNames()) + self.assertEqual(attribute_names, expected_attribute_names) - test_dict = attribute_container.CopyToDict() - self.assertEqual(test_dict, expected_dict) +class OperatingSystemArtifactTest(shared_test_lib.BaseTestCase): + """Tests for the operating system artifact.""" + + # pylint: disable=protected-access + + def testVersionTuple(self): + """Tests the version_tuplele property.""" + attribute_container = artifacts.OperatingSystemArtifact(version="5.1") + self.assertEqual(attribute_container.version_tuple, (5, 1)) + + attribute_container = artifacts.OperatingSystemArtifact() + self.assertIsNone(attribute_container.version_tuple) + + attribute_container = artifacts.OperatingSystemArtifact(version="5.a") + self.assertIsNone(attribute_container.version_tuple) + + def testGetNameFromProduct(self): + """Tests the _GetNameFromProduct function.""" + attribute_container = artifacts.OperatingSystemArtifact( + product='Windows Server 2012 R2 Standard') + + name = attribute_container._GetNameFromProduct() + self.assertEqual(name, 'Windows 2012 R2') + + attribute_container = artifacts.OperatingSystemArtifact( + product='Microsoft Windows Server 2003') + + name = attribute_container._GetNameFromProduct() + self.assertEqual(name, 'Windows 2003') + + def testIsEquivalent(self): + """Tests the IsEquivalent function.""" + win2k12_container = artifacts.OperatingSystemArtifact( + product='Windows 2012') + winxp_container = artifacts.OperatingSystemArtifact(product='Windows XP') + + self.assertFalse(win2k12_container.IsEquivalent(winxp_container)) + self.assertFalse(winxp_container.IsEquivalent(win2k12_container)) + + winnt62_container = artifacts.OperatingSystemArtifact( + family=definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, version='6.2') + winnt51_container = artifacts.OperatingSystemArtifact( + family=definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, version='5.1') + + self.assertFalse(winnt62_container.IsEquivalent(winnt51_container)) + self.assertFalse(winnt51_container.IsEquivalent(winnt62_container)) + + win9x_container = artifacts.OperatingSystemArtifact( + family=definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_9x) + winnt_container = artifacts.OperatingSystemArtifact( + family=definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT) + + self.assertFalse(win9x_container.IsEquivalent(winnt_container)) + self.assertFalse(winnt_container.IsEquivalent(win9x_container)) + + winnt51_container = artifacts.OperatingSystemArtifact( + family=definitions.OPERATING_SYSTEM_FAMILY_WINDOWS_NT, version='5.1') + winxp_container = artifacts.OperatingSystemArtifact(product='Windows XP') + + self.assertTrue(winnt51_container.IsEquivalent(winxp_container)) + self.assertTrue(winxp_container.IsEquivalent(winnt51_container)) + + def testGetAttributeNames(self): + """Tests the GetAttributeNames function.""" + attribute_container = artifacts.OperatingSystemArtifact() + + expected_attribute_names = ['family', 'name', 'product', 'version'] + + attribute_names = sorted(attribute_container.GetAttributeNames()) + self.assertEqual(attribute_names, expected_attribute_names) class SystemConfigurationArtifactTest(shared_test_lib.BaseTestCase): """Tests for the system configuration artifact.""" - # TODO: replace by GetAttributeNames test - def testCopyToDict(self): - """Tests the CopyToDict function.""" - attribute_container = artifacts.SystemConfigurationArtifact( - code_page='cp1252', time_zone='UTC') - - self.assertEqual(attribute_container.time_zone, 'UTC') - - expected_dict = { - 'code_page': 'cp1252', - 'time_zone': 'UTC', - 'user_accounts': []} + def testGetAttributeNames(self): + """Tests the GetAttributeNames function.""" + attribute_container = artifacts.SystemConfigurationArtifact() + + expected_attribute_names = [ + 'code_page', 'hostname', 'keyboard_layout', 'operating_system', + 'operating_system_product', 'operating_system_version', 'time_zone', + 'user_accounts'] - test_dict = attribute_container.CopyToDict() - - self.assertEqual(test_dict, expected_dict) + attribute_names = sorted(attribute_container.GetAttributeNames()) + self.assertEqual(attribute_names, expected_attribute_names) class UserAccountArtifactTest(shared_test_lib.BaseTestCase): """Tests for the user account artifact.""" - # TODO: replace by GetAttributeNames test - def testCopyToDict(self): - """Tests the CopyToDict function.""" - attribute_container = artifacts.UserAccountArtifact( - full_name='Full Name', group_identifier=1001, identifier=1000, - user_directory='/home/username', username='username') - - self.assertEqual(attribute_container.username, 'username') - - expected_dict = { - 'full_name': 'Full Name', - 'group_identifier': 1001, - 'identifier': 1000, - 'user_directory': '/home/username', - 'username': 'username'} - - test_dict = attribute_container.CopyToDict() + def testGetAttributeNames(self): + """Tests the GetAttributeNames function.""" + attribute_container = artifacts.UserAccountArtifact() + + expected_attribute_names = [ + 'full_name', 'group_identifier', 'identifier', 'user_directory', + 'username'] - self.assertEqual(test_dict, expected_dict) + attribute_names = sorted(attribute_container.GetAttributeNames()) + self.assertEqual(attribute_names, expected_attribute_names) if __name__ == '__main__': diff -Nru plaso-20181219/tests/engine/artifact_filters.py plaso-20190131/tests/engine/artifact_filters.py --- plaso-20181219/tests/engine/artifact_filters.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/engine/artifact_filters.py 2019-01-31 19:09:43.000000000 +0000 @@ -4,6 +4,7 @@ from __future__ import unicode_literals +import itertools import unittest from artifacts import definitions as artifact_types @@ -83,15 +84,20 @@ environment_variables=[environment_variable]) find_specs_per_source_type = knowledge_base.GetValue( test_filter_file.KNOWLEDGE_BASE_VALUE) - find_specs = find_specs_per_source_type.get( + file_find_specs = find_specs_per_source_type.get( artifact_types.TYPE_INDICATOR_FILE, []) - # Should build 15 FindSpec entries. - self.assertEqual(len(find_specs), 15) + # Should build 15 file FindSpec entries. + self.assertEqual(len(file_find_specs), 15) + + all_find_specs = iter(find_specs_per_source_type.values()) + all_find_specs = itertools.chain(*all_find_specs) + find_spec_count = len(list(all_find_specs)) + self.assertEqual(find_spec_count, 15) # Last find_spec should contain the testuser2 profile path. location_segments = sorted([ - find_spec._location_segments for find_spec in find_specs]) + find_spec._location_segments for find_spec in file_find_specs]) path_segments = [ 'Users', 'testuser2', 'Documents', 'WindowsPowerShell', 'profile\\.ps1'] self.assertEqual(location_segments[2], path_segments) @@ -102,7 +108,7 @@ searcher = file_system_searcher.FileSystemSearcher( file_system, path_spec) - path_spec_generator = searcher.Find(find_specs=find_specs) + path_spec_generator = searcher.Find(find_specs=file_find_specs) self.assertIsNotNone(path_spec_generator) path_specs = list(path_spec_generator) @@ -114,20 +120,85 @@ file_system.Close() @shared_test_lib.skipUnlessHasTestFile(['artifacts']) + @shared_test_lib.skipUnlessHasTestFile(['System.evtx']) + @shared_test_lib.skipUnlessHasTestFile(['testdir', 'filter_1.txt']) + @shared_test_lib.skipUnlessHasTestFile(['testdir', 'filter_3.txt']) + def testBuildFindSpecsWithFileSystemAndGroup(self): + """Tests the BuildFindSpecs function for file type artifacts.""" + knowledge_base = knowledge_base_engine.KnowledgeBase() + + testuser1 = artifacts.UserAccountArtifact( + identifier='1000', + user_directory='C:\\Users\\testuser1', + username='testuser1') + knowledge_base.AddUserAccount(testuser1) + + testuser2 = artifacts.UserAccountArtifact( + identifier='1001', + user_directory='C:\\Users\\testuser2', + username='testuser2') + knowledge_base.AddUserAccount(testuser2) + + filter_helper = self._CreateTestArtifactDefinitionsFilterHelper( + ['TestGroupExtract'], knowledge_base) + + environment_variable = artifacts.EnvironmentVariableArtifact( + case_sensitive=False, name='SystemDrive', value='C:') + + filter_helper.BuildFindSpecs( + environment_variables=[environment_variable]) + find_specs_per_source_type = knowledge_base.GetValue( + filter_helper.KNOWLEDGE_BASE_VALUE) + file_find_specs = find_specs_per_source_type.get( + artifact_types.TYPE_INDICATOR_FILE, []) + + # Should build 15 file FindSpec entries. + self.assertEqual(len(file_find_specs), 15) + + all_find_specs = iter(find_specs_per_source_type.values()) + all_find_specs = itertools.chain(*all_find_specs) + find_spec_count = len(list(all_find_specs)) + self.assertEqual(find_spec_count, 15) + + path_spec = path_spec_factory.Factory.NewPathSpec( + dfvfs_definitions.TYPE_INDICATOR_OS, location='.') + file_system = path_spec_resolver.Resolver.OpenFileSystem(path_spec) + searcher = file_system_searcher.FileSystemSearcher( + file_system, path_spec) + + path_spec_generator = searcher.Find(find_specs=file_find_specs) + self.assertIsNotNone(path_spec_generator) + + path_specs = list(path_spec_generator) + + # Two evtx, one symbolic link to evtx, one AUTHORS, two filter_*.txt + # files, + # total 6 path specifications. + self.assertEqual(len(path_specs), 6) + + file_system.Close() + + @shared_test_lib.skipUnlessHasTestFile(['artifacts']) @shared_test_lib.skipUnlessHasTestFile(['SYSTEM']) def testBuildFindSpecsWithRegistry(self): - """Tests the BuildFindSpecs function on Windows Registry artifacts.""" + """Tests the BuildFindSpecs function on Windows Registry sources.""" knowledge_base = knowledge_base_engine.KnowledgeBase() test_filter_file = self._CreateTestArtifactDefinitionsFilterHelper( - ['TestRegistry'], knowledge_base) + ['TestRegistry', 'TestRegistryValue'], knowledge_base) test_filter_file.BuildFindSpecs(environment_variables=None) find_specs_per_source_type = knowledge_base.GetValue( test_filter_file.KNOWLEDGE_BASE_VALUE) - find_specs = find_specs_per_source_type.get( + registry_find_specs = find_specs_per_source_type.get( artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY, []) - self.assertEqual(len(find_specs), 1) + self.assertEqual(len(registry_find_specs), 3) + + all_find_specs = iter(find_specs_per_source_type.values()) + all_find_specs = itertools.chain(*all_find_specs) + find_spec_count = len(list(all_find_specs)) + self.assertEqual(find_spec_count, 3) + win_registry_reader = ( windows_registry_parser.FileObjectWinRegistryFileReader()) @@ -143,12 +214,11 @@ win_registry.MapFile(key_path_prefix, registry_file) searcher = dfwinreg_registry_searcher.WinRegistrySearcher(win_registry) - key_paths = list(searcher.Find(find_specs=find_specs)) + key_paths = list(searcher.Find(find_specs=registry_find_specs)) self.assertIsNotNone(key_paths) - # Three key paths found. - self.assertEqual(len(key_paths), 3) + self.assertEqual(len(key_paths), 5) def testCheckKeyCompatibility(self): """Tests the CheckKeyCompatibility function""" @@ -167,7 +237,7 @@ self.assertFalse(compatible_key) def testBuildFindSpecsFromFileArtifact(self): - """Tests the BuildFindSpecsFromFileArtifact function for file artifacts.""" + """Tests the BuildFindSpecsFromFileArtifact function on file sources.""" knowledge_base = knowledge_base_engine.KnowledgeBase() test_filter_file = self._CreateTestArtifactDefinitionsFilterHelper( [], knowledge_base) @@ -180,7 +250,7 @@ environment_variable = [artifacts.EnvironmentVariableArtifact( case_sensitive=False, name='SystemRoot', value='C:\\Windows')] - find_specs = test_filter_file.BuildFindSpecsFromFileArtifact( + find_specs = test_filter_file._BuildFindSpecsFromFileSourcePath( path_entry, separator, environment_variable, user_accounts) # Should build 1 find_spec. @@ -198,7 +268,7 @@ # Test expansion of globs. path_entry = '\\test_data\\**' - find_specs = test_filter_file.BuildFindSpecsFromFileArtifact( + find_specs = test_filter_file._BuildFindSpecsFromFileSourcePath( path_entry, separator, environment_variable, user_accounts) # Glob expansion should by default recurse ten levels. @@ -226,7 +296,7 @@ user_accounts = [testuser1, testuser2] path_entry = '%%users.homedir%%/.thumbnails/**3' - find_specs = test_filter_file.BuildFindSpecsFromFileArtifact( + find_specs = test_filter_file._BuildFindSpecsFromFileSourcePath( path_entry, separator, environment_variable, user_accounts) # Six total find specs should be created for testuser1 and testuser2. @@ -245,7 +315,7 @@ user_accounts = [testuser1, testuser2] path_entry = '%%users.homedir%%\\AppData\\**4' - find_specs = test_filter_file.BuildFindSpecsFromFileArtifact( + find_specs = test_filter_file._BuildFindSpecsFromFileSourcePath( path_entry, separator, environment_variable, user_accounts) # Eight find specs should be created for testuser1 and testuser2. diff -Nru plaso-20181219/tests/engine/engine.py plaso-20190131/tests/engine/engine.py --- plaso-20181219/tests/engine/engine.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/engine/engine.py 2019-01-31 19:09:43.000000000 +0000 @@ -91,7 +91,8 @@ file_system, mount_point) operating_system = test_engine._DetermineOperatingSystem(searcher) - self.assertEqual(operating_system, definitions.OPERATING_SYSTEM_UNKNOWN) + self.assertEqual( + operating_system, definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN) def testStartStopProfiling(self): """Tests the _StartProfiling and _StopProfiling functions.""" @@ -161,7 +162,7 @@ test_engine.PreprocessSources(registry, [source_path_spec]) operating_system = test_engine.knowledge_base.GetValue('operating_system') - self.assertEqual(operating_system, 'Windows') + self.assertEqual(operating_system, 'Windows NT') test_engine.PreprocessSources(registry, [None]) diff -Nru plaso-20181219/tests/filters/test_lib.py plaso-20190131/tests/filters/test_lib.py --- plaso-20181219/tests/filters/test_lib.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/filters/test_lib.py 2019-01-31 19:09:43.000000000 +0000 @@ -20,7 +20,7 @@ Raises: WrongPlugin: if the filter could not be compiled. """ - pass + pass # pylint: disable=unnecessary-pass class FilterTestCase(shared_test_lib.BaseTestCase): diff -Nru plaso-20181219/tests/formatters/hachoir.py plaso-20190131/tests/formatters/hachoir.py --- plaso-20181219/tests/formatters/hachoir.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/formatters/hachoir.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -"""Tests for the Hachoir event formatter.""" - -from __future__ import unicode_literals - -import unittest - -from plaso.formatters import hachoir - -from tests.formatters import test_lib - - -class HachoirFormatterTest(test_lib.EventFormatterTestCase): - """Tests for the Hachoir event formatter.""" - - def testInitialization(self): - """Tests the initialization.""" - event_formatter = hachoir.HachoirFormatter() - self.assertIsNotNone(event_formatter) - - def testGetFormatStringAttributeNames(self): - """Tests the GetFormatStringAttributeNames function.""" - event_formatter = hachoir.HachoirFormatter() - - expected_attribute_names = ['data'] - - self._TestGetFormatStringAttributeNames( - event_formatter, expected_attribute_names) - - # TODO: add test for GetMessages. - - -if __name__ == '__main__': - unittest.main() diff -Nru plaso-20181219/tests/multi_processing/psort.py plaso-20190131/tests/multi_processing/psort.py --- plaso-20181219/tests/multi_processing/psort.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/multi_processing/psort.py 2019-01-31 19:09:43.000000000 +0000 @@ -60,7 +60,7 @@ analysis plugins and other components, such as storage and dfvfs. event (EventObject): event. """ - pass + pass # pylint: disable=unnecessary-pass class TestEvent(events.EventObject): @@ -135,7 +135,7 @@ def WriteHeader(self): """Writes the header to the output.""" - pass + pass # pylint: disable=unnecessary-pass def WriteEventMACBGroup(self, event_macb_group): """Writes an event MACB group to the output. diff -Nru plaso-20181219/tests/multi_processing/worker_process.py plaso-20190131/tests/multi_processing/worker_process.py --- plaso-20181219/tests/multi_processing/worker_process.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/multi_processing/worker_process.py 2019-01-31 19:09:43.000000000 +0000 @@ -32,7 +32,7 @@ other components, such as storage and dfvfs. path_spec (dfvfs.PathSpec): path specification. """ - pass + pass # pylint: disable=unnecessary-pass class TestFailureEventExtractionWorker(worker.EventExtractionWorker): diff -Nru plaso-20181219/tests/output/fake_mysqldb.py plaso-20190131/tests/output/fake_mysqldb.py --- plaso-20181219/tests/output/fake_mysqldb.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/output/fake_mysqldb.py 2019-01-31 19:09:43.000000000 +0000 @@ -94,7 +94,7 @@ """ if (not self.query_results or self._result_index < 0 or self._result_index >= len(self.query_results)): - return + return None row = self.query_results[self._result_index] self._result_index += 1 diff -Nru plaso-20181219/tests/output/manager.py plaso-20190131/tests/output/manager.py --- plaso-20181219/tests/output/manager.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/output/manager.py 2019-01-31 19:09:43.000000000 +0000 @@ -23,7 +23,7 @@ Args: event (EventObject): event that contains the event values. """ - pass + pass # pylint: disable=unnecessary-pass class OutputManagerTest(unittest.TestCase): diff -Nru plaso-20181219/tests/parsers/bsm.py plaso-20190131/tests/parsers/bsm.py --- plaso-20181219/tests/parsers/bsm.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/parsers/bsm.py 2019-01-31 19:09:43.000000000 +0000 @@ -22,7 +22,7 @@ """Tests the Parse function on a MacOS BSM file.""" parser = bsm.BSMParser() knowledge_base_values = { - 'operating_system': definitions.OPERATING_SYSTEM_MACOS} + 'operating_system': definitions.OPERATING_SYSTEM_FAMILY_MACOS} storage_writer = self._ParseFile( ['apple.bsm'], parser, knowledge_base_values=knowledge_base_values) @@ -177,7 +177,7 @@ """Tests the Parse function on a "generic" BSM file.""" parser = bsm.BSMParser() knowledge_base_values = { - 'operating_system': definitions.OPERATING_SYSTEM_LINUX} + 'operating_system': definitions.OPERATING_SYSTEM_FAMILY_LINUX} storage_writer = self._ParseFile( ['openbsm.bsm'], parser, knowledge_base_values=knowledge_base_values) diff -Nru plaso-20181219/tests/parsers/dtfabric_parser.py plaso-20190131/tests/parsers/dtfabric_parser.py --- plaso-20181219/tests/parsers/dtfabric_parser.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/parsers/dtfabric_parser.py 2019-01-31 19:09:43.000000000 +0000 @@ -31,6 +31,7 @@ Raises: IOError: for testing. + OSError: for testing. """ raise IOError('Unable to read for testing purposes.') diff -Nru plaso-20181219/tests/parsers/manager.py plaso-20190131/tests/parsers/manager.py --- plaso-20181219/tests/parsers/manager.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/parsers/manager.py 2019-01-31 19:09:43.000000000 +0000 @@ -197,6 +197,7 @@ self.assertIn('winreg', parsers_names) + @shared_test_lib.skipUnlessHasTestFile(['presets.yaml']) def testGetParserAndPluginNames(self): """Tests the GetParserAndPluginNames function.""" TestParserWithPlugins.RegisterPlugin(TestPlugin) @@ -231,6 +232,9 @@ manager.ParsersManager.DeregisterParser(TestParser) # Test with a preset name. + test_path = self._GetTestFilePath(['presets.yaml']) + manager.ParsersManager.ReadPresetsFromFile(test_path) + parser_names = manager.ParsersManager.GetParserAndPluginNames( parser_filter_expression='win_gen') self.assertIn('lnk', parser_names) @@ -298,6 +302,7 @@ manager.ParsersManager.DeregisterParser(TestParserWithPlugins) manager.ParsersManager.DeregisterParser(TestParser) + @shared_test_lib.skipUnlessHasTestFile(['presets.yaml']) def testGetParsers(self): """Tests the GetParsers function.""" TestParserWithPlugins.RegisterPlugin(TestPlugin) @@ -335,6 +340,9 @@ manager.ParsersManager.DeregisterParser(TestParser) # Test with a preset name. + test_path = self._GetTestFilePath(['presets.yaml']) + manager.ParsersManager.ReadPresetsFromFile(test_path) + expected_parser_names = [ 'bencode', 'binary_cookies', 'chrome_cache', 'chrome_preferences', 'czip', 'esedb', 'filestat', 'firefox_cache', 'gdrive_synclog', @@ -374,7 +382,7 @@ TestParserWithPlugins.DeregisterPlugin(TestPlugin) - # TODO: add tests for GetPresetForOperatingSystem. + # TODO: add tests for GetPresetsForOperatingSystem. def testPluginRegistration(self): """Tests the RegisterPlugin and DeregisterPlugin functions.""" diff -Nru plaso-20181219/tests/parsers/olecf_plugins/dtfabric_plugin.py plaso-20190131/tests/parsers/olecf_plugins/dtfabric_plugin.py --- plaso-20181219/tests/parsers/olecf_plugins/dtfabric_plugin.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/parsers/olecf_plugins/dtfabric_plugin.py 2019-01-31 19:09:43.000000000 +0000 @@ -31,6 +31,7 @@ Raises: IOError: for testing. + OSError: for testing. """ raise IOError('Unable to read for testing purposes.') diff -Nru plaso-20181219/tests/parsers/presets.py plaso-20190131/tests/parsers/presets.py --- plaso-20181219/tests/parsers/presets.py 1970-01-01 00:00:00.000000000 +0000 +++ plaso-20190131/tests/parsers/presets.py 2019-01-31 19:09:43.000000000 +0000 @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Tests for parser and parser plugin presets.""" + +from __future__ import unicode_literals + +import unittest + +from plaso.containers import artifacts +from plaso.parsers import presets + +from tests import test_lib as shared_test_lib + + +class ParserPresetTest(shared_test_lib.BaseTestCase): + """Tests for the parser and parser plugin preset.""" + + def testInitialize(self): + """Tests the __init__ function.""" + test_definition = presets.ParserPreset('test', ['parser1', 'parser2']) + self.assertIsNotNone(test_definition) + + +@shared_test_lib.skipUnlessHasTestFile(['presets.yaml']) +class ParserPresetsManagerTest(shared_test_lib.BaseTestCase): + """Tests for the parser and parser plugin presets manager.""" + + # TODO add tests for _ReadPresetDefinitionValues + # TODO add tests for _ReadPresetsFromFileObject + + def testGetNames(self): + """Tests the GetNames function.""" + test_manager = presets.ParserPresetsManager() + + test_path = self._GetTestFilePath(['presets.yaml']) + test_manager.ReadFromFile(test_path) + + test_names = list(test_manager.GetNames()) + self.assertEqual(len(test_names), 7) + + expected_names = sorted([ + 'android', 'linux', 'macos', 'webhist', 'win7', 'win_gen', 'winxp']) + self.assertEqual(test_names, expected_names) + + def testGetPresetByName(self): + """Tests the GetPresetByName function.""" + test_manager = presets.ParserPresetsManager() + + test_path = self._GetTestFilePath(['presets.yaml']) + test_manager.ReadFromFile(test_path) + + test_preset = test_manager.GetPresetByName('linux') + self.assertIsNotNone(test_preset) + self.assertEqual(test_preset.name, 'linux') + + expected_parsers = [ + 'bash_history', + 'bencode', + 'czip/oxml', + 'dockerjson', + 'dpkg', + 'filestat', + 'gdrive_synclog', + 'java_idx', + 'olecf', + 'pls_recall', + 'popularity_contest', + 'selinux', + 'sqlite/google_drive', + 'sqlite/skype', + 'sqlite/zeitgeist', + 'syslog', + 'systemd_journal', + 'utmp', + 'webhist', + 'xchatlog', + 'xchatscrollback', + 'zsh_extended_history'] + + self.assertEqual(test_preset.parsers, expected_parsers) + + test_preset = test_manager.GetPresetByName('bogus') + self.assertIsNone(test_preset) + + def testGetPresetsByOperatingSystem(self): + """Tests the GetPresetsByOperatingSystem function.""" + test_manager = presets.ParserPresetsManager() + + test_path = self._GetTestFilePath(['presets.yaml']) + test_manager.ReadFromFile(test_path) + + operating_system = artifacts.OperatingSystemArtifact(family='MacOS') + + test_presets = test_manager.GetPresetsByOperatingSystem(operating_system) + self.assertEqual(len(test_presets), 1) + self.assertEqual(test_presets[0].name, 'macos') + + expected_parsers = [ + 'asl_log', + 'bash_history', + 'bencode', + 'bsm_log', + 'cups_ipp', + 'czip/oxml', + 'filestat', + 'fseventsd', + 'gdrive_synclog', + 'java_idx', + 'mac_appfirewall_log', + 'mac_keychain', + 'mac_securityd', + 'macwifi', + 'olecf', + 'plist', + 'sqlite/appusage', + 'sqlite/google_drive', + 'sqlite/imessage', + 'sqlite/ls_quarantine', + 'sqlite/mac_document_versions', + 'sqlite/mackeeper_cache', + 'sqlite/skype', + 'syslog', + 'utmpx', + 'webhist', + 'zsh_extended_history'] + + self.assertEqual(test_presets[0].parsers, expected_parsers) + + operating_system = artifacts.OperatingSystemArtifact(family='bogus') + + test_presets = test_manager.GetPresetsByOperatingSystem(operating_system) + self.assertEqual(len(test_presets), 0) + + def testGetPresets(self): + """Tests the GetPresets function.""" + test_manager = presets.ParserPresetsManager() + + test_path = self._GetTestFilePath(['presets.yaml']) + test_manager.ReadFromFile(test_path) + + test_presets = list(test_manager.GetPresets()) + self.assertEqual(len(test_presets), 7) + + # TODO add tests for ReadFromFile + + +if __name__ == '__main__': + unittest.main() diff -Nru plaso-20181219/tests/parsers/text_parser.py plaso-20190131/tests/parsers/text_parser.py --- plaso-20181219/tests/parsers/text_parser.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/parsers/text_parser.py 2019-01-31 19:09:43.000000000 +0000 @@ -45,9 +45,6 @@ text_parser.PyparsingConstants.IPV4_ADDRESS.parseString('1.1.34.2')) with self.assertRaises(pyparsing.ParseException): - text_parser.PyparsingConstants.IPV4_ADDRESS.parseString('1.1.34.258') - - with self.assertRaises(pyparsing.ParseException): text_parser.PyparsingConstants.IPV4_ADDRESS.parseString('a.1.34.258') with self.assertRaises(pyparsing.ParseException): @@ -56,30 +53,6 @@ with self.assertRaises(pyparsing.ParseException): text_parser.PyparsingConstants.IPV4_ADDRESS.parseString('34.258') - with self.assertRaises(pyparsing.ParseException): - text_parser.PyparsingConstants.IPV4_ADDRESS.parseString('10.52.34.258') - - def testConstantOctet(self): - """Tests parsing with the IPV4_OCTET constant.""" - self.assertTrue( - text_parser.PyparsingConstants.IPV4_OCTET.parseString('0')) - - self.assertTrue( - text_parser.PyparsingConstants.IPV4_OCTET.parseString('123')) - - self.assertTrue( - text_parser.PyparsingConstants.IPV4_OCTET.parseString('255')) - - with self.assertRaises(pyparsing.ParseException): - text_parser.PyparsingConstants.IPV4_OCTET.parseString('526') - - with self.assertRaises(pyparsing.ParseException): - text_parser.PyparsingConstants.IPV4_OCTET.parseString('1026') - - with self.assertRaises(pyparsing.ParseException): - text_parser.PyparsingConstants.IPV4_OCTET.parseString( - 'a9', parseAll=True) - class PyparsingSingleLineTextParserTest(unittest.TestCase): """Tests for the single line PyParsing-based text parser.""" diff -Nru plaso-20181219/tests/parsers/winreg_plugins/dtfabric_plugin.py plaso-20190131/tests/parsers/winreg_plugins/dtfabric_plugin.py --- plaso-20181219/tests/parsers/winreg_plugins/dtfabric_plugin.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/parsers/winreg_plugins/dtfabric_plugin.py 2019-01-31 19:09:43.000000000 +0000 @@ -31,6 +31,7 @@ Raises: IOError: for testing. + OSError: for testing. """ raise IOError('Unable to read for testing purposes.') diff -Nru plaso-20181219/tests/parsers/winreg_plugins/test_lib.py plaso-20190131/tests/parsers/winreg_plugins/test_lib.py --- plaso-20181219/tests/parsers/winreg_plugins/test_lib.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/parsers/winreg_plugins/test_lib.py 2019-01-31 19:09:43.000000000 +0000 @@ -73,13 +73,13 @@ """ file_object = file_entry.GetFileObject() if not file_object: - return + return None win_registry_reader = winreg.FileObjectWinRegistryFileReader() registry_file = win_registry_reader.Open(file_object) if not registry_file: file_object.close() - return + return None win_registry = dfwinreg_registry.WinRegistry() key_path_prefix = win_registry.GetRegistryFileMapping(registry_file) diff -Nru plaso-20181219/tests/test_lib.py plaso-20190131/tests/test_lib.py --- plaso-20181219/tests/test_lib.py 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/tests/test_lib.py 2019-01-31 19:09:43.000000000 +0000 @@ -132,6 +132,7 @@ module_name, _, _ = filename.partition('.') import_expression = re.compile(r' import {0:s}\b'.format(module_name)) + # pylint: disable=deprecated-method self.assertRegexpMatches( init_content, import_expression, '{0:s} not imported in {1:s}'.format(module_name, init_path)) diff -Nru plaso-20181219/.travis.yml plaso-20190131/.travis.yml --- plaso-20181219/.travis.yml 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/.travis.yml 2019-01-31 19:09:43.000000000 +0000 @@ -2,11 +2,11 @@ include: - env: TARGET="pylint" os: linux - dist: trusty + dist: xenial sudo: required group: edge language: python - python: 2.7 + python: 3.5 virtualenv: system_site_packages: true - env: TARGET="linux-python27" @@ -76,24 +76,6 @@ os: osx osx_image: xcode9.2 language: generic - - env: TARGET="trusty-python27" - os: linux - dist: trusty - sudo: required - group: edge - language: python - python: 2.7 - virtualenv: - system_site_packages: true - - env: TARGET="trusty-python34" - os: linux - dist: trusty - sudo: required - group: edge - language: python - python: 3.4 - virtualenv: - system_site_packages: true - env: TARGET="jenkins" os: linux dist: trusty @@ -105,4 +87,5 @@ script: - ./config/travis/run_with_timeout.sh 30 ./config/travis/runtests.sh after_success: +- if ! test -f /usr/bin/coverage; then sudo ln -s /usr/bin/python-coverage /usr/bin/coverage; fi - if test ${TARGET} = "linux-python27"; then curl -o codecov.sh -s https://codecov.io/bash && /bin/bash ./codecov.sh; fi diff -Nru plaso-20181219/utils/prep_dist.sh plaso-20190131/utils/prep_dist.sh --- plaso-20181219/utils/prep_dist.sh 2018-12-20 05:42:12.000000000 +0000 +++ plaso-20190131/utils/prep_dist.sh 2019-01-31 19:09:43.000000000 +0000 @@ -4,16 +4,6 @@ EXIT_FAILURE=1; EXIT_SUCCESS=0; -# Remove support for hachoir which is GPLv2 and cannot be distributed -# in binary form. Leave the formatter because it does not link in the -# hachoir code. - -rm -f plaso/parsers/hachoir* - -sed -i"~" -e "s/'hachoir', //" plaso/parsers/presets.py - -sed -i"~" '/hachoir_/d' plaso/dependencies.py - if ! test -d ../l2tdevtools; then echo "Unable to locate l2tdevtools."; @@ -32,7 +22,7 @@ cp "../l2tdevtools/data/licenses/LICENSE.${DEPENDENCY}" config/licenses/ done -rm -f config/licenses/LICENSE.hachoir-* +# Remove debug, test and yet unused dependencies. rm -f config/licenses/LICENSE.guppy rm -f config/licenses/LICENSE.libexe rm -f config/licenses/LICENSE.libwrc