#!/usr/bin/env python3
# SPDX-License-Identifier: 0BSD

# Dependencies:
# * Python 3
# * urllib3 (for HTTP requests with Keep Alive support)
# * lxml (for HTML parsing)
# * dnspython (for DNS queries)


import argparse
import functools
import json
import random
import re
import socket
import ssl
import string
import sys
import urllib.parse

import dns.query
import dns.resolver
import dns.zone
import lxml.etree  # noqa: DUO107
import lxml.html  # noqa: DUO107
import urllib3

STANDARD_PHP_FILES = ["index.php", "wp-config.php", "configuration.php",
                      "config.php", "config.inc.php", "settings.php"]


# initializing global state variables
duplicate_preventer = []
mainpage_cache = {}
dns_cache = {}

# This disables warnings about the lack of certificate verification.
# Usually this is a bad idea, but for this tool we want to find HTTPS leaks
# even if they are shipped with invalid certificates.
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)  # noqa: DUO131


def DEFAULT(f):
    f._is_default_test = True
    return f


def INFO(f):
    f._is_info_test = True
    return f


def HOSTNAME(f):
    f._is_hostname_test = True
    return f


def pdebug(msg):
    if args.debug:
        print(f"[[debug]] {msg}")


def pout(cause, url, misc="", noisymsg=False):
    if noisymsg and not args.noisy:
        return
    # we're storing URL without protocol/www-prefix and cause to avoid
    # duplicates on same host
    dup_check = cause + "__" + re.sub(r"http[s]?://(www\.)?", "", url) + misc
    if dup_check not in duplicate_preventer:
        duplicate_preventer.append(dup_check)
        if args.json:
            json_out.append({"cause": cause, "url": url, "misc": misc})
        elif misc:
            print(f"[{cause}] {url} {misc}")
        else:
            print(f"[{cause}] {url}")


def randstring():
    return "".join(random.SystemRandom().choice(string.ascii_lowercase) for i in range(8))


# a random string that stays the same during one execution of snallygaster,
# makes duplicate detection easier.
@functools.cache
def staticrandstring():
    return "".join(random.SystemRandom().choice(string.ascii_lowercase) for i in range(8))


def escape(msg):
    return repr(msg)[1:-1]


def fetcher(fullurl, binary=False, getredir=False, geterrpage=False):
    data = ""
    redir = ""
    try:
        r = pool.request("GET", fullurl, retries=False, redirect=False)
        if getredir:
            headers = {k.lower(): v for k, v in r.headers.items()}
            redir = headers.get("location", "")
        elif (r.status != 200 and not geterrpage):
            data = ""
        elif binary:
            data = r.data
        else:
            data = r.data.decode("ascii", errors="ignore")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        data = ""
    if getredir:
        return data, redir
    return data


def fetchpartial(fullurl, size, returnsize=False, binary=False):
    try:
        r = pool.request("GET", fullurl, retries=False, redirect=False,
                         preload_content=False)
        if r.status == 200:
            ret = r.read(size)
            if binary:
                rv = ret
            else:
                rv = ret.decode("ascii", errors="ignore")
            if returnsize:
                size = r.headers.getlist("content-length")
                if size == []:
                    size = 0
                else:
                    size = int(size[0])
                r.release_conn()
                return (rv, size)
            r.release_conn()
            return rv
        r.release_conn()
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass
    if returnsize:
        return ("", 0)
    return ""


@functools.cache
def check404(url):
    rndurl = url + "/" + staticrandstring() + ".htm"

    pdebug(f"Checking 404 page state of {rndurl}")
    try:
        r = pool.request("GET", rndurl, retries=False, redirect=False)
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        return False
    what404 = {}
    what404["rndurl"] = rndurl
    what404["state"] = r.status != 200
    what404["content"] = r.data.decode("ascii", errors="ignore")
    if any(m in what404["content"] for m in ["<?php", "<?="]):
        # we're getting php code for 404 pages
        pout("errorpage_with_php", rndurl)
        what404["php"] = True
    else:
        what404["php"] = False
    if "INSERT INTO" in what404["content"]:
        # we're getting sql code for 404 pages
        pout("errorpage_with_sql", rndurl)
        what404["sql"] = True
    else:
        what404["sql"] = False

    return what404


def getmainpage(url):
    if url in mainpage_cache:
        return mainpage_cache[url]["content"]

    try:
        r = pool.request("GET", url, retries=False, redirect=True)
        if r.status == 200:
            data = r.data.decode("ascii", errors="ignore")
        else:
            data = ""
        headers = {k.lower(): v for k, v in r.headers.items()}
        redir = headers.get("location", "")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        mainpage_cache[url] = {}
        mainpage_cache[url]["httpcode"] = 0
        mainpage_cache[url]["location"] = ""
        mainpage_cache[url]["content"] = ""
        return ""

    mainpage_cache[url] = {}
    mainpage_cache[url]["httpcode"] = r.status
    mainpage_cache[url]["location"] = redir
    mainpage_cache[url]["content"] = data
    return data


def dnscache(qhost):
    if qhost in dns_cache:
        return dns_cache[qhost]
    try:
        socket.inet_aton(qhost)
        dns_cache[qhost] = qhost
        return qhost
    except OSError:
        pass
    try:
        if "resolve" in dir(dns.resolver):
            dnsanswer = dns.resolver.resolve(qhost, "A")
        else:  # dnspython before 2.0
            dnsanswer = dns.resolver.query(qhost, "A")
    except (dns.exception.DNSException, ConnectionResetError):
        dns_cache[qhost] = None
        return None
    dns_cache[qhost] = dnsanswer.rrset
    return dnsanswer.rrset


@DEFAULT
def test_lfm_php(url):
    r = fetcher(url + "/lfm.php")
    if "Lazy File Manager" in r:
        pout("lfm_php", url + "/lfm.php")


@DEFAULT
def test_idea(url):
    r = fetcher(url + "/.idea/WebServers.xml")
    if 'name="WebServers"' in r:
        pout("idea", url + "/.idea/WebServers.xml")


@DEFAULT
def test_symfony_databases_yml(url):
    r = fetcher(url + "/config/databases.yml")
    if "class:" in r and "param:" in r:
        pout("symfony_databases_yml", url + "/config/databases.yml")


@DEFAULT
def test_rails_database_yml(url):
    r = fetcher(url + "/config/database.yml")
    if "adapter:" in r and "database:" in r:
        pout("rails_database_yml", url + "/config/database.yml")


@DEFAULT
def test_git_dir(url):
    r = fetcher(url + "/.git/config")
    if "[core]" in r:
        pout("git_dir", url + "/.git/config")


@DEFAULT
def test_svn_dir(url):
    r = fetcher(url + "/.svn/entries")
    try:
        if (str)((int)(r)) + "\n" == r:
            pout("svn_dir", url + "/.svn/entries")
    except ValueError:
        pass


@DEFAULT
def test_apache_server_status(url):
    r = fetcher(url + "/server-status")
    if "Apache Status" in r:
        pout("apache_server_status", url + "/server-status")


@DEFAULT
def test_apache_server_info(url):
    r = fetcher(url + "/server-info")
    if "Apache Server Information</h1>" in r:
        pout("apache_server_info", url + "/server-info")


@DEFAULT
def test_coredump(url):
    r = fetchpartial(url + "/core", 20, binary=True)
    if r and r[0:4] == b"\x7fELF":
        pout("coredump", url + "/core")


@DEFAULT
def test_sftp_config(url):
    r = fetcher(url + "/sftp-config.json")
    if '"type":' in r and "ftp" in r and '"save_before_upload"' in r:
        pout("sftp_config", url + "/sftp-config.json")


@DEFAULT
def test_wsftp_ini(url):
    for fn in ["WS_FTP.ini", "ws_ftp.ini", "WS_FTP.INI"]:
        r = fetcher(url + "/" + fn)
        if "[_config_]" in r:
            pout("wsftp_ini", url + "/" + fn)


@DEFAULT
def test_filezilla_xml(url):
    for fn in ["filezilla.xml", "sitemanager.xml", "FileZilla.xml"]:
        r = fetcher(url + "/" + fn)
        if "<FileZilla" in r:
            pout("filezilla_xml", url + "/" + fn)


@DEFAULT
def test_winscp_ini(url):
    for fn in ["winscp.ini", "WinSCP.ini"]:
        r = fetcher(url + "/" + fn)
        if "[Configuration]" in r:
            pout("winscp_ini", f"{url}/{fn}")


@DEFAULT
def test_ds_store(url):
    r = fetcher(url + "/.DS_Store", binary=True)
    if r[0:8] == b"\x00\x00\x00\x01Bud1":
        pout("ds_store", url + "/.DS_Store")


@DEFAULT
def test_php_cs_fixer(url):
    r = fetcher(url + "/.php_cs.cache")
    if r[0:8] == '{"php":"':
        pout("php_cs_cache", url + "/.php_cs.cache")
    r = fetcher(url + "/.php-cs-fixer.cache")
    if r[0:8] == '{"php":"':
        pout("php_cs_cache", url + "/.php-cs-fixer.cache")


@DEFAULT
def test_backupfiles(url):
    what404 = check404(url)
    if not what404:
        return
    if what404["php"] and not what404["state"]:
        # we don't get proper 404 replies and the default page contains PHP
        # code, check doesn't make sense.
        return
    for f in STANDARD_PHP_FILES:
        for ps in ["_FILE_.bak", "_FILE_~", "._FILE_.swp", "%23_FILE_%23", "_FILE_.save",
                   "_FILE_.orig"]:
            furl = url + "/" + ps.replace("_FILE_", f)
            r = fetcher(furl)
            if any(m in r for m in ["<?php", "<?="]):
                pout("backupfiles", furl)


@DEFAULT
def test_backup_archive(url):
    apexhost = re.sub("^www.", "", re.sub("(.*//|/.*)", "", url))
    wwwhost = "www." + apexhost

    for fn in ["backup", "www", "wwwdata", "db", "htdocs", apexhost, wwwhost]:
        r = fetchpartial(url + "/" + fn + ".zip", 2, binary=True)
        if r == b"PK":
            pout("backup_archive", url + "/" + fn + ".zip")
        r = fetchpartial(url + "/" + fn + ".tar.gz", 3, binary=True)
        if r == b"\x1f\x8b\x08":
            pout("backup_archive", url + "/" + fn + ".tar.gz")
        r = fetchpartial(url + "/" + fn + ".tar.bz2", 3, binary=True)
        if r in [b"BZh", b"BZ0"]:
            pout("backup_archive", url + "/" + fn + ".tar.bz2")
        r = fetchpartial(url + "/" + fn + ".tar.xz", 6, binary=True)
        if r == b"\xFD7zXZ\x00":
            pout("backup_archive", url + "/" + fn + ".tar.xz")


@DEFAULT
def test_deadjoe(url):
    r = fetcher(url + "/DEADJOE")
    if "in JOE when it aborted" in r:
        pout("deadjoe", url + "/DEADJOE")


@DEFAULT
def test_sql_dump(url):
    what404 = check404(url)
    if not what404:
        return
    for f in ["dump.sql", "database.sql", "1.sql", "backup.sql", "data.sql",
              "db_backup.sql", "dbdump.sql", "db.sql", "localhost.sql",
              "mysql.sql", "site.sql", "sql.sql", "temp.sql", "users.sql",
              "translate.sql", "mysqldump.sql"]:
        if not what404["sql"] or what404["state"]:
            # if we don't get proper 404 replies and the default page contains
            # SQL code, check doesn't make sense.
            r = fetchpartial(url + "/" + f, 4000)
            if any(m in r for m in ["INSERT INTO"]):
                pout("sql_dump", url + "/" + f)
        r = fetchpartial(url + "/" + f + ".gz", 3, binary=True)
        if r == b"\x1f\x8b\x08":
            pout("sql_dump_gz", url + "/" + f + ".gz")
        r = fetchpartial(url + "/" + f + ".bz2", 3, binary=True)
        if r in [b"BZh", b"BZ0"]:
            pout("sql_dump_bz", url + "/" + f + ".bz2")
        r = fetchpartial(url + "/" + f + ".xz", 6, binary=True)
        if r == b"\xFD7zXZ\x00":
            pout("sql_dump_xz", url + "/" + f + ".xz")


@DEFAULT
def test_bitcoin_wallet(url):
    r = fetchpartial(url + "/wallet.dat", 16, binary=True)
    if r and len(r) == 16 and r[12:] == b"b1\x05\x00":
        pout("bitcoin_wallet", url + "/wallet.dat")


@DEFAULT
def test_drupal_backup_migrate(url):
    r = fetcher(url + "/sites/default/private/files/backup_migrate/scheduled/test.txt")
    if "this file should not be publicly accessible" in r:
        pout("drupal_backup_migrate",
             url + "/sites/default/private/files/backup_migrate/scheduled/test.txt")


@DEFAULT
def test_magento_config(url):
    r = fetcher(url + "/app/etc/local.xml")
    if "<config" in r and "Mage" in r:
        pout("magento_config", url + "/app/etc/local.xml")


@DEFAULT
def test_xaa(url):
    r, s = fetchpartial(url + "/xaa", 4000, returnsize=True, binary=True)
    if r is None:
        return
    # we need some heuristics here to avoid false positives.
    # If output is larger than 10 MB it's likely a true positive.
    if s > 10000000:
        pout("xaa", url + "/xaa")
        return
    # Check for signs of common compression formats (gz, bzip2, xz, zstd, zip).
    if (r[0:3] == b"\x1f\x8b\x08"
            or r[0:3] in [b"BZh", b"BZ0"]
            or r[0:6] == b"\xFD7zXZ\x00"
            or r[0:4] in [b"\x28\xB5\x2F\xFD", b"PK\x03\x04"]):
        pout("xaa", url + "/xaa")


@DEFAULT
def test_optionsbleed(url):
    try:
        r = pool.request("OPTIONS", url, retries=False, redirect=False)
    except (ConnectionRefusedError, urllib3.exceptions.HTTPError,
            UnicodeError):
        return
    try:
        allow = str(r.headers["Allow"])
    except KeyError:
        return
    if allow == "":
        pout("options_empty", url, noisymsg=True)
        return

    # catch some obvious cases first
    if (",," in allow) or (allow[0] == ",") or (allow[-1] == ","):
        pout("optionsbleed", url, escape(allow))
        return

    z = [x.strip() for x in allow.split(",")]
    if re.match("^[a-zA-Z-]*$", "".join(z)):
        if len(z) > len(set(z)):
            pout("options_duplicates", url, escape(allow), noisymsg=True)
        return
    if re.match("^[a-zA-Z- ]*$", allow):
        pout("options_spaces", url, escape(allow))
        return
    pout("optionsbleed", url, escape(allow))


@DEFAULT
def test_privatekey(url):
    hostkey = re.sub("^www.", "", re.sub("(.*//|/.*)", "", url)) + ".key"
    wwwkey = "www." + hostkey
    for fn in ["server.key", "privatekey.key", "myserver.key", "key.pem",
               hostkey, wwwkey]:
        r = fetcher(url + "/" + fn)
        if "BEGIN PRIVATE KEY" in r:
            pout("privatekey_pkcs8", f"{url}/{fn}")
        if "BEGIN RSA PRIVATE KEY" in r:
            pout("privatekey_rsa", f"{url}/{fn}")
        if "BEGIN DSA PRIVATE KEY" in r:
            pout("privatekey_dsa", f"{url}/{fn}")
        if "BEGIN EC PRIVATE KEY" in r:
            pout("privatekey_ec", f"{url}/{fn}")


@DEFAULT
def test_sshkey(url):
    for fn in ["id_rsa", "id_dsa", ".ssh/id_rsa", ".ssh/id_dsa"]:
        r = fetcher(url + "/" + fn)
        if "BEGIN" in r and "PRIVATE KEY" in r:
            pout("sshkey", f"{url}/{fn}")


@DEFAULT
def test_dotenv(url):
    r = fetcher(url + "/.env")
    if "APP_ENV=" in r or "DB_PASSWORD=" in r:
        pout("dotenv", url + "/.env")


@DEFAULT
def test_invalidsrc(url):
    r = getmainpage(url)
    try:
        p = lxml.html.document_fromstring(r.encode())
    except lxml.etree.ParserError:
        return

    g = p.xpath("//*[@src]/@src")
    srcs = sorted(set(g))

    checkeddomains = []
    for src in srcs:
        try:
            realurl = urllib.parse.urljoin(url, src)
            domain = urllib.parse.urlparse(realurl).hostname
            protocol = urllib.parse.urlparse(realurl).scheme
        except ValueError:
            pout("invalidsrc_brokenurl", escape(url), src)
            continue
        if domain is None:
            continue
        if protocol not in ["https", "http"]:
            continue

        # We avoid double-checking multiple requests to the same host.
        # This is a compromise between speed and in-depth scanning.
        if domain in checkeddomains:
            continue
        checkeddomains.append(domain)
        pdebug(f"Checking url {realurl}")

        if dnscache(domain) is None:
            pout("invalidsrc_dns", url, escape(src))
            continue

        try:
            r = pool.request("GET", realurl, retries=False, redirect=False)
            if r.status >= 400:
                pout("invalidsrc_http", url, escape(src))
        except UnicodeEncodeError:
            pass
        except (ConnectionRefusedError, ConnectionResetError,
                urllib3.exceptions.HTTPError):
            pout("invalidsrc_http_connfail", url, escape(src))


@DEFAULT
def test_ilias_defaultpw(url):
    getmainpage(url)
    if (url + "/ilias.php" in mainpage_cache[url]["location"]
            or (url + "/login.php" in mainpage_cache[url]["location"]
                and "powered by ILIAS" in fetcher(mainpage_cache[url]["location"]))):
        # we're confident we found an ILIAS installation
        pdebug("Ilias found")
        try:
            login = pool.request("POST", url + "/ilias.php?cmd=post&baseClass=ilStartUpGUI",
                                 fields={"username": "root",
                                         "password": "homer",
                                         "cmd[doStandardAuthentication]": "Login"},
                                 headers={"Cookie": "iltest=;PHPSESSID=" + randstring()})
            data = login.data.decode("ascii", errors="ignore")
            if (('class="ilFailureMessage"' not in data)
                    and ('name="il_message_focus"' not in data)
                    and (('class="ilBlockContent"' in data)
                         or ('class="ilAdminRow"' in data))):
                pout("ilias_defaultpw", url, "root/homer")
        except (ConnectionRefusedError, ConnectionResetError,
                urllib3.exceptions.HTTPError):
            pass


@DEFAULT
def test_cgiecho(url):
    for pre in [url + "/cgi-bin/cgiecho", url + "/cgi-sys/cgiecho"]:
        try:
            r = pool.request("GET", pre + "/" + randstring())
            if r.status == 500 and "<P><EM>cgiemail" in r.data.decode("ascii", errors="ignore"):
                pout("cgiecho", pre)
        except (ConnectionRefusedError, ConnectionResetError,
                urllib3.exceptions.HTTPError):
            pass


@DEFAULT
def test_phpunit_eval(url):
    try:
        r = pool.request("POST", url + "/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php",
                         body='<?php echo(substr_replace("hello", "12", 2, 2));')
        data = r.data.decode("ascii", errors="ignore")
        if data == "he12o":
            pout("phpunit_eval", url + "/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php")
    except (ConnectionRefusedError, ConnectionResetError,
            urllib3.exceptions.HTTPError):
        pass


@DEFAULT
def test_acmereflect(url):
    reflect = randstring()
    try:
        r = pool.request("GET", url + "/.well-known/acme-challenge/<html>" + reflect,
                         retries=False, redirect=False)
        if not r.data.decode("ascii", errors="ignore").startswith("<html>" + reflect):
            return
        headers = {k.lower(): v for k, v in r.headers.items()}
        if ("content-type" in headers) and headers["content-type"].startswith("text/plain"):
            return
        pout("acmereflect", url + "/.well-known/acme-challenge/reflect")

    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        return


@DEFAULT
def test_drupaldb(url):
    r = fetchpartial(url + "/sites/default/files/.ht.sqlite", 20, binary=True)
    if r and r[0:13] == b"SQLite format":
        pout("drupaldb", url + "/sites/default/files/.ht.sqlite")


@DEFAULT
def test_phpwarnings(url):
    try:
        r = pool.request("GET", url, headers={"Cookie": "PHPSESSID=in_vålíd"})
        if ("The session id is too long or contains illegal characters"
                in r.data.decode("ascii", errors="ignore")):
            pout("phpwarnings", url)
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@DEFAULT
def test_adminer(url):
    r = fetcher(url + "/adminer.php")
    if "adminer.org" in r:
        pout("adminer", url + "/adminer.php")


@DEFAULT
def test_elmah(url):
    r = fetcher(url + "/elmah.axd")
    if "Error Log for" in r:
        pout("elmah", url + "/elmah.axd")
    r = fetcher(url + "/scripts/elmah.axd")
    if "Error Log for" in r:
        pout("elmah", url + "/scripts/elmah.axd")


@DEFAULT
def test_citrix_rce(url):
    try:
        r = pool.request("GET", url + "/vpn/../vpns/portal/tips.html",
                         retries=False, redirect=False,
                         headers={"NSC_USER": "x", "NSC_NONCE": "x"})
        if '<div id="nexttip">' in r.data.decode("ascii", errors="ignore"):
            pout("citrix_rce", url + "/vpn/../vpns/portal/tips.html")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@DEFAULT
def test_installer(url):
    r = getmainpage(url)
    if (mainpage_cache[url]["location"].endswith("wp-admin/setup-config.php")
       or 'href="wp-admin/css/install.css"' in r):
        pout("installer_wordpress", url)
    elif mainpage_cache[url]["location"].endswith("installation/index.php"):
        pout("installer_joomla", url)
    elif mainpage_cache[url]["location"].endswith("typo3/install.php"):
        pout("installer_typo3", url)
    elif mainpage_cache[url]["location"].endswith("install.php"):
        pout("installer_drupal", url)
    elif mainpage_cache[url]["location"].endswith("serendipity_admin.php"):
        pout("installer_s9y", url)
    elif "LocalSettings.php not found" in r:
        pout("installer_mediawiki", url)
    elif "8 easy steps and will take around 5 minutes" in r:
        pout("installer_matomo", url)
    elif "Create an <strong>admin account</strong>" in r:
        pout("installer_nextcloud", url)


@DEFAULT
def test_wpsubdir(url):
    r, redir = fetcher(url + "/wordpress/", getredir=True)
    if (redir.endswith("wp-admin/setup-config.php")
       or 'href="wp-admin/css/install.css"' in r):
        pout("wpsubdir", url + "/wordpress/")


@DEFAULT
def test_telescope(url):
    r = fetcher(url + "/telescope", geterrpage=True)
    if "<strong>Laravel</strong> Telescope" in r:
        pout("telescope", url + "/telescope")
    elif "The Telescope assets are not published" in r:
        pout("telescope_inactive", url + "/telescope")


@DEFAULT
def test_vb_test(url):
    r = fetcher(url + "/vb_test.php")
    if "<title>vBulletin Test Script" in r:
        pout("vb_test", url + "/vb_test.php")


@DEFAULT
def test_headerinject(url):
    rnd = randstring()
    try:
        r = pool.request("GET", f"{url}/%0D%0A{rnd}:1", retries=False, redirect=False)
        if rnd in r.headers:
            pout("headerinject", f"{url}/%0D%0A{rnd}:1")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@DEFAULT
def test_wpdebug(url):
    r = fetcher(url + "/wp-content/debug.log")
    if re.match(r"^\[\d\d-\w\w\w-\d\d\d\d ", r):
        pout("wpdebug", url + "/wp-content/debug.log")


@DEFAULT
def test_thumbsdb(url):
    r = fetcher(url + "/Thumbs.db", binary=True)
    if r and r[0:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1":
        pout("thumbsdb", url + "/Thumbs.db")


@DEFAULT
def test_duplicator(url):
    for fn in ["installer.php", "installer-backup.php"]:
        r = fetcher(f"{url}/{fn}")
        if "/dup-installer/main.installer.php" in r:
            pout("duplicator", f"{url}/{fn}")
    for fn in ["backups-dup-pro", "backups-dup-lite"]:
        r = fetcher(f"{url}/wp-content/{fn}/")
        if ">Index of /" in r:
            pout("duplicator_dirlisting", f"{url}/wp-content/{fn}/")


@DEFAULT
def test_desktopini(url):
    r = fetcher(url + "/desktop.ini")
    if "[\x00.\x00S\x00h\x00e\x00l\x00l\x00C\x00l\x00a\x00s\x00s" in r:
        pout("desktopini", url + "/desktop.ini")
    r = fetcher(url + "/Desktop.ini")
    if "[\x00.\x00S\x00h\x00e\x00l\x00l\x00C\x00l\x00a\x00s\x00s" in r:
        pout("desktopini", url + "/Desktop.ini")


@DEFAULT
def test_postdebug(url):
    try:
        r = pool.request("POST", url, retries=False, redirect=False)
        data = r.data.decode("ascii", errors="ignore")
        if (
            "The POST method is not supported for" in data
            and "Symfony\\Component\\HttpKernel\\Exception" in data
        ):
            pout("postdebug_laravel", url + " POST")
        elif "Symfony Exception" in data and '<div class="exception-' in data:
            pout("postdebug_symfony", url + " POST")
        elif "<title>Action Controller: Exception caught" in data:
            pout("postdebug_rails", url + " POST")
    except (urllib3.exceptions.HTTPError, UnicodeError, ConnectionRefusedError):
        pass


@DEFAULT
def test_djangodebug(url):
    what404 = check404(url)
    if what404 and "you have <code>DEBUG = True</code>" in what404["content"]:
        pout("djangodebug", what404["rndurl"])


@DEFAULT
def test_symfonydebug(url):
    what404 = check404(url)
    if what404 and "<title>No route found" in what404["content"] and \
       "Symfony Exception" in what404["content"]:
        pout("symfonydebug", what404["rndurl"])


@DEFAULT
@HOSTNAME
def test_axfr(qhost):
    try:
        if "resolve" in dir(dns.resolver):
            ns = dns.resolver.resolve(qhost, "NS")
        else:  # dnspython before 2.0
            ns = dns.resolver.query(qhost, "NS")
    except (dns.exception.DNSException, dns.exception.Timeout,
            ConnectionResetError, ConnectionRefusedError,
            EOFError, socket.gaierror, TimeoutError, OSError):
        return
    for rr in ns.rrset:
        r = str(rr)
        ipv4 = []
        ipv6 = []
        try:
            if "resolve" in dir(dns.resolver):
                ipv4 = dns.resolver.resolve(r, "a").rrset
                ipv6 = dns.resolver.resolve(r, "aaaa").rrset
            else:  # dnspython before 2.0
                ipv4 = dns.resolver.query(r, "a").rrset
                ipv6 = dns.resolver.query(r, "aaaa").rrset
        except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN,
                dns.resolver.LifetimeTimeout):
            pass
        ips = []
        for ip in ipv4:
            ips.append(str(ip))
        for ip in ipv6:
            ips.append(str(ip))
        for ip in ips:
            try:
                axfr = dns.zone.from_xfr(dns.query.xfr(ip, qhost))
                if axfr:
                    pout("axfr", qhost, r)
            except (dns.exception.DNSException, dns.exception.Timeout,
                    ConnectionResetError, ConnectionRefusedError,
                    EOFError, socket.gaierror, TimeoutError, OSError):
                pass


@DEFAULT
@HOSTNAME
def test_openmonit(qhost):
    url = f"http://{qhost}:2812/"
    headers = urllib3.util.make_headers(basic_auth="admin:monit")
    try:
        r = pool.request("GET", url, headers=headers)
        if "<title>Monit:" in r.data.decode("ascii", errors="ignore"):
            pout("openmonit", url)
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@DEFAULT
@HOSTNAME
def test_openelasticsearch(qhost):
    headers = urllib3.util.make_headers(basic_auth="admin:admin")
    try:
        r = pool.request("GET", f"http://{qhost}:9200", headers=headers)
        if '"cluster_name" :' in r.data.decode("ascii", errors="ignore"):
            pout("openelasticsearch", f"http://{qhost}:9200")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass
    try:
        r = pool.request("GET", f"https://{qhost}:9200", headers=headers)
        if '"cluster_name" :' in r.data.decode("ascii", errors="ignore"):
            pout("openelasticsearch", f"https://{qhost}:9200")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@INFO
def test_drupal(url):
    r = fetcher(url + "/CHANGELOG.txt")
    try:
        if r != "":
            result = re.findall("Drupal [0-9.]*", r)
            version = result[0][7:]
            pout("drupal", url, version)
        r = fetcher(url + "/core/CHANGELOG.txt")
        if r != "":
            result = re.findall("Drupal [0-9.]*", r)
            version = result[0][7:]
            pout("drupal", url, version)
    except IndexError:
        pass


@INFO
def test_wordpress(url):
    r = getmainpage(url)
    try:
        p = lxml.html.document_fromstring(r.encode())
    except lxml.etree.ParserError:
        return
    g = p.xpath("//meta[@name='generator']/@content")
    if g and g[0].startswith("WordPress "):
        version = g[0].split(" ", 1)[1]
        if set(version).issubset("0123456789."):
            pout("wordpress", url, version)


@INFO
def test_mailman(url):
    murl = f"{url}/mailman/listinfo"
    r = fetcher(murl)
    if "Delivered by Mailman" in r:
        ver = re.findall("version ([0-9.]+)", r)
        if len(ver) > 0:
            ver = ver[0]
        else:
            ver = "unknown"
        if "There currently are no publicly-advertised" in r:
            pout("mailman_unused", f"{murl} {ver}")
        else:
            pout("mailman", f"{murl} {ver}")


@INFO
def test_django_staticfiles_json(url):
    furl = url + "/static/staticfiles.json"
    data = fetcher(furl)
    try:
        parsed = json.loads(data)
    except json.JSONDecodeError:
        pass
    else:
        if isinstance(parsed, dict) and "paths" in parsed:
            pout("django_staticfiles_json", furl)


@INFO
def test_composer(url):
    for c in ["composer.json", "composer.lock"]:
        furl = url + "/" + c
        r = fetcher(furl)
        if '"require":' in r or '"packages":' in r:
            pout("composer", furl)


@INFO
def test_phpinfo(url):
    for fn in ["phpinfo.php", "info.php", "i.php", "test.php"]:
        r = fetcher(url + "/" + fn)
        if "phpinfo()" in r:
            pout("phpinfo", url + "/" + fn)


def new_excepthook(etype, value, traceback):
    if etype is KeyboardInterrupt:
        pdebug("Interrupted by user...")
        sys.exit(1)

    print("Oh oh... an unhandled exception has happened. This shouldn't be.")
    print("Please report a bug and include all output.")
    print()
    print("called with")
    print(" ".join(sys.argv))
    print()
    sys.__excepthook__(etype, value, traceback)


sys.excepthook = new_excepthook


parser = argparse.ArgumentParser()
parser.add_argument("hosts", nargs="+", help="hostname to scan")
parser.add_argument("-t", "--tests", help="Comma-separated tests to run.")
parser.add_argument("--useragent", help="User agent to send")
parser.add_argument("--nowww", action="store_true",
                    help="Skip scanning www.[host]")
parser.add_argument("--nohttp", action="store_true",
                    help="Don't scan http")
parser.add_argument("--nohttps", action="store_true",
                    help="Don't scan https")
parser.add_argument("-i", "--info", action="store_true",
                    help="Enable all info tests (no bugs/security vulnerabilities)")
parser.add_argument("-n", "--noisy", action="store_true",
                    help="Show noisy messages that indicate boring bugs, but no security issue")
parser.add_argument("-p", "--path", default="", action="store", type=str,
                    help="Base path on server (scans root dir by default)")
parser.add_argument("-j", "--json", action="store_true",
                    help="Produce JSON output")
parser.add_argument("-d", "--debug", action="store_true",
                    help="Show detailed debugging info")
args = parser.parse_args()

# Initializing global pool manager
user_agent = {"user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0"}
if args.useragent:
    user_agent = {"user-agent": args.useragent}
urllib3_major = int(urllib3.__version__.split(".", maxsplit=1)[0])
if urllib3_major >= 2:
    pool = urllib3.PoolManager(10, headers=user_agent, cert_reqs="CERT_NONE",  # noqa: DUO132
                               retries=False, timeout=2, ssl_minimum_version=ssl.TLSVersion.SSLv3)
else:
    pool = urllib3.PoolManager(10, headers=user_agent, cert_reqs="CERT_NONE",  # noqa: DUO132
                               retries=False, timeout=2)

# This is necessary for directory traversal attacks like citrix_cve
urllib3.util.url.NORMALIZABLE_SCHEMES = ()

if args.tests is None:
    tests = [g for f, g in locals().items() if hasattr(g, "_is_default_test")]
else:
    tests = []
    try:
        for x in args.tests.split(","):
            tests.append(locals()["test_" + x])
    except KeyError:
        print(f"Test {x} does not exist")
        sys.exit(1)

if args.info:
    tests += [g for f, g in locals().items() if hasattr(g, "_is_info_test")]

path = args.path.rstrip("/")
if path != "" and path[0] != "/":
    path = "/" + path
if path != "":
    pdebug(f"Path: {path}")

hosts = list(args.hosts)
if not args.nowww:
    for h in args.hosts:
        hosts.append("www." + h)

for i, h in enumerate(hosts):
    if h.startswith(("http://", "https://")):
        print("ERROR: Please run snallygaster with a hostname, not a URL.")
        sys.exit(1)
    try:
        hosts[i] = h.encode("idna").decode("ascii")
    except UnicodeError:
        print("ERROR: Invalid hostname")
        sys.exit(1)
    if h != hosts[i]:
        pdebug(f"Converted {h} to {hosts[i]}")

pdebug(f"All hosts: {','.join(hosts)}")


json_out = []
for host in hosts:
    pdebug(f"Scanning {host}")
    for test in tests:
        pdebug(f"Running {test.__name__} test")
        if hasattr(test, "_is_hostname_test"):
            test(host)
        else:
            if not args.nohttp:
                test("http://" + host + path)
            if not args.nohttps:
                test("https://" + host + path)

# clear all sockets
pool.clear()

if args.json:
    print(json.dumps(json_out))
