From cc2c63159982a2dc3cf1f79f9b1a9c49f3167022 Mon Sep 17 00:00:00 2001 From: pl77 Date: Fri, 11 Jan 2019 11:19:02 -0500 Subject: [PATCH 1/4] Added .gitignore to keep my IDE cache out of the repo. --- .gitignore | 160 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc1a227 --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Created by .ignore support plugin (hsz.mobi) +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/dictionaries +.idea/**/shelf + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + From 3c24deb92252233558caf989f63c0b54358c2156 Mon Sep 17 00:00:00 2001 From: pl77 Date: Fri, 11 Jan 2019 11:21:00 -0500 Subject: [PATCH 2/4] Added try/except block to 'for loop' to handle nested list exception. Also PEP8ified formatting and updated script version number. --- ctfr.py | 97 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 53 insertions(+), 44 deletions(-) diff --git a/ctfr.py b/ctfr.py index 91629dc..47358bb 100644 --- a/ctfr.py +++ b/ctfr.py @@ -2,80 +2,89 @@ # -*- coding: utf-8 -*- """ ------------------------------------------------------------------------------ - CTFR - 04.03.18.02.10.00 - Sheila A. Berta (UnaPibaGeek) + CTFR - 011.01.19.11.16.00 - Sheila A. Berta (UnaPibaGeek) ------------------------------------------------------------------------------ -""" -## # LIBRARIES # ## +# ## LIBRARIES ## # +""" import re import json import requests +import argparse + +# ## CONTEXT VARIABLES ## # +version = 1.3 -## # CONTEXT VARIABLES # ## -version = 1.2 +# ## MAIN FUNCTIONS ## # -## # MAIN FUNCTIONS # ## def parse_args(): - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--domain', type=str, required=True, help="Target domain.") - parser.add_argument('-o', '--output', type=str, help="Output file.") - return parser.parse_args() + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--domain', type=str, required=True, help="Target domain.") + parser.add_argument('-o', '--output', type=str, help="Output file.") + return parser.parse_args() + def banner(): - global version - b = ''' + global version + b = ''' ____ _____ _____ ____ / ___|_ _| ___| _ \ | | | | | |_ | |_) | | |___ | | | _| | _ < \____| |_| |_| |_| \_\\ - + Version {v} - Hey don't miss AXFR! Made by Sheila A. Berta (UnaPibaGeek) - '''.format(v=version) - print(b) - + '''.format(v=version) + print(b) + + def clear_url(target): - return re.sub('.*www\.','',target,1).split('/')[0].strip() + return re.sub('.*www\.', '', target, 1).split('/')[0].strip() -def save_subdomains(subdomain,output_file): - with open(output_file,"a") as f: - f.write(subdomain + '\n') - f.close() -def main(): - banner() - args = parse_args() +def save_subdomains(subdomain, output_file): + with open(output_file, "a") as f: + f.write(subdomain + '\n') + f.close() - subdomains = [] - target = clear_url(args.domain) - output = args.output - req = requests.get("https://crt.sh/?q=%.{d}&output=json".format(d=target)) +def main(): + banner() + args = parse_args() + + subdomains = [] + target = clear_url(args.domain) + output = args.output - if req.status_code != 200: - print("[X] Information not available!") - exit(1) + req = requests.get("https://crt.sh/?q=%.{d}&output=json".format(d=target)) - json_data = json.loads('[{}]'.format(req.text.replace('}{', '},{'))) + if req.status_code != 200: + print("[X] Information not available!") + exit(1) - for (key,value) in enumerate(json_data): - subdomains.append(value['name_value']) + json_data = json.loads('[{}]'.format(req.text.replace('}{', '},{'))) - - print("\n[!] ---- TARGET: {d} ---- [!] \n".format(d=target)) + # json.loads seems to now return a list, which resulted in nesting and broke the 'for' loop. + # rewrote with a try/except loop to maintain compatibility with previous versions. + try: + for (key, value) in enumerate(json_data): + subdomains.append(value['name_value']) + except TypeError: + json_data = json.loads('{}'.format(req.text.replace('}{', '},{'))) + for (key, value) in enumerate(json_data): + subdomains.append(value['name_value']) + print("\n[!] ---- TARGET: {d} ---- [!] \n".format(d=target)) - subdomains = sorted(set(subdomains)) + subdomains = sorted(set(subdomains)) - for subdomain in subdomains: - print("[-] {s}".format(s=subdomain)) - if output is not None: - save_subdomains(subdomain,output) + for subdomain in subdomains: + print("[-] {s}".format(s=subdomain)) + if output is not None: + save_subdomains(subdomain, output) - print("\n\n[!] Done. Have a nice day! ;).") + print("\n\n[!] Done. Have a nice day! ;).") main() - From 95231ef0ca2c2bfa8d9d3b282ee114a2a9ed3249 Mon Sep 17 00:00:00 2001 From: pl77 Date: Fri, 11 Jan 2019 16:21:34 -0500 Subject: [PATCH 3/4] Option created for grabbing additional domain information from censys.io --- README.md | 12 +++++- ctfr.py | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 133 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 41731eb..f5e3e52 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # CTFR Do you miss AXFR technique? This tool allows to get the subdomains from a HTTP**S** website in a few seconds. -How it works? CTFR does not use neither dictionary attack nor brute-force, it just abuses of Certificate Transparency logs. +How it works? CTFR does not use neither dictionary attack nor brute-force, it just abuses of Certificate Transparency logs. +Additional domains are grabbed from the `Names` field of the record stored at [censys.io](https://censys.io). For more information about CT logs, check www.certificate-transparency.org and [crt.sh](https://crt.sh/). ## Getting Started @@ -19,6 +20,11 @@ $ git clone https://github.com/UnaPibaGeek/ctfr.git $ cd ctfr $ pip3 install -r requirements.txt ``` +If you want to also search for additional domains from censys.io, you also need to install BeautifulSoup4: +```bash +$ pip3 install beautifulsoup4 +``` +This was left out of the requirements.txt file so you weren't forced to install it. ### Running ```bash @@ -33,6 +39,7 @@ Parameters and examples of use. ``` -d --domain [target_domain] (required) -o --output [output_file] (optional) +-a --alt (optional, single switch) ``` ### Examples @@ -42,6 +49,9 @@ $ python3 ctfr.py -d starbucks.com ```bash $ python3 ctfr.py -d facebook.com -o /home/shei/subdomains_fb.txt ``` +```bash +$ python3 ctfr.py -d facebook.com -a -o /home/shei/subdomains_fb.txt +``` ### With Docker I think it's a little bit crazy to use Docker for running such a little python script, but if you want to do it anyway, you can download [this lightweight (97.8MB) Docker image](https://hub.docker.com/r/johnpaulada/ctfr/) made by John Paulada. diff --git a/ctfr.py b/ctfr.py index 47358bb..37fd10c 100644 --- a/ctfr.py +++ b/ctfr.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ ------------------------------------------------------------------------------ - CTFR - 011.01.19.11.16.00 - Sheila A. Berta (UnaPibaGeek) + CTFR - 011.01.19.15.25.00 - Sheila A. Berta (UnaPibaGeek) ------------------------------------------------------------------------------ # ## LIBRARIES ## # @@ -10,10 +10,17 @@ import re import json import requests +from requests import Session import argparse +try: + from bs4 import BeautifulSoup + bs4_present = True +except ImportError: + bs4_present = False + # ## CONTEXT VARIABLES ## # -version = 1.3 +version = 2.0 # ## MAIN FUNCTIONS ## # @@ -21,6 +28,7 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-d', '--domain', type=str, required=True, help="Target domain.") + parser.add_argument('-a', '--alt', action='store_true', help="Alternate domains. Takes longer to complete.") parser.add_argument('-o', '--output', type=str, help="Output file.") return parser.parse_args() @@ -50,13 +58,107 @@ def save_subdomains(subdomain, output_file): f.close() +def grab_alt_detail(alternate_domains, subdomains, cert_ids, existing_domains, new_domains, output): + if alternate_domains and bs4_present: + print("\n[!] ---- ADDITIONAL DOMAIN SEARCH ---- [!] \n") + print(" (this could take quite a while for sites with hundreds of subdomains...)\n") + + cert_queue = set() + sub_queue = dict() + for subdomain in subdomains: + sub_queue[subdomain] = 0 + for cid in cert_ids: + if cid['min_cert_id'] > sub_queue[cid['name_value']]: + sub_queue[cid['name_value']] = cid['min_cert_id'] + for subdomain in sub_queue.keys(): + cert_queue.add(sub_queue[subdomain]) + cert_queue = sorted(cert_queue) + additional_domains = set() + for cert in cert_queue: + with Session() as rsess: + crt_id_url = "https://crt.sh/?id={}".format(cert) + headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", + "referer": crt_id_url} + req = rsess.get(crt_id_url, headers=headers) + rawhtml = req.text + soup = BeautifulSoup(rawhtml, features="html.parser") + body = soup.html.body + crt_links = body.select('a') + cen_url = raw_url = None + for link in crt_links: + try: + crt_tmp = link.attrs['href'] + if crt_tmp.startswith('//censys.io/certificates/'): + cen_url = "https:{}".format(crt_tmp) + raw_url = "{}/raw".format(cen_url) + break + else: + continue + except KeyError: + continue + if cen_url is not None: + # first_ref = "https://censys.io/certificates/{}".format(cert) + headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", + "referer": cen_url} + # first_url = "https://censys.io/certificates/{}/raw".format(cert) + req = rsess.get(raw_url, headers=headers) + rawhtml = req.text + soup = BeautifulSoup(rawhtml, features="html.parser") + json_chunk = soup.select_one('code[class="json"]') + try: + jsontxt = json_chunk.text + except AttributeError: + continue + json_data = json.loads(jsontxt) + try: + for name in json_data['parsed']['names']: + if name in existing_domains: + continue + new_domains.add(name) + if name not in subdomains: + subdomains.append(name) + additional_domains.add(name) + print("[-] {s}".format(s=name)) + if output is not None: + save_subdomains(name, output) + existing_domains.add(name) + except KeyError: + pass + + additional_domains = sorted(additional_domains) + + print("\n[!] ---- ADDITIONAL DOMAINS FOUND: {d} ---- [!]".format(d=len(additional_domains))) + + elif alternate_domains and not bs4_present: + print("BeautifulSoup 4 not installed. Ignoring any -a/--alt switches.") + print("If you wish to parse additional details, try 'pip install beautifulsoup4' and run script again.") + return new_domains + + def main(): banner() args = parse_args() - subdomains = [] + subdomains = list() + cert_ids = list() + target = clear_url(args.domain) output = args.output + alternate_domains = args.alt + existing_domains = set() + new_domains = set() + + if output is not None: + try: + with open(output, "r") as f: + file_domains = f.readlines() + for fd in file_domains: + existing_domains.add(fd.strip()) + print("Found", len(existing_domains), "domains in the", target, "file. Appending any new ones we find.") + except FileNotFoundError: + pass req = requests.get("https://crt.sh/?q=%.{d}&output=json".format(d=target)) @@ -71,20 +173,35 @@ def main(): try: for (key, value) in enumerate(json_data): subdomains.append(value['name_value']) + cert_ids.append(value) except TypeError: json_data = json.loads('{}'.format(req.text.replace('}{', '},{'))) for (key, value) in enumerate(json_data): subdomains.append(value['name_value']) + cert_ids.append(value) + print("\n[!] ---- TARGET: {d} ---- [!] \n".format(d=target)) subdomains = sorted(set(subdomains)) for subdomain in subdomains: + if subdomain in existing_domains: + continue print("[-] {s}".format(s=subdomain)) + new_domains.add(subdomain) if output is not None: save_subdomains(subdomain, output) + existing_domains.add(subdomain) + + print("\n[!] ---- NEW DOMAINS FOUND: {d} ---- [!] \n".format(d=len(new_domains))) + + new_domains = grab_alt_detail(alternate_domains, subdomains, cert_ids, existing_domains, new_domains, output) - print("\n\n[!] Done. Have a nice day! ;).") + print("\n\n[!] ---- TOTAL NEW DOMAINS FOUND: {d} ---- [!]".format(d=len(new_domains))) + print("\n[!] Done. Have a nice day! ;).") -main() +try: + main() +except KeyboardInterrupt: + print("Keyboard Interrupt detected, exiting program. Have a great day!") From 6b22685cc4ed408fa65c345b17b5857bdb3b9dd4 Mon Sep 17 00:00:00 2001 From: pl77 Date: Fri, 11 Jan 2019 16:23:10 -0500 Subject: [PATCH 4/4] Added text files to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index dc1a227..eed0d54 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,5 @@ fabric.properties # Editor-based Rest Client .idea/httpRequests +# Generated text files. +*.txt \ No newline at end of file