meerkat/meerkat.py

import json
import asyncio
import socket
from aiohttp import ClientSession
from concurrent.futures import ThreadPoolExecutor

def reverse_dns_lookup_sync(ip):
    """
    Perform a reverse DNS lookup (synchronous, for multithreading).
    """
    try:
        print(f"Performing synchronous DNS lookup for IP: {ip}")
        hostname = socket.gethostbyaddr(ip)[0]
        print(f"DNS lookup successful for IP {ip}: {hostname}")
        return hostname
    except socket.herror:
        print(f"DNS lookup failed for IP: {ip}")
        return None

async def reverse_dns_lookup(ip, executor):
    """
    Perform a reverse DNS lookup using a thread pool.
    """
    print(f"Starting DNS lookup for IP: {ip}")
    loop = asyncio.get_event_loop()
    hostname = await loop.run_in_executor(executor, reverse_dns_lookup_sync, ip)
    return hostname

async def fetch_website_info(session, ip, hostname):
    """
    Fetch website title and description from the IP.
    """
    print(f"Fetching website info for IP: {ip} (Hostname: {hostname or 'Unknown'})")
    endpoints = [f"http://{ip}", f"http://{hostname}" if hostname else None]
    for url in filter(None, endpoints):
        try:
            print(f"Trying URL: {url}")
            async with session.get(url, timeout=10) as response:
                if response.status == 200:
                    print(f"Successfully fetched data from {url}")
                    html = await response.text()
                    title = (
                        html.split("<title>")[1].split("</title>")[0]
                        if "<title>" in html and "</title>" in html else "No Title"
                    )
                    description = "No Description"
                    if '<meta name="description"' in html:
                        desc_split = html.split('<meta name="description"')[1]
                        description = desc_split.split('content="')[1].split('"')[0]
                    print(f"Extracted title: {title}, description: {description}")
                    return {"title": title, "description": description}
        except Exception as e:
            print(f"Failed to fetch from {url}: {e}")
            continue
    print(f"No valid website info found for IP: {ip}")
    return {"title": "No Title", "description": "No Description"}

async def analyze_ip(ip, session, executor):
    """
    Analyze a single IP address: reverse DNS and fetch website info.
    """
    print(f"Analyzing IP: {ip}")
    hostname = await reverse_dns_lookup(ip, executor)
    website_info = await fetch_website_info(session, ip, hostname)
    result = {"ip": ip, "hostname": hostname or "Unknown", **website_info}
    print(f"Analysis complete for IP: {ip}")
    return result

async def main():
    # Load the previous results and last scanned IP
    try:
        with open("output.json", "r") as file:
            data = json.load(file)
            if isinstance(data, dict):  # Check if the loaded data is a dictionary
                results = data.get("results", [])
                last_scanned_ip = data.get("last_scanned_ip", None)
            else:  # In case it's a list or incorrectly formatted
                results = []
                last_scanned_ip = None
    except (FileNotFoundError, json.JSONDecodeError):
        results = []
        last_scanned_ip = None

    # Load IPs from the JSON file
    with open("ips_up.json", "r") as file:
        ips = json.load(file)

    # If there's a last scanned IP, find the index of the next IP to scan
    start_index = 0
    if last_scanned_ip:
        try:
            start_index = ips.index(last_scanned_ip) + 1
            print(f"Resuming from IP: {last_scanned_ip}")
        except ValueError:
            print(f"Last scanned IP {last_scanned_ip} not found in the list, starting from the beginning.")

    print(f"Loaded {len(ips)} IPs for analysis.")

    # Limit the number of threads for DNS lookups
    max_threads = 20  # Adjust based on your system's resources
    executor = ThreadPoolExecutor(max_threads)

    # Use a single aiohttp session for all requests
    async with ClientSession() as session:
        tasks = [
            analyze_ip(ip, session, executor) for ip in ips[start_index:]
        ]
        # Run tasks concurrently
        try:
            batch_size = 10  # Write to file every 10 IPs
            batch_count = 0
            for coro in asyncio.as_completed(tasks):
                result = await coro
                results.append(result)
                print(f"Result: {result}")
                batch_count += 1
                if batch_count >= batch_size:
                    # Save progress to file after every batch of 10 results
                    with open("output.json", "w") as file:
                        json.dump({"results": results, "last_scanned_ip": result["ip"]}, file, indent=4)
                    print(f"Progress saved to 'output.json'. Last scanned IP: {result['ip']}")
                    batch_count = 0  # Reset the batch count

            # Final progress save
            with open("output.json", "w") as file:
                json.dump({"results": results, "last_scanned_ip": results[-1]["ip"]}, file, indent=4)
            print("Final progress saved to 'output.json'.")
        except KeyboardInterrupt:
            print("\nProcess interrupted. Saving current progress...")
            with open("output.json", "w") as file:
                json.dump({"results": results, "last_scanned_ip": ips[start_index - 1]}, file, indent=4)
            print("Progress saved successfully. Exiting gracefully.")
            return

    print("Analysis complete. Final results saved to 'output.json'.")

if __name__ == "__main__":
    asyncio.run(main())