before analyzing

.gitignore

@@ -0,0 +1,176 @@
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+dev.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc

README.md

@@ -0,0 +1,51 @@
+# Available Subnets
+
+## Вступление
+Для борьбы с блокировками вида Whitelist, мы неизбежно встречаемся с вопросом: к каким айпи и подсетям доступ разрешён? Если к каким-то ресурсам есть доступ, значит ли это что можно найти и свой "счастливый IP"? Этим вопросом задалась команда **Malenia-Research**. 
+
+Целью исследования можно установить **поиск доступных в условиях whitelist-ограничений IP, подсетей и компаний-хостеров, к которым эти адреса относятся**.
+
+## Этап 1: Подготовка диапазона сканирования
+
+**Цель:** Сопоставить две базы данных IP2Location (ASN и DB11), чтобы извлечь список всех CIDR-подсетей для заданной страны (например, RU) и сохранить их в текстовый файл (по одному CIDR на строку).
+### Алгоритм
+1. **Предварительная загрузка (DB11):**  
+    База геолокации загружается в оперативную память в виде двух параллельных массивов:
+    - ip_starts — отсортированный список начальных IP-адресов (ip_from).
+    - geo_data — список кортежей с конечным IP и кодом страны (ip_to, country_code).
+2. **Бинарный поиск (ASN):**  
+    Скрипт построчно читает базу ASN. Для начального IP каждой подсети применяется встроенная функция bisect.bisect_right() к массиву ip_starts. Это позволяет за доли миллисекунды найти индекс блока геолокации, к которому относится данный IP.
+3. **Валидация и фильтрация:**  
+    Найдя нужный блок, скрипт делает две проверки:
+    - Не выходит ли IP за верхнюю границу диапазона (ip_from <= db_ip_to).
+    - Совпадает ли код страны с искомым (например, RU).
+
+Eсли условия выполнены, подсеть (CIDR) записывается в итоговый .txt файл.
+## Этап 2: Сканирование
+
+**Цель:** Получить из списка подсетей конкретные IP-адреса, доступные при белых списках.
+
+Этот шаг может быть выполнен по разному, в данном документе будет описано то, как сканирование проводилось в рамках данного исследования.
+
+В зоне работы белых списков, на телефоне с сим-картой оператора Т2-Мобайл включался режим "Точки Доступа". К этой сети Wi-Fi подключался ноутбук с включенной Windows Subsystem for Linux (Ubuntu). Внутри папки `scanning` выполнялась следующая команда:
+
+```bash
+sudo masscan -iL russian_subnets.txt -p443,22 --excludefile exclude.txt --rate=5000 -oG results.txt
+```
+
+Порты выбраны не диапазоном для увеличения эффективности алгоритма. 22 и 443 были выбраны в качестве наиболее эффективных, т.к. практически на каждом сервере установлен сервер ssh, оперирующий на порте 22. Также, мы использовали 443, т.к. nginx тоже является одним из самых распространённых сервисов на виртуальных серверах.
+
+Важно принять во внимание, что даже при выбраном `rate=5000` (5000 пакетов в секунду), список IP-адресов, по которому выполнялся наш поиск, будет обрабатываться около 5-7 часов.
+
+В результате мы получаем `results.txt`, содержащий данные в формате
+
+```
+Timestamp *    Host: * ()    Ports: *
+```
+
+Мы перевели эти данные в упрощённый вид IP-адресов:
+
+```bash
+grep -oP 'Host: \K[^ ]+ ' results.txt | aggregate > subnets.txt
+```
+## Этап 3: Обработка данных

analyzing_mascan_results/analyze.py

@@ -0,0 +1,44 @@
+import geoip2.database
+import geoip2.errors
+import csv
+
+# File names
+INPUT_FILE = 'mascan_results.txt'       # Your list of IPs, one per line
+OUTPUT_FILE = 'results.csv'  # The CSV file that will be created
+DB_FILE = 'GeoLite2-ASN.mmdb' # The MaxMind database file
+
+# We use 'with' statements to ensure files and databases close automatically
+with geoip2.database.Reader(DB_FILE) as reader, \
+     open(INPUT_FILE, 'r') as infile, \
+     open(OUTPUT_FILE, 'w', newline='') as outfile:
+    
+    # Initialize the CSV writer
+    writer = csv.writer(outfile)
+    
+    # Write the column headers to the CSV
+    writer.writerow(['IP Address', 'ISP / Provider', 'Subnet', 'ASN'])
+    
+    # Loop through each line in your input text file
+    for line in infile:
+        ip = line.strip() # Remove spaces and line breaks
+        
+        if not ip:
+            continue # Skip blank lines
+            
+        try:
+            # Query the Radix Tree database
+            response = reader.asn(ip)
+            provider = response.autonomous_system_organization
+            subnet = str(response.network) # Convert subnet object to string
+            asn = response.autonomous_system_number
+            
+            # Write the successful result to the CSV
+            writer.writerow([ip, provider, subnet, asn])
+            
+        except geoip2.errors.AddressNotFoundError:
+            # IP isn't in the database
+            writer.writerow([ip, 'Not Found', 'Not Found', 'Not Found'])
+            
+        except ValueError:
+            # The text on this line wasn't a valid IP address format
+            writer.writerow([ip, 'Invalid IP format', 'N/A', 'N/A'])

analyzing_mascan_results/mascan_results.txt

@@ -0,0 +1,44 @@
+213.87.45.151
+51.250.32.53
+185.138.255.120
+91.142.135.249
+185.138.254.19
+194.35.48.89
+195.3.183.35
+51.250.36.135
+84.201.149.9
+213.59.197.148
+176.112.172.163
+45.136.21.184
+51.250.68.84
+79.174.93.218
+151.236.108.136
+94.181.203.49
+87.250.251.94
+158.160.60.16
+84.201.130.113
+84.201.162.116
+178.248.232.99
+51.250.125.50
+46.42.185.125
+95.163.251.181
+158.160.54.141
+185.141.224.76
+151.236.75.97
+51.250.104.104
+158.160.225.118
+158.160.176.92
+5.188.113.209
+51.250.15.30
+151.236.117.31
+176.109.76.103
+185.51.158.56
+46.254.18.103
+155.212.204.157
+158.160.167.138
+185.65.149.243
+178.250.242.63
+185.32.249.55
+158.160.90.8
+51.250.77.135
+109.238.88.226

analyzing_mascan_results/results.csv

@@ -0,0 +1,45 @@
+IP Address,ISP / Provider,Subnet,ASN
+213.87.45.151,MTS PJSC,213.87.44.0/22,13174
+51.250.32.53,Yandex.Cloud LLC,51.250.0.0/17,200350
+185.138.255.120,LLC Wildberries,185.138.252.0/22,57073
+91.142.135.249,PJSC NK Rosneft,91.142.135.0/24,41706
+185.138.254.19,LLC Wildberries,185.138.252.0/22,57073
+194.35.48.89,Lime HD LLC,194.35.48.0/23,47481
+195.3.183.35,GKU TO CIT,195.3.183.0/24,39562
+51.250.36.135,Yandex.Cloud LLC,51.250.0.0/17,200350
+84.201.149.9,Yandex.Cloud LLC,84.201.128.0/18,200350
+213.59.197.148,Rostelecom,213.59.192.0/18,12389
+176.112.172.163,LLC VK,176.112.168.0/21,47764
+45.136.21.184,LLC VK,45.136.20.0/22,47764
+51.250.68.84,Yandex.Cloud LLC,51.250.0.0/17,200350
+79.174.93.218,"Domain names registrar REG.RU, Ltd",79.174.80.0/20,197695
+151.236.108.136,Global Cloud Network LLC,151.236.108.0/24,204720
+94.181.203.49,JSC ER-Telecom Holding,94.181.192.0/18,41727
+87.250.251.94,YANDEX LLC,87.250.248.0/21,13238
+158.160.60.16,Yandex.Cloud LLC,158.160.0.0/16,200350
+84.201.130.113,Yandex.Cloud LLC,84.201.128.0/18,200350
+84.201.162.116,Yandex.Cloud LLC,84.201.128.0/18,200350
+178.248.232.99,HLL LLC,178.248.232.0/21,51115
+51.250.125.50,Yandex.Cloud LLC,51.250.0.0/17,200350
+46.42.185.125,CDNvideo LLC,46.42.185.0/24,57363
+95.163.251.181,LLC VK,95.163.248.0/21,47764
+158.160.54.141,Yandex.Cloud LLC,158.160.0.0/16,200350
+185.141.224.76,Global Cloud Network LLC,185.141.224.0/24,204720
+151.236.75.97,CDNvideo LLC,151.236.72.0/22,57363
+51.250.104.104,Yandex.Cloud LLC,51.250.0.0/17,200350
+158.160.225.118,Yandex.Cloud LLC,158.160.0.0/16,200350
+158.160.176.92,Yandex.Cloud LLC,158.160.0.0/16,200350
+5.188.113.209,JSC Selectel,5.188.113.0/24,49505
+51.250.15.30,Yandex.Cloud LLC,51.250.0.0/17,200350
+151.236.117.31,Global Cloud Network LLC,151.236.116.0/22,204720
+176.109.76.103,MTS PJSC,176.109.72.0/21,60490
+185.51.158.56,LLC IMT,185.51.156.0/22,60162
+46.254.18.103,EuroByte LLC,46.254.16.0/21,210079
+155.212.204.157,LLC VK,155.212.192.0/20,47764
+158.160.167.138,Yandex.Cloud LLC,158.160.0.0/16,200350
+185.65.149.243,HLL LLC,185.65.148.0/22,51115
+178.250.242.63,Hosting Ltd,178.250.240.0/21,43362
+185.32.249.55,LLC VK,185.32.249.0/24,28709
+158.160.90.8,Yandex.Cloud LLC,158.160.0.0/16,200350
+51.250.77.135,Yandex.Cloud LLC,51.250.0.0/17,200350
+109.238.88.226,Servicepipe LLC,109.238.88.0/24,201706

parse_subnets/main.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+IP2Location ASN & Geolocation Subnet Parser
+Designed for research purposes. 
+
+This script cross-references the IP2LOCATION-LITE-ASN and IP2LOCATION-LITE-DB11
+databases to extract all CIDR subnets belonging to a specific country code (default: RU).
+It outputs only the CIDR subnets to a plain text file.
+"""
+
+import csv
+import bisect
+import argparse
+import logging
+import sys
+from typing import List, Tuple
+
+# Configure logging for research tracking
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S"
+)
+
+def load_geolocation_data(db11_path: str) -> Tuple[List[int], List[Tuple[int, str]]]:
+    """
+    Loads the DB11 database into memory for fast binary search.
+    
+    Args:
+        db11_path (str): Path to the IP2LOCATION-LITE-DB11.CSV file.
+        
+    Returns:
+        Tuple containing:
+            - A sorted list of starting IPs (ip_from).
+            - A corresponding list of tuples (ip_to, country_code).
+    """
+    ip_starts: List[int] = []
+    geo_data: List[Tuple[int, str]] = []
+    
+    logging.info(f"Loading Geolocation database: {db11_path}")
+    
+    try:
+        with open(db11_path, mode='r', encoding='utf-8') as f:
+            reader = csv.reader(f)
+            for row_num, row in enumerate(reader, start=1):
+                try:
+                    ip_from = int(row[0])
+                    ip_to = int(row[1])
+                    country_code = row[2]
+                    
+                    ip_starts.append(ip_from)
+                    geo_data.append((ip_to, country_code))
+                except (ValueError, IndexError):
+                    logging.warning(f"Skipping malformed row {row_num} in DB11.")
+                    
+    except FileNotFoundError:
+        logging.error(f"File not found: {db11_path}")
+        sys.exit(1)
+
+    logging.info(f"Successfully loaded {len(ip_starts):,} geolocation records.")
+    return ip_starts, geo_data
+
+def extract_country_subnets(
+    asn_path: str, 
+    ip_starts: List[int], 
+    geo_data: List[Tuple[int, str]], 
+    target_country: str, 
+    output_path: str
+) -> None:
+    """
+    Parses the ASN database, cross-references with DB11 data, and saves matching CIDRs to a TXT file.
+    
+    Args:
+        asn_path (str): Path to the IP2LOCATION-LITE-ASN.CSV file.
+        ip_starts (List[int]): Sorted list of starting IPs from DB11.
+        geo_data (List[Tuple[int, str]]): Corresponding DB11 data (ip_to, country_code).
+        target_country (str): ISO 3166-1 alpha-2 country code (e.g., 'RU').
+        output_path (str): Filepath for the resulting TXT file.
+    """
+    logging.info(f"Parsing ASN database ({asn_path}) for country code: '{target_country}'...")
+    
+    match_count = 0
+    total_count = 0
+
+    try:
+        with open(asn_path, mode='r', encoding='utf-8') as infile, \
+             open(output_path, mode='w', encoding='utf-8') as outfile:
+             
+            reader = csv.reader(infile)
+            
+            for row in reader:
+                total_count += 1
+                try:
+                    ip_from = int(row[0])
+                    cidr = row[2]
+                    
+                    # Binary search to find the matching geolocation block
+                    # bisect_right returns the insertion point. The actual range is at index - 1
+                    idx = bisect.bisect_right(ip_starts, ip_from) - 1
+                    
+                    if idx >= 0:
+                        db_ip_to, country_code = geo_data[idx]
+                        
+                        # Verify that the IP falls within the bounds and matches the country
+                        if ip_from <= db_ip_to and country_code == target_country:
+                            outfile.write(f"{cidr}\n")
+                            match_count += 1
+                            
+                except (ValueError, IndexError):
+                    logging.warning(f"Skipping malformed row {total_count} in ASN DB.")
+
+    except FileNotFoundError:
+        logging.error(f"File not found: {asn_path}")
+        sys.exit(1)
+
+    logging.info(f"Processing complete. Parsed {total_count:,} ASN records.")
+    logging.info(f"Found {match_count:,} subnets assigned to '{target_country}'.")
+    logging.info(f"Results saved to: {output_path}")
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Cross-reference IP2Location ASN and DB11 databases to extract specific country CIDRs."
+    )
+    
+    parser.add_argument(
+        "--asn", 
+        required=True, 
+        help="Path to the IP2LOCATION-LITE-ASN.CSV file"
+    )
+    parser.add_argument(
+        "--db11", 
+        required=True, 
+        help="Path to the IP2LOCATION-LITE-DB11.CSV file"
+    )
+    parser.add_argument(
+        "--output", 
+        default="russian_subnets.txt", 
+        help="Output TXT filename (default: russian_subnets.txt)"
+    )
+    parser.add_argument(
+        "--country", 
+        default="RU", 
+        help="ISO 3166-1 alpha-2 country code to filter by (default: RU)"
+    )
+    
+    args = parser.parse_args()
+
+    # Step 1: Load DB11 into memory
+    ip_starts, geo_data = load_geolocation_data(args.db11)
+    
+    # Step 2: Parse ASN, filter by country, and output to TXT
+    extract_country_subnets(args.asn, ip_starts, geo_data, args.country.upper(), args.output)
+
+if __name__ == "__main__":
+    main()

scanning/exclude.txt

@@ -0,0 +1,9 @@
+0.0.0.0/8
+10.0.0.0/8
+100.64.0.0/10
+127.0.0.0/8
+169.254.0.0/16
+172.16.0.0/12
+198.18.0.0/15
+224.0.0.0/4
+255.255.255.255/32