Initial commit

Author: gatufo

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include wappalyzer/data *

README.md

@@ -0,0 +1,84 @@
+# wappalyzer-python
+Python wrapper for [Wappalizer](https://wappalyzer.com/) (utility that uncovers the technologies used on websites)
+
+# How to use it
+
+```python
+>>> from wappalyzer import Wappalyzer
+>>> w = Wappalyzer()
+
+>>> w.analyze('http://wikipedia.org')
+{u'Apache': {u'confidence': 100, u'version': u'', u'categories': [u'web-servers']}, 
+u'Varnish': {u'confidence': 100, u'version': u'', u'categories': [u'cache-tools']}}
+
+>>> w.analyze('http://tripadvisor.com')
+{u'Apache': {u'confidence': 100, u'version': u'', u'categories': [u'web-servers']}, 
+u'Google Analytics': {u'confidence': 100, u'version': u'', u'categories': [u'analytics']}, 
+u'comScore': {u'confidence': 100, u'version': u'', u'categories': [u'analytics']}}
+
+>>> w.analyze('http://facebook.com')
+{u'reCAPTCHA': {u'confidence': 100, u'version': u'', u'categories': [u'captchas']}}
+```
+
+You can specify the User-Agent to use:
+```python
+>>> w.analyze('http://www.google.com', user_agent='your_user_agent')
+```
+
+Or analyze from already downloaded pages (in this case you'll need to have the url and response headers too):
+```python
+>>> w.analyze_from_data(url=the_url, html=the_html, headers=the_response_headers)
+```
+
+Apps and Categories are available as dict objects:
+```python
+>>> w.apps
+{u'Google Wallet': {u'website': u'wallet.google.com', u'cats': [41], u'script': [u'checkout\\.google\\.com',
+u'wallet\\.google\\.com']}, u'Lockerz Share': ...}
+
+>>> w.categories
+{u'42': u'tag-managers', u'48': u'network-storage', u'43': u'paywalls', u'49': u'feed-readers', u'24': 
+u'rich-text-editors', u'25': u'javascript-graphics', u'26': u'mobile-frameworks', ...}
+
+```
+
+
+Data can be also updated with the latest version available from the [Wappalyzer Github repo](https://github.com/AliasIO/Wappalyzer):
+
+```python
+>>> from wappalyzer import updater
+>>> updater.update_all()
+```
+By default app icons will be updated to the `data/icons` folder, in case you need them somewhere else you can specify the destination folder:
+
+```python
+>>> from wappalyzer import updater
+>>> updater.update_all(icons_folder='your_icons_folder')
+```
+
+Or update them individually:
+
+```python
+>>> updater.update_icons(icons_folder='your_icons_folder')
+```
+
+# Requirements
+
+* [Requests](https://github.com/kennethreitz/requests)
+* [PyV8](https://github.com/okoye/PyV8)
+* [lxml](https://github.com/lxml/lxml)
+
+# Install
+
+Using setup:
+
+```python
+>>> python setup.py install
+```
+
+Using pypi:
+
+```python
+>>> pip install wappalyzer-python
+```
+

requirements.txt

@@ -0,0 +1,3 @@
+requests
+PyV8
+lxml

setup.py

@@ -0,0 +1,29 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='wappalyzer-python',
+    version='0.1.2',
+    url='https://github.com/gatufo/wappalyzer-python',
+    description='Python wrapper for Wappalyzer (utility that uncovers the technologies used on websites)',
+    author='Javier Casas',
+    author_email='[email protected]',
+    packages=find_packages(),
+    package_data={'': ['data/*.*', 'data/icons/*.*']},
+    include_package_data=True,
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python :: 2.7',
+        'Topic :: Internet :: WWW/HTTP',
+    ],
+    keywords=['wappalyzer', 'scraping', 'crawling', 'site'],
+    install_requires=[
+        'requests',
+        'PyV8',
+        'lxml',
+    ],
+)
+
+

wappalyzer/__init__.py

@@ -0,0 +1,68 @@
+from urlparse import urlparse
+import logging
+
+import requests
+try:
+    import PyV8
+except ImportError:
+    from pyv8 import PyV8
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+from . import settings
+
+
+logger = logging.getLogger(__name__)
+
+
+class Wappalyzer(object):
+
+    def __init__(self):
+        logger.debug('Initializing Wappalyzer...')
+        with open(settings.FILENAME_APPS_JSON, 'r') as f:
+            data = json.loads(f.read())
+        self.categories = data['categories']
+        self.apps = data['apps']
+
+    def analyze(self, url, user_agent=None):
+        logger.debug('Fetching: %s' % url)
+
+        response = requests.get(
+            url=url,
+            headers={'User-Agent': user_agent or settings.USER_AGENT}
+        )
+        return self.analyze_from_data(
+            url=url,
+            html=response.text,
+            headers=dict(response.headers))
+
+    def analyze_from_data(self, url, html, headers):
+        logger.debug('Analyzing: %s' % url)
+
+        ctxt = PyV8.JSContext()
+        ctxt.enter()
+
+        with open(settings.FILENAME_WAPPALIZER_JS) as f:
+            ctxt.eval(f.read())
+
+        with open(settings.FILENAME_DRIVER_JS) as f:
+            ctxt.eval(f.read())
+
+        apps = json.dumps(self.apps)
+        categories = json.dumps(self.categories)
+        data = {
+            'host': urlparse(url).hostname,
+            'url': url,
+            'html': html,
+            'headers': headers
+        }
+
+        return json.loads(ctxt.eval(
+            "w.apps={apps}; w.categories={categories}; w.driver.data={data}; w.driver.init();".format(
+                apps=apps,
+                categories=categories,
+                data=json.dumps(data)
+                )
+        ))