added code to query api documentation (via beautifulsoup4) to build a current api listing

Author: mahtin

CloudFlare/api_decode_from_web.py

@@ -0,0 +1,43 @@
+""" API extras for Cloudflare API"""
+
+from bs4 import BeautifulSoup, Comment
+
+API_TYPES = ['GET', 'POST', 'PATCH', 'PUT', 'DELETE']
+
+def do_section(section):
+    """ API extras for Cloudflare API"""
+
+    cmds = []
+    # look for deprecated first in section
+    deprecated = False
+    for tag2 in section.find_all('h3'):
+        if 'Deprecation Warning' in str(tag2):
+            deprecated = True
+    # look for all API calls in section
+    for tag2 in section.find_all('pre'):
+        cmd = []
+        for child in tag2.children:
+            if isinstance(child, Comment):
+                # remove <!-- react-text ... -> parts
+                continue
+            cmd.append(child.strip())
+        if len(cmd) == 0:
+            continue
+        action = cmd[0]
+        cmd = '/' + ''.join(cmd[1:])
+        if action == '' or action not in API_TYPES:
+            continue
+        v = {'deprecated': deprecated, 'action': action, 'cmd': cmd}
+        cmds.append(v)
+    return cmds
+
+def api_decode_from_web(content):
+    """ API extras for Cloudflare API"""
+
+    soup = BeautifulSoup(content, 'html.parser')
+
+    all_cmds = []
+    for section in soup.find_all('section'):
+        all_cmds += do_section(section)
+
+    return sorted(all_cmds, key=lambda v: v['cmd'])

CloudFlare/cloudflare.py

@@ -9,6 +9,7 @@
 from .read_configs import read_configs
 from .api_v4 import api_v4
 from .api_extras import api_extras
+from .api_decode_from_web import api_decode_from_web
 from .exceptions import CloudFlareError, CloudFlareAPIError, CloudFlareInternalError
 
 BASE_URL = 'https://api.cloudflare.com/client/v4'
@@ -154,7 +155,7 @@ def call_with_certauth(self, method, parts,
                               identifier1, identifier2, identifier3,
                               params, data, files)
 
-        def _connection(self, method, url, headers, params, data, files):
+        def _connection(self, method, url, headers=None, params=None, data=None, files=None):
             """ Cloudflare v4 API"""
 
             if self.use_sessions:
@@ -254,10 +255,9 @@ def _network(self, method, headers, parts,
                 self.logger.debug('Call: method and url %s %s', str(method), str(url))
                 self.logger.debug('Call: headers %s', str(sanitize_secrets(headers)))
 
-            if self.logger:
-                self.logger.debug('Call: doit!')
-
             try:
+                if self.logger:
+                    self.logger.debug('Call: doit!')
                 response = self._connection(method, url, headers, params, data, files)
                 if self.logger:
                     self.logger.debug('Call: done!')
@@ -595,6 +595,25 @@ def _call_unwrapped(self, method, headers, parts,
             result = response_data
             return result
 
+        def _api_from_web(self):
+            """ Cloudflare v4 API"""
+
+            # base url isn't enough; we need less
+            url = '/'.join(self.base_url.split('/')[0:3])
+
+            try:
+                if self.logger:
+                    self.logger.debug('Call: doit!')
+                response = self._connection("GET", url)
+                if self.logger:
+                    self.logger.debug('Call: done!')
+            except Exception as e:
+                if self.logger:
+                    self.logger.debug('Call: exception! "%s"' % (e))
+                raise CloudFlareAPIError(0, 'connection failed.')
+
+            return response.text
+
     class _AddUnused(object):
         """ Cloudflare v4 API"""
 
@@ -919,6 +938,11 @@ def api_list(self, m=None, s=''):
                 w = w + self.api_list(a, s + '/' + n)
         return w
 
+    def api_from_web(self):
+        """ Cloudflare v4 API"""
+
+        return api_decode_from_web(self._base._api_from_web())
+
     def __init__(self, email=None, token=None, certtoken=None, debug=False, raw=False, use_sessions=True, profile=None):
         """ Cloudflare v4 API"""
 

cli4/cli4.py

@@ -23,6 +23,15 @@ def dump_commands():
     w = cf.api_list()
     sys.stdout.write('\n'.join(w) + '\n')
 
+def dump_commands_from_web():
+    """dump a tree of all the known API commands - from web"""
+    cf = CloudFlare.CloudFlare()
+    w = cf.api_from_web()
+    for r in w:
+        if r['deprecated']:
+            continue
+        sys.stdout.write('%-6s %s\n' % (r['action'], r['cmd']))
+
 def run_command(cf, method, command, params=None, content=None, files=None):
     """run the command line"""
     # remove leading and trailing /'s
@@ -239,6 +248,7 @@ def do_it(args):
     output = 'json'
     raw = False
     dump = False
+    dump_from_web = False
     profile = None
     method = 'GET'
 
@@ -247,19 +257,20 @@ def do_it(args):
              + '[-j|--json] [-y|--yaml] [-n|ndjson] '
              + '[-r|--raw] '
              + '[-d|--dump] '
+             + '[-a|--api] '
              + '[-p|--profile profile-name] '
              + '[--get|--patch|--post|--put|--delete] '
              + '[item=value|item=@filename|@filename ...] '
              + '/command ...')
 
     try:
         opts, args = getopt.getopt(args,
-                                   'Vhvqjyrdp:GPOUD',
+                                   'Vhvqjyrdap:GPOUD',
                                    [
                                        'version',
                                        'help', 'verbose', 'quiet', 'json', 'yaml', 'ndjson',
                                        'raw',
-                                       'dump',
+                                       'dump', 'api',
                                        'profile=',
                                        'get', 'patch', 'post', 'put', 'delete'
                                    ])
@@ -290,6 +301,8 @@ def do_it(args):
             profile = arg
         elif opt in ('-d', '--dump'):
             dump = True
+        elif opt in ('-a', '--api'):
+            dump_from_web = True
         elif opt in ('-G', '--get'):
             method = 'GET'
         elif opt in ('-P', '--patch'):
@@ -305,6 +318,10 @@ def do_it(args):
         dump_commands()
         sys.exit(0)
 
+    if dump_from_web:
+        dump_commands_from_web()
+        sys.exit(0)
+
     digits_only = re.compile('^-?[0-9]+$')
     floats_only = re.compile('^-?[0-9.]+$')
 

examples/example_list_api_from_web.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+"""Cloudflare API code - example"""
+
+from __future__ import print_function
+
+import os
+import sys
+import json
+
+sys.path.insert(0, os.path.abspath('..'))
+import CloudFlare
+
+def main():
+    """Cloudflare API code - example"""
+
+    cf = CloudFlare.CloudFlare()
+    try:
+        r = cf.api_from_web()
+    except Exception as e:
+        exit('api_from_web: - %s - api call connection failed' % (e))
+
+    print(json.dumps(r))
+    exit(0)
+
+if __name__ == '__main__':
+    main()
+

requirements.txt

@@ -2,3 +2,4 @@ requests>=2.4.2
 future
 pyyaml
 jsonlines
+beautifulsoup4

setup.py

@@ -32,7 +32,7 @@ def main():
         #package_data={'cloudflare-examples': ["examples/*"]},
         include_package_data=True,
         data_files = [('man/man1', ['cli4/cli4.man'])],
-        install_requires=['requests', 'future', 'pyyaml', 'jsonlines'],
+        install_requires=['requests', 'future', 'pyyaml', 'jsonlines', 'beautifulsoup4'],
         keywords='cloudflare',
         entry_points={
             'console_scripts': [