Merge pull request #69 from techgaun/feat/output-formats

feat: add structured output formats

Author: techgaun

README.md

@@ -62,6 +62,12 @@ GH_TOKEN=<github_token> github-dorks -u dev-nepal              # search using au
 GH_URL=https://github.example.com github-dorks -u dev-nepal    # search a GitHub Enterprise instance
 
 github-dorks -r techgaun/github-dorks --max-retries 5         # retry recoverable failures up to five times
+
+github-dorks -r techgaun/github-dorks --format json           # stream JSON to stdout
+
+github-dorks -u dev-nepal --format jsonl -o results.jsonl     # write newline-delimited JSON
+
+github-dorks -u dev-nepal --format csv -o results.csv --force # explicitly replace an existing file
 ```
 
 The legacy `python github-dork.py ...` invocation remains available for
@@ -72,6 +78,13 @@ elapsed time. The command exits with status `0` after a complete scan, `2` when
 one or more queries failed, and `1` for fatal configuration, file, or
 authentication errors.
 
+Supported result formats are `text`, `csv`, `json`, and `jsonl`. Text is the
+default for terminal output; using `-o/--output` without `--format` preserves
+the historical CSV default. Existing files are protected unless `--force` is
+provided. Use `--quiet` to suppress progress and summaries, or `--verbose` to
+print every query as it runs. Structured stdout remains machine-readable because
+status messages are sent to stderr.
+
 ### Development
 
 Run the dependency-free unit test suite with:

github_dorks/cli.py

@@ -60,8 +60,26 @@ def build_parser():
         help='GitHub dorks file. Eg: github-dorks.txt',
     )
     parser.add_argument(
-        '-o', '--outputFile', dest='output_filename',
-        help='CSV file to write results to. This overwrites the provided file.',
+        '-o', '--output', '--outputFile', dest='output_filename',
+        help='Write results to this file instead of stdout',
+    )
+    parser.add_argument(
+        '--format', choices=('text', 'csv', 'json', 'jsonl'),
+        dest='output_format',
+        help='Result format (default: text, or CSV when -o is used)',
+    )
+    parser.add_argument(
+        '-f', '--force', action='store_true',
+        help='Overwrite an existing output file',
+    )
+    detail_group = parser.add_mutually_exclusive_group()
+    detail_group.add_argument(
+        '-q', '--quiet', action='store_true',
+        help='Suppress progress and summary messages',
+    )
+    detail_group.add_argument(
+        '--verbose', action='store_true',
+        help='Report each query as it runs',
     )
     parser.add_argument(
         '--max-retries', type=int, default=3,
@@ -84,6 +102,10 @@ def main():
             user_to_search=args.user_to_search,
             gh_dorks_file=args.gh_dorks_file,
             output_filename=args.output_filename,
+            output_format=args.output_format,
+            force=args.force,
+            quiet=args.quiet,
+            verbose=args.verbose,
             client=create_client(),
             max_retries=args.max_retries,
         )

github_dorks/output.py

@@ -0,0 +1,107 @@
+"""Streaming result writers for supported output formats."""
+
+import csv
+import json
+from dataclasses import asdict
+
+
+FIELDNAMES = ('dork', 'text_matches', 'path', 'score', 'url')
+CSV_HEADERS = (
+    'Issue Type (Dork)', 'Text Matches', 'File Path',
+    'Score/Relevance', 'URL of File',
+)
+
+
+class TextWriter:
+    def __init__(self, stream):
+        self.stream = stream
+
+    def start(self):
+        pass
+
+    def write(self, result):
+        self.stream.write('\n'.join([
+            'Found result for {dork}',
+            'Text matches: {text_matches}',
+            'File path: {path}',
+            'Score/Relevance: {score}',
+            'URL of File: {url}',
+            '',
+        ]).format(**result) + '\n')
+
+    def finish(self, stats):
+        pass
+
+
+class CsvWriter:
+    def __init__(self, stream):
+        self.writer = csv.DictWriter(stream, fieldnames=FIELDNAMES)
+
+    def start(self):
+        self.writer.writerow(dict(zip(FIELDNAMES, CSV_HEADERS)))
+
+    def write(self, result):
+        self.writer.writerow(result)
+
+    def finish(self, stats):
+        pass
+
+
+class JsonWriter:
+    def __init__(self, stream):
+        self.stream = stream
+        self.first_result = True
+
+    def start(self):
+        self.stream.write('{"results":[')
+
+    def write(self, result):
+        if not self.first_result:
+            self.stream.write(',')
+        json.dump(result, self.stream, default=str, separators=(',', ':'))
+        self.first_result = False
+
+    def finish(self, stats):
+        self.stream.write('],"summary":')
+        json.dump(asdict(stats), self.stream, separators=(',', ':'))
+        self.stream.write('}\n')
+
+
+class JsonLinesWriter:
+    def __init__(self, stream):
+        self.stream = stream
+
+    def start(self):
+        pass
+
+    def write(self, result):
+        json.dump(
+            {'type': 'result', **result}, self.stream,
+            default=str, separators=(',', ':'),
+        )
+        self.stream.write('\n')
+
+    def finish(self, stats):
+        json.dump(
+            {'type': 'summary', **asdict(stats)}, self.stream,
+            separators=(',', ':'),
+        )
+        self.stream.write('\n')
+
+
+WRITERS = {
+    'text': TextWriter,
+    'csv': CsvWriter,
+    'json': JsonWriter,
+    'jsonl': JsonLinesWriter,
+}
+
+
+def create_writer(output_format, stream):
+    try:
+        return WRITERS[output_format](stream)
+    except KeyError as error:
+        supported = ', '.join(WRITERS)
+        raise ValueError(
+            f'unsupported output format {output_format!r}; choose from {supported}'
+        ) from error

github_dorks/search.py

@@ -1,6 +1,5 @@
 """GitHub client construction and reliable code-search execution."""
 
-import csv
 import os
 import time
 from contextlib import nullcontext
@@ -11,6 +10,8 @@
 import github3 as github
 from requests import exceptions as requests_exceptions
 
+from github_dorks.output import create_writer
+
 
 @dataclass
 class ScanStats:
@@ -128,29 +129,19 @@ def iter_search_results(client, query, stats, max_retries=3,
             sleep(delay)
 
 
-def _write_result(result, query, csv_writer, stdout):
-    values = {
+def _result_record(result, query):
+    return {
         'dork': query,
         'text_matches': result.text_matches,
         'path': result.path,
         'score': result.score,
         'url': result.html_url,
     }
-    if csv_writer:
-        csv_writer.writerow(values.values())
-        return
-    stdout.write('\n'.join([
-        'Found result for {dork}',
-        'Text matches: {text_matches}',
-        'File path: {path}',
-        'Score/Relevance: {score}',
-        'URL of File: {url}',
-        '',
-    ]).format(**values) + '\n')
 
 
 def search(repo_to_search=None, user_to_search=None, gh_dorks_file=None,
-           output_filename=None, client=None, max_retries=3,
+           output_filename=None, output_format=None, force=False,
+           quiet=False, verbose=False, client=None, max_retries=3,
            sleep=time.sleep, now=time.time, monotonic=time.monotonic,
            stdout=None, stderr=None):
     """Run every dork and return statistics, isolating per-query failures."""
@@ -162,50 +153,63 @@ def search(repo_to_search=None, user_to_search=None, gh_dorks_file=None,
         raise ValueError('exactly one repository or user scope is required')
     client = create_client() if client is None else client
     dorks_path = find_dorks_file(gh_dorks_file)
+    output_format = output_format or ('csv' if output_filename else 'text')
     stats = ScanStats()
     started_at = monotonic()
     scope = f' repo:{repo_to_search}' if repo_to_search else f' user:{user_to_search}'
     label = 'Repo' if repo_to_search else 'User'
-    stdout.write(f'Scanning {label}: {scope.split(":", 1)[1]}\n')
+    status_stream = stderr if not output_filename and output_format != 'text' else stdout
+    if not quiet:
+        status_stream.write(f'Scanning {label}: {scope.split(":", 1)[1]}\n')
 
     output_context = (
-        open(output_filename, 'w', newline='', encoding='utf-8')
-        if output_filename else nullcontext(None)
+        open(
+            output_filename, 'w' if force else 'x', newline='', encoding='utf-8'
+        ) if output_filename else nullcontext(stdout)
     )
     with dorks_path.open(encoding='utf-8') as dork_file, output_context as output_file:
-        csv_writer = csv.writer(output_file) if output_file else None
-        if csv_writer:
-            csv_writer.writerow([
-                'Issue Type (Dork)', 'Text Matches', 'File Path',
-                'Score/Relevance', 'URL of File',
-            ])
-        for line in dork_file:
-            dork = line.strip()
-            if not dork or dork[0] in '#;':
-                continue
-            query = dork + scope
-            stats.queries += 1
-            try:
-                for result in iter_search_results(
+        writer = create_writer(output_format, output_file)
+        writer.start()
+        try:
+            for line in dork_file:
+                dork = line.strip()
+                if not dork or dork[0] in '#;':
+                    continue
+                query = dork + scope
+                stats.queries += 1
+                if verbose and not quiet:
+                    status_stream.write(f'Searching: {query}\n')
+                results = iter_search_results(
                     client, query, stats, max_retries, sleep, now, stderr
-                ):
-                    stats.matches += 1
-                    _write_result(result, query, csv_writer, stdout)
-            except Exception as error:
-                authentication_error = getattr(
-                    github.exceptions, 'AuthenticationFailed', ()
                 )
-                if authentication_error and isinstance(error, authentication_error):
-                    raise
-                stats.failures += 1
-                stderr.write(f'Query failed: {query}\n{error}\n')
-
-    stats.elapsed_seconds = monotonic() - started_at
-    if not stats.matches and not stats.failures:
-        stdout.write(f'No results for your dork search{scope}. Hurray!\n')
-    stdout.write(
-        f'Summary: {stats.queries} queries, {stats.matches} matches, '
-        f'{stats.failures} failures, {stats.retries} retries, '
-        f'{stats.elapsed_seconds:.1f}s elapsed\n'
-    )
+                while True:
+                    try:
+                        result = next(results)
+                    except StopIteration:
+                        break
+                    except Exception as error:
+                        authentication_error = getattr(
+                            github.exceptions, 'AuthenticationFailed', ()
+                        )
+                        if authentication_error and isinstance(
+                            error, authentication_error
+                        ):
+                            raise
+                        stats.failures += 1
+                        stderr.write(f'Query failed: {query}\n{error}\n')
+                        break
+                    stats.matches += 1
+                    writer.write(_result_record(result, query))
+        finally:
+            stats.elapsed_seconds = monotonic() - started_at
+            writer.finish(stats)
+
+    if not quiet:
+        if not stats.matches and not stats.failures:
+            status_stream.write(f'No results for your dork search{scope}. Hurray!\n')
+        status_stream.write(
+            f'Summary: {stats.queries} queries, {stats.matches} matches, '
+            f'{stats.failures} failures, {stats.retries} retries, '
+            f'{stats.elapsed_seconds:.1f}s elapsed\n'
+        )
     return stats