|
| 1 | +# frozen_string_literal: true |
| 2 | + |
| 3 | +# Spike: check the traffic our request specs already generate against the |
| 4 | +# OpenAPI description in docs/openapi. |
| 5 | +# |
| 6 | +# The request specs drive the real rack app through rack-test, so every |
| 7 | +# example is a request/response pair the description either covers or |
| 8 | +# doesn't. openapi_first wraps that app and records the verdict, which gives |
| 9 | +# us contract checking without a deployed CF, a proxy, or a traffic capture. |
| 10 | +# |
| 11 | +# Off unless OPENAPI_CONFORMANCE is set, so `rake spec` is untouched -- the |
| 12 | +# gem isn't even required. |
| 13 | +# |
| 14 | +# OPENAPI_CONFORMANCE=1 record violations, fail nothing |
| 15 | +# OPENAPI_CONFORMANCE=strict raise on the first request/response that |
| 16 | +# doesn't conform |
| 17 | +# |
| 18 | +# It reads the bundled description: openapi_first takes a single file and |
| 19 | +# docs/openapi is a multi-file redocly source, so `yarn build` has to run first. |
| 20 | +module OpenapiConformance |
| 21 | + DESCRIPTION = File.expand_path('../../docs/openapi/dist/latest/openapi.yaml', __dir__) |
| 22 | + OUT_DIR = File.expand_path('../../out', __dir__) |
| 23 | + HTML_REPORT = File.join(OUT_DIR, 'openapi_coverage.html') |
| 24 | + MARKDOWN_REPORT = File.join(OUT_DIR, 'openapi_conformance.md') |
| 25 | + |
| 26 | + # Rows past this get cut from the markdown -- the HTML report has them all, |
| 27 | + # and a step summary that long is unreadable anyway. |
| 28 | + MAX_ROWS = 50 |
| 29 | + |
| 30 | + # `format: uuid` failures are ours, not the API's: 248 places in |
| 31 | + # spec/request hand a model a literal guid like 'app1_guid', and specs |
| 32 | + # assert on those literals, so they can't just be swapped for real UUIDs. |
| 33 | + # Real Cloud Controller guids are SecureRandom.uuid, so this check only has |
| 34 | + # signal against recorded traffic, not against fixtures. Counted, not listed. |
| 35 | + UUID_FORMAT_ERROR = /does not match format: uuid/ |
| 36 | + |
| 37 | + # Only suppress when every clause is a uuid complaint -- a compound error |
| 38 | + # like "... format: uuid. value at `/x` is not an object" still matters. |
| 39 | + def self.uuid_noise?(message) |
| 40 | + return false if message.nil? |
| 41 | + |
| 42 | + clauses = message.sub(/\A(Request body invalid|Response body is invalid|Path segment is invalid): /, '').split('. ') |
| 43 | + clauses.all? { |clause| clause.match?(UUID_FORMAT_ERROR) } |
| 44 | + end |
| 45 | + |
| 46 | + # openapi_first takes one reporter. This one writes the HTML report to read |
| 47 | + # after the fact, plus a short markdown digest for the CI log and the job |
| 48 | + # summary. The gem's own terminal reporter prints every route not fully |
| 49 | + # covered -- around a thousand lines -- so it only runs in CI, folded away. |
| 50 | + class Reporter |
| 51 | + def initialize(**_options); end |
| 52 | + |
| 53 | + def report(coverage_result) |
| 54 | + FileUtils.mkdir_p(OUT_DIR) |
| 55 | + OpenapiFirst::Test::Coverage::HtmlReporter.new(output: HTML_REPORT).report(coverage_result) |
| 56 | + |
| 57 | + markdown = render(coverage_result) |
| 58 | + File.write(MARKDOWN_REPORT, markdown) |
| 59 | + puts markdown |
| 60 | + print_details(coverage_result) if ENV['GITHUB_ACTIONS'] == 'true' |
| 61 | + end |
| 62 | + |
| 63 | + private |
| 64 | + |
| 65 | + # The per-route detail otherwise only exists in the HTML artifact. In |
| 66 | + # GitHub Actions it goes into a collapsed log group, so it can be read in |
| 67 | + # the job log without downloading anything and doesn't bury the digest. |
| 68 | + # Local runs skip it: the HTML report is right there in out/. |
| 69 | + def print_details(coverage_result) |
| 70 | + puts '::group::OpenAPI coverage details (every route not fully covered)' |
| 71 | + OpenapiFirst::Test::Coverage::TerminalReporter.new.report(coverage_result) |
| 72 | + puts '::endgroup::' |
| 73 | + end |
| 74 | + |
| 75 | + def render(coverage_result) |
| 76 | + lines = ["## OpenAPI conformance\n"] |
| 77 | + coverage_result.plans.each { |plan| lines.concat(render_plan(plan)) } |
| 78 | + lines.join("\n") |
| 79 | + end |
| 80 | + |
| 81 | + def render_plan(plan) |
| 82 | + exercised = plan.routes.count { |route| route.requests.any?(&:requested?) } |
| 83 | + violations, suppressed = partition_violations(plan) |
| 84 | + |
| 85 | + lines = [ |
| 86 | + "- Coverage: **#{plan.coverage.round(2)}%**", |
| 87 | + "- Described routes exercised: **#{exercised}/#{plan.routes.size}**", |
| 88 | + "- Responses skipped (5xx): #{plan.skipped_responses_count}", |
| 89 | + "- Conformance violations: **#{violations.size + suppressed.size}**" |
| 90 | + ] |
| 91 | + lines << " (#{suppressed.size} of them `format: uuid` vs. synthetic test guids, not listed)" if suppressed.any? |
| 92 | + lines << '' |
| 93 | + return lines if violations.empty? |
| 94 | + |
| 95 | + lines << '| Route | What | Error |' |
| 96 | + lines << '| --- | --- | --- |' |
| 97 | + violations.first(MAX_ROWS).each do |route, what, error| |
| 98 | + lines << "| #{route} | #{what} | #{error} |" |
| 99 | + end |
| 100 | + lines << "\n_#{violations.size - MAX_ROWS} more omitted; see the HTML report._" if violations.size > MAX_ROWS |
| 101 | + lines |
| 102 | + end |
| 103 | + |
| 104 | + # A violation is something the specs actually exercised where every |
| 105 | + # attempt was rejected. Routes nothing touched are a coverage gap, not a |
| 106 | + # contract breach, and they are the overwhelming majority right now. |
| 107 | + def violations(plan) |
| 108 | + plan.routes.flat_map do |route| |
| 109 | + label = "`#{route.request_method.upcase} #{route.path}`" |
| 110 | + bad_requests = route.requests.select { |req| req.requested? && !req.any_valid_request? } |
| 111 | + bad_responses = route.responses.select { |res| res.responded? && !res.any_valid_response? } |
| 112 | + |
| 113 | + bad_requests.map { |req| [label, 'request', req.last_error_message] } + |
| 114 | + bad_responses.map { |res| [label, "response #{res.status}", res.last_error_message] } |
| 115 | + end |
| 116 | + end |
| 117 | + |
| 118 | + def partition_violations(plan) |
| 119 | + violations(plan).partition { |(_route, _what, error)| !OpenapiConformance.uuid_noise?(error) } |
| 120 | + end |
| 121 | + end |
| 122 | + |
| 123 | + class << self |
| 124 | + def enabled? |
| 125 | + !ENV['OPENAPI_CONFORMANCE'].to_s.empty? |
| 126 | + end |
| 127 | + |
| 128 | + def strict? |
| 129 | + ENV['OPENAPI_CONFORMANCE'] == 'strict' |
| 130 | + end |
| 131 | + |
| 132 | + # Wraps the app the request specs drive. Returns it untouched when the |
| 133 | + # check is off, so a normal run pays nothing. |
| 134 | + def wrap(rack_app) |
| 135 | + return rack_app unless enabled? |
| 136 | + |
| 137 | + OpenapiFirst::Test.app(rack_app, api: :cloud_controller) |
| 138 | + end |
| 139 | + |
| 140 | + def setup! |
| 141 | + return unless enabled? |
| 142 | + |
| 143 | + unless File.exist?(DESCRIPTION) |
| 144 | + raise "OPENAPI_CONFORMANCE is set but #{DESCRIPTION} is missing. " \ |
| 145 | + 'Build it first: cd docs/openapi && yarn install && yarn build' |
| 146 | + end |
| 147 | + |
| 148 | + require 'openapi_first' |
| 149 | + |
| 150 | + OpenapiFirst::Test.setup do |test| |
| 151 | + test.register(DESCRIPTION, as: :cloud_controller) |
| 152 | + |
| 153 | + # The description covers part of the API, so the request specs hit |
| 154 | + # plenty of endpoints it says nothing about. That is the description |
| 155 | + # being incomplete, not the app being wrong. |
| 156 | + test.ignore_unknown_requests = true |
| 157 | + |
| 158 | + # Default is to turn a non-conforming request or response into a spec |
| 159 | + # failure. Off unless asked, so one violation doesn't hide the rest of |
| 160 | + # the picture -- and so this can be added without breaking the suite. |
| 161 | + if strict? |
| 162 | + # Even here the uuid complaints stay out of the way -- they are test |
| 163 | + # fixtures, so failing on them would make strict mode unusable. |
| 164 | + test.ignore_request_error { |request| uuid_noise?(request.error&.message) } |
| 165 | + test.ignore_response_error { |response, _rack_request| uuid_noise?(response.error&.message) } |
| 166 | + else |
| 167 | + test.ignore_request_error { true } |
| 168 | + test.response_raise_error = false |
| 169 | + end |
| 170 | + |
| 171 | + # 5xx responses are described on nearly every operation, but request |
| 172 | + # specs almost never produce them -- 245 of the 950 untested responses |
| 173 | + # in the first run. Skipping takes them out of the coverage number and |
| 174 | + # the report shows them as skipped rather than untested. The cost: a |
| 175 | + # skipped response isn't tracked at all, so a malformed 5xx body would |
| 176 | + # not show up as a violation. |
| 177 | + test.skip_response_coverage { |response| response.status.to_s.start_with?('5') } |
| 178 | + |
| 179 | + # :warn reports without the exit 2 that an incomplete description |
| 180 | + # would otherwise trigger. Finding out what the number actually is is |
| 181 | + # the point of the spike. |
| 182 | + test.report_coverage = :warn |
| 183 | + test.coverage_reporter = Reporter |
| 184 | + end |
| 185 | + end |
| 186 | + end |
| 187 | +end |
| 188 | + |
| 189 | +OpenapiConformance.setup! |
0 commit comments