Load related collections from YAML files (#1902)

Author: mre

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
           REPO: ${{ github.repository }}
         run: |
           gh api "repos/$REPO/issues/$PR/comments" \
-            -f body="README.md was edited directly. The README is generated from the YAML files in \`data/tools/\`. Please add or edit the corresponding file in \`data/tools/\` instead and do not touch README.md." \
+            -f body="README.md was edited directly. Please edit tool entries in \`data/tools/\`, related collections in \`data/collections/\`, or text and structure in \`ci/render/templates/README.md\` instead. Leave the generated README.md out of your pull request." \
             --silent
           echo "README.md must not be edited directly." >&2
           exit 1

CONTRIBUTING.md

@@ -57,6 +57,20 @@ Finally, create a pull request with all your changes. You can call `make
 render` to check for errors before.  This is optional, because it will also be
 done when creating a pull request.
 
+### Related collections
+
+To add or update a link in “More collections”, edit a YAML file in
+`data/collections/`, not the README template. Each file contains `name`,
+`homepage`, and `description`:
+
+```yaml
+name: Example collection
+homepage: https://example.com/collection
+description: A collection of static analysis tools for a specific language.
+```
+
+Collections are listed alphabetically and are separate from individual tool entries.
+
 ### How to mark a tool as unmaintained/deprecated
 
 Sometimes a tool becomes unmaintained and there's nothing wrong with that.  

Makefile

@@ -16,10 +16,10 @@ help:
 
 # Main rendering targets
 render:
-	cargo run --manifest-path ci/Cargo.toml -p render -- --tags data/tags.yml --tools data/tools --md-out README.md --json-out data/api
+	cargo run --manifest-path ci/Cargo.toml -p render -- --tags data/tags.yml --tools data/tools --collections data/collections --md-out README.md --json-out data/api
 
 render-skip-deprecated:
-	cargo run --manifest-path ci/Cargo.toml -p render -- --tags data/tags.yml --tools data/tools --md-out README.md --json-out data/api --skip-deprecated
+	cargo run --manifest-path ci/Cargo.toml -p render -- --tags data/tags.yml --tools data/tools --collections data/collections --md-out README.md --json-out data/api --skip-deprecated
 
 # Development targets
 check:

ci/render/src/bin/main.rs

@@ -1,8 +1,9 @@
 use anyhow::{Context, Result};
 use askama::Template;
 use pico_args::Arguments;
-use render::types::{Entry, ParsedEntry, Tag, Tags, Type};
+use render::types::{Collection, Entry, ParsedEntry, Tag, Tags, Type};
 use render::{check_deprecated, create_api, create_catalog};
+use serde::de::DeserializeOwned;
 use slug::slugify;
 use std::collections::BTreeMap;
 use std::env;
@@ -14,6 +15,7 @@ use std::path::PathBuf;
 struct Args {
     tags: PathBuf,
     tools: PathBuf,
+    collections: PathBuf,
     md_out: PathBuf,
     json_out: PathBuf,
     skip_deprecated: bool,
@@ -30,7 +32,7 @@ fn read_tags(path: PathBuf) -> Result<Tags> {
     Ok(serde_saphyr::from_reader(f)?)
 }
 
-fn read_tools(path: PathBuf) -> Result<Vec<ParsedEntry>> {
+fn read_entries<T: DeserializeOwned>(path: PathBuf) -> Result<Vec<T>> {
     let dir: std::fs::ReadDir = std::fs::read_dir(path)?;
 
     let files = dir
@@ -47,10 +49,11 @@ fn read_tools(path: PathBuf) -> Result<Vec<ParsedEntry>> {
         .inspect(|p| println!("Checking {}", p.display()))
         .map(|p| {
             let file = std::fs::File::open(p)?;
-            let entry: ParsedEntry = serde_saphyr::from_reader(file)?;
+            let entry = serde_saphyr::from_reader(file)
+                .with_context(|| format!("Cannot parse {}", p.display()))?;
             Ok(entry)
         })
-        .collect::<Result<Vec<ParsedEntry>, _>>()
+        .collect()
 }
 
 /// Backfills the deprecated field in the tools data from the old tools data.
@@ -81,14 +84,18 @@ async fn main() -> Result<()> {
     let args = Args {
         tags: args.value_from_os_str("--tags", parse_path)?,
         tools: args.value_from_os_str("--tools", parse_path)?,
+        collections: args.value_from_os_str("--collections", parse_path)?,
         md_out: args.value_from_os_str("--md-out", parse_path)?,
         json_out: args.value_from_os_str("--json-out", parse_path)?,
         skip_deprecated: args.contains("--skip-deprecated"),
     };
 
     let tags = read_tags(args.tags)?;
 
-    let parsed_tools = read_tools(args.tools)?;
+    let mut collections: Vec<Collection> = read_entries(args.collections)?;
+    collections.sort_by_cached_key(|collection| collection.name.to_lowercase());
+
+    let parsed_tools: Vec<ParsedEntry> = read_entries(args.tools)?;
     let tools: Result<Vec<Entry>> = parsed_tools
         .into_iter()
         .map(|t| Entry::from_parsed(t, &tags))
@@ -119,7 +126,7 @@ async fn main() -> Result<()> {
 
     let other_tags: Vec<Tag> = tags.into_iter().filter(|t| t.kind == Type::Other).collect();
 
-    let catalog = create_catalog(&tools, &languages, &other_tags);
+    let catalog = create_catalog(&tools, &languages, &other_tags, collections);
     fs::write(&args.md_out, catalog.render()?).context(format!(
         "Cannot write Markdown output to {}",
         args.md_out.display()
@@ -168,7 +175,20 @@ mod tests {
     fn parses_catalog() -> Result<()> {
         let data = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../data");
         let tags = read_tags(data.join("tags.yml"))?;
-        let tools = read_tools(data.join("tools"))?;
+        let tools: Vec<ParsedEntry> = read_entries(data.join("tools"))?;
+        let collections: Vec<Collection> = read_entries(data.join("collections"))?;
+        assert!(!collections.is_empty());
+        let catalog = create_catalog(&[], &[], &[], collections);
+        let markdown = catalog.render()?;
+        for collection in &catalog.collections {
+            assert!(!collection.name.is_empty());
+            reqwest::Url::parse(&collection.homepage)?;
+            assert!(!collection.description.is_empty());
+            assert!(markdown.contains(&format!(
+                "- [{}]({}) — {}",
+                collection.name, collection.homepage, collection.description
+            )));
+        }
         assert!(!tags.is_empty());
         assert!(!tools.is_empty());
         for tool in tools {

ci/render/src/lib.rs

@@ -10,7 +10,7 @@ pub mod stats;
 pub mod types;
 
 use std::collections::BTreeMap;
-use types::{Api, ApiEntry, Catalog, Entry, ParsedEntry, Tag, Type};
+use types::{Api, ApiEntry, Catalog, Collection, Entry, ParsedEntry, Tag, Type};
 
 fn valid(entry: &ParsedEntry, tags: &[Tag]) -> Result<()> {
     let lints = [lints::name, lints::min_one_tag];
@@ -113,7 +113,12 @@ pub async fn check_deprecated(token: &str, entries: &mut [Entry]) -> Result<()>
 
 /// Groups normalized entries for the generated README.
 #[must_use]
-pub fn create_catalog(entries: &[Entry], languages: &[Tag], other_tags: &[Tag]) -> Catalog {
+pub fn create_catalog(
+    entries: &[Entry],
+    languages: &[Tag],
+    other_tags: &[Tag],
+    collections: Vec<Collection>,
+) -> Catalog {
     // Multi-language tools get their own primary section instead of being repeated under
     // every language. They still belong in applicable non-language tag sections.
     let (multi, single_language): (Vec<Entry>, Vec<Entry>) =
@@ -159,6 +164,7 @@ pub fn create_catalog(entries: &[Entry], languages: &[Tag], other_tags: &[Tag])
         linters,
         others,
         multi,
+        collections,
     }
 }
 
@@ -292,6 +298,7 @@ mod tests {
             linters: BTreeMap::from([(tag("Rust", "rust", Type::Language), tools.clone())]),
             others: BTreeMap::from([(tag("Security", "security", Type::Other), tools.clone())]),
             multi: tools,
+            collections: vec![],
         };
         let markdown = catalog.render()?;
 
@@ -343,6 +350,7 @@ mod tests {
                 linters: BTreeMap::new(),
                 others: BTreeMap::new(),
                 multi: tools,
+                collections: vec![],
             }
             .render()?;
             assert!(!markdown.contains("Show Deprecated"));
@@ -358,6 +366,7 @@ mod tests {
             linters: BTreeMap::new(),
             others: BTreeMap::new(),
             multi: vec![tool],
+            collections: vec![],
         }
         .render()?;
         assert_eq!(
@@ -427,7 +436,7 @@ mod tests {
         let languages = [python, rust];
         let other_tags = [ai_generated.clone()];
 
-        let catalog = create_catalog(std::slice::from_ref(&tool), &languages, &other_tags);
+        let catalog = create_catalog(std::slice::from_ref(&tool), &languages, &other_tags, vec![]);
 
         assert!(catalog.linters.is_empty());
         assert_eq!(catalog.multi.len(), 1);
@@ -451,6 +460,7 @@ mod tests {
             std::slice::from_ref(&tool),
             &[c.clone(), cpp.clone()],
             std::slice::from_ref(&security),
+            vec![],
         );
 
         assert!(catalog.multi.is_empty());

ci/render/src/types.rs

@@ -212,12 +212,22 @@ impl Ord for Entry {
 
 pub type EntryMap = BTreeMap<Tag, Vec<Entry>>;
 
+/// A related collection listed separately from individual tools.
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Collection {
+    pub name: String,
+    pub homepage: String,
+    pub description: String,
+}
+
 #[derive(Debug, Serialize, Deserialize, Template)]
 #[template(path = "README.md")]
 pub struct Catalog {
     pub linters: EntryMap,
     pub others: EntryMap,
     pub multi: Vec<Entry>,
+    pub collections: Vec<Collection>,
 }
 
 impl Catalog {

ci/render/templates/README.md

@@ -109,14 +109,9 @@ Also check out the sister project, [awesome-dynamic-analysis](https://github.com
 
 ## More collections
 
-- [Clean code linters](https://github.com/collections/clean-code-linters) — A collection of linters in github collections
-- [Code Quality Checker Tools For PHP Projects](https://github.com/collections/code-quality-in-php) — A collection of PHP linters in github collections
-- [go-tools](https://github.com/dominikh/go-tools) — A collection of tools and libraries for working with Go code, including linters and static analysis
-- [linters](https://github.com/mcandre/linters) — An introduction to static code analysis
-- [OWASP Source Code Analysis Tools](https://owasp.org/www-community/Source_Code_Analysis_Tools) — List of tools maintained by the Open Web Application Security Project
-- [php-static-analysis-tools](https://github.com/exakat/php-static-analysis-tools) — A reviewed list of useful PHP static analysis tools
-- [AppSec Santa — SAST Tools](https://appsecsanta.com/sast-tools) — Independent comparison of 30+ static analysis security testing tools with features, pricing, and alternatives
-- [Wikipedia](http://en.wikipedia.org/wiki/List_of_tools_for_static_code_analysis) — A list of tools for static code analysis.
+{% for collection in collections -%}
+- [{{ collection.name }}]({{ collection.homepage }}) — {{ collection.description }}
+{% endfor %}
 
 ## License
 

data/README.md

@@ -1,6 +1,7 @@
 This folder contains the source data for the list.
 
 - `tools/` - one YAML file per tool
+- `collections/` - one YAML file per related collection in the README's “More collections” section
 - `tags.yml` - all valid tags that can be used in tool entries
 - `api/` - generated JSON output (do not edit manually)
 

data/collections/appsec-santa.yml

@@ -0,0 +1,3 @@
+name: AppSec Santa — SAST Tools
+homepage: https://appsecsanta.com/sast-tools
+description: Independent comparison of 30+ static analysis security testing tools with features, pricing, and alternatives

data/collections/clean-code-linters.yml

@@ -0,0 +1,3 @@
+name: Clean code linters
+homepage: https://github.com/collections/clean-code-linters
+description: A collection of linters in github collections