Merge pull request #190 from dotkernel/missing-md

updated llms generation

Author: arhimede

.gitignore

@@ -54,6 +54,7 @@ package-lock.json
 # Generated by bin/generate-packages (daily cron); not tracked so it never conflicts on pull
 /public/dotkernel-packages.json
 /public/dotkernel-packages.json.tmp
+/public/dotkernel-packages-oss-lifecycle.md
 
 /public/llms-full.txt
 /public/llms.txt

bin/generate-packages

@@ -43,6 +43,10 @@ printf(
     PHP_EOL
 );
 
+if ($packageGenerator->getMarkdownFile() !== null) {
+    printf('Markdown listing written to %s%s', $packageGenerator->getMarkdownFile(), PHP_EOL);
+}
+
 if ($report['skipped'] !== []) {
     printf('Skipped by ignoreRepos: %s%s', implode(', ', $report['skipped']), PHP_EOL);
 }

config/autoload/packages.global.php

@@ -4,8 +4,9 @@
  * Dotkernel packages listing.
  *
  * Consumed by `bin/generate-packages`, which queries the GitHub organisation and writes the
- * result to `dataFile`. Credentials are intentionally NOT stored here - the GitHub token lives
- * in `config/autoload/local.php` under the `github` key, which is ignored by git.
+ * result to `dataFile`, and the same data to `markdownFile` for `llms.txt`/`llms-full.txt` to
+ * pick up. Credentials are intentionally NOT stored here - the GitHub token lives in
+ * `config/autoload/local.php` under the `github` key, which is ignored by git.
  *
  * `local.php` is merged after every `*.global.php`, so any value below can be overridden locally.
  */
@@ -70,6 +71,7 @@
             'zf1',
         ],
         'dataFile'        => __DIR__ . '/../../public/dotkernel-packages.json',
+        'markdownFile'    => __DIR__ . '/../../public/dotkernel-packages-oss-lifecycle.md',
         'includeArchived' => true,
         'timeout'         => 10,
         'connectTimeout'  => 5,

public/dotkernel-packages-oss-lifecycle.md

@@ -1,5 +1,5 @@
 ---
-title: "SQL queries using Zend_Db – SELECT"
+title: "SQL queries using Zend_Db - SELECT"
 description: "How to write SELECT queries with JOINs and WHERE IN clauses using Zend_Db, alongside their equivalent raw SQL."
 author: "Teo"
 date_published: "2010-06-15"
@@ -8,7 +8,7 @@ category: "Best Practice"
 language: "en"
 ---
 
-# SQL queries using Zend_Db – SELECT
+# SQL queries using Zend_Db - SELECT
 
 ## TL;DR
 

public/md-articles/best-practice/sql-queries-using-zend-db-select.md

@@ -11,6 +11,7 @@
 use function assert;
 use function is_array;
 use function is_scalar;
+use function is_string;
 
 class PackageGeneratorFactory
 {
@@ -39,12 +40,18 @@ public function __invoke(ContainerInterface $container): PackageGenerator
             }
         }
 
+        $markdownFile = isset($packages['markdownFile']) && is_string($packages['markdownFile'])
+            ? $packages['markdownFile']
+            : null;
+
         return new PackageGenerator(
             $client,
             (string) ($packages['dataFile'] ?? 'data/dotkernel-packages.json'),
             (string) ($github['org'] ?? 'dotkernel'),
             $ignoreRepos,
             (bool) ($packages['includeArchived'] ?? true),
+            $markdownFile,
+            (string) ($config['application']['url'] ?? ''),
         );
     }
 }

src/App/src/Factory/PackageGeneratorFactory.php

@@ -2704,7 +2704,7 @@
         ]
     },
     {
-        "name": "Architecture",
+        "name": "Architecture Deep Dive",
         "slug": "architecture",
         "isVisible": true,
         "articles": [

src/App/src/Fixture/articles_cleaned.json

@@ -16,6 +16,7 @@
 use function file_put_contents;
 use function glob;
 use function implode;
+use function in_array;
 use function preg_match;
 use function sprintf;
 use function strcasecmp;
@@ -35,6 +36,8 @@ class LlmsGenerator
         'middleware',
         'architecture',
         'design-pattern',
+        'how-to',
+        'best-practice',
     ];
 
     /**
@@ -43,8 +46,6 @@ class LlmsGenerator
      * @var list<non-empty-string>
      */
     private const OPTIONAL_CATEGORIES = [
-        'best-practice',
-        'how-to',
         'javascript',
         'php-troubleshooting',
         'licensing',
@@ -70,6 +71,24 @@ class LlmsGenerator
         'design-pattern'      => 'naming conventions for PSR-15 handlers',
     ];
 
+    /**
+     * Static page slugs listed under `## Docs` instead of `## Pages` - reference/setup content
+     * rather than product pages.
+     *
+     * @var list<non-empty-string>
+     */
+    private const array DOCS_PAGES = [
+        'architecture',
+        'wsl2',
+        'dotboost',
+    ];
+
+    /**
+     * Slug of the packages lifecycle page, hardcoded under `## Docs` in `buildHeader()` -
+     * excluded from `## Pages` here so it is not listed twice.
+     */
+    private const string OSS_LIFECYCLE_SLUG = 'dotkernel-packages-oss-lifecycle';
+
     public function __construct(
         private readonly PostRepository $postRepository,
         private readonly string $sourceDir,
@@ -106,7 +125,7 @@ public function write(): int
             }
             $optionalCategoryCount++;
             foreach ($postsByCategory[$slug] as $entry) {
-                $optionalLines[] = $this->buildPostLink($slug, $entry);
+                $optionalLines[] = $this->buildOptionalPostLink($slug, $entry);
             }
             unset($postsByCategory[$slug]);
         }
@@ -197,6 +216,22 @@ private function buildPostLink(string $slug, array $entry): string
         );
     }
 
+    /**
+     * @param array{post: Post, title: string, description: string} $entry
+     */
+    private function buildOptionalPostLink(string $slug, array $entry): string
+    {
+        return sprintf(
+            '- [%s](%s/%s/%s.md) *(%s)*: %s',
+            $entry['title'],
+            $this->baseUrl,
+            $slug,
+            $entry['post']->getSlug(),
+            $entry['post']->getCategory()->getName(),
+            $entry['description'],
+        );
+    }
+
     /**
      * @return array{title: string, description: string}
      */
@@ -223,6 +258,11 @@ private function buildPagesSection(): string
 
         $pages = [];
         foreach (glob($this->pagesDir . '/*.md') ?: [] as $path) {
+            $slug = basename($path, '.md');
+            if ($slug === self::OSS_LIFECYCLE_SLUG || in_array($slug, self::DOCS_PAGES, true)) {
+                continue;
+            }
+
             $contents = file_get_contents($path);
             if ($contents === false) {
                 continue;
@@ -236,7 +276,7 @@ private function buildPagesSection(): string
 
             $pages[] = [
                 'title'       => trim(explode('|', $title, 2)[0]),
-                'slug'        => basename($path, '.md'),
+                'slug'        => $slug,
                 'description' => $description,
             ];
         }
@@ -261,6 +301,41 @@ private function buildPagesSection(): string
         return implode("\n", $lines) . "\n\n";
     }
 
+    /**
+     * Adds `self::DOCS_PAGES` under `## Docs`
+     */
+    private function buildDocsPageLinks(): string
+    {
+        if ($this->pagesDir === null) {
+            return '';
+        }
+
+        $lines = [];
+        foreach (self::DOCS_PAGES as $slug) {
+            $path     = sprintf('%s/%s.md', $this->pagesDir, $slug);
+            $contents = @file_get_contents($path);
+            if ($contents === false) {
+                continue;
+            }
+
+            $title       = $this->extractFrontMatterField($contents, 'title');
+            $description = $this->extractFrontMatterField($contents, 'description');
+            if ($title === null || $description === null) {
+                continue;
+            }
+
+            $lines[] = sprintf(
+                '- [%s](%s/%s.md): %s',
+                trim(explode('|', $title, 2)[0]),
+                $this->baseUrl,
+                $slug,
+                $description,
+            );
+        }
+
+        return $lines === [] ? '' : implode("\n", $lines) . "\n";
+    }
+
     private function extractFrontMatterField(string $contents, string $field): ?string
     {
         if (preg_match(sprintf('/^%s:\s*"(.*)"\s*$/m', $field), $contents, $matches) === 1) {
@@ -272,15 +347,16 @@ private function extractFrontMatterField(string $contents, string $field): ?stri
 
     private function buildHeader(): string
     {
-        $intro = '> Dotkernel is the technical blog for the Dotkernel headless PHP platform - a PSR-15 '
+        $intro = '> dotkernel.com is the technical blog for the Dotkernel headless PHP platform - a PSR-15 '
             . 'compliant application built on Mezzio and Laminas components. It publishes architecture '
             . 'write-ups, how-tos, and release notes for the platform applications - Dotkernel API, Admin '
             . 'and Queue - and for the standalone Dotkernel Light skeleton.';
 
         $body = 'Content spans foundational PHP/middleware architecture (PSR-7, PSR-15, request lifecycle, '
             . 'dependency injection), practical how-tos (Doctrine migrations, CORS, authentication, caching), '
             . 'and the history/release notes of the Dotkernel ecosystem going back to 2008. Posts are organized '
-            . 'by category and attributed to an author; URLs follow the pattern `/{category-slug}/{post-slug}/`.';
+            . 'by category and attributed to an author; URLs follow the pattern `/{category-slug}/{post-slug}/`. '
+            . 'Each post also has a Markdown version at `/{category-slug}/{post-slug}.md`.';
 
         $about = 'the team behind Dotkernel - how the team works, its commitment to open source and the PHP '
             . 'community, and how it uses AI under guardrails';
@@ -289,10 +365,8 @@ private function buildHeader(): string
             . $intro . "\n\n"
             . $body . "\n\n"
             . "## Docs\n\n"
-            . "- [Blog]({$this->baseUrl}/blog/): full list of posts, most recent first, paginated\n"
-            . "- [Categories]({$this->baseUrl}/categories/): all categories with post counts\n"
-            . "- [OSS Package Lifecycle]({$this->baseUrl}/dotkernel-packages-oss-lifecycle/): "
+            . "- [OSS Package Lifecycle]({$this->baseUrl}/" . self::OSS_LIFECYCLE_SLUG . ".md): "
             . "support/maintenance status of Dotkernel's open-source packages\n"
-            . "- [Contact]({$this->baseUrl}/contact/)\n\n";
+            . $this->buildDocsPageLinks() . "\n";
     }
 }