Merge pull request #13 from ptkvaibhav/v5

Author: ptkvaibhav

src/core/entity-detector.js

@@ -1,64 +1,72 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { askAI } from "./ai.js";
 
-const ENTITY_MARKERS = [
-  ".git",
-  "node_modules",
-  "package.json",
-  "Cargo.toml",
-  "requirements.txt",
-  ".vbox",
-  "AppxManifest.xml",
-  "Contents/Info.plist", // macOS apps
-  "bin/Release",
-  "Release"
-];
-
-const APPLICATION_NAME_PATTERNS = [
-  /kali-linux/i,
-  /juice-shop/i,
-  /flutter_windows/i,
-  /apache-maven/i,
-  /DS4Windows/i,
-  /DVWA/i,
-  /VMware/i,
-  /Ollama/i
-];
+// In-memory cache to prevent spamming the LLM with identical directory structures.
+const entityCache = new Map();
 
 /**
- * Checks if a directory should be treated as a single cohesive entity 
- * (like an app or project) rather than a collection of individual files.
+ * Uses the local AI to dynamically determine if a directory should be treated 
+ * as a single cohesive entity (like an app, project, cache, or backup) 
+ * rather than a collection of individual files.
  */
 export async function detectCohesiveEntity(absoluteDirPath) {
   try {
-    const entries = await fs.readdir(absoluteDirPath);
-    const entrySet = new Set(entries);
+    const dirName = path.basename(absoluteDirPath);
+    
+    // Ignore root drives or empty names
+    if (!dirName || dirName.length <= 1) return { isEntity: false };
 
-    // Check for file markers
-    for (const marker of ENTITY_MARKERS) {
-      if (entrySet.has(marker)) {
-        return {
-          isEntity: true,
-          type: "software_project",
-          marker
-        };
-      }
+    const entries = await fs.readdir(absoluteDirPath, { withFileTypes: true });
+    
+    // Sample contents (up to 12 items) to give the AI context
+    const sampleItems = entries.slice(0, 12).map(e => e.isDirectory() ? `[DIR] ${e.name}` : e.name);
+    const sample = sampleItems.length > 0 ? sampleItems.join(", ") : "(empty)";
+    
+    // Cache key based on directory name and a sample of its contents
+    const cacheKey = `${dirName}::${sample}`;
+    if (entityCache.has(cacheKey)) {
+      return entityCache.get(cacheKey);
     }
 
-    // Check for application name patterns in the folder itself
-    const dirName = path.basename(absoluteDirPath);
-    for (const pattern of APPLICATION_NAME_PATTERNS) {
-      if (pattern.test(dirName)) {
-        return {
-          isEntity: true,
-          type: "application",
-          pattern: pattern.toString()
-        };
-      }
-    }
+    const prompt = `You are a deterministic file system analysis AI. Your job is to decide if a directory is a "Cohesive Entity".
+A Cohesive Entity is a folder that should NOT have its internal files separated or moved individually. 
+Examples of Cohesive Entities:
+- Software projects (contain source code, config files)
+- Installed applications or games
+- System caches, temporary folders, or metadata folders (e.g., .thumbnails, .cache, .git)
+- Device backups or OS images
 
-    return { isEntity: false };
-  } catch {
+Directory Name: "${dirName}"
+Contents Sample: ${sample}
+
+Determine if this directory is a Cohesive Entity.
+Return ONLY a raw JSON string matching this exact structure: {"isEntity": true/false, "type": "software|app|cache|backup|none", "reasoning": "Brief explanation"}`;
+
+    const response = await askAI(prompt, "You are a deterministic system AI returning valid JSON only. Do not use markdown blocks like ```json. Return raw JSON.");
+    
+    // Clean JSON if the AI hallucinates markdown
+    const clean = response.replace(/```(?:json)?\s*([\s\S]*?)\s*```/g, "$1").trim();
+    
+    let result;
+    try {
+      result = JSON.parse(clean);
+    } catch {
+      // If parsing fails, use a safe default but don't cache the failure globally
+      return { isEntity: false };
+    }
+    
+    const finalResult = {
+      isEntity: !!result.isEntity,
+      type: result.type || "unknown",
+      marker: "AI Determined",
+      reasoning: result.reasoning || "AI classified based on name and contents."
+    };
+    
+    entityCache.set(cacheKey, finalResult);
+    return finalResult;
+  } catch (error) {
+    console.error("AI Entity Detection failed for", absoluteDirPath, error.message);
     return { isEntity: false };
   }
 }

src/organization/local-audit.js

@@ -1,6 +1,7 @@
 import { Catalog } from "../core/catalog.js";
 import { fingerprintFile } from "../core/fingerprint.js";
 import { loadEngagement } from "../engagement/parser.js";
+import path from "node:path";
 import { findDuplicateGroups } from "./duplicates.js";
 import { findIrrelevanceFindings } from "./irrelevance.js";
 import { buildOrganizationProposals } from "./proposals.js";
@@ -57,14 +58,30 @@ export async function buildLocalAudit({
       continue;
     }
 
-    const profile = await fingerprintFile(scannedFile.absolutePath);
-    
-    // Extract content for deep intelligence (PDFs, txt, etc.)
-    if (skippedFiles.includes(scannedFile.absolutePath)) {
-      profile.extractedText = "";
+    let profile;
+    if (scannedFile.isEntity) {
+      // For folders (entities), we don't read bytes. 
+      // We use a deterministic hash based on path and type.
+      profile = {
+        absolutePath: scannedFile.absolutePath,
+        baseName: path.basename(scannedFile.absolutePath),
+        extension: "",
+        sizeBytes: 0,
+        modifiedAt: scannedFile.modifiedAt,
+        sha256: `entity:${scannedFile.entityType}:${scannedFile.absolutePath}`,
+        extractedText: ""
+      };
     } else {
-      profile.extractedText = await extractContent(scannedFile.absolutePath);
+      profile = await fingerprintFile(scannedFile.absolutePath);
+      
+      // Extract content for deep intelligence (PDFs, txt, etc.)
+      if (skippedFiles.includes(scannedFile.absolutePath)) {
+        profile.extractedText = "";
+      } else {
+        profile.extractedText = await extractContent(scannedFile.absolutePath);
+      }
     }
+
     const passwordRequired = profile.extractedText === "[[PASSWORD_REQUIRED]]";
 
     if (passwordRequired) {

src/organization/purpose-rules.js

@@ -72,8 +72,8 @@ export const PURPOSE_RULES = [
   }
 ];
 
-// Supports: _v1, -v1, .v1, (1), - Copy, - Copy (1)
-export const VERSION_PATTERN = /([._-]v(\d+))|(\((\d+)\))|([- ]Copy( \((\d+)\))?)$/i;
+// Supports: _v1, -v1, .v1, (1), - Copy, - Copy (1), 2024.4, 1.2.3
+export const VERSION_PATTERN = /([._-]v?(\d+([._]\d+)*))|(\((\d+)\))|([- ]Copy( \((\d+)\))?)$/i;
 
 export const CODE_EXTENSIONS = new Set([
   ".js", ".ts", ".jsx", ".tsx", ".py", ".java", ".c", ".cpp", ".h", ".hpp", ".cs", ".go", ".rs", ".rb", ".php", ".html", ".css", ".sql", ".sh", ".bat", ".ps1", ".yml", ".yaml", ".json", ".xml", ".md", ".sol"

src/organization/versions.js

@@ -24,11 +24,11 @@ export function identifyVersionGroups(files) {
     archivable: []
   };
 
-  for (const [key, groupFiles] of groups.entries()) {
+  for (const groupFiles of groups.values()) {
     if (groupFiles.length <= 1) continue;
 
-    // Sort by version descending
-    groupFiles.sort((a, b) => b.extractedVersion - a.extractedVersion);
+    // Sort by version descending using semantic comparison
+    groupFiles.sort((a, b) => compareVersions(b.extractedVersion, a.extractedVersion));
 
     const [latest, ...older] = groupFiles;
     findings.latest.push(latest);
@@ -45,23 +45,41 @@ function parseVersionInfo(fileName) {
   }
 
   const versionSegment = match[0];
-  let versionNumber = 0;
+  let version = "0";
 
-  // Group 2: v(\d+)
+  // Group 2: v?(\d+(\.\d+)*)
   if (match[2]) {
-    versionNumber = parseInt(match[2], 10);
+    version = match[2].replaceAll("_", ".");
   } 
   // Group 4: (\d+) from (1)
   else if (match[4]) {
-    versionNumber = parseInt(match[4], 10);
+    version = match[4];
   }
   // Group 5: Copy suffix
   else if (match[5]) {
     // Group 8: (\d+) from Copy (1)
-    versionNumber = match[8] ? parseInt(match[8], 10) : 1;
+    version = match[8] ? match[8] : "1";
   }
 
   const baseIdentity = fileName.replace(versionSegment, "").trim();
 
-  return { baseIdentity, version: versionNumber };
+  return { baseIdentity, version };
+}
+
+/**
+ * Compares two version strings (e.g. "2024.4" vs "1.2.3").
+ * Returns > 0 if v1 > v2, < 0 if v1 < v2, 0 if equal.
+ */
+function compareVersions(v1, v2) {
+  const parts1 = String(v1).split(".").map(Number);
+  const parts2 = String(v2).split(".").map(Number);
+  const maxLen = Math.max(parts1.length, parts2.length);
+
+  for (let i = 0; i < maxLen; i++) {
+    const num1 = parts1[i] || 0;
+    const num2 = parts2[i] || 0;
+    if (num1 > num2) return 1;
+    if (num2 > num1) return -1;
+  }
+  return 0;
 }

src/server.js

@@ -18,12 +18,52 @@ export async function startServer({ port = 3030, dbPath = DEFAULT_DB_PATH } = {}
 
   // API Routes
 
-  let currentScanProgress = { current: 0, total: 0, file: "" };
+  let currentScanProgress = { current: 0, total: 0, file: "", status: "idle" };
+  let scanAbortController = null;
 
   app.get("/api/scan/progress", (req, res) => {
     res.json(currentScanProgress);
   });
 
+  app.post("/api/scan/start", async (req, res) => {
+    const { directory, skippedFiles } = req.body;
+    if (!directory) return res.status(400).json({ error: "Directory path required" });
+
+    // Prevent multiple concurrent scans for now
+    if (currentScanProgress.status === "running") {
+      return res.status(409).json({ error: "A scan is already in progress" });
+    }
+
+    currentScanProgress = { current: 0, total: 0, file: "", status: "running" };
+
+    // Fire and forget the scan job
+    (async () => {
+      try {
+        const audit = await buildLocalAudit({ 
+          dbPath,
+          targetDirectory: directory,
+          skippedFiles: skippedFiles || [],
+          onProgress: (current, total, file) => {
+             currentScanProgress = { current, total, file, status: "running" };
+          }
+        });
+
+        if (audit.needsPassword) {
+          currentScanProgress.status = "needs_password";
+          currentScanProgress.passwordFile = audit.passwordFile;
+        } else {
+          currentScanProgress.status = "complete";
+        }
+      } catch (error) {
+        console.error("Background Scan Error:", error);
+        currentScanProgress.status = "failed";
+        currentScanProgress.error = error.message;
+      }
+    })();
+
+    res.json({ success: true, message: "Scan started in background" });
+  });
+
   app.get("/api/select-directory", async (req, res) => {
     try {
       const { execSync } = await import("node:child_process");
@@ -42,74 +82,6 @@ export async function startServer({ port = 3030, dbPath = DEFAULT_DB_PATH } = {}
     }
   });
 
-  app.post("/api/add-password", async (req, res) => {
-    try {
-      const { password, filePath } = req.body;
-      
-      // Test the password immediately if filePath is provided
-      if (filePath) {
-        const fs = await import("node:fs/promises");
-        const pdf = (await import("pdf-parse")).default;
-        const dataBuffer = await fs.readFile(filePath);
-        
-        let isValid = false;
-        
-        const originalWarn = console.warn;
-        console.warn = () => {};
-        try {
-          await pdf(dataBuffer, { password });
-          isValid = true;
-        } catch (e) {
-          isValid = false;
-        } finally {
-          console.warn = originalWarn;
-        }
-        
-        if (!isValid) {
-          return res.json({ success: false, error: "Incorrect password for this file" });
-        }
-      }
-
-      const { loadConfig, saveConfig } = await import("./core/config.js");
-      const { config, configPath } = await loadConfig();
-      if (!config.pdfPasswords) config.pdfPasswords = [];
-      if (!config.pdfPasswords.includes(password)) {
-        config.pdfPasswords.push(password);
-        await saveConfig(config, configPath);
-      }
-      res.json({ success: true });
-    } catch (e) {
-      res.status(500).json({ error: e.message });
-    }
-  });
-
-  // Step 1 & 2: Start Scan
-  app.post("/api/scan/start", async (req, res) => {
-    try {
-      const { directory, skippedFiles } = req.body;
-      if (!directory) return res.status(400).json({ error: "Directory path required" });
-      
-      currentScanProgress = { current: 0, total: 0, file: "" };
-      const audit = await buildLocalAudit({ 
-        dbPath,
-        targetDirectory: directory,
-        skippedFiles: skippedFiles || [],
-        onProgress: (current, total, file) => {
-           currentScanProgress = { current, total, file };
-        }
-      });
-      
-      if (audit.needsPassword) {
-        res.json({ success: true, message: "Password required", needsPassword: true, passwordFile: audit.passwordFile });
-        return;
-      }
-      
-      res.json({ success: true, message: "Scan complete", needsPassword: false });
-    } catch (error) {
-      res.status(500).json({ error: error.message });
-    }
-  });
-
   function cleanJSON(str) {
     try {
       const match = str.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
@@ -198,6 +170,17 @@ Return ONLY a raw JSON string like {"proposedName": "file.pdf", "reasoning": "wh
     }
   });
 
+  app.post("/api/items/:id/reject", async (req, res) => {
+    try {
+      const { id } = req.params;
+      catalog.db.prepare("UPDATE review_items SET status = 'rejected', approved = 0, updated_at = ? WHERE id = ?")
+        .run(new Date().toISOString(), id);
+      res.json({ success: true });
+    } catch (error) {
+      res.status(500).json({ error: error.message });
+    }
+  });
+
   app.post("/api/apply", async (req, res) => {
     try {
       const result = await applyApprovedReview({ catalog });

ui/dist/assets/index-CiiTxZ18.js

@@ -5,8 +5,8 @@
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>ui</title>
-    <script type="module" crossorigin src="/assets/index-Ma2OvLvk.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-xYV3nBDd.css">
+    <script type="module" crossorigin src="/assets/index-CiiTxZ18.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-Dbuct-A1.css">
   </head>
   <body>
     <div id="root"></div>