Fixed formatting, also added script to run dependents_calculator.sql before pushing in index_new_repo, also fixed the formetter file

Author: RohanVashisht1234

.github/workflows/codeberg_automatic_update.yaml

@@ -25,11 +25,11 @@ jobs:
 
       - name: uv sync and install important things
         run: |
-           cd ./src/readme_keyword_extraction
+          cd ./src/readme_keyword_extraction
 
-           uv sync
+          uv sync
 
-           uv run uvicorn main:app --host 0.0.0.0 --port 8000 &
+          uv run uvicorn main:app --host 0.0.0.0 --port 8000 &
 
       - name: wait for 20 seconds before starting, for server to start
         run: |
@@ -39,7 +39,6 @@ jobs:
       - name: install hf
         run: curl -LsSf https://hf.co/cli/install.sh | bash
 
-
       - name: Download cron-codeberg binary
         run: wget -O cron-codeberg https://github.com/Zigistry/database/releases/download/executables/cron-codeberg
 
@@ -54,4 +53,3 @@ jobs:
 
       - name: push updated database
         run: hf cp ./zigistry.db hf://buckets/Zigistry/Zigistry/zigistry.db
-

.github/workflows/github_automatic_update.yaml

@@ -25,11 +25,11 @@ jobs:
 
       - name: uv sync and install important things
         run: |
-           cd ./src/readme_keyword_extraction
+          cd ./src/readme_keyword_extraction
 
-           uv sync
+          uv sync
 
-           uv run uvicorn main:app --host 0.0.0.0 --port 8000 &
+          uv run uvicorn main:app --host 0.0.0.0 --port 8000 &
 
       - name: wait for 20 seconds before starting, for server to start
         run: |

.github/workflows/index_safe_repos_automatic_update.yaml

@@ -26,11 +26,11 @@ jobs:
 
       - name: uv sync and install important things
         run: |
-           cd ./src/readme_keyword_extraction
+          cd ./src/readme_keyword_extraction
 
-           uv sync
+          uv sync
 
-           uv run uvicorn main:app --host 0.0.0.0 --port 8000 &
+          uv run uvicorn main:app --host 0.0.0.0 --port 8000 &
 
       - name: wait for 20 seconds before starting, for server to start
         run: |
@@ -40,7 +40,6 @@ jobs:
       - name: install hf
         run: curl -LsSf https://hf.co/cli/install.sh | bash
 
-
       - name: Download index-safe-repos binary
         run: wget -O index-safe-repos https://github.com/Zigistry/database/releases/download/executables/index-safe-repos
 
@@ -53,6 +52,8 @@ jobs:
       - name: Run index-safe-repos
         run: ./index-safe-repos
 
+      - name: Run dependents calculator
+        run: sqlite3 ./zigistry.db < ./Database_SQL_Files/dependents_calculator.sql
+
       - name: push updated database
         run: hf cp ./zigistry.db hf://buckets/Zigistry/Zigistry/zigistry.db
-

Database_SQL_Files/database_schema.sql

@@ -4,6 +4,7 @@ CREATE TABLE users (
   platform VARCHAR(10) NOT NULL,
   bio VARCHAR(260)
 );
+
 CREATE TABLE repos (
   -- The username is 40 characters at max, repo name is 100 and the key and slashes is 5
   -- Hence, I can keep this at 150.
@@ -24,15 +25,9 @@ CREATE TABLE repos (
   created_at TIMESTAMP NOT NULL,
   -- I just realised that the boolean type is like an integer only.
   -- I added check that it should only be either true or false. i.e 0, 1
-  is_archived BOOLEAN NOT NULL CHECK (
-    is_archived IN (0, 1)
-  ),
-  is_disabled BOOLEAN NOT NULL CHECK (
-    is_disabled IN (0, 1)
-  ),
-  is_fork BOOLEAN NOT NULL CHECK (
-    is_fork IN (0, 1)
-  ),
+  is_archived BOOLEAN NOT NULL CHECK (is_archived IN (0, 1)),
+  is_disabled BOOLEAN NOT NULL CHECK (is_disabled IN (0, 1)),
+  is_fork BOOLEAN NOT NULL CHECK (is_fork IN (0, 1)),
   -- Making this only 30 characters limit, it shouldn't be more than that.
   -- Simple google tells, its just 36 characters, so, ok?
   license VARCHAR(40) NOT NULL,
@@ -42,9 +37,9 @@ CREATE TABLE repos (
   last_updated_in_this_database TIMESTAMP NOT NULL,
   FOREIGN KEY (owner) REFERENCES users (id) ON DELETE CASCADE
 );
-CREATE VIRTUAL TABLE repo_search USING fts5 (
-  repo_id, keywords, tokenize = 'porter'
-);
+
+CREATE VIRTUAL TABLE repo_search USING fts5 (repo_id, keywords, tokenize = 'porter');
+
 -- Doing this to make sure, no duplicate repo_id is added.
 -- I will be using this query:
 -- INSERT OR REPLACE INTO repo_search (repo_id, keywords) VALUES (?, ?)
@@ -55,12 +50,14 @@ CREATE TABLE repo_topics (
   FOREIGN KEY (repo_id) REFERENCES repos (id) ON DELETE CASCADE,
   UNIQUE (repo_id, topic)
 );
+
 CREATE TABLE repo_dependents (
   repo_id VARCHAR(150) NOT NULL,
   dependent VARCHAR(260) NOT NULL,
   FOREIGN KEY (repo_id) REFERENCES repos (id) ON DELETE CASCADE,
   UNIQUE (repo_id, dependent)
 );
+
 CREATE TABLE releases (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   repo_id VARCHAR(150) NOT NULL,
@@ -74,32 +71,35 @@ CREATE TABLE releases (
   FOREIGN KEY (repo_id) REFERENCES repos (id) ON DELETE CASCADE,
   UNIQUE (repo_id, version)
 );
+
 CREATE TABLE release_dependencies (
   release_id INTEGER NOT NULL,
   name VARCHAR(260) NOT NULL,
   hash VARCHAR(260) NOT NULL,
-  is_lazy BOOLEAN NOT NULL CHECK (
-    is_lazy IN (0, 1)
-  ),
+  is_lazy BOOLEAN NOT NULL CHECK (is_lazy IN (0, 1)),
   url VARCHAR(260) NOT NULL,
   path VARCHAR(260) NOT NULL,
   FOREIGN KEY (release_id) REFERENCES releases (id) ON DELETE CASCADE,
   UNIQUE (release_id, name)
 );
+
 CREATE TABLE index_sections (
   section_name VARCHAR(10) NOT NULL,
   repo_id VARCHAR(150) NOT NULL,
   FOREIGN KEY (repo_id) REFERENCES repos (id) ON DELETE CASCADE,
   UNIQUE (section_name, repo_id)
 );
+
 CREATE TABLE packages (
   repo_id VARCHAR(150) NOT NULL PRIMARY KEY,
   FOREIGN KEY (repo_id) REFERENCES repos (id) ON DELETE CASCADE
 );
+
 CREATE TABLE programs (
   repo_id VARCHAR(150) NOT NULL PRIMARY KEY,
   FOREIGN KEY (repo_id) REFERENCES repos (id) ON DELETE CASCADE
 );
+
 CREATE TABLE index_new_repo (
   id VARCHAR(150) PRIMARY KEY,
   type_of_repo VARCHAR(10) NOT NULL -- This is like, package or program.
@@ -109,44 +109,51 @@ CREATE TABLE safe_to_index_new_repo (
   id VARCHAR(150) PRIMARY KEY,
   type_of_repo VARCHAR(10) NOT NULL -- This is like, package or program.
 );
+
 -- For this specifically, I will create a AI model, I will train it to detect
 -- all the scam repos, if my AI algorithm flags a repo
 -- to review it will be added to this table, if no problem, it will continue.
 CREATE TABLE quarantined_repos (
   id VARCHAR(150) PRIMARY KEY,
   type_of_repo VARCHAR(10) NOT NULL
 );
+
 -- These are only the users who are either spamming or
 -- shipping malware.
-CREATE TABLE banned_user_list (
-  id VARCHAR(45) PRIMARY KEY
-);
+CREATE TABLE banned_user_list (id VARCHAR(45) PRIMARY KEY);
+
 -- This is for making it easier to fetch
 -- details of these repositories
 -- to improve the accuracy of the algorithm
 -- which detects such repos.
-CREATE TABLE banned_repo_list (
-  id VARCHAR(150) PRIMARY KEY
-);
+CREATE TABLE banned_repo_list (id VARCHAR(150) PRIMARY KEY);
+
 -- Needs update
 CREATE TABLE needs_updates (
   id VARCHAR(150) PRIMARY KEY,
   -- All the repos that need updates.
   type_of_repo VARCHAR(10) NOT NULL -- This is like, package or program.
-  );
+);
+
 -- I do search for "All repos where username is username"
 CREATE INDEX repo_owner_index ON repos (owner);
+
 -- Again, I do this, specially for the top 10 latest repos
 CREATE INDEX repo_created_at_index ON repos (created_at);
+
 -- again, for repos with most stars
 CREATE INDEX repo_star_count_index ON repos (stargazer_count);
+
 -- Whenever someone visits a repo page, that page
 -- needs "ALl the releases with of that repo."
 CREATE INDEX releases_repo_index ON releases (repo_id, published_at DESC);
+
 -- Also, for every release, I need to get
 -- all the dependencies of that release
 CREATE INDEX release_dependencies_release_id_index ON release_dependencies (release_id);
+
 -- Ok, on every package I also need number of repo topics.
 CREATE INDEX repo_topics_repo_id_index ON repo_topics (repo_id);
+
 -- Ok, on every package I also need number of repo dependents.
 CREATE INDEX repo_dependents_repo_id_index ON repo_dependents (repo_id);

GitHub_GQL_API_Files/fetch-search-query-everything.gql

@@ -75,4 +75,4 @@ fragment RepoFields on Repository {
       name
     }
   }
-}
+}

GitHub_GQL_API_Files/fetch-search-query-partial.gql

@@ -34,4 +34,4 @@ fragment RepoFields on Repository {
       name
     }
   }
-}
+}

formater.sh

@@ -3,4 +3,4 @@ taplo fmt
 npx prettier --write "**/*.{md,yaml,yml,gql}"
 npx sql-formatter -l sqlite --fix ./Database_SQL_Files/database_schema.sql
 npx sql-formatter -l sqlite --fix ./Database_SQL_Files/dependents_calculator.sql
-zig fmt ./src/search_preparer.zig
+

src/bin/fetch-readmes-and-process.rs

@@ -63,7 +63,12 @@ async fn get_readme(
             )
             .await;
             let (_, content) = codeberg::helper_functions::get_readme_url(
-                &owner, &name, &branch, false, true, &directory_files,
+                &owner,
+                &name,
+                &branch,
+                false,
+                true,
+                &directory_files,
             )
             .await;
             content