0.32.0 - Upgrade for API V1 for Debiai

Author: Tomansion

.github/workflows/pull-request-check.yml

@@ -131,7 +131,13 @@ jobs:
   python-tests: # Install dependencies and run tests with pytest
     name: Python tests
     needs:
-      [black-format-check, prettier-check, cspell-check, version-upgrade-check]
+      [
+        black-format-check,
+        flake8-check,
+        prettier-check,
+        cspell-check,
+        version-upgrade-check,
+      ]
     runs-on: ubuntu-latest
 
     strategy:

.gitignore

@@ -27,3 +27,6 @@ deployment/
 
 # Databases
 *.db
+
+# Project files
+TODO

Dockerfile

@@ -2,6 +2,11 @@
 FROM node:16.13-alpine AS build-stage
 WORKDIR /frontend
 COPY frontend/ .
+
+# Set the environment variable for auth token local storage key
+ARG VUE_APP_AUTH_TOKEN_KEY=debiai_auth_token
+ENV VUE_APP_AUTH_TOKEN_KEY=${VUE_APP_AUTH_TOKEN_KEY}
+
 RUN npm install
 RUN npm run build
 

cspell.json

@@ -6,6 +6,7 @@
     "applymap",
     "astype",
     "bdist",
+    "blocklevels",
     "buildx",
     "cacheout",
     "ckdtree",
@@ -27,6 +28,7 @@
     "kilocode",
     "kneighbors",
     "kraskov",
+    "LOIC",
     "nats",
     "ndarray",
     "nextafter",
@@ -80,6 +82,9 @@
     "__pycache__/",
     "node_modules/",
     "./setup.py",
-    "TODO"
+    "TODO",
+    "todo",
+    "*egg-info/*",
+    "*.db"
   ]
 }

debiaiServer/api/__init__.py

@@ -0,0 +1,124 @@
+#############################################################################
+# Imports
+#############################################################################
+from debiaiServer.config.init_config import get_config
+import debiaiServer.modules.dataProviders.dataProviderManager as data_provider_manager
+from debiaiServer.modules.dataProviders.DataProviderException import (
+    DataProviderException,
+)
+from debiaiServer.api.v1.debiai.utils import make_hash
+
+
+def format_data_provider_info(data_provider):
+    data = {
+        "id": data_provider.id,
+        "status": True,
+        "name": data_provider.name,
+        "type": data_provider.type,
+        "tags": [],
+        "metadata": {},
+        "metrics": {},
+    }
+    print("NAME ", data_provider.name)
+
+    provider_info = data_provider.get_info()
+    if "canDelete" in provider_info and provider_info["canDelete"]["models"]:
+        data["tags"].append("canDeleteModels")
+    if "canDelete" in provider_info and provider_info["canDelete"]["projects"]:
+        data["tags"].append("canDeleteProject")
+    if "canDelete" in provider_info and provider_info["canDelete"]["selections"]:
+        data["tags"].append("canDeleteSelections")
+    if "version" in provider_info:
+        data["metadata"]["version"] = provider_info["version"]
+
+    if "maxResultByRequest" in provider_info:
+        data["metrics"]["maxResultByRequest"] = provider_info["maxResultByRequest"]
+    if "maxSampleDataByRequest" in provider_info:
+        data["metrics"]["maxSampleDataByRequest"] = provider_info[
+            "maxSampleDataByRequest"
+        ]
+    if "maxSampleIdByRequest" in provider_info:
+        data["metrics"]["maxSampleIdByRequest"] = provider_info["maxSampleIdByRequest"]
+
+    if data_provider.name != "Python module Data Provider":
+        data["metadata"]["external_url"] = data_provider.url
+
+    return data
+
+
+def get_data_provider_info(dataProviderId):
+
+    # As Parquet not yet supported we force json_block:
+    if dataProviderId == "json_block":
+        dataProviderId = "Python module Data Provider"
+
+    try:
+        data_provider = data_provider_manager.get_single_data_provider(dataProviderId)
+        data = format_data_provider_info(data_provider)
+
+        return data, 200
+    except DataProviderException as e:
+        return e.message, e.status_code
+
+
+def get_data_providers(prev_hash_content=None):
+    data_provider_list = data_provider_manager.get_data_provider_list()
+    providers_formatted = []
+
+    for data_provider in data_provider_list:
+        try:
+            data = format_data_provider_info(data_provider)
+        except DataProviderException as e:
+            data = {
+                "id": data_provider.id,
+                "status": False,
+                "name": data_provider.name,
+                "type": data_provider.type,
+                "tags": [],
+                "metadata": {
+                    "status_code": e.status_code,
+                    "message": e.message,
+                },
+                "metrics": {},
+            }
+
+        providers_formatted.append(data)
+
+    new_hash = "data_" + str(make_hash(providers_formatted))
+    # We add a prefix to avoir empty string
+
+    print(
+        new_hash,
+        " <=> ",
+        prev_hash_content,
+        type(new_hash),
+        type(prev_hash_content),
+        new_hash == prev_hash_content,
+    )
+    if new_hash == prev_hash_content:
+        return None, 304
+    else:
+        providers_answer = {
+            "dataproviders": providers_formatted,
+            "hash_content": new_hash,
+        }
+        return providers_answer, 200
+
+
+def delete_data_providers(dataProviderId):
+    # Check if we are allowed to add data providers from the config file
+    config = get_config()
+    deletion_allowed = config["DATA_PROVIDERS_CONFIG"]["deletion"]
+    if not deletion_allowed:
+        return "Data provider deletion is not allowed", 403
+
+    # To remove when legacy API V0 removal
+    if dataProviderId == "json_block":
+        dataProviderId = "Python module Data Provider"
+
+    # Delete data provider
+    try:
+        data_provider_manager.delete(dataProviderId)
+        return None, 204
+    except DataProviderException as e:
+        return e.message, e.status_code

debiaiServer/api/v1/__init__.py

@@ -0,0 +1,38 @@
+#############################################################################
+# Imports
+#############################################################################
+import debiaiServer.utils.layouts.layouts as layoutsUtils
+from debiaiServer.api.v1.debiai.utils import make_hash
+
+#############################################################################
+# Analysis dashboard layout Management
+#############################################################################
+
+
+def get_layouts(prev_hash_content=None):
+    layouts_overview = layoutsUtils.get_layouts()
+    for layout in layouts_overview:
+        layout.pop("dataProviderId", None)
+        print(layout)
+
+    new_hash = "layout_" + str(make_hash(layouts_overview))
+    print(new_hash, " <=> ", prev_hash_content, new_hash == prev_hash_content)
+
+    if new_hash == prev_hash_content:
+        return None, 304
+    else:
+        layouts_answer = {
+            "layouts": layouts_overview,
+            "hash_content": new_hash,
+        }
+        return layouts_answer, 200
+
+
+def post_layout(data):
+    layoutsUtils.add_layout(data)
+    return None, 204
+
+
+def delete_layout(id):
+    layoutsUtils.delete_layout(id)
+    return None, 204

debiaiServer/api/v1/debiai/__init__.py

@@ -0,0 +1,78 @@
+#############################################################################
+# Imports
+#############################################################################
+from debiaiServer.modules.dataProviders.DataProviderException import (
+    DataProviderException,
+)
+import debiaiServer.modules.dataProviders.dataProviderManager as data_provider_manager
+from debiaiServer.api.v1.debiai.utils import make_hash
+
+#############################################################################
+# PROJECTS Management
+#############################################################################
+
+
+def change_project_overview_v1(project_info):
+    v1_project_info = {
+        "id": project_info["id"],
+        "dataProviderProjectId": project_info["id"],  # same id for test
+        "name": project_info["name"],
+        "creationDate": project_info["creationDate"],
+        "updateDate": project_info["updateDate"],
+        "tags": [],
+        "metadata": {},
+        "metrics": {
+            "nbModels": project_info["nbModels"],
+            "nbSelections": project_info["nbSelections"],
+            "nbSamples": project_info["nbSamples"],
+        },
+        # "columns": column_info,
+        # projectColumns, get from statistic et suppress
+        # "blockLevelInfo": projectBlockLevel, suppress here keep
+    }
+    return v1_project_info
+
+
+def get_projects(prev_hash_content=None):
+    # Return a list of project overviews from all the data providers
+    data_providers_list = data_provider_manager.get_data_provider_list()
+    projectOverviews = {}
+    projectList = []
+    print("data_provider Get projects")
+    for data_provider in data_providers_list:
+        print(data_provider)
+        try:
+            projects = data_provider.get_projects()
+
+            if projects is not None:
+                # Adding data provider id to projects
+                for project in projects:
+                    project_inf = change_project_overview_v1(project)
+                    projectList.append(project_inf)
+
+                    if data_provider.name != "Python module Data Provider":
+                        project_inf["dataProviderId"] = data_provider.name
+                    else:
+                        project_inf["dataProviderId"] = "json_block"
+
+        except DataProviderException as e:
+            print("Warning get DP projects : " + e.message)
+
+    new_hash = "prj_" + str(
+        +make_hash(projectList)
+    )  # We add a prefix to avoir empty string
+
+    print(
+        new_hash,
+        " <=> ",
+        prev_hash_content,
+        type(new_hash),
+        type(prev_hash_content),
+        new_hash == prev_hash_content,
+    )
+    if new_hash == prev_hash_content:
+        return None, 304
+    else:
+        projectOverviews["projects"] = projectList
+        projectOverviews["hash_content"] = new_hash
+        return projectOverviews, 200