Refactoring Tests

src/AlgebraicRelations.jl

@@ -4,11 +4,25 @@ using Reexport
 
 include("Schemas.jl")
 # include("Queries.jl")
-include("sqlacsets/SQLACSets.jl")
+using ACSets
+using Catlab
+
+using MLStyle
+using FunSQL
+using DataFrames
+using DBInterface
+
+# hand-rolled SQL syntax. necessary for DML operations, since FunSQL does not provide that
+include("syntax.jl") 
+
+# defines the Data Fabric concept
+include("fabric/Fabric.jl")
+
+# the VirtualACSet
+include("methods.jl")
 
 @reexport using .Schemas
-# @reexport using .Queries
-# query db with acsets
-@reexport using .SQLACSets
+@reexport using .SQLACSetSyntax
+@reexport using .Fabric
 
 end

src/sqlacsets/fabric/catalog.jl → src/fabric/Catalog.jl

@@ -1,3 +1,5 @@
+module Catalog
+
 # CATALOG
 emptyMaybe(x) = isempty(x) ? nothing : Some(x)
 
@@ -24,6 +26,34 @@ function (c::Catalog)(value)
     c
 end
 
+# TODO get_fields from Schemas
+"""    Catalog(p::Presentation; types::Union{Dict, Nothing}=nothing)
+"""
+function Catalog(p::Presentation; types::Union{Dict, Nothing}=nothing)
+    catalog = Catalog()
+    fields = get_fields(p, types)
+    tables = keys(fields)
+    table_to_index = Dict{Symbol, Int64}()
+    for table in tables
+        table_idx = add_part!(catalog, :Table, tname="$table")
+        add_part!(catalog, :Column, table_idx, cname="$(table)_id", type="TODO")
+        table_to_index[table] = table_idx
+    end
+    # ingest columns
+    for table in tables
+        table_idx = only(incident(catalog, "$table", :tname))
+        for column in fields[table]
+            if column[1] == :Hom
+                column_idx = add_part!(catalog, :Column, table = table_idx, cname = "$(column[3])", type="TODO")
+                add_part!(catalog, :FK, to=column_idx, from=table_to_index[column[2]])
+            else
+                type = SQLType(column[2])
+                add_part!(catalog, table = table_idx, cnmame = "$(c[3])", type=type)
+            end
+        end
+    end
+end
+
 function table_to_fields(s::SQLSchema{T}) where T
     Dict([ 
           Symbol(subpart(s, table, :tname)) => map(incident(s, table, :table)) do column
@@ -85,3 +115,4 @@ end
 #     out
 # end
 
+end

src/sqlacsets/fabric/Fabric.jl → src/fabric/Fabric.jl

@@ -1,15 +1,17 @@
-module Fabric
-
-using ...Schemas
-using ..SQLACSetSyntax
-
 # The DataFabric is an edge-labeled graph of data sources and schema-schema interrelations
 # which implements the ACSet interface. It may "virtualize" data by querying it
 # into memory.
 #
 # ## Colimiting: 
 # If all the data sources have known database schema, then we can assembly the
 # data into a single ACSet schema.
+module Fabric
+
+using ...Schemas
+using ..SQLACSetSyntax
+
+using Reexport
+
 using Catlab
 using Catlab.Graphics.Graphviz
 using ACSets
@@ -26,18 +28,15 @@ import FunSQL: render
 
 using PrettyTables
 using StructEquality
-using Reexport
 
+include("catalog.jl")
 
 function columntypes end
 export columntypes
 
 struct PK end
 export PK
 
-get_sqlite_schema(::Any) = []
-export get_sqlite_schema
-
 # foreign key wrapper
 # TODO as_record
 @struct_hash_equal struct FK{T<:ACSet} 
@@ -58,7 +57,6 @@ export to_sql
 function from_sql end
 export from_sql
 
-include("catalog.jl")
 # Data Source Graph
 
 # TODO move to Catlab. This is a labeled graph whose edges are also labeled
@@ -103,55 +101,27 @@ function Catlab.Graphics.Graphviz.view_graphviz(g::DataSourceGraph)
 end
 export view_graphviz
 
-# TODO change Any to AbstractResult
-QueryResultDSGraph = DataSourceGraph{Symbol, Union{DataFrame, Nothing}, Symbol}
-
-struct QueryResultWrapper
-    qg::QueryResultDSGraph
-    # query
-end
-export QueryResultWrapper
-
-function QueryResultWrapper(g::DataSourceGraph)
-    qg = QueryResultDSGraph()
-    add_parts!(qg, :V, nparts(g, :V), label=subpart(g, :label))
-    edges = parts(g, :E)
-    for e in edges
-        foot1 = subpart(g, e, :src)
-        foot2 = subpart(g, e, :tgt)
-        label1 = subpart(g, foot1, :label)
-        label2 = subpart(g, foot2, :label)
-        apex = add_part!(qg, :V, label=Symbol("$label1⨝$label2"))
-        add_parts!(qg, :E, 2, src=[apex, apex], tgt=[foot1, foot2], edgelabel=[label1, label2])
-    end
-    QueryResultWrapper(qg)
-end
-export QueryResultWrapper
-
-# DataFabric
-struct Log
-    time::DateTime
-    event
-    Log(event::DataType) = new(Dates.now(), event)
-end
-export Log
-
 using TraitInterfaces
 import Catlab: ACSet
 
 # TODO derive as a trait
 
 @interface ThDataSource begin
     @import ACSet::TYPE
+    @import Presentation::TYPE
     @import Vector::TYPE
     @import AbstractString::TYPE
     Source::TYPE # Type of data 
     reconnect!(s::Source)::Source
     # incident(s::Source, r::Row, c::Column)::Vector{Row}
     execute!(d::Source, stmt::AbstractString)::ACSet # TODO stmt, formatter
-    schema(d::Source)::ACSet
+    # This retrieves the source schema in terms of an ACSet
+    schema(d::Source)::Presentation
+    # Upload structured data into source. TODO should return result
+    upload(d::Source, table::AbstractString, filename::AbstractString)::AbstractString
 end
 export ThDataSource, reconnect!, execute!
+# LibPQ.execute(conn, "COPY '$table' FROM '$filename' DELIMITER ',' CSV HEADER;")
 
 @kwdef mutable struct DataFabric
     # this will store the connections, their schema, and values
@@ -170,6 +140,7 @@ export catalog
 queries(fabric::DataFabric) = fabric.queries
 export queries
 
+# TODO should not export generic name like `trait`
 function trait end
 export trait
 
@@ -195,6 +166,9 @@ TraitInterfaces.@instance ThDataSource{Source=DataFabric} [model::FabricTrait] b
     function schema(fabric::DataFabric)
         nothing
     end
+    """ TODO """
+    function upload(d::Source, tabke::AbstractString, filename::AbstractString)
+    end
 end
 
 function reflect_source!(fabric::DataFabric, vs::Vector{Int})
@@ -259,16 +233,31 @@ end
 export add_fk!
 
 
+struct PagingInfo 
+    startIndex::Int
+    batchSize::Union{Int, Nothing}
+end
+
+# DataFabric
+struct Log
+    time::DateTime
+    event
+    Log(event::DataType) = new(Dates.now(), event)
+end
+export Log
+
+
 # Executing commands on data fabric
 
 """ """
 function render end
 export render
 
-
 # ACSet Interface for the Fabric. It determines which data source to dispatch the ACSet function on
 include("acset_interface.jl")
-include("queryplanning.jl")
+
+# query
+include("query/Query.jl")
 
 include("datasources/database/DatabaseDS.jl")
 include("datasources/inmemory/InMemoryDS.jl")

src/sqlacsets/fabric/datasources/database/DatabaseDS.jl → src/fabric/datasources/database/DatabaseDS.jl

@@ -45,6 +45,9 @@ TraitInterfaces.@instance ThDataSource{Source=DBSource} [model::DBSourceTrait] b
     function schema(source::DBSource)
         source.schema
     end
+    function upload(source::DBSource, table::AbstractString, filename::AbstractString)
+        nothing # TODO
+    end
 end
 
 Base.nameof(source::DBSource) = nothing
@@ -58,10 +61,10 @@ function Fabric.columntypes(source::DBSource)
 end
 
 # TODO could probably implement `isDML(::AbstractSQLTerm) = true` for types that are
-function Fabric.execute!(db::DBSource, stmt::AbstractSQLTerm, formatter=DataFrame)
+function Fabric.execute!(db::DBSource, stmt::Union{DML, AbstractString}, formatter=DataFrame)
     # @match statement because of DBInterface.execute
     result = @match stmt begin
-        ::ACSetInsert || ::ACSetUpdate => DBInterface.execute(db.conn.raw, render(db, stmt))
+        ::DML => DBInterface.execute(db.conn.raw, render(db, stmt))
         _ => DBInterface.execute(db.conn, render(db, stmt))
     end
     reconnect!(db)

src/sqlacsets/fabric/datasources/database/acsets_interface.jl → src/fabric/datasources/database/acsets_interface.jl

@@ -2,6 +2,8 @@
 # # ACSets Interface
 # ####################
 
+# TODO DML / DML
+
 # get the number of rows
 function ACSetInterface.nparts(db::DBSource, table::Symbol; formatter=identity)
     query = From(table) |> Group() |> Select(Agg.count())
@@ -87,7 +89,7 @@ end
 # add_part!
 
 function ACSetInterface.add_part!(db::DBSource, table::Symbol, values::Vector{<:NamedTuple{T}}) where T 
-    execute!(db, ACSetInsert(table, values))
+    execute!(db, DML(table, values))
 end
 
 function ACSetInterface.add_part!(db::DBSource, table, value::NamedTuple{T}) where T
@@ -98,7 +100,7 @@ end
 
 function ACSetInterface.set_subpart!(db::DBSource, 
         table::Symbol, values::Vector{<:NamedTuple{T}}; wheres::Union{WhereClause, Nothing}=nothing) where T 
-    query = execute!(db, ACSetUpdate(table, values, wheres))
+    query = execute!(db, DML(table, values, wheres))
     df = DataFrames.DataFrame(query); metadata!(df, "ob", table, style=:note)
     df
 end