From 7b70680dad9bcdbb0a7c4c7d40eb542661846a9a Mon Sep 17 00:00:00 2001 From: Matt Cuffaro Date: Sun, 14 Sep 2025 08:16:36 -0400 Subject: [PATCH] WIP: refactoring tests --- dev.md | 10 - src/AlgebraicRelations.jl | 22 ++- src/Interface.jl | 50 ----- .../fabric/catalog.jl => fabric/Catalog.jl} | 31 ++++ src/{sqlacsets => }/fabric/Fabric.jl | 81 ++++----- src/{sqlacsets => }/fabric/acset_interface.jl | 0 .../fabric/datasources/database/DatabaseDS.jl | 7 +- .../datasources/database/acsets_interface.jl | 6 +- .../fabric/datasources/inmemory/InMemoryDS.jl | 30 ++- .../fabric/datasources/webapi/WebApiDS.jl | 0 src/{Queries.jl => fabric/query/Query.jl} | 36 +++- .../query/planning.jl} | 1 - src/{sqlacsets => }/fabric/show.jl | 0 src/sqlacsets/SQLACSets.jl | 26 --- src/sqlacsets/methods.jl | 171 ------------------ src/sqlacsets/syntax.jl | 129 ------------- .../chained_accessors.jl | 0 .../datasources/sqlite.jl | 0 test/{sqlacsets => fabric}/reflection.jl | 0 test/schemas/BusinessSchema.jl | 36 ++-- 20 files changed, 163 insertions(+), 473 deletions(-) delete mode 100644 dev.md delete mode 100644 src/Interface.jl rename src/{sqlacsets/fabric/catalog.jl => fabric/Catalog.jl} (75%) rename src/{sqlacsets => }/fabric/Fabric.jl (88%) rename src/{sqlacsets => }/fabric/acset_interface.jl (100%) rename src/{sqlacsets => }/fabric/datasources/database/DatabaseDS.jl (88%) rename src/{sqlacsets => }/fabric/datasources/database/acsets_interface.jl (97%) rename src/{sqlacsets => }/fabric/datasources/inmemory/InMemoryDS.jl (75%) rename src/{sqlacsets => }/fabric/datasources/webapi/WebApiDS.jl (100%) rename src/{Queries.jl => fabric/query/Query.jl} (72%) rename src/{sqlacsets/fabric/queryplanning.jl => fabric/query/planning.jl} (99%) rename src/{sqlacsets => }/fabric/show.jl (100%) delete mode 100644 src/sqlacsets/SQLACSets.jl delete mode 100644 src/sqlacsets/methods.jl delete mode 100644 src/sqlacsets/syntax.jl rename test/{sqlacsets => fabric}/chained_accessors.jl (100%) rename test/{sqlacsets => fabric}/datasources/sqlite.jl (100%) rename test/{sqlacsets => fabric}/reflection.jl (100%) diff --git a/dev.md b/dev.md deleted file mode 100644 index c06e174..0000000 --- a/dev.md +++ /dev/null @@ -1,10 +0,0 @@ -# Notes - - * Result of a data query is a subobject of the DSGraph inside of the Fabric. - * `decide_source` should somehow be renamed to indicate its fetching the source - * dismbiguate `reflect!`, `recatalog` and `reload` and if necessary rename to enforce consistency - * `reload` is how we reload (reconnect) the DB connection: at backend interface level - * `reconnect!`: updates the DB catalog in the backend connection - * `reflect!`: updates catalog in the fabric - * need to figure out when we open/close connections to a DB. - * get rid of `render` at Fabric.jl level diff --git a/src/AlgebraicRelations.jl b/src/AlgebraicRelations.jl index 15dff2b..919047a 100644 --- a/src/AlgebraicRelations.jl +++ b/src/AlgebraicRelations.jl @@ -4,11 +4,25 @@ using Reexport include("Schemas.jl") # include("Queries.jl") -include("sqlacsets/SQLACSets.jl") +using ACSets +using Catlab + +using MLStyle +using FunSQL +using DataFrames +using DBInterface + +# hand-rolled SQL syntax. necessary for DML operations, since FunSQL does not provide that +include("syntax.jl") + +# defines the Data Fabric concept +include("fabric/Fabric.jl") + +# the VirtualACSet +include("methods.jl") @reexport using .Schemas -# @reexport using .Queries -# query db with acsets -@reexport using .SQLACSets +@reexport using .SQLACSetSyntax +@reexport using .Fabric end diff --git a/src/Interface.jl b/src/Interface.jl deleted file mode 100644 index ad5f13b..0000000 --- a/src/Interface.jl +++ /dev/null @@ -1,50 +0,0 @@ -module Interface - -export init_tables, prepare, execute, Connection, format_form -using Catlab -using ..DB, ..Queries -using LibPQ, DataFrames -import LibPQ: Connection, Result, Statement - -""" init_tables: -This function will initialize a database with the tables required for the -given schema argument -""" -function init_tables(conn::Connection, schema) - st = generate_schema_sql(schema) - result = LibPQ.execute(conn, st) -end - -""" upload_csv: -This function uploads data from a CSV to the connected database -""" -function upload_csv(conn::Connection, table::String, filename::String) - LibPQ.execute(conn, "COPY '$table' FROM '$filename' DELIMITER ',' CSV HEADER;") -end - -# prepare: -# This function creates a prepared statement which can be executed later -# with variable inserted into the query -function prepare(conn::Connection, q::Query)::Statement - uid = LibPQ.unique_id(conn) - query, n_args = to_prepared_sql(q, uid) - res = LibPQ.execute(conn, query) - Statement(conn, uid, query, res, n_args) -end - -# execute: -# This function gets the results running the provided query on the connected -# database -function execute(conn::Connection, q::Query)::DataFrame - query = to_sql(q) - DataFrame(LibPQ.execute(conn, query)) -end - -# execute(Statement, Array): -# This function runs a prepared SQL statement using the input array as -# arguments -function execute(st::Statement, input::AbstractArray)::DataFrame - DataFrame(LibPQ.execute(st, input)) -end - -end diff --git a/src/sqlacsets/fabric/catalog.jl b/src/fabric/Catalog.jl similarity index 75% rename from src/sqlacsets/fabric/catalog.jl rename to src/fabric/Catalog.jl index af08a48..1f308bc 100644 --- a/src/sqlacsets/fabric/catalog.jl +++ b/src/fabric/Catalog.jl @@ -1,3 +1,5 @@ +module Catalog + # CATALOG emptyMaybe(x) = isempty(x) ? nothing : Some(x) @@ -24,6 +26,34 @@ function (c::Catalog)(value) c end +# TODO get_fields from Schemas +""" Catalog(p::Presentation; types::Union{Dict, Nothing}=nothing) +""" +function Catalog(p::Presentation; types::Union{Dict, Nothing}=nothing) + catalog = Catalog() + fields = get_fields(p, types) + tables = keys(fields) + table_to_index = Dict{Symbol, Int64}() + for table in tables + table_idx = add_part!(catalog, :Table, tname="$table") + add_part!(catalog, :Column, table_idx, cname="$(table)_id", type="TODO") + table_to_index[table] = table_idx + end + # ingest columns + for table in tables + table_idx = only(incident(catalog, "$table", :tname)) + for column in fields[table] + if column[1] == :Hom + column_idx = add_part!(catalog, :Column, table = table_idx, cname = "$(column[3])", type="TODO") + add_part!(catalog, :FK, to=column_idx, from=table_to_index[column[2]]) + else + type = SQLType(column[2]) + add_part!(catalog, table = table_idx, cnmame = "$(c[3])", type=type) + end + end + end +end + function table_to_fields(s::SQLSchema{T}) where T Dict([ Symbol(subpart(s, table, :tname)) => map(incident(s, table, :table)) do column @@ -85,3 +115,4 @@ end # out # end +end diff --git a/src/sqlacsets/fabric/Fabric.jl b/src/fabric/Fabric.jl similarity index 88% rename from src/sqlacsets/fabric/Fabric.jl rename to src/fabric/Fabric.jl index 519bf4b..b303138 100644 --- a/src/sqlacsets/fabric/Fabric.jl +++ b/src/fabric/Fabric.jl @@ -1,8 +1,3 @@ -module Fabric - -using ...Schemas -using ..SQLACSetSyntax - # The DataFabric is an edge-labeled graph of data sources and schema-schema interrelations # which implements the ACSet interface. It may "virtualize" data by querying it # into memory. @@ -10,6 +5,13 @@ using ..SQLACSetSyntax # ## Colimiting: # If all the data sources have known database schema, then we can assembly the # data into a single ACSet schema. +module Fabric + +using ...Schemas +using ..SQLACSetSyntax + +using Reexport + using Catlab using Catlab.Graphics.Graphviz using ACSets @@ -26,8 +28,8 @@ import FunSQL: render using PrettyTables using StructEquality -using Reexport +include("catalog.jl") function columntypes end export columntypes @@ -35,9 +37,6 @@ export columntypes struct PK end export PK -get_sqlite_schema(::Any) = [] -export get_sqlite_schema - # foreign key wrapper # TODO as_record @struct_hash_equal struct FK{T<:ACSet} @@ -58,7 +57,6 @@ export to_sql function from_sql end export from_sql -include("catalog.jl") # Data Source Graph # TODO move to Catlab. This is a labeled graph whose edges are also labeled @@ -103,39 +101,6 @@ function Catlab.Graphics.Graphviz.view_graphviz(g::DataSourceGraph) end export view_graphviz -# TODO change Any to AbstractResult -QueryResultDSGraph = DataSourceGraph{Symbol, Union{DataFrame, Nothing}, Symbol} - -struct QueryResultWrapper - qg::QueryResultDSGraph - # query -end -export QueryResultWrapper - -function QueryResultWrapper(g::DataSourceGraph) - qg = QueryResultDSGraph() - add_parts!(qg, :V, nparts(g, :V), label=subpart(g, :label)) - edges = parts(g, :E) - for e in edges - foot1 = subpart(g, e, :src) - foot2 = subpart(g, e, :tgt) - label1 = subpart(g, foot1, :label) - label2 = subpart(g, foot2, :label) - apex = add_part!(qg, :V, label=Symbol("$label1⨝$label2")) - add_parts!(qg, :E, 2, src=[apex, apex], tgt=[foot1, foot2], edgelabel=[label1, label2]) - end - QueryResultWrapper(qg) -end -export QueryResultWrapper - -# DataFabric -struct Log - time::DateTime - event - Log(event::DataType) = new(Dates.now(), event) -end -export Log - using TraitInterfaces import Catlab: ACSet @@ -143,15 +108,20 @@ import Catlab: ACSet @interface ThDataSource begin @import ACSet::TYPE + @import Presentation::TYPE @import Vector::TYPE @import AbstractString::TYPE Source::TYPE # Type of data reconnect!(s::Source)::Source # incident(s::Source, r::Row, c::Column)::Vector{Row} execute!(d::Source, stmt::AbstractString)::ACSet # TODO stmt, formatter - schema(d::Source)::ACSet + # This retrieves the source schema in terms of an ACSet + schema(d::Source)::Presentation + # Upload structured data into source. TODO should return result + upload(d::Source, table::AbstractString, filename::AbstractString)::AbstractString end export ThDataSource, reconnect!, execute! +# LibPQ.execute(conn, "COPY '$table' FROM '$filename' DELIMITER ',' CSV HEADER;") @kwdef mutable struct DataFabric # this will store the connections, their schema, and values @@ -170,6 +140,7 @@ export catalog queries(fabric::DataFabric) = fabric.queries export queries +# TODO should not export generic name like `trait` function trait end export trait @@ -195,6 +166,9 @@ TraitInterfaces.@instance ThDataSource{Source=DataFabric} [model::FabricTrait] b function schema(fabric::DataFabric) nothing end + """ TODO """ + function upload(d::Source, tabke::AbstractString, filename::AbstractString) + end end function reflect_source!(fabric::DataFabric, vs::Vector{Int}) @@ -259,16 +233,31 @@ end export add_fk! +struct PagingInfo + startIndex::Int + batchSize::Union{Int, Nothing} +end + +# DataFabric +struct Log + time::DateTime + event + Log(event::DataType) = new(Dates.now(), event) +end +export Log + + # Executing commands on data fabric """ """ function render end export render - # ACSet Interface for the Fabric. It determines which data source to dispatch the ACSet function on include("acset_interface.jl") -include("queryplanning.jl") + +# query +include("query/Query.jl") include("datasources/database/DatabaseDS.jl") include("datasources/inmemory/InMemoryDS.jl") diff --git a/src/sqlacsets/fabric/acset_interface.jl b/src/fabric/acset_interface.jl similarity index 100% rename from src/sqlacsets/fabric/acset_interface.jl rename to src/fabric/acset_interface.jl diff --git a/src/sqlacsets/fabric/datasources/database/DatabaseDS.jl b/src/fabric/datasources/database/DatabaseDS.jl similarity index 88% rename from src/sqlacsets/fabric/datasources/database/DatabaseDS.jl rename to src/fabric/datasources/database/DatabaseDS.jl index e649ae3..a5cfcc2 100644 --- a/src/sqlacsets/fabric/datasources/database/DatabaseDS.jl +++ b/src/fabric/datasources/database/DatabaseDS.jl @@ -45,6 +45,9 @@ TraitInterfaces.@instance ThDataSource{Source=DBSource} [model::DBSourceTrait] b function schema(source::DBSource) source.schema end + function upload(source::DBSource, table::AbstractString, filename::AbstractString) + nothing # TODO + end end Base.nameof(source::DBSource) = nothing @@ -58,10 +61,10 @@ function Fabric.columntypes(source::DBSource) end # TODO could probably implement `isDML(::AbstractSQLTerm) = true` for types that are -function Fabric.execute!(db::DBSource, stmt::AbstractSQLTerm, formatter=DataFrame) +function Fabric.execute!(db::DBSource, stmt::Union{DML, AbstractString}, formatter=DataFrame) # @match statement because of DBInterface.execute result = @match stmt begin - ::ACSetInsert || ::ACSetUpdate => DBInterface.execute(db.conn.raw, render(db, stmt)) + ::DML => DBInterface.execute(db.conn.raw, render(db, stmt)) _ => DBInterface.execute(db.conn, render(db, stmt)) end reconnect!(db) diff --git a/src/sqlacsets/fabric/datasources/database/acsets_interface.jl b/src/fabric/datasources/database/acsets_interface.jl similarity index 97% rename from src/sqlacsets/fabric/datasources/database/acsets_interface.jl rename to src/fabric/datasources/database/acsets_interface.jl index d880ef3..1ed8b4f 100644 --- a/src/sqlacsets/fabric/datasources/database/acsets_interface.jl +++ b/src/fabric/datasources/database/acsets_interface.jl @@ -2,6 +2,8 @@ # # ACSets Interface # #################### +# TODO DML / DML + # get the number of rows function ACSetInterface.nparts(db::DBSource, table::Symbol; formatter=identity) query = From(table) |> Group() |> Select(Agg.count()) @@ -87,7 +89,7 @@ end # add_part! function ACSetInterface.add_part!(db::DBSource, table::Symbol, values::Vector{<:NamedTuple{T}}) where T - execute!(db, ACSetInsert(table, values)) + execute!(db, DML(table, values)) end function ACSetInterface.add_part!(db::DBSource, table, value::NamedTuple{T}) where T @@ -98,7 +100,7 @@ end function ACSetInterface.set_subpart!(db::DBSource, table::Symbol, values::Vector{<:NamedTuple{T}}; wheres::Union{WhereClause, Nothing}=nothing) where T - query = execute!(db, ACSetUpdate(table, values, wheres)) + query = execute!(db, DML(table, values, wheres)) df = DataFrames.DataFrame(query); metadata!(df, "ob", table, style=:note) df end diff --git a/src/sqlacsets/fabric/datasources/inmemory/InMemoryDS.jl b/src/fabric/datasources/inmemory/InMemoryDS.jl similarity index 75% rename from src/sqlacsets/fabric/datasources/inmemory/InMemoryDS.jl rename to src/fabric/datasources/inmemory/InMemoryDS.jl index 3e4865c..02e3f93 100644 --- a/src/sqlacsets/fabric/datasources/inmemory/InMemoryDS.jl +++ b/src/fabric/datasources/inmemory/InMemoryDS.jl @@ -27,6 +27,7 @@ TraitInterfaces.@instance ThDataSource{Source=InMemory} [model::InMemoryTrait] b reconnect!(m::InMemory)::InMemory = m execute!(m::InMemory, stmt::AbstractString)::Vector{Int} = Int[] schema(m::InMemory) = DenseACSets.acset_schema(value) + upload(m::InMemory, table::AbstractString, filename::AbstractString) = nothing # TODO end Base.nameof(m::InMemory) = nameof(typeof(m.value)) @@ -40,9 +41,9 @@ function Fabric.columntypes(x::ACSet) Dict([name => attrtype_mapping[attrtype] for (name, _, attrtype) in acset_schema(x).attrs]...) end -# function DenseACSets.acset_schema(m::InMemory) -# acset_schema(m.value) -# end +function DenseACSets.acset_schema(m::InMemory) + acset_schema(m.value) +end function ACSetInterface.nparts(m::InMemory, args...) nparts(m.value, args...) @@ -84,5 +85,28 @@ function ACSetInterface.incident(m::InMemory, parts, f::T; formatter=identity) w formatter(out) end +function getattrs(m::InMemory, table::Symbol) + first.(filter(attrs(acset_schema(m))) do (attr, tbl, _) + table == tbl + end) +end +export getattrs + +function gethoms(m::InMemory, table::Symbol) + first.(homs(acset_schema(m); from=table)) +end +export gethoms + +# FIXME Set +function colnames(x::ACSet, table::Symbol) + homnames = first.(homs(acset_schema(x); from=table)) + gattrs = getattrs(x, table) + # I don't like this as it assumes the order of the columns would agree + cols = [:_id, (homnames ∪ gattrs)...] + """($(join(cols, ", ")))""" +end +export colnames + + end diff --git a/src/sqlacsets/fabric/datasources/webapi/WebApiDS.jl b/src/fabric/datasources/webapi/WebApiDS.jl similarity index 100% rename from src/sqlacsets/fabric/datasources/webapi/WebApiDS.jl rename to src/fabric/datasources/webapi/WebApiDS.jl diff --git a/src/Queries.jl b/src/fabric/query/Query.jl similarity index 72% rename from src/Queries.jl rename to src/fabric/query/Query.jl index 1fc7763..eac5c10 100644 --- a/src/Queries.jl +++ b/src/fabric/query/Query.jl @@ -1,4 +1,4 @@ -module Queries +module Query export to_funsql, SQLTable @@ -12,14 +12,6 @@ import FunSQL: SQLTable using ..Schemas -# TODO Change this to Catalog -function SQLTable(sch::SQLSchema) - Dict{Symbol, Union{SQLTable, SQLNode}}(map(parts(sch, :Table)) do t - tname = Symbol(lowercase(sch[t, :tname])) - tname => SQLTable(tname, columns = Symbol.(sch[incident(sch, t, :table), :cname])) - end) -end - # convert Rel to FunSQL function to_funsql(rel, sch::SQLSchema; queries::Dict{Symbol, SQLNode} = Dict{Symbol, SQLNode}()) to_funsql(rel, merge(SQLTable(sch), queries)) @@ -68,4 +60,30 @@ end # TODO convert FunSQL to Rel function relation(catalog::FunSQL.SQLCatalog, n::FunSQL.SQLNode) end + +# TODO change Any to AbstractResult +QueryResultDSGraph = DataSourceGraph{Symbol, Union{DataFrame, Nothing}, Symbol} + +struct QueryResultWrapper + qg::QueryResultDSGraph + # query +end +export QueryResultWrapper + +function QueryResultWrapper(g::DataSourceGraph) + qg = QueryResultDSGraph() + add_parts!(qg, :V, nparts(g, :V), label=subpart(g, :label)) + edges = parts(g, :E) + for e in edges + foot1 = subpart(g, e, :src) + foot2 = subpart(g, e, :tgt) + label1 = subpart(g, foot1, :label) + label2 = subpart(g, foot2, :label) + apex = add_part!(qg, :V, label=Symbol("$label1⨝$label2")) + add_parts!(qg, :E, 2, src=[apex, apex], tgt=[foot1, foot2], edgelabel=[label1, label2]) + end + QueryResultWrapper(qg) +end +export QueryResultWrapper + end diff --git a/src/sqlacsets/fabric/queryplanning.jl b/src/fabric/query/planning.jl similarity index 99% rename from src/sqlacsets/fabric/queryplanning.jl rename to src/fabric/query/planning.jl index d7c3fc5..02bb7c9 100644 --- a/src/sqlacsets/fabric/queryplanning.jl +++ b/src/fabric/query/planning.jl @@ -23,7 +23,6 @@ function Base.getindex(iter::PairIterator, idx::Int64) idx ≤ length(iter) ? Tuple(iter.data[idx:idx+1]) : throw(BoundsError(length(iter), idx)) end Base.lastindex(iter::PairIterator) = length(iter) -# iter[i:j] function arity(diag::UntypedNamedRelationDiagram, i::Int, label::Symbol=:junction) length(incident(diag, i, label)) diff --git a/src/sqlacsets/fabric/show.jl b/src/fabric/show.jl similarity index 100% rename from src/sqlacsets/fabric/show.jl rename to src/fabric/show.jl diff --git a/src/sqlacsets/SQLACSets.jl b/src/sqlacsets/SQLACSets.jl deleted file mode 100644 index 568cab0..0000000 --- a/src/sqlacsets/SQLACSets.jl +++ /dev/null @@ -1,26 +0,0 @@ -module SQLACSets - -using ACSets -using Catlab - -using MLStyle -using FunSQL -using DataFrames -using DBInterface - -using Reexport - -# hand-rolled SQL syntax. necessary for DML operations, since FunSQL does not provide that -include("syntax.jl") - -# defines the Data Fabric concept -include("fabric/Fabric.jl") - -# the VirtualACSet -include("methods.jl") - -@reexport using .SQLACSetSyntax -@reexport using .Fabric - - -end diff --git a/src/sqlacsets/methods.jl b/src/sqlacsets/methods.jl deleted file mode 100644 index 413bacd..0000000 --- a/src/sqlacsets/methods.jl +++ /dev/null @@ -1,171 +0,0 @@ -using .SQLACSetSyntax - -using MLStyle -using Dates -using DataFrames -import FunSQL: render - -# VAS - -struct PagingInfo - startIndex::Int - batchSize::Union{Int, Nothing} -end - -abstract type AbstractVirtualACSet end - -# ### Reading large data materialized elsewhere -@kwdef mutable struct VirtualACSet{Conn} <: AbstractVirtualACSet - conn::FunSQL.SQLConnection{Conn} - # TODO diagram of basic schemas - acsettype::Union{Type{<:ACSet}, Nothing} = nothing - # the schema is an optional value which mirrors data in the database - schema::Union{Type{<:ACSet}, Schema, Nothing} = nothing - view::Union{DataFrames.DataFrame, Nothing} = nothing -end -export VirtualACSet -# TODO we need to convert the `view` into an ACSet - -function VirtualACSet(conn::Conn) where Conn - c = FunSQL.DB(conn, catalog=FunSQL.reflect(conn)) - VirtualACSet{Conn}(conn=c) -end - -function VirtualACSet(conn::FunSQL.SQLConnection{Conn}, acs::ACSet) where Conn - VirtualACSet{Conn}(conn=conn, schema=acset_schema(acs)) -end - -function VirtualACSet(conn::Conn, x::ACSet) where {Conn<:DBInterface.Connection} - c = FunSQL.DB(conn, catalog=FunSQL.reflect(conn)) - VirtualACSet{Conn}(conn=c, schema=acset_schema(x)) -end - -# how do we know which view we are looking at? - -# Reading data distributed over many data sources. We know how the data -# should interact. - - -# blends homs and attrs together. not ideal -function namesrctgt(schema::BasicSchema) - Dict([name => src => tgt for (name, src, tgt) in schema.homs ∪ schema.attrs]) -end - -function toacset(vas::VirtualACSet{Conn}) where Conn - isnothing(vas.view) && return acset - # get columns in schema - kvs = namesrctgt(acset.schema) - # exclude primary key column - subview = vas.view[!, Not(:_id)] - cols = propertynames(subview) - ob = first.(getindex.(Ref(kvs), cols)) |> unique |> only - # instantiate - add_parts!(acset, ob, nrow(vas.view)) - for (name, column) in pairs(eachcol(subview)) - ensure_size!(acset, kvs[name].second, Int64(maximum(column))) - set_subpart!(acset, name, Int64.(vas.view[!, name])) - end - acset -end -export toacset - - - -# ###### - -function reload! end -export reload! - -function render(vas::VirtualACSet{Conn}, args...; kwargs...) where Conn end -export render - -# function execute! end -# export execute! - -# # TODO generate multiple statements, then decide to execute single or multiple -# function execute!(vas::VirtualACSet{Conn}, stmt::AbstractString; formatter::Union{Nothing, DataType, Function}=DataFrame) where Conn -# result = DBInterface.execute(vas.conn.raw, stmt) -# isnothing(formatter) && return result -# formatter(result) -# end - -# function execute!(vas::VirtualACSet{Conn}, query::AbstractSQLTerm; formatter::Union{Nothing, DataType, Function}=DataFrame) where Conn -# result = @match query begin -# # wants a prepared FunSQL statement -# ::ACSetInsert || ::ACSetUpdate || ::ACSetDelete || ::ShowTables => DBInterface.execute(vas.conn.raw, render(vas, query)) -# _ => DBInterface.execute(vas.conn, render(vas, query)) -# end -# isnothing(formatter) && return result -# formatter(result) -# # check status -# push!(vas.log, Log(typeof(query))) -# end - -function ACSet!(vas::VirtualACSet{Conn}, query::SQLTerms) where Conn - vas.view = execute!(vas, query) - ACSet(vas) -end - -## TODO - -function create!(vas::VirtualACSet{Conn}, x::ACSet) where Conn - stmt = render(vas, ACSetCreate(x)) - DBInterface.executemultiple(conn, stmt) -end -export create! - -function create!(vas::VirtualACSet{Conn}) where Conn - query = render(vas, vas.schema) - DBInterface.execute(v.conn.raw, query) -end - -function insert!(vas::VirtualACSet{Conn}, acset::ACSet) where Conn - insert_stmts = render.(Ref(vas), ACSetInsert(v.conn, acset)) - query = DBInterface.executemultiple(vas.conn.raw, insert_stmts) - DataFrames.DataFrame(query) -end - -function update!(vas::VirtualACSet, acset::ACSet) - update_stmts = render(vas, ACSetUpdate(vas.conn, acset)) - query = DBInterface.executemultiple(vas.conn.raw, update_stmts) - DataFrames.DataFrame(query) -end - -function tosql end -export tosql - -# get attrs -function getattrs(g::ACSet, table::Symbol) - first.(filter(attrs(acset_schema(g))) do (attr, tbl, _) - table == tbl - end) -end -export getattrs - -gethoms(x::ACSet, table::Symbol) = first.(homs(acset_schema(x); from=table)) -export gethoms - -# Values should have a method which turns single values into "(1)" -function getrows(vas::VirtualACSet{Conn}, x::ACSet, table::Symbol) where Conn - cols = gethoms(x, table) ∪ getattrs(x, table) - x = map(parts(x, table)) do id - (;zip([:_id, cols...], [id, tosql.(Ref(vas), subpart.(Ref(x), Ref(id), cols))...])...) - end - Values(table, x) -end -export getrows - -# FIXME Set -function colnames(x::ACSet, table::Symbol) - homnames = first.(homs(acset_schema(x); from=table)) - gattrs = getattrs(x, table) - # I don't like this as it assumes the order of the columns would agree - cols = [:_id, (homnames ∪ gattrs)...] - """($(join(cols, ", ")))""" -end -export colnames - -function wrap(stmt::String, left::String, right::String) - join([left, stmt, right], " ") -end -export wrap diff --git a/src/sqlacsets/syntax.jl b/src/sqlacsets/syntax.jl deleted file mode 100644 index 5c7b32d..0000000 --- a/src/sqlacsets/syntax.jl +++ /dev/null @@ -1,129 +0,0 @@ -module SQLACSetSyntax - -using ACSets - -using MLStyle -using DataFrames -using DBInterface -using MySQL - -function tostring end -export tostring - -tostring(conn, nothing) = "" - -# this typing ensures that named tuples have the same keys -struct Values{T} - table::Union{Symbol, Nothing} - vals::Vector{<:NamedTuple{T}} -end -export Values - -Base.length(v::Values{T}) where T = length(v.vals) -Base.iterate(v::Values{T}, args...) where T = iterate(v.vals, args...) -Base.broadcast(f, v::Values{T}) where T = Values{T}(v.table, broadcast(f, v.vals)) - -columns(v::Values{T}) where T = T -export columns - -@as_record struct WhereClause - operator::Symbol - clauses::Union{Pair{Symbol, <:Any}, Vector{<:WhereClause}} -end -export WhereClause - -@kwdef struct SQLEquation - lhs::Pair{Symbol, Symbol} - rhs::Pair{Symbol, Symbol} - op::Symbol = :(==) -end -export SQLEquation - -SQLEquation(lhs, rhs) = SQLEquation(lhs=lhs, rhs=rhs) - -# select expr from dual; -# SQLite supports SELECT * FROM A, B, C ON (A.x = B.y AND B.y = C.z) -# select * from t1 inner join - -@data SQLSelectQuantity begin - SelectAll() # default - SelectDistinct() - SelectDistinctRow() - # Pair{Symbol, Symbol} is table.column relation - SelectColumns(::Vector{Union{Symbol, Pair{Symbol, Symbol}}}) -end -export SQLSelectQuantity, SelectAll, SelectDistinct, SelectDistinctRow, SelectColumns - -SelectColumns(t::Union{Symbol, Pair{Symbol, Symbol}}) = SelectColumns([t]) -SelectColumns(varargs...) = SelectColumns([varargs...]) - -function SelectColumns(t::Vector{Pair{Symbol, Any}}) - xs = Vector{Pair{Symbol, Symbol}}() - foreach(t) do (k, v) - v isa Symbol ? push!(xs, k => v) : push!.(Ref(xs), (=>).(Ref(k), v)) - end - SelectColumns(xs) -end - -struct ACSetJoin - type::Symbol - table::Symbol - on::Union{Vector{SQLEquation}, Nothing} - function ACSetJoin(type::Symbol, table::Symbol, on::SQLEquation) - new(type, table, [on]) - end -end -export ACSetJoin - -abstract type AbstractSQLTerm end -export AbstractSQLTerm - -@data SQLTerms <: AbstractSQLTerm begin - ACSetInsert(table::Symbol, values::Values, wheres::Union{WhereClause, Nothing}) - ACSetUpdate(table::Symbol, values::Values, wheres::Union{WhereClause, Nothing}) - ACSetSelect(qty::SQLSelectQuantity, - from::Union{Symbol, Vector{Symbol}}, # TODO could be subquery - join::Union{ACSetJoin, Nothing}, - wheres::Union{ACSetSelect, WhereClause, Nothing}) - ACSetAlter(table::Symbol, refdom::Symbol, refcodom::Symbol) - ACSetCreate(schema::BasicSchema{Symbol}) - ACSetDelete(table::Symbol, ids::Vector{Int}) -end -export SQLTerms, Values, ACSetInsert, ACSetUpdate, ACSetSelect, ACSetAlter, ACSetCreate, ACSetDelete - -## Constructors - -function ACSetSelect(from::Union{Symbol, Vector{Symbol}}; - what::SQLSelectQuantity=SelectAll(), - on::Union{Vector{SQLEquation}, Nothing}=nothing, - wheres::Union{WhereClause, Nothing}=nothing) - ACSetSelect(what, from, on, wheres) -end - -function ACSetAlter(table::Symbol, arrow::Pair{Symbol, Symbol}) - ACSetAlter(table, arrow.first, arrow.second) -end - -function ACSetCreate(acset::SimpleACSet) - ACSetCreate(acset_schema(acset)) -end - -function ACSetInsert(table::Symbol, vs::Vector{<:NamedTuple{T}}, wheres::Union{WhereClause, Nothing}=nothing) where T - ACSetInsert(table, Values(table, vs), wheres) -end - -function ACSetUpdate(table::Symbol, vs::Vector{<:NamedTuple{T}}, wheres::Union{WhereClause, Nothing}=nothing) where T - ACSetUpdate(table, Values(table, vs), wheres) -end - -abstract type DatabaseEnvironmentConfig <: AbstractSQLTerm end - -struct ShowTables <: DatabaseEnvironmentConfig end -export ShowTables - -struct ForeignKeyChecks <: DatabaseEnvironmentConfig - bool::Bool -end -export ForeignKeyChecks - -end diff --git a/test/sqlacsets/chained_accessors.jl b/test/fabric/chained_accessors.jl similarity index 100% rename from test/sqlacsets/chained_accessors.jl rename to test/fabric/chained_accessors.jl diff --git a/test/sqlacsets/datasources/sqlite.jl b/test/fabric/datasources/sqlite.jl similarity index 100% rename from test/sqlacsets/datasources/sqlite.jl rename to test/fabric/datasources/sqlite.jl diff --git a/test/sqlacsets/reflection.jl b/test/fabric/reflection.jl similarity index 100% rename from test/sqlacsets/reflection.jl rename to test/fabric/reflection.jl diff --git a/test/schemas/BusinessSchema.jl b/test/schemas/BusinessSchema.jl index 4d38181..0045dfd 100644 --- a/test/schemas/BusinessSchema.jl +++ b/test/schemas/BusinessSchema.jl @@ -10,30 +10,26 @@ using DataFrames using SQLite using FunSQL: render, SQLDialect -@present Business(FreeSchema) begin - (val!Salary, Name)::AttrType +fabric = DataFabric() + +@present SchBusiness(FreeSchema) begin + (Salary, Name)::AttrType (Employee, Manager, Income, Salary)::Ob name::Attr(Employee, Name) - # - (man!employee, man!manager)::Hom(Manager, Employee) - # - inc!employee::Hom(Income, Employee) - inc!salary::Hom(Income, Salary) - # - sal!salary::Attr(Salary, val!Salary) + (employee, manager)::Hom(Manager, Employee) + employee::Hom(Income, Employee) + salary::Hom(Income, Salary) + salary::Attr(Salary, Salary) end - -busSchema = SQLSchema(Business; types = Dict(:val!Salary => Float64, :Name => String)) - - - +@acset_type Business(SchBusiness) +business = InMemory(Business{Int, Symbol}) +fabric = add_source!(fabric, business) @testset "Generate DB Schema" begin - -for stmt in splt_stmts - @test execute!(vas, stmt) isa SQLite.Query -end - + # TODO break Business part of fabric into INSERT statments + for stmt in splt_stmts + @test execute!(vas, stmt) isa SQLite.Query + end end insert_stmts = [ @@ -55,7 +51,7 @@ insert_stmts = [ "INSERT OR IGNORE INTO income (employee, salary, id) VALUES (4, 4, 4);"]; for stmt in insert_stmts - DBInterface.execute(db, stmt) + execute(fabric, stmt) end tab = SQLTable(busSchema)