diff --git a/collection.go b/collection.go index 6e9d2b9..96d5ff4 100644 --- a/collection.go +++ b/collection.go @@ -379,28 +379,28 @@ func (c *Collection) Close() error { // --------------------------- Primary Key ---------------------------- // InsertKey inserts a row given its corresponding primary key. -func (c *Collection) InsertKey(key string, fn func(Row) error) error { +func (c *Collection) InsertKey(key int64, fn func(Row) error) error { return c.Query(func(txn *Txn) error { return txn.InsertKey(key, fn) }) } // UpsertKey inserts or updates a row given its corresponding primary key. -func (c *Collection) UpsertKey(key string, fn func(Row) error) error { +func (c *Collection) UpsertKey(key int64, fn func(Row) error) error { return c.Query(func(txn *Txn) error { return txn.UpsertKey(key, fn) }) } // QueryKey queries/updates a row given its corresponding primary key. -func (c *Collection) QueryKey(key string, fn func(Row) error) error { +func (c *Collection) QueryKey(key int64, fn func(Row) error) error { return c.Query(func(txn *Txn) error { return txn.QueryKey(key, fn) }) } // DeleteKey deletes a row for a given primary key. -func (c *Collection) DeleteKey(key string) error { +func (c *Collection) DeleteKey(key int64) error { return c.Query(func(txn *Txn) error { return txn.DeleteKey(key) }) diff --git a/column_keys.go b/column_keys.go new file mode 100644 index 0000000..c80f4f4 --- /dev/null +++ b/column_keys.go @@ -0,0 +1,99 @@ +// Copyright (c) Roman Atachiants and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for details. + +package column + +import ( + "fmt" + "sync" + + "github.com/kelindar/column/commit" +) + +// --------------------------- Key ---------------------------- + +// columnKey represents the primary key column implementation +type columnKey struct { + numericColumn[int64] + name string // Name of the column + lock sync.RWMutex // Lock to protect the lookup table + seek map[int64]uint32 // Lookup table for O(1) index seek +} + +// makeKey creates a new primary key column +func makeKey() Column { + col := makeInt64s().(*numericColumn[int64]) + return &columnKey{ + seek: make(map[int64]uint32, 64), + numericColumn: *col, + } +} + +// Apply applies a set of operations to the column. +func (c *columnKey) Apply(chunk commit.Chunk, r *commit.Reader) { + fill, data := c.chunkAt(chunk) + from := chunk.Min() + + for r.Next() { + offset := r.Offset - int32(from) + switch r.Type { + case commit.Put: + value := r.Int64() + + fill[offset>>6] |= 1 << (offset & 0x3f) + data[offset] = value + c.lock.Lock() + c.seek[value] = uint32(r.Offset) + c.lock.Unlock() + + case commit.Delete: + fill.Remove(uint32(offset)) + c.lock.Lock() + delete(c.seek, data[offset]) + c.lock.Unlock() + } + } +} + +// OffsetOf returns the offset for a particular value +func (c *columnKey) OffsetOf(v int64) (uint32, bool) { + c.lock.RLock() + idx, ok := c.seek[v] + c.lock.RUnlock() + return idx, ok +} + +// rwKey represents read-write accessor for primary keys. +type rwKey struct { + cursor *uint32 + writer *commit.Buffer + reader *columnKey +} + +// Set sets the value at the current transaction index +func (s rwKey) Set(value int64) error { + if _, ok := s.reader.OffsetOf(value); !ok { + s.writer.PutInt64(commit.Put, *s.cursor, value) + return nil + } + + return fmt.Errorf("column: unable to set duplicate key '%d'", value) +} + +// Get loads the value at the current transaction index +func (s rwKey) Get() (int64, bool) { + return s.reader.LoadInt64(*s.cursor) +} + +// Enum returns a enumerable column accessor +func (txn *Txn) Key() rwKey { + if txn.owner.pk == nil { + panic(fmt.Errorf("column: primary key column does not exist")) + } + + return rwKey{ + cursor: &txn.cursor, + writer: txn.bufferFor(txn.owner.pk.name), + reader: txn.owner.pk, + } +} diff --git a/column_strings.go b/column_strings.go index ccc5422..29e00ff 100644 --- a/column_strings.go +++ b/column_strings.go @@ -4,9 +4,7 @@ package column import ( - "fmt" "math" - "sync" "github.com/kelindar/bitmap" "github.com/kelindar/column/commit" @@ -256,95 +254,6 @@ func (txn *Txn) String(columnName string) rwString { } } -// --------------------------- Key ---------------------------- - -// columnKey represents the primary key column implementation -type columnKey struct { - columnString - name string // Name of the column - lock sync.RWMutex // Lock to protect the lookup table - seek map[string]uint32 // Lookup table for O(1) index seek -} - -// makeKey creates a new primary key column -func makeKey() Column { - return &columnKey{ - seek: make(map[string]uint32, 64), - columnString: columnString{ - chunks: make(chunks[string], 0, 4), - }, - } -} - -// Apply applies a set of operations to the column. -func (c *columnKey) Apply(chunk commit.Chunk, r *commit.Reader) { - fill, data := c.chunkAt(chunk) - from := chunk.Min() - - for r.Next() { - offset := r.Offset - int32(from) - switch r.Type { - case commit.Put: - value := string(r.Bytes()) - - fill[offset>>6] |= 1 << (offset & 0x3f) - data[offset] = value - c.lock.Lock() - c.seek[value] = uint32(r.Offset) - c.lock.Unlock() - - case commit.Delete: - fill.Remove(uint32(offset)) - c.lock.Lock() - delete(c.seek, string(data[offset])) - c.lock.Unlock() - } - } -} - -// OffsetOf returns the offset for a particular value -func (c *columnKey) OffsetOf(v string) (uint32, bool) { - c.lock.RLock() - idx, ok := c.seek[v] - c.lock.RUnlock() - return idx, ok -} - -// rwKey represents read-write accessor for primary keys. -type rwKey struct { - cursor *uint32 - writer *commit.Buffer - reader *columnKey -} - -// Set sets the value at the current transaction index -func (s rwKey) Set(value string) error { - if _, ok := s.reader.OffsetOf(value); !ok { - s.writer.PutString(commit.Put, *s.cursor, value) - return nil - } - - return fmt.Errorf("column: unable to set duplicate key '%s'", value) -} - -// Get loads the value at the current transaction index -func (s rwKey) Get() (string, bool) { - return s.reader.LoadString(*s.cursor) -} - -// Enum returns a enumerable column accessor -func (txn *Txn) Key() rwKey { - if txn.owner.pk == nil { - panic(fmt.Errorf("column: primary key column does not exist")) - } - - return rwKey{ - cursor: &txn.cursor, - writer: txn.bufferFor(txn.owner.pk.name), - reader: txn.owner.pk, - } -} - // --------------------------- Reader ---------------------------- // rdString represents a read-only accessor for strings diff --git a/column_test.go b/column_test.go index b681886..81b2e4c 100644 --- a/column_test.go +++ b/column_test.go @@ -255,7 +255,7 @@ func TestForKindInvalid(t *testing.T) { } func TestAtKey(t *testing.T) { - const testKey = "key=20" + const testKey = 20 // Update a name players := loadPlayers(500) @@ -263,7 +263,7 @@ func TestAtKey(t *testing.T) { assert.NoError(t, players.Query(func(txn *Txn) error { pk := txn.Key() return txn.Range(func(idx uint32) { - pk.Set(fmt.Sprintf("key=%d", idx)) + pk.Set(int64(idx)) }) })) @@ -290,7 +290,7 @@ func TestAtKey(t *testing.T) { func TestUpdateAtKeyWithoutPK(t *testing.T) { col := NewCollection() - assert.Error(t, col.QueryKey("test", func(r Row) error { + assert.Error(t, col.QueryKey(1234, func(r Row) error { r.SetEnum("name", "Roman") return nil })) @@ -298,23 +298,23 @@ func TestUpdateAtKeyWithoutPK(t *testing.T) { func TestSelectAtKeyWithoutPK(t *testing.T) { col := NewCollection() - assert.Error(t, col.QueryKey("test", func(r Row) error { return nil })) - assert.Error(t, col.InsertKey("test", func(r Row) error { return nil })) - assert.Error(t, col.UpsertKey("test", func(r Row) error { return nil })) - assert.Error(t, col.DeleteKey("test")) + assert.Error(t, col.QueryKey(1234, func(r Row) error { return nil })) + assert.Error(t, col.InsertKey(1234, func(r Row) error { return nil })) + assert.Error(t, col.UpsertKey(1234, func(r Row) error { return nil })) + assert.Error(t, col.DeleteKey(1234)) } func TestBulkUpdateDuplicatePK(t *testing.T) { col := NewCollection() col.CreateColumn("key", ForKey()) - assert.NoError(t, col.InsertKey("1", func(r Row) error { return nil })) - assert.NoError(t, col.InsertKey("2", func(r Row) error { return nil })) + assert.NoError(t, col.InsertKey(1, func(r Row) error { return nil })) + assert.NoError(t, col.InsertKey(2, func(r Row) error { return nil })) // If we attempt to change to an already persisted key, we should get an error assert.NoError(t, col.Query(func(txn *Txn) error { pk := txn.Key() - assert.Error(t, txn.QueryKey("1", func(Row) error { - return pk.Set("2") + assert.Error(t, txn.QueryKey(1, func(Row) error { + return pk.Set(2) })) return nil })) @@ -489,8 +489,9 @@ func TestPKAccessor(t *testing.T) { col := NewCollection() assert.NoError(t, col.CreateColumn("name", ForKey())) + const Roman = 1234 // Insert a primary key value - err := col.InsertKey("Roman", func(r Row) error { + err := col.InsertKey(Roman, func(r Row) error { return nil }) assert.NoError(t, err) @@ -499,7 +500,7 @@ func TestPKAccessor(t *testing.T) { col.QueryAt(0, func(r Row) error { value, ok := r.txn.Key().Get() assert.True(t, ok) - assert.Equal(t, "Roman", value) + assert.Equal(t, Roman, value) return nil }) } @@ -529,13 +530,14 @@ func TestDuplicatePK(t *testing.T) { col := NewCollection() assert.NoError(t, col.CreateColumn("name", ForKey())) + const Roman = 1234 // Insert a primary key value - assert.NoError(t, col.InsertKey("Roman", func(r Row) error { + assert.NoError(t, col.InsertKey(Roman, func(r Row) error { return nil })) // Insert a duplicate - assert.Error(t, col.InsertKey("Roman", func(r Row) error { + assert.Error(t, col.InsertKey(Roman, func(r Row) error { return nil })) diff --git a/examples/bench/bench.go b/examples/bench/bench.go index 86b71a3..2a8cf6e 100644 --- a/examples/bench/bench.go +++ b/examples/bench/bench.go @@ -6,6 +6,7 @@ package main import ( "context" "fmt" + "runtime" "sync" "sync/atomic" "time" @@ -23,7 +24,7 @@ var ( ) func main() { - amount := 1000000 + amount := 1_000_000 players := column.NewCollection(column.Options{ Capacity: amount, }) @@ -59,10 +60,10 @@ func main() { func runBenchmark(name string, fn func(bool) (int, int)) { fmt.Printf("Benchmarking %v ...\n", name) fmt.Printf("%7v\t%6v\t%17v\t%13v\n", "WORK", "PROCS", "READ RATE", "WRITE RATE") - for _, workload := range []int{0, 10, 50, 90, 100} { + for _, workload := range []int{0, 10, 25, 50, 75, 90, 100} { // Iterate over various concurrency levels - for _, n := range []int{1, 2, 4, 8, 16, 32, 64, 128, 256, 512} { + for n := 1; n <= runtime.NumCPU()*2; n *= 2 { work := make(chan async.Task, n) pool := async.Consume(context.Background(), n, work) diff --git a/examples/cache/cache.go b/examples/cache/cache.go index 1aec6f9..dffce8c 100644 --- a/examples/cache/cache.go +++ b/examples/cache/cache.go @@ -24,7 +24,7 @@ func New() *Cache { } // Get attempts to retrieve a value for a key -func (c *Cache) Get(key string) (value string, found bool) { +func (c *Cache) Get(key int64) (value string, found bool) { c.store.QueryKey(key, func(r column.Row) error { value, found = r.String("val") return nil @@ -33,7 +33,7 @@ func (c *Cache) Get(key string) (value string, found bool) { } // Set updates or inserts a new value -func (c *Cache) Set(key, value string) { +func (c *Cache) Set(key int64, value string) { if err := c.store.UpsertKey(key, func(r column.Row) error { r.SetString("val", value) return nil diff --git a/examples/cache/main.go b/examples/cache/main.go index 6cbb88c..49110fc 100644 --- a/examples/cache/main.go +++ b/examples/cache/main.go @@ -17,7 +17,7 @@ func main() { measure("insert", fmt.Sprintf("%v rows", amount), func() { for i := 0; i < amount; i++ { - key := fmt.Sprintf("user_%d", i) + key := int64(i) val := fmt.Sprintf("Hi, User %d", i) cache.Set(key, val) @@ -27,8 +27,8 @@ func main() { } }, 1) - key := fmt.Sprintf("user_%d", xxrand.Intn(amount)) - measure("query", key, func() { + key := int64(xxrand.Intn(amount)) + measure("query", fmt.Sprint(key), func() { xxrand.Intn(amount) fmt.Println(cache.Get(key)) }, 100000) diff --git a/examples/simple/main.go b/examples/simple/main.go index c072e2c..42f0126 100644 --- a/examples/simple/main.go +++ b/examples/simple/main.go @@ -2,7 +2,10 @@ package main import ( "encoding/json" + "log" "os" + "strings" + "time" "github.com/kelindar/column" ) @@ -65,13 +68,17 @@ func main() { }) // Run an indexed query + oldMages := []string{} + start := time.Now() players.Query(func(txn *column.Txn) error { name := txn.String("name") return txn.With("human", "mage", "old").Range(func(idx uint32) { value, _ := name.Get() - println("old mage, human:", value) + oldMages = append(oldMages, value) }) }) + log.Printf("query took %s", time.Since(start)) + log.Printf("old mages, %s", strings.Join(oldMages, ", ")) } // loadFixture loads a fixture by its name @@ -92,7 +99,7 @@ func loadFixture(name string) []Player { // --------------------------- Player ---------------------------- type Player struct { - Serial string `json:"serial"` + Serial int64 `json:"serial"` Name string `json:"name"` Active bool `json:"active"` Class string `json:"class"` diff --git a/txn.go b/txn.go index 7c07665..a243c00 100644 --- a/txn.go +++ b/txn.go @@ -434,23 +434,23 @@ func (txn *Txn) DeleteAll() { // --------------------------- Primary Key ---------------------------- // InsertKey inserts a row given its corresponding primary key. -func (txn *Txn) InsertKey(key string, fn func(Row) error) error { +func (txn *Txn) InsertKey(key int64, fn func(Row) error) error { if txn.owner.pk == nil { return errNoKey } if idx, ok := txn.owner.pk.OffsetOf(key); ok { - return fmt.Errorf("column: key '%s' already exists at offset %d", key, idx) + return fmt.Errorf("column: key '%d' already exists at offset %d", key, idx) } // If not found, insert at a new index idx, err := txn.insert(fn, 0) - txn.bufferFor(txn.owner.pk.name).PutString(commit.Put, idx, key) + txn.bufferFor(txn.owner.pk.name).PutInt64(commit.Put, idx, key) return err } // UpsertKey inserts or updates a row given its corresponding primary key. -func (txn *Txn) UpsertKey(key string, fn func(Row) error) error { +func (txn *Txn) UpsertKey(key int64, fn func(Row) error) error { if txn.owner.pk == nil { return errNoKey } @@ -461,12 +461,12 @@ func (txn *Txn) UpsertKey(key string, fn func(Row) error) error { // If not found, insert at a new index idx, err := txn.insert(fn, 0) - txn.bufferFor(txn.owner.pk.name).PutString(commit.Put, idx, key) + txn.bufferFor(txn.owner.pk.name).PutInt64(commit.Put, idx, key) return err } // QueryKey queries/updates a row given its corresponding primary key. -func (txn *Txn) QueryKey(key string, fn func(Row) error) error { +func (txn *Txn) QueryKey(key int64, fn func(Row) error) error { if txn.owner.pk == nil { return errNoKey } @@ -475,11 +475,11 @@ func (txn *Txn) QueryKey(key string, fn func(Row) error) error { return txn.QueryAt(idx, fn) } - return fmt.Errorf("column: key '%s' was not found", key) + return fmt.Errorf("column: key '%d' was not found", key) } // DeleteKey deletes a row for a given primary key. -func (txn *Txn) DeleteKey(key string) error { +func (txn *Txn) DeleteKey(key int64) error { if txn.owner.pk == nil { return errNoKey } @@ -489,7 +489,7 @@ func (txn *Txn) DeleteKey(key string) error { return nil } - return fmt.Errorf("column: key '%s' was not found", key) + return fmt.Errorf("column: key '%d' was not found", key) } // --------------------------- Commit & Rollback ---------------------------- diff --git a/txn_row.go b/txn_row.go index a3b3fc6..05a97a7 100644 --- a/txn_row.go +++ b/txn_row.go @@ -175,15 +175,15 @@ func (r Row) MergeFloat64(columnName string, value float64) { // --------------------------- Strings ---------------------------- // Key loads a primary key value at a particular column -func (r Row) Key() (v string, ok bool) { +func (r Row) Key() (v int64, ok bool) { if pk := r.txn.owner.pk; pk != nil { - v, ok = pk.LoadString(r.txn.cursor) + v, ok = pk.LoadInt64(r.txn.cursor) } return } // SetKey stores a primary key value at a particular column -func (r Row) SetKey(key string) { +func (r Row) SetKey(key int64) { r.txn.Key().Set(key) } diff --git a/txn_test.go b/txn_test.go index 7ec0cca..900ce3b 100644 --- a/txn_test.go +++ b/txn_test.go @@ -689,13 +689,13 @@ func TestUpsertKey(t *testing.T) { c := NewCollection() c.CreateColumn("key", ForKey()) c.CreateColumn("val", ForString()) - assert.NoError(t, c.UpsertKey("1", func(r Row) error { + assert.NoError(t, c.UpsertKey(1, func(r Row) error { r.SetString("val", "Roman") return nil })) count := 0 - assert.NoError(t, c.UpsertKey("1", func(r Row) error { + assert.NoError(t, c.UpsertKey(1, func(r Row) error { count++ return nil })) @@ -708,7 +708,7 @@ func TestUpsertKeyNoColumn(t *testing.T) { c.CreateColumn("key", ForKey()) assert.Panics(t, func() { - c.UpsertKey("1", func(r Row) error { + c.UpsertKey(1, func(r Row) error { r.Enum("xxx") return nil }) @@ -719,14 +719,14 @@ func TestDeleteKey(t *testing.T) { c := NewCollection() c.CreateColumn("key", ForKey()) c.CreateColumn("val", ForString()) - assert.NoError(t, c.InsertKey("1", func(r Row) error { + assert.NoError(t, c.InsertKey(1, func(r Row) error { r.SetString("val", "Roman") return nil })) // Only one should succeed - assert.NoError(t, c.DeleteKey("1")) - assert.Error(t, c.DeleteKey("1")) + assert.NoError(t, c.DeleteKey(1)) + assert.Error(t, c.DeleteKey(1)) assert.Equal(t, 0, c.Count()) } @@ -735,10 +735,10 @@ func TestInsertKey(t *testing.T) { c.CreateColumn("key", ForKey()) // Only one should succeed - assert.NoError(t, c.InsertKey("1", func(r Row) error { + assert.NoError(t, c.InsertKey(1, func(r Row) error { return nil })) - assert.Error(t, c.InsertKey("1", func(r Row) error { + assert.Error(t, c.InsertKey(1, func(r Row) error { return nil })) assert.Equal(t, 1, c.Count()) @@ -749,16 +749,16 @@ func TestQueryKey(t *testing.T) { c.CreateColumn("key", ForKey()) c.CreateColumn("val", ForString()) - assert.Error(t, c.QueryKey("1", func(r Row) error { + assert.Error(t, c.QueryKey(1, func(r Row) error { return nil })) - assert.NoError(t, c.InsertKey("1", func(r Row) error { + assert.NoError(t, c.InsertKey(1, func(r Row) error { r.SetString("val", "Roman") return nil })) - assert.NoError(t, c.QueryKey("1", func(r Row) error { + assert.NoError(t, c.QueryKey(1, func(r Row) error { return nil })) } @@ -768,14 +768,14 @@ func TestChangeKey(t *testing.T) { c.CreateColumn("key", ForKey()) // Try to change the key from "1" to "2" - assert.NoError(t, c.InsertKey("1", func(r Row) error { return nil })) - assert.NoError(t, c.QueryKey("1", func(r Row) error { - r.SetKey("2") + assert.NoError(t, c.InsertKey(1, func(r Row) error { return nil })) + assert.NoError(t, c.QueryKey(1, func(r Row) error { + r.SetKey(2) return nil })) // Must now have "2" - assert.NoError(t, c.QueryKey("2", func(r Row) error { return nil })) + assert.NoError(t, c.QueryKey(2, func(r Row) error { return nil })) assert.Equal(t, 1, c.Count()) } @@ -834,7 +834,8 @@ func TestRowMethods(t *testing.T) { c.CreateColumn("float32", ForFloat32()) c.CreateColumn("float64", ForFloat64()) - c.InsertKey("key", func(r Row) error { + key := int64(1) + c.InsertKey(key, func(r Row) error { r.SetBool("bool", true) r.SetAny("name", "Roman") @@ -869,7 +870,7 @@ func TestRowMethods(t *testing.T) { assert.True(t, ok) } - c.QueryKey("key", func(r Row) error { + c.QueryKey(key, func(r Row) error { assert.True(t, r.Bool("bool")) exists(r.Key()) exists(r.Any("name")) @@ -894,8 +895,10 @@ func TestRow(t *testing.T) { var wg sync.WaitGroup wg.Add(2) + const Roman = 1234 + go c.Query(func(txn *Txn) error { - txn.InsertKey("Roman", func(r Row) error { + txn.InsertKey(Roman, func(r Row) error { return nil }) wg.Done()