Start & end location timestamps (#126)

* Start & end location timestamps

* Remove comments

* endTimestamp -> non-optional

* StoredLocation Metadata

* Small refactor

* Fixes

* Fixes

* Small UI fixes

* Handle out of order timestamps

* Fix index

Author: jackpf

server/.idea/misc.xml

@@ -6,6 +6,7 @@ import com.jackpf.locationhistory.admin_service.{
   DeleteDeviceRequest,
   DeleteDeviceResponse,
   ListDevicesRequest,
+  ListLocationsRequest,
   LoginRequest,
   SendNotificationRequest,
   SendNotificationResponse
@@ -15,9 +16,11 @@ import com.jackpf.locationhistory.beacon_service.{
   RegisterDeviceRequest,
   RegisterDeviceResponse,
   RegisterPushHandlerRequest,
-  RegisterPushHandlerResponse
+  RegisterPushHandlerResponse,
+  SetLocationRequest,
+  SetLocationResponse
 }
-import com.jackpf.locationhistory.common.{Device, PushHandler}
+import com.jackpf.locationhistory.common.{Device, Location, PushHandler, StoredLocation}
 import com.jackpf.locationhistory.server.testutil.{GrpcMatchers, IntegrationTest, TestServer}
 import io.grpc.Status.Code
 import org.mockito.ArgumentMatchers.{any, anyString}
@@ -108,12 +111,10 @@ class AdminTest extends IntegrationTest with GrpcMatchers {
       }
 
       "fail to approve already approved device" >> in(new RegisteredDeviceContext {}) { context =>
-        // First approval should succeed
         context.adminClient.approveDevice(
           ApproveDeviceRequest(deviceId = context.device.id)
         ) === ApproveDeviceResponse(success = true)
 
-        // Second approval should fail
         context.adminClient.approveDevice(
           ApproveDeviceRequest(deviceId = context.device.id)
         ) must throwAGrpcRuntimeException(
@@ -200,5 +201,66 @@ class AdminTest extends IntegrationTest with GrpcMatchers {
         )
       }
     }
+
+    "list locations endpoint" >> {
+      trait ApprovedDeviceContext extends RegisteredDeviceContext {
+        adminClient.approveDevice(
+          ApproveDeviceRequest(deviceId = device.id)
+        ) === ApproveDeviceResponse(success = true)
+      }
+
+      "list locations with metadata fields" >> in(new ApprovedDeviceContext {}) { context =>
+        val timestamp = System.currentTimeMillis()
+        val location = Location(lat = 51.5007, lon = -0.1246, accuracy = 10.0)
+
+        context.client.setLocation(
+          SetLocationRequest(
+            timestamp = timestamp,
+            deviceId = context.device.id,
+            location = Some(location)
+          )
+        ) === SetLocationResponse(success = true)
+
+        val response = context.adminClient.listLocations(
+          ListLocationsRequest(deviceId = context.device.id)
+        )
+
+        response.locations must haveSize(1)
+        val storedLocation = response.locations.head
+        storedLocation.location must beSome(location)
+        storedLocation.startTimestamp === timestamp
+        storedLocation.endTimestamp === timestamp
+        storedLocation.count === 1L
+      }
+
+      "list locations with updated metadata after duplicates" >> in(new ApprovedDeviceContext {}) {
+        context =>
+          context.client.setLocation(
+            SetLocationRequest(
+              timestamp = 1000L,
+              deviceId = context.device.id,
+              location = Some(Location(lat = 51.5007, lon = -0.1246, accuracy = 10.0))
+            )
+          ) === SetLocationResponse(success = true)
+
+          context.client.setLocation(
+            SetLocationRequest(
+              timestamp = 2000L,
+              deviceId = context.device.id,
+              location = Some(Location(lat = 51.5007, lon = -0.1246, accuracy = 10.0))
+            )
+          ) === SetLocationResponse(success = true)
+
+          val response = context.adminClient.listLocations(
+            ListLocationsRequest(deviceId = context.device.id)
+          )
+
+          response.locations must haveSize(1)
+          val storedLocation = response.locations.head
+          storedLocation.startTimestamp === 1000L
+          storedLocation.endTimestamp === 2000L
+          storedLocation.count === 2L
+      }
+    }
   }
 }

server/src/it/scala/com/jackpf/locationhistory/server/AdminTest.scala

@@ -103,7 +103,9 @@ class LocationTest extends IntegrationTest with GrpcMatchers {
       listLocationsResponse.locations must haveSize(1)
       listLocationsResponse.locations.head === StoredLocation(
         location = Some(location),
-        timestamp = timestamp
+        startTimestamp = timestamp,
+        endTimestamp = timestamp,
+        count = 1L
       )
     }
 
@@ -134,11 +136,15 @@ class LocationTest extends IntegrationTest with GrpcMatchers {
       listLocationsResponse.locations === Seq(
         StoredLocation(
           location = Some(Location(lat = 51.500800, lon = -0.124500, accuracy = 0.2)),
-          timestamp = 3L
+          startTimestamp = 1L,
+          endTimestamp = 3L,
+          count = 3L
         ),
         StoredLocation(
           location = Some(Location(lat = 35.659500, lon = 139.700500, accuracy = 0.1)),
-          timestamp = 4L
+          startTimestamp = 4L,
+          endTimestamp = 4L,
+          count = 1L
         )
       )
     }

server/src/it/scala/com/jackpf/locationhistory/server/LocationTest.scala

@@ -3,13 +3,42 @@ package com.jackpf.locationhistory.server.model
 import com.jackpf.locationhistory.common.StoredLocation as ProtoStoredLocation
 
 object StoredLocation {
-  def fromLocation(location: Location, id: Long, timestamp: Long): StoredLocation =
-    StoredLocation(id, location, timestamp)
+  object Metadata {
+    def initial(timestamp: Long): Metadata =
+      Metadata(startTimestamp = timestamp, endTimestamp = timestamp, count = 1L)
+  }
+
+  case class Metadata(
+      startTimestamp: Long,
+      endTimestamp: Long,
+      count: Long
+  ) {
+    def updated(newTimestamp: Long): Metadata = {
+      // Handle mismatched newTimestamp (e.g. out-of-order events)
+      val newStartTimestamp = math.min(startTimestamp, newTimestamp)
+      val newEndTimestamp = math.max(endTimestamp, newTimestamp)
+
+      copy(startTimestamp = newStartTimestamp, endTimestamp = newEndTimestamp, count = count + 1)
+    }
+  }
+
+  def fromLocation(
+      location: Location,
+      id: Long,
+      metadata: Metadata
+  ): StoredLocation =
+    StoredLocation(id, location, metadata)
 }
 
-case class StoredLocation(id: Long, location: Location, timestamp: Long) {
+case class StoredLocation(
+    id: Long,
+    location: Location,
+    metadata: StoredLocation.Metadata
+) {
   def toProto: ProtoStoredLocation = ProtoStoredLocation(
     location = Some(location.toProto),
-    timestamp = timestamp
+    startTimestamp = metadata.startTimestamp,
+    endTimestamp = metadata.endTimestamp,
+    count = metadata.count
   )
 }

server/src/main/scala/com/jackpf/locationhistory/server/model/StoredLocation.scala

@@ -28,10 +28,10 @@ class InMemoryLocationRepo(maxItemsPerDevice: Long = DefaultMaxItemsPerDevice)
   override def storeDeviceLocation(
       deviceId: DeviceId.Type,
       location: Location,
-      timestamp: Long
+      metadata: StoredLocation.Metadata
   ): Future[Try[Unit]] = Future.successful {
     val storedLocation =
-      StoredLocation.fromLocation(location, id = generateId(), timestamp = timestamp)
+      StoredLocation.fromLocation(location, id = generateId(), metadata = metadata)
 
     storedLocations.updateWith(deviceId) {
       case Some(existingLocations) =>

server/src/main/scala/com/jackpf/locationhistory/server/repo/InMemoryLocationRepo.scala

@@ -11,7 +11,7 @@ trait LocationRepo extends LocationRepoExtensions {
   def storeDeviceLocation(
       deviceId: DeviceId.Type,
       location: Location,
-      timestamp: Long
+      metadata: StoredLocation.Metadata
   ): Future[Try[Unit]]
 
   def getForDevice(deviceId: DeviceId.Type, limit: Option[Int]): Future[Vector[StoredLocation]]

server/src/main/scala/com/jackpf/locationhistory/server/repo/LocationRepo.scala

@@ -11,14 +11,13 @@ object LocationRepoExtensions {
 }
 
 trait LocationRepoExtensions { self: LocationRepo =>
-  // TODO We might want to update an endTimestamp and count so we don't lose info of when the location was first seen
   private def updatePreviousLocation(
       newLocation: Location,
       newTimestamp: Long,
       storedLocation: StoredLocation
   ): StoredLocation = storedLocation.copy(
     location = newLocation,
-    timestamp = newTimestamp
+    metadata = storedLocation.metadata.updated(newTimestamp)
   )
 
   /** Note that this is a "best effort" approach and not strictly thread safe:
@@ -43,7 +42,7 @@ trait LocationRepoExtensions { self: LocationRepo =>
             storedLocation => updatePreviousLocation(location, timestamp, storedLocation)
         )
       case _ =>
-        storeDeviceLocation(deviceId, location, timestamp)
+        storeDeviceLocation(deviceId, location, StoredLocation.Metadata.initial(timestamp))
     }
   }
 

server/src/main/scala/com/jackpf/locationhistory/server/repo/LocationRepoExtensions.scala

@@ -19,13 +19,19 @@ private case class StoredLocationRow(
     lat: Double,
     lon: Double,
     accuracy: Double,
-    timestamp: Long,
+    startTimestamp: Long,
+    endTimestamp: Long,
+    count: Long,
     metadata: JsonColumn[Map[String, String]]
 ) {
   def toStoredLocation: StoredLocation = StoredLocation(
     id = id,
     location = Location(lat = lat, lon = lon, accuracy = accuracy, metadata.value),
-    timestamp = timestamp
+    metadata = StoredLocation.Metadata(
+      startTimestamp = startTimestamp,
+      endTimestamp = endTimestamp,
+      count = count
+    )
   )
 }
 private object StoredLocationTable extends SimpleTable[StoredLocationRow]
@@ -42,12 +48,14 @@ class SQLiteLocationRepo(db: DbClient.DataSource)(using executionContext: Execut
             lat DOUBLE,
             lon DOUBLE,
             accuracy DOUBLE,
-            timestamp UNSIGNED BIG INT,
+            start_timestamp UNSIGNED BIG INT,
+            end_timestamp UNSIGNED BIG INT,
+            count UNSIGNED BIG INT,
             metadata TEXT
           );"""
         )
         val _ = db.updateRaw(
-          """CREATE INDEX IF NOT EXISTS idx_device_time ON stored_location_table (device_id, timestamp);"""
+          """CREATE INDEX IF NOT EXISTS idx_device_time ON stored_location_table (device_id, end_timestamp);"""
         )
       }
     }
@@ -56,7 +64,7 @@ class SQLiteLocationRepo(db: DbClient.DataSource)(using executionContext: Execut
   override def storeDeviceLocation(
       deviceId: DeviceId.Type,
       location: Location,
-      timestamp: Long
+      metadata: StoredLocation.Metadata
   ): Future[Try[Unit]] = Future {
     db.transaction { implicit db =>
       Try {
@@ -67,8 +75,10 @@ class SQLiteLocationRepo(db: DbClient.DataSource)(using executionContext: Execut
               _.lat := location.lat,
               _.lon := location.lon,
               _.accuracy := location.accuracy,
-              _.timestamp := timestamp,
-              _.metadata := JsonColumn(location.metadata)
+              _.metadata := JsonColumn(location.metadata),
+              _.startTimestamp := metadata.startTimestamp,
+              _.endTimestamp := metadata.endTimestamp,
+              _.count := metadata.count
             )
           )
           ()
@@ -87,7 +97,7 @@ class SQLiteLocationRepo(db: DbClient.DataSource)(using executionContext: Execut
           {
             val q = StoredLocationTable.select
               .filter(_.deviceId === deviceId.toString)
-              .sortBy(_.timestamp)
+              .sortBy(_.endTimestamp)
               .desc
 
             limit match {
@@ -129,8 +139,10 @@ class SQLiteLocationRepo(db: DbClient.DataSource)(using executionContext: Execut
                     _.lat := updatedStoredDevice.location.lat,
                     _.lon := updatedStoredDevice.location.lon,
                     _.accuracy := updatedStoredDevice.location.accuracy,
-                    _.timestamp := updatedStoredDevice.timestamp,
-                    _.metadata := JsonColumn(updatedStoredDevice.location.metadata)
+                    _.metadata := JsonColumn(updatedStoredDevice.location.metadata),
+                    _.startTimestamp := updatedStoredDevice.metadata.startTimestamp,
+                    _.endTimestamp := updatedStoredDevice.metadata.endTimestamp,
+                    _.count := updatedStoredDevice.metadata.count
                   )
               )
             }
@@ -175,7 +187,7 @@ class SQLiteLocationRepo(db: DbClient.DataSource)(using executionContext: Execut
           val result = db.runSql[StoredLocationRow](sql"""
           SELECT * FROM (
             SELECT *,
-              ROW_NUMBER() OVER (PARTITION BY device_id ORDER BY timestamp DESC) as rn
+              ROW_NUMBER() OVER (PARTITION BY device_id ORDER BY end_timestamp DESC) as rn
             FROM stored_location_table
             WHERE device_id IN ($deviceIds)
           )

server/src/main/scala/com/jackpf/locationhistory/server/repo/SQLiteLocationRepo.scala

@@ -239,13 +239,13 @@ class AdminServiceImplTest(implicit ee: ExecutionEnv)
               1L,
               MockModels
                 .location(lat = 0.1, lon = 0.2, accuracy = 0.3, metadata = Map("k1" -> "v1")),
-              timestamp = 1L
+              model.StoredLocation.Metadata(startTimestamp = 1L, endTimestamp = 2L, count = 3L)
             ),
             MockModels.storedLocation(
               2L,
               MockModels
                 .location(lat = 0.4, lon = 0.5, accuracy = 0.6, metadata = Map("k2" -> "v2")),
-              timestamp = 2L
+              model.StoredLocation.Metadata.initial(2L)
             )
           )
         )
@@ -255,11 +255,15 @@ class AdminServiceImplTest(implicit ee: ExecutionEnv)
             Seq(
               StoredLocation(
                 Some(Location(lat = 0.1, lon = 0.2, accuracy = 0.3, metadata = Map("k1" -> "v1"))),
-                timestamp = 1L
+                startTimestamp = 1L,
+                endTimestamp = 2L,
+                count = 3L
               ),
               StoredLocation(
                 Some(Location(lat = 0.4, lon = 0.5, accuracy = 0.6, metadata = Map("k2" -> "v2"))),
-                timestamp = 2L
+                startTimestamp = 2L,
+                endTimestamp = 2L,
+                count = 1L
               )
             )
           )

server/src/test/scala/com/jackpf/locationhistory/server/grpc/AdminServiceImplTest.scala

@@ -1,6 +1,6 @@
 package com.jackpf.locationhistory.server.repo
 
-import com.jackpf.locationhistory.server.model.DeviceId
+import com.jackpf.locationhistory.server.model.{DeviceId, StoredLocation}
 import com.jackpf.locationhistory.server.testutil.MockModels
 import org.specs2.concurrent.ExecutionEnv
 
@@ -21,7 +21,7 @@ class InMemoryLocationRepoTest(implicit ee: ExecutionEnv) extends LocationRepoTe
         context.locationRepo.storeDeviceLocation(
           deviceId,
           MockModels.location(),
-          ts
+          StoredLocation.Metadata.initial(ts)
         )
 
       {
@@ -35,7 +35,7 @@ class InMemoryLocationRepoTest(implicit ee: ExecutionEnv) extends LocationRepoTe
           locations <- context.locationRepo.getForDevice(deviceId, limit = None)
         } yield {
           locations must haveSize(4)
-          locations.map(_.timestamp) must beEqualTo(
+          locations.map(_.metadata.startTimestamp) must beEqualTo(
             Seq(
               3L,
               4L,