- CM_ET/BT strictly on raw data and robust w.r.t. drop-outs and non-number readings

- better timeout handling
- prevents stacking of stock updates
- disable automatic stock updates during recording
- avoid unnecessary re-authentication in ON (black plus) mode
- lib updates

Author: MAKOMO

src/artisanlib/main.py

@@ -301,8 +301,8 @@ def stateChanged(self, state:Qt.ApplicationState) -> None:
                               aw.plus_account is not None and aw.qmc.roastUUID is not None and aw.curFile is not None):
                             plus.sync.getUpdate(aw.qmc.roastUUID, aw.curFile) # sync the loaded profile data if any
 
-                        if aw.schedule_window is not None and aw.plus_account is not None:
-                            # only if scheduler is active and plus connected we update the stock on app raise which triggers a scheduler redraw implicitly
+                        if aw.schedule_window is not None and aw.plus_account is not None and not aw.qmc.flagstart:
+                            # only if not recording, scheduler is active and plus connected we update the stock on app raise which triggers a scheduler redraw implicitly
                             # NOTE the scheduler redraw is also happening if stock was not updated due to the update request time limit
                             plus.stock.update() # stock update (frequency limited by plus/config.py:stock_cache_expiration)
 
@@ -7281,11 +7281,13 @@ def replNone(a:'npt.NDArray[numpy.double]', nv:'npt.NDArray[numpy.int64]') -> 'n
 
         return result
 
-    # computes the similarity between BT and backgroundBT as well as ET and backgroundET
+    # computes the similarity between BT and backgroundBT as well as ET and backgroundET on raw readings
     # known as CM (idea by Hungary roasting company Casino Mocca)
-    # computes from profile DRY END as set in Phases dialog through DROP
+    # computes from profile DRY END temperature as set in Phases dialog through DROP
     # returns None in case no similarity can be computed
     # refactored to use numpy arrays.
+    # NOTE: the results can differ if foreground and background curves are swapped as the DRY END of the foreground profiles determines the number of readings
+    # to be compared which might differ from that of the background profile
     def curveSimilarity(self) -> tuple[float|None, float|None]: # pylint: disable=no-self-use
         try:
             # if background profile is loaded and both profiles have a DROP event set
@@ -7294,27 +7296,16 @@ def curveSimilarity(self) -> tuple[float|None, float|None]: # pylint: disable=no
 #                _log.debug(f"curveSimilarity: {self.qmc.profile_sampling_interval=}")  #pylint: disable=logging-fstring-interpolation
 #                _log.debug(f"curveSimilarity: {self.qmc.background_profile_sampling_interval=}")  #pylint: disable=logging-fstring-interpolation
 
-                # create arrays using smoothed data if available
-                if len(self.qmc.stemp1) == len(self.qmc.temp1):
-                    # take smoothed data if available
-                    np_et = numpy.array(self.qmc.stemp1)
-                else:
-                    np_et = numpy.array(self.qmc.temp1)
-                    _log.debug('curveSimilarity: using non-smoothed ET')
-                if len(self.qmc.stemp2) == len(self.qmc.temp2):
-                    # take smoothed data if available
-                    np_bt = numpy.array(self.qmc.stemp2)
-                else:
-                    np_bt = numpy.array(self.qmc.temp2)
-                    _log.debug('curveSimilarity: using non-smoothed BT')
+                np_et = numpy.array(self.qmc.temp1)
+                np_bt = numpy.array(self.qmc.temp2)
 
                 # CM is based on the Phases Dry not marked Dry
                 # Find the DRY point
                 # create a view of the original with a stride that accesses it in reverse order
                 rev_np_bt = np_bt[::-1]
                 # Find TP or if there is not one then find the minimum temp before DROP
                 # Note - CHARGE is not considered
-                len_bt = len(self.qmc.stemp2)
+                len_bt = len(self.qmc.temp2)
                 rev_drop_idx:int = len_bt - self.qmc.timeindex[6]
                 BTlimit = self.qmc.phases[1]
                 if len(rev_np_bt[rev_drop_idx:]) == 0:
@@ -7344,25 +7335,38 @@ def curveSimilarity(self) -> tuple[float|None, float|None]: # pylint: disable=no
                 # these are not the smoothed background temps, which is how the old CM was done
                 np_etb = numpy.array(self.qmc.temp1B)
                 np_btb = numpy.array(self.qmc.temp2B)
-                np_timeB = numpy.array(self.qmc.timeB) + dropTimeDelta
-
-                # hack to work like OLD method where any temp before timeB[0] is -1
-                np_etb = numpy.insert(np_etb,0,-1)
-                np_btb = numpy.insert(np_btb,0,-1)
-                np_timeB = numpy.insert(np_timeB,0,np_timeB[0]-0.1)
-
-                interp_np_etb = numpy.interp(np_timex,np_timeB,np_etb)
-                interp_np_btb = numpy.interp(np_timex,np_timeB,np_btb)
-
-                det = numpy.sqrt(numpy.mean(numpy.square(np_et - interp_np_etb)))
-                dbt = numpy.sqrt(numpy.mean(numpy.square(np_bt - interp_np_btb)))
-
-                if numpy.isnan(det):
-                    det = None
-                if numpy.isnan(dbt):
-                    dbt = None
-
-                return det,dbt
+                np_timeB = numpy.array(self.qmc.timeB) + dropTimeDelta # shift background times such that they are aligned with foreground profile @ DROP
+
+                # we masked the -1 error values
+                np_etb_masked = numpy.ma.masked_equal(np_etb, -1) # type:ignore[no-untyped-call]
+                np_btb_masked = numpy.ma.masked_equal(np_btb, -1) # type:ignore[no-untyped-call]
+                np_timeB_etb_masked = numpy.ma.masked_array(np_timeB, np_etb_masked.mask) # type:ignore[no-untyped-call] # pylint:disable=no-member
+                np_timeB_btb_masked = numpy.ma.masked_array(np_timeB, np_btb_masked.mask) # type:ignore[no-untyped-call] # pylint:disable=no-member
+                # ignore the masked error values on computing the interpolation and fill (especially on the left) with -1 values
+                interp_np_etb = numpy.interp(np_timex,np_timeB_etb_masked.compressed(),np_etb_masked.compressed(),left=-1,right=-1) # pyright:ignore[reportUnknownArgumentType]  # pylint:disable=no-member
+                interp_np_btb = numpy.interp(np_timex,np_timeB_btb_masked.compressed(),np_btb_masked.compressed(),left=-1,right=-1) # pyright:ignore[reportUnknownArgumentType]  # pylint:disable=no-member
+
+                # at his point the background arrays interp_np_etb/interp_np_btb have the same length then their foreground counter parts np_et/np_bt
+                # however, all those errors may contain -1 error values and inf/nan readings. Let's mask them to be ignored in the computation.
+
+                # mask the -1 padding resulting from interpolating the background data as well as the inf/nan readings
+                interp_np_etb_masked = numpy.ma.masked_equal(numpy.ma.masked_invalid(interp_np_etb), -1) # type:ignore[no-untyped-call] # mask -1, inf, nan to be ignored
+                interp_np_btb_masked = numpy.ma.masked_equal(numpy.ma.masked_invalid(interp_np_btb), -1) # type:ignore[no-untyped-call] # mask -1, inf, nan to be ignored
+
+                # mask the -1 error values as well as the inf/nan readings
+                np_et_masked = numpy.ma.masked_equal(numpy.ma.masked_invalid(np_et), -1) # type:ignore[no-untyped-call] # mask -1, inf, nan to be ignored
+                np_bt_masked = numpy.ma.masked_equal(numpy.ma.masked_invalid(np_bt), -1) # type:ignore[no-untyped-call] # mask -1, inf, nan to be ignored
+
+                # all readings that are masked in the one or the other array are ignored and do not contribute in the following
+                RMSE_et = numpy.sqrt(numpy.mean(numpy.square(np_et_masked - interp_np_etb_masked))) # pyright:ignore[reportUnknownArgumentType]
+                RMSE_bt = numpy.sqrt(numpy.mean(numpy.square(np_bt_masked - interp_np_btb_masked))) # pyright:ignore[reportUnknownArgumentType]
+
+                if numpy.isnan(RMSE_et):
+                    RMSE_et = None
+                if numpy.isnan(RMSE_bt):
+                    RMSE_bt = None
+
+                return RMSE_et,RMSE_bt
 
             # no DROP event registered
             return None, None

src/artisanlib/roast_properties.py

@@ -1677,13 +1677,11 @@ def __init__(self, parent:QWidget, aw:'ApplicationWindow', activeTab:int = 0) ->
         self.stockWorker:plus.stock.Worker|None = None
         try:
             if self.aw.plus_account is not None:
-                if plus.controller.is_connected():
-                    self.stockWorker= plus.stock.getWorker()
-                    if self.stockWorker is not None:
-                        self.updateStockSignalConnection = self.stockWorker.updatedSignal.connect(self.populatePlusCoffeeBlendCombos)
-                        QTimer.singleShot(10, plus.stock.update)
-                else: # we are in ON mode, but not connected, we connect which triggers a stock update if successful
-                    plus.controller.connect(interactive=False)
+                self.stockWorker= plus.stock.getWorker()
+                if self.stockWorker is not None:
+                    self.updateStockSignalConnection = self.stockWorker.updatedSignal.connect(self.populatePlusCoffeeBlendCombos)
+                    QTimer.singleShot(10, plus.stock.update)
+
         except Exception as e:  # pylint: disable=broad-except
             _log.exception(e)
         if platform.system() != 'Windows':

src/includes/Machines/Santoker/Cube_Bluetooth.aset

@@ -53,9 +53,9 @@ buttonactions=5, 5, 5, 0, 5, 0, 5, 0
 buttonactionstrings="santoker(80,1)", "santoker(81,1)", "santoker(82,1)", , "santoker(83,1)", , "santoker(84,1)", 
 buttonvisibility=true, true, true, true, true, false, true, false
 extrabuttonactions=5, 0, 0
-extrabuttonactionstrings="santoker(85,0)", , 
-xextrabuttonactions=0, 0
-xextrabuttonactionstrings=, 
+extrabuttonactionstrings="santoker(85,0);sleep(4);santoker(80,1)", , 
+xextrabuttonactions=0, 5
+xextrabuttonactionstrings=, "santoker(85,0);sleep(4);santoker(80,1)"
 
 [Quantifiers]
 clusterEventsFlag=false

src/plus/blend.py

@@ -126,7 +126,7 @@ def __init__(self, parent:'QWidget', aw:'ApplicationWindow', inWeight:float, wei
 
         # configure UI
         self.ui = BlendDialog.Ui_customBlendDialog()
-        self.ui.setupUi(self) # OFF type:ignore[no-untyped-call]
+        self.ui.setupUi(self) # type:ignore[no-untyped-call,unused-ignore]
         self.setWindowTitle(QApplication.translate('Form Caption','Custom Blend'))
         self.ui.buttonBox.setStandardButtons(QDialogButtonBox.StandardButton.Cancel|QDialogButtonBox.StandardButton.Apply)
         # hack to assign the Apply button the AcceptRole without losing default system translations

src/plus/config.py

@@ -59,7 +59,8 @@
 #verify_ssl: Final[bool] = False
 verify_ssl: Final[bool] = True
 connect_timeout: Final[int] = 6  # in seconds
-read_timeout: Final[int] = 6  # in seconds
+read_timeout: Final[int] = 12  # in seconds
+read_timeout_max: Final[int] = 30  # in seconds
 min_passwd_len: Final[int] = 4
 min_login_len: Final[int] = 6
 compress_posts: Final[bool] = True
@@ -76,7 +77,7 @@
 # Cache and queue parameters
 
 # Note: stock_cache_expiration should be larger than schedule_cache_expiration
-stock_cache_expiration: Final[int] = 30   # expiration period in seconds for full stock updates (expensive)
+stock_cache_expiration: Final[int] = 35   # expiration period in seconds for full stock updates (expensive)
 schedule_cache_expiration: Final[int] = 5 # expiration period in seconds for full stock updates only in case the schedule on the server has changed
 
 queue_start_delay: Final[int] = 5  # startup time of queue in seconds

src/plus/connection.py

@@ -35,6 +35,7 @@
 import logging
 import dateutil.parser
 import requests
+import requests.models
 import requests.exceptions
 
 from plus import config, account, util
@@ -48,6 +49,22 @@
     1
 )  # protects access to the session token which is manipulated only here
 
+# request timeout
+
+request_read_timeout_step:Final[int] = 2 # step size to decrease request_read_timeout on success in seconds
+request_read_timeout:int = config.read_timeout # dynamic read_timeout, updated on successful communication and timeouts
+
+def getReadTimeout() -> int:
+    return request_read_timeout
+def updateReadTimeoutOnSuccess() -> None:
+    global request_read_timeout # pylint:disable=global-statement
+    request_read_timeout = max(config.read_timeout, request_read_timeout - request_read_timeout_step)
+def updateReadTimeoutOnTimeout() -> None:
+    global request_read_timeout # pylint:disable=global-statement
+    request_read_timeout = config.read_timeout_max
+
+#
+
 def getToken() -> str|None:
     try:
         token_semaphore.acquire(1)
@@ -376,95 +393,111 @@ def sendData(
     verb: str, # POST or PUT
     authorized: bool = True,
     compress: bool = config.compress_posts,
-) -> Any:
+) -> requests.models.Response:
     # don't log POST data as it might contain credentials!
     _log.debug('sendData(%s,_data_,%s,%s)', url, verb, authorized)
     jsondata = json.dumps(data, indent=None, separators=(',', ':'), ensure_ascii=False).encode('utf8')
     _log.debug('-> size %s', len(jsondata))
 #    _log.debug("PRINT jsondata: %s",jsondata)
     headers, postdata = getHeadersAndData(authorized, compress, jsondata, verb)
-    if verb == 'POST':
-        r = requests.post(
-            url,
-            headers=headers,
-            data=postdata,
-            verify=config.verify_ssl,
-            timeout=(config.connect_timeout, config.read_timeout),
-        )
-    else:
-        r = requests.put(
-            url,
-            headers=headers,
-            data=postdata,
-            verify=config.verify_ssl,
-            timeout=(config.connect_timeout, config.read_timeout),
-        )
-    _log.debug('-> status %s, time %s', r.status_code, r.elapsed.total_seconds())
-    if authorized and r.status_code == 401:  # authorisation failed
-        _log.debug('-> session token outdated (401)')
-        # we re-authentify by renewing the session token and try again
-        if authentify():
-            time.sleep(0.3) # a little delay not to stress out the server too much
-            headers, postdata = getHeadersAndData(
-                authorized, compress, jsondata, verb
-            )  # recreate header with new token
-            if verb == 'POST':
-                r = requests.post(
-                    url,
-                    headers=headers,
-                    data=postdata,
-                    verify=config.verify_ssl,
-                    timeout=(config.connect_timeout, config.read_timeout),
-                )
-            else:
-                r = requests.put(
-                    url,
-                    headers=headers,
-                    data=postdata,
-                    verify=config.verify_ssl,
-                    timeout=(config.connect_timeout, config.read_timeout),
-                )
-            _log.debug('-> status %s, time %s', r.status_code, r.elapsed.total_seconds())
-    return r
+
+    try:
+        if verb == 'POST':
+            r = requests.post(
+                url,
+                headers=headers,
+                data=postdata,
+                verify=config.verify_ssl,
+                timeout=(config.connect_timeout, getReadTimeout()),
+            )
+        else:
+            r = requests.put(
+                url,
+                headers=headers,
+                data=postdata,
+                verify=config.verify_ssl,
+                timeout=(config.connect_timeout, getReadTimeout()),
+            )
+        updateReadTimeoutOnSuccess()
+        _log.debug('-> status %s, time %s', r.status_code, r.elapsed.total_seconds())
+        if authorized and r.status_code == 401:  # authorisation failed
+            _log.debug('-> session token outdated (401)')
+            # we re-authentify by renewing the session token and try again
+            if authentify():
+                time.sleep(0.3) # a little delay not to stress out the server too much
+                headers, postdata = getHeadersAndData(
+                    authorized, compress, jsondata, verb
+                )  # recreate header with new token
+                if verb == 'POST':
+                    r = requests.post(
+                        url,
+                        headers=headers,
+                        data=postdata,
+                        verify=config.verify_ssl,
+                        timeout=(config.connect_timeout, getReadTimeout()),
+                    )
+                else:
+                    r = requests.put(
+                        url,
+                        headers=headers,
+                        data=postdata,
+                        verify=config.verify_ssl,
+                        timeout=(config.connect_timeout, getReadTimeout()),
+                    )
+                updateReadTimeoutOnSuccess()
+                _log.debug('on retry: -> status %s, time %s', r.status_code, r.elapsed.total_seconds())
+        return r
+    except requests.exceptions.Timeout as e:
+        _log.error(e)
+        updateReadTimeoutOnTimeout()
+        raise e
 
 
-def getData(url: str, authorized: bool = True, params:dict[str,str]|None = None) -> Any:
+def getData(url: str, authorized: bool = True, params:dict[str,str]|None = None) -> requests.models.Response|None:
     _log.debug('getData(%s,%s,%s)', url, authorized, params)
     headers = getHeaders(authorized)
     params = params or {}
     #    _log.debug("-> request headers %s",headers)
-    r = requests.get(
-        url,
-        headers=headers,
-        verify=config.verify_ssl,
-        params=params,
-        timeout=(config.connect_timeout, config.read_timeout),
-    )
-    _log.debug('-> status %s', r.status_code)
-    # _log.debug("-> headers %s",r.headers)
-    _log.debug('-> time %s', r.elapsed.total_seconds())
-    if authorized and r.status_code == 401:  # authorisation failed
-        _log.debug(
-            '-> session token outdated (404) - re-authentify'
-        )
-        # we re-authentify by renewing the session token and try again
-        authentify()
-        headers = getHeaders(authorized)  # recreate header with new token
-        r = requests.get(
+    try:
+        r:requests.models.Response = requests.get(
             url,
             headers=headers,
             verify=config.verify_ssl,
             params=params,
-            timeout=(config.connect_timeout, config.read_timeout),
+            timeout=(config.connect_timeout, getReadTimeout()),
         )
+        updateReadTimeoutOnSuccess()
         _log.debug('-> status %s', r.status_code)
-        #        _log.debug("-> headers %s",r.headers)
-        _log.debug(
-            '-> time %s', r.elapsed.total_seconds()
-        )
-    try:
-        _log.debug('-> size %s', len(r.content))
-#        _log.debug("-> data %s",r.json())
-    except Exception:  # pylint: disable=broad-except
-        pass
-    return r
+        # _log.debug("-> headers %s",r.headers)
+        _log.debug('-> time %s', r.elapsed.total_seconds())
+        if authorized and r.status_code == 401:  # authorisation failed
+            _log.debug(
+                '-> session token outdated (404) - re-authentify'
+            )
+            # we re-authentify by renewing the session token and try again
+            if authentify():
+                time.sleep(0.3) # a little delay not to stress out the server too much
+                headers = getHeaders(authorized)  # recreate header with new token
+                r = requests.get(
+                    url,
+                    headers=headers,
+                    verify=config.verify_ssl,
+                    params=params,
+                    timeout=(config.connect_timeout, getReadTimeout()),
+                )
+                updateReadTimeoutOnSuccess()
+                _log.debug('-> status %s', r.status_code)
+                #        _log.debug("-> headers %s",r.headers)
+                _log.debug(
+                    'on retry: -> time %s', r.elapsed.total_seconds()
+                )
+        try:
+            _log.debug('-> size %s', len(r.content))
+    #        _log.debug("-> data %s",r.json())
+        except Exception:  # pylint: disable=broad-except
+            pass
+        return r
+    except requests.exceptions.Timeout as e:
+        _log.error(e)
+        updateReadTimeoutOnTimeout()
+        return None