Connection robustness improvements (#813)

* Clear heartbeat and queue when disconnected

* Give clearer error in case configure fails due to a lost connection. Used to throw 'Packet does not exist'

* If the queue processing error is due to a lost connection, throw it instead of looping endlessly

* In case we send a disconnection event we don't need to also throw

* Catch heartbeat errors

* Also handle invalid state errors

* Handle socket timeouts

* Log heartbeat failures

* Make linter happy

* Transform stream being a singleton prevented reconnection attempts

* Adapt tests to not using singleton

* Aborting already ends the connection

packages/core/src/meshDevice.ts

@@ -64,6 +64,11 @@ export class MeshDevice {
         this.isConfigured = true;
       } else if (status === DeviceStatusEnum.DeviceConfiguring) {
         this.isConfigured = false;
+      } else if (status === DeviceStatusEnum.DeviceDisconnected) {
+        if (this._heartbeatIntervalId !== undefined) {
+          clearInterval(this._heartbeatIntervalId);
+        }
+        this.complete();
       }
     });
 
@@ -770,7 +775,14 @@ export class MeshDevice {
       },
     });
 
-    return this.sendRaw(toBinary(Protobuf.Mesh.ToRadioSchema, toRadio));
+    return this.sendRaw(toBinary(Protobuf.Mesh.ToRadioSchema, toRadio)).catch(
+      (e) => {
+        if (this.deviceStatus === DeviceStatusEnum.DeviceDisconnected) {
+          throw new Error("Device connection lost");
+        }
+        throw e;
+      },
+    );
   }
 
   /**
@@ -797,7 +809,12 @@ export class MeshDevice {
       clearInterval(this._heartbeatIntervalId);
     }
     this._heartbeatIntervalId = setInterval(() => {
-      this.heartbeat();
+      this.heartbeat().catch((err) => {
+        this.log.error(
+          Emitter[Emitter.Ping],
+          `⚠️ Unable to send heartbeat: ${err.message}`,
+        );
+      });
     }, interval);
   }
 

packages/core/src/utils/queue.ts

@@ -117,6 +117,14 @@ export class Queue {
             await writer.write(item.data);
             item.sent = true;
           } catch (error) {
+            if (
+              error?.code === "ECONNRESET" ||
+              error?.code === "ERR_INVALID_STATE"
+            ) {
+              writer.releaseLock();
+              this.lock = false;
+              throw error;
+            }
             console.error(`Error sending packet ${item.id}`, error);
           }
         }

packages/core/src/utils/transform/toDevice.ts

@@ -1,16 +1,18 @@
 /**
  * Pads packets with appropriate framing information before writing to the output stream.
  */
-export const toDeviceStream: TransformStream<Uint8Array, Uint8Array> =
-  new TransformStream<Uint8Array, Uint8Array>({
-    transform(chunk: Uint8Array, controller): void {
-      const bufLen = chunk.length;
-      const header = new Uint8Array([
-        0x94,
-        0xc3,
-        (bufLen >> 8) & 0xff,
-        bufLen & 0xff,
-      ]);
-      controller.enqueue(new Uint8Array([...header, ...chunk]));
-    },
-  });
+export const toDeviceStream: () => TransformStream<Uint8Array, Uint8Array> =
+  () => {
+    return new TransformStream<Uint8Array, Uint8Array>({
+      transform(chunk: Uint8Array, controller): void {
+        const bufLen = chunk.length;
+        const header = new Uint8Array([
+          0x94,
+          0xc3,
+          (bufLen >> 8) & 0xff,
+          bufLen & 0xff,
+        ]);
+        controller.enqueue(new Uint8Array([...header, ...chunk]));
+      },
+    });
+  };