feat(chrome_extension): write dom changes to database (using sockets) during openadapt.record

.gitignore

@@ -15,6 +15,8 @@ cache
 *.db
 *.db-journal
 
+*.html
+
 # VSCode
 .VSCode
 .vsCode

alembic/versions/1a19ce257672_add_browserevent_message.py

@@ -0,0 +1,49 @@
+"""add BrowserEvent.message
+
+Revision ID: 1a19ce257672
+Revises: 8713b142f5de
+Create Date: 2023-08-28 17:01:07.703670
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+import openadapt
+
+# revision identifiers, used by Alembic.
+revision = "1a19ce257672"
+down_revision = "8713b142f5de"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "browser_event",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "recording_timestamp",
+            openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False),
+            nullable=True,
+        ),
+        sa.Column("message", sa.JSON(), nullable=True),
+        sa.Column(
+            "timestamp",
+            openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["recording_timestamp"],
+            ["recording.timestamp"],
+            name=op.f("fk_browser_event_recording_timestamp_recording"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_browser_event")),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("browser_event")
+    # ### end Alembic commands ###

chrome/background.js

@@ -0,0 +1,54 @@
+/**
+ * @file background.js
+ * @description This file is responsible for communicating with the native
+ * messaging host and the content script.
+ * @see https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p
+ */
+
+const hostName = "openadapt";
+var port = null; // Native Messaging port
+var lastMsg = null;
+
+/*
+ * Handle received messages from browser.js
+ */
+function onReceived(response) {
+  console.log(response);
+}
+
+function onDisconnected() {
+  msg = "Failed to connect: " + chrome.runtime.lastError.message; // silence error
+  port = null;
+}
+
+function connect() {
+  port = chrome.runtime.connectNative(hostName);
+  port.onMessage.addListener(onReceived);
+  port.onDisconnect.addListener(onDisconnected);
+}
+
+/*
+ * Message listener for content script
+ */
+function messageListener(message, sender, sendResponse) {
+  const timestampThreshold = 30; // arbitrary threshold in milliseconds
+
+  try {
+    if (lastMsg !== null) {
+      if (
+        Math.abs(message.timestamp - lastMsg.timestamp) < timestampThreshold &&
+        message.tagName === lastMsg.tagName &&
+        message.action === lastMsg.action
+      ) {
+        return;
+      }
+    }
+    console.log({ message, sender, sendResponse });
+    port.postMessage(message); // send to browser.py (native messaging host)
+    lastMsg = message;
+  } catch (e) {
+    connect();
+  }
+}
+connect();
+chrome.runtime.onMessage.addListener(messageListener);

chrome/browser.bat

@@ -0,0 +1,3 @@
+@echo off
+
+python -u "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\chrome\\browser.py"

chrome/browser.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3 # noqa: D205
+
+"""Script for communicating with the browser extension. # noqa: D205 D415
+Usage:
+    See `native_chrome_extension/browser.bat`.
+"""
+
+# Note that running python with the `-u` flag is required on Windows,
+# in order to ensure that stdin and stdout are opened in binary, rather
+# than text, mode.
+
+import json
+import sqlite3
+import struct
+import sys
+
+from openadapt import config, sockets
+
+SOCKETS = True
+DBG_DATABASE = False
+
+
+def get_message() -> dict:
+    """Read a message from stdin and decode it.
+
+    Returns:
+        A dictionary representing the decoded message.
+    """
+    raw_length = sys.stdin.buffer.read(4)
+    if len(raw_length) == 0:
+        sys.exit(0)
+    message_length = struct.unpack("@I", raw_length)[0]
+    message = sys.stdin.buffer.read(message_length).decode("utf-8")
+    return json.loads(message)
+
+
+def encode_message(message_content: any) -> dict:
+    """Encode a message for transmission, given its content.
+
+    Args:
+        message_content: The content of the message to be encoded.
+
+    Returns:
+        A dictionary containing the encoded message.
+    """
+    # https://docs.python.org/3/library/json.html#basic-usage
+    # To get the most compact JSON representation, you should specify
+    # (',', ':') to eliminate whitespace.
+    # We want the most compact representation because the browser rejects
+    # messages that exceed 1 MB.
+    encoded_content = json.dumps(message_content, separators=(",", ":")).encode("utf-8")
+    encoded_length = struct.pack("@I", len(encoded_content))
+    return {"length": encoded_length, "content": encoded_content}
+
+
+def send_message(encoded_message: dict) -> None:
+    """Send an encoded message to stdout.
+
+    Args:
+        encoded_message: The encoded message to be sent.
+    """
+    sys.stdout.buffer.write(encoded_message["length"])
+    sys.stdout.buffer.write(encoded_message["content"])
+    sys.stdout.buffer.flush()
+
+
+def send_message_to_client(conn: sockets.Connection, message: dict) -> None:
+    """Send a message to the client.
+
+    Args:
+        conn: The connection to the client.
+        message: The message to be sent.
+    """
+    try:
+        conn.send(message)
+    except Exception as exc:
+        print(f"Error sending message to client: {exc}")
+
+
+def main() -> None:
+    """Main function."""
+    if DBG_DATABASE:
+        db_conn = sqlite3.connect("messages.db")
+        c = db_conn.cursor()
+        c.execute("""
+            CREATE TABLE IF NOT EXISTS messages (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                message TEXT NOT NULL
+            )
+            """)
+    if SOCKETS:
+        conn = sockets.create_server_connection(config.SOCKET_PORT)
+    while True:
+        if SOCKETS and conn.closed:
+            conn = sockets.create_server_connection(config.SOCKET_PORT)
+        message = get_message()
+
+        # Log the message to the database
+        if DBG_DATABASE:
+            c.execute(
+                "INSERT INTO messages (message) VALUES (?)", (json.dumps(message),)
+            )
+            db_conn.commit()
+        response = {"message": "Data received and logged successfully!"}
+        if message:
+            encoded_response = encode_message(response)
+            send_message(encoded_response)
+            if SOCKETS:
+                send_message_to_client(conn, message)
+
+
+if __name__ == "__main__":
+    main()

chrome/content.js

@@ -0,0 +1,124 @@
+/**
+ * @file content.js
+ * @description This file is injected into the web page and is responsible for
+ * capturing DOM changes and sending them to the background script.
+ */
+
+let logged = false;
+let ignoreAttributes = new Set();
+const elements = {};
+
+/*
+ * Function to send a message to the background script
+ */
+function sendMessageToBackgroundScript(message) {
+  chrome.runtime.sendMessage(message);
+}
+
+/*
+ * Function to capture initial document state and
+ * send it to the background script
+ */
+function captureDocumentState() {
+  const documentBody = document.body.outerHTML;
+  const documentHead = document.head.outerHTML;
+  const page_url = window.location.href;
+
+  sendMessageToBackgroundScript({
+    action: "captureDocumentState",
+    documentBody: documentBody,
+    documentHead: documentHead,
+    elements: elements,
+    url: page_url,
+    timestamp: Date.now(),
+  });
+}
+
+function handleElementClick(event) {
+  const element = event.target;
+  const tagName = element.tagName;
+  const { x, y } = elements[element.id] || {};
+  const value = elements[element.id]?.value || "";
+  const attributes = {};
+
+  for (const attr of element.attributes) {
+    attributes[attr.name] = attr.value;
+  }
+
+  sendMessageToBackgroundScript({
+    action: "elementClicked",
+    tagName: tagName,
+    attributes: attributes,
+    x: x,
+    y: y,
+    value: value,
+    url: window.location.href,
+    timestamp: Date.now(),
+  });
+}
+
+function debounce(func, delay) {
+  let timerId;
+  return function (...args) {
+    if (timerId) {
+      clearTimeout(timerId);
+    }
+    timerId = setTimeout(() => {
+      func.apply(this, args);
+      timerId = null;
+    }, delay);
+  };
+}
+
+function handleDebouncedInput(event) {
+  const element = event.target;
+  const { x, y } = elements[element.id];
+  const value = elements[element.id].element.value;
+  const tagName = element.tagName;
+  const attributes = {};
+
+  for (const attr of element.attributes) {
+    attributes[attr.name] = attr.value;
+  }
+
+  sendMessageToBackgroundScript({
+    action: "elementInput",
+    tagName: tagName,
+    attributes: attributes,
+    x: x,
+    y: y,
+    value: value,
+    url: window.location.href,
+    timestamp: Date.now(),
+  });
+}
+
+const debouncedInputHandler = debounce(handleDebouncedInput, 500);
+
+function handleElementInput(event) {
+  debouncedInputHandler(event);
+}
+
+function addElement(element) {
+  const rect = element.getBoundingClientRect();
+  const x = rect.left + window.scrollX;
+  const y = rect.top + window.scrollY;
+  const value = element.value;
+  if (!element.id) {
+    element.id = element.tagName + "_" + x + "_" + y;
+  }
+  elements[element.id] = { element, x, y, value };
+  element.addEventListener("click", handleElementClick);
+  element.addEventListener("input", debounce(handleDebouncedInput, 500));
+}
+
+function addEventListeners() {
+  const elements = document.getElementsByTagName("*");
+
+  for (const element of elements) {
+    addElement(element);
+  }
+}
+
+addEventListeners();
+captureDocumentState();