From 8ac5676991691c718e5992be97386e73c2ccc2d5 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 5 May 2023 19:34:21 -0400 Subject: [PATCH 01/89] Added native_chrome_extension folder --- .gitignore | 3 +++ native_chrome_extension/Icons/MLDSAI_logo.png | Bin 0 -> 4060 bytes native_chrome_extension/manifest.json | 15 +++++++++++++++ native_chrome_extension/popup.html | 12 ++++++++++++ 4 files changed, 30 insertions(+) create mode 100644 native_chrome_extension/Icons/MLDSAI_logo.png create mode 100644 native_chrome_extension/manifest.json create mode 100644 native_chrome_extension/popup.html diff --git a/.gitignore b/.gitignore index fea93ba97..4c0b428a9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ cache # db *.db + +# VSCode +*.vscode diff --git a/native_chrome_extension/Icons/MLDSAI_logo.png b/native_chrome_extension/Icons/MLDSAI_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..62627b203a1b69f276de57509baad9cd1f68aa34 GIT binary patch literal 4060 zcmcJSX*d+#+sExhnh{!zEo+o0BwP0ECOc!-&oZ`Am|{#sh_Xcz24!r?$Ye~6r80%A z(b&e2WEo6SX4F{bnV#o)^?&{Qzc}Yy=bZbTbDis)`~H6J%&QKT!h#Zl92^|N)>h_D z?6&W35#VFjjUJeC4h}$-wYiDQt+EY@u#3x5JO7VfV2gJN`i-GYP4JY-A-(j1h)%;|ZAuK{bxD&t9yCKa`HJdq-~KS+*PC-{z_pt!jB*2rTc&NX*G9qjiuGNk+ag-*-9>8-3yCACs~zkl}$ zs_?VN=?X3!X+CzYECLtV$y^ir`3vNaqupT&9MAjWZ)9V2tDZvM+M=3y;m;`_KfZ7v zDrnKiBB-bct@bOEZ|>}+3Hh}^3@d$2`OvnH`Js%BK6vCJI4KMJe}m>1V7^-Vb{E`h z9WX8)uXEE5msPyvynb`@icHd8>vU7;1;qzk+{j2L(YoBLoGR)~&)VDl=(M+~GPNqs zrb9F7TcT_>IV-bJ?Gb(MxuVLiyrk2tnyEPfaf2!s7tnjZ~QKn zkdUB98g+ASBz)Nay=(OLO7~%?mVSL*9Z@MRMUQ={fyei$ojXU?$>+`iUap_q(GSSI zk1&A;-d?gf1jSJ=Hi{Q2dmAg*7DZGxxOW%OtvN%pySoAQ6$fQ6HYqoau%5_DX;VIP zY|M&?C*okQ*lp4th}+@tGu#*O$fczuGjZ$ri}&YCHh^4~2V zwq~NL%5=D+YNGOv-(1goEEN8c*$M-r(F%WfiR#^ZFi2-Y2C9L(`j8AT)2%e>#=9s3 z-k^aZGeH+V8#CjQADHmOt@k>Gj~<0gCKyWr^OUGm&kbh!c`DIBX?Y~#vk);g`NX{} z$OlZ!i5bGq2@s9z{3%&Jv?zm{9~*>(LZ4)jxIf0Mkq<(NJQ$ZC$;l-L)X{POxt(ZEy&x&*FvfWa z2;?|Ydztw6B(DgSGiNIFVsCw@*$hz8o?JftlArU7u`$R08u9QXb94Ud-7@AxRr=mO zauqjOgOA_;8Yo5S_@jx^b+%esa=q9%GTYzP$={QB$GQ|K#R)K7~643Bl__`A@l zm$iK)(st7Bm$$jjp`k!E&kLxi*}7+oy(8mY$X@E+pEFsXUmSbKLv)IY!ffx(?_9I> zo)wB^ZYv53`=DbzgLRQiA3=c=Q_FM&{byzTuV>RY);=q;gGgI{kG;j67@%f0=(stF z6t5Mt%u{XK)?|{(9G_>Elfo`VQJBgIY5C1uznKOrPLLsFkU$gy<(+Z2nX2N1KvGhI z9V1?(&&96*A3bs{>1ajlU|PZ_!{Cxl!QqqV(7iG{MRZqJ*YWl^!t~0W zR~YV!N6r)mJZ>lPakc@dcMa4xVOnu(>k&<3qoZ{!l-NUIkgsz~*901U)_(;&6x$9G zlVQP2eGz|doPYnm(;5<~7Y>_=9Sx3qF2xnd2W`T}LZ80?>%~7lI6+cU_y)i-hEbM- zb03&=MCe=xF7_*t_l(=J2N0+kNoI(kF8iEbM=WNyeJ0>D0<}*Tgq@fE_~>9tZR=py z^N7xiNPB^Myn%rSc1Em3uZf-6m>#i)-s1cB?^{>of@LhiuE(b6Ro>{sJBDj!jmf8EkF~=aT_(O$ z$j;3Wh#blrl=Ir!H(oIvs~8{6J8q_Q`+MkNoCcz z@SDvdcp}krd+(RL-Idmk4(R|}wiqG(d?-y3Zkv9rgc67H4l4_bd6lbe39_;d-@m?R zJu58SsYidaOjt-eujT)yYL-rjjnA`J+oIFJCMK!V8ZZZkJCrq75D(E*P0Fbrx*4$g zb1AS@K`AO|(kdlok1amCWA{WjzpdybBh}4kxFsH@92M0pQtQn@Rh`tFyGmPmrN~}9 zH|L9m2L<5*am7^~9a%Y&f9xh!7RNJ=sg;>{c zf3X2mu%Wi`1g0@=Hp$1<2JBn~M@4jLyn0a#v1e-uN z7z_1KS^9o1*LjV$1AX$%@9;m;B&Ya(cHemmbPr^R?()*HhMhm>dg3uc(1MH`ZX?sMgD^L4H!%v9)(T(Gq8~o5Emz0`uQ}? zGl8)&%*a9RGA>NdQ3!M`B-4h33Yku2(?v{-B6!4d@U~A-koFL&bTHk7v^Ilt5(MOi zWTdT6bcX#zsO$~lxx)r_JlrsekFnUFkxmUW3zBGIwBZMLKcwxzW7yMEX2ErOlIaso z^Ja5%i%`<@ow}BmGRhTPBYGt9=QoS29N!O&qGw0pfOpQWu0sql9oT_gt%DC5KQ~hw zYkjQXNc9AU^1f(U*F8U9e!4TuOOA#dF7LDxI*!338p9OqjL~Lyk4OmW}om5D>9PN!aY~?o8lw zrT4)z9lB<%!{7R^iPs%2Oiu=WnrIxL8|i3iIVJ9SMTG7QqX^7{qH@rYVes!{yRu6U z4Gd=Kw2{lZDTeyq*f0`;dE0$5r|=QZF<-(&_Z`22+*eP}-2(9NT9SjkVP83%95IFl zGSD8tj31o&4B4TzwcFKf)yB?)j6{)RXh(R1aG`Rn@xy|Zg2(OvFUqpL z=W{2*9qFZ~tFGR3!LzBAaiqFNdK6EY^@gqtEBwFNr%yj)G6x4c+$Je4DybMhRz{bU znAYp}P?axbtn=jsX^@#~v z@2|U&C(?zR9N9?kMwYJ2jpoeEl8D4b3QPVzku9CA3mLYnGds(^l%U|lr0R>O0_3T& zI^>RQwn~UeZq&lS#TsQdZd}HPq1YtNYp$oTHrrOv)i_S694ck?yolUhTRJ6V6}x=` z(s3)*b)AO}Tn3)e7=8R98}uB;Vvt#*WxD;rFlGH+uF|9PgJg2SF|~wGVR0$#0-|}2 zRq1^*ii$=~zH{?~b2%-O=cU^quaqzp`W?MPqIsUL|TKl@qdA=Oy<#8)p zji!i7Mb}6d($#$Q%ox09wZlV03)Ql-yH$K1s>Z#(x#=q{GQEdDOV~?5anRP=AmE!4 z?d#}IQ$gP}=BYcf9JGHq&-{-JHdaASN@T0z8B7XUdMueSrmc1{se&cU!NFot=JaSi UIzKkqSv!Zdg@ZZ9)Ia5a01m;BI{*Lx literal 0 HcmV?d00001 diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json new file mode 100644 index 000000000..8a73efb66 --- /dev/null +++ b/native_chrome_extension/manifest.json @@ -0,0 +1,15 @@ +{ + "name": "puterbot", + "version": "1.0", + "description": "Tokenizer for ScreenShots", + "manifest_version": 3, + "permissions": [ + "nativeMessaging" + ], + "": { + "default_popup": "popup.html" + }, + "icons": { + "128": "./Icons/MLDSAI_logo.png" + } +} \ No newline at end of file diff --git a/native_chrome_extension/popup.html b/native_chrome_extension/popup.html new file mode 100644 index 000000000..324138a79 --- /dev/null +++ b/native_chrome_extension/popup.html @@ -0,0 +1,12 @@ + + + + + + + puterbot + + +

Test

+ + \ No newline at end of file From 5c7f573ef64228777b18b8b1979a3ba97f1848d1 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 5 May 2023 19:55:27 -0400 Subject: [PATCH 02/89] Implementing Issue #83 --- install_puterbot.ps1 | 30 ++++++++++++++++++++++++++++++ install_puterbot.sh | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 install_puterbot.ps1 create mode 100644 install_puterbot.sh diff --git a/install_puterbot.ps1 b/install_puterbot.ps1 new file mode 100644 index 000000000..7d90839cc --- /dev/null +++ b/install_puterbot.ps1 @@ -0,0 +1,30 @@ +# Check if Git is installed +if (!(Test-Path -Path "C:\Program Files\Git\cmd\git.exe")) { + Write-Host "Git is not installed. Installing Git..." -ForegroundColor Yellow + Start-Process -FilePath "https://git-scm.com/download/win" -Wait +} + +# Check if Python 3.10 is installed +if (!(Test-Path -Path "C:\Program Files\Python310\python.exe")) { + Write-Host "Python 3.10 is not installed. Installing Python 3.10..." -ForegroundColor Yellow + Start-Process -FilePath "https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe" -Wait +} + +# Clone the OpenAdapt repository +git clone https://github.com/MLDSAI/puterbot.git +cd puterbot + +# Create and activate a Python virtual environment +python -m venv .venv +.\.venv\Scripts\Activate.ps1 + +# Install required packages and libraries +pip install wheel +pip install -r requirements.txt +pip install -e . + +# Run the database migration +alembic upgrade head + +# Run the test suite +pytest diff --git a/install_puterbot.sh b/install_puterbot.sh new file mode 100644 index 000000000..9946ba43a --- /dev/null +++ b/install_puterbot.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Check if Git is installed +if ! [ -x "$(command -v git)" ]; then + echo "Git is not installed. Installing Git..." + sudo apt-get update + sudo apt-get install -y git +fi + +# Check if Python 3.10 is installed +if ! [ -x "$(command -v python3.10)" ]; then + echo "Python 3.10 is not installed. Installing Python 3.10..." + sudo apt-get update + sudo apt-get install -y software-properties-common + sudo add-apt-repository -y ppa:deadsnakes/ppa + sudo apt-get update + sudo apt-get install -y python3.10 python3.10-venv +fi + +# Clone the OpenAdapt repository +git clone https://github.com/MLDSAI/puterbot.git +cd puterbot + +# Create and activate a Python virtual environment +python3.10 -m venv .venv +source .venv/bin/activate + +# Install required packages and libraries +pip install wheel +pip install -r requirements.txt +pip install -e . + +# Run the database migration +alembic upgrade head + +# Run the test suite +pytest From 1572d45c05477ccaa32633c6149f53ab8d731ecf Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 5 May 2023 20:26:47 -0400 Subject: [PATCH 03/89] Minor changes for the .sh installation script --- install_puterbot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_puterbot.sh b/install_puterbot.sh index 9946ba43a..69952cc44 100644 --- a/install_puterbot.sh +++ b/install_puterbot.sh @@ -19,7 +19,7 @@ fi # Clone the OpenAdapt repository git clone https://github.com/MLDSAI/puterbot.git -cd puterbot +cd puterbot || exit # Create and activate a Python virtual environment python3.10 -m venv .venv From 224f3212baa2d649f58e3923ae067f85202eb463 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 5 May 2023 23:25:58 -0400 Subject: [PATCH 04/89] Implementing Scrubbing Issue #47 --- puterbot/scrub.py | 47 ++++++++++++++++++++++++++++++++++++ tests/puterbot/test_scrub.py | 36 +++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 puterbot/scrub.py create mode 100644 tests/puterbot/test_scrub.py diff --git a/puterbot/scrub.py b/puterbot/scrub.py new file mode 100644 index 000000000..fa2f62569 --- /dev/null +++ b/puterbot/scrub.py @@ -0,0 +1,47 @@ +import re + +def scrub(text: str) -> str: + """ + This function takes in a string of text as input and returns a scrubbed version of the text with any potential personally + identifiable information (PII) or protected health information (PHI) replaced with asterisks. + + The function replaces email addresses, phone numbers, credit card numbers, and dates of birth (or any dates) + with asterisks. It uses regular expressions to identify the patterns of these types of information in the text and + replaces them with the corresponding number of asterisks. + + Usage example: + python -m puterbot.scrub + + :param text: a string of text that may contain PII/PHI + :return: a scrubbed version of the input text with PII/PHI replaced with asterisks. + """ + # "\b = a word boundary/complete word" + scrubbed_text = text + # Replace email addresses: XYZ@XYZ.XYZ [Format] [XYZ stands for Alphanumeric characters] + scrubbed_text = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '***@***.***', scrubbed_text) + + # Replace phone numbers XXX-XXX-XXXX [Format] [X stands for A digit from 0-9] + scrubbed_text = re.sub(r'\b\d{3}[-\s]?\d{3}[-\s]?\d{4}\b', '***-***-****', scrubbed_text) + + # Replace credit card numbers XXXX-XXXX-XXXX-XXXX [Format] [X stands for A digit from 0-9] + scrubbed_text = re.sub(r'\b\d{4}-\d{4}-\d{4}-\d{4}\b', '****-****-****-****', scrubbed_text) + + # Replace dates of birth (or any Dates) mm/dd/yyyy or dd/mm/yyyy [Format] + scrubbed_text = re.sub(r'\b\d{2}/\d{2}/\d{4}\b', '**/**/****', scrubbed_text) + + # TODO: Add any other PII or PHI field for scrubbing. + + + return scrubbed_text + + +if __name__ == '__main__': + import sys + inp_text = sys.stdin.read() + scrubbed_text = scrub(inp_text) + sys.stdout.write(scrubbed_text) + + # Note: This script uses the sys.stdin.read() method to read input from the console, + # which means you'll need to + # press Ctrl + D on Linux/Mac or + # Enter then Ctrl + Z then again hit Enter, on Windows to signal the end of the input when you're done typing or pasting in your text. diff --git a/tests/puterbot/test_scrub.py b/tests/puterbot/test_scrub.py new file mode 100644 index 000000000..1a9eccf69 --- /dev/null +++ b/tests/puterbot/test_scrub.py @@ -0,0 +1,36 @@ +import pytest +from puterbot.scrub import scrub + +def test_scrub_email(): + text = "test email is test@utoronto.ca " + expected_output = "test email is ***@***.*** " + assert scrub(text) == expected_output + + +def test_scrub_phone_number(): + text = "test phone number is 123-456-7890 and my" + expected_output = "test phone number is ***-***-**** and my" + assert scrub(text) == expected_output + + +def test_scrub_credit_card_number(): + text = "test credit card number is 1234-1234-1234-1234 sf" + expected_output = "test credit card number is ****-****-****-**** sf" + assert scrub(text) == expected_output + + +def test_scrub_date_of_birth(): + text = "test date of birth is 01/01/1990" + expected_output = "test date of birth is **/**/****" + assert scrub(text) == expected_output + + +# Add tests for other fields for newly added PII/PHI in scrub.py [if any] + + +def test_all_together(): + text = "My email is john@example.com, my phone number is 123-456-7890, " \ + "my credit card number is 1234-5678-9012-3456, and my date of birth is 01/01/1990." + expected_output = "My email is ***@***.***, my phone number is ***-***-****, " \ + "my credit card number is ****-****-****-****, and my date of birth is **/**/****." + assert scrub(text) == expected_output \ No newline at end of file From 5f3dd57335a1a08983ab243ce689a7959b2669f5 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 6 May 2023 12:24:39 -0400 Subject: [PATCH 05/89] remove native folder --- native_chrome_extension/Icons/MLDSAI_logo.png | Bin 4060 -> 0 bytes native_chrome_extension/manifest.json | 15 --------------- native_chrome_extension/popup.html | 12 ------------ 3 files changed, 27 deletions(-) delete mode 100644 native_chrome_extension/Icons/MLDSAI_logo.png delete mode 100644 native_chrome_extension/manifest.json delete mode 100644 native_chrome_extension/popup.html diff --git a/native_chrome_extension/Icons/MLDSAI_logo.png b/native_chrome_extension/Icons/MLDSAI_logo.png deleted file mode 100644 index 62627b203a1b69f276de57509baad9cd1f68aa34..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4060 zcmcJSX*d+#+sExhnh{!zEo+o0BwP0ECOc!-&oZ`Am|{#sh_Xcz24!r?$Ye~6r80%A z(b&e2WEo6SX4F{bnV#o)^?&{Qzc}Yy=bZbTbDis)`~H6J%&QKT!h#Zl92^|N)>h_D z?6&W35#VFjjUJeC4h}$-wYiDQt+EY@u#3x5JO7VfV2gJN`i-GYP4JY-A-(j1h)%;|ZAuK{bxD&t9yCKa`HJdq-~KS+*PC-{z_pt!jB*2rTc&NX*G9qjiuGNk+ag-*-9>8-3yCACs~zkl}$ zs_?VN=?X3!X+CzYECLtV$y^ir`3vNaqupT&9MAjWZ)9V2tDZvM+M=3y;m;`_KfZ7v zDrnKiBB-bct@bOEZ|>}+3Hh}^3@d$2`OvnH`Js%BK6vCJI4KMJe}m>1V7^-Vb{E`h z9WX8)uXEE5msPyvynb`@icHd8>vU7;1;qzk+{j2L(YoBLoGR)~&)VDl=(M+~GPNqs zrb9F7TcT_>IV-bJ?Gb(MxuVLiyrk2tnyEPfaf2!s7tnjZ~QKn zkdUB98g+ASBz)Nay=(OLO7~%?mVSL*9Z@MRMUQ={fyei$ojXU?$>+`iUap_q(GSSI zk1&A;-d?gf1jSJ=Hi{Q2dmAg*7DZGxxOW%OtvN%pySoAQ6$fQ6HYqoau%5_DX;VIP zY|M&?C*okQ*lp4th}+@tGu#*O$fczuGjZ$ri}&YCHh^4~2V zwq~NL%5=D+YNGOv-(1goEEN8c*$M-r(F%WfiR#^ZFi2-Y2C9L(`j8AT)2%e>#=9s3 z-k^aZGeH+V8#CjQADHmOt@k>Gj~<0gCKyWr^OUGm&kbh!c`DIBX?Y~#vk);g`NX{} z$OlZ!i5bGq2@s9z{3%&Jv?zm{9~*>(LZ4)jxIf0Mkq<(NJQ$ZC$;l-L)X{POxt(ZEy&x&*FvfWa z2;?|Ydztw6B(DgSGiNIFVsCw@*$hz8o?JftlArU7u`$R08u9QXb94Ud-7@AxRr=mO zauqjOgOA_;8Yo5S_@jx^b+%esa=q9%GTYzP$={QB$GQ|K#R)K7~643Bl__`A@l zm$iK)(st7Bm$$jjp`k!E&kLxi*}7+oy(8mY$X@E+pEFsXUmSbKLv)IY!ffx(?_9I> zo)wB^ZYv53`=DbzgLRQiA3=c=Q_FM&{byzTuV>RY);=q;gGgI{kG;j67@%f0=(stF z6t5Mt%u{XK)?|{(9G_>Elfo`VQJBgIY5C1uznKOrPLLsFkU$gy<(+Z2nX2N1KvGhI z9V1?(&&96*A3bs{>1ajlU|PZ_!{Cxl!QqqV(7iG{MRZqJ*YWl^!t~0W zR~YV!N6r)mJZ>lPakc@dcMa4xVOnu(>k&<3qoZ{!l-NUIkgsz~*901U)_(;&6x$9G zlVQP2eGz|doPYnm(;5<~7Y>_=9Sx3qF2xnd2W`T}LZ80?>%~7lI6+cU_y)i-hEbM- zb03&=MCe=xF7_*t_l(=J2N0+kNoI(kF8iEbM=WNyeJ0>D0<}*Tgq@fE_~>9tZR=py z^N7xiNPB^Myn%rSc1Em3uZf-6m>#i)-s1cB?^{>of@LhiuE(b6Ro>{sJBDj!jmf8EkF~=aT_(O$ z$j;3Wh#blrl=Ir!H(oIvs~8{6J8q_Q`+MkNoCcz z@SDvdcp}krd+(RL-Idmk4(R|}wiqG(d?-y3Zkv9rgc67H4l4_bd6lbe39_;d-@m?R zJu58SsYidaOjt-eujT)yYL-rjjnA`J+oIFJCMK!V8ZZZkJCrq75D(E*P0Fbrx*4$g zb1AS@K`AO|(kdlok1amCWA{WjzpdybBh}4kxFsH@92M0pQtQn@Rh`tFyGmPmrN~}9 zH|L9m2L<5*am7^~9a%Y&f9xh!7RNJ=sg;>{c zf3X2mu%Wi`1g0@=Hp$1<2JBn~M@4jLyn0a#v1e-uN z7z_1KS^9o1*LjV$1AX$%@9;m;B&Ya(cHemmbPr^R?()*HhMhm>dg3uc(1MH`ZX?sMgD^L4H!%v9)(T(Gq8~o5Emz0`uQ}? zGl8)&%*a9RGA>NdQ3!M`B-4h33Yku2(?v{-B6!4d@U~A-koFL&bTHk7v^Ilt5(MOi zWTdT6bcX#zsO$~lxx)r_JlrsekFnUFkxmUW3zBGIwBZMLKcwxzW7yMEX2ErOlIaso z^Ja5%i%`<@ow}BmGRhTPBYGt9=QoS29N!O&qGw0pfOpQWu0sql9oT_gt%DC5KQ~hw zYkjQXNc9AU^1f(U*F8U9e!4TuOOA#dF7LDxI*!338p9OqjL~Lyk4OmW}om5D>9PN!aY~?o8lw zrT4)z9lB<%!{7R^iPs%2Oiu=WnrIxL8|i3iIVJ9SMTG7QqX^7{qH@rYVes!{yRu6U z4Gd=Kw2{lZDTeyq*f0`;dE0$5r|=QZF<-(&_Z`22+*eP}-2(9NT9SjkVP83%95IFl zGSD8tj31o&4B4TzwcFKf)yB?)j6{)RXh(R1aG`Rn@xy|Zg2(OvFUqpL z=W{2*9qFZ~tFGR3!LzBAaiqFNdK6EY^@gqtEBwFNr%yj)G6x4c+$Je4DybMhRz{bU znAYp}P?axbtn=jsX^@#~v z@2|U&C(?zR9N9?kMwYJ2jpoeEl8D4b3QPVzku9CA3mLYnGds(^l%U|lr0R>O0_3T& zI^>RQwn~UeZq&lS#TsQdZd}HPq1YtNYp$oTHrrOv)i_S694ck?yolUhTRJ6V6}x=` z(s3)*b)AO}Tn3)e7=8R98}uB;Vvt#*WxD;rFlGH+uF|9PgJg2SF|~wGVR0$#0-|}2 zRq1^*ii$=~zH{?~b2%-O=cU^quaqzp`W?MPqIsUL|TKl@qdA=Oy<#8)p zji!i7Mb}6d($#$Q%ox09wZlV03)Ql-yH$K1s>Z#(x#=q{GQEdDOV~?5anRP=AmE!4 z?d#}IQ$gP}=BYcf9JGHq&-{-JHdaASN@T0z8B7XUdMueSrmc1{se&cU!NFot=JaSi UIzKkqSv!Zdg@ZZ9)Ia5a01m;BI{*Lx diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json deleted file mode 100644 index 8a73efb66..000000000 --- a/native_chrome_extension/manifest.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "puterbot", - "version": "1.0", - "description": "Tokenizer for ScreenShots", - "manifest_version": 3, - "permissions": [ - "nativeMessaging" - ], - "": { - "default_popup": "popup.html" - }, - "icons": { - "128": "./Icons/MLDSAI_logo.png" - } -} \ No newline at end of file diff --git a/native_chrome_extension/popup.html b/native_chrome_extension/popup.html deleted file mode 100644 index 324138a79..000000000 --- a/native_chrome_extension/popup.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - puterbot - - -

Test

- - \ No newline at end of file From 49123971924045db8367c3f496fa48a7caa02216 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 6 May 2023 12:25:54 -0400 Subject: [PATCH 06/89] add native folder --- native_chrome_extension/Icons/MLDSAI_logo.png | Bin 0 -> 4060 bytes native_chrome_extension/manifest.json | 15 +++++++++++++++ native_chrome_extension/popup.html | 12 ++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 native_chrome_extension/Icons/MLDSAI_logo.png create mode 100644 native_chrome_extension/manifest.json create mode 100644 native_chrome_extension/popup.html diff --git a/native_chrome_extension/Icons/MLDSAI_logo.png b/native_chrome_extension/Icons/MLDSAI_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..62627b203a1b69f276de57509baad9cd1f68aa34 GIT binary patch literal 4060 zcmcJSX*d+#+sExhnh{!zEo+o0BwP0ECOc!-&oZ`Am|{#sh_Xcz24!r?$Ye~6r80%A z(b&e2WEo6SX4F{bnV#o)^?&{Qzc}Yy=bZbTbDis)`~H6J%&QKT!h#Zl92^|N)>h_D z?6&W35#VFjjUJeC4h}$-wYiDQt+EY@u#3x5JO7VfV2gJN`i-GYP4JY-A-(j1h)%;|ZAuK{bxD&t9yCKa`HJdq-~KS+*PC-{z_pt!jB*2rTc&NX*G9qjiuGNk+ag-*-9>8-3yCACs~zkl}$ zs_?VN=?X3!X+CzYECLtV$y^ir`3vNaqupT&9MAjWZ)9V2tDZvM+M=3y;m;`_KfZ7v zDrnKiBB-bct@bOEZ|>}+3Hh}^3@d$2`OvnH`Js%BK6vCJI4KMJe}m>1V7^-Vb{E`h z9WX8)uXEE5msPyvynb`@icHd8>vU7;1;qzk+{j2L(YoBLoGR)~&)VDl=(M+~GPNqs zrb9F7TcT_>IV-bJ?Gb(MxuVLiyrk2tnyEPfaf2!s7tnjZ~QKn zkdUB98g+ASBz)Nay=(OLO7~%?mVSL*9Z@MRMUQ={fyei$ojXU?$>+`iUap_q(GSSI zk1&A;-d?gf1jSJ=Hi{Q2dmAg*7DZGxxOW%OtvN%pySoAQ6$fQ6HYqoau%5_DX;VIP zY|M&?C*okQ*lp4th}+@tGu#*O$fczuGjZ$ri}&YCHh^4~2V zwq~NL%5=D+YNGOv-(1goEEN8c*$M-r(F%WfiR#^ZFi2-Y2C9L(`j8AT)2%e>#=9s3 z-k^aZGeH+V8#CjQADHmOt@k>Gj~<0gCKyWr^OUGm&kbh!c`DIBX?Y~#vk);g`NX{} z$OlZ!i5bGq2@s9z{3%&Jv?zm{9~*>(LZ4)jxIf0Mkq<(NJQ$ZC$;l-L)X{POxt(ZEy&x&*FvfWa z2;?|Ydztw6B(DgSGiNIFVsCw@*$hz8o?JftlArU7u`$R08u9QXb94Ud-7@AxRr=mO zauqjOgOA_;8Yo5S_@jx^b+%esa=q9%GTYzP$={QB$GQ|K#R)K7~643Bl__`A@l zm$iK)(st7Bm$$jjp`k!E&kLxi*}7+oy(8mY$X@E+pEFsXUmSbKLv)IY!ffx(?_9I> zo)wB^ZYv53`=DbzgLRQiA3=c=Q_FM&{byzTuV>RY);=q;gGgI{kG;j67@%f0=(stF z6t5Mt%u{XK)?|{(9G_>Elfo`VQJBgIY5C1uznKOrPLLsFkU$gy<(+Z2nX2N1KvGhI z9V1?(&&96*A3bs{>1ajlU|PZ_!{Cxl!QqqV(7iG{MRZqJ*YWl^!t~0W zR~YV!N6r)mJZ>lPakc@dcMa4xVOnu(>k&<3qoZ{!l-NUIkgsz~*901U)_(;&6x$9G zlVQP2eGz|doPYnm(;5<~7Y>_=9Sx3qF2xnd2W`T}LZ80?>%~7lI6+cU_y)i-hEbM- zb03&=MCe=xF7_*t_l(=J2N0+kNoI(kF8iEbM=WNyeJ0>D0<}*Tgq@fE_~>9tZR=py z^N7xiNPB^Myn%rSc1Em3uZf-6m>#i)-s1cB?^{>of@LhiuE(b6Ro>{sJBDj!jmf8EkF~=aT_(O$ z$j;3Wh#blrl=Ir!H(oIvs~8{6J8q_Q`+MkNoCcz z@SDvdcp}krd+(RL-Idmk4(R|}wiqG(d?-y3Zkv9rgc67H4l4_bd6lbe39_;d-@m?R zJu58SsYidaOjt-eujT)yYL-rjjnA`J+oIFJCMK!V8ZZZkJCrq75D(E*P0Fbrx*4$g zb1AS@K`AO|(kdlok1amCWA{WjzpdybBh}4kxFsH@92M0pQtQn@Rh`tFyGmPmrN~}9 zH|L9m2L<5*am7^~9a%Y&f9xh!7RNJ=sg;>{c zf3X2mu%Wi`1g0@=Hp$1<2JBn~M@4jLyn0a#v1e-uN z7z_1KS^9o1*LjV$1AX$%@9;m;B&Ya(cHemmbPr^R?()*HhMhm>dg3uc(1MH`ZX?sMgD^L4H!%v9)(T(Gq8~o5Emz0`uQ}? zGl8)&%*a9RGA>NdQ3!M`B-4h33Yku2(?v{-B6!4d@U~A-koFL&bTHk7v^Ilt5(MOi zWTdT6bcX#zsO$~lxx)r_JlrsekFnUFkxmUW3zBGIwBZMLKcwxzW7yMEX2ErOlIaso z^Ja5%i%`<@ow}BmGRhTPBYGt9=QoS29N!O&qGw0pfOpQWu0sql9oT_gt%DC5KQ~hw zYkjQXNc9AU^1f(U*F8U9e!4TuOOA#dF7LDxI*!338p9OqjL~Lyk4OmW}om5D>9PN!aY~?o8lw zrT4)z9lB<%!{7R^iPs%2Oiu=WnrIxL8|i3iIVJ9SMTG7QqX^7{qH@rYVes!{yRu6U z4Gd=Kw2{lZDTeyq*f0`;dE0$5r|=QZF<-(&_Z`22+*eP}-2(9NT9SjkVP83%95IFl zGSD8tj31o&4B4TzwcFKf)yB?)j6{)RXh(R1aG`Rn@xy|Zg2(OvFUqpL z=W{2*9qFZ~tFGR3!LzBAaiqFNdK6EY^@gqtEBwFNr%yj)G6x4c+$Je4DybMhRz{bU znAYp}P?axbtn=jsX^@#~v z@2|U&C(?zR9N9?kMwYJ2jpoeEl8D4b3QPVzku9CA3mLYnGds(^l%U|lr0R>O0_3T& zI^>RQwn~UeZq&lS#TsQdZd}HPq1YtNYp$oTHrrOv)i_S694ck?yolUhTRJ6V6}x=` z(s3)*b)AO}Tn3)e7=8R98}uB;Vvt#*WxD;rFlGH+uF|9PgJg2SF|~wGVR0$#0-|}2 zRq1^*ii$=~zH{?~b2%-O=cU^quaqzp`W?MPqIsUL|TKl@qdA=Oy<#8)p zji!i7Mb}6d($#$Q%ox09wZlV03)Ql-yH$K1s>Z#(x#=q{GQEdDOV~?5anRP=AmE!4 z?d#}IQ$gP}=BYcf9JGHq&-{-JHdaASN@T0z8B7XUdMueSrmc1{se&cU!NFot=JaSi UIzKkqSv!Zdg@ZZ9)Ia5a01m;BI{*Lx literal 0 HcmV?d00001 diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json new file mode 100644 index 000000000..8a73efb66 --- /dev/null +++ b/native_chrome_extension/manifest.json @@ -0,0 +1,15 @@ +{ + "name": "puterbot", + "version": "1.0", + "description": "Tokenizer for ScreenShots", + "manifest_version": 3, + "permissions": [ + "nativeMessaging" + ], + "": { + "default_popup": "popup.html" + }, + "icons": { + "128": "./Icons/MLDSAI_logo.png" + } +} \ No newline at end of file diff --git a/native_chrome_extension/popup.html b/native_chrome_extension/popup.html new file mode 100644 index 000000000..324138a79 --- /dev/null +++ b/native_chrome_extension/popup.html @@ -0,0 +1,12 @@ + + + + + + + puterbot + + +

Test

+ + \ No newline at end of file From 114b4cc58fd3c218400ef2d6693673d9da71d7e6 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 6 May 2023 15:19:56 -0400 Subject: [PATCH 07/89] Add briwer.py, native-manifest.json, --- native_chrome_extension/background.js | 15 ++++++++++++ native_chrome_extension/browser.py | 23 ++++++++++++++++++ native_chrome_extension/content.js | 25 ++++++++++++++++++++ native_chrome_extension/manifest.json | 24 ++++++++++++------- native_chrome_extension/native-manifest.json | 9 +++++++ native_chrome_extension/popup.html | 12 ---------- 6 files changed, 88 insertions(+), 20 deletions(-) create mode 100644 native_chrome_extension/background.js create mode 100644 native_chrome_extension/browser.py create mode 100644 native_chrome_extension/content.js create mode 100644 native_chrome_extension/native-manifest.json delete mode 100644 native_chrome_extension/popup.html diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js new file mode 100644 index 000000000..335e8bd5e --- /dev/null +++ b/native_chrome_extension/background.js @@ -0,0 +1,15 @@ +/** +the background.js file listens for messages from the content script, and +sends them to PuterBot via native messaging +*/ + +chrome.runtime.onMessageExternal.addListener(function(message, sender, sendResponse) { + if (sender.id !== chrome.runtime.id) { + return; + } + + // Send the message to PuterBot + chrome.runtime.sendNativeMessage("puterbot", message, function(response) { + console.log(response); + }); +}); \ No newline at end of file diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py new file mode 100644 index 000000000..c8bcecf88 --- /dev/null +++ b/native_chrome_extension/browser.py @@ -0,0 +1,23 @@ +import json +import nativemessaging + +def main(): + messages = [] + + while True: + # Receive a message from the Chrome Extension + message = nativemessaging.get_message() + if not message: + break + + # Parse the message as JSON + message_data = json.loads(message) + + # Store the message in a list + messages.append(message_data) + + # Do something with the collected messages + print(messages) + +if __name__ == "__main__": + main() diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js new file mode 100644 index 000000000..fc62ed4e1 --- /dev/null +++ b/native_chrome_extension/content.js @@ -0,0 +1,25 @@ +/** +The content.js file listens for changes to the DOM, and +sends a message to the background script when a change is detected. +*/ + +// Connect to the background script +const port = chrome.runtime.connect(); + +// Listen for changes to the DOM +// Ref.: https://stackoverflow.com/questions/8882502/how-to-track-dom-change-in-chrome-extension solution + +const observer = new MutationObserver(mutations => { + mutations.forEach(mutation => { // change occurs in DOM + try { + port.postMessage(mutation); // then, Send a message to the background script: background.js + } catch (e) { + console.error(e); + port.disconnect(); + const newPort = chrome.runtime.connect(); + newPort.onMessage.addListener(msg => console.log(msg)); + } + }); +}); + +observer.observe(document, { childList: true, subtree: true }); diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json index 8a73efb66..af7b6e698 100644 --- a/native_chrome_extension/manifest.json +++ b/native_chrome_extension/manifest.json @@ -1,15 +1,23 @@ { - "name": "puterbot", + "name": "PuterBot DOM Listener", "version": "1.0", - "description": "Tokenizer for ScreenShots", "manifest_version": 3, + "icons": { + "128": "./Icons/MLDSAI_logo.png" + }, "permissions": [ + "activeTab", + "tabs", "nativeMessaging" ], - "": { - "default_popup": "popup.html" + "background": { + "service_worker": "background.js" }, - "icons": { - "128": "./Icons/MLDSAI_logo.png" - } -} \ No newline at end of file + "content_scripts": [ + { + "matches": [""], + "all_frames": true, + "js": ["content.js"] + } + ] +} diff --git a/native_chrome_extension/native-manifest.json b/native_chrome_extension/native-manifest.json new file mode 100644 index 000000000..b5109a18f --- /dev/null +++ b/native_chrome_extension/native-manifest.json @@ -0,0 +1,9 @@ +{ + "name": "browser", + "description": "PuterBot DOM Listener", + "path": "./browser.py", + "type": "stdio", + "allowed_origins": [ + "chrome-extension://aeedjbflenakjffcnhodfddbphbnpemf/" + ] +} \ No newline at end of file diff --git a/native_chrome_extension/popup.html b/native_chrome_extension/popup.html deleted file mode 100644 index 324138a79..000000000 --- a/native_chrome_extension/popup.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - puterbot - - -

Test

- - \ No newline at end of file From afcb2965cc34be2d9e7a1255efc5fadbf1fda74a Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sun, 7 May 2023 12:06:41 -0400 Subject: [PATCH 08/89] remove uneccessary commits --- install_puterbot.ps1 | 30 ----------------------- install_puterbot.sh | 37 ---------------------------- puterbot/scrub.py | 47 ------------------------------------ tests/puterbot/test_scrub.py | 36 --------------------------- 4 files changed, 150 deletions(-) delete mode 100644 install_puterbot.ps1 delete mode 100644 install_puterbot.sh delete mode 100644 puterbot/scrub.py delete mode 100644 tests/puterbot/test_scrub.py diff --git a/install_puterbot.ps1 b/install_puterbot.ps1 deleted file mode 100644 index 7d90839cc..000000000 --- a/install_puterbot.ps1 +++ /dev/null @@ -1,30 +0,0 @@ -# Check if Git is installed -if (!(Test-Path -Path "C:\Program Files\Git\cmd\git.exe")) { - Write-Host "Git is not installed. Installing Git..." -ForegroundColor Yellow - Start-Process -FilePath "https://git-scm.com/download/win" -Wait -} - -# Check if Python 3.10 is installed -if (!(Test-Path -Path "C:\Program Files\Python310\python.exe")) { - Write-Host "Python 3.10 is not installed. Installing Python 3.10..." -ForegroundColor Yellow - Start-Process -FilePath "https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe" -Wait -} - -# Clone the OpenAdapt repository -git clone https://github.com/MLDSAI/puterbot.git -cd puterbot - -# Create and activate a Python virtual environment -python -m venv .venv -.\.venv\Scripts\Activate.ps1 - -# Install required packages and libraries -pip install wheel -pip install -r requirements.txt -pip install -e . - -# Run the database migration -alembic upgrade head - -# Run the test suite -pytest diff --git a/install_puterbot.sh b/install_puterbot.sh deleted file mode 100644 index 69952cc44..000000000 --- a/install_puterbot.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Check if Git is installed -if ! [ -x "$(command -v git)" ]; then - echo "Git is not installed. Installing Git..." - sudo apt-get update - sudo apt-get install -y git -fi - -# Check if Python 3.10 is installed -if ! [ -x "$(command -v python3.10)" ]; then - echo "Python 3.10 is not installed. Installing Python 3.10..." - sudo apt-get update - sudo apt-get install -y software-properties-common - sudo add-apt-repository -y ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install -y python3.10 python3.10-venv -fi - -# Clone the OpenAdapt repository -git clone https://github.com/MLDSAI/puterbot.git -cd puterbot || exit - -# Create and activate a Python virtual environment -python3.10 -m venv .venv -source .venv/bin/activate - -# Install required packages and libraries -pip install wheel -pip install -r requirements.txt -pip install -e . - -# Run the database migration -alembic upgrade head - -# Run the test suite -pytest diff --git a/puterbot/scrub.py b/puterbot/scrub.py deleted file mode 100644 index fa2f62569..000000000 --- a/puterbot/scrub.py +++ /dev/null @@ -1,47 +0,0 @@ -import re - -def scrub(text: str) -> str: - """ - This function takes in a string of text as input and returns a scrubbed version of the text with any potential personally - identifiable information (PII) or protected health information (PHI) replaced with asterisks. - - The function replaces email addresses, phone numbers, credit card numbers, and dates of birth (or any dates) - with asterisks. It uses regular expressions to identify the patterns of these types of information in the text and - replaces them with the corresponding number of asterisks. - - Usage example: - python -m puterbot.scrub - - :param text: a string of text that may contain PII/PHI - :return: a scrubbed version of the input text with PII/PHI replaced with asterisks. - """ - # "\b = a word boundary/complete word" - scrubbed_text = text - # Replace email addresses: XYZ@XYZ.XYZ [Format] [XYZ stands for Alphanumeric characters] - scrubbed_text = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '***@***.***', scrubbed_text) - - # Replace phone numbers XXX-XXX-XXXX [Format] [X stands for A digit from 0-9] - scrubbed_text = re.sub(r'\b\d{3}[-\s]?\d{3}[-\s]?\d{4}\b', '***-***-****', scrubbed_text) - - # Replace credit card numbers XXXX-XXXX-XXXX-XXXX [Format] [X stands for A digit from 0-9] - scrubbed_text = re.sub(r'\b\d{4}-\d{4}-\d{4}-\d{4}\b', '****-****-****-****', scrubbed_text) - - # Replace dates of birth (or any Dates) mm/dd/yyyy or dd/mm/yyyy [Format] - scrubbed_text = re.sub(r'\b\d{2}/\d{2}/\d{4}\b', '**/**/****', scrubbed_text) - - # TODO: Add any other PII or PHI field for scrubbing. - - - return scrubbed_text - - -if __name__ == '__main__': - import sys - inp_text = sys.stdin.read() - scrubbed_text = scrub(inp_text) - sys.stdout.write(scrubbed_text) - - # Note: This script uses the sys.stdin.read() method to read input from the console, - # which means you'll need to - # press Ctrl + D on Linux/Mac or - # Enter then Ctrl + Z then again hit Enter, on Windows to signal the end of the input when you're done typing or pasting in your text. diff --git a/tests/puterbot/test_scrub.py b/tests/puterbot/test_scrub.py deleted file mode 100644 index 1a9eccf69..000000000 --- a/tests/puterbot/test_scrub.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest -from puterbot.scrub import scrub - -def test_scrub_email(): - text = "test email is test@utoronto.ca " - expected_output = "test email is ***@***.*** " - assert scrub(text) == expected_output - - -def test_scrub_phone_number(): - text = "test phone number is 123-456-7890 and my" - expected_output = "test phone number is ***-***-**** and my" - assert scrub(text) == expected_output - - -def test_scrub_credit_card_number(): - text = "test credit card number is 1234-1234-1234-1234 sf" - expected_output = "test credit card number is ****-****-****-**** sf" - assert scrub(text) == expected_output - - -def test_scrub_date_of_birth(): - text = "test date of birth is 01/01/1990" - expected_output = "test date of birth is **/**/****" - assert scrub(text) == expected_output - - -# Add tests for other fields for newly added PII/PHI in scrub.py [if any] - - -def test_all_together(): - text = "My email is john@example.com, my phone number is 123-456-7890, " \ - "my credit card number is 1234-5678-9012-3456, and my date of birth is 01/01/1990." - expected_output = "My email is ***@***.***, my phone number is ***-***-****, " \ - "my credit card number is ****-****-****-****, and my date of birth is **/**/****." - assert scrub(text) == expected_output \ No newline at end of file From 10a55f601fb41304b09e58f6e1ae1cb82222848d Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 19 May 2023 18:01:52 -0400 Subject: [PATCH 09/89] Progress on Issue #51 --- native_chrome_extension/background.js | 35 ++++++++++++++----- native_chrome_extension/browser.py | 23 +++++------- .../com.openadapt.domlistener.bat | 2 ++ .../com.openadapt.domlistener_chrome.json | 9 +++++ native_chrome_extension/content.js | 24 ++++--------- native_chrome_extension/manifest.json | 18 +++++----- native_chrome_extension/native-manifest.json | 8 ++--- native_chrome_extension/popup.html | 15 ++++++++ requirements.txt | 1 + 9 files changed, 82 insertions(+), 53 deletions(-) create mode 100644 native_chrome_extension/com.openadapt.domlistener.bat create mode 100644 native_chrome_extension/com.openadapt.domlistener_chrome.json create mode 100644 native_chrome_extension/popup.html diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 335e8bd5e..bcb447c4d 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -1,15 +1,32 @@ /** the background.js file listens for messages from the content script, and -sends them to PuterBot via native messaging +sends them to OpenAdapt via native messaging */ -chrome.runtime.onMessageExternal.addListener(function(message, sender, sendResponse) { - if (sender.id !== chrome.runtime.id) { - return; - } +// Connect to the native messaging host +let port = chrome.runtime.connectNative("com.openadapt.domlistener"); + +// Receive messages from the native messaging host +port.onMessage.addListener(onReceived); + +// Send initial message to the native messaging host +port.postMessage("hello"); - // Send the message to PuterBot - chrome.runtime.sendNativeMessage("puterbot", message, function(response) { - console.log(response); - }); +// Handle received messages +function onReceived(response) { + console.log(response); +} + +// Listen for messages from the content script +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + // Forward the message to the native messaging host + port.postMessage(message); +}); + +// Listen for DOM changes in the active tab +chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { + if (changeInfo.status === "complete") { + // Inject content script to the updated tab + chrome.tabs.executeScript(tabId, { file: "content.js" }); + } }); \ No newline at end of file diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index c8bcecf88..bc0ad542d 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -1,23 +1,18 @@ -import json +""" Module for the browser native messaging host. """ import nativemessaging -def main(): - messages = [] +def main() -> None: + """ + Main function for the browser native messaging host. + """ + reply_num = 0 while True: - # Receive a message from the Chrome Extension message = nativemessaging.get_message() - if not message: - break + print(message) + nativemessaging.send_message(nativemessaging.encode_message(str(reply_num))) + reply_num += 1 - # Parse the message as JSON - message_data = json.loads(message) - - # Store the message in a list - messages.append(message_data) - - # Do something with the collected messages - print(messages) if __name__ == "__main__": main() diff --git a/native_chrome_extension/com.openadapt.domlistener.bat b/native_chrome_extension/com.openadapt.domlistener.bat new file mode 100644 index 000000000..37a2b189d --- /dev/null +++ b/native_chrome_extension/com.openadapt.domlistener.bat @@ -0,0 +1,2 @@ +@echo off +python -u "P:\OpenAdapt AI - MLDS AI\cloned_repo\OpenAdapt\native_chrome_extension\browser.py" \ No newline at end of file diff --git a/native_chrome_extension/com.openadapt.domlistener_chrome.json b/native_chrome_extension/com.openadapt.domlistener_chrome.json new file mode 100644 index 000000000..bd4ca3c07 --- /dev/null +++ b/native_chrome_extension/com.openadapt.domlistener_chrome.json @@ -0,0 +1,9 @@ +{ + "name": "com.openadapt.domlistener", + "description": "OpenAdapt Native Messaging Host", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\com.openadapt.domlistener.bat", + "type": "stdio", + "allowed_origins": [ + "chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/" + ] +} \ No newline at end of file diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index fc62ed4e1..9c919cd4f 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -3,23 +3,13 @@ The content.js file listens for changes to the DOM, and sends a message to the background script when a change is detected. */ -// Connect to the background script -const port = chrome.runtime.connect(); - -// Listen for changes to the DOM -// Ref.: https://stackoverflow.com/questions/8882502/how-to-track-dom-change-in-chrome-extension solution - -const observer = new MutationObserver(mutations => { - mutations.forEach(mutation => { // change occurs in DOM - try { - port.postMessage(mutation); // then, Send a message to the background script: background.js - } catch (e) { - console.error(e); - port.disconnect(); - const newPort = chrome.runtime.connect(); - newPort.onMessage.addListener(msg => console.log(msg)); +// DOM mutation observer +let observer = new MutationObserver(mutations => { + for (let mutation of mutations) { + // Send the mutation to the background script + chrome.runtime.sendMessage(mutation); } - }); }); -observer.observe(document, { childList: true, subtree: true }); +// Start observing DOM mutations +observer.observe(document, { childList: true, subtree: true }); \ No newline at end of file diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json index af7b6e698..64cd96208 100644 --- a/native_chrome_extension/manifest.json +++ b/native_chrome_extension/manifest.json @@ -1,5 +1,5 @@ { - "name": "PuterBot DOM Listener", + "name": "OpenAdapt DOM Listener", "version": "1.0", "manifest_version": 3, "icons": { @@ -13,11 +13,11 @@ "background": { "service_worker": "background.js" }, - "content_scripts": [ - { - "matches": [""], - "all_frames": true, - "js": ["content.js"] - } - ] -} + "action": { + "default_popup": "popup.html" + }, + "content_scripts": [{ + "matches": [""], + "js": ["content.js"] + }] +} \ No newline at end of file diff --git a/native_chrome_extension/native-manifest.json b/native_chrome_extension/native-manifest.json index b5109a18f..1d47e31cd 100644 --- a/native_chrome_extension/native-manifest.json +++ b/native_chrome_extension/native-manifest.json @@ -1,9 +1,9 @@ { - "name": "browser", - "description": "PuterBot DOM Listener", - "path": "./browser.py", + "name": "com.openadapt.domlistener", + "description": "OpenAdapt Native Messaging Host", + "path": "browser.py", "type": "stdio", "allowed_origins": [ - "chrome-extension://aeedjbflenakjffcnhodfddbphbnpemf/" + "chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/" ] } \ No newline at end of file diff --git a/native_chrome_extension/popup.html b/native_chrome_extension/popup.html new file mode 100644 index 000000000..07f86a5c7 --- /dev/null +++ b/native_chrome_extension/popup.html @@ -0,0 +1,15 @@ + + + + + + + OpenAdapt DOM Listener + + + +

PuterBot DOM Listener

+

Extension is actively listening for DOM changes.

+ + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index de647807c..9e75e6c4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ sqlalchemy==1.4.43 torch==2.0.0 tqdm==4.64.0 transformers==4.28.1 +nativemessaging==1.0.1 \ No newline at end of file From 8a2a11fbb12a3b59b56f7b51c8b096d5598f5138 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 22 May 2023 12:05:10 -0400 Subject: [PATCH 10/89] Add nativemessaging --- .../nativemessaging/__init__.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 native_chrome_extension/nativemessaging/__init__.py diff --git a/native_chrome_extension/nativemessaging/__init__.py b/native_chrome_extension/nativemessaging/__init__.py new file mode 100644 index 000000000..7c7a1a9eb --- /dev/null +++ b/native_chrome_extension/nativemessaging/__init__.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +import json +import sys +import struct + + +# from https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Native_messaging +def get_message(): + raw_length = sys.stdin.buffer.read(4) + if len(raw_length) == 0: + sys.exit(0) + message_length = struct.unpack("@I", raw_length)[0] + message = sys.stdin.buffer.read(message_length).decode("utf-8") + return json.loads(message) + + +# Encode a message for transmission, given its content. +def encode_message(message_content): + encoded_content = json.dumps(message_content).encode("utf-8") + encoded_length = struct.pack("@I", len(encoded_content)) + return {"length": encoded_length, "content": encoded_content} + + +# Send an encoded message to stdout. +def send_message(encoded_message): + sys.stdout.buffer.write(encoded_message["length"]) + sys.stdout.buffer.write(encoded_message["content"]) + sys.stdout.buffer.flush() + \ No newline at end of file From daac09ea97974cd7c01a9db43713cf6593944398 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 23 May 2023 15:52:25 -0400 Subject: [PATCH 11/89] remove all --- native_chrome_extension/background.js | 32 ------------------- native_chrome_extension/browser.py | 18 ----------- .../com.openadapt.domlistener.bat | 2 -- .../com.openadapt.domlistener_chrome.json | 9 ------ native_chrome_extension/content.js | 15 --------- native_chrome_extension/manifest.json | 23 ------------- native_chrome_extension/native-manifest.json | 9 ------ .../nativemessaging/__init__.py | 29 ----------------- native_chrome_extension/popup.html | 15 --------- 9 files changed, 152 deletions(-) delete mode 100644 native_chrome_extension/background.js delete mode 100644 native_chrome_extension/browser.py delete mode 100644 native_chrome_extension/com.openadapt.domlistener.bat delete mode 100644 native_chrome_extension/com.openadapt.domlistener_chrome.json delete mode 100644 native_chrome_extension/content.js delete mode 100644 native_chrome_extension/manifest.json delete mode 100644 native_chrome_extension/native-manifest.json delete mode 100644 native_chrome_extension/nativemessaging/__init__.py delete mode 100644 native_chrome_extension/popup.html diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js deleted file mode 100644 index bcb447c4d..000000000 --- a/native_chrome_extension/background.js +++ /dev/null @@ -1,32 +0,0 @@ -/** -the background.js file listens for messages from the content script, and -sends them to OpenAdapt via native messaging -*/ - -// Connect to the native messaging host -let port = chrome.runtime.connectNative("com.openadapt.domlistener"); - -// Receive messages from the native messaging host -port.onMessage.addListener(onReceived); - -// Send initial message to the native messaging host -port.postMessage("hello"); - -// Handle received messages -function onReceived(response) { - console.log(response); -} - -// Listen for messages from the content script -chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { - // Forward the message to the native messaging host - port.postMessage(message); -}); - -// Listen for DOM changes in the active tab -chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { - if (changeInfo.status === "complete") { - // Inject content script to the updated tab - chrome.tabs.executeScript(tabId, { file: "content.js" }); - } -}); \ No newline at end of file diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py deleted file mode 100644 index bc0ad542d..000000000 --- a/native_chrome_extension/browser.py +++ /dev/null @@ -1,18 +0,0 @@ -""" Module for the browser native messaging host. """ -import nativemessaging - - -def main() -> None: - """ - Main function for the browser native messaging host. - """ - reply_num = 0 - while True: - message = nativemessaging.get_message() - print(message) - nativemessaging.send_message(nativemessaging.encode_message(str(reply_num))) - reply_num += 1 - - -if __name__ == "__main__": - main() diff --git a/native_chrome_extension/com.openadapt.domlistener.bat b/native_chrome_extension/com.openadapt.domlistener.bat deleted file mode 100644 index 37a2b189d..000000000 --- a/native_chrome_extension/com.openadapt.domlistener.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -python -u "P:\OpenAdapt AI - MLDS AI\cloned_repo\OpenAdapt\native_chrome_extension\browser.py" \ No newline at end of file diff --git a/native_chrome_extension/com.openadapt.domlistener_chrome.json b/native_chrome_extension/com.openadapt.domlistener_chrome.json deleted file mode 100644 index bd4ca3c07..000000000 --- a/native_chrome_extension/com.openadapt.domlistener_chrome.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "com.openadapt.domlistener", - "description": "OpenAdapt Native Messaging Host", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\com.openadapt.domlistener.bat", - "type": "stdio", - "allowed_origins": [ - "chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/" - ] -} \ No newline at end of file diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js deleted file mode 100644 index 9c919cd4f..000000000 --- a/native_chrome_extension/content.js +++ /dev/null @@ -1,15 +0,0 @@ -/** -The content.js file listens for changes to the DOM, and -sends a message to the background script when a change is detected. -*/ - -// DOM mutation observer -let observer = new MutationObserver(mutations => { - for (let mutation of mutations) { - // Send the mutation to the background script - chrome.runtime.sendMessage(mutation); - } -}); - -// Start observing DOM mutations -observer.observe(document, { childList: true, subtree: true }); \ No newline at end of file diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json deleted file mode 100644 index 64cd96208..000000000 --- a/native_chrome_extension/manifest.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "OpenAdapt DOM Listener", - "version": "1.0", - "manifest_version": 3, - "icons": { - "128": "./Icons/MLDSAI_logo.png" - }, - "permissions": [ - "activeTab", - "tabs", - "nativeMessaging" - ], - "background": { - "service_worker": "background.js" - }, - "action": { - "default_popup": "popup.html" - }, - "content_scripts": [{ - "matches": [""], - "js": ["content.js"] - }] -} \ No newline at end of file diff --git a/native_chrome_extension/native-manifest.json b/native_chrome_extension/native-manifest.json deleted file mode 100644 index 1d47e31cd..000000000 --- a/native_chrome_extension/native-manifest.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "com.openadapt.domlistener", - "description": "OpenAdapt Native Messaging Host", - "path": "browser.py", - "type": "stdio", - "allowed_origins": [ - "chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/" - ] -} \ No newline at end of file diff --git a/native_chrome_extension/nativemessaging/__init__.py b/native_chrome_extension/nativemessaging/__init__.py deleted file mode 100644 index 7c7a1a9eb..000000000 --- a/native_chrome_extension/nativemessaging/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -import json -import sys -import struct - - -# from https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Native_messaging -def get_message(): - raw_length = sys.stdin.buffer.read(4) - if len(raw_length) == 0: - sys.exit(0) - message_length = struct.unpack("@I", raw_length)[0] - message = sys.stdin.buffer.read(message_length).decode("utf-8") - return json.loads(message) - - -# Encode a message for transmission, given its content. -def encode_message(message_content): - encoded_content = json.dumps(message_content).encode("utf-8") - encoded_length = struct.pack("@I", len(encoded_content)) - return {"length": encoded_length, "content": encoded_content} - - -# Send an encoded message to stdout. -def send_message(encoded_message): - sys.stdout.buffer.write(encoded_message["length"]) - sys.stdout.buffer.write(encoded_message["content"]) - sys.stdout.buffer.flush() - \ No newline at end of file diff --git a/native_chrome_extension/popup.html b/native_chrome_extension/popup.html deleted file mode 100644 index 07f86a5c7..000000000 --- a/native_chrome_extension/popup.html +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - OpenAdapt DOM Listener - - - -

PuterBot DOM Listener

-

Extension is actively listening for DOM changes.

- - - \ No newline at end of file From 8a719a9a49ec0134a23a95e0887125c47af53b6e Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 24 May 2023 11:00:09 -0400 Subject: [PATCH 12/89] test_extension files --- native_chrome_extension/browser.bat | 2 ++ native_chrome_extension/browser.py | 7 +++++++ native_chrome_extension/manifest.json | 21 +++++++++++++++++++ native_chrome_extension/native-manifest.json | 7 +++++++ native_chrome_extension/openadapt_chrome.json | 9 ++++++++ native_chrome_extension/scripts/content.js | 11 ++++++++++ .../scripts/service-worker.js | 11 ++++++++++ 7 files changed, 68 insertions(+) create mode 100644 native_chrome_extension/browser.bat create mode 100644 native_chrome_extension/browser.py create mode 100644 native_chrome_extension/manifest.json create mode 100644 native_chrome_extension/native-manifest.json create mode 100644 native_chrome_extension/openadapt_chrome.json create mode 100644 native_chrome_extension/scripts/content.js create mode 100644 native_chrome_extension/scripts/service-worker.js diff --git a/native_chrome_extension/browser.bat b/native_chrome_extension/browser.bat new file mode 100644 index 000000000..286582558 --- /dev/null +++ b/native_chrome_extension/browser.bat @@ -0,0 +1,2 @@ +@echo off +python -u "P:\OpenAdapt AI - MLDS AI\cloned_repo\OpenAdapt\native_chrome_extension\browser.py" diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py new file mode 100644 index 000000000..fdf3dcddb --- /dev/null +++ b/native_chrome_extension/browser.py @@ -0,0 +1,7 @@ +import nativemessaging + +while True: + message = nativemessaging.get_message() + print(message) + if message == "hello": + nativemessaging.send_message(nativemessaging.encode_message("world")) diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json new file mode 100644 index 000000000..be0033c20 --- /dev/null +++ b/native_chrome_extension/manifest.json @@ -0,0 +1,21 @@ +{ + "manifest_version": 3, + "name": "Reading time", + "version": "1.0", + "description": "Add the reading time to Chrome Extension documentation articles", + "icons": { + "128": "Icons/MLDSAI_logo.png" + }, + "content_scripts": [ + { + "js": ["scripts/content.js"], + "matches": [""] + } + ], + "permissions": [ + "activeTab", + "tabs", + "nativeMessaging" + ] +} + diff --git a/native_chrome_extension/native-manifest.json b/native_chrome_extension/native-manifest.json new file mode 100644 index 000000000..db0467005 --- /dev/null +++ b/native_chrome_extension/native-manifest.json @@ -0,0 +1,7 @@ +{ + "name": "openadapt", + "description": "OpenAdapt DOM Listener", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", + "type": "stdio", + "allowed_origins": ["chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/"] + } \ No newline at end of file diff --git a/native_chrome_extension/openadapt_chrome.json b/native_chrome_extension/openadapt_chrome.json new file mode 100644 index 000000000..5d3365583 --- /dev/null +++ b/native_chrome_extension/openadapt_chrome.json @@ -0,0 +1,9 @@ +{ + "name": "openadapt", + "description": "OpenAdapt DOM Listener", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", + "type": "stdio", + "allowed_origins": [ + "chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/" + ] +} \ No newline at end of file diff --git a/native_chrome_extension/scripts/content.js b/native_chrome_extension/scripts/content.js new file mode 100644 index 000000000..a2cfd1e31 --- /dev/null +++ b/native_chrome_extension/scripts/content.js @@ -0,0 +1,11 @@ +function onReceived(response) { + console.log(response); +} + +// runtime.connectNative +var port = browser.runtime.connectNative("openadapt"); +port.onMessage.addListener(onReceived); +port.postMessage("hello"); + +// runtime.sendNativeMessage +browser.runtime.sendNativeMessage("openadapt", "hello").then(onReceived); diff --git a/native_chrome_extension/scripts/service-worker.js b/native_chrome_extension/scripts/service-worker.js new file mode 100644 index 000000000..151a8a3c7 --- /dev/null +++ b/native_chrome_extension/scripts/service-worker.js @@ -0,0 +1,11 @@ +var port = chrome.runtime.connectNative('openadapt'); + +port.onMessage.addListener(function (msg) { + console.log('Received' + msg); +}); + +port.onDisconnect.addListener(function () { + console.log('Disconnected'); +}); + +port.postMessage({text: 'Hello, openadapt'}); \ No newline at end of file From e6b1e9f7c2f63d3b748519b100996566d2635ce2 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 24 May 2023 11:11:55 -0400 Subject: [PATCH 13/89] browser to chrome --- native_chrome_extension/scripts/content.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native_chrome_extension/scripts/content.js b/native_chrome_extension/scripts/content.js index a2cfd1e31..b919517ec 100644 --- a/native_chrome_extension/scripts/content.js +++ b/native_chrome_extension/scripts/content.js @@ -3,7 +3,7 @@ function onReceived(response) { } // runtime.connectNative -var port = browser.runtime.connectNative("openadapt"); +var port = chrome.runtime.connectNative("openadapt"); port.onMessage.addListener(onReceived); port.postMessage("hello"); From 8a86f496333939e47123a998ec7ea0e0bdd8ab7e Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 24 May 2023 11:15:52 -0400 Subject: [PATCH 14/89] browser -> chrome --- native_chrome_extension/scripts/content.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native_chrome_extension/scripts/content.js b/native_chrome_extension/scripts/content.js index b919517ec..c45623e2c 100644 --- a/native_chrome_extension/scripts/content.js +++ b/native_chrome_extension/scripts/content.js @@ -8,4 +8,4 @@ port.onMessage.addListener(onReceived); port.postMessage("hello"); // runtime.sendNativeMessage -browser.runtime.sendNativeMessage("openadapt", "hello").then(onReceived); +chrome.runtime.sendNativeMessage("openadapt", "hello").then(onReceived); From bfeb34d00cb7b971445b3bb0d1428aa9ca675c4d Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 24 May 2023 12:08:44 -0400 Subject: [PATCH 15/89] test native applicatoin --- native_chrome_extension/browser.bat | 2 +- native_chrome_extension/manifest.json | 8 ++++++-- native_chrome_extension/native-manifest.json | 2 +- native_chrome_extension/scripts/background.js | 18 ++++++++++++++++++ native_chrome_extension/scripts/content.js | 4 ++-- .../scripts/service-worker.js | 11 ----------- 6 files changed, 28 insertions(+), 17 deletions(-) create mode 100644 native_chrome_extension/scripts/background.js delete mode 100644 native_chrome_extension/scripts/service-worker.js diff --git a/native_chrome_extension/browser.bat b/native_chrome_extension/browser.bat index 286582558..b10826375 100644 --- a/native_chrome_extension/browser.bat +++ b/native_chrome_extension/browser.bat @@ -1,2 +1,2 @@ @echo off -python -u "P:\OpenAdapt AI - MLDS AI\cloned_repo\OpenAdapt\native_chrome_extension\browser.py" +call python "P:\OpenAdapt AI - MLDS AI\cloned_repo\OpenAdapt\native_chrome_extension\browser.py" diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json index be0033c20..57a68436f 100644 --- a/native_chrome_extension/manifest.json +++ b/native_chrome_extension/manifest.json @@ -1,6 +1,6 @@ { "manifest_version": 3, - "name": "Reading time", + "name": "OpenAdapt DOM Listener", "version": "1.0", "description": "Add the reading time to Chrome Extension documentation articles", "icons": { @@ -16,6 +16,10 @@ "activeTab", "tabs", "nativeMessaging" - ] + ], + "background": { + "service_worker": "scripts/background.js" + }, + "action": {} } diff --git a/native_chrome_extension/native-manifest.json b/native_chrome_extension/native-manifest.json index db0467005..19b40fc56 100644 --- a/native_chrome_extension/native-manifest.json +++ b/native_chrome_extension/native-manifest.json @@ -1,7 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.py", "type": "stdio", "allowed_origins": ["chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/"] } \ No newline at end of file diff --git a/native_chrome_extension/scripts/background.js b/native_chrome_extension/scripts/background.js new file mode 100644 index 000000000..21fb05354 --- /dev/null +++ b/native_chrome_extension/scripts/background.js @@ -0,0 +1,18 @@ +// service-worker.js + +// Establish a connection with the native app +var port = chrome.runtime.connectNative('openadapt'); + +port.onMessage.addListener(function (msg) { + console.log('Received: ' + msg); +}); + +port.onDisconnect.addListener(function () { + console.log('Disconnected'); +}); + +// On a click on the action, send a message to the native app +chrome.action.onClicked.addListener(function () { + console.log('Sending: hi'); + port.postMessage('hi'); +}); diff --git a/native_chrome_extension/scripts/content.js b/native_chrome_extension/scripts/content.js index c45623e2c..2e391a67a 100644 --- a/native_chrome_extension/scripts/content.js +++ b/native_chrome_extension/scripts/content.js @@ -3,9 +3,9 @@ function onReceived(response) { } // runtime.connectNative -var port = chrome.runtime.connectNative("openadapt"); +var port = browser.runtime.connectNative("application_name"); port.onMessage.addListener(onReceived); port.postMessage("hello"); // runtime.sendNativeMessage -chrome.runtime.sendNativeMessage("openadapt", "hello").then(onReceived); +browser.runtime.sendNativeMessage("application_name", "hello").then(onReceived); \ No newline at end of file diff --git a/native_chrome_extension/scripts/service-worker.js b/native_chrome_extension/scripts/service-worker.js deleted file mode 100644 index 151a8a3c7..000000000 --- a/native_chrome_extension/scripts/service-worker.js +++ /dev/null @@ -1,11 +0,0 @@ -var port = chrome.runtime.connectNative('openadapt'); - -port.onMessage.addListener(function (msg) { - console.log('Received' + msg); -}); - -port.onDisconnect.addListener(function () { - console.log('Disconnected'); -}); - -port.postMessage({text: 'Hello, openadapt'}); \ No newline at end of file From 7a48fa9bf4eb0f99949d25f7c3b5fc62bbe3888a Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 24 May 2023 12:21:50 -0400 Subject: [PATCH 16/89] fixes --- native_chrome_extension/scripts/background.js | 2 +- native_chrome_extension/scripts/content.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/native_chrome_extension/scripts/background.js b/native_chrome_extension/scripts/background.js index 21fb05354..1a39eb044 100644 --- a/native_chrome_extension/scripts/background.js +++ b/native_chrome_extension/scripts/background.js @@ -1,4 +1,4 @@ -// service-worker.js +// background.js // Establish a connection with the native app var port = chrome.runtime.connectNative('openadapt'); diff --git a/native_chrome_extension/scripts/content.js b/native_chrome_extension/scripts/content.js index 2e391a67a..575f28daf 100644 --- a/native_chrome_extension/scripts/content.js +++ b/native_chrome_extension/scripts/content.js @@ -3,9 +3,9 @@ function onReceived(response) { } // runtime.connectNative -var port = browser.runtime.connectNative("application_name"); +var port = browser.runtime.connectNative("openadapt"); port.onMessage.addListener(onReceived); port.postMessage("hello"); // runtime.sendNativeMessage -browser.runtime.sendNativeMessage("application_name", "hello").then(onReceived); \ No newline at end of file +browser.runtime.sendNativeMessage("openadapt", "hello").then(onReceived); \ No newline at end of file From 7fbe220b721b55c4f5cda37b65bdcad3401390ce Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 25 May 2023 20:39:36 -0400 Subject: [PATCH 17/89] add bat in antive json --- native_chrome_extension/native-manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/native_chrome_extension/native-manifest.json b/native_chrome_extension/native-manifest.json index 19b40fc56..e0c4dd49f 100644 --- a/native_chrome_extension/native-manifest.json +++ b/native_chrome_extension/native-manifest.json @@ -1,7 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.py", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", "type": "stdio", "allowed_origins": ["chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/"] - } \ No newline at end of file +} From a3dc6106a321e928b8691382ae5a91e25d0c209c Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 2 Jun 2023 15:51:22 -0400 Subject: [PATCH 18/89] resolving merge conflicts --- .gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4c0b428a9..1be7eef74 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,10 @@ cache *.db # VSCode -*.vscode +.VSCode + +# Generated performance charts +performance + +# Generated when adding editable dependencies in requirements.txt (-e) +src \ No newline at end of file From 8e93a83c8298b1c6d92857639a8074db56b77091 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 5 Jun 2023 07:23:16 -0400 Subject: [PATCH 19/89] attempting ping pong example on chrome --- native_chrome_extension/add-on/background.js | 19 +++++++++ .../{Icons => add-on/icons}/MLDSAI_logo.png | Bin native_chrome_extension/add-on/manifest.json | 29 +++++++++++++ native_chrome_extension/app/browser.bat | 3 ++ native_chrome_extension/app/browser.py | 40 ++++++++++++++++++ .../{ => app}/openadapt_chrome.json | 2 +- native_chrome_extension/browser.bat | 2 - native_chrome_extension/browser.py | 7 --- native_chrome_extension/manifest.json | 25 ----------- native_chrome_extension/native-manifest.json | 7 --- native_chrome_extension/scripts/background.js | 18 -------- native_chrome_extension/scripts/content.js | 11 ----- 12 files changed, 92 insertions(+), 71 deletions(-) create mode 100644 native_chrome_extension/add-on/background.js rename native_chrome_extension/{Icons => add-on/icons}/MLDSAI_logo.png (100%) create mode 100644 native_chrome_extension/add-on/manifest.json create mode 100644 native_chrome_extension/app/browser.bat create mode 100644 native_chrome_extension/app/browser.py rename native_chrome_extension/{ => app}/openadapt_chrome.json (84%) delete mode 100644 native_chrome_extension/browser.bat delete mode 100644 native_chrome_extension/browser.py delete mode 100644 native_chrome_extension/manifest.json delete mode 100644 native_chrome_extension/native-manifest.json delete mode 100644 native_chrome_extension/scripts/background.js delete mode 100644 native_chrome_extension/scripts/content.js diff --git a/native_chrome_extension/add-on/background.js b/native_chrome_extension/add-on/background.js new file mode 100644 index 000000000..020e6edde --- /dev/null +++ b/native_chrome_extension/add-on/background.js @@ -0,0 +1,19 @@ +/* +On startup, connect to the "ping_pong" app. +*/ +let port = browser.runtime.connectNative("ping_pong"); + +/* +Listen for messages from the app. +*/ +port.onMessage.addListener((response) => { + console.log("Received: " + response); +}); + +/* +On a click on the browser action, send the app a message. +*/ +browser.browserAction.onClicked.addListener(() => { + console.log("Sending: ping"); + port.postMessage("ping"); +}); diff --git a/native_chrome_extension/Icons/MLDSAI_logo.png b/native_chrome_extension/add-on/icons/MLDSAI_logo.png similarity index 100% rename from native_chrome_extension/Icons/MLDSAI_logo.png rename to native_chrome_extension/add-on/icons/MLDSAI_logo.png diff --git a/native_chrome_extension/add-on/manifest.json b/native_chrome_extension/add-on/manifest.json new file mode 100644 index 000000000..2f41f7e4f --- /dev/null +++ b/native_chrome_extension/add-on/manifest.json @@ -0,0 +1,29 @@ +{ + + "description": "Native messaging example add-on", + "manifest_version": 2, + "name": "Native messaging example", + "version": "1.0", + "icons": { + "48": "icons/message.svg" + }, + + "browser_specific_settings": { + "gecko": { + "id": "ping_pong@example.org", + "strict_min_version": "50.0" + } + }, + + "background": { + "scripts": ["background.js"] + }, + + "browser_action": { + "default_icon": "icons/message.svg" + }, + + "permissions": ["nativeMessaging"] + + } + \ No newline at end of file diff --git a/native_chrome_extension/app/browser.bat b/native_chrome_extension/app/browser.bat new file mode 100644 index 000000000..92ce50f2d --- /dev/null +++ b/native_chrome_extension/app/browser.bat @@ -0,0 +1,3 @@ +@echo off + +python -u "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\app\\browser.py" diff --git a/native_chrome_extension/app/browser.py b/native_chrome_extension/app/browser.py new file mode 100644 index 000000000..92a340425 --- /dev/null +++ b/native_chrome_extension/app/browser.py @@ -0,0 +1,40 @@ +#!/usr/bin/env -S python3 -u + +# Note that running python with the `-u` flag is required on Windows, +# in order to ensure that stdin and stdout are opened in binary, rather +# than text, mode. + +import sys +import json +import struct + +# Read a message from stdin and decode it. +def getMessage(): + rawLength = sys.stdin.buffer.read(4) + if len(rawLength) == 0: + sys.exit(0) + messageLength = struct.unpack('@I', rawLength)[0] + message = sys.stdin.buffer.read(messageLength).decode('utf-8') + return json.loads(message) + +# Encode a message for transmission, +# given its content. +def encodeMessage(messageContent): + # https://docs.python.org/3/library/json.html#basic-usage + # To get the most compact JSON representation, you should specify + # (',', ':') to eliminate whitespace. + # We want the most compact representation because the browser rejects # messages that exceed 1 MB. + encodedContent = json.dumps(messageContent, separators=(',', ':')).encode('utf-8') + encodedLength = struct.pack('@I', len(encodedContent)) + return {'length': encodedLength, 'content': encodedContent} + +# Send an encoded message to stdout +def sendMessage(encodedMessage): + sys.stdout.buffer.write(encodedMessage['length']) + sys.stdout.buffer.write(encodedMessage['content']) + sys.stdout.buffer.flush() + +while True: + receivedMessage = getMessage() + if receivedMessage == "ping": + sendMessage(encodeMessage("pong")) diff --git a/native_chrome_extension/openadapt_chrome.json b/native_chrome_extension/app/openadapt_chrome.json similarity index 84% rename from native_chrome_extension/openadapt_chrome.json rename to native_chrome_extension/app/openadapt_chrome.json index 5d3365583..a0e3074d8 100644 --- a/native_chrome_extension/openadapt_chrome.json +++ b/native_chrome_extension/app/openadapt_chrome.json @@ -1,7 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\app\\browser.bat", "type": "stdio", "allowed_origins": [ "chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/" diff --git a/native_chrome_extension/browser.bat b/native_chrome_extension/browser.bat deleted file mode 100644 index b10826375..000000000 --- a/native_chrome_extension/browser.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -call python "P:\OpenAdapt AI - MLDS AI\cloned_repo\OpenAdapt\native_chrome_extension\browser.py" diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py deleted file mode 100644 index fdf3dcddb..000000000 --- a/native_chrome_extension/browser.py +++ /dev/null @@ -1,7 +0,0 @@ -import nativemessaging - -while True: - message = nativemessaging.get_message() - print(message) - if message == "hello": - nativemessaging.send_message(nativemessaging.encode_message("world")) diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json deleted file mode 100644 index 57a68436f..000000000 --- a/native_chrome_extension/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "manifest_version": 3, - "name": "OpenAdapt DOM Listener", - "version": "1.0", - "description": "Add the reading time to Chrome Extension documentation articles", - "icons": { - "128": "Icons/MLDSAI_logo.png" - }, - "content_scripts": [ - { - "js": ["scripts/content.js"], - "matches": [""] - } - ], - "permissions": [ - "activeTab", - "tabs", - "nativeMessaging" - ], - "background": { - "service_worker": "scripts/background.js" - }, - "action": {} -} - diff --git a/native_chrome_extension/native-manifest.json b/native_chrome_extension/native-manifest.json deleted file mode 100644 index e0c4dd49f..000000000 --- a/native_chrome_extension/native-manifest.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "openadapt", - "description": "OpenAdapt DOM Listener", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", - "type": "stdio", - "allowed_origins": ["chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/"] -} diff --git a/native_chrome_extension/scripts/background.js b/native_chrome_extension/scripts/background.js deleted file mode 100644 index 1a39eb044..000000000 --- a/native_chrome_extension/scripts/background.js +++ /dev/null @@ -1,18 +0,0 @@ -// background.js - -// Establish a connection with the native app -var port = chrome.runtime.connectNative('openadapt'); - -port.onMessage.addListener(function (msg) { - console.log('Received: ' + msg); -}); - -port.onDisconnect.addListener(function () { - console.log('Disconnected'); -}); - -// On a click on the action, send a message to the native app -chrome.action.onClicked.addListener(function () { - console.log('Sending: hi'); - port.postMessage('hi'); -}); diff --git a/native_chrome_extension/scripts/content.js b/native_chrome_extension/scripts/content.js deleted file mode 100644 index 575f28daf..000000000 --- a/native_chrome_extension/scripts/content.js +++ /dev/null @@ -1,11 +0,0 @@ -function onReceived(response) { - console.log(response); -} - -// runtime.connectNative -var port = browser.runtime.connectNative("openadapt"); -port.onMessage.addListener(onReceived); -port.postMessage("hello"); - -// runtime.sendNativeMessage -browser.runtime.sendNativeMessage("openadapt", "hello").then(onReceived); \ No newline at end of file From 34f237c9551533ceb9e9306a3411df3476faa847 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 5 Jun 2023 20:16:15 -0400 Subject: [PATCH 20/89] ping pong example on chrome --- native_chrome_extension/add-on/background.js | 19 ------- native_chrome_extension/add-on/manifest.json | 29 ---------- native_chrome_extension/background.js | 50 ++++++++++++++++++ native_chrome_extension/{app => }/browser.bat | 2 +- native_chrome_extension/{app => }/browser.py | 4 +- .../{add-on => }/icons/MLDSAI_logo.png | Bin native_chrome_extension/manifest.json | 22 ++++++++ .../{app => }/openadapt_chrome.json | 6 +-- 8 files changed, 77 insertions(+), 55 deletions(-) delete mode 100644 native_chrome_extension/add-on/background.js delete mode 100644 native_chrome_extension/add-on/manifest.json create mode 100644 native_chrome_extension/background.js rename native_chrome_extension/{app => }/browser.bat (64%) rename native_chrome_extension/{app => }/browser.py (94%) rename native_chrome_extension/{add-on => }/icons/MLDSAI_logo.png (100%) create mode 100644 native_chrome_extension/manifest.json rename native_chrome_extension/{app => }/openadapt_chrome.json (53%) diff --git a/native_chrome_extension/add-on/background.js b/native_chrome_extension/add-on/background.js deleted file mode 100644 index 020e6edde..000000000 --- a/native_chrome_extension/add-on/background.js +++ /dev/null @@ -1,19 +0,0 @@ -/* -On startup, connect to the "ping_pong" app. -*/ -let port = browser.runtime.connectNative("ping_pong"); - -/* -Listen for messages from the app. -*/ -port.onMessage.addListener((response) => { - console.log("Received: " + response); -}); - -/* -On a click on the browser action, send the app a message. -*/ -browser.browserAction.onClicked.addListener(() => { - console.log("Sending: ping"); - port.postMessage("ping"); -}); diff --git a/native_chrome_extension/add-on/manifest.json b/native_chrome_extension/add-on/manifest.json deleted file mode 100644 index 2f41f7e4f..000000000 --- a/native_chrome_extension/add-on/manifest.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - - "description": "Native messaging example add-on", - "manifest_version": 2, - "name": "Native messaging example", - "version": "1.0", - "icons": { - "48": "icons/message.svg" - }, - - "browser_specific_settings": { - "gecko": { - "id": "ping_pong@example.org", - "strict_min_version": "50.0" - } - }, - - "background": { - "scripts": ["background.js"] - }, - - "browser_action": { - "default_icon": "icons/message.svg" - }, - - "permissions": ["nativeMessaging"] - - } - \ No newline at end of file diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js new file mode 100644 index 000000000..641ef7feb --- /dev/null +++ b/native_chrome_extension/background.js @@ -0,0 +1,50 @@ +/* +On startup, connect to the "ping_pong" app. +*/ +let port = chrome.runtime.connectNative("openadapt"); + +// Receive messages from the native messaging host +port.onMessage.addListener(onReceived); + +// Handle received messages +function onReceived(response) { + console.log(response); +} + +// /* +// Listen for messages from the app. +// */ +// port.onMessage.addListener((response) => { +// console.log("Received: " + response); +// // port.postMessage(message); +// }); + +// Listen for messages from the content script +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + // Forward the message to the native messaging host + port.postMessage(message); +}); + + +// Send initial message to the native messaging host +port.postMessage("hello"); + + +/* +On a click on the browser action, send the app a message. +*/ +chrome.action.onClicked.addListener((tab) => { + port.postMessage("Thanks! For using OpenAdapt DOM Listener."); +}); + + +// DOM mutation observer +let observer = new MutationObserver(mutations => { + for (let mutation of mutations) { + // Send the mutation to the background script + port.postMessage(mutation); + } +}); + +// Start observing DOM mutations +observer.observe(document, { childList: true, subtree: true }); diff --git a/native_chrome_extension/app/browser.bat b/native_chrome_extension/browser.bat similarity index 64% rename from native_chrome_extension/app/browser.bat rename to native_chrome_extension/browser.bat index 92ce50f2d..788947cc7 100644 --- a/native_chrome_extension/app/browser.bat +++ b/native_chrome_extension/browser.bat @@ -1,3 +1,3 @@ @echo off -python -u "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\app\\browser.py" +python -u "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.py" diff --git a/native_chrome_extension/app/browser.py b/native_chrome_extension/browser.py similarity index 94% rename from native_chrome_extension/app/browser.py rename to native_chrome_extension/browser.py index 92a340425..2d8fc39ad 100644 --- a/native_chrome_extension/app/browser.py +++ b/native_chrome_extension/browser.py @@ -35,6 +35,6 @@ def sendMessage(encodedMessage): sys.stdout.buffer.flush() while True: + reply_num = 0 receivedMessage = getMessage() - if receivedMessage == "ping": - sendMessage(encodeMessage("pong")) + sendMessage(encodeMessage(str(receivedMessage))) diff --git a/native_chrome_extension/add-on/icons/MLDSAI_logo.png b/native_chrome_extension/icons/MLDSAI_logo.png similarity index 100% rename from native_chrome_extension/add-on/icons/MLDSAI_logo.png rename to native_chrome_extension/icons/MLDSAI_logo.png diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json new file mode 100644 index 000000000..02b1a0f25 --- /dev/null +++ b/native_chrome_extension/manifest.json @@ -0,0 +1,22 @@ +{ + + "name": "openadapt", + "description": "Native messaging example add-on", + "version": "1.0", + "manifest_version": 3, + "icons": { + "48": "icons/MLDSAI_logo.png" + }, + "action": { + "default_icon": "icons/MLDSAI_logo.png" + }, + "background": { + "service_worker": "./background.js" + }, + "permissions": [ + "nativeMessaging", + "scripting", + "activeTab" + ] + } + \ No newline at end of file diff --git a/native_chrome_extension/app/openadapt_chrome.json b/native_chrome_extension/openadapt_chrome.json similarity index 53% rename from native_chrome_extension/app/openadapt_chrome.json rename to native_chrome_extension/openadapt_chrome.json index a0e3074d8..4ef90ccda 100644 --- a/native_chrome_extension/app/openadapt_chrome.json +++ b/native_chrome_extension/openadapt_chrome.json @@ -1,9 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\app\\browser.bat", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", "type": "stdio", - "allowed_origins": [ - "chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/" - ] + "allowed_origins": ["chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/"] } \ No newline at end of file From 64df009a6fbcc01888a2b263a6a6f81929dd9e8c Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 7 Jun 2023 07:01:57 -0400 Subject: [PATCH 21/89] New Tab Listner --- native_chrome_extension/background.js | 22 +--------------------- native_chrome_extension/content.js | 1 + native_chrome_extension/manifest.json | 13 +++++++++---- 3 files changed, 11 insertions(+), 25 deletions(-) create mode 100644 native_chrome_extension/content.js diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 641ef7feb..7db69268f 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -1,6 +1,4 @@ -/* -On startup, connect to the "ping_pong" app. -*/ + let port = chrome.runtime.connectNative("openadapt"); // Receive messages from the native messaging host @@ -11,14 +9,6 @@ function onReceived(response) { console.log(response); } -// /* -// Listen for messages from the app. -// */ -// port.onMessage.addListener((response) => { -// console.log("Received: " + response); -// // port.postMessage(message); -// }); - // Listen for messages from the content script chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { // Forward the message to the native messaging host @@ -38,13 +28,3 @@ chrome.action.onClicked.addListener((tab) => { }); -// DOM mutation observer -let observer = new MutationObserver(mutations => { - for (let mutation of mutations) { - // Send the mutation to the background script - port.postMessage(mutation); - } -}); - -// Start observing DOM mutations -observer.observe(document, { childList: true, subtree: true }); diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js new file mode 100644 index 000000000..562e031b5 --- /dev/null +++ b/native_chrome_extension/content.js @@ -0,0 +1 @@ +chrome.runtime.sendMessage("Hello from content.js!"); \ No newline at end of file diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json index 02b1a0f25..f6b16967f 100644 --- a/native_chrome_extension/manifest.json +++ b/native_chrome_extension/manifest.json @@ -11,12 +11,17 @@ "default_icon": "icons/MLDSAI_logo.png" }, "background": { - "service_worker": "./background.js" + "service_worker": "background.js" }, "permissions": [ + "activeTab", + "tabs", "nativeMessaging", - "scripting", - "activeTab" - ] + "scripting" + ], + "content_scripts": [{ + "matches": [""], + "js": ["content.js"] + }] } \ No newline at end of file From c0e4e4807ed2bfe3e63600bfca161c8f6a2e1f61 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 7 Jun 2023 10:37:05 -0400 Subject: [PATCH 22/89] add content script for accessing document object --- native_chrome_extension/background.js | 71 +++++++++++++++++++++------ native_chrome_extension/browser.py | 10 +++- native_chrome_extension/content.js | 19 ++++++- 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 7db69268f..8d0013a64 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -1,30 +1,71 @@ +// // Handle received messages +// function onReceived(response) { +// console.log(response); +// } -let port = chrome.runtime.connectNative("openadapt"); +// let port = chrome.runtime.connectNative("openadapt"); +// port.onMessage.addListener(onReceived); // Receive messages from the native messaging host +// port.postMessage("hello"); -// Receive messages from the native messaging host -port.onMessage.addListener(onReceived); +// /* +// On a click on the browser action, send the app a message. +// */ +// chrome.action.onClicked.addListener((tab) => { +// port.postMessage("Thanks! For using OpenAdapt DOM Listener."); +// }); + + +// let observer = new MutationObserver(mutations => { +// for(let mutation of mutations) { +// chrome.runtime.sendNativeMessage("openadapt", mutation).then(onReceived); +// } +// }); +// observer.observe(document, { childList: true, subtree: true }); + + +// Native Messaging port +let port = null; // Handle received messages function onReceived(response) { - console.log(response); + console.log(response); } -// Listen for messages from the content script -chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { - // Forward the message to the native messaging host - port.postMessage(message); -}); +const hostName = 'openadapt'; // Replace with your Native Messaging host name +port = chrome.runtime.connectNative(hostName); +port.onMessage.addListener(onReceived); // Receive messages from the Native Messaging host +port.postMessage('hello'); + + +// Send document to the Native Messaging host +function sendDocument(documentHTML) { + if (port) { + const message = { + type: 'document', + html: documentHTML, + }; + + port.postMessage(message); + console.log('Sent document to Native Messaging host:', message); + } +} + +// Message listener for content script +function messageListener(message, sender, sendResponse) { + if (message.type === 'document') { + sendDocument(message.html); + } +} -// Send initial message to the native messaging host -port.postMessage("hello"); + +// Listen for messages from content scripts +chrome.runtime.onMessage.addListener(messageListener); /* -On a click on the browser action, send the app a message. +On a click on the browser action, send a message to the active tab's content script. */ chrome.action.onClicked.addListener((tab) => { - port.postMessage("Thanks! For using OpenAdapt DOM Listener."); + chrome.tabs.sendMessage(tab.id, { type: 'getDocument' }); }); - - diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 2d8fc39ad..5bb26015b 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -8,6 +8,7 @@ import json import struct + # Read a message from stdin and decode it. def getMessage(): rawLength = sys.stdin.buffer.read(4) @@ -17,6 +18,7 @@ def getMessage(): message = sys.stdin.buffer.read(messageLength).decode('utf-8') return json.loads(message) + # Encode a message for transmission, # given its content. def encodeMessage(messageContent): @@ -28,13 +30,17 @@ def encodeMessage(messageContent): encodedLength = struct.pack('@I', len(encodedContent)) return {'length': encodedLength, 'content': encodedContent} + # Send an encoded message to stdout def sendMessage(encodedMessage): sys.stdout.buffer.write(encodedMessage['length']) sys.stdout.buffer.write(encodedMessage['content']) sys.stdout.buffer.flush() + +reply_num = 0 while True: - reply_num = 0 receivedMessage = getMessage() - sendMessage(encodeMessage(str(receivedMessage))) + reply_num += 1 + sendMessage(encodeMessage(str(reply_num))) + sendMessage(encodeMessage(receivedMessage)) diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 562e031b5..aa0583935 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -1 +1,18 @@ -chrome.runtime.sendMessage("Hello from content.js!"); \ No newline at end of file +chrome.runtime.sendMessage("Hello from content.js!"); + + +// Function to send the document to the background script +function sendDocument(doc) { + chrome.runtime.sendMessage({ type: 'document', document: doc }); +} + + +// Listen for DOM changes and send the document to the background script +document.addEventListener('DOMContentLoaded', () => { + + // Send the initial document + sendDocument(document); + + // Listen for DOM changes + document.addEventListener('DOMSubtreeModified', () => sendDocument(document)); +}); From b484f7eccb959872171c708ba03cfec8816f23bc Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 7 Jun 2023 10:41:33 -0400 Subject: [PATCH 23/89] remove old code --- native_chrome_extension/background.js | 36 ++------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 8d0013a64..7eab428b9 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -1,27 +1,3 @@ -// // Handle received messages -// function onReceived(response) { -// console.log(response); -// } - -// let port = chrome.runtime.connectNative("openadapt"); -// port.onMessage.addListener(onReceived); // Receive messages from the native messaging host -// port.postMessage("hello"); - -// /* -// On a click on the browser action, send the app a message. -// */ -// chrome.action.onClicked.addListener((tab) => { -// port.postMessage("Thanks! For using OpenAdapt DOM Listener."); -// }); - - -// let observer = new MutationObserver(mutations => { -// for(let mutation of mutations) { -// chrome.runtime.sendNativeMessage("openadapt", mutation).then(onReceived); -// } -// }); -// observer.observe(document, { childList: true, subtree: true }); - // Native Messaging port let port = null; @@ -32,9 +8,9 @@ function onReceived(response) { } -const hostName = 'openadapt'; // Replace with your Native Messaging host name +const hostName = 'openadapt'; port = chrome.runtime.connectNative(hostName); -port.onMessage.addListener(onReceived); // Receive messages from the Native Messaging host +port.onMessage.addListener(onReceived); port.postMessage('hello'); @@ -61,11 +37,3 @@ function messageListener(message, sender, sendResponse) { // Listen for messages from content scripts chrome.runtime.onMessage.addListener(messageListener); - - -/* -On a click on the browser action, send a message to the active tab's content script. -*/ -chrome.action.onClicked.addListener((tab) => { - chrome.tabs.sendMessage(tab.id, { type: 'getDocument' }); -}); From 2847d1af8529d4295e2bad1ddb071ebe1309c06b Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 7 Jun 2023 11:11:32 -0400 Subject: [PATCH 24/89] Now, the extensions sends 1). the document 2). reply_num (gets added by 1 on every change) on every mutation of DOM --- native_chrome_extension/background.js | 32 +++++---------------------- native_chrome_extension/content.js | 28 ++++++++++++----------- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 7eab428b9..ed7f96785 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -1,6 +1,7 @@ - // Native Messaging port let port = null; +const hostName = 'openadapt'; + // Handle received messages function onReceived(response) { @@ -8,32 +9,11 @@ function onReceived(response) { } -const hostName = 'openadapt'; -port = chrome.runtime.connectNative(hostName); -port.onMessage.addListener(onReceived); -port.postMessage('hello'); - - -// Send document to the Native Messaging host -function sendDocument(documentHTML) { - if (port) { - const message = { - type: 'document', - html: documentHTML, - }; - - port.postMessage(message); - console.log('Sent document to Native Messaging host:', message); - } -} - // Message listener for content script function messageListener(message, sender, sendResponse) { - if (message.type === 'document') { - sendDocument(message.html); - } + port.postMessage(message); } - -// Listen for messages from content scripts -chrome.runtime.onMessage.addListener(messageListener); +port = chrome.runtime.connectNative(hostName); +port.onMessage.addListener(onReceived); +chrome.runtime.onMessage.addListener(messageListener); // Listen for messages from content scripts diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index aa0583935..0e04d84d8 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -1,18 +1,20 @@ -chrome.runtime.sendMessage("Hello from content.js!"); - +// Function to extract relevant information from the document +function extractDocumentInfo() { + const documentInfo = { + title: document.title, + url: window.location.href, + html: document.documentElement.outerHTML + }; + return documentInfo; +} -// Function to send the document to the background script -function sendDocument(doc) { - chrome.runtime.sendMessage({ type: 'document', document: doc }); +// Send the document info to the background script +function sendDocumentInfo() { + const documentInfo = extractDocumentInfo(); + chrome.runtime.sendMessage({ type: 'document', documentInfo }); } -// Listen for DOM changes and send the document to the background script -document.addEventListener('DOMContentLoaded', () => { +sendDocumentInfo(); // Send the initial document info - // Send the initial document - sendDocument(document); - - // Listen for DOM changes - document.addEventListener('DOMSubtreeModified', () => sendDocument(document)); -}); +document.addEventListener('DOMSubtreeModified', sendDocumentInfo); // Listen for DOM changes and send the updated document info From aa44f3848a8b40b53412dd8b8372375e46ce5416 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 30 Jun 2023 13:21:00 -0400 Subject: [PATCH 25/89] Mutation Observer in content.js --- native_chrome_extension/content.js | 43 ++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 0e04d84d8..16f820640 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -1,20 +1,35 @@ -// Function to extract relevant information from the document -function extractDocumentInfo() { - const documentInfo = { - title: document.title, - url: window.location.href, - html: document.documentElement.outerHTML - }; - return documentInfo; +let port = null; // Native Messaging port +const hostName = 'openadapt'; + + +/* + * Handle received messages +*/ +function onReceived(response) { + console.log(response); } -// Send the document info to the background script -function sendDocumentInfo() { - const documentInfo = extractDocumentInfo(); - chrome.runtime.sendMessage({ type: 'document', documentInfo }); +/* + * Handle disconnection from native app +*/ +function onDisconnectedFromNativeHost() { + console.log('Disconnected from native host'); + port = null; } -sendDocumentInfo(); // Send the initial document info +port = chrome.runtime.connectNative(hostName); +port.onMessage.addListener(onReceived); +port.onDisconnect.addListener(onDisconnectedFromNativeHost); +console.log('Connected to native messaging host: ' + hostName); +port.postMessage({ text: "Hello, my_application" }); -document.addEventListener('DOMSubtreeModified', sendDocumentInfo); // Listen for DOM changes and send the updated document info +/* +* Forward messages from content scripts to the native app +*/ +chrome.runtime.onMessage.addListener(function (message, sender, sendResponse) { + if (message.action === 'logDOMChange') { + port.postMessage(message.documentInfo); + console.log('Message forwarded to native app'); + } +}); From 3acd453f00fd5ae6e306ac7de4c88268648ad312 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 30 Jun 2023 13:23:53 -0400 Subject: [PATCH 26/89] working DOM Mutation Observer COde --- native_chrome_extension/content.js | 51 ++++++++++++++---------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 16f820640..52db086e5 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -1,35 +1,32 @@ -let port = null; // Native Messaging port -const hostName = 'openadapt'; - - /* - * Handle received messages + * Function to send a message to the background script */ -function onReceived(response) { - console.log(response); +function sendMessageToBackgroundScript(message) { + chrome.runtime.sendMessage(message); } + /* - * Handle disconnection from native app + * Function to detect DOM changes */ -function onDisconnectedFromNativeHost() { - console.log('Disconnected from native host'); - port = null; -} +function detectDOMChanges() { + // Create a mutation observer + const observer = new MutationObserver(function(mutationsList) { + // Send a message to the background script when a DOM change is detected + sendMessageToBackgroundScript({ + action: 'logDOMChange', + documentObject: document, + }); + }); + // Start observing DOM changes + observer.observe(document, { + subtree: true, + childList: true, + attributes: true, + characterData: true, + }); +} -port = chrome.runtime.connectNative(hostName); -port.onMessage.addListener(onReceived); -port.onDisconnect.addListener(onDisconnectedFromNativeHost); -console.log('Connected to native messaging host: ' + hostName); -port.postMessage({ text: "Hello, my_application" }); - -/* -* Forward messages from content scripts to the native app -*/ -chrome.runtime.onMessage.addListener(function (message, sender, sendResponse) { - if (message.action === 'logDOMChange') { - port.postMessage(message.documentInfo); - console.log('Message forwarded to native app'); - } -}); +// Call the function to start detecting DOM changes +detectDOMChanges(); From 4efc2183ced87c0969617c77f55c01bbb041c4ac Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 30 Jun 2023 14:18:35 -0400 Subject: [PATCH 27/89] Chrome Extension - Progress: --using DOM Mutation Observer https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p --- native_chrome_extension/background.js | 4 +++- native_chrome_extension/content.js | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index ed7f96785..54f663431 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -3,14 +3,16 @@ let port = null; const hostName = 'openadapt'; -// Handle received messages +// Handle received messages from content.js function onReceived(response) { console.log(response); } + // Message listener for content script function messageListener(message, sender, sendResponse) { + console.log({ message, sender, sendResponse }); port.postMessage(message); } diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 52db086e5..a01fe8138 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -13,9 +13,10 @@ function detectDOMChanges() { // Create a mutation observer const observer = new MutationObserver(function(mutationsList) { // Send a message to the background script when a DOM change is detected + console.log({ mutationsList }) sendMessageToBackgroundScript({ action: 'logDOMChange', - documentObject: document, + documentObject: document.body.innerHTML, }); }); From 062e5af542f50ecf191f79c61fb3fde7a49b03a5 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 30 Jun 2023 14:19:33 -0400 Subject: [PATCH 28/89] add link for the system diagram --- native_chrome_extension/background.js | 4 ++++ native_chrome_extension/content.js | 3 +++ 2 files changed, 7 insertions(+) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 54f663431..706dd98ca 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -1,4 +1,8 @@ +// System Diagram: https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p + // Native Messaging port + + let port = null; const hostName = 'openadapt'; diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index a01fe8138..47ce1b82c 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -1,3 +1,6 @@ +// System Diagram: https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p + + /* * Function to send a message to the background script */ From 3eda05fe905b7ab8d3f2c9c815192f81153a8c25 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 30 Jun 2023 15:27:51 -0400 Subject: [PATCH 29/89] Update: -- sending document.outerHTML -- sending document.innerHTML Extension working properly --- native_chrome_extension/browser.py | 1 + native_chrome_extension/content.js | 24 +++++++++++++++++++++++- openadapt/models.py | 6 ++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 5bb26015b..d92807ed1 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -41,6 +41,7 @@ def sendMessage(encodedMessage): reply_num = 0 while True: receivedMessage = getMessage() + # pipe.send(receivedMessage) reply_num += 1 sendMessage(encodeMessage(str(reply_num))) sendMessage(encodeMessage(receivedMessage)) diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 47ce1b82c..ea076ca2a 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -17,9 +17,11 @@ function detectDOMChanges() { const observer = new MutationObserver(function(mutationsList) { // Send a message to the background script when a DOM change is detected console.log({ mutationsList }) + getElementPositions(); sendMessageToBackgroundScript({ action: 'logDOMChange', - documentObject: document.body.innerHTML, + documentBody: document.body.outerHTML, + documentHead: document.head.outerHTML }); }); @@ -34,3 +36,23 @@ function detectDOMChanges() { // Call the function to start detecting DOM changes detectDOMChanges(); + + +function getElementPositions() { + // iterate over all elements in the page + const elements = document.getElementsByTagName("*"); + + for (const element of elements) { + const rect = element.getBoundingClientRect(); + console.log({ rect }); + const attrs = ['top', 'right', 'bottom', 'left', 'width', 'height']; + for (const attr of attrs) { + element.setAttribute(`data-${attr}`, rect[attr]); + } + + } + // later, improve performance: + // - ignore elements that are not in the viewport + // - on update, use mutations to only update elements that have changed +} + diff --git a/openadapt/models.py b/openadapt/models.py index fea3a952a..d6b19ffbb 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -306,3 +306,9 @@ class PerformanceStat(db.Base): start_time = sa.Column(sa.Integer) end_time = sa.Column(sa.Integer) window_id = sa.Column(sa.String) + +class BrowserEvent(db.Base): + + bodyHTML = sa.Column(sa.String) + headHTML = sa.Column(sa.String) + \ No newline at end of file From 64134639fb70a7e2cf5908d8ade16cd0a795fc0e Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 6 Jul 2023 11:29:54 -0400 Subject: [PATCH 30/89] add ignoreAtributes --- native_chrome_extension/content.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index ea076ca2a..57bf2e815 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -16,7 +16,14 @@ function detectDOMChanges() { // Create a mutation observer const observer = new MutationObserver(function(mutationsList) { // Send a message to the background script when a DOM change is detected - console.log({ mutationsList }) + if (mutationsList.length === 0) { + return; + } + if (!logged) { + console.log({ mutationsList }) + // logged = true; + // debugger; + } getElementPositions(); sendMessageToBackgroundScript({ action: 'logDOMChange', @@ -36,6 +43,8 @@ function detectDOMChanges() { // Call the function to start detecting DOM changes detectDOMChanges(); +let logged = true; +let ignoreAttributes = new Set() function getElementPositions() { @@ -44,13 +53,14 @@ function getElementPositions() { for (const element of elements) { const rect = element.getBoundingClientRect(); - console.log({ rect }); + // console.log({ rect }); const attrs = ['top', 'right', 'bottom', 'left', 'width', 'height']; for (const attr of attrs) { element.setAttribute(`data-${attr}`, rect[attr]); + ignoreAttributes.add(`data-${attr}`) } - } + // later, improve performance: // - ignore elements that are not in the viewport // - on update, use mutations to only update elements that have changed From 0212e32bcaf20d49fe929f7bbd389dc6e5638585 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 6 Jul 2023 12:32:51 -0400 Subject: [PATCH 31/89] add hostPermissions --- native_chrome_extension/manifest.json | 1 + 1 file changed, 1 insertion(+) diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json index f6b16967f..76058d3f9 100644 --- a/native_chrome_extension/manifest.json +++ b/native_chrome_extension/manifest.json @@ -19,6 +19,7 @@ "nativeMessaging", "scripting" ], + "host_permissions": [""], "content_scripts": [{ "matches": [""], "js": ["content.js"] From b6219235181fd45216bcd06c4d9ab7bf122c4b04 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 6 Jul 2023 14:19:46 -0400 Subject: [PATCH 32/89] Now the Extensions sends: --document HTML on new tab and change in active Tab --and thereafter sends mutationList (emoty due to chrome security) --Bug: the extension doe snot work properly in the tab in which it was started --- native_chrome_extension/background.js | 32 +++++- native_chrome_extension/content.js | 142 ++++++++++++++++++++------ 2 files changed, 140 insertions(+), 34 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 706dd98ca..db5177214 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -7,19 +7,43 @@ let port = null; const hostName = 'openadapt'; -// Handle received messages from content.js +// Handle received messages from browser.js function onReceived(response) { console.log(response); } - // Message listener for content script function messageListener(message, sender, sendResponse) { - console.log({ message, sender, sendResponse }); + // console.log({ message, sender, sendResponse }); port.postMessage(message); } + +// Event listener for tab switch or new tab open +function handleTabEvent() { + chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) { + const activeTab = tabs[0]; + const message = { + action: 'afterTabEvent', + tabId: activeTab.id + }; + + // Check if the tab is ready to receive messages or content script + if (activeTab.status === 'complete') { + chrome.tabs.sendMessage(activeTab.id, message); + } + }); +} + + port = chrome.runtime.connectNative(hostName); port.onMessage.addListener(onReceived); -chrome.runtime.onMessage.addListener(messageListener); // Listen for messages from content scripts +chrome.runtime.onMessage.addListener(messageListener); + +// Event listeners for tab switch or new tab open +chrome.tabs.onActivated.addListener(handleTabEvent); +chrome.tabs.onCreated.addListener(handleTabEvent); +// chrome.tabs.onUpdated.addListener(handleTabEvent); +// chrome.tabs.onReplaced.addListener(handleTabEvent); +// chrome.tabs.onDetached.addListener(handleTabEvent); diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 57bf2e815..f374637f9 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -1,9 +1,11 @@ -// System Diagram: https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p +let logged = false; +let ignoreAttributes = new Set(); +let observer = null; // Reference to the MutationObserver /* * Function to send a message to the background script -*/ + */ function sendMessageToBackgroundScript(message) { chrome.runtime.sendMessage(message); } @@ -11,58 +13,138 @@ function sendMessageToBackgroundScript(message) { /* * Function to detect DOM changes -*/ -function detectDOMChanges() { - // Create a mutation observer - const observer = new MutationObserver(function(mutationsList) { - // Send a message to the background script when a DOM change is detected - if (mutationsList.length === 0) { + */ +function detectDOMChanges(mutationsList) { + // Send a message to the background script when a DOM change is detected + if (mutationsList.length === 0) { + return; + } + if (!logged) { + console.log({ mutationsList }); + // logged = true; + // debugger; + } + sendMessageToBackgroundScript({ + action: 'logDOMChange', + mutationsList: mutationsList + }); +} + + +/* + * Mutation observer callback function + */ +function handleMutation(mutationsList) { + const filteredMutations = []; + + // Filter out mutations related to ignored attributes + mutationsList.forEach((mutation) => { + if (ignoreAttributes.has(mutation.attributeName)) { return; } - if (!logged) { - console.log({ mutationsList }) - // logged = true; - // debugger; - } - getElementPositions(); - sendMessageToBackgroundScript({ - action: 'logDOMChange', - documentBody: document.body.outerHTML, - documentHead: document.head.outerHTML - }); + filteredMutations.push(mutation); }); + detectDOMChanges(filteredMutations); +} + + +/* + * Function to start observing DOM changes + */ +function startObservingDOMChanges() { + // Create a new mutation observer if it doesn't exist + if (!observer) { + observer = new MutationObserver(handleMutation); + } + // Start observing DOM changes observer.observe(document, { subtree: true, childList: true, attributes: true, - characterData: true, + attributeFilter: Array.from(ignoreAttributes), + characterData: true }); } -// Call the function to start detecting DOM changes -detectDOMChanges(); -let logged = true; -let ignoreAttributes = new Set() + +/* + * Function to stop observing DOM changes + */ +function stopObservingDOMChanges() { + if (observer) { + observer.disconnect(); + observer = null; + } +} +/* + * Function to capture initial document state and send it to the background script + */ +function captureInitialDocumentState() { + const documentBody = document.body.outerHTML; + const documentHead = document.head.outerHTML; + + sendMessageToBackgroundScript({ + action: 'captureInitialDocumentState', + documentBody: documentBody, + documentHead: documentHead + }); +} + + +/* + * Function to get element positions + */ function getElementPositions() { - // iterate over all elements in the page const elements = document.getElementsByTagName("*"); for (const element of elements) { const rect = element.getBoundingClientRect(); - // console.log({ rect }); const attrs = ['top', 'right', 'bottom', 'left', 'width', 'height']; for (const attr of attrs) { element.setAttribute(`data-${attr}`, rect[attr]); - ignoreAttributes.add(`data-${attr}`) + ignoreAttributes.add(`data-${attr}`); } } +} + + +/* + * After Tab Event processing +*/ +function afterTabEvent(message) { + // Perform additional actions based on the message received + console.log('A Tab Event was occurred and now content script is ready for receiving end'); + + // Stop observing DOM changes when switching to another tab or opening a new tab + stopObservingDOMChanges(); + + // Capture and send the initial document state + captureInitialDocumentState(); + + // Get element positions + getElementPositions(); - // later, improve performance: - // - ignore elements that are not in the viewport - // - on update, use mutations to only update elements that have changed + // Resume observing DOM changes + startObservingDOMChanges(); } + +// Message listener for messages from the background script +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + if (message.action === 'afterTabEvent') { + afterTabEvent(message); + } +}); + +// Call the function to capture initial document state +captureInitialDocumentState(); + +// Call the function to get element positions +getElementPositions(); + +// Call the function to start observing DOM changes +startObservingDOMChanges(); From c18f61382ddad1d44ddb4e87b61c12cccf170c03 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 6 Jul 2023 14:34:28 -0400 Subject: [PATCH 33/89] before test code --- native_chrome_extension/browser.py | 37 ++++++++++++++++++++++++------ native_chrome_extension/client.py | 15 ++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 native_chrome_extension/client.py diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index d92807ed1..aef9b4768 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -38,10 +38,33 @@ def sendMessage(encodedMessage): sys.stdout.buffer.flush() -reply_num = 0 -while True: - receivedMessage = getMessage() - # pipe.send(receivedMessage) - reply_num += 1 - sendMessage(encodeMessage(str(reply_num))) - sendMessage(encodeMessage(receivedMessage)) +# reply_num = 0 +# while True: +# receivedMessage = getMessage() +# # pipe.send(receivedMessage) +# reply_num += 1 +# sendMessage(encodeMessage(str(reply_num))) +# sendMessage(encodeMessage(receivedMessage)) + + + +# Create a new txt file to show the browser has connected +with open('browser_connected.txt', 'w') as f: + f.write('Browser connected') + +with NPopen('wt', name='test') as pipe: # writable text pipe + stream = pipe.wait() + sendMessage(encodeMessage("Client connected: " + str(stream))) + dom_change_num = 0 + while True: + receivedMessage = getMessage() + dom_change_num += 1 + + # Check if client (record.py) is still connected + if pipe.stream: + # Write message to the stream + pipe.stream.write(receivedMessage) + pipe.stream.flush() + + sendMessage(encodeMessage(str(dom_change_num))) + sendMessage(encodeMessage(receivedMessage)) \ No newline at end of file diff --git a/native_chrome_extension/client.py b/native_chrome_extension/client.py new file mode 100644 index 000000000..3d2da2c07 --- /dev/null +++ b/native_chrome_extension/client.py @@ -0,0 +1,15 @@ +import os +import subprocess as sp + +from namedpipe import NPopen + + +def client(): + with open('\\.\pipe\test', 'rt') as f: + while not f.closed: + line = f.readline() + if len(line) > 0: + print(line) + +if __name__ == '__main__': + client() From 0562292c82465c1d80ea5c2dd287dded5629190e Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 6 Jul 2023 14:35:48 -0400 Subject: [PATCH 34/89] resolve merge conflicts --- openadapt/models.py | 37 ++++++++++++++++++++++++------------- requirements.txt | 31 ------------------------------- 2 files changed, 24 insertions(+), 44 deletions(-) delete mode 100644 requirements.txt diff --git a/openadapt/models.py b/openadapt/models.py index d6b19ffbb..f76528200 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -6,7 +6,7 @@ import numpy as np import sqlalchemy as sa -from openadapt import db, utils, window +from openadapt import config, db, utils, window # https://groups.google.com/g/sqlalchemy/c/wlr7sShU6-k @@ -133,9 +133,9 @@ def canonical_key(self): ) def _text(self, canonical=False): - sep = self._text_sep - name_prefix = self._text_name_prefix - name_suffix = self._text_name_suffix + sep = config.ACTION_TEXT_SEP + name_prefix = config.ACTION_TEXT_NAME_PREFIX + name_suffix = config.ACTION_TEXT_NAME_SUFFIX if canonical: key_attr = self.canonical_key key_name_attr = self.canonical_key_name @@ -201,10 +201,6 @@ def __str__(self): rval = " ".join(attrs) return rval - _text_sep = "-" - _text_name_prefix = "<" - _text_name_suffix = ">" - @classmethod def from_children(cls, children_dicts): children = [ @@ -274,6 +270,18 @@ def take_screenshot(cls): screenshot = Screenshot(sct_img=sct_img) return screenshot + def crop_active_window(self, action_event): + window_event = action_event.window_event + width_ratio, height_ratio = utils.get_scale_ratios(action_event) + + x0 = window_event.left * width_ratio + y0 = window_event.top * height_ratio + x1 = x0 + window_event.width * width_ratio + y1 = y0 + window_event.height * height_ratio + + box = (x0, y0, x1, y1) + self._image = self._image.crop(box) + class WindowEvent(db.Base): __tablename__ = "window_event" @@ -307,8 +315,11 @@ class PerformanceStat(db.Base): end_time = sa.Column(sa.Integer) window_id = sa.Column(sa.String) -class BrowserEvent(db.Base): - - bodyHTML = sa.Column(sa.String) - headHTML = sa.Column(sa.String) - \ No newline at end of file + +class MemoryStat(db.Base): + __tablename__ = "memory_stat" + + id = sa.Column(sa.Integer, primary_key=True) + recording_timestamp = sa.Column(sa.Integer) + memory_usage_bytes = sa.Column(ForceFloat) + timestamp = sa.Column(ForceFloat) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 102862b12..000000000 --- a/requirements.txt +++ /dev/null @@ -1,31 +0,0 @@ -alembic==1.8.1 -ascii_magic==2.3.0 --e git+https://github.com/abrichr/atomacos.git; sys_platform == 'darwin' -bokeh==2.4.3 -clipboard==0.0.4 -deepdiff[optimize]==6.3.0 -dictalchemy3==1.0.0 -fire==0.4.0 -ipdb==0.13.11 -loguru==0.6.0 -matplotlib==3.6.2 -mss==6.1.0 -openai==0.27.5 -pandas==2.0.0 -pygetwindow==0.0.9; sys_platform == 'win32' -pyinstaller -setuptools-lint -sphinx -python-dotenv==1.0.0 -pywin32==306; sys_platform == 'win32' --e git+https://github.com/abrichr/pynput.git#egg=pynput -pytest==7.1.3 -rapidocr-onnxruntime==1.2.3 -scikit-learn==1.2.2 -scipy==1.9.3 -sqlalchemy==1.4.43 -tiktoken==0.4.0 -torch==2.0.0 -tqdm==4.64.0 -transformers==4.28.1 -nativemessaging==1.0.1 \ No newline at end of file From 3b7f2b7477fc424805c9c673dadced8f01444737 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 6 Jul 2023 16:05:23 -0400 Subject: [PATCH 35/89] add serve.py t o show pipe is working normally but not with extension --- native_chrome_extension/browser.py | 5 +--- native_chrome_extension/client.py | 15 +++++------- native_chrome_extension/server.py | 37 ++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 native_chrome_extension/server.py diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index aef9b4768..4ced98848 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -48,11 +48,8 @@ def sendMessage(encodedMessage): -# Create a new txt file to show the browser has connected -with open('browser_connected.txt', 'w') as f: - f.write('Browser connected') - with NPopen('wt', name='test') as pipe: # writable text pipe + sendMessage(encodeMessage("Waiting for client...")) stream = pipe.wait() sendMessage(encodeMessage("Client connected: " + str(stream))) dom_change_num = 0 diff --git a/native_chrome_extension/client.py b/native_chrome_extension/client.py index 3d2da2c07..585fc4b55 100644 --- a/native_chrome_extension/client.py +++ b/native_chrome_extension/client.py @@ -4,12 +4,9 @@ from namedpipe import NPopen -def client(): - with open('\\.\pipe\test', 'rt') as f: - while not f.closed: - line = f.readline() - if len(line) > 0: - print(line) - -if __name__ == '__main__': - client() +with open(r'\\.\pipe\test', 'rt') as f: + while not f.closed: + line = f.readline() + if len(line) > 0: + print(line) + diff --git a/native_chrome_extension/server.py b/native_chrome_extension/server.py new file mode 100644 index 000000000..c6eb15de3 --- /dev/null +++ b/native_chrome_extension/server.py @@ -0,0 +1,37 @@ +import subprocess as sp +from namedpipe import NPopen +import time + + +def get_dom_changes(n): + if n < 20: # test sending 20 dom_changes messages + return 'dom_change' + else: + return 'quit' + + +def server(): + with NPopen('wt', name='test') as pipe: # writable text pipe + print('Pipe Created = ', pipe.path) + + print('waiting for client to connect...') + stream = pipe.wait() + print('client connected') + + n = 0 + while True: + n += 1 + time.sleep(1) + message = get_dom_changes(n) + if message == 'quit': + break + + # Check if client (record.py) has connected + if pipe.stream: + message = message + '\n' + # Write message to the stream + pipe.stream.write('sent ' + str(n) + ' ' + message) + pipe.stream.flush() + +if __name__ == '__main__': + server() \ No newline at end of file From 9a7c29c6b19038f5395a2131a493c07fc5c0a245 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 7 Jul 2023 14:42:01 -0400 Subject: [PATCH 36/89] working Extension demo with sockets --- native_chrome_extension/background.js | 24 --------- native_chrome_extension/browser.py | 41 +++++++------- native_chrome_extension/client.py | 12 ----- native_chrome_extension/content.js | 68 +++++++----------------- native_chrome_extension/server.py | 37 ------------- native_chrome_extension/socket_client.py | 25 +++++++++ poetry.lock | 2 + 7 files changed, 64 insertions(+), 145 deletions(-) delete mode 100644 native_chrome_extension/client.py delete mode 100644 native_chrome_extension/server.py create mode 100644 native_chrome_extension/socket_client.py diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index db5177214..9deed9326 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -20,30 +20,6 @@ function messageListener(message, sender, sendResponse) { } -// Event listener for tab switch or new tab open -function handleTabEvent() { - chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) { - const activeTab = tabs[0]; - const message = { - action: 'afterTabEvent', - tabId: activeTab.id - }; - - // Check if the tab is ready to receive messages or content script - if (activeTab.status === 'complete') { - chrome.tabs.sendMessage(activeTab.id, message); - } - }); -} - - port = chrome.runtime.connectNative(hostName); port.onMessage.addListener(onReceived); chrome.runtime.onMessage.addListener(messageListener); - -// Event listeners for tab switch or new tab open -chrome.tabs.onActivated.addListener(handleTabEvent); -chrome.tabs.onCreated.addListener(handleTabEvent); -// chrome.tabs.onUpdated.addListener(handleTabEvent); -// chrome.tabs.onReplaced.addListener(handleTabEvent); -// chrome.tabs.onDetached.addListener(handleTabEvent); diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 4ced98848..f786c1a44 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -4,10 +4,16 @@ # in order to ensure that stdin and stdout are opened in binary, rather # than text, mode. +from loguru import logger +from multiprocessing.connection import Listener import sys import json import struct +import os +import subprocess as sp + +from namedpipe import NPopen # Read a message from stdin and decode it. def getMessage(): @@ -38,30 +44,19 @@ def sendMessage(encodedMessage): sys.stdout.buffer.flush() -# reply_num = 0 -# while True: -# receivedMessage = getMessage() -# # pipe.send(receivedMessage) -# reply_num += 1 -# sendMessage(encodeMessage(str(reply_num))) -# sendMessage(encodeMessage(receivedMessage)) - +SERVER_SENDS = True +PORT = 6001 +if __name__ == '__main__': + address = ('localhost', PORT) # family is deduced to be 'AF_INET' + listener = Listener(address, authkey=b'secret password') + conn = listener.accept() + logger.info(f'connection accepted from {listener.last_accepted=}') -with NPopen('wt', name='test') as pipe: # writable text pipe - sendMessage(encodeMessage("Waiting for client...")) - stream = pipe.wait() - sendMessage(encodeMessage("Client connected: " + str(stream))) - dom_change_num = 0 + reply_num = 0 while True: receivedMessage = getMessage() - dom_change_num += 1 - - # Check if client (record.py) is still connected - if pipe.stream: - # Write message to the stream - pipe.stream.write(receivedMessage) - pipe.stream.flush() - - sendMessage(encodeMessage(str(dom_change_num))) - sendMessage(encodeMessage(receivedMessage)) \ No newline at end of file + reply_num += 1 + conn.send(receivedMessage) + sendMessage(encodeMessage(str(reply_num))) + sendMessage(encodeMessage(receivedMessage)) diff --git a/native_chrome_extension/client.py b/native_chrome_extension/client.py deleted file mode 100644 index 585fc4b55..000000000 --- a/native_chrome_extension/client.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -import subprocess as sp - -from namedpipe import NPopen - - -with open(r'\\.\pipe\test', 'rt') as f: - while not f.closed: - line = f.readline() - if len(line) > 0: - print(line) - diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index f374637f9..f32ba9c22 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -11,6 +11,21 @@ function sendMessageToBackgroundScript(message) { } +/* + * Function to capture initial document state and send it to the background script + */ +function captureDocumentState() { + const documentBody = document.body.outerHTML; + const documentHead = document.head.outerHTML; + + sendMessageToBackgroundScript({ + action: 'captureDocumentState', + documentBody: documentBody, + documentHead: documentHead + }); +} + + /* * Function to detect DOM changes */ @@ -24,10 +39,7 @@ function detectDOMChanges(mutationsList) { // logged = true; // debugger; } - sendMessageToBackgroundScript({ - action: 'logDOMChange', - mutationsList: mutationsList - }); + captureDocumentState(); } @@ -80,21 +92,6 @@ function stopObservingDOMChanges() { } -/* - * Function to capture initial document state and send it to the background script - */ -function captureInitialDocumentState() { - const documentBody = document.body.outerHTML; - const documentHead = document.head.outerHTML; - - sendMessageToBackgroundScript({ - action: 'captureInitialDocumentState', - documentBody: documentBody, - documentHead: documentHead - }); -} - - /* * Function to get element positions */ @@ -112,39 +109,12 @@ function getElementPositions() { } -/* - * After Tab Event processing -*/ -function afterTabEvent(message) { - // Perform additional actions based on the message received - console.log('A Tab Event was occurred and now content script is ready for receiving end'); - - // Stop observing DOM changes when switching to another tab or opening a new tab - stopObservingDOMChanges(); - - // Capture and send the initial document state - captureInitialDocumentState(); - - // Get element positions - getElementPositions(); - - // Resume observing DOM changes - startObservingDOMChanges(); -} - - -// Message listener for messages from the background script -chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { - if (message.action === 'afterTabEvent') { - afterTabEvent(message); - } -}); - -// Call the function to capture initial document state -captureInitialDocumentState(); // Call the function to get element positions getElementPositions(); +// Call the function to capture initial document state +captureDocumentState(); + // Call the function to start observing DOM changes startObservingDOMChanges(); diff --git a/native_chrome_extension/server.py b/native_chrome_extension/server.py deleted file mode 100644 index c6eb15de3..000000000 --- a/native_chrome_extension/server.py +++ /dev/null @@ -1,37 +0,0 @@ -import subprocess as sp -from namedpipe import NPopen -import time - - -def get_dom_changes(n): - if n < 20: # test sending 20 dom_changes messages - return 'dom_change' - else: - return 'quit' - - -def server(): - with NPopen('wt', name='test') as pipe: # writable text pipe - print('Pipe Created = ', pipe.path) - - print('waiting for client to connect...') - stream = pipe.wait() - print('client connected') - - n = 0 - while True: - n += 1 - time.sleep(1) - message = get_dom_changes(n) - if message == 'quit': - break - - # Check if client (record.py) has connected - if pipe.stream: - message = message + '\n' - # Write message to the stream - pipe.stream.write('sent ' + str(n) + ' ' + message) - pipe.stream.flush() - -if __name__ == '__main__': - server() \ No newline at end of file diff --git a/native_chrome_extension/socket_client.py b/native_chrome_extension/socket_client.py new file mode 100644 index 000000000..efcbb7404 --- /dev/null +++ b/native_chrome_extension/socket_client.py @@ -0,0 +1,25 @@ +from multiprocessing.connection import Client + +from loguru import logger + +import time + +SERVER_SENDS = True +PORT = 6001 + +if __name__ == "__main__": + address = ('localhost', PORT) + conn = Client(address, authkey=b'secret password') + while True: + if SERVER_SENDS: + logger.info(f"waiting for message...") + msg = conn.recv() + logger.info(f"{msg=}") + else: + t = time.time() + logger.info(f"sending {t=}") + conn.send(t) + time.sleep(1) + # can also send arbitrary objects: + # conn.send(['a', 2.5, None, int, sum]) + conn.close() diff --git a/poetry.lock b/poetry.lock index 4a0ccd962..2189ae7f2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2642,6 +2642,8 @@ python-versions = ">=3.6" files = [ {file = "opencv-python-4.8.0.74.tar.gz", hash = "sha256:009e3ce356a0cd2d7423723e00a32fd3d3cc5bb5970ed27a9a1f8a8f221d1db5"}, {file = "opencv_python-4.8.0.74-cp37-abi3-macosx_10_16_x86_64.whl", hash = "sha256:31d0d59fc8fdf703de4cec46c79b9f8d026fdde9d23d6e2e6a66809feeebbda9"}, + {file = "opencv_python-4.8.0.74-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:66eadb5882ee56848b67f9fb57aadcaca2f4c9d9d00a0ef11043041925b51291"}, + {file = "opencv_python-4.8.0.74-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:038ba7075e55cb8e2846663ae970f0fb776a45b48ee69a887bf4ee15e2570083"}, {file = "opencv_python-4.8.0.74-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43dd0dfe331fb95767af581bf3b2781d7a72cf6560ddf2f55949fe547f3e5c9f"}, {file = "opencv_python-4.8.0.74-cp37-abi3-win32.whl", hash = "sha256:458e5dc377f15fcf769d80314f3d885bd95457b1a2891bee67df2eb24a1d3a52"}, {file = "opencv_python-4.8.0.74-cp37-abi3-win_amd64.whl", hash = "sha256:8fe0018d0056a5187c57120b6b3f6c3e706c13b45c48e54e86d245a9a16fac84"}, From 1dff338c1f8ecb483ef9ca4b9d2c6430c15a45eb Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 7 Jul 2023 14:42:43 -0400 Subject: [PATCH 37/89] add ToDO --- native_chrome_extension/browser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index f786c1a44..ab33f3cfb 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -44,6 +44,9 @@ def sendMessage(encodedMessage): sys.stdout.buffer.flush() +# TODO: send the Javascript memory state to the server + + SERVER_SENDS = True PORT = 6001 From 59a10293e5873ca1618316f2e3f302cf494e693e Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 7 Jul 2023 14:43:59 -0400 Subject: [PATCH 38/89] add todos --- native_chrome_extension/browser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index ab33f3cfb..20d1bbebc 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -45,6 +45,8 @@ def sendMessage(encodedMessage): # TODO: send the Javascript memory state to the server +# TODO: also send the window.location.href to the server + SERVER_SENDS = True From 863dee3eabcd33f1da697cde305a593a6d226361 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 8 Jul 2023 10:39:26 -0400 Subject: [PATCH 39/89] add Todos --- openadapt/models.py | 8 ++++++++ openadapt/record.py | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/openadapt/models.py b/openadapt/models.py index f76528200..aa9e2c4b9 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -323,3 +323,11 @@ class MemoryStat(db.Base): recording_timestamp = sa.Column(sa.Integer) memory_usage_bytes = sa.Column(ForceFloat) timestamp = sa.Column(ForceFloat) + + +class BrowserEvent(db.Base): + + bodyHTML = sa.Column(sa.String) + headHTML = sa.Column(sa.String) + url = sa.Column(sa.String) + timestamp = sa.Column(ForceFloat) diff --git a/openadapt/record.py b/openadapt/record.py index 1668e205d..d53845b1f 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -153,6 +153,7 @@ def process_events( prev_screen_event = event elif event.type == "window": prev_window_event = event + # TODO: elif event.type == "browser" elif event.type == "action": if prev_screen_event is None: logger.warning("discarding action that came before screen") @@ -162,6 +163,7 @@ def process_events( continue event.data["screenshot_timestamp"] = prev_screen_event.timestamp event.data["window_event_timestamp"] = prev_window_event.timestamp + # TODO: save browser_event_timestamp to event.data process_event( event, action_write_q, @@ -713,6 +715,8 @@ def record( ) window_event_reader.start() + # TODO: browser_event_reader + screen_event_reader = threading.Thread( target=read_screen_events, args=(event_q, terminate_event, recording_timestamp), @@ -759,6 +763,8 @@ def record( ) screen_event_writer.start() + # TODO: browser_event_writer + action_event_writer = multiprocessing.Process( target=write_events, args=( From 8b4ad900d9ff16107eb4c2b689ad10349450bb3e Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 8 Jul 2023 11:09:32 -0400 Subject: [PATCH 40/89] format Js files --- native_chrome_extension/background.js | 11 +++++++---- native_chrome_extension/content.js | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 9deed9326..f3e55b963 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -1,19 +1,22 @@ // System Diagram: https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p -// Native Messaging port - +// Native Messaging port let port = null; const hostName = 'openadapt'; -// Handle received messages from browser.js +/* + * Handle received messages from browser.js +*/ function onReceived(response) { console.log(response); } -// Message listener for content script +/* + * Message listener for content script +*/ function messageListener(message, sender, sendResponse) { // console.log({ message, sender, sendResponse }); port.postMessage(message); diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index f32ba9c22..49995489a 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -17,11 +17,13 @@ function sendMessageToBackgroundScript(message) { function captureDocumentState() { const documentBody = document.body.outerHTML; const documentHead = document.head.outerHTML; + const page_url = window.location.href; sendMessageToBackgroundScript({ action: 'captureDocumentState', documentBody: documentBody, - documentHead: documentHead + documentHead: documentHead, + url: page_url }); } @@ -109,7 +111,6 @@ function getElementPositions() { } - // Call the function to get element positions getElementPositions(); From f761da7478ab3deab1091ed7abf245534a643ed7 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 8 Jul 2023 11:15:33 -0400 Subject: [PATCH 41/89] add documentatoin --- native_chrome_extension/background.js | 7 ++++++- native_chrome_extension/content.js | 25 ++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index f3e55b963..e3cded6ab 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -1,4 +1,9 @@ -// System Diagram: https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p +/** + * @file background.js + * @description This file is responsible for communicating with the native + * messaging host and the content script. + * @see https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p + */ // Native Messaging port diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 49995489a..9004eb864 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -1,11 +1,17 @@ +/** + * @file content.js + * @description This file is injected into the web page and is responsible for + * capturing DOM changes and sending them to the background script. +*/ + let logged = false; let ignoreAttributes = new Set(); -let observer = null; // Reference to the MutationObserver +let observer = null; /* * Function to send a message to the background script - */ +*/ function sendMessageToBackgroundScript(message) { chrome.runtime.sendMessage(message); } @@ -13,7 +19,7 @@ function sendMessageToBackgroundScript(message) { /* * Function to capture initial document state and send it to the background script - */ +*/ function captureDocumentState() { const documentBody = document.body.outerHTML; const documentHead = document.head.outerHTML; @@ -30,7 +36,7 @@ function captureDocumentState() { /* * Function to detect DOM changes - */ +*/ function detectDOMChanges(mutationsList) { // Send a message to the background script when a DOM change is detected if (mutationsList.length === 0) { @@ -47,7 +53,7 @@ function detectDOMChanges(mutationsList) { /* * Mutation observer callback function - */ +*/ function handleMutation(mutationsList) { const filteredMutations = []; @@ -65,7 +71,7 @@ function handleMutation(mutationsList) { /* * Function to start observing DOM changes - */ +*/ function startObservingDOMChanges() { // Create a new mutation observer if it doesn't exist if (!observer) { @@ -85,7 +91,7 @@ function startObservingDOMChanges() { /* * Function to stop observing DOM changes - */ +*/ function stopObservingDOMChanges() { if (observer) { observer.disconnect(); @@ -96,7 +102,7 @@ function stopObservingDOMChanges() { /* * Function to get element positions - */ +*/ function getElementPositions() { const elements = document.getElementsByTagName("*"); @@ -111,11 +117,8 @@ function getElementPositions() { } -// Call the function to get element positions getElementPositions(); -// Call the function to capture initial document state captureDocumentState(); -// Call the function to start observing DOM changes startObservingDOMChanges(); From 56a320e5367dbdca6b6b7a6c91b5fab506465972 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 8 Jul 2023 11:17:10 -0400 Subject: [PATCH 42/89] minor changes in formatting --- native_chrome_extension/background.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index e3cded6ab..97501b284 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -6,8 +6,7 @@ */ -// Native Messaging port -let port = null; +let port = null; // Native Messaging port const hostName = 'openadapt'; @@ -29,5 +28,6 @@ function messageListener(message, sender, sendResponse) { port = chrome.runtime.connectNative(hostName); + port.onMessage.addListener(onReceived); chrome.runtime.onMessage.addListener(messageListener); From 346772e3f4dae9862bf12a9ecc31efeec4dea1f5 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 8 Jul 2023 11:43:55 -0400 Subject: [PATCH 43/89] format multiple files --- native_chrome_extension/background.js | 2 +- native_chrome_extension/browser.py | 88 +++++++++++++++++++-------- native_chrome_extension/content.js | 3 +- openadapt/config.py | 2 + 4 files changed, 66 insertions(+), 29 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 97501b284..737b5d418 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -23,7 +23,7 @@ function onReceived(response) { */ function messageListener(message, sender, sendResponse) { // console.log({ message, sender, sendResponse }); - port.postMessage(message); + port.postMessage(message); // send to browser.py (native messaging host) } diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 20d1bbebc..0eb694c90 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -1,46 +1,81 @@ #!/usr/bin/env -S python3 -u +"""Script for communicating with the browser extension. + +Usage: + + See `native_chrome_extension/browser.bat`. + +""" + # Note that running python with the `-u` flag is required on Windows, # in order to ensure that stdin and stdout are opened in binary, rather # than text, mode. -from loguru import logger from multiprocessing.connection import Listener -import sys +from typing import Any import json +import os import struct +import sys -import os +from loguru import logger import subprocess as sp -from namedpipe import NPopen +from openadapt import config + -# Read a message from stdin and decode it. -def getMessage(): +SERVER_SENDS = True + + +def getMessage() -> Any: + """Read a message from stdin and decode it. + + Args: + None + + Returns: + A Python object representing the message. + """ rawLength = sys.stdin.buffer.read(4) if len(rawLength) == 0: sys.exit(0) - messageLength = struct.unpack('@I', rawLength)[0] - message = sys.stdin.buffer.read(messageLength).decode('utf-8') + messageLength = struct.unpack("@I", rawLength)[0] + message = sys.stdin.buffer.read(messageLength).decode("utf-8") return json.loads(message) -# Encode a message for transmission, -# given its content. -def encodeMessage(messageContent): +def encodeMessage(messageContent: Any) -> dict[bytes, str]: + """Encode a message for transmission, given its content. + + Args: + messageContent: The content of the message to be encoded. + + Returns: + A dictionary containing the encoded message. + """ # https://docs.python.org/3/library/json.html#basic-usage - # To get the most compact JSON representation, you should specify + # To get the most compact JSON representation, you should specify # (',', ':') to eliminate whitespace. # We want the most compact representation because the browser rejects # messages that exceed 1 MB. - encodedContent = json.dumps(messageContent, separators=(',', ':')).encode('utf-8') - encodedLength = struct.pack('@I', len(encodedContent)) - return {'length': encodedLength, 'content': encodedContent} + encodedContent = json.dumps(messageContent, separators=(",", ":")).encode( + "utf-8" + ) + encodedLength = struct.pack("@I", len(encodedContent)) + return {"length": encodedLength, "content": encodedContent} -# Send an encoded message to stdout -def sendMessage(encodedMessage): - sys.stdout.buffer.write(encodedMessage['length']) - sys.stdout.buffer.write(encodedMessage['content']) +def sendMessage(encodedMessage: dict[bytes, str]) -> None: + """Send an encoded message to stdout + + Args: + encodedMessage: The encoded message to be sent. + + Returns: + None + """ + sys.stdout.buffer.write(encodedMessage["length"]) + sys.stdout.buffer.write(encodedMessage["content"]) sys.stdout.buffer.flush() @@ -48,15 +83,14 @@ def sendMessage(encodedMessage): # TODO: also send the window.location.href to the server - -SERVER_SENDS = True -PORT = 6001 - -if __name__ == '__main__': - address = ('localhost', PORT) # family is deduced to be 'AF_INET' - listener = Listener(address, authkey=b'secret password') +if __name__ == "__main__": + address = ( + "localhost", + config.SERVER_PORT, + ) # family is deduced to be 'AF_INET' + listener = Listener(address, authkey=config.SERVER_AUTHKEY) conn = listener.accept() - logger.info(f'connection accepted from {listener.last_accepted=}') + logger.info(f"connection accepted from {listener.last_accepted=}") reply_num = 0 while True: diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 9004eb864..4b3953e4e 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -18,7 +18,8 @@ function sendMessageToBackgroundScript(message) { /* - * Function to capture initial document state and send it to the background script + * Function to capture initial document state and + * send it to the background script */ function captureDocumentState() { const documentBody = document.body.outerHTML; diff --git a/openadapt/config.py b/openadapt/config.py index 695570186..f6aafd9ad 100644 --- a/openadapt/config.py +++ b/openadapt/config.py @@ -84,6 +84,8 @@ "children", ], "PLOT_PERFORMANCE": True, + "SOCKET_PORT": 6001, + "SERVER_AUTHKEY": b"openadapt", } # each string in STOP_STRS should only contain strings that don't contain special characters From aea191b30ef141fb8d253fcd1b17d53d96e98e6c Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Sat, 8 Jul 2023 11:48:48 -0400 Subject: [PATCH 44/89] format pylint browser --- native_chrome_extension/browser.py | 44 ++++++++++++++---------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 0eb694c90..eef0d11b7 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -15,12 +15,10 @@ from multiprocessing.connection import Listener from typing import Any import json -import os import struct import sys from loguru import logger -import subprocess as sp from openadapt import config @@ -28,7 +26,7 @@ SERVER_SENDS = True -def getMessage() -> Any: +def get_message() -> Any: """Read a message from stdin and decode it. Args: @@ -37,19 +35,19 @@ def getMessage() -> Any: Returns: A Python object representing the message. """ - rawLength = sys.stdin.buffer.read(4) - if len(rawLength) == 0: + raw_length = sys.stdin.buffer.read(4) + if len(raw_length) == 0: sys.exit(0) - messageLength = struct.unpack("@I", rawLength)[0] - message = sys.stdin.buffer.read(messageLength).decode("utf-8") + message_length = struct.unpack("@I", raw_length)[0] + message = sys.stdin.buffer.read(message_length).decode("utf-8") return json.loads(message) -def encodeMessage(messageContent: Any) -> dict[bytes, str]: +def encode_message(message_content: Any) -> dict[bytes, str]: """Encode a message for transmission, given its content. Args: - messageContent: The content of the message to be encoded. + message_content: The content of the message to be encoded. Returns: A dictionary containing the encoded message. @@ -57,30 +55,30 @@ def encodeMessage(messageContent: Any) -> dict[bytes, str]: # https://docs.python.org/3/library/json.html#basic-usage # To get the most compact JSON representation, you should specify # (',', ':') to eliminate whitespace. - # We want the most compact representation because the browser rejects # messages that exceed 1 MB. - encodedContent = json.dumps(messageContent, separators=(",", ":")).encode( + # We want the most compact representation because the browser rejects + # messages that exceed 1 MB. + encoded_content = json.dumps(message_content, separators=(",", ":")).encode( "utf-8" ) - encodedLength = struct.pack("@I", len(encodedContent)) - return {"length": encodedLength, "content": encodedContent} + encoded_length = struct.pack("@I", len(encoded_content)) + return {"length": encoded_length, "content": encoded_content} -def sendMessage(encodedMessage: dict[bytes, str]) -> None: +def send_message(encoded_message: dict[bytes, str]) -> None: """Send an encoded message to stdout Args: - encodedMessage: The encoded message to be sent. + encoded_message: The encoded message to be sent. Returns: None """ - sys.stdout.buffer.write(encodedMessage["length"]) - sys.stdout.buffer.write(encodedMessage["content"]) + sys.stdout.buffer.write(encoded_message["length"]) + sys.stdout.buffer.write(encoded_message["content"]) sys.stdout.buffer.flush() # TODO: send the Javascript memory state to the server -# TODO: also send the window.location.href to the server if __name__ == "__main__": @@ -92,10 +90,10 @@ def sendMessage(encodedMessage: dict[bytes, str]) -> None: conn = listener.accept() logger.info(f"connection accepted from {listener.last_accepted=}") - reply_num = 0 + DOM_CHANGE_NUM = 0 while True: - receivedMessage = getMessage() - reply_num += 1 + receivedMessage = get_message() + DOM_CHANGE_NUM += 1 conn.send(receivedMessage) - sendMessage(encodeMessage(str(reply_num))) - sendMessage(encodeMessage(receivedMessage)) + send_message(encode_message(str(DOM_CHANGE_NUM))) + send_message(encode_message(receivedMessage)) From 75b776fe0ef77ee679ae24e09ea3a807442987b9 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 10 Jul 2023 08:00:11 -0400 Subject: [PATCH 45/89] add read_browser_events and browser evnet queues --- openadapt/config.py | 2 +- openadapt/record.py | 62 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/openadapt/config.py b/openadapt/config.py index f6aafd9ad..be18e4cff 100644 --- a/openadapt/config.py +++ b/openadapt/config.py @@ -135,7 +135,7 @@ def obfuscate(val, pct_reveal=0.1, char="*"): -_OBFUSCATE_KEY_PARTS = ("KEY", "PASSWORD", "TOKEN") +_OBFUSCATE_KEY_PARTS = ("KEY", "PASSWORD", "TOKEN", "AUTHKEY") if multiprocessing.current_process().name == "MainProcess": for key, val in dict(locals()).items(): if not key.startswith("_") and key.isupper(): diff --git a/openadapt/record.py b/openadapt/record.py index d53845b1f..5010b0d30 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -499,6 +499,42 @@ def read_window_events( prev_window_data = window_data +def read_browser_events( + event_q: queue.Queue, + terminate_event: multiprocessing.Event, + recording_timestamp: float, +) -> None: + """ + Read browser events and add them to the event queue. + + Args: + event_q: A queue for adding window events. + terminate_event: An event to signal the termination of the process. + recording_timestamp: The timestamp of the recording. + """ + utils.configure_logging(logger, LOG_LEVEL) + utils.set_start_time(recording_timestamp) + logger.info(f"starting") + prev_browser_data = {} + + while not terminate_event.is_set(): + browser_data = # TODO (if the current window is Chrome then, get the DOM change data from the extension) + if not browser_data: + continue + + if browser_data != prev_browser_data: + # logger.info(f"{browser_data=}") + logger.debug("queuing browser event for writing") + event_q.put( + Event( + utils.get_timestamp(), + "browser", + browser_data, + ) + ) + prev_browser_data = browser_data + + @trace(logger) def performance_stats_writer( perf_q: sq.SynchronizedQueue, @@ -701,6 +737,7 @@ def record( screen_write_q = sq.SynchronizedQueue() action_write_q = sq.SynchronizedQueue() window_write_q = sq.SynchronizedQueue() + browser_write_q = sq.SynchronizedQueue() # TODO: save write times to DB; display performance plot in visualize.py perf_q = sq.SynchronizedQueue() terminate_event = multiprocessing.Event() @@ -708,6 +745,7 @@ def record( term_pipe_parent_window, term_pipe_child_window = multiprocessing.Pipe() term_pipe_parent_screen, term_pipe_child_screen = multiprocessing.Pipe() term_pipe_parent_action, term_pipe_child_action = multiprocessing.Pipe() + term_pipe_parent_browser, term_pipe_child_browser = multiprocessing.Pipe() window_event_reader = threading.Thread( target=read_window_events, @@ -715,7 +753,11 @@ def record( ) window_event_reader.start() - # TODO: browser_event_reader + browser_event_reader = threading.Thread( + target=read_browser_events, + args(event_q, terminate_event, recording_timestamp), + ) + browser_event_reader.start() screen_event_reader = threading.Thread( target=read_screen_events, @@ -742,6 +784,7 @@ def record( screen_write_q, action_write_q, window_write_q, + browser_write_q, perf_q, recording_timestamp, terminate_event, @@ -763,7 +806,19 @@ def record( ) screen_event_writer.start() - # TODO: browser_event_writer + browser_event_writer = multiprocessing.Process( + target=write_events, + args=( + "browser", + write_browser_event, + browser_event_q, + perf_q, + recording_timestamp, + terminate_event, + term_pipe_child_action + ), + ) + browser_event_writer.start() action_event_writer = multiprocessing.Process( target=write_events, @@ -832,16 +887,19 @@ def record( term_pipe_parent_window.send(window_write_q.qsize()) term_pipe_parent_action.send(action_write_q.qsize()) term_pipe_parent_screen.send(screen_write_q.qsize()) + term_pipe_parent_browser.send(browser_write_q.qsize()) logger.info(f"joining...") keyboard_event_reader.join() mouse_event_reader.join() screen_event_reader.join() window_event_reader.join() + browser_event_reader.join() event_processor.join() screen_event_writer.join() action_event_writer.join() window_event_writer.join() + browser_event_writer.join() terminate_perf_event.set() if PLOT_PERFORMANCE: From 1efa3e492e1331cefd4662145b11124c1b44ac23 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 10 Jul 2023 09:23:33 -0400 Subject: [PATCH 46/89] add browser writer function --- openadapt/record.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/openadapt/record.py b/openadapt/record.py index 5010b0d30..67f3a2423 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -256,6 +256,25 @@ def write_window_event( perf_q.put((event.type, event.timestamp, utils.get_timestamp())) +def write_browser_event( + recording_timestamp: float, + event: Event, + perf_q: sq.SynchronizedQueue, +): + """ + Write a browser event to the database and update the performance queue. + + Args: + recording_timestamp: The timestamp of the recording. + event: A browser event to be written. + perf_q: A queue for collecting performance data. + """ + + assert event.type == "browser", event + crud.insert_browser_event(recording_timestamp, event.timestamp, event.data) + perf_q.put((event.type, event.timestamp, utils.get_timestamp())) + + @trace(logger) def write_events( event_type: str, @@ -518,7 +537,7 @@ def read_browser_events( prev_browser_data = {} while not terminate_event.is_set(): - browser_data = # TODO (if the current window is Chrome then, get the DOM change data from the extension) + # browser_data = # TODO (if the current window is Chrome then, get the DOM change data from the extension) if not browser_data: continue From db1c2493322762f16f29cd611a1500bfa502dd22 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 10 Jul 2023 11:01:17 -0400 Subject: [PATCH 47/89] changes to the extension --- native_chrome_extension/background.js | 2 +- native_chrome_extension/browser.py | 8 +++----- native_chrome_extension/socket_client.py | 4 ++-- openadapt/config.py | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 737b5d418..ceea97bb7 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -22,7 +22,7 @@ function onReceived(response) { * Message listener for content script */ function messageListener(message, sender, sendResponse) { - // console.log({ message, sender, sendResponse }); + console.log({ message, sender, sendResponse }); port.postMessage(message); // send to browser.py (native messaging host) } diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index eef0d11b7..315a210dc 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -82,11 +82,9 @@ def send_message(encoded_message: dict[bytes, str]) -> None: if __name__ == "__main__": - address = ( - "localhost", - config.SERVER_PORT, - ) # family is deduced to be 'AF_INET' - listener = Listener(address, authkey=config.SERVER_AUTHKEY) + address = ("localhost", 6001) + # family is deduced to be 'AF_INET' + listener = Listener(address, authkey=config.SOCKET_AUTHKEY) conn = listener.accept() logger.info(f"connection accepted from {listener.last_accepted=}") diff --git a/native_chrome_extension/socket_client.py b/native_chrome_extension/socket_client.py index efcbb7404..ffaa9603c 100644 --- a/native_chrome_extension/socket_client.py +++ b/native_chrome_extension/socket_client.py @@ -8,8 +8,8 @@ PORT = 6001 if __name__ == "__main__": - address = ('localhost', PORT) - conn = Client(address, authkey=b'secret password') + address = ('localhost', config.PORT) + conn = Client(address, authkey=b'openadapt') while True: if SERVER_SENDS: logger.info(f"waiting for message...") diff --git a/openadapt/config.py b/openadapt/config.py index be18e4cff..7ec21b6c9 100644 --- a/openadapt/config.py +++ b/openadapt/config.py @@ -85,7 +85,7 @@ ], "PLOT_PERFORMANCE": True, "SOCKET_PORT": 6001, - "SERVER_AUTHKEY": b"openadapt", + "SOCKET_AUTHKEY": b"openadapt", } # each string in STOP_STRS should only contain strings that don't contain special characters From 1aade8e03a15fe9f3944252915454760faac1eac Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 11 Jul 2023 10:35:56 -0400 Subject: [PATCH 48/89] add browser events in multiple files --- openadapt/browser/__init__.py | 11 ++++++++++ openadapt/crud.py | 34 +++++++++++++++++++++++++++++ openadapt/events.py | 41 +++++++++++++++++++++++++---------- openadapt/models.py | 38 +++++++++++++++++++++++++------- openadapt/record.py | 18 ++++++++++++--- 5 files changed, 120 insertions(+), 22 deletions(-) create mode 100644 openadapt/browser/__init__.py diff --git a/openadapt/browser/__init__.py b/openadapt/browser/__init__.py new file mode 100644 index 000000000..59f7c2e40 --- /dev/null +++ b/openadapt/browser/__init__.py @@ -0,0 +1,11 @@ + +from typing import Any + + +# TODO: implement extension +# def get_active_chrome_data() -> Any: +# try: +# # TODO: implement +# except Exception as exc: +# logger.warning(f"{exc=}") +# return None diff --git a/openadapt/crud.py b/openadapt/crud.py index 9db3fa49f..359d40b4e 100644 --- a/openadapt/crud.py +++ b/openadapt/crud.py @@ -1,3 +1,5 @@ +from typing import Any, List + from loguru import logger import sqlalchemy as sa @@ -76,6 +78,26 @@ def insert_window_event(recording_timestamp, event_timestamp, event_data): _insert(event_data, WindowEvent, window_events) +def insert_browser_event( + recording_timestamp, + event_timestamp, + event_data +) -> None: + """Insert a browser event into the database. + + Args: + recording_timestamp (int): The timestamp of the recording. + event_timestamp (int): The timestamp of the event. + event_data (dict): The data of the event. + """ + event_data = { + **event_data, + "timestamp": event_timestamp, + "recording_timestamp": recording_timestamp, + } + _insert(event_data, BrowserEvent, browser_events) + + def insert_perf_stat(recording_timestamp, event_type, start_time, end_time): """ Insert event performance stat into db @@ -239,3 +261,15 @@ def get_screenshots(recording, precompute_diffs=False): def get_window_events(recording): return _get(WindowEvent, recording.timestamp) + + +def get_browser_events(recording) -> list[BrowserEvent]: + """Get browser events for a given recording + + Args: + recording (Recording): recording object + + Returns: + List[BrowserEvent]: list of browser events + """ + return _get(BrowserEvent, recording.timestamp) diff --git a/openadapt/events.py b/openadapt/events.py index d4eaf5bcb..5e3bc5a88 100644 --- a/openadapt/events.py +++ b/openadapt/events.py @@ -20,6 +20,7 @@ def get_events( action_events = crud.get_action_events(recording) window_events = crud.get_window_events(recording) screenshots = crud.get_screenshots(recording) + browser_events = crud.get_browser_events(recording) raw_action_event_dicts = utils.rows2dicts(action_events) logger.debug(f"raw_action_event_dicts=\n{pformat(raw_action_event_dicts)}") @@ -27,10 +28,12 @@ def get_events( num_action_events = len(action_events) num_window_events = len(window_events) num_screenshots = len(screenshots) + num_browser_events = len(browser_events) num_action_events_raw = num_action_events num_window_events_raw = num_window_events num_screenshots_raw = num_screenshots + num_browser_events_raw = num_browser_events duration_raw = action_events[-1].timestamp - action_events[0].timestamp num_process_iters = 0 @@ -41,20 +44,23 @@ def get_events( f"{num_action_events=} " f"{num_window_events=} " f"{num_screenshots=}" + f"{num_browser_events=}" ) - action_events, window_events, screenshots = process_events( - action_events, window_events, screenshots, + action_events, window_events, screenshots, browser_events = process_events( + action_events, window_events, screenshots, browser_events ) if ( len(action_events) == num_action_events and len(window_events) == num_window_events and - len(screenshots) == num_screenshots + len(screenshots) == num_screenshots and + len(browser_events) == num_browser_events ): break num_process_iters += 1 num_action_events = len(action_events) num_window_events = len(window_events) num_screenshots = len(screenshots) + num_browser_events = len(browser_events) if num_process_iters == MAX_PROCESS_ITERS: break @@ -72,6 +78,9 @@ def get_events( meta["num_screenshots"] = format_num( num_screenshots, num_screenshots_raw, ) + meta["num_browser_events"] = format_num( + num_browser_events, num_browser_events_raw, + ) duration = action_events[-1].timestamp - action_events[0].timestamp if len(action_events) > 1: @@ -82,7 +91,7 @@ def get_events( duration = end_time - start_time logger.info(f"{duration=}") - return action_events # , window_events, screenshots + return action_events # , window_events, screenshots, browser_events def make_parent_event(child, extra=None): @@ -93,9 +102,11 @@ def make_parent_event(child, extra=None): "recording_timestamp": child.recording_timestamp, "window_event_timestamp": child.window_event_timestamp, "screenshot_timestamp": child.screenshot_timestamp, + "browser_event_timestamp": child.browser_event_timestamp, "recording": child.recording, "window_event": child.window_event, "screenshot": child.screenshot, + "browser_event": child.browser_event, } extra = extra or {} for key, val in extra.items(): @@ -587,18 +598,20 @@ def discard_unused_events( return referred_events -def process_events(action_events, window_events, screenshots): +def process_events(action_events, window_events, screenshots, browser_events): # for debugging _action_events = action_events _window_events = window_events _screenshots = screenshots + _browser_events = browser_events num_action_events = len(action_events) num_window_events = len(window_events) num_screenshots = len(screenshots) - num_total = num_action_events + num_window_events + num_screenshots + num_browser_events = len(browser_events) + num_total = num_action_events + num_window_events + num_screenshots + num_browser_events logger.info( - f"before {num_action_events=} {num_window_events=} {num_screenshots=} " + f"before {num_action_events=} {num_window_events=} {num_screenshots=} {num_browser_events=} " f"{num_total=}" ) process_fns = [ @@ -625,21 +638,27 @@ def process_events(action_events, window_events, screenshots): screenshots = discard_unused_events( screenshots, action_events, "screenshot_timestamp", ) + browser_events = discard_unused_events( + browser_events, action_events, "browser_event_timestamp", + ) + num_action_events_ = len(action_events) num_window_events_ = len(window_events) num_screenshots_ = len(screenshots) - num_total_ = num_action_events_ + num_window_events_ + num_screenshots_ + num_browser_events_ = len(browser_events) + num_total_ = num_action_events_ + num_window_events_ + num_screenshots_ + num_browser_events_ pct_action_events = num_action_events_ / num_action_events pct_window_events = num_window_events_ / num_window_events pct_screenshots = num_screenshots_ / num_screenshots + pct_browser_events = num_browser_events_ / num_browser_events pct_total = num_total_ / num_total logger.info( - f"after {num_action_events_=} {num_window_events_=} {num_screenshots_=} " + f"after {num_action_events_=} {num_window_events_=} {num_screenshots_=} {num_browser_events_} " f"{num_total=}" ) logger.info( - f"{pct_action_events=} {pct_window_events=} {pct_screenshots=} " + f"{pct_action_events=} {pct_window_events=} {pct_screenshots=} {pct_browser_events=} " f"{pct_total=}" ) - return action_events, window_events, screenshots + return action_events, window_events, screenshots, browser_events diff --git a/openadapt/models.py b/openadapt/models.py index aa9e2c4b9..941a4589e 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -1,4 +1,5 @@ import io +from typing import Any from loguru import logger from pynput import keyboard @@ -47,6 +48,11 @@ class Recording(db.Base): back_populates="recording", order_by="WindowEvent.timestamp", ) + browser_events = sa.orm.relationship( + "BrowserEvent", + back_populates="recording", + order_by="BrowserEvent.timestamp", + ) _processed_action_events = None @@ -68,6 +74,7 @@ class ActionEvent(db.Base): recording_timestamp = sa.Column(sa.ForeignKey("recording.timestamp")) screenshot_timestamp = sa.Column(sa.ForeignKey("screenshot.timestamp")) window_event_timestamp = sa.Column(sa.ForeignKey("window_event.timestamp")) + browser_event_timestamp = sa.Column(sa.ForeignKey("browser_event.timestamp")) mouse_x = sa.Column(sa.Numeric(asdecimal=False)) mouse_y = sa.Column(sa.Numeric(asdecimal=False)) mouse_dx = sa.Column(sa.Numeric(asdecimal=False)) @@ -92,6 +99,7 @@ class ActionEvent(db.Base): recording = sa.orm.relationship("Recording", back_populates="action_events") screenshot = sa.orm.relationship("Screenshot", back_populates="action_event") window_event = sa.orm.relationship("WindowEvent", back_populates="action_events") + browser_event = sa.orm.relationship("BrowserEvent", back_populates="action_event") # TODO: playback_timestamp / original_timestamp @@ -305,6 +313,28 @@ def get_active_window_event(cls): return WindowEvent(**window.get_active_window_data()) +class BrowserEvent(db.Base): + """Class representing a browser event in the database.""" + + __tablename__ = 'browser_event' + + id = sa.Column(sa.Integer, primary_key=True) + recording_timestamp = sa.Column(sa.ForeignKey("recording.timestamp")) + timestamp = sa.Column(ForceFloat) + bodyHTML = sa.Column(sa.String) + headHTML = sa.Column(sa.String) + url = sa.Column(sa.String) + + recording = sa.orm.relationship("Recording", back_populates="browser_events") + action_event = sa.orm.relationship("ActionEvent", back_populates="browser_event") + + # TODO: implement for extension + # @classmethod + # def get_active_window_event(cls: Any) -> Any: + # """Get the active chrome tab window's DOM""" + # return BrowserEvent(**get_active_chrome_data()) + + class PerformanceStat(db.Base): __tablename__ = "performance_stat" @@ -323,11 +353,3 @@ class MemoryStat(db.Base): recording_timestamp = sa.Column(sa.Integer) memory_usage_bytes = sa.Column(ForceFloat) timestamp = sa.Column(ForceFloat) - - -class BrowserEvent(db.Base): - - bodyHTML = sa.Column(sa.String) - headHTML = sa.Column(sa.String) - url = sa.Column(sa.String) - timestamp = sa.Column(ForceFloat) diff --git a/openadapt/record.py b/openadapt/record.py index 67f3a2423..53c52caa5 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -141,8 +141,10 @@ def process_events( prev_event = None prev_screen_event = None prev_window_event = None + prev_browser_event = None prev_saved_screen_timestamp = 0 prev_saved_window_timestamp = 0 + prev_saved_browser_timestamp = 0 while not terminate_event.is_set() or not event_q.empty(): event = event_q.get() logger.trace(f"{event=}") @@ -153,7 +155,8 @@ def process_events( prev_screen_event = event elif event.type == "window": prev_window_event = event - # TODO: elif event.type == "browser" + elif event.type == "browser": + prev_browser_event = event elif event.type == "action": if prev_screen_event is None: logger.warning("discarding action that came before screen") @@ -163,7 +166,7 @@ def process_events( continue event.data["screenshot_timestamp"] = prev_screen_event.timestamp event.data["window_event_timestamp"] = prev_window_event.timestamp - # TODO: save browser_event_timestamp to event.data + event.data["browser_event_timestamp"] = prev_browser_event.timestamp process_event( event, action_write_q, @@ -189,6 +192,15 @@ def process_events( perf_q, ) prev_saved_window_timestamp = prev_window_event.timestamp + if prev_saved_browser_timestamp < prev_browser_event.timestamp: + process_event( + prev_browser_event, + browser_write_q, + write_browser_event, + recording_timestamp, + perf_q, + ) + prev_saved_browser_timestamp = prev_browser_event.timestamp else: raise Exception(f"unhandled {event.type=}") del prev_event @@ -774,7 +786,7 @@ def record( browser_event_reader = threading.Thread( target=read_browser_events, - args(event_q, terminate_event, recording_timestamp), + args=(event_q, terminate_event, recording_timestamp), ) browser_event_reader.start() From ba220be409484d871336c71d9b5468a2f1553484 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 11 Jul 2023 11:20:20 -0400 Subject: [PATCH 49/89] add missing browser events --- openadapt/crud.py | 2 ++ openadapt/models.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/openadapt/crud.py b/openadapt/crud.py index 359d40b4e..7efcc68ff 100644 --- a/openadapt/crud.py +++ b/openadapt/crud.py @@ -9,6 +9,7 @@ Screenshot, Recording, WindowEvent, + BrowserEvent, PerformanceStat, MemoryStat ) @@ -20,6 +21,7 @@ action_events = [] screenshots = [] window_events = [] +browser_events = [] performance_stats = [] memory_stats = [] diff --git a/openadapt/models.py b/openadapt/models.py index 941a4589e..9e69a672a 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -330,7 +330,7 @@ class BrowserEvent(db.Base): # TODO: implement for extension # @classmethod - # def get_active_window_event(cls: Any) -> Any: + # def get_active_browser_event(cls: Any) -> Any: # """Get the active chrome tab window's DOM""" # return BrowserEvent(**get_active_chrome_data()) From 598ee369995f3cbfa87b644591cfc736488a9460 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 11 Jul 2023 11:26:23 -0400 Subject: [PATCH 50/89] add browser_event_dict in visualize.py --- openadapt/visualize.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index 9a2e96f3d..5d87d7c27 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -221,10 +221,12 @@ def main(): action_event_dict = row2dict(action_event) window_event_dict = row2dict(action_event.window_event) + browser_event_dict = row2dict(action_event.browser_event) if SCRUB: action_event_dict = scrub.scrub_dict(action_event_dict) window_event_dict = scrub.scrub_dict(window_event_dict) + browser_event_dict = scrub.scrub_dict(browser_event_dict) rows.append( [ @@ -254,6 +256,9 @@ def main(): {dict2html(window_event_dict , None)}
+ + {dict2html(browser_event_dict , None)} +
""", ), Div( From 7ec25674c736981de989537868cd9f2c9fcb433a Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 12 Jul 2023 11:54:42 -0400 Subject: [PATCH 51/89] fix: socket_client script on closing and reopening the browser --- native_chrome_extension/browser.py | 4 +- native_chrome_extension/socket_client.py | 56 +++++++++++++++++------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 315a210dc..149890360 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -82,9 +82,9 @@ def send_message(encoded_message: dict[bytes, str]) -> None: if __name__ == "__main__": - address = ("localhost", 6001) + address = ("localhost", config.SOCKET_PORT) # family is deduced to be 'AF_INET' - listener = Listener(address, authkey=config.SOCKET_AUTHKEY) + listener = Listener(address, authkey=b"openadapt") conn = listener.accept() logger.info(f"connection accepted from {listener.last_accepted=}") diff --git a/native_chrome_extension/socket_client.py b/native_chrome_extension/socket_client.py index ffaa9603c..8d9021271 100644 --- a/native_chrome_extension/socket_client.py +++ b/native_chrome_extension/socket_client.py @@ -1,25 +1,51 @@ from multiprocessing.connection import Client - from loguru import logger - import time +from openadapt import config SERVER_SENDS = True PORT = 6001 +RETRY_INTERVAL = 5 # seconds -if __name__ == "__main__": - address = ('localhost', config.PORT) +def establish_connection(): + address = ('localhost', config.SOCKET_PORT) conn = Client(address, authkey=b'openadapt') + logger.info("Connected to the server.") + return conn + +def main(): + conn = establish_connection() + while True: - if SERVER_SENDS: - logger.info(f"waiting for message...") - msg = conn.recv() - logger.info(f"{msg=}") - else: - t = time.time() - logger.info(f"sending {t=}") - conn.send(t) - time.sleep(1) - # can also send arbitrary objects: - # conn.send(['a', 2.5, None, int, sum]) + try: + if SERVER_SENDS: + logger.info("Waiting for message...") + msg = conn.recv() + logger.info(f"{msg=}") + else: + t = time.time() + logger.info(f"Sending {t=}") + conn.send(t) + time.sleep(1) + except EOFError: + logger.warning("Connection closed. Reconnecting...") + while True: + try: + conn = establish_connection() + break + except Exception as exc: + logger.warning(f"Failed to reconnect: {exc}") + time.sleep(RETRY_INTERVAL) + except Exception as exc: + logger.warning(f"Error during communication: {exc}") + time.sleep(RETRY_INTERVAL) + conn.close() + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + logger.info("Script terminated by user.") + except Exception as exc: + logger.error(f"An unexpected error occurred: {exc}") \ No newline at end of file From 562d240b12ffb1d5f4a8d7c69e914e5a3f718328 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 12 Jul 2023 23:50:23 -0400 Subject: [PATCH 52/89] add code for sockets.py and make related changes --- native_chrome_extension/browser.py | 27 +-- native_chrome_extension/socket_client.py | 55 ++---- openadapt/config.py | 4 +- openadapt/record.py | 29 ++- openadapt/sockets.py | 227 +++++++++++++++++++++++ openadapt/utils.py | 19 ++ 6 files changed, 290 insertions(+), 71 deletions(-) create mode 100644 openadapt/sockets.py diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 149890360..4959bc4af 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -20,7 +20,7 @@ from loguru import logger -from openadapt import config +from openadapt import config, sockets SERVER_SENDS = True @@ -82,16 +82,21 @@ def send_message(encoded_message: dict[bytes, str]) -> None: if __name__ == "__main__": - address = ("localhost", config.SOCKET_PORT) - # family is deduced to be 'AF_INET' - listener = Listener(address, authkey=b"openadapt") - conn = listener.accept() + + # Establish a server connection + listener = sockets.create_server_connection(6001) + logger.info(f"connection accepted from {listener.last_accepted=}") - DOM_CHANGE_NUM = 0 + # Start the event loop while True: - receivedMessage = get_message() - DOM_CHANGE_NUM += 1 - conn.send(receivedMessage) - send_message(encode_message(str(DOM_CHANGE_NUM))) - send_message(encode_message(receivedMessage)) + received_message = get_message() + + # Sending message to Client + sockets.client_send_message(6001, received_message) + + # # Receiving message from Server + # response = sockets.server_receive_message(config.SOCKET_PORT) + + # Sending the received message back to background.js + send_message(encode_message(received_message)) diff --git a/native_chrome_extension/socket_client.py b/native_chrome_extension/socket_client.py index 8d9021271..fd7bb5fdc 100644 --- a/native_chrome_extension/socket_client.py +++ b/native_chrome_extension/socket_client.py @@ -1,51 +1,22 @@ from multiprocessing.connection import Client from loguru import logger -import time -from openadapt import config +from openadapt import config, sockets -SERVER_SENDS = True -PORT = 6001 -RETRY_INTERVAL = 5 # seconds - -def establish_connection(): - address = ('localhost', config.SOCKET_PORT) - conn = Client(address, authkey=b'openadapt') - logger.info("Connected to the server.") - return conn def main(): - conn = establish_connection() + conn = sockets.create_client_connection(config.SOCKET_PORT) + logger.info(f"connection accepted from {conn.last_accepted=}") + try: + # Start the event loop + while True: + # Receiving message from Server + response = sockets.client_receive_message(config.SOCKET_PORT) - while True: - try: - if SERVER_SENDS: - logger.info("Waiting for message...") - msg = conn.recv() - logger.info(f"{msg=}") - else: - t = time.time() - logger.info(f"Sending {t=}") - conn.send(t) - time.sleep(1) - except EOFError: - logger.warning("Connection closed. Reconnecting...") - while True: - try: - conn = establish_connection() - break - except Exception as exc: - logger.warning(f"Failed to reconnect: {exc}") - time.sleep(RETRY_INTERVAL) - except Exception as exc: - logger.warning(f"Error during communication: {exc}") - time.sleep(RETRY_INTERVAL) + logger.info(f"received {response=}") + except KeyboardInterrupt: + logger.info("Script terminated by user.") + conn.close() - conn.close() if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - logger.info("Script terminated by user.") - except Exception as exc: - logger.error(f"An unexpected error occurred: {exc}") \ No newline at end of file + main() diff --git a/openadapt/config.py b/openadapt/config.py index 7ec21b6c9..fd3ac2ff1 100644 --- a/openadapt/config.py +++ b/openadapt/config.py @@ -86,6 +86,8 @@ "PLOT_PERFORMANCE": True, "SOCKET_PORT": 6001, "SOCKET_AUTHKEY": b"openadapt", + "SOCKET_ADDRESS": "localhost", + "SOCKET_RETRY_INTERVAL": 5, # seconds } # each string in STOP_STRS should only contain strings that don't contain special characters @@ -135,7 +137,7 @@ def obfuscate(val, pct_reveal=0.1, char="*"): -_OBFUSCATE_KEY_PARTS = ("KEY", "PASSWORD", "TOKEN", "AUTHKEY") +_OBFUSCATE_KEY_PARTS = ("KEY", "PASSWORD", "TOKEN", "AUTHKEY", "PORT", "ADDRESS") if multiprocessing.current_process().name == "MainProcess": for key, val in dict(locals()).items(): if not key.startswith("_") and key.isupper(): diff --git a/openadapt/record.py b/openadapt/record.py index 53c52caa5..590dc620e 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -546,25 +546,20 @@ def read_browser_events( utils.configure_logging(logger, LOG_LEVEL) utils.set_start_time(recording_timestamp) logger.info(f"starting") - prev_browser_data = {} - - while not terminate_event.is_set(): - # browser_data = # TODO (if the current window is Chrome then, get the DOM change data from the extension) - if not browser_data: - continue - if browser_data != prev_browser_data: - # logger.info(f"{browser_data=}") - logger.debug("queuing browser event for writing") - event_q.put( - Event( - utils.get_timestamp(), - "browser", - browser_data, - ) + while not terminate_event.is_set(): + # browser_data = get_message() + + # logger.info(f"{browser_data=}") + logger.debug("queuing browser event for writing") + event_q.put( + Event( + utils.get_timestamp(), + "browser", + browser_data, ) - prev_browser_data = browser_data - + ) + @trace(logger) def performance_stats_writer( diff --git a/openadapt/sockets.py b/openadapt/sockets.py new file mode 100644 index 000000000..e75cdabba --- /dev/null +++ b/openadapt/sockets.py @@ -0,0 +1,227 @@ +"""Module for managing socket connections and communication.""" + +from multiprocessing import Queue +from multiprocessing.connection import Client, Listener +import time +from typing import Optional + +from loguru import logger + +from openadapt import config + + +client_by_port = {} +server_by_port = {} +queue_by_port = {} + + +def client_send_message(port: int, msg: str) -> None: + """ + Send a message to the client connection associated with the given port. + + Args: + port: The port number associated with the client connection. + msg: The message to be sent. + + Returns: + None + """ + client_conn = client_by_port.get(port) + if client_conn: + client_conn.send(msg) + + +def server_send_message(port: int, msg: str) -> None: + """ + Send a message to the server connection associated with the given port. + + Args: + port: The port number associated with the server connection. + msg: The message to be sent. + + Returns: + None + """ + server_conn = server_by_port.get(port) + if server_conn: + server_conn.send(msg) + + +def client_receive_message(port: int) -> Optional[str]: + """ + Receive a message from the client connection associated with the given port. + + Args: + port: The port number associated with the client connection. + + Returns: + The received message as a string, or None if no message is available. + """ + client_conn = client_by_port.get(port) + while True: + if client_conn: + try: + message = client_conn.recv() + return message + except EOFError: + logger.warning("Connection closed. Reconnecting...") + while True: + try: + server_conn = create_server_connection(port) + break + except Exception as exc: + logger.warning(f"Failed to reconnect: {exc}") + time.sleep(config.SOCKET_RETRY_INTERVAL) + return None + + +def server_receive_message(port: int) -> Optional[str]: + """ + Receive a message from the server connection associated with the given port. + + Args: + port: The port number associated with the server connection. + + Returns: + The received message as a string, or None if no message is available. + """ + server_conn = server_by_port.get(port) + while True: + if server_conn: + try: + message = server_conn.recv() + return message + except EOFError: + logger.warning("Connection closed. Reconnecting...") + while True: + try: + server_conn = create_server_connection(port) + break + except Exception as exc: + logger.warning(f"Failed to reconnect: {exc}") + time.sleep(config.SOCKET_RETRY_INTERVAL) + return None + + +def client_add_sink(port: int, queue: Queue) -> None: + """ + Add a sink queue to the specified client port. + + Args: + port: The port number to associate with the sink queue. + queue: The queue to be added as a sink. + + Raises: + ValueError: If the specified port already has a sink assigned. + + Returns: + None + """ + if port in queue_by_port: + raise ValueError(f"Port {port} already has a sink assigned.") + queue_by_port[port] = queue + + +def server_add_sink(port: int, queue: Queue) -> None: + """ + Add a sink queue to the specified server port. + + Args: + port: The port number to associate with the sink queue. + queue: The queue to be added as a sink. + + Raises: + ValueError: If the specified port already has a sink assigned. + + Returns: + None + """ + if port in queue_by_port: + raise ValueError(f"Port {port} already has a sink assigned.") + queue_by_port[port] = queue + + +_terminate_event: Optional[bool] = None + + +def set_terminate_event(terminate_event) -> None: + """ + Set the termination event to control the event loop. + + Args: + terminate_event: The termination event object. + + Returns: + None + """ + global _terminate_event + _terminate_event = terminate_event + + +def create_client_connection(port: int) -> Client: + """ + Create a client connection and establish a connection to the specified port. + + Args: + port: The port number to connect to. + + Returns: + The created client connection object. + """ + address = (config.SOCKET_ADDRESS, port) + conn = Client(address, authkey=config.SOCKET_AUTHKEY) + logger.info("Connected to the Client.") + return conn + + +def create_server_connection(port: int) -> Listener: + """ + Create a server connection and start listening for connections on the specified port. + + Args: + port: The port number to bind the server connection to. + + Returns: + The created server connection object. + """ + address = (config.SOCKET_ADDRESS, port) + conn = Listener(address, authkey=config.SOCKET_AUTHKEY) + logger.info("Connected to the Server.") + return conn + + +def event_loop() -> None: + """ + The event loop for receiving and handling messages. + + Raises: + AssertionError: If `_terminate_event` is not set. + + Returns: + None + """ + assert _terminate_event, "You must call set_terminate_event" + while not _terminate_event.is_set(): + for port, client_conn in client_by_port.items(): + try: + message = client_conn.recv() + if message: + queue = queue_by_port.get(port) + if queue: + queue.put(message) + except EOFError: + # Handle connection closed or error + del client_by_port[port] + del queue_by_port[port] + + for port, server_conn in server_by_port.items(): + try: + message = server_conn.recv() + if message: + queue = queue_by_port.get(port) + if queue: + queue.put(message) + except EOFError: + # Handle connection closed or error + del server_by_port[port] + del queue_by_port[port] diff --git a/openadapt/utils.py b/openadapt/utils.py index 4646b2900..961721b8a 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -5,6 +5,7 @@ import fire import inspect import os +import socket import sys import time @@ -503,6 +504,24 @@ def strip_element_state(action_event): return action_event +def get_free_port() -> int: + """ + Get a free port number on the local machine. + + Returns: + An available free port number. + + Raises: + OSError: If a free port number cannot be obtained. + """ + # Create a temporary socket to find a free port + temp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + temp_socket.bind(('localhost', 0)) + _, port = temp_socket.getsockname() + temp_socket.close() + return port + + def get_functions(name) -> dict: """ Get a dictionary of function names to functions for all non-private functions From e59e6a652c957efa203d164795bb6789f06b2abc Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 13 Jul 2023 09:11:56 -0400 Subject: [PATCH 53/89] merge main to continue_extension --- openadapt/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/config.py b/openadapt/config.py index fd3ac2ff1..cee390b7b 100644 --- a/openadapt/config.py +++ b/openadapt/config.py @@ -38,7 +38,7 @@ "ACTION_TEXT_NAME_PREFIX": "<", "ACTION_TEXT_NAME_SUFFIX": ">", # SCRUBBING CONFIGURATIONS - "SCRUB_ENABLED": True, + "SCRUB_ENABLED": False, "SCRUB_CHAR": "*", "SCRUB_LANGUAGE": "en", # TODO support lists in getenv_fallback From 5fc3859c8463d6bb5f0b45eb72170d13eba41f29 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 13 Jul 2023 09:12:25 -0400 Subject: [PATCH 54/89] update the sockets.py script --- native_chrome_extension/browser.py | 9 ++---- native_chrome_extension/socket_client.py | 38 +++++++++++++++++------- openadapt/sockets.py | 2 ++ 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 4959bc4af..a955b89a0 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -23,9 +23,6 @@ from openadapt import config, sockets -SERVER_SENDS = True - - def get_message() -> Any: """Read a message from stdin and decode it. @@ -84,16 +81,16 @@ def send_message(encoded_message: dict[bytes, str]) -> None: if __name__ == "__main__": # Establish a server connection - listener = sockets.create_server_connection(6001) + listener = sockets.create_server_connection(config.SOCKET_PORT) - logger.info(f"connection accepted from {listener.last_accepted=}") + # logger.info(f"connection accepted from {listener.last_accepted=}") # Start the event loop while True: received_message = get_message() # Sending message to Client - sockets.client_send_message(6001, received_message) + sockets.client_send_message(config.SOCKET_PORT, received_message) # # Receiving message from Server # response = sockets.server_receive_message(config.SOCKET_PORT) diff --git a/native_chrome_extension/socket_client.py b/native_chrome_extension/socket_client.py index fd7bb5fdc..3976930ee 100644 --- a/native_chrome_extension/socket_client.py +++ b/native_chrome_extension/socket_client.py @@ -1,21 +1,37 @@ from multiprocessing.connection import Client from loguru import logger from openadapt import config, sockets +import time +RETRY_INTERVAL=5 def main(): - conn = sockets.create_client_connection(config.SOCKET_PORT) - logger.info(f"connection accepted from {conn.last_accepted=}") - try: - # Start the event loop - while True: - # Receiving message from Server - response = sockets.client_receive_message(config.SOCKET_PORT) + conn = Client((config.SOCKET_ADDRESS,config.SOCKET_PORT) , authkey=config.SOCKET_AUTHKEY) + while True: + try: + if SERVER_SENDS: + logger.info("Waiting for message...") + msg = conn.recv() + logger.info(f"{msg=}") + else: + t = time.time() + logger.info(f"Sending {t=}") + conn.send(t) + time.sleep(1) + except EOFError: + logger.warning("Connection closed. Reconnecting...") + while True: + try: + conn = establish_connection() + break + except Exception as exc: + logger.warning(f"Failed to reconnect: {exc}") + time.sleep(RETRY_INTERVAL) + except Exception as exc: + logger.warning(f"Error during communication: {exc}") + time.sleep(RETRY_INTERVAL) - logger.info(f"received {response=}") - except KeyboardInterrupt: - logger.info("Script terminated by user.") - conn.close() + conn.close() if __name__ == "__main__": diff --git a/openadapt/sockets.py b/openadapt/sockets.py index e75cdabba..eae1fafbc 100644 --- a/openadapt/sockets.py +++ b/openadapt/sockets.py @@ -170,6 +170,7 @@ def create_client_connection(port: int) -> Client: """ address = (config.SOCKET_ADDRESS, port) conn = Client(address, authkey=config.SOCKET_AUTHKEY) + client_by_port[port] = conn logger.info("Connected to the Client.") return conn @@ -186,6 +187,7 @@ def create_server_connection(port: int) -> Listener: """ address = (config.SOCKET_ADDRESS, port) conn = Listener(address, authkey=config.SOCKET_AUTHKEY) + server_by_port[port] = conn logger.info("Connected to the Server.") return conn From b4bb98447882427bff8ee250fd5fe13152fa610f Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 14 Jul 2023 13:36:31 -0400 Subject: [PATCH 55/89] rename to mock_client --- native_chrome_extension/{socket_client.py => mock_client.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename native_chrome_extension/{socket_client.py => mock_client.py} (88%) diff --git a/native_chrome_extension/socket_client.py b/native_chrome_extension/mock_client.py similarity index 88% rename from native_chrome_extension/socket_client.py rename to native_chrome_extension/mock_client.py index 3976930ee..2e652dffa 100644 --- a/native_chrome_extension/socket_client.py +++ b/native_chrome_extension/mock_client.py @@ -6,12 +6,12 @@ RETRY_INTERVAL=5 def main(): - conn = Client((config.SOCKET_ADDRESS,config.SOCKET_PORT) , authkey=config.SOCKET_AUTHKEY) + conn = sockets.create_client_connection(config.SOCKET_PORT) while True: try: if SERVER_SENDS: logger.info("Waiting for message...") - msg = conn.recv() + msg = sockets.client_receive_message(config.SOCKET_PORT) logger.info(f"{msg=}") else: t = time.time() From 3e68d62bd9bf9c2366af945b3eeb7bc77f63d60b Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 14 Jul 2023 13:53:02 -0400 Subject: [PATCH 56/89] fix: server send message --- native_chrome_extension/browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index a955b89a0..a0a02175a 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -90,7 +90,7 @@ def send_message(encoded_message: dict[bytes, str]) -> None: received_message = get_message() # Sending message to Client - sockets.client_send_message(config.SOCKET_PORT, received_message) + sockets.server_send_message(config.SOCKET_PORT, received_message) # # Receiving message from Server # response = sockets.server_receive_message(config.SOCKET_PORT) From 11bce463ffcac14a446653896e1f18f602f5c98e Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Fri, 14 Jul 2023 14:25:36 -0400 Subject: [PATCH 57/89] now the extensoin works using the sockets.py --- native_chrome_extension/browser.py | 8 ++++++-- native_chrome_extension/mock_client.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index a0a02175a..296acc92d 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -81,7 +81,11 @@ def send_message(encoded_message: dict[bytes, str]) -> None: if __name__ == "__main__": # Establish a server connection - listener = sockets.create_server_connection(config.SOCKET_PORT) + # listener = sockets.create_server_connection(config.SOCKET_PORT) + + address = (config.SOCKET_ADDRESS, 6001) + conn = Listener(address, authkey=config.SOCKET_AUTHKEY) + conn = conn.accept() # logger.info(f"connection accepted from {listener.last_accepted=}") @@ -90,7 +94,7 @@ def send_message(encoded_message: dict[bytes, str]) -> None: received_message = get_message() # Sending message to Client - sockets.server_send_message(config.SOCKET_PORT, received_message) + conn.send(received_message) # # Receiving message from Server # response = sockets.server_receive_message(config.SOCKET_PORT) diff --git a/native_chrome_extension/mock_client.py b/native_chrome_extension/mock_client.py index 2e652dffa..c5d753d36 100644 --- a/native_chrome_extension/mock_client.py +++ b/native_chrome_extension/mock_client.py @@ -4,6 +4,7 @@ import time RETRY_INTERVAL=5 +SERVER_SENDS = True def main(): conn = sockets.create_client_connection(config.SOCKET_PORT) From f13f885864bc801b1dcaa3a2b90ccadf50052728 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 17 Jul 2023 09:21:38 -0400 Subject: [PATCH 58/89] update browser.py to use sockets.py --- native_chrome_extension/browser.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 296acc92d..cb5c9e6f5 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -81,23 +81,15 @@ def send_message(encoded_message: dict[bytes, str]) -> None: if __name__ == "__main__": # Establish a server connection - # listener = sockets.create_server_connection(config.SOCKET_PORT) - - address = (config.SOCKET_ADDRESS, 6001) - conn = Listener(address, authkey=config.SOCKET_AUTHKEY) + conn = sockets.create_server_connection(config.SOCKET_PORT) conn = conn.accept() - # logger.info(f"connection accepted from {listener.last_accepted=}") - # Start the event loop while True: received_message = get_message() # Sending message to Client - conn.send(received_message) - - # # Receiving message from Server - # response = sockets.server_receive_message(config.SOCKET_PORT) + sockets.server_send_message(config.SOCKET_PORT, received_message) # Sending the received message back to background.js send_message(encode_message(received_message)) From fcd742866044f015e7675fd8ced716f189a538a3 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 17 Jul 2023 09:42:21 -0400 Subject: [PATCH 59/89] now the browser.py and mock_client use sockects.py to communicate --- native_chrome_extension/browser.py | 1 - openadapt/sockets.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index cb5c9e6f5..81c8d3a60 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -82,7 +82,6 @@ def send_message(encoded_message: dict[bytes, str]) -> None: # Establish a server connection conn = sockets.create_server_connection(config.SOCKET_PORT) - conn = conn.accept() # Start the event loop while True: diff --git a/openadapt/sockets.py b/openadapt/sockets.py index eae1fafbc..273581e59 100644 --- a/openadapt/sockets.py +++ b/openadapt/sockets.py @@ -1,7 +1,7 @@ """Module for managing socket connections and communication.""" from multiprocessing import Queue -from multiprocessing.connection import Client, Listener +from multiprocessing.connection import Client, Listener, Connection import time from typing import Optional @@ -175,7 +175,7 @@ def create_client_connection(port: int) -> Client: return conn -def create_server_connection(port: int) -> Listener: +def create_server_connection(port: int) -> Connection: """ Create a server connection and start listening for connections on the specified port. @@ -187,6 +187,7 @@ def create_server_connection(port: int) -> Listener: """ address = (config.SOCKET_ADDRESS, port) conn = Listener(address, authkey=config.SOCKET_AUTHKEY) + conn = conn.accept() server_by_port[port] = conn logger.info("Connected to the Server.") return conn From 9aec8b9b5d57bd60f4877d262292cba225ec426a Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 17 Jul 2023 09:45:39 -0400 Subject: [PATCH 60/89] add mock_client code to record --- openadapt/record.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 590dc620e..f1430327d 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -548,9 +548,32 @@ def read_browser_events( logger.info(f"starting") while not terminate_event.is_set(): - # browser_data = get_message() + conn = sockets.create_client_connection(config.SOCKET_PORT) + while True: + try: + if SERVER_SENDS: + logger.info("Waiting for message...") + msg = sockets.client_receive_message(config.SOCKET_PORT) + logger.info(f"{msg=}") + else: + t = time.time() + logger.info(f"Sending {t=}") + conn.send(t) + time.sleep(1) + except EOFError: + logger.warning("Connection closed. Reconnecting...") + while True: + try: + conn = establish_connection() + break + except Exception as exc: + logger.warning(f"Failed to reconnect: {exc}") + time.sleep(RETRY_INTERVAL) + except Exception as exc: + logger.warning(f"Error during communication: {exc}") + time.sleep(RETRY_INTERVAL) + conn.close() - # logger.info(f"{browser_data=}") logger.debug("queuing browser event for writing") event_q.put( Event( From 056087fd71f9a369cdd034f61c5f4c77b907b9bf Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 17 Jul 2023 10:13:03 -0400 Subject: [PATCH 61/89] try to fix errors during record eith extension --- openadapt/record.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index f1430327d..7083141b7 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -9,6 +9,7 @@ from collections import namedtuple from functools import partial, wraps from typing import Any, Callable, Dict +from multiprocessing.connection import Client import multiprocessing import os import queue @@ -26,7 +27,7 @@ import mss.tools import psutil -from openadapt import config, crud, utils, window +from openadapt import config, crud, utils, window, sockets from openadapt.extensions import synchronized_queue as sq Event = namedtuple("Event", ("timestamp", "type", "data")) @@ -48,6 +49,8 @@ tracemalloc.start() utils.configure_logging(logger, LOG_LEVEL) +SERVER_SENDS = True + def collect_stats(): performance_snapshots.append(tracemalloc.take_snapshot()) @@ -117,6 +120,7 @@ def process_events( screen_write_q: sq.SynchronizedQueue, action_write_q: sq.SynchronizedQueue, window_write_q: sq.SynchronizedQueue, + browser_write_q: sq.SynchronizedQueue, perf_q: sq.SynchronizedQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, @@ -555,6 +559,15 @@ def read_browser_events( logger.info("Waiting for message...") msg = sockets.client_receive_message(config.SOCKET_PORT) logger.info(f"{msg=}") + browser_data = msg + logger.debug("queuing browser event for writing") + event_q.put( + Event( + utils.get_timestamp(), + "browser", + browser_data, + ) + ) else: t = time.time() logger.info(f"Sending {t=}") @@ -568,21 +581,12 @@ def read_browser_events( break except Exception as exc: logger.warning(f"Failed to reconnect: {exc}") - time.sleep(RETRY_INTERVAL) + time.sleep(config.SOCKET_RETRY_INTERVAL) except Exception as exc: logger.warning(f"Error during communication: {exc}") - time.sleep(RETRY_INTERVAL) + time.sleep(config.SOCKET_RETRY_INTERVAL) conn.close() - logger.debug("queuing browser event for writing") - event_q.put( - Event( - utils.get_timestamp(), - "browser", - browser_data, - ) - ) - @trace(logger) def performance_stats_writer( @@ -860,7 +864,7 @@ def record( args=( "browser", write_browser_event, - browser_event_q, + browser_write_q, perf_q, recording_timestamp, terminate_event, From c313199e67af35c0bd9aa9df2cedd119a125754a Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 17 Jul 2023 10:31:41 -0400 Subject: [PATCH 62/89] try to fix communication closed error when extension --- openadapt/record.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 7083141b7..4ebd47471 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -551,8 +551,8 @@ def read_browser_events( utils.set_start_time(recording_timestamp) logger.info(f"starting") + conn = sockets.create_client_connection(config.SOCKET_PORT) while not terminate_event.is_set(): - conn = sockets.create_client_connection(config.SOCKET_PORT) while True: try: if SERVER_SENDS: @@ -585,7 +585,8 @@ def read_browser_events( except Exception as exc: logger.warning(f"Error during communication: {exc}") time.sleep(config.SOCKET_RETRY_INTERVAL) - conn.close() + conn.close() + conn.close() @trace(logger) From de76ab1908333b7da9485b60c3641b38444adf6a Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 18 Jul 2023 09:51:00 -0400 Subject: [PATCH 63/89] try to fix error :- on ctrl + C read_browser_event is still reading from sockets --- openadapt/record.py | 28 +++++++++++++++------------- openadapt/sockets.py | 15 ++++++++------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 4ebd47471..11945c51b 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -573,20 +573,22 @@ def read_browser_events( logger.info(f"Sending {t=}") conn.send(t) time.sleep(1) - except EOFError: - logger.warning("Connection closed. Reconnecting...") - while True: - try: - conn = establish_connection() - break - except Exception as exc: - logger.warning(f"Failed to reconnect: {exc}") - time.sleep(config.SOCKET_RETRY_INTERVAL) except Exception as exc: - logger.warning(f"Error during communication: {exc}") - time.sleep(config.SOCKET_RETRY_INTERVAL) - conn.close() - conn.close() + logger.warning("Connection closed. Reconnecting...") + break + # while True: + # try: + # conn = establish_connection() + # break + # except Exception as exc: + # logger.warning(f"Failed to reconnect: {exc}") + # time.sleep(config.SOCKET_RETRY_INTERVAL) + # except Exception as exc: + # logger.warning(f"Error during communication: {exc}") + # time.sleep(config.SOCKET_RETRY_INTERVAL) + conn.close() + break + # conn.close() @trace(logger) diff --git a/openadapt/sockets.py b/openadapt/sockets.py index 273581e59..dfbfc9594 100644 --- a/openadapt/sockets.py +++ b/openadapt/sockets.py @@ -65,13 +65,14 @@ def client_receive_message(port: int) -> Optional[str]: return message except EOFError: logger.warning("Connection closed. Reconnecting...") - while True: - try: - server_conn = create_server_connection(port) - break - except Exception as exc: - logger.warning(f"Failed to reconnect: {exc}") - time.sleep(config.SOCKET_RETRY_INTERVAL) + break + # while True: + # try: + # server_conn = create_server_connection(port) + # break + # except Exception as exc: + # logger.warning(f"Failed to reconnect: {exc}") + # time.sleep(config.SOCKET_RETRY_INTERVAL) return None From 109b62083458bbb07c290f4b15e81da8382c349d Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 19 Jul 2023 11:59:20 -0400 Subject: [PATCH 64/89] attempt to observe the execution of sockets during record --- native_chrome_extension/browser.py | 3 +- openadapt/record.py | 50 +++++++++----------- openadapt/sockets.py | 74 ++++++++++++++++-------------- 3 files changed, 64 insertions(+), 63 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 81c8d3a60..21e74d367 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -88,7 +88,8 @@ def send_message(encoded_message: dict[bytes, str]) -> None: received_message = get_message() # Sending message to Client - sockets.server_send_message(config.SOCKET_PORT, received_message) + # sockets.server_send_message(config.SOCKET_PORT, received_message) + sockets.server_sends(conn, received_message) # Sending the received message back to background.js send_message(encode_message(received_message)) diff --git a/openadapt/record.py b/openadapt/record.py index 512e8e11d..32f498c9f 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -49,8 +49,6 @@ tracemalloc.start() utils.configure_logging(logger, LOG_LEVEL) -SERVER_SENDS = True - def collect_stats(): performance_snapshots.append(tracemalloc.take_snapshot()) @@ -546,32 +544,29 @@ def read_browser_events( utils.configure_logging(logger, LOG_LEVEL) utils.set_start_time(recording_timestamp) logger.info(f"starting") - conn = sockets.create_client_connection(config.SOCKET_PORT) - while not terminate_event.is_set(): - while True: - try: - if SERVER_SENDS: - logger.info("Waiting for message...") - msg = sockets.client_receive_message(config.SOCKET_PORT) - logger.info(f"{msg=}") - browser_data = msg - logger.debug("queuing browser event for writing") - event_q.put( - Event( - utils.get_timestamp(), - "browser", - browser_data, - ) + while not terminate_event.is_set() and conn is not None: + try: + logger.info("Waiting for message...") + # msg = sockets.client_receive_message(config.SOCKET_PORT) + msg = sockets.client_receive(conn) + # logger.info(f"{msg=}") + if msg is not None: + logger.info("Received DOM message.") + browser_data = msg + logger.debug("queuing browser event for writing") + event_q.put( + Event( + utils.get_timestamp(), + "browser", + browser_data, ) - else: - t = time.time() - logger.info(f"Sending {t=}") - conn.send(t) - time.sleep(1) - except Exception as exc: - logger.warning("Connection closed. Reconnecting...") - break + ) + else: + logger.info("No message received or received None Type Message.") + except Exception as exc: + logger.warning("Connection closed.") + break # while True: # try: # conn = establish_connection() @@ -582,9 +577,8 @@ def read_browser_events( # except Exception as exc: # logger.warning(f"Error during communication: {exc}") # time.sleep(config.SOCKET_RETRY_INTERVAL) + if conn: conn.close() - break - # conn.close() @trace(logger) diff --git a/openadapt/sockets.py b/openadapt/sockets.py index dfbfc9594..3f8e43588 100644 --- a/openadapt/sockets.py +++ b/openadapt/sockets.py @@ -15,7 +15,7 @@ queue_by_port = {} -def client_send_message(port: int, msg: str) -> None: +def client_send_message(port: int, msg: Any) -> None: """ Send a message to the client connection associated with the given port. @@ -31,7 +31,7 @@ def client_send_message(port: int, msg: str) -> None: client_conn.send(msg) -def server_send_message(port: int, msg: str) -> None: +def server_send_message(port: int, msg: Any) -> None: """ Send a message to the server connection associated with the given port. @@ -58,22 +58,13 @@ def client_receive_message(port: int) -> Optional[str]: The received message as a string, or None if no message is available. """ client_conn = client_by_port.get(port) - while True: - if client_conn: - try: - message = client_conn.recv() - return message - except EOFError: - logger.warning("Connection closed. Reconnecting...") - break - # while True: - # try: - # server_conn = create_server_connection(port) - # break - # except Exception as exc: - # logger.warning(f"Failed to reconnect: {exc}") - # time.sleep(config.SOCKET_RETRY_INTERVAL) - return None + if client_conn: + try: + message = client_conn.recv() + return message + except Exception as exc: + logger.warning("Connection was closed.") + del client_by_port[port] def server_receive_message(port: int) -> Optional[str]: @@ -159,7 +150,7 @@ def set_terminate_event(terminate_event) -> None: _terminate_event = terminate_event -def create_client_connection(port: int) -> Client: +def create_client_connection(port: int) -> Connection: """ Create a client connection and establish a connection to the specified port. @@ -171,6 +162,7 @@ def create_client_connection(port: int) -> Client: """ address = (config.SOCKET_ADDRESS, port) conn = Client(address, authkey=config.SOCKET_AUTHKEY) + conn = conn.accept() client_by_port[port] = conn logger.info("Connected to the Client.") return conn @@ -209,23 +201,37 @@ def event_loop() -> None: for port, client_conn in client_by_port.items(): try: message = client_conn.recv() - if message: - queue = queue_by_port.get(port) - if queue: - queue.put(message) + # if message: + # TODO: Handle message + except EOFError: # Handle connection closed or error del client_by_port[port] del queue_by_port[port] - for port, server_conn in server_by_port.items(): - try: - message = server_conn.recv() - if message: - queue = queue_by_port.get(port) - if queue: - queue.put(message) - except EOFError: - # Handle connection closed or error - del server_by_port[port] - del queue_by_port[port] + # for port, server_conn in server_by_port.items(): + # try: + # message = server_conn.recv() + # if message: + # queue = queue_by_port.get(port) + # if queue: + # queue.put(message) + # except EOFError: + # # Handle connection closed or error + # del server_by_port[port] + # del queue_by_port[port] + + + +def server_sends(conn: Connection, message: Any) -> None: + if conn: + conn.send(message) + +def client_receive(conn: Connection) -> Any: + if conn: + try: + message = conn.recv() + return message + except EOFError: + logger.warning("Connection was closed.") + return None From 06d7ed8f9b9088af26527b2f6986d102080ae064 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 19 Jul 2023 12:31:43 -0400 Subject: [PATCH 65/89] try to fix the infinite running server --- native_chrome_extension/browser.py | 5 +++++ openadapt/record.py | 6 +++++- openadapt/sockets.py | 3 +-- openadapt/utils.py | 18 ++++++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 21e74d367..391f7bb03 100644 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -82,6 +82,10 @@ def send_message(encoded_message: dict[bytes, str]) -> None: # Establish a server connection conn = sockets.create_server_connection(config.SOCKET_PORT) + + # address = (config.SOCKET_ADDRESS, config.SOCKET_PORT) + # conn = Listener(address, authkey=config.SOCKET_AUTHKEY) + # conn = conn.accept() # Start the event loop while True: @@ -90,6 +94,7 @@ def send_message(encoded_message: dict[bytes, str]) -> None: # Sending message to Client # sockets.server_send_message(config.SOCKET_PORT, received_message) sockets.server_sends(conn, received_message) + # conn.send(received_message) # Sending the received message back to background.js send_message(encode_message(received_message)) diff --git a/openadapt/record.py b/openadapt/record.py index 32f498c9f..61fd17f75 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -167,7 +167,7 @@ def process_events( continue event.data["screenshot_timestamp"] = prev_screen_event.timestamp event.data["window_event_timestamp"] = prev_window_event.timestamp - event.data["browser_event_timestamp"] = prev_browser_event.timestamp + event.data["browser_event_timestamp"] = prev_browser_event.timestamp if not prev_browser_event is None else None process_event( event, action_write_q, @@ -580,6 +580,10 @@ def read_browser_events( if conn: conn.close() + pid = utils.get_pid_by_name("browser.py") + utils.send_kill_signal(pid) + logger.info("done") + @trace(logger) def performance_stats_writer( diff --git a/openadapt/sockets.py b/openadapt/sockets.py index 3f8e43588..8f016831a 100644 --- a/openadapt/sockets.py +++ b/openadapt/sockets.py @@ -3,7 +3,7 @@ from multiprocessing import Queue from multiprocessing.connection import Client, Listener, Connection import time -from typing import Optional +from typing import Optional, Any from loguru import logger @@ -162,7 +162,6 @@ def create_client_connection(port: int) -> Connection: """ address = (config.SOCKET_ADDRESS, port) conn = Client(address, authkey=config.SOCKET_AUTHKEY) - conn = conn.accept() client_by_port[port] = conn logger.info("Connected to the Client.") return conn diff --git a/openadapt/utils.py b/openadapt/utils.py index 09fb924bf..bca7e41d0 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -5,6 +5,8 @@ import fire import inspect import os +import psutil +import signal import socket import sys import threading @@ -527,6 +529,22 @@ def get_free_port() -> int: return port +def send_kill_signal(pid): + try: + # Send the kill signal (SIGTERM) to the process identified by the PID + os.kill(pid, signal.SIGTERM) + logger.info("Kill signal sent successfully.") + except OSError as e: + logger.info(f"Failed to send kill signal: {e}") + + +def get_pid_by_name(process_name): + for process in psutil.process_iter(['pid', 'name']): + if process.info['name'] == process_name: + return process.info['pid'] + return None + + def get_functions(name) -> dict: """ Get a dictionary of function names to functions for all non-private functions From d2e43015882383dc5453eba19682b258057f8976 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Fri, 28 Jul 2023 10:37:12 -0400 Subject: [PATCH 66/89] Create browser.sh --- native_chrome_extension/browser.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 native_chrome_extension/browser.sh diff --git a/native_chrome_extension/browser.sh b/native_chrome_extension/browser.sh new file mode 100755 index 000000000..0fdf2a45b --- /dev/null +++ b/native_chrome_extension/browser.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +dir=$(dirname "$0") +python3 -u "$dir/browser.py" From b33f6a09b9ab7a4b53ecb434cff371a0e6111886 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 31 Jul 2023 14:04:39 -0400 Subject: [PATCH 67/89] mac version --- native_chrome_extension/background.js | 26 ++++++++++++++---- native_chrome_extension/browser.sh | 2 +- native_chrome_extension/icons/logo.png | Bin 0 -> 4001 bytes .../mac_nativehost_file.json | 7 +++++ native_chrome_extension/manifest.json | 4 +-- native_chrome_extension/openadapt_chrome.json | 4 +-- pyproject.toml | 2 +- 7 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 native_chrome_extension/icons/logo.png create mode 100644 native_chrome_extension/mac_nativehost_file.json diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index ceea97bb7..afbed7066 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -17,17 +17,31 @@ function onReceived(response) { console.log(response); } +function onDisconnected() { + msg = 'Failed to connect: ' + chrome.runtime.lastError.message; // silence error + port = null; +} + +function connect() { + port = chrome.runtime.connectNative(hostName); + port.onMessage.addListener(onReceived); + port.onDisconnect.addListener(onDisconnected); +} + /* * Message listener for content script */ function messageListener(message, sender, sendResponse) { - console.log({ message, sender, sendResponse }); - port.postMessage(message); // send to browser.py (native messaging host) + try { + console.log({ message, sender, sendResponse }); + port.postMessage(message); // send to browser.py (native messaging host) + } + catch (e) { + connect(); + } } - -port = chrome.runtime.connectNative(hostName); - -port.onMessage.addListener(onReceived); chrome.runtime.onMessage.addListener(messageListener); + +connect(); \ No newline at end of file diff --git a/native_chrome_extension/browser.sh b/native_chrome_extension/browser.sh index 0fdf2a45b..33025ecb4 100755 --- a/native_chrome_extension/browser.sh +++ b/native_chrome_extension/browser.sh @@ -1,4 +1,4 @@ #!/bin/bash dir=$(dirname "$0") -python3 -u "$dir/browser.py" +python3 -u "$dir/browser.py" \ No newline at end of file diff --git a/native_chrome_extension/icons/logo.png b/native_chrome_extension/icons/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..64aa029bdc8e5bd6e0fd433081d9bf57d0fc0798 GIT binary patch literal 4001 zcmb7Hdo+}5+rOU~W*lPnl$4yuIFuYhjPuBd5z#?Nm<~ol1{K@!nbd1i8;#0wq7o@> zBj-KgQL{TSo9K0jm{e>;lw$_(w7!47_g#B`>wEvW*L|(uy4H1H>%On+cmJ+So_n3; zWVXlv0OY7Hl>GoeViN*V81Z%_>KspmJW#*lS7;0osYS^cxf5>rDox9b9>u2LcwVHoJVYTL`7p^P0t|; zjea)o1`n9FK8I+Bc3@%O-&Fv8V>HO|YWx2KC8LJOcg?XNBthtp`cm3-2k|MBZ zMBB}0zIo(D-J7zwj#@RB|rjuwR63O8^rx%cm_{v8Q-fp+Yl5Y_k73#sF9`_qD z-&Zz4kz`M}>F$*;mx?YTvqwN6T7K`FkG9BpR;=;9c=Mm+;U_Lu67~3At;jBWC_SCU z+fWy(WC>(C)s`Z3SA;_!b&*>)rLR4OJX!Mv$x73qsxdXF*m#OfSEPM6x&p2DT=Io)RFKoHXvig*VzWW*^+9? zR7V?4G)(M7)B<*pPTz#C3?+P57Xe;%Vn53gJp1B*eLh^81fEr7=&$uhxjbt!fUwBo zWOn73KBeVa2^H{R;j8Qa7U~}^) zrR65=<3l%KG)>gRDI1bQhTRhV2=ed@FU6)5ve?bXna+kvTY}w&dHI%xuR7*}O=d$b zEzyfmE_u)Z?sNZvYzqp%(snLL_3OGKfS*yBK3bGvmi&0KRYnN(D3k%0g;LZ6d_Zg1 zS#{yK?aWrP1JNDZgnC?&GGYNro%lOdtHX0+F`k($A6$Xk2GrJvsPu0_qYQMB@#Gww z1$%V5^_7kv5{Ba|i4K0|#&(i=ImHb~_DMC2m68S2thAb`&}s^wcjC*G)E@paswN8h zFAMB%YqI=LgBBu3>Imxbwip9**9Xjn-F!iV?3n9SQ$XeI`Ry7#<_In+zdRQ?M*Y&} z@8X@754l*$)T`uN=PBy%t~6yHqXYK2(2Vax{FI^2ZRd16`RmA?N0IADLEBxQzO;Pl zD@hoQF+yIdckHxZ4;|IB$I2{5v+M^NqE$3-*2gO?_lFPdhQp2?@$1sFOgE4$V(u1Z~?!{x{&>?vDVs%aul{)zk-x+GWB2eb@c%rTk^*Ryx>kB6o%f z9sA?`)N9=SQH+L{OFH-~_?eP}bF&Z6H?KLc?W#BnM-=h6`TYXoI&(mrM2lp(fm4Uu zYTy5YJg@t96X#-OLpU_QuOofS$E{@R#q8mhh!1tT+=HDH(VHIaOpyh+B>fJWBh3>M zwnRuOpO9Lo18f!_qEcj;4iYf7+3SR-W~S{AXjh6f^D_hn(y-0{4E7M}P?3Cy+ zw94X)Pe~^Zy0XPJiYgq*F6~FH9`3|eDbVHNChFV#k)!vP!#)^TLz_^0h>az6Ppk-I zG@-n(u5RW=2ol8ppKhIYaFL{6vg(Rq|S ztmYDDAh??Rs6C|+SDrb$q*3QT=kmFQbc&vy-mVr#$IWv0DWe3&f=Nm_!4xFOI+aX`7q_j>$!KX!|VX}=#N8#l3ax(i6hvA zjXsL_3EsF%rvoC5yhQSU8!xR-YG*WfRo1ZpvGOGPsaa| zJke8IvEyfjSP#kohnLA-BG8@_hFxh;yLx!|@)UaTeanz7mW+FQ*(Vi^3o;B^cUnUJ zGAlKD(|)x0l?cOog+B?^qk2p{Cz!G)OIGY#{oM>yfNdJf3m+-Nz}RW`4m?GywT4Bk z-W*;|9toJRSPz(i*r}KPt>B2PiVD7N)UbHBJnTjownqHmDyFVUcGgo@S`wu|1#jQ( z@AgmyC^_-Jvlf+U$wFDOT-VS}v(fL-TEof>m&-V48Xyc`V);m^UDNt@s(2sq>$ImWQD<7LPD|CXXl8 z46e+aY{)B_x5P)rQ3kSk8u6M~1a{)TntwwJ6y8CSM;@N!p?okRHEs-L`z$`WH^G87 zf=&6XN5yswq{VNuu5I+mUQHblt$t<2YQ*fYzCF)_RDqT(ET~x+Q6Fp?fhs zr>l$l8bZe$*dTM(ie|}Oc(Z5Q-m59sl20?eNW9P5PdYi?%lHL~FMU42k z1T4uqM%7U_Bv@!A6f}e?72U?#E5ykd>sYFvzE7`W>cgfb9cwqtQMyk2v(o+5=?d0p zuJlrLr+gA@KP^-? zgy2Q_7jV*Iu0FSVVJ;B2oyC$92UHVx?1U+J>u!7F>U`6xAAF~Oc}LiZ1i7O1>;UP6 zli$vI^c? zJFFce4v=&1Kt=`HbiFxGkFaWTRIJzoq^|kCx=8+UQjoB^z(b>UN-Qa*sJyF52Qsvg zTs6_BD4j~Jh6HhUmS z%Hz5#>jg@*ZEtZxuHBSK(jrU={8@8PO+o)hr9AB>HPOaMuo=M!->~WpVl=yru&^ZK z93|n0ZI6W$O$dhV{ue&IzYOOFHA%%%DFs<&5y}8drYoU^hlMNn_s`3`+n`L}65u1F zvH{;dH&7vW1+7uF4r1VpjR-f#D?NTBP~Mt?>{AofUP2eCWxW18y!>I{VlRUP#;45$ zPpz8@c|R#DV#!P0DBazB{|4F0_mSXQxR&6w-OwNYEV(RnQCRY3{8|oAO4$%gZm`>k zFBGRGm)`@O=zeVlKbQ$fGKb{hHY+Ju8I~MuZP$4s&S@Cbqe%opH}O=v%xEZBS}_oy zElN*G3U;ZBC9iQiz!bgfT0240WC)3Qi<*p+6q|x-WBr?G8ks0fLM{HpurW?VHP1^3 zcHdk$>zjZ5C1mW$rbEHdD5w~G*5BO`_P(*a?n$|xJj~28Lz8Zd%nCaMN!9WsLqb#T zU1R<8Xc~pscu@?akuT2tjk$XsU5y4;GK>8aYKHbRehiTl|0#U`jnw{98vm_F?;6@G Xv~_V#NjpY7UIVJrUdpe#Xy^Y0CJFjO literal 0 HcmV?d00001 diff --git a/native_chrome_extension/mac_nativehost_file.json b/native_chrome_extension/mac_nativehost_file.json new file mode 100644 index 000000000..5398f85d1 --- /dev/null +++ b/native_chrome_extension/mac_nativehost_file.json @@ -0,0 +1,7 @@ +{ + "name": "openadapt", + "description": "OpenAdapt DOM Listener", + "path": "/Users/aaron/Documents/GitHub/OpenAdapt/native_chrome_extension/browser.sh", + "type": "stdio", + "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] +} \ No newline at end of file diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json index 76058d3f9..5306894cd 100644 --- a/native_chrome_extension/manifest.json +++ b/native_chrome_extension/manifest.json @@ -5,10 +5,10 @@ "version": "1.0", "manifest_version": 3, "icons": { - "48": "icons/MLDSAI_logo.png" + "48": "icons/logo.png" }, "action": { - "default_icon": "icons/MLDSAI_logo.png" + "default_icon": "icons/logo.png" }, "background": { "service_worker": "background.js" diff --git a/native_chrome_extension/openadapt_chrome.json b/native_chrome_extension/openadapt_chrome.json index 4ef90ccda..5398f85d1 100644 --- a/native_chrome_extension/openadapt_chrome.json +++ b/native_chrome_extension/openadapt_chrome.json @@ -1,7 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", + "path": "/Users/aaron/Documents/GitHub/OpenAdapt/native_chrome_extension/browser.sh", "type": "stdio", - "allowed_origins": ["chrome-extension://fipijfheidmdeingpecoiflbjefbnfjn/"] + "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5be4fea15..6cb085c1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ flake8-annotations = "^3.0.1" pre-commit = "^3.3.3" pympler = "^1.0.1" psutil = "^5.9.5" -pyobjc-framework-avfoundation = "^9.2" +pyobjc-framework-avfoundation = {version = "^9.2", markers = "sys_platform == 'darwin'"} fastapi = "0.98.0" [tool.poetry.dependencies.en_core_web_trf] From 1d693441213272fc61feed7d40320df503ce5707 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 21 Aug 2023 21:53:21 -0400 Subject: [PATCH 68/89] write documentstates to db and visualize --- native_chrome_extension/background.js | 10 +- native_chrome_extension/browser.py | 91 ++++++++----------- native_chrome_extension/browser.sh | 4 - .../mac_nativehost_file.json | 4 +- native_chrome_extension/view_messages.py | 40 ++++++++ openadapt/sockets.py | 13 ++- 6 files changed, 92 insertions(+), 70 deletions(-) mode change 100644 => 100755 native_chrome_extension/browser.py delete mode 100755 native_chrome_extension/browser.sh create mode 100644 native_chrome_extension/view_messages.py diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index afbed7066..7f17e19e0 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -6,11 +6,11 @@ */ -let port = null; // Native Messaging port +var port = null; // Native Messaging port const hostName = 'openadapt'; -/* +/* * Handle received messages from browser.js */ function onReceived(response) { @@ -29,7 +29,7 @@ function connect() { } -/* +/* * Message listener for content script */ function messageListener(message, sender, sendResponse) { @@ -41,7 +41,5 @@ function messageListener(message, sender, sendResponse) { connect(); } } - +connect(); chrome.runtime.onMessage.addListener(messageListener); - -connect(); \ No newline at end of file diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py old mode 100644 new mode 100755 index 391f7bb03..ed42c2fae --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -1,36 +1,18 @@ -#!/usr/bin/env -S python3 -u +#!/usr/bin/env python3 -"""Script for communicating with the browser extension. - -Usage: - - See `native_chrome_extension/browser.bat`. - -""" - -# Note that running python with the `-u` flag is required on Windows, -# in order to ensure that stdin and stdout are opened in binary, rather -# than text, mode. - -from multiprocessing.connection import Listener -from typing import Any import json +import sqlite3 import struct import sys -from loguru import logger +STORE_DATA = False -from openadapt import config, sockets - -def get_message() -> Any: +def get_message() -> dict: """Read a message from stdin and decode it. - Args: - None - Returns: - A Python object representing the message. + A dictionary representing the decoded message. """ raw_length = sys.stdin.buffer.read(4) if len(raw_length) == 0: @@ -40,7 +22,7 @@ def get_message() -> Any: return json.loads(message) -def encode_message(message_content: Any) -> dict[bytes, str]: +def encode_message(message_content: any) -> dict: """Encode a message for transmission, given its content. Args: @@ -54,47 +36,54 @@ def encode_message(message_content: Any) -> dict[bytes, str]: # (',', ':') to eliminate whitespace. # We want the most compact representation because the browser rejects # messages that exceed 1 MB. - encoded_content = json.dumps(message_content, separators=(",", ":")).encode( - "utf-8" - ) + encoded_content = json.dumps(message_content, separators=(",", ":")).encode("utf-8") encoded_length = struct.pack("@I", len(encoded_content)) return {"length": encoded_length, "content": encoded_content} -def send_message(encoded_message: dict[bytes, str]) -> None: +def send_message(encoded_message: dict) -> None: """Send an encoded message to stdout Args: encoded_message: The encoded message to be sent. - - Returns: - None """ sys.stdout.buffer.write(encoded_message["length"]) sys.stdout.buffer.write(encoded_message["content"]) sys.stdout.buffer.flush() -# TODO: send the Javascript memory state to the server - - -if __name__ == "__main__": - - # Establish a server connection - conn = sockets.create_server_connection(config.SOCKET_PORT) - - # address = (config.SOCKET_ADDRESS, config.SOCKET_PORT) - # conn = Listener(address, authkey=config.SOCKET_AUTHKEY) - # conn = conn.accept() - - # Start the event loop +def main() -> None: + # Connect to the database + conn = sqlite3.connect("messages.db") + c = conn.cursor() + # Create the messages table if it doesn't exist + c.execute(""" + CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message TEXT NOT NULL + ) + """) + # Keep track of the last message that was logged + last_message = None while True: - received_message = get_message() + message = get_message() + # Check if the message is the same as the last one + if message == last_message: + response = {"message": "Duplicate message received and ignored."} + else: + if STORE_DATA: + # Log the message to the database + c.execute( + "INSERT INTO messages (message) VALUES (?)", (json.dumps(message),) + ) + conn.commit() + response = {"message": "Data received and logged successfully!"} + last_message = message + # Send a response to the extension + encoded_response = encode_message(response) + send_message(encoded_response) + sys.stdout.buffer.flush() - # Sending message to Client - # sockets.server_send_message(config.SOCKET_PORT, received_message) - sockets.server_sends(conn, received_message) - # conn.send(received_message) - # Sending the received message back to background.js - send_message(encode_message(received_message)) +if __name__ == "__main__": + main() diff --git a/native_chrome_extension/browser.sh b/native_chrome_extension/browser.sh deleted file mode 100755 index 33025ecb4..000000000 --- a/native_chrome_extension/browser.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -dir=$(dirname "$0") -python3 -u "$dir/browser.py" \ No newline at end of file diff --git a/native_chrome_extension/mac_nativehost_file.json b/native_chrome_extension/mac_nativehost_file.json index 5398f85d1..984aff676 100644 --- a/native_chrome_extension/mac_nativehost_file.json +++ b/native_chrome_extension/mac_nativehost_file.json @@ -1,7 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "/Users/aaron/Documents/GitHub/OpenAdapt/native_chrome_extension/browser.sh", + "path": "/Users/aaron/Documents/GitHub/OpenAdapt/native_chrome_extension/browser.py", "type": "stdio", "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] -} \ No newline at end of file +} diff --git a/native_chrome_extension/view_messages.py b/native_chrome_extension/view_messages.py new file mode 100644 index 000000000..7fb725b43 --- /dev/null +++ b/native_chrome_extension/view_messages.py @@ -0,0 +1,40 @@ +import html +import json +import sqlite3 + +from nicegui import ui + +conn = sqlite3.connect("messages.db") +c = conn.cursor() + +c.execute("SELECT * FROM messages") +messages = c.fetchall() + +conn.close() + +dicts = [] +trees = [] + +# 28 : dict_keys(['action', 'documentBody', 'documentHead', 'url']) + + +for msg in messages: + dicts.append({"id": msg[0], "message": json.loads(msg[1])}) + +for d in dicts: + tree = [{"id": "id: " + f'{d["id"]} : url: {d["message"]["url"]}'}] + children = [] + children.append({"id": "action", "children": [{"id": str(d["message"]["action"])}]}) + document_body = html.escape(d["message"]["documentBody"]) + children.append( + {"id": "documentBody", "children": [{"id": "
" + document_body + "
"}]} + ) + document_head = html.escape(d["message"]["documentHead"]) + children.append( + {"id": "documentHead", "children": [{"id": "
" + document_head + "
"}]} + ) + children.append({"id": "url", "children": [{"id": str(d["message"]["url"])}]}) + tree[0]["children"] = children + ui.tree(tree, label_key="id") + +ui.run(port=7777) diff --git a/openadapt/sockets.py b/openadapt/sockets.py index 8f016831a..6a5928d2c 100644 --- a/openadapt/sockets.py +++ b/openadapt/sockets.py @@ -1,15 +1,14 @@ """Module for managing socket connections and communication.""" from multiprocessing import Queue -from multiprocessing.connection import Client, Listener, Connection +from multiprocessing.connection import Client, Connection, Listener +from typing import Any, Optional import time -from typing import Optional, Any from loguru import logger from openadapt import config - client_by_port = {} server_by_port = {} queue_by_port = {} @@ -60,8 +59,8 @@ def client_receive_message(port: int) -> Optional[str]: client_conn = client_by_port.get(port) if client_conn: try: - message = client_conn.recv() - return message + if message := client_conn.recv(): + return message except Exception as exc: logger.warning("Connection was closed.") del client_by_port[port] @@ -201,7 +200,7 @@ def event_loop() -> None: try: message = client_conn.recv() # if message: - # TODO: Handle message + # TODO: Handle message except EOFError: # Handle connection closed or error @@ -221,11 +220,11 @@ def event_loop() -> None: # del queue_by_port[port] - def server_sends(conn: Connection, message: Any) -> None: if conn: conn.send(message) + def client_receive(conn: Connection) -> Any: if conn: try: From c44ca0ac9f384e13eb5032c7a485453b41e76589 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Thu, 24 Aug 2023 18:30:05 -0400 Subject: [PATCH 69/89] remove duplicate messages, add exception handling, todo: look for better visualization --- native_chrome_extension/background.js | 27 ++++-- native_chrome_extension/browser.py | 27 ++---- native_chrome_extension/content.js | 116 ++++++++--------------- native_chrome_extension/view_messages.py | 24 +++-- native_chrome_extension/visualize.py | 0 5 files changed, 83 insertions(+), 111 deletions(-) create mode 100644 native_chrome_extension/visualize.py diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 7f17e19e0..0908a0fd4 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -5,20 +5,19 @@ * @see https://docs.google.com/presentation/d/106AXW3sBe7-7E-zIggnMnaUKUXWAj_aAuSxBspTDcGk/edit#slide=id.p */ - +const hostName = "openadapt"; var port = null; // Native Messaging port -const hostName = 'openadapt'; - +var lastMsg = null; /* * Handle received messages from browser.js -*/ + */ function onReceived(response) { console.log(response); } function onDisconnected() { - msg = 'Failed to connect: ' + chrome.runtime.lastError.message; // silence error + msg = "Failed to connect: " + chrome.runtime.lastError.message; // silence error port = null; } @@ -28,16 +27,26 @@ function connect() { port.onDisconnect.addListener(onDisconnected); } - /* * Message listener for content script -*/ + */ function messageListener(message, sender, sendResponse) { + const timestampThreshold = 3; // arbitrary threshold in milliseconds + try { + if (lastMsg !== null) { + if ( + Math.abs(message.timestamp - lastMsg.timestamp) < timestampThreshold && + message.tagName === lastMsg.tagName && + message.action === lastMsg.action + ) { + return; + } + } console.log({ message, sender, sendResponse }); port.postMessage(message); // send to browser.py (native messaging host) - } - catch (e) { + lastMsg = message; + } catch (e) { connect(); } } diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index ed42c2fae..0096931db 100755 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -5,7 +5,7 @@ import struct import sys -STORE_DATA = False +STORE_DATA = True def get_message() -> dict: @@ -63,25 +63,18 @@ def main() -> None: message TEXT NOT NULL ) """) - # Keep track of the last message that was logged - last_message = None + while True: message = get_message() - # Check if the message is the same as the last one - if message == last_message: - response = {"message": "Duplicate message received and ignored."} - else: - if STORE_DATA: - # Log the message to the database - c.execute( - "INSERT INTO messages (message) VALUES (?)", (json.dumps(message),) - ) - conn.commit() + if STORE_DATA: + # Log the message to the database + c.execute( + "INSERT INTO messages (message) VALUES (?)", (json.dumps(message),) + ) + conn.commit() response = {"message": "Data received and logged successfully!"} - last_message = message - # Send a response to the extension - encoded_response = encode_message(response) - send_message(encoded_response) + encoded_response = encode_message(response) + send_message(encoded_response) sys.stdout.buffer.flush() diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 4b3953e4e..6b0a261f0 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -2,124 +2,82 @@ * @file content.js * @description This file is injected into the web page and is responsible for * capturing DOM changes and sending them to the background script. -*/ + */ let logged = false; let ignoreAttributes = new Set(); let observer = null; - /* * Function to send a message to the background script -*/ + */ function sendMessageToBackgroundScript(message) { - chrome.runtime.sendMessage(message); + if (typeof chrome.app.isInstalled !== "undefined") + chrome.runtime.sendMessage(message); } - /* - * Function to capture initial document state and + * Function to capture initial document state and * send it to the background script -*/ + */ function captureDocumentState() { const documentBody = document.body.outerHTML; const documentHead = document.head.outerHTML; const page_url = window.location.href; sendMessageToBackgroundScript({ - action: 'captureDocumentState', + action: "captureDocumentState", documentBody: documentBody, documentHead: documentHead, - url: page_url + url: page_url, + timestamp: Date.now(), }); } +function handleElementClick(event) { + const element = event.target; + const tagName = element.tagName; + const attributes = {}; -/* - * Function to detect DOM changes -*/ -function detectDOMChanges(mutationsList) { - // Send a message to the background script when a DOM change is detected - if (mutationsList.length === 0) { - return; + for (const attr of element.attributes) { + attributes[attr.name] = attr.value; } - if (!logged) { - console.log({ mutationsList }); - // logged = true; - // debugger; - } - captureDocumentState(); -} - -/* - * Mutation observer callback function -*/ -function handleMutation(mutationsList) { - const filteredMutations = []; - - // Filter out mutations related to ignored attributes - mutationsList.forEach((mutation) => { - if (ignoreAttributes.has(mutation.attributeName)) { - return; - } - filteredMutations.push(mutation); + sendMessageToBackgroundScript({ + action: "elementClicked", + tagName: tagName, + attributes: attributes, + url: window.location.href, + timestamp: Date.now(), }); - - detectDOMChanges(filteredMutations); } +function handleElementInput(event) { + const element = event.target; + const tagName = element.tagName; + const attributes = {}; -/* - * Function to start observing DOM changes -*/ -function startObservingDOMChanges() { - // Create a new mutation observer if it doesn't exist - if (!observer) { - observer = new MutationObserver(handleMutation); + for (const attr of element.attributes) { + attributes[attr.name] = attr.value; } - // Start observing DOM changes - observer.observe(document, { - subtree: true, - childList: true, - attributes: true, - attributeFilter: Array.from(ignoreAttributes), - characterData: true + sendMessageToBackgroundScript({ + action: "elementInput", + tagName: tagName, + attributes: attributes, + url: window.location.href, + timestamp: Date.now(), }); } - -/* - * Function to stop observing DOM changes -*/ -function stopObservingDOMChanges() { - if (observer) { - observer.disconnect(); - observer = null; - } -} - - -/* - * Function to get element positions -*/ -function getElementPositions() { +function addEventListeners() { const elements = document.getElementsByTagName("*"); for (const element of elements) { - const rect = element.getBoundingClientRect(); - const attrs = ['top', 'right', 'bottom', 'left', 'width', 'height']; - for (const attr of attrs) { - element.setAttribute(`data-${attr}`, rect[attr]); - ignoreAttributes.add(`data-${attr}`); - } + element.addEventListener("click", handleElementClick); + element.addEventListener("input", handleElementInput); } } - -getElementPositions(); - +addEventListeners(); captureDocumentState(); - -startObservingDOMChanges(); diff --git a/native_chrome_extension/view_messages.py b/native_chrome_extension/view_messages.py index 7fb725b43..552023209 100644 --- a/native_chrome_extension/view_messages.py +++ b/native_chrome_extension/view_messages.py @@ -24,17 +24,29 @@ for d in dicts: tree = [{"id": "id: " + f'{d["id"]} : url: {d["message"]["url"]}'}] children = [] + children.append({"id": "action", "children": [{"id": str(d["message"]["action"])}]}) - document_body = html.escape(d["message"]["documentBody"]) - children.append( - {"id": "documentBody", "children": [{"id": "
" + document_body + "
"}]} - ) - document_head = html.escape(d["message"]["documentHead"]) + + if d["message"]["action"] == "captureDocumentState": + document_body = html.escape(d["message"]["documentBody"]) + children.append( + { + "id": "documentBody", + "children": [{"id": "
" + document_body + "
"}], + } + ) + document_head = html.escape(d["message"]["documentHead"]) + else: + tagName = d["message"]["tagName"] + children.append({"id": "tagName", "children": [{"id": str(tagName)}]}) + + attributes = d["message"]["attributes"] + children.append({"id": "attributes", "children": [{"id": str(attributes)}]}) children.append( {"id": "documentHead", "children": [{"id": "
" + document_head + "
"}]} ) children.append({"id": "url", "children": [{"id": str(d["message"]["url"])}]}) tree[0]["children"] = children - ui.tree(tree, label_key="id") + ui.tree(tree, label_key="id")._props["default-expand-all"] = True ui.run(port=7777) diff --git a/native_chrome_extension/visualize.py b/native_chrome_extension/visualize.py new file mode 100644 index 000000000..e69de29bb From 791e3eea19848129baf1f534d787c418f317dfb7 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Thu, 24 Aug 2023 19:29:26 -0400 Subject: [PATCH 70/89] xy + input value + debounce --- native_chrome_extension/content.js | 38 ++++++++++++++++++++++-- native_chrome_extension/view_messages.py | 20 ++++++++++--- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 6b0a261f0..193b16aa4 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -12,8 +12,7 @@ let observer = null; * Function to send a message to the background script */ function sendMessageToBackgroundScript(message) { - if (typeof chrome.app.isInstalled !== "undefined") - chrome.runtime.sendMessage(message); + chrome.runtime.sendMessage(message); } /* @@ -43,16 +42,35 @@ function handleElementClick(event) { attributes[attr.name] = attr.value; } + const rect = element.getBoundingClientRect(); + const x = rect.left + window.scrollX; + const y = rect.top + window.scrollY; + sendMessageToBackgroundScript({ action: "elementClicked", tagName: tagName, attributes: attributes, + x: x, + y: y, url: window.location.href, timestamp: Date.now(), }); } -function handleElementInput(event) { +function debounce(func, delay) { + let timerId; + return function (...args) { + if (timerId) { + clearTimeout(timerId); + } + timerId = setTimeout(() => { + func.apply(this, args); + timerId = null; + }, delay); + }; +} + +function handleDebouncedInput(event) { const element = event.target; const tagName = element.tagName; const attributes = {}; @@ -61,15 +79,29 @@ function handleElementInput(event) { attributes[attr.name] = attr.value; } + const rect = element.getBoundingClientRect(); + const x = rect.left + window.scrollX; + const y = rect.top + window.scrollY; + const value = element.value; + sendMessageToBackgroundScript({ action: "elementInput", tagName: tagName, attributes: attributes, + x: x, + y: y, + value: value, url: window.location.href, timestamp: Date.now(), }); } +const debouncedInputHandler = debounce(handleDebouncedInput, 500); + +function handleElementInput(event) { + debouncedInputHandler(event); +} + function addEventListeners() { const elements = document.getElementsByTagName("*"); diff --git a/native_chrome_extension/view_messages.py b/native_chrome_extension/view_messages.py index 552023209..3f45d4db3 100644 --- a/native_chrome_extension/view_messages.py +++ b/native_chrome_extension/view_messages.py @@ -36,16 +36,28 @@ } ) document_head = html.escape(d["message"]["documentHead"]) + children.append( + { + "id": "documentHead", + "children": [{"id": "
" + document_head + "
"}], + } + ) + children.append({"id": "url", "children": [{"id": str(d["message"]["url"])}]}) else: tagName = d["message"]["tagName"] children.append({"id": "tagName", "children": [{"id": str(tagName)}]}) + if d["message"]["action"] == "elementInput": + value = d["message"]["value"] + children.append({"id": "value", "children": [{"id": str(value)}]}) + + x, y = d["message"]["x"], d["message"]["y"] + children.append({"id": "x", "children": [{"id": str(x)}]}) + children.append({"id": "y", "children": [{"id": str(y)}]}) + attributes = d["message"]["attributes"] children.append({"id": "attributes", "children": [{"id": str(attributes)}]}) - children.append( - {"id": "documentHead", "children": [{"id": "
" + document_head + "
"}]} - ) - children.append({"id": "url", "children": [{"id": str(d["message"]["url"])}]}) + tree[0]["children"] = children ui.tree(tree, label_key="id")._props["default-expand-all"] = True From 09cab0202e385ffd5b0c6b86076458c37b8daf91 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Thu, 24 Aug 2023 19:40:42 -0400 Subject: [PATCH 71/89] add nativemessaginghost install script --- native_chrome_extension/install.sh | 41 ++++++++++++++++++++++++++++ native_chrome_extension/visualize.py | 0 2 files changed, 41 insertions(+) create mode 100755 native_chrome_extension/install.sh delete mode 100644 native_chrome_extension/visualize.py diff --git a/native_chrome_extension/install.sh b/native_chrome_extension/install.sh new file mode 100755 index 000000000..163666000 --- /dev/null +++ b/native_chrome_extension/install.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Set the name of your Native Messaging Host +HOST_NAME=openadapt + +# Set the path to your Native Messaging Host manifest file +MANIFEST_PATH=mac_nativehost_file.json + +# Check if the systemwide Native Messaging Host directory exists +if [ -d "/Library/Google/Chrome/NativeMessagingHosts/" ]; then + if [ "$1" == "install" ]; then + # Copy the Native Messaging Host manifest file to the directory + sudo cp "$MANIFEST_PATH" "/Library/Google/Chrome/NativeMessagingHosts/$HOST_NAME.json" + + echo "Native Messaging Host installed systemwide successfully!" + elif [ "$1" == "uninstall" ]; then + # Remove the Native Messaging Host manifest file from the directory + sudo rm "/Library/Google/Chrome/NativeMessagingHosts/$HOST_NAME.json" + + echo "Native Messaging Host uninstalled systemwide successfully!" + else + echo "Usage: $0 [install|uninstall]" + fi +fi + +# Check if the user Native Messaging Host directory exists +if [ -d "$HOME/Library/Application Support/Google/Chrome/NativeMessagingHosts/" ]; then + if [ "$1" == "install" ]; then + # Copy the Native Messaging Host manifest file to the directory + cp "$MANIFEST_PATH" "$HOME/Library/Application Support/Google/Chrome/NativeMessagingHosts/$HOST_NAME.json" + + echo "Native Messaging Host installed for user successfully!" + elif [ "$1" == "uninstall" ]; then + # Remove the Native Messaging Host manifest file from the directory + rm "$HOME/Library/Application Support/Google/Chrome/NativeMessagingHosts/$HOST_NAME.json" + + echo "Native Messaging Host uninstalled for user successfully!" + else + echo "Usage: $0 [install|uninstall]" + fi +fi diff --git a/native_chrome_extension/visualize.py b/native_chrome_extension/visualize.py deleted file mode 100644 index e69de29bb..000000000 From 781d531d39227598f8bfe47f877fe0a6ce025e9d Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Thu, 24 Aug 2023 21:08:10 -0400 Subject: [PATCH 72/89] Create view_messages_table.py --- .../view_messages_table.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 native_chrome_extension/view_messages_table.py diff --git a/native_chrome_extension/view_messages_table.py b/native_chrome_extension/view_messages_table.py new file mode 100644 index 000000000..18cf726dc --- /dev/null +++ b/native_chrome_extension/view_messages_table.py @@ -0,0 +1,53 @@ +import base64 +import datetime +import html +import json +import os +import sqlite3 + +conn = sqlite3.connect("messages.db") +c = conn.cursor() + +c.execute("SELECT * FROM messages") + +messages = c.fetchall() + +conn.close() + +logo_path = "icons/logo.png" +with open(logo_path, "rb") as f: + logo_data = f.read() +logo_base64 = base64.b64encode(logo_data).decode("utf-8") + +page = f""" + +Exported database: messages +""" + +for i, message in enumerate(messages): + message_json = json.loads(message[1]) + if message_json["action"] == "captureDocumentState": + message_json["documentBody"] = "..." + message_json["documentHead"] = "..." + + message_html = json.dumps(message_json, indent=4, ensure_ascii=False) + message_html = html.escape(message_html) + message_html = message_html.replace("\n", "
") + message_html = message_html.replace(" ", " ") + page += ( + f"{message[0]}" + ) + +page += f""" +
Table: messages
#id
INTEGER		message
TEXT
{i+1}{message_html}
+ +

Document generated by OpenAdapt on {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} +""" + +name = f"messages-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}.html" + +with open(name, "w") as f: + f.write(page) + +os.system(f"open {name}") From ad68e189f9279f37d6f2305d3b65f12ee92199c5 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Fri, 25 Aug 2023 14:12:22 -0400 Subject: [PATCH 73/89] add comment --- native_chrome_extension/background.js | 2 +- native_chrome_extension/view_messages.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/native_chrome_extension/background.js b/native_chrome_extension/background.js index 0908a0fd4..0de8782a1 100644 --- a/native_chrome_extension/background.js +++ b/native_chrome_extension/background.js @@ -31,7 +31,7 @@ function connect() { * Message listener for content script */ function messageListener(message, sender, sendResponse) { - const timestampThreshold = 3; // arbitrary threshold in milliseconds + const timestampThreshold = 30; // arbitrary threshold in milliseconds try { if (lastMsg !== null) { diff --git a/native_chrome_extension/view_messages.py b/native_chrome_extension/view_messages.py index 3f45d4db3..4b0a98930 100644 --- a/native_chrome_extension/view_messages.py +++ b/native_chrome_extension/view_messages.py @@ -1,3 +1,7 @@ +""" +nicegui app to view messages from the native chrome extension +""" + import html import json import sqlite3 From 9f2932eab6b85919c93757c6358e39b81ad8cfed Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Fri, 25 Aug 2023 14:19:29 -0400 Subject: [PATCH 74/89] Update browser.py --- native_chrome_extension/browser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 0096931db..d463bee3f 100755 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -53,6 +53,7 @@ def send_message(encoded_message: dict) -> None: def main() -> None: + # TODO: use sockets to communicate with openadapt client # Connect to the database conn = sqlite3.connect("messages.db") c = conn.cursor() From 33718cfe7d25430527c81b729b85e7c0fd4782d3 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Sun, 27 Aug 2023 20:51:34 -0400 Subject: [PATCH 75/89] add mutationObserverImplementation --- .../contentMutationObserverImplementation.js | 72 +++++++++++++++++++ ...w_messages_table.py => export_messages.py} | 11 ++- 2 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 native_chrome_extension/contentMutationObserverImplementation.js rename native_chrome_extension/{view_messages_table.py => export_messages.py} (81%) diff --git a/native_chrome_extension/contentMutationObserverImplementation.js b/native_chrome_extension/contentMutationObserverImplementation.js new file mode 100644 index 000000000..75e5683a9 --- /dev/null +++ b/native_chrome_extension/contentMutationObserverImplementation.js @@ -0,0 +1,72 @@ +/** + * @file content.js + * @description This file is injected into the web page and is responsible for + * capturing DOM changes and sending them to the background script. + */ + +let logged = false; +let ignoreAttributes = new Set(); + +/* + * Function to send a message to the background script + */ +function sendMessageToBackgroundScript(message) { + chrome.runtime.sendMessage(message); +} + +/* + * Function to capture initial document state and + * send it to the background script + */ +function captureDocumentState() { + const documentBody = document.body.outerHTML; + const documentHead = document.head.outerHTML; + const page_url = window.location.href; + + sendMessageToBackgroundScript({ + action: "captureDocumentState", + documentBody: documentBody, + documentHead: documentHead, + url: page_url, + timestamp: Date.now(), + }); +} + +const observer = new MutationObserver((mutations) => { + mutations.forEach((mutation) => { + const { type, target } = mutation; + const tagName = target.tagName.toLowerCase(); + const attributes = {}; + + for (const attr of target.attributes) { + attributes[attr.name] = attr.value; + } + + const rect = target.getBoundingClientRect(); + const x = rect.left + window.scrollX; + const y = rect.top + window.scrollY; + const value = target.value; + + sendMessageToBackgroundScript({ + action: "elementMutation", + type: type, + tagName: tagName, + attributes: attributes, + x: x, + y: y, + value: value, + url: window.location.href, + timestamp: Date.now(), + }); + }); +}); + +observer.observe(document.body, { + childList: true, + subtree: true, + attributes: true, + attributeOldValue: true, + characterData: true, + characterDataOldValue: true, +}); +captureDocumentState(); diff --git a/native_chrome_extension/view_messages_table.py b/native_chrome_extension/export_messages.py similarity index 81% rename from native_chrome_extension/view_messages_table.py rename to native_chrome_extension/export_messages.py index 18cf726dc..0f94132e1 100644 --- a/native_chrome_extension/view_messages_table.py +++ b/native_chrome_extension/export_messages.py @@ -5,6 +5,8 @@ import os import sqlite3 +from loguru import logger + conn = sqlite3.connect("messages.db") c = conn.cursor() @@ -21,7 +23,7 @@ page = f""" -Exported database: messages +Exported database: messages
Table: messages
#id
INTEGER		message
TEXT
""" for i, message in enumerate(messages): @@ -45,9 +47,14 @@

Document generated by OpenAdapt on {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} """ -name = f"messages-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}.html" +try: + name = f"messages-{json.loads(messages[0][1])['timestamp']}.html" +except Exception as e: + logger.error(e) + name = f"messages-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}.html" with open(name, "w") as f: f.write(page) os.system(f"open {name}") +logger.info(f"Saved messages to {name}") From 7dd21885924a8b8be03afb95c9a387b3461a668c Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Sun, 27 Aug 2023 21:34:10 -0400 Subject: [PATCH 76/89] work on socket --- native_chrome_extension/browser.py | 30 ++++++++++++++++++++++++-- native_chrome_extension/mock_client.py | 20 ++++++++++++----- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index d463bee3f..06e86951f 100755 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -1,11 +1,23 @@ #!/usr/bin/env python3 +"""Script for communicating with the browser extension. +Usage: + See `native_chrome_extension/browser.bat`. +""" + +# Note that running python with the `-u` flag is required on Windows, +# in order to ensure that stdin and stdout are opened in binary, rather +# than text, mode. + +from multiprocessing.connection import Listener import json import sqlite3 import struct import sys -STORE_DATA = True +from openadapt import config, sockets + +STORE_DATA = False def get_message() -> dict: @@ -52,8 +64,20 @@ def send_message(encoded_message: dict) -> None: sys.stdout.buffer.flush() +def send_message_to_client(message): + # check if client connection exists + # if not, create one + try: + conn = sockets.create_client_connection(config.SOCKET_PORT) + conn.send(message) + conn.close() + except Exception as exc: + print(f"Error sending message to client: {exc}") + + def main() -> None: # TODO: use sockets to communicate with openadapt client + # Connect to the database conn = sqlite3.connect("messages.db") c = conn.cursor() @@ -66,6 +90,7 @@ def main() -> None: """) while True: + conn = sockets.create_server_connection(config.SOCKET_PORT) message = get_message() if STORE_DATA: # Log the message to the database @@ -76,7 +101,8 @@ def main() -> None: response = {"message": "Data received and logged successfully!"} encoded_response = encode_message(response) send_message(encoded_response) - sys.stdout.buffer.flush() + send_message_to_client(message) + sys.stdout.buffer.flush() if __name__ == "__main__": diff --git a/native_chrome_extension/mock_client.py b/native_chrome_extension/mock_client.py index c5d753d36..f1dfb3238 100644 --- a/native_chrome_extension/mock_client.py +++ b/native_chrome_extension/mock_client.py @@ -1,19 +1,29 @@ from multiprocessing.connection import Client +import time + from loguru import logger + from openadapt import config, sockets -import time -RETRY_INTERVAL=5 +RETRY_INTERVAL = 5 SERVER_SENDS = True + +def establish_connection(): + return sockets.create_client_connection(config.SOCKET_PORT) + + def main(): conn = sockets.create_client_connection(config.SOCKET_PORT) while True: try: if SERVER_SENDS: - logger.info("Waiting for message...") - msg = sockets.client_receive_message(config.SOCKET_PORT) - logger.info(f"{msg=}") + if conn.closed: + conn = establish_connection() + else: + logger.info("Waiting for message...") + msg = conn.recv() + logger.info(f"{msg=}") else: t = time.time() logger.info(f"Sending {t=}") From a80004cfc6ee16b53aff4778d38977dfcfa175dc Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:51:16 -0400 Subject: [PATCH 77/89] fixed socket --- native_chrome_extension/browser.py | 18 +++++++----------- native_chrome_extension/export_messages.py | 5 ++--- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 06e86951f..9c2019b91 100755 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -17,7 +17,7 @@ from openadapt import config, sockets -STORE_DATA = False +STORE_DATA = True def get_message() -> dict: @@ -64,13 +64,11 @@ def send_message(encoded_message: dict) -> None: sys.stdout.buffer.flush() -def send_message_to_client(message): +def send_message_to_client(conn, message): # check if client connection exists # if not, create one try: - conn = sockets.create_client_connection(config.SOCKET_PORT) conn.send(message) - conn.close() except Exception as exc: print(f"Error sending message to client: {exc}") @@ -79,8 +77,8 @@ def main() -> None: # TODO: use sockets to communicate with openadapt client # Connect to the database - conn = sqlite3.connect("messages.db") - c = conn.cursor() + db_conn = sqlite3.connect("messages.db") + c = db_conn.cursor() # Create the messages table if it doesn't exist c.execute(""" CREATE TABLE IF NOT EXISTS messages ( @@ -88,21 +86,19 @@ def main() -> None: message TEXT NOT NULL ) """) - + conn = sockets.create_server_connection(config.SOCKET_PORT) while True: - conn = sockets.create_server_connection(config.SOCKET_PORT) message = get_message() if STORE_DATA: # Log the message to the database c.execute( "INSERT INTO messages (message) VALUES (?)", (json.dumps(message),) ) - conn.commit() + db_conn.commit() response = {"message": "Data received and logged successfully!"} encoded_response = encode_message(response) send_message(encoded_response) - send_message_to_client(message) - sys.stdout.buffer.flush() + send_message_to_client(conn, message) if __name__ == "__main__": diff --git a/native_chrome_extension/export_messages.py b/native_chrome_extension/export_messages.py index 0f94132e1..6002939af 100644 --- a/native_chrome_extension/export_messages.py +++ b/native_chrome_extension/export_messages.py @@ -47,10 +47,9 @@

Document generated by OpenAdapt on {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} """ -try: +if len(messages) > 0: name = f"messages-{json.loads(messages[0][1])['timestamp']}.html" -except Exception as e: - logger.error(e) +else: name = f"messages-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}.html" with open(name, "w") as f: From 3a468f8640a9667ecf03bd06188ea045ab1b4722 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:52:02 -0400 Subject: [PATCH 78/89] Update browser.py --- native_chrome_extension/browser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 9c2019b91..7f7ba29ca 100755 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -88,6 +88,8 @@ def main() -> None: """) conn = sockets.create_server_connection(config.SOCKET_PORT) while True: + if conn.closed: + conn = sockets.create_server_connection(config.SOCKET_PORT) message = get_message() if STORE_DATA: # Log the message to the database From 478e3254083895b34d92c7250ebef1ad57f4248b Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:57:28 -0400 Subject: [PATCH 79/89] update windows path & clean --- native_chrome_extension/browser.py | 2 -- native_chrome_extension/content.js | 1 - native_chrome_extension/openadapt_chrome.json | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index 7f7ba29ca..affde0f68 100755 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -74,8 +74,6 @@ def send_message_to_client(conn, message): def main() -> None: - # TODO: use sockets to communicate with openadapt client - # Connect to the database db_conn = sqlite3.connect("messages.db") c = db_conn.cursor() diff --git a/native_chrome_extension/content.js b/native_chrome_extension/content.js index 193b16aa4..f4e11c1ed 100644 --- a/native_chrome_extension/content.js +++ b/native_chrome_extension/content.js @@ -6,7 +6,6 @@ let logged = false; let ignoreAttributes = new Set(); -let observer = null; /* * Function to send a message to the background script diff --git a/native_chrome_extension/openadapt_chrome.json b/native_chrome_extension/openadapt_chrome.json index 5398f85d1..5bd0f87d6 100644 --- a/native_chrome_extension/openadapt_chrome.json +++ b/native_chrome_extension/openadapt_chrome.json @@ -1,7 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "/Users/aaron/Documents/GitHub/OpenAdapt/native_chrome_extension/browser.sh", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", "type": "stdio", "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] -} \ No newline at end of file +} From 4b8bf275f370417e4bfd9e4e95ed696268080076 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 12:22:49 -0400 Subject: [PATCH 80/89] styling --- .gitignore | 2 + native_chrome_extension/browser.py | 18 +++++--- native_chrome_extension/export_messages.py | 16 ++++--- native_chrome_extension/manifest.json | 45 +++++++++---------- native_chrome_extension/mock_client.py | 10 +++-- native_chrome_extension/openadapt_chrome.json | 10 ++--- native_chrome_extension/view_messages.py | 4 +- openadapt/browser/__init__.py | 2 - openadapt/playback.py | 2 +- openadapt/strategies/mixins/sam.py | 2 +- openadapt/window/_macos.py | 2 +- 11 files changed, 58 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index 78f088041..6bacf997e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ cache *.db *.db-journal +*.html + # VSCode .VSCode .vsCode diff --git a/native_chrome_extension/browser.py b/native_chrome_extension/browser.py index affde0f68..d1ef38eb4 100755 --- a/native_chrome_extension/browser.py +++ b/native_chrome_extension/browser.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3 # noqa: D205 -"""Script for communicating with the browser extension. +"""Script for communicating with the browser extension. # noqa: D205 D415 Usage: See `native_chrome_extension/browser.bat`. """ @@ -9,7 +9,6 @@ # in order to ensure that stdin and stdout are opened in binary, rather # than text, mode. -from multiprocessing.connection import Listener import json import sqlite3 import struct @@ -54,7 +53,7 @@ def encode_message(message_content: any) -> dict: def send_message(encoded_message: dict) -> None: - """Send an encoded message to stdout + """Send an encoded message to stdout. Args: encoded_message: The encoded message to be sent. @@ -64,9 +63,13 @@ def send_message(encoded_message: dict) -> None: sys.stdout.buffer.flush() -def send_message_to_client(conn, message): - # check if client connection exists - # if not, create one +def send_message_to_client(conn: sockets.Connection, message: dict) -> None: + """Send a message to the client. + + Args: + conn: The connection to the client. + message: The message to be sent. + """ try: conn.send(message) except Exception as exc: @@ -74,6 +77,7 @@ def send_message_to_client(conn, message): def main() -> None: + """Main function.""" # Connect to the database db_conn = sqlite3.connect("messages.db") c = db_conn.cursor() diff --git a/native_chrome_extension/export_messages.py b/native_chrome_extension/export_messages.py index 6002939af..3a9d850a1 100644 --- a/native_chrome_extension/export_messages.py +++ b/native_chrome_extension/export_messages.py @@ -1,3 +1,5 @@ +"""Script to export messages from the database to an HTML file.""" + import base64 import datetime import html @@ -24,7 +26,7 @@ page = f""" Exported database: messages
Table: messages
#id
INTEGER		message
TEXT
-""" +""" # noqa for i, message in enumerate(messages): message_json = json.loads(message[1]) @@ -41,11 +43,13 @@ f" align='right'>{message[0]}" ) -page += f""" -
Table: messages
#id
INTEGER		message
TEXT
{message_html}
- -

Document generated by OpenAdapt on {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} -""" +page += ( + "" + '' + "

Document generated by OpenAdapt on" + f" {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" +) if len(messages) > 0: name = f"messages-{json.loads(messages[0][1])['timestamp']}.html" diff --git a/native_chrome_extension/manifest.json b/native_chrome_extension/manifest.json index 5306894cd..bbd4b7742 100644 --- a/native_chrome_extension/manifest.json +++ b/native_chrome_extension/manifest.json @@ -1,28 +1,23 @@ { - - "name": "openadapt", - "description": "Native messaging example add-on", - "version": "1.0", - "manifest_version": 3, - "icons": { - "48": "icons/logo.png" - }, - "action": { - "default_icon": "icons/logo.png" - }, - "background": { - "service_worker": "background.js" - }, - "permissions": [ - "activeTab", - "tabs", - "nativeMessaging", - "scripting" - ], - "host_permissions": [""], - "content_scripts": [{ + "name": "openadapt", + "description": "Native messaging example add-on", + "version": "1.0", + "manifest_version": 3, + "icons": { + "48": "icons/logo.png" + }, + "action": { + "default_icon": "icons/logo.png" + }, + "background": { + "service_worker": "background.js" + }, + "permissions": ["activeTab", "tabs", "nativeMessaging", "scripting"], + "host_permissions": [""], + "content_scripts": [ + { "matches": [""], "js": ["content.js"] - }] - } - \ No newline at end of file + } + ] +} diff --git a/native_chrome_extension/mock_client.py b/native_chrome_extension/mock_client.py index f1dfb3238..91a95117e 100644 --- a/native_chrome_extension/mock_client.py +++ b/native_chrome_extension/mock_client.py @@ -1,4 +1,4 @@ -from multiprocessing.connection import Client +"""Mock client for testing the server.""" import time from loguru import logger @@ -9,12 +9,14 @@ SERVER_SENDS = True -def establish_connection(): +def establish_connection() -> sockets.Connection: + """Establish a connection to the server.""" return sockets.create_client_connection(config.SOCKET_PORT) -def main(): - conn = sockets.create_client_connection(config.SOCKET_PORT) +def main() -> None: + """Main function.""" + conn = establish_connection() while True: try: if SERVER_SENDS: diff --git a/native_chrome_extension/openadapt_chrome.json b/native_chrome_extension/openadapt_chrome.json index 5bd0f87d6..483b11aff 100644 --- a/native_chrome_extension/openadapt_chrome.json +++ b/native_chrome_extension/openadapt_chrome.json @@ -1,7 +1,7 @@ { - "name": "openadapt", - "description": "OpenAdapt DOM Listener", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", - "type": "stdio", - "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] + "name": "openadapt", + "description": "OpenAdapt DOM Listener", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", + "type": "stdio", + "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] } diff --git a/native_chrome_extension/view_messages.py b/native_chrome_extension/view_messages.py index 4b0a98930..93c183b5f 100644 --- a/native_chrome_extension/view_messages.py +++ b/native_chrome_extension/view_messages.py @@ -1,6 +1,4 @@ -""" -nicegui app to view messages from the native chrome extension -""" +"""nicegui app to view messages from the native chrome extension.""" import html import json diff --git a/openadapt/browser/__init__.py b/openadapt/browser/__init__.py index 59f7c2e40..50ac45131 100644 --- a/openadapt/browser/__init__.py +++ b/openadapt/browser/__init__.py @@ -1,7 +1,5 @@ - from typing import Any - # TODO: implement extension # def get_active_chrome_data() -> Any: # try: diff --git a/openadapt/playback.py b/openadapt/playback.py index 0e7889ee9..36a0b6450 100644 --- a/openadapt/playback.py +++ b/openadapt/playback.py @@ -1,7 +1,7 @@ """Utilities for playing back ActionEvents.""" from loguru import logger -from oa_pynput import mouse, keyboard +from oa_pynput import keyboard, mouse from openadapt.common import KEY_EVENTS, MOUSE_EVENTS from openadapt.models import ActionEvent diff --git a/openadapt/strategies/mixins/sam.py b/openadapt/strategies/mixins/sam.py index 5f1a193ac..4be3ca835 100644 --- a/openadapt/strategies/mixins/sam.py +++ b/openadapt/strategies/mixins/sam.py @@ -14,9 +14,9 @@ class MyReplayStrategy(SAMReplayStrategyMixin): from loguru import logger from PIL import Image from segment_anything import ( - modeling, SamAutomaticMaskGenerator, SamPredictor, + modeling, sam_model_registry, ) import matplotlib.axes as axes diff --git a/openadapt/window/_macos.py b/openadapt/window/_macos.py index f17b88aea..037276021 100644 --- a/openadapt/window/_macos.py +++ b/openadapt/window/_macos.py @@ -7,8 +7,8 @@ from loguru import logger import AppKit import ApplicationServices -import oa_atomacos import Foundation +import oa_atomacos import Quartz From 1a74aa157d12673ee3e6e572a8f3aed635e656b3 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 12:55:59 -0400 Subject: [PATCH 81/89] more styling --- openadapt/browser/__init__.py | 8 ------- openadapt/crud.py | 8 ++++--- openadapt/events.py | 8 ++++--- openadapt/models.py | 2 +- openadapt/record.py | 18 +++++++--------- openadapt/sockets.py | 39 ++++++++++++++--------------------- openadapt/utils.py | 28 ++++++++++++++++++------- 7 files changed, 56 insertions(+), 55 deletions(-) diff --git a/openadapt/browser/__init__.py b/openadapt/browser/__init__.py index 50ac45131..4d21fdb54 100644 --- a/openadapt/browser/__init__.py +++ b/openadapt/browser/__init__.py @@ -1,9 +1 @@ -from typing import Any - -# TODO: implement extension -# def get_active_chrome_data() -> Any: -# try: # # TODO: implement -# except Exception as exc: -# logger.warning(f"{exc=}") -# return None diff --git a/openadapt/crud.py b/openadapt/crud.py index c534cc2ae..2ce6f24ec 100644 --- a/openadapt/crud.py +++ b/openadapt/crud.py @@ -126,7 +126,9 @@ def insert_window_event( _insert(event_data, WindowEvent, window_events) -def insert_browser_event(recording_timestamp, event_timestamp, event_data) -> None: +def insert_browser_event( + recording_timestamp: int, event_timestamp: int, event_data: dict[str, Any] = None +) -> None: """Insert a browser event into the database. Args: @@ -412,8 +414,8 @@ def get_window_events(recording: Recording) -> list[WindowEvent]: return _get(WindowEvent, recording.timestamp) -def get_browser_events(recording) -> list[BrowserEvent]: - """Get browser events for a given recording +def get_browser_events(recording: Recording) -> list[BrowserEvent]: + """Get browser events for a given recording. Args: recording (Recording): recording object diff --git a/openadapt/events.py b/openadapt/events.py index b04f8ac8c..b0135721c 100644 --- a/openadapt/events.py +++ b/openadapt/events.py @@ -731,7 +731,8 @@ def process_events( ) logger.info( "before" - f" {num_action_events=} {num_window_events=} {num_screenshots=} {num_browser_events=} " + f" {num_action_events=} {num_window_events=}" + f" {num_screenshots=} {num_browser_events=} " f"{num_total=}" ) process_fns = [ @@ -782,11 +783,12 @@ def process_events( pct_action_events = num_action_events_ / num_action_events pct_window_events = num_window_events_ / num_window_events pct_screenshots = num_screenshots_ / num_screenshots - pct_browser_events = num_browser_events_ / num_browser_events + # pct_browser_events = num_browser_events_ / num_browser_events pct_total = num_total_ / num_total logger.info( "after" - f" {num_action_events_=} {num_window_events_=} {num_screenshots_=} {num_browser_events_} " + f" {num_action_events_=} {num_window_events_=}" + f" {num_screenshots_=} {num_browser_events_} " f"{num_total=}" ) logger.info( diff --git a/openadapt/models.py b/openadapt/models.py index 179cbbf64..b3c1b21a2 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -1,6 +1,6 @@ """This module defines the models used in the OpenAdapt system.""" -from typing import Any, Union +from typing import Union import io from loguru import logger diff --git a/openadapt/record.py b/openadapt/record.py index b71305132..8344076ca 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -8,8 +8,7 @@ from collections import namedtuple from functools import partial, wraps -from multiprocessing.connection import Client -from typing import Any, Callable, Dict +from typing import Any, Callable, Union import multiprocessing import os import queue @@ -175,6 +174,7 @@ def process_events( screen_write_q: A queue for writing screen events. action_write_q: A queue for writing action events. window_write_q: A queue for writing window events. + browser_write_q: A queue for writing browser events. perf_q: A queue for collecting performance data. recording_timestamp: The timestamp of the recording. terminate_event: An event to signal the termination of the process. @@ -214,7 +214,7 @@ def process_events( event.data["screenshot_timestamp"] = prev_screen_event.timestamp event.data["window_event_timestamp"] = prev_window_event.timestamp event.data["browser_event_timestamp"] = ( - prev_browser_event.timestamp if not prev_browser_event is None else None + prev_browser_event.timestamp if prev_browser_event is not None else None ) process_event( event, @@ -315,16 +315,14 @@ def write_browser_event( recording_timestamp: float, event: Event, perf_q: sq.SynchronizedQueue, -): - """ - Write a browser event to the database and update the performance queue. +) -> None: + """Write a browser event to the database and update the performance queue. Args: recording_timestamp: The timestamp of the recording. event: A browser event to be written. perf_q: A queue for collecting performance data. """ - assert event.type == "browser", event crud.insert_browser_event(recording_timestamp, event.timestamp, event.data) perf_q.put((event.type, event.timestamp, utils.get_timestamp())) @@ -606,8 +604,7 @@ def read_browser_events( terminate_event: multiprocessing.Event, recording_timestamp: float, ) -> None: - """ - Read browser events and add them to the event queue. + """Read browser events and add them to the event queue. Args: event_q: A queue for adding window events. @@ -616,7 +613,7 @@ def read_browser_events( """ utils.configure_logging(logger, LOG_LEVEL) utils.set_start_time(recording_timestamp) - logger.info(f"starting") + logger.info("starting") conn = sockets.create_client_connection(config.SOCKET_PORT) while not terminate_event.is_set() and conn is not None: try: @@ -639,6 +636,7 @@ def read_browser_events( logger.info("No message received or received None Type Message.") except Exception as exc: logger.warning("Connection closed.") + logger.warning(exc) break # while True: # try: diff --git a/openadapt/sockets.py b/openadapt/sockets.py index 6a5928d2c..1fde00be0 100644 --- a/openadapt/sockets.py +++ b/openadapt/sockets.py @@ -15,8 +15,7 @@ def client_send_message(port: int, msg: Any) -> None: - """ - Send a message to the client connection associated with the given port. + """Send a message to the client connection associated with the given port. Args: port: The port number associated with the client connection. @@ -31,8 +30,7 @@ def client_send_message(port: int, msg: Any) -> None: def server_send_message(port: int, msg: Any) -> None: - """ - Send a message to the server connection associated with the given port. + """Send a message to the server connection associated with the given port. Args: port: The port number associated with the server connection. @@ -47,8 +45,7 @@ def server_send_message(port: int, msg: Any) -> None: def client_receive_message(port: int) -> Optional[str]: - """ - Receive a message from the client connection associated with the given port. + """Receive a message from the client connection associated with the given port. Args: port: The port number associated with the client connection. @@ -62,13 +59,13 @@ def client_receive_message(port: int) -> Optional[str]: if message := client_conn.recv(): return message except Exception as exc: - logger.warning("Connection was closed.") + logger.error("Connection was closed.") + logger.error(exc) del client_by_port[port] def server_receive_message(port: int) -> Optional[str]: - """ - Receive a message from the server connection associated with the given port. + """Receive a message from the server connection associated with the given port. Args: port: The port number associated with the server connection. @@ -95,8 +92,7 @@ def server_receive_message(port: int) -> Optional[str]: def client_add_sink(port: int, queue: Queue) -> None: - """ - Add a sink queue to the specified client port. + """Add a sink queue to the specified client port. Args: port: The port number to associate with the sink queue. @@ -114,8 +110,7 @@ def client_add_sink(port: int, queue: Queue) -> None: def server_add_sink(port: int, queue: Queue) -> None: - """ - Add a sink queue to the specified server port. + """Add a sink queue to the specified server port. Args: port: The port number to associate with the sink queue. @@ -135,9 +130,8 @@ def server_add_sink(port: int, queue: Queue) -> None: _terminate_event: Optional[bool] = None -def set_terminate_event(terminate_event) -> None: - """ - Set the termination event to control the event loop. +def set_terminate_event(terminate_event: bool) -> None: + """Set the termination event to control the event loop. Args: terminate_event: The termination event object. @@ -150,8 +144,7 @@ def set_terminate_event(terminate_event) -> None: def create_client_connection(port: int) -> Connection: - """ - Create a client connection and establish a connection to the specified port. + """Create a client connection and establish a connection to the specified port. Args: port: The port number to connect to. @@ -167,8 +160,7 @@ def create_client_connection(port: int) -> Connection: def create_server_connection(port: int) -> Connection: - """ - Create a server connection and start listening for connections on the specified port. + """Create and listen for connections on the specified port. Args: port: The port number to bind the server connection to. @@ -185,8 +177,7 @@ def create_server_connection(port: int) -> Connection: def event_loop() -> None: - """ - The event loop for receiving and handling messages. + """The event loop for receiving and handling messages. Raises: AssertionError: If `_terminate_event` is not set. @@ -198,7 +189,7 @@ def event_loop() -> None: while not _terminate_event.is_set(): for port, client_conn in client_by_port.items(): try: - message = client_conn.recv() + message = client_conn.recv() # noqa: F841 # if message: # TODO: Handle message @@ -221,11 +212,13 @@ def event_loop() -> None: def server_sends(conn: Connection, message: Any) -> None: + """Send a message to the server connection associated with the given port.""" if conn: conn.send(message) def client_receive(conn: Connection) -> Any: + """Receive a message from the client connection associated with the given port.""" if conn: try: message = conn.recv() diff --git a/openadapt/utils.py b/openadapt/utils.py index 51faceabe..c087eb84f 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -746,8 +746,7 @@ def strip_element_state(action_event: ActionEvent) -> ActionEvent: def get_free_port() -> int: - """ - Get a free port number on the local machine. + """Get a free port number on the local machine. Returns: An available free port number. @@ -763,7 +762,15 @@ def get_free_port() -> int: return port -def send_kill_signal(pid): +def send_kill_signal(pid: int) -> None: + """Send a kill signal to the process identified by the PID. + + Args: + pid (int): The PID of the process. + + Raises: + OSError: If the kill signal cannot be sent. + """ try: # Send the kill signal (SIGTERM) to the process identified by the PID os.kill(pid, signal.SIGTERM) @@ -772,16 +779,23 @@ def send_kill_signal(pid): logger.info(f"Failed to send kill signal: {e}") -def get_pid_by_name(process_name): +def get_pid_by_name(process_name: str) -> int: + """Get the PID of the process with the given name. + + Args: + process_name (str): The name of the process. + + Returns: + int: The PID of the process. + """ for process in psutil.process_iter(["pid", "name"]): if process.info["name"] == process_name: return process.info["pid"] return None -def get_functions(name) -> dict: - """ - Get a dictionary of function names to functions for all non-private functions +def get_functions(name: str) -> dict: + """Get a dictionary of function names to functions for all non-private functions. Usage: From bc2d6a65c764c38a250cfe42f35b0c4563c7a8d0 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 12:56:08 -0400 Subject: [PATCH 82/89] delete init --- openadapt/browser/__init__.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 openadapt/browser/__init__.py diff --git a/openadapt/browser/__init__.py b/openadapt/browser/__init__.py deleted file mode 100644 index 4d21fdb54..000000000 --- a/openadapt/browser/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# # TODO: implement From a5daeb4683f504daffb6dcb8475bef587723e78a Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 13:07:07 -0400 Subject: [PATCH 83/89] rename directory --- {native_chrome_extension => chrome}/background.js | 0 chrome/browser.bat | 3 +++ {native_chrome_extension => chrome}/browser.py | 0 {native_chrome_extension => chrome}/content.js | 0 .../contentMutationObserverImplementation.js | 0 .../export_messages.py | 0 .../icons/MLDSAI_logo.png | Bin {native_chrome_extension => chrome}/icons/logo.png | Bin {native_chrome_extension => chrome}/install.sh | 0 .../mac_nativehost_file.json | 2 +- {native_chrome_extension => chrome}/manifest.json | 0 {native_chrome_extension => chrome}/mock_client.py | 0 .../openadapt_chrome.json | 2 +- .../view_messages.py | 0 native_chrome_extension/browser.bat | 3 --- 15 files changed, 5 insertions(+), 5 deletions(-) rename {native_chrome_extension => chrome}/background.js (100%) create mode 100644 chrome/browser.bat rename {native_chrome_extension => chrome}/browser.py (100%) rename {native_chrome_extension => chrome}/content.js (100%) rename {native_chrome_extension => chrome}/contentMutationObserverImplementation.js (100%) rename {native_chrome_extension => chrome}/export_messages.py (100%) rename {native_chrome_extension => chrome}/icons/MLDSAI_logo.png (100%) rename {native_chrome_extension => chrome}/icons/logo.png (100%) rename {native_chrome_extension => chrome}/install.sh (100%) rename {native_chrome_extension => chrome}/mac_nativehost_file.json (66%) rename {native_chrome_extension => chrome}/manifest.json (100%) rename {native_chrome_extension => chrome}/mock_client.py (100%) rename {native_chrome_extension => chrome}/openadapt_chrome.json (62%) rename {native_chrome_extension => chrome}/view_messages.py (100%) delete mode 100644 native_chrome_extension/browser.bat diff --git a/native_chrome_extension/background.js b/chrome/background.js similarity index 100% rename from native_chrome_extension/background.js rename to chrome/background.js diff --git a/chrome/browser.bat b/chrome/browser.bat new file mode 100644 index 000000000..75a4394bb --- /dev/null +++ b/chrome/browser.bat @@ -0,0 +1,3 @@ +@echo off + +python -u "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\chrome\\browser.py" diff --git a/native_chrome_extension/browser.py b/chrome/browser.py similarity index 100% rename from native_chrome_extension/browser.py rename to chrome/browser.py diff --git a/native_chrome_extension/content.js b/chrome/content.js similarity index 100% rename from native_chrome_extension/content.js rename to chrome/content.js diff --git a/native_chrome_extension/contentMutationObserverImplementation.js b/chrome/contentMutationObserverImplementation.js similarity index 100% rename from native_chrome_extension/contentMutationObserverImplementation.js rename to chrome/contentMutationObserverImplementation.js diff --git a/native_chrome_extension/export_messages.py b/chrome/export_messages.py similarity index 100% rename from native_chrome_extension/export_messages.py rename to chrome/export_messages.py diff --git a/native_chrome_extension/icons/MLDSAI_logo.png b/chrome/icons/MLDSAI_logo.png similarity index 100% rename from native_chrome_extension/icons/MLDSAI_logo.png rename to chrome/icons/MLDSAI_logo.png diff --git a/native_chrome_extension/icons/logo.png b/chrome/icons/logo.png similarity index 100% rename from native_chrome_extension/icons/logo.png rename to chrome/icons/logo.png diff --git a/native_chrome_extension/install.sh b/chrome/install.sh similarity index 100% rename from native_chrome_extension/install.sh rename to chrome/install.sh diff --git a/native_chrome_extension/mac_nativehost_file.json b/chrome/mac_nativehost_file.json similarity index 66% rename from native_chrome_extension/mac_nativehost_file.json rename to chrome/mac_nativehost_file.json index 984aff676..7a38d1d31 100644 --- a/native_chrome_extension/mac_nativehost_file.json +++ b/chrome/mac_nativehost_file.json @@ -1,7 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "/Users/aaron/Documents/GitHub/OpenAdapt/native_chrome_extension/browser.py", + "path": "/Users/aaron/Documents/GitHub/OpenAdapt/chrome/browser.py", "type": "stdio", "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] } diff --git a/native_chrome_extension/manifest.json b/chrome/manifest.json similarity index 100% rename from native_chrome_extension/manifest.json rename to chrome/manifest.json diff --git a/native_chrome_extension/mock_client.py b/chrome/mock_client.py similarity index 100% rename from native_chrome_extension/mock_client.py rename to chrome/mock_client.py diff --git a/native_chrome_extension/openadapt_chrome.json b/chrome/openadapt_chrome.json similarity index 62% rename from native_chrome_extension/openadapt_chrome.json rename to chrome/openadapt_chrome.json index 483b11aff..81db51887 100644 --- a/native_chrome_extension/openadapt_chrome.json +++ b/chrome/openadapt_chrome.json @@ -1,7 +1,7 @@ { "name": "openadapt", "description": "OpenAdapt DOM Listener", - "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.bat", + "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\chrome\\browser.bat", "type": "stdio", "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] } diff --git a/native_chrome_extension/view_messages.py b/chrome/view_messages.py similarity index 100% rename from native_chrome_extension/view_messages.py rename to chrome/view_messages.py diff --git a/native_chrome_extension/browser.bat b/native_chrome_extension/browser.bat deleted file mode 100644 index 788947cc7..000000000 --- a/native_chrome_extension/browser.bat +++ /dev/null @@ -1,3 +0,0 @@ -@echo off - -python -u "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\native_chrome_extension\\browser.py" From 59be3e492403d2bbd4ff268d134c53a30b7b33c7 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 13:21:03 -0400 Subject: [PATCH 84/89] disable socket for now --- chrome/browser.py | 10 +++++++--- chrome/mac_nativehost_file.json | 2 +- chrome/openadapt_chrome.json | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/chrome/browser.py b/chrome/browser.py index d1ef38eb4..180ee2707 100755 --- a/chrome/browser.py +++ b/chrome/browser.py @@ -17,6 +17,7 @@ from openadapt import config, sockets STORE_DATA = True +SOCKETS = False def get_message() -> dict: @@ -88,9 +89,10 @@ def main() -> None: message TEXT NOT NULL ) """) - conn = sockets.create_server_connection(config.SOCKET_PORT) + if SOCKETS: + conn = sockets.create_server_connection(config.SOCKET_PORT) while True: - if conn.closed: + if SOCKETS and conn.closed: conn = sockets.create_server_connection(config.SOCKET_PORT) message = get_message() if STORE_DATA: @@ -102,7 +104,9 @@ def main() -> None: response = {"message": "Data received and logged successfully!"} encoded_response = encode_message(response) send_message(encoded_response) - send_message_to_client(conn, message) + + if SOCKETS: + send_message_to_client(conn, message) if __name__ == "__main__": diff --git a/chrome/mac_nativehost_file.json b/chrome/mac_nativehost_file.json index 7a38d1d31..d436fc653 100644 --- a/chrome/mac_nativehost_file.json +++ b/chrome/mac_nativehost_file.json @@ -3,5 +3,5 @@ "description": "OpenAdapt DOM Listener", "path": "/Users/aaron/Documents/GitHub/OpenAdapt/chrome/browser.py", "type": "stdio", - "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] + "allowed_origins": ["chrome-extension://ghlnbigmccnehdmcnbgjnlfehfkcieff/"] } diff --git a/chrome/openadapt_chrome.json b/chrome/openadapt_chrome.json index 81db51887..008960c6b 100644 --- a/chrome/openadapt_chrome.json +++ b/chrome/openadapt_chrome.json @@ -3,5 +3,5 @@ "description": "OpenAdapt DOM Listener", "path": "P:\\OpenAdapt AI - MLDS AI\\cloned_repo\\OpenAdapt\\chrome\\browser.bat", "type": "stdio", - "allowed_origins": ["chrome-extension://nhbpabaagkpjeppjkhopppfhdjpelchg/"] + "allowed_origins": ["chrome-extension://ghlnbigmccnehdmcnbgjnlfehfkcieff/"] } From ac1ac8d23686b8538c83631c38d567bf397aa8d2 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 13:33:59 -0400 Subject: [PATCH 85/89] use last message --- chrome/export_messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chrome/export_messages.py b/chrome/export_messages.py index 3a9d850a1..f11925ac7 100644 --- a/chrome/export_messages.py +++ b/chrome/export_messages.py @@ -52,7 +52,7 @@ ) if len(messages) > 0: - name = f"messages-{json.loads(messages[0][1])['timestamp']}.html" + name = f"messages-{json.loads(messages[-1][1])['timestamp']}.html" else: name = f"messages-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}.html" From bbc4a3aa23b91f3d475006a30682c356315c09a5 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 28 Aug 2023 16:31:35 -0400 Subject: [PATCH 86/89] add elements --- chrome/content.js | 32 ++++++++++++------- .../contentMutationObserverImplementation.js | 2 +- chrome/export_messages.py | 1 + 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/chrome/content.js b/chrome/content.js index f4e11c1ed..aae0946bb 100644 --- a/chrome/content.js +++ b/chrome/content.js @@ -6,6 +6,7 @@ let logged = false; let ignoreAttributes = new Set(); +const elements = {}; /* * Function to send a message to the background script @@ -27,6 +28,7 @@ function captureDocumentState() { action: "captureDocumentState", documentBody: documentBody, documentHead: documentHead, + elements: elements, url: page_url, timestamp: Date.now(), }); @@ -35,22 +37,21 @@ function captureDocumentState() { function handleElementClick(event) { const element = event.target; const tagName = element.tagName; + const { x, y } = elements[element.id]; + const value = elements[element.id].element.value; const attributes = {}; for (const attr of element.attributes) { attributes[attr.name] = attr.value; } - const rect = element.getBoundingClientRect(); - const x = rect.left + window.scrollX; - const y = rect.top + window.scrollY; - sendMessageToBackgroundScript({ action: "elementClicked", tagName: tagName, attributes: attributes, x: x, y: y, + value: value, url: window.location.href, timestamp: Date.now(), }); @@ -71,6 +72,8 @@ function debounce(func, delay) { function handleDebouncedInput(event) { const element = event.target; + const { x, y } = elements[element.id]; + const value = elements[element.id].element.value; const tagName = element.tagName; const attributes = {}; @@ -78,11 +81,6 @@ function handleDebouncedInput(event) { attributes[attr.name] = attr.value; } - const rect = element.getBoundingClientRect(); - const x = rect.left + window.scrollX; - const y = rect.top + window.scrollY; - const value = element.value; - sendMessageToBackgroundScript({ action: "elementInput", tagName: tagName, @@ -101,12 +99,24 @@ function handleElementInput(event) { debouncedInputHandler(event); } +function addElement(element) { + const rect = element.getBoundingClientRect(); + const x = rect.left + window.scrollX; + const y = rect.top + window.scrollY; + const value = element.value; + if (!element.id) { + element.id = element.tagName + "_" + x + "_" + y; + } + elements[element.id] = { element, x, y, value }; + element.addEventListener("click", handleElementClick); + element.addEventListener("input", debounce(handleDebouncedInput, 500)); +} + function addEventListeners() { const elements = document.getElementsByTagName("*"); for (const element of elements) { - element.addEventListener("click", handleElementClick); - element.addEventListener("input", handleElementInput); + addElement(element); } } diff --git a/chrome/contentMutationObserverImplementation.js b/chrome/contentMutationObserverImplementation.js index 75e5683a9..f914988d6 100644 --- a/chrome/contentMutationObserverImplementation.js +++ b/chrome/contentMutationObserverImplementation.js @@ -48,7 +48,7 @@ const observer = new MutationObserver((mutations) => { const value = target.value; sendMessageToBackgroundScript({ - action: "elementMutation", + action: "mutation", type: type, tagName: tagName, attributes: attributes, diff --git a/chrome/export_messages.py b/chrome/export_messages.py index f11925ac7..42cdf3f77 100644 --- a/chrome/export_messages.py +++ b/chrome/export_messages.py @@ -33,6 +33,7 @@ if message_json["action"] == "captureDocumentState": message_json["documentBody"] = "..." message_json["documentHead"] = "..." + message_json["elements"] = "..." message_html = json.dumps(message_json, indent=4, ensure_ascii=False) message_html = html.escape(message_html) From c67975be32882b4de17326ea77bcd0539256ac57 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Thu, 31 Aug 2023 16:09:52 -0400 Subject: [PATCH 87/89] progress on recording + db table --- chrome/browser.py | 30 +++++++++++++------------- chrome/content.js | 4 ++-- openadapt/models.py | 24 ++++++++++----------- openadapt/record.py | 51 ++++++++++++++++++++++++++------------------- 4 files changed, 58 insertions(+), 51 deletions(-) diff --git a/chrome/browser.py b/chrome/browser.py index 180ee2707..f51c97476 100755 --- a/chrome/browser.py +++ b/chrome/browser.py @@ -16,8 +16,8 @@ from openadapt import config, sockets -STORE_DATA = True -SOCKETS = False +SOCKETS = True +DBG_DATABASE = False def get_message() -> dict: @@ -79,32 +79,32 @@ def send_message_to_client(conn: sockets.Connection, message: dict) -> None: def main() -> None: """Main function.""" - # Connect to the database - db_conn = sqlite3.connect("messages.db") - c = db_conn.cursor() - # Create the messages table if it doesn't exist - c.execute(""" - CREATE TABLE IF NOT EXISTS messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message TEXT NOT NULL - ) - """) + if DBG_DATABASE: + db_conn = sqlite3.connect("messages.db") + c = db_conn.cursor() + c.execute(""" + CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message TEXT NOT NULL + ) + """) if SOCKETS: conn = sockets.create_server_connection(config.SOCKET_PORT) while True: if SOCKETS and conn.closed: conn = sockets.create_server_connection(config.SOCKET_PORT) message = get_message() - if STORE_DATA: - # Log the message to the database + + # Log the message to the database + if DBG_DATABASE: c.execute( "INSERT INTO messages (message) VALUES (?)", (json.dumps(message),) ) db_conn.commit() response = {"message": "Data received and logged successfully!"} + if message: encoded_response = encode_message(response) send_message(encoded_response) - if SOCKETS: send_message_to_client(conn, message) diff --git a/chrome/content.js b/chrome/content.js index aae0946bb..bc25ba335 100644 --- a/chrome/content.js +++ b/chrome/content.js @@ -37,8 +37,8 @@ function captureDocumentState() { function handleElementClick(event) { const element = event.target; const tagName = element.tagName; - const { x, y } = elements[element.id]; - const value = elements[element.id].element.value; + const { x, y } = elements[element.id] || {}; + const value = elements[element.id]?.value || ""; const attributes = {}; for (const attr of element.attributes) { diff --git a/openadapt/models.py b/openadapt/models.py index b3c1b21a2..381422ff5 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -88,7 +88,7 @@ class ActionEvent(db.Base): recording_timestamp = sa.Column(sa.ForeignKey("recording.timestamp")) screenshot_timestamp = sa.Column(sa.ForeignKey("screenshot.timestamp")) window_event_timestamp = sa.Column(sa.ForeignKey("window_event.timestamp")) - browser_event_timestamp = sa.Column(sa.ForeignKey("browser_event.timestamp")) + # browser_event_timestamp = sa.Column(sa.ForeignKey("browser_event.timestamp")) mouse_x = sa.Column(sa.Numeric(asdecimal=False)) mouse_y = sa.Column(sa.Numeric(asdecimal=False)) mouse_dx = sa.Column(sa.Numeric(asdecimal=False)) @@ -117,7 +117,7 @@ class ActionEvent(db.Base): recording = sa.orm.relationship("Recording", back_populates="action_events") screenshot = sa.orm.relationship("Screenshot", back_populates="action_event") window_event = sa.orm.relationship("WindowEvent", back_populates="action_events") - browser_event = sa.orm.relationship("BrowserEvent", back_populates="action_event") + # browser_event = sa.orm.relationship("BrowserEvent", back_populates="action_event") # TODO: playback_timestamp / original_timestamp @@ -391,19 +391,19 @@ class BrowserEvent(db.Base): id = sa.Column(sa.Integer, primary_key=True) recording_timestamp = sa.Column(sa.ForeignKey("recording.timestamp")) + recording = sa.orm.relationship("Recording", back_populates="browser_events") + message = sa.Column(sa.JSON) timestamp = sa.Column(ForceFloat) - bodyHTML = sa.Column(sa.String) - headHTML = sa.Column(sa.String) - url = sa.Column(sa.String) - recording = sa.orm.relationship("Recording", back_populates="browser_events") - action_event = sa.orm.relationship("ActionEvent", back_populates="browser_event") - # TODO: implement for extension - # @classmethod - # def get_active_browser_event(cls: Any) -> Any: - # """Get the active chrome tab window's DOM""" - # return BrowserEvent(**get_active_chrome_data()) +# recording = sa.orm.relationship("Recording", back_populates="browser_events") +# action_event = sa.orm.relationship("ActionEvent", back_populates="browser_event") + +# TODO: implement for extension +# @classmethod +# def get_active_browser_event(cls: Any) -> Any: +# """Get the active chrome tab window's DOM""" +# return BrowserEvent(**get_active_chrome_data()) class PerformanceStat(db.Base): diff --git a/openadapt/record.py b/openadapt/record.py index 8344076ca..bddb2f76b 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -213,9 +213,12 @@ def process_events( continue event.data["screenshot_timestamp"] = prev_screen_event.timestamp event.data["window_event_timestamp"] = prev_window_event.timestamp - event.data["browser_event_timestamp"] = ( - prev_browser_event.timestamp if prev_browser_event is not None else None - ) + if prev_browser_event is not None: + event.data["browser_event_timestamp"] = ( + prev_browser_event.message["timestamp"] + if prev_browser_event is not None + else None + ) process_event( event, action_write_q, @@ -241,15 +244,19 @@ def process_events( perf_q, ) prev_saved_window_timestamp = prev_window_event.timestamp - if prev_saved_browser_timestamp < prev_browser_event.timestamp: - process_event( - prev_browser_event, - browser_write_q, - write_browser_event, - recording_timestamp, - perf_q, - ) - prev_saved_browser_timestamp = prev_browser_event.timestamp + if prev_browser_event is not None: + if prev_saved_browser_timestamp < prev_browser_event.msg["timestamp"]: + process_event( + prev_browser_event, + browser_write_q, + write_browser_event, + recording_timestamp, + perf_q, + ) + if prev_browser_event is not None: + prev_saved_browser_timestamp = prev_browser_event.message[ + "timestamp" + ] else: raise Exception(f"unhandled {event.type=}") del prev_event @@ -615,14 +622,16 @@ def read_browser_events( utils.set_start_time(recording_timestamp) logger.info("starting") conn = sockets.create_client_connection(config.SOCKET_PORT) - while not terminate_event.is_set() and conn is not None: + while not terminate_event.is_set(): try: - logger.info("Waiting for message...") - # msg = sockets.client_receive_message(config.SOCKET_PORT) - msg = sockets.client_receive(conn) - # logger.info(f"{msg=}") + if not conn.closed: + logger.info("Waiting for message...") + msg = conn.recv() + logger.info(f"{msg=}") + else: + conn = sockets.create_client_connection(config.SOCKET_PORT) if msg is not None: - logger.info("Received DOM message.") + logger.info("Received message.") browser_data = msg logger.debug("queuing browser event for writing") event_q.put( @@ -648,11 +657,9 @@ def read_browser_events( # except Exception as exc: # logger.warning(f"Error during communication: {exc}") # time.sleep(config.SOCKET_RETRY_INTERVAL) - if conn: - conn.close() + # if conn: + # conn.close() - pid = utils.get_pid_by_name("browser.py") - utils.send_kill_signal(pid) logger.info("done") From 475972562e7228c1498a284e4b84da81628c6c43 Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Thu, 31 Aug 2023 16:09:55 -0400 Subject: [PATCH 88/89] Create 1a19ce257672_add_browserevent_message.py --- .../1a19ce257672_add_browserevent_message.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 alembic/versions/1a19ce257672_add_browserevent_message.py diff --git a/alembic/versions/1a19ce257672_add_browserevent_message.py b/alembic/versions/1a19ce257672_add_browserevent_message.py new file mode 100644 index 000000000..9f7341ae2 --- /dev/null +++ b/alembic/versions/1a19ce257672_add_browserevent_message.py @@ -0,0 +1,49 @@ +"""add BrowserEvent.message + +Revision ID: 1a19ce257672 +Revises: 8713b142f5de +Create Date: 2023-08-28 17:01:07.703670 + +""" +import sqlalchemy as sa + +from alembic import op +import openadapt + +# revision identifiers, used by Alembic. +revision = "1a19ce257672" +down_revision = "8713b142f5de" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "browser_event", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column( + "recording_timestamp", + openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), + nullable=True, + ), + sa.Column("message", sa.JSON(), nullable=True), + sa.Column( + "timestamp", + openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["recording_timestamp"], + ["recording.timestamp"], + name=op.f("fk_browser_event_recording_timestamp_recording"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_browser_event")), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("browser_event") + # ### end Alembic commands ### From cafc053ceba5de0ce2de2984c9d36b285d8ab16c Mon Sep 17 00:00:00 2001 From: Aaron <57018940+0dm@users.noreply.github.com> Date: Mon, 4 Sep 2023 12:03:23 -0400 Subject: [PATCH 89/89] fix error in browser --- chrome/browser.py | 2 +- openadapt/record.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/chrome/browser.py b/chrome/browser.py index f51c97476..d25cc8d7e 100755 --- a/chrome/browser.py +++ b/chrome/browser.py @@ -101,7 +101,7 @@ def main() -> None: "INSERT INTO messages (message) VALUES (?)", (json.dumps(message),) ) db_conn.commit() - response = {"message": "Data received and logged successfully!"} + response = {"message": "Data received and logged successfully!"} if message: encoded_response = encode_message(response) send_message(encoded_response) diff --git a/openadapt/record.py b/openadapt/record.py index bddb2f76b..863c61ea2 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -624,12 +624,13 @@ def read_browser_events( conn = sockets.create_client_connection(config.SOCKET_PORT) while not terminate_event.is_set(): try: - if not conn.closed: + if conn.closed: + conn = sockets.create_client_connection(config.SOCKET_PORT) + else: logger.info("Waiting for message...") msg = conn.recv() logger.info(f"{msg=}") - else: - conn = sockets.create_client_connection(config.SOCKET_PORT) + if msg is not None: logger.info("Received message.") browser_data = msg @@ -643,7 +644,7 @@ def read_browser_events( ) else: logger.info("No message received or received None Type Message.") - except Exception as exc: + except EOFError as exc: logger.warning("Connection closed.") logger.warning(exc) break