Add README and dedupe data entries

3 years ago · 43e0c6c7f8
parent 73fa9a8976
commit 43e0c6c7f8
9 changed files with 160 additions and 85 deletions
--- a/README.md
+++ b/README.md
@ -0,0 +1,7 @@
+# Problematic-requests, aka ICD scanner
+
+Wtyczka pokazująca, jakie dane zostały ~~wykradzione~~ wysłane do podmiotów trzecich przez odwiedzane strony.
+
+## TODO:
+
+- Używać https://github.com/InteractiveAdvertisingBureau/iabtcf-es/tree/master/modules/core#iabtcfcore do wizualizacji "zgód" zebranych przez CMP-y od IAB
--- a/extended-request.ts
+++ b/extended-request.ts
@ -1,12 +1,19 @@
 import { StolenDataEntry } from "./request-cluster";
-import { getshorthost, Request } from "./util";
+import { getshorthost, parseCookie, Request } from "./util";

 export default class ExtendedRequest {
  public tabId: number;
  public url: string;
  public requestHeaders: Request["requestHeaders"];
+  public origin: string;
+  public initialized = false;

-  async getOrigin() {
+  async init() {
+    await this.cacheOrigin();
+    this.initialized = true;
+  }
+
+  async cacheOrigin(): Promise<void> {
    let url: string;
    if (this.data.tabId && this.data.tabId >= 0) {
      const tab = await browser.tabs.get(this.data.tabId);
@ -14,12 +21,19 @@ export default class ExtendedRequest {
    } else {
      url = (this.data as any).frameAncestors[0].url;
    }
-    return url;
+    this.origin = url;
+  }
+
+  getOrigin(): string {
+    if (!this.initialized) {
+      throw new Error("initialize first!!");
+    }
+    return this.origin;
  }

-  async isThirdParty() {
+  isThirdParty() {
    const request_url = new URL(this.data.url);
-    const origin_url = new URL(await this.getOrigin());
+    const origin_url = new URL(this.getOrigin());
    if (request_url.host.includes(origin_url.host)) {
      return false;
    }
@ -37,8 +51,33 @@ export default class ExtendedRequest {
      .value;
  }

-  async exposesOrigin() {
-    return this.getReferer().includes(new URL(await this.getOrigin()).host);
+  exposesOrigin() {
+    const url = new URL(this.getOrigin());
+    const host = url.host;
+    const path = url.pathname;
+    return (
+      this.getReferer().includes(host) ||
+      this.getAllStolenData().filter(
+        (entry) => entry.value.includes(host) || entry.value.includes(path)
+      ).length > 0
+    );
+  }
+
+  getAllStolenData(): StolenDataEntry[] {
+    return [
+      ...this.getPathParams(),
+      ...this.getCookieData(),
+      ...this.getQueryParams(),
+    ];
+  }
+
+  getCookieData(): StolenDataEntry[] {
+    if (!this.hasCookie() || this.getCookie() === undefined) {
+      return [];
+    }
+    return Object.entries(parseCookie(this.getCookie()))
+      .map(([key, value]) => [key, value || ""])
+      .map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
  }

  hasReferer() {
@ -49,7 +88,7 @@ export default class ExtendedRequest {
    return this.data.requestHeaders.some((h) => h.name === "Cookie");
  }

-  getCookie() {
+  getCookie(): string {
    return this.requestHeaders.find((h) => h.name == "Cookie")?.value;
  }

@ -62,12 +101,25 @@ export default class ExtendedRequest {
    return path
      .split(";")
      .map((e) => e.split("="))
+      .map(([key, value]) => [key, value || ""])
      .map(
        ([key, value]) =>
-          new StolenDataEntry("pathname", key, decodeURIComponent(value))
+          new StolenDataEntry(this, "pathname", key, decodeURIComponent(value))
      );
  }

+  getQueryParams(): StolenDataEntry[] {
+    const url = new URL(this.data.url);
+    return Array.from((url.searchParams as any).entries())
+      .map(([key, value]) => [key, value || ""])
+      .map(([key, value]) => {
+        try {
+          value = decodeURIComponent(value);
+        } catch (e) {}
+        return new StolenDataEntry(this, "queryparams", key, value);
+      });
+  }
+
  constructor(public data: Request) {
    this.tabId = data.tabId;
    this.url = data.url;
--- a/memory.ts
+++ b/memory.ts
@ -6,11 +6,8 @@ import { RequestCluster } from "./request-cluster";
 class Memory extends EventEmitter {
  tab_to_history = {} as Record<string, Record<string, RequestCluster>>;
  async register(request: ExtendedRequest) {
-    if (
-      (await request.isThirdParty()) &&
-      request.hasReferer() &&
-      (await request.exposesOrigin())
-    ) {
+    await request.init();
+    if (request.isThirdParty() && request.exposesOrigin()) {
      if (!this.tab_to_history[request.tabId]) {
        this.tab_to_history[request.tabId] = {};
      }
--- a/package-lock.json
+++ b/package-lock.json
@ -9,10 +9,12 @@
      "version": "1.0.0",
      "license": "ISC",
      "dependencies": {
+        "consent-string": "^1.5.2",
        "esbuild": "^0.13.3",
        "events": "^3.3.0",
        "react": "^17.0.2",
-        "react-dom": "^17.0.2"
+        "react-dom": "^17.0.2",
+        "tai-password-strength": "^1.1.3"
      },
      "devDependencies": {
        "@types/events": "^3.0.0",
@ -58,6 +60,19 @@
      "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
      "dev": true
    },
+    "node_modules/base-64": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
+      "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
+    },
+    "node_modules/consent-string": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
+      "integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
+      "dependencies": {
+        "base-64": "^0.1.0"
+      }
+    },
    "node_modules/csstype": {
      "version": "3.0.9",
      "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
@ -349,6 +364,11 @@
        "object-assign": "^4.1.1"
      }
    },
+    "node_modules/tai-password-strength": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
+      "integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
+    },
    "node_modules/web-ext-types": {
      "version": "3.2.1",
      "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
@ -395,6 +415,19 @@
      "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
      "dev": true
    },
+    "base-64": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
+      "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
+    },
+    "consent-string": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
+      "integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
+      "requires": {
+        "base-64": "^0.1.0"
+      }
+    },
    "csstype": {
      "version": "3.0.9",
      "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
@ -571,6 +604,11 @@
        "object-assign": "^4.1.1"
      }
    },
+    "tai-password-strength": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
+      "integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
+    },
    "web-ext-types": {
      "version": "3.2.1",
      "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
--- a/package.json
+++ b/package.json
@ -14,10 +14,12 @@
  "author": "",
  "license": "ISC",
  "dependencies": {
+    "consent-string": "^1.5.2",
    "esbuild": "^0.13.3",
    "events": "^3.3.0",
    "react": "^17.0.2",
-    "react-dom": "^17.0.2"
+    "react-dom": "^17.0.2",
+    "tai-password-strength": "^1.1.3"
  },
  "devDependencies": {
    "@types/events": "^3.0.0",
--- a/problematic.js
+++ b/problematic.js
@ -7,7 +7,6 @@ function gethost(url) {
 }

 function getshorthost(host) {
-  console.log("getshort", host);
  return host.split(".").slice(-2).join(".");
 }

@ -16,11 +15,6 @@ async function isThirdParty(request) {
  const request_url = new URL(request.url);
  const origin_url = new URL(await getOrigin(request));
  /* console.log(request_url.ho, origin_url, request_url.includes(origin_url)); */
-  console.log(
-    request_url.host,
-    origin_url.host,
-    request_url.host.includes(origin_url.host)
-  );
  if (request_url.host.includes(origin_url.host)) {
    return false;
  }
@ -85,11 +79,9 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
  if (sender.tab) {
    return;
  }
-  console.log("got message!", request);
  if (request?.msg === "get_memory") {
    sendResponse(memory);
  } else if (request?.msg === "clear_memory") {
-    console.log("memory cleared");
    memory = {};
  }
 });
--- a/request-cluster.ts
+++ b/request-cluster.ts
@ -1,9 +1,25 @@
 import { EventEmitter } from "events";
 import ExtendedRequest from "./extended-request";
-import { parseCookie } from "./util";
+
+export type Sources = "cookie" | "pathname" | "queryparams";

 export class StolenDataEntry {
-  constructor(public type: string, public name: string, public value: string) {}
+  constructor(
+    public request: ExtendedRequest,
+    public source: Sources,
+    public name: string,
+    public value: string
+  ) {}
+
+  getPriority() {
+    let priority = 0;
+    priority += this.value.length;
+    const url = new URL(this.request.getOrigin());
+    if (this.value.includes(url.host) || this.value.includes(url.pathname)) {
+      priority += 100;
+    }
+    return priority;
+  }
 }

 export class RequestCluster extends EventEmitter {
@ -25,56 +41,28 @@ export class RequestCluster extends EventEmitter {
    return false;
  }

-  getCookiesContent({
-    minValueLength,
-  }: {
-    minValueLength: number;
-  }): StolenDataEntry[] {
-    this.getQueryParamsContent({ minValueLength });
-    const cookieValues = new Set<string>();
-    for (const request of this.requests) {
-      if (request.hasCookie()) {
-        cookieValues.add(request.getCookie());
-      }
-    }
-    return Array.from(cookieValues.values())
-      .map(parseCookie)
-      .map((o) => Object.entries(o))
+  getStolenData(filter: { minValueLength: number }): StolenDataEntry[] {
+    return this.requests
+      .map((request) => request.getAllStolenData())
      .reduce((a, b) => a.concat(b), [])
-      .map(([key, value]) => new StolenDataEntry("cookie", key, value))
-      .filter((e) => e.value.length >= minValueLength);
-  }
-
-  getQueryParamsContent({
-    minValueLength,
-  }: {
-    minValueLength: number;
-  }): StolenDataEntry[] {
-    const result = [];
-    for (const request of this.requests) {
-      console.log(request.data.url);
-    }
-    return result;
-  }
-
-  getPathnameParamsContent({
-    minValueLength,
-  }: {
-    minValueLength: number;
-  }): StolenDataEntry[] {
-    let result = [];
-    for (const request of this.requests) {
-      result = [...result, ...request.getPathParams()];
-    }
-    console.log("PATHNAME PARAMS FOR", this.id, result);
-    return result;
-  }
-
-  getStolenData(filter: { minValueLength: number }) {
-    return [
-      ...this.getCookiesContent(filter),
-      ...this.getPathnameParamsContent(filter),
-    ];
+      .filter((entry) => {
+        return entry.value.length >= filter.minValueLength;
+      })
+      .sort((entry1, entry2) =>
+        entry1.getPriority() > entry2.getPriority() ? -1 : 1
+      )
+      .filter((element, index, array) => {
+        // remove duplicate neighbours
+        if (index == 0) {
+          return true;
+        }
+        if (
+          element.name != array[index - 1].name ||
+          element.value != array[index - 1].value
+        ) {
+          return true;
+        }
+      });
  }

  static sortCompare(a: RequestCluster, b: RequestCluster) {
--- a/sidebar.tsx
+++ b/sidebar.tsx
@ -1,7 +1,7 @@
 import React, { useEffect, useState } from "react";
 import ReactDOM from "react-dom";
 import memory from "./memory";
-import { RequestCluster } from "./request-cluster";
+import { RequestCluster, Sources } from "./request-cluster";
 import { Tab, useEmitter } from "./util";

 async function getTabByID(id: number) {
@ -10,7 +10,6 @@ async function getTabByID(id: number) {
 }

 async function getCurrentTab() {
-  console.log("getCurrentTab");
  const [tab] = await browser.tabs.query({
    active: true,
    windowId: browser.windows.WINDOW_ID_CURRENT,
@ -27,7 +26,6 @@ const TabDropdown = ({
 }) => {
  const [tabs, setTabs] = useState([]);
  useEffect(() => {
-    console.log("useEffect!");
    browser.tabs.query({ currentWindow: true }).then(setTabs);
  }, []);
  return (
@ -35,7 +33,6 @@ const TabDropdown = ({
      id="tab_dropdown"
      value={pickedTab}
      onChange={async (e) => {
-        console.log(e.target.value);
        setPickedTab(parseInt(e.target.value));
      }}
    >
@ -60,6 +57,11 @@ const StolenDataRow = ({
  minValueLength: number;
 }) => {
  const cluster = memory.getClustersForTab(tabID)[shorthost];
+  const icons: Record<Sources, string> = {
+    cookie: "🍪",
+    pathname: "🛣",
+    queryparams: "🅿",
+  };
  return (
    <div>
      <h2>
@ -73,7 +75,8 @@ const StolenDataRow = ({
              <th style={{ maxWidth: "200px", wordWrap: "break-word" }}>
                {entry.name}
              </th>
-              <td>{entry.value}</td>
+              <td>{icons[entry.source]}</td>
+              <td style={{ wordWrap: "anywhere" as any }}>{entry.value}</td>
            </tr>
          ))}
        </tbody>
@ -140,9 +143,8 @@ const Options = ({ minValueLength, setMinValueLength }) => {
 };

 const Sidebar = () => {
-  console.log("rendering!");
  const [pickedTab, setPickedTab] = useState<number | null>(null);
-  const [minValueLength, setMinValueLength] = useState<number | null>(5);
+  const [minValueLength, setMinValueLength] = useState<number | null>(7);
  const counter = useEmitter(memory);
  return (
    <>
--- a/util.ts
+++ b/util.ts
@ -16,11 +16,8 @@ export function getshorthost(host: string) {
 export function useEmitter(e: EventEmitter) {
  const [counter, setCounter] = useState<number>(0);
  useEffect(() => {
-    console.log("useEmitter!");
    const callback = () => {
-      console.log("Detected memory change!");
      setCounter((counter) => counter + 1);
-      console.log("RT:", counter + 1);
    };
    e.on("change", callback);
    return () => {