feat(scripts): sync pre-commit linter versions with conda in update.py

.pre-commit-config.yaml

@@ -1,15 +1,15 @@
 repos:
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.13.2
+    rev: 8.0.1
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 24.10.0
+    rev: 26.1.0
     hooks:
       - id: black
         language_version: python3
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.19.1
     hooks:
       - id: mypy
         additional_dependencies:

custom_command.py

@@ -1,4 +1,4 @@
-""" This file aims to demonstrate how to write custom commands in OpenWPM
+"""This file aims to demonstrate how to write custom commands in OpenWPM
 
 Steps to have a custom command run as part of a CommandSequence
 

docs/Release-Checklist.md

@@ -4,10 +4,11 @@ We aim to release a new version of OpenWPM with each new Firefox release (~1 rel
 
 1. Run `python scripts/update.py` — this will:
     - Repin the conda environment
+    - Sync linter versions in `.pre-commit-config.yaml` to match `environment.yaml` (black, isort, mypy)
     - Bump all npm dependencies to their latest compatible versions
     - Rebuild the extension
     - Automatically detect and apply any new Firefox release to `scripts/install-firefox.sh`
-2. Check if we can unpin our python version (this requires Linux to test) See <https://github.com/openwpm/OpenWPM/issues/1111>
+2. Verify no linter drift warnings were printed by `update.py`. If any `WARNING` lines mention a linter that could not be synced, resolve the mismatch manually.
 3. Run `./scripts/install-firefox.sh` to install the updated Firefox locally and verify the test suite passes.
 4. Increment the version number in [VERSION](../VERSION)
 5. Add a summary of changes since the last version to [CHANGELOG](../CHANGELOG.md)

openwpm/browser_manager.py

@@ -156,7 +156,7 @@ def check_queue(launch_status: Dict[str, bool]) -> Any:
                 "BROWSER %i: Spawn attempt %i " % (self.browser_id, unsuccessful_spawns)
             )
             # Resets the command/status queues
-            (self.command_queue, self.status_queue) = (Queue(), Queue())
+            self.command_queue, self.status_queue = (Queue(), Queue())
 
             # builds and launches the browser_manager
 

openwpm/deploy_browsers/configure_firefox.py

@@ -1,4 +1,4 @@
-""" Set prefs and load extensions in Firefox """
+"""Set prefs and load extensions in Firefox"""
 
 from selenium.webdriver.firefox.options import Options
 

openwpm/deploy_browsers/deploy_firefox.py

@@ -88,12 +88,10 @@ def deploy_firefox(
             display.start()
             display_pid, display_port = display.pid, display.display
         except EasyProcessError:
-            raise RuntimeError(
-                "Xvfb could not be started. \
+            raise RuntimeError("Xvfb could not be started. \
                 Please ensure it's on your path. \
                 See www.X.org for full details. \
-                Commonly solved on ubuntu with `sudo apt install xvfb`"
-            )
+                Commonly solved on ubuntu with `sudo apt install xvfb`")
     # Must do this for all display modes,
     # because status_queue is read off no matter what.
     status_queue.put(("STATUS", "Display", (display_pid, display_port)))

openwpm/errors.py

@@ -1,4 +1,4 @@
-""" OpenWPM Custom Errors """
+"""OpenWPM Custom Errors"""
 
 
 class CommandExecutionError(Exception):

openwpm/js_instrumentation.py

@@ -32,12 +32,10 @@ def _validate(python_list_to_validate):
             propertiesToInstrument = set(propertiesToInstrument)
             excludedProperties = set(setting["logSettings"]["excludedProperties"])
             if len(propertiesToInstrument.intersection(excludedProperties)) != 0:
-                raise ValueError(
-                    f"excludedProperties and \
+                raise ValueError(f"excludedProperties and \
                     propertiesToInstrument collide. This \
                     may have occurred after a merge. \
-                    Setting with collision: {setting}."
-                )
+                    Setting with collision: {setting}.")
     return True
 
 
@@ -127,11 +125,9 @@ def _build_full_settings_object(setting):
 
     elif isinstance(setting, dict):
         if len(setting.keys()) != 1:
-            raise ValueError(
-                f"Invalid settings request. \
+            raise ValueError(f"Invalid settings request. \
                 Settings item is a dictionary with more \
-                than one key. Received {setting}."
-            )
+                than one key. Received {setting}.")
 
         setting_key = list(setting.keys())[0]
         props = setting[setting_key]
@@ -142,18 +138,14 @@ def _build_full_settings_object(setting):
             for k, v in props.items():
                 logSettings[k] = v
         else:
-            raise ValueError(
-                f"Invalid settings request. \
+            raise ValueError(f"Invalid settings request. \
                 Setting was a dictionary. Settings value \
                 must be a list or a dictionary. Received \
-                {setting}."
-            )
+                {setting}.")
     else:
-        raise ValueError(
-            f"Invalid settings request. \
+        raise ValueError(f"Invalid settings request. \
             Must be a string or a dictionary. \
-            Received {setting}."
-        )
+            Received {setting}.")
 
     return {
         "object": obj,
@@ -228,10 +220,8 @@ def clean_js_instrumentation_settings(
 
     """
     if not isinstance(user_requested_settings, list):
-        raise TypeError(
-            f"js_instrumentation_settings must be a list. \
-              Received {user_requested_settings}"
-        )
+        raise TypeError(f"js_instrumentation_settings must be a list. \
+              Received {user_requested_settings}")
     settings = []
     for setting in user_requested_settings:
         if isinstance(setting, str) and (setting in shortcut_specs):

openwpm/socket_interface.py

@@ -41,7 +41,7 @@ def _accept(self):
         """Listen for connections and pass handling to a new thread"""
         while True:
             try:
-                (client, address) = self.sock.accept()
+                client, address = self.sock.accept()
                 thread = threading.Thread(
                     target=self._handle_conn, args=(client, address)
                 )

openwpm/storage/storage_controller.py

@@ -122,20 +122,16 @@ async def handler(
             record_type, data = record
 
             if record_type == RECORD_TYPE_CREATE:
-                raise RuntimeError(
-                    f"""{RECORD_TYPE_CREATE} is no longer supported.
+                raise RuntimeError(f"""{RECORD_TYPE_CREATE} is no longer supported.
                     Please change the schema before starting the StorageController.
                     For an example of that see test/test_custom_function.py
-                    """
-                )
+                    """)
 
             if record_type == RECORD_TYPE_CONTENT:
                 assert len(data) == 2
                 if self.unstructured_storage is None:
-                    self.logger.error(
-                        """Tried to save content while not having
-                        provided any unstructured storage provider."""
-                    )
+                    self.logger.error("""Tried to save content while not having
+                        provided any unstructured storage provider.""")
                     continue
                 content, content_hash = data
                 content = base64.b64decode(content)

openwpm/utilities/build_cookie_table.py

@@ -153,17 +153,14 @@ def build_http_cookie_table(database, verbose=False):
     cur1 = con.cursor()
     cur2 = con.cursor()
 
-    cur1.execute(
-        "CREATE TABLE IF NOT EXISTS http_request_cookies ( \
+    cur1.execute("CREATE TABLE IF NOT EXISTS http_request_cookies ( \
                     id INTEGER PRIMARY KEY AUTOINCREMENT, \
                     browser_id INTEGER NOT NULL, \
                     header_id INTEGER NOT NULL, \
                     name VARCHAR(200) NOT NULL, \
                     value TEXT NOT NULL, \
-                    accessed DATETIME);"
-    )
-    cur1.execute(
-        "CREATE TABLE IF NOT EXISTS http_response_cookies ( \
+                    accessed DATETIME);")
+    cur1.execute("CREATE TABLE IF NOT EXISTS http_response_cookies ( \
                     id INTEGER PRIMARY KEY AUTOINCREMENT, \
                     browser_id INTEGER NOT NULL, \
                     header_id INTEGER NOT NULL, \
@@ -177,20 +174,17 @@ def build_http_cookie_table(database, verbose=False):
                     secure BOOLEAN, \
                     comment VARCHAR(200), \
                     version VARCHAR(100), \
-                    accessed DATETIME);"
-    )
+                    accessed DATETIME);")
     con.commit()
 
     # Parse http request cookies
     commit = 0
     last_commit = 0
 
-    cur1.execute(
-        """SELECT id, browser_id, headers, time_stamp
+    cur1.execute("""SELECT id, browser_id, headers, time_stamp
                     FROM http_requests
                     WHERE id NOT IN (SELECT header_id FROM
-                    http_request_cookies)"""
-    )
+                    http_request_cookies)""")
 
     row = cur1.fetchone()
     if row is None:
@@ -225,12 +219,10 @@ def build_http_cookie_table(database, verbose=False):
     # Parse http response cookies
     commit = 0
     last_commit = 0
-    cur1.execute(
-        """SELECT id, browser_id, url, headers, time_stamp
+    cur1.execute("""SELECT id, browser_id, url, headers, time_stamp
                     FROM http_responses
                     WHERE id NOT IN (SELECT header_id
-                    FROM http_response_cookies)"""
-    )
+                    FROM http_response_cookies)""")
 
     row = cur1.fetchone()
     if row is None:

openwpm/utilities/cookie.py

@@ -207,7 +207,6 @@
 Finis.
 """
 
-
 # Import our required modules
 #
 
@@ -641,9 +640,7 @@ def js_output(self, attrs=None):
         document.cookie = \"%s\";
         // end hiding -->
         </script>
-        """ % (
-            self.OutputString(attrs).replace('"', r"\""),
-        )
+        """ % (self.OutputString(attrs).replace('"', r"\""),)
 
     # end js_output()
 

scripts/update.py

@@ -23,6 +23,14 @@
 
 _MAX_RESOLVE_ATTEMPTS = 10
 
+# Mapping: conda package name → (pre-commit repo URL, rev prefix)
+# The prefix is prepended to the conda version to form the pre-commit rev tag.
+_LINTER_MAP: dict[str, tuple[str, str]] = {
+    "black": ("https://github.com/psf/black", ""),
+    "isort": ("https://github.com/timothycrosley/isort", ""),
+    "mypy": ("https://github.com/pre-commit/mirrors-mypy", "v"),
+}
+
 
 def run(*cmd: str, cwd: Path = ROOT) -> None:
     print(f"+ {' '.join(cmd)}")
@@ -33,6 +41,80 @@ def conda_run(*cmd: str, cwd: Path = ROOT) -> None:
     run("conda", "run", "-n", "openwpm", *cmd, cwd=cwd)
 
 
+def _parse_conda_versions(env_yaml: Path) -> dict[str, str]:
+    """Extract version-pinned package versions from environment.yaml.
+
+    Parses lines like ``- black=26.1.0`` and returns ``{"black": "26.1.0"}``.
+    Skips pip-style ``==`` pins and non-versioned entries.
+    """
+    versions: dict[str, str] = {}
+    for line in env_yaml.read_text().splitlines():
+        stripped = line.strip()
+        if not stripped.startswith("- ") or stripped.startswith("- pip"):
+            continue
+        spec = stripped[2:]
+        if "==" in spec or "=" not in spec:
+            continue
+        name, _, version = spec.partition("=")
+        versions[name] = version
+    return versions
+
+
+def sync_precommit_linter_versions() -> None:
+    """Sync linter revs in .pre-commit-config.yaml to match environment.yaml."""
+    env_yaml = ROOT / "environment.yaml"
+    precommit_yaml = ROOT / ".pre-commit-config.yaml"
+
+    print("\n=== Syncing pre-commit linter versions with conda ===")
+
+    conda_versions = _parse_conda_versions(env_yaml)
+    content = precommit_yaml.read_text()
+
+    updated = False
+    for pkg, (repo_url, prefix) in _LINTER_MAP.items():
+        if pkg not in conda_versions:
+            print(
+                f"WARNING: {pkg} not found in environment.yaml, skipping",
+                file=sys.stderr,
+            )
+            continue
+
+        target_rev = f"{prefix}{conda_versions[pkg]}"
+
+        # Match the repo URL line followed by the rev line, preserving whitespace.
+        pattern = re.compile(
+            rf"(- repo: {re.escape(repo_url)}\s*\n\s*rev:\s*)\S+",
+        )
+        match = pattern.search(content)
+        if not match:
+            print(
+                f"WARNING: repo {repo_url} ({pkg}) not found in "
+                f".pre-commit-config.yaml",
+                file=sys.stderr,
+            )
+            continue
+
+        current_rev = content[match.start(0) + len(match.group(1)) : match.end(0)]
+        if current_rev == target_rev:
+            print(f"  {pkg}: already in sync ({current_rev})")
+            continue
+
+        print(f"  {pkg}: {current_rev} -> {target_rev}")
+        content = (
+            content[: match.start(0)]
+            + match.group(1)
+            + target_rev
+            + content[match.end(0) :]
+        )
+        updated = True
+
+    if updated:
+        precommit_yaml.write_text(content)
+        print("Updated .pre-commit-config.yaml")
+    else:
+        print("All linter versions already in sync.")
+
+
 def _npm_install(cwd: Path) -> tuple[str, bool]:
     """Run npm install and return (combined output, success).
 
@@ -150,6 +232,9 @@ def main() -> None:
     # Repin the conda environment from unpinned sources
     run("./repin.sh", cwd=SCRIPTS)
 
+    # Sync pre-commit linter versions to match the freshly pinned conda env
+    sync_precommit_linter_versions()
+
     # Bump npm deps to latest and resolve peer dep conflicts
     npm_bump_and_resolve(ROOT)
     npm_bump_and_resolve(ROOT / "Extension")

test/extension/test_logging.py

@@ -4,9 +4,7 @@ def test_extension_logging(task_manager_creator, default_params, tmp_path):
     manager_params, browser_params = default_params
     manager_params.log_path = log_path
     for browser_param in browser_params:
-        browser_param.custom_params[
-            "pre_instrumentation_code"
-        ] = f"""
+        browser_param.custom_params["pre_instrumentation_code"] = f"""
             // Weird name needed due to webpack name mangling
             loggingDB.logWarn("{test_msg}"); 
         """

test/extension/test_startup_timeout.py

@@ -7,9 +7,7 @@ def test_extension_startup_timeout(task_manager_creator, default_params):
     manager_params, browser_params = default_params
     manager_params.num_browsers = 1
     for browser_param in browser_params:
-        browser_param.custom_params[
-            "pre_instrumentation_code"
-        ] = """
+        browser_param.custom_params["pre_instrumentation_code"] = """
            const startTime = Date.now();
            while (Date.now() - startTime < 20000) { // Delaying for 20s
             console.log("delaying startup");

test/storage/test_values.py

@@ -1,4 +1,4 @@
-""" This file should contain one entry for every table
+"""This file should contain one entry for every table
 so that we can test storing and loading for every single entry
 for every structured storage provider.