Add esmify script

This seems to produce decent output on many of our files, but it totally
misses some import statements. Not sure why that is yet.

scripts/esmify/.gitignore

@@ -0,0 +1 @@
+__pycache__/

scripts/esmify/README.txt

@@ -0,0 +1,12 @@
+This is a modified version of the esmify tool from the fx115 tree:
+  https://searchfox.org/mozilla-esr115/source/tools/esmify
+
+Zotero modifications:
+  - Make esmify.py work independently of Mach
+  - Remove ESLint step
+  - Fix VCS integration for our structure (and remove Mercurial)
+  - Replace references to nonexistent "pathlib.os"
+  - Add handling for Components.utils.import(), not just Cu.import()
+
+Run:
+  scripts/esmify/esmify.py [dir]

scripts/esmify/esmify.py

@@ -0,0 +1,803 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import logging
+import os
+import pathlib
+import re
+import subprocess
+import sys
+import argparse
+
+
+def path_sep_to_native(path_str):
+    """Make separators in the path OS native."""
+    return os.sep.join(path_str.split("/"))
+
+
+def path_sep_from_native(path):
+    """Make separators in the path OS native."""
+    return "/".join(str(path).split(os.sep))
+
+
+excluded_from_convert_prefix = list(
+    map(
+        path_sep_to_native,
+        [
+            # Testcases for actors.
+            "toolkit/actors/TestProcessActorChild.jsm",
+            "toolkit/actors/TestProcessActorParent.jsm",
+            "toolkit/actors/TestWindowChild.jsm",
+            "toolkit/actors/TestWindowParent.jsm",
+            "js/xpconnect/tests/unit/",
+            # Testcase for build system.
+            "python/mozbuild/mozbuild/test/",
+        ],
+    )
+)
+
+
+def is_excluded_from_convert(path):
+    """Returns true if the JSM file shouldn't be converted to ESM."""
+    path_str = str(path)
+    for prefix in excluded_from_convert_prefix:
+        if path_str.startswith(prefix):
+            return True
+
+    return False
+
+
+excluded_from_imports_prefix = list(
+    map(
+        path_sep_to_native,
+        [
+            # Vendored or auto-generated files.
+            "browser/components/pocket/content/panels/js/vendor.bundle.js",
+            "devtools/client/debugger/dist/parser-worker.js",
+            "devtools/client/debugger/test/mochitest/examples/react/build/main.js",
+            "devtools/client/debugger/test/mochitest/examples/sourcemapped/polyfill-bundle.js",
+            "devtools/client/inspector/markup/test/shadowdom_open_debugger.min.js",
+            "devtools/client/shared/source-map-loader/test/browser/fixtures/bundle.js",
+            "layout/style/test/property_database.js",
+            "services/fxaccounts/FxAccountsPairingChannel.js",
+            "testing/talos/talos/tests/devtools/addon/content/pages/custom/debugger/static/js/main.js",  # noqa E501
+            "testing/web-platform/",
+            # Unrelated testcases that has edge case syntax.
+            "browser/components/sessionstore/test/unit/data/",
+            "devtools/client/debugger/src/workers/parser/tests/fixtures/",
+            "devtools/client/debugger/test/mochitest/examples/sourcemapped/fixtures/",
+            "devtools/client/webconsole/test/browser/test-syntaxerror-worklet.js",
+            "devtools/server/tests/xpcshell/test_framebindings-03.js",
+            "devtools/server/tests/xpcshell/test_framebindings-04.js",
+            "devtools/shared/tests/xpcshell/test_eventemitter_basic.js",
+            "devtools/shared/tests/xpcshell/test_eventemitter_static.js",
+            "dom/base/crashtests/module-with-syntax-error.js",
+            "dom/base/test/file_bug687859-16.js",
+            "dom/base/test/file_bug687859-16.js",
+            "dom/base/test/file_js_cache_syntax_error.js",
+            "dom/base/test/jsmodules/module_badSyntax.js",
+            "dom/canvas/test/reftest/webgl-utils.js",
+            "dom/encoding/test/file_utf16_be_bom.js",
+            "dom/encoding/test/file_utf16_le_bom.js",
+            "dom/html/test/bug649134/file_bug649134-1.sjs",
+            "dom/html/test/bug649134/file_bug649134-2.sjs",
+            "dom/media/webrtc/tests/mochitests/identity/idp-bad.js",
+            "dom/serviceworkers/test/file_js_cache_syntax_error.js",
+            "dom/serviceworkers/test/parse_error_worker.js",
+            "dom/workers/test/importScripts_worker_imported3.js",
+            "dom/workers/test/invalid.js",
+            "dom/workers/test/threadErrors_worker1.js",
+            "dom/xhr/tests/browser_blobFromFile.js",
+            "image/test/browser/browser_image.js",
+            "js/xpconnect/tests/chrome/test_bug732665_meta.js",
+            "js/xpconnect/tests/mochitest/class_static_worker.js",
+            "js/xpconnect/tests/unit/bug451678_subscript.js",
+            "js/xpconnect/tests/unit/error_other.sys.mjs",
+            "js/xpconnect/tests/unit/es6module_parse_error.js",
+            "js/xpconnect/tests/unit/recursive_importA.jsm",
+            "js/xpconnect/tests/unit/recursive_importB.jsm",
+            "js/xpconnect/tests/unit/syntax_error.jsm",
+            "js/xpconnect/tests/unit/test_defineModuleGetter.js",
+            "js/xpconnect/tests/unit/test_import.js",
+            "js/xpconnect/tests/unit/test_import_shim.js",
+            "js/xpconnect/tests/unit/test_recursive_import.js",
+            "js/xpconnect/tests/unit/test_unload.js",
+            "modules/libpref/test/unit/data/testParser.js",
+            "python/mozbuild/mozbuild/test/",
+            "remote/shared/messagehandler/test/browser/resources/modules/root/invalid.sys.mjs",
+            "testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js",
+            "testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js",
+            "toolkit/components/reader/Readerable.sys.mjs",
+            "toolkit/components/workerloader/tests/moduleF-syntax-error.js",
+            "tools/lint/test/",
+            "tools/update-packaging/test/",
+            # SpiderMonkey internals.
+            "js/examples/",
+            "js/src/",
+            # Files has macro.
+            "browser/app/profile/firefox.js",
+            "browser/branding/official/pref/firefox-branding.js",
+            "browser/components/enterprisepolicies/schemas/schema.sys.mjs",
+            "browser/locales/en-US/firefox-l10n.js",
+            "mobile/android/app/geckoview-prefs.js",
+            "mobile/android/app/mobile.js",
+            "mobile/android/locales/en-US/mobile-l10n.js",
+            "modules/libpref/greprefs.js",
+            "modules/libpref/init/all.js",
+            "testing/condprofile/condprof/tests/profile/user.js",
+            "testing/mozbase/mozprofile/tests/files/prefs_with_comments.js",
+            "toolkit/modules/AppConstants.sys.mjs",
+            "toolkit/mozapps/update/tests/data/xpcshellConstantsPP.js",
+            # Uniffi templates
+            "toolkit/components/uniffi-bindgen-gecko-js/src/templates/js/",
+            "node_modules/",
+            "resource/tinymce/",
+            "resource/react",
+        ],
+    )
+)
+
+EXCLUSION_FILES = []
+
+
+def load_exclusion_files():
+    for path in EXCLUSION_FILES:
+        with open(path, "r") as f:
+            for line in f:
+                p = path_sep_to_native(re.sub("\*$", "", line.strip()))
+                excluded_from_imports_prefix.append(p)
+
+
+def is_excluded_from_imports(path):
+    """Returns true if the JS file content shouldn't be handled by
+    jscodeshift.
+
+    This filter is necessary because jscodeshift cannot handle some
+    syntax edge cases and results in unexpected rewrite."""
+    path_str = str(path)
+    for prefix in excluded_from_imports_prefix:
+        if path_str.startswith(prefix):
+            return True
+
+    return False
+
+
+# Wrapper for hg/git operations
+class VCSUtils:
+    def run(self, cmd):
+        # Do not pass check=True because the pattern can match no file.
+        lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode()
+        return filter(lambda x: x != "", lines.split("\n"))
+
+
+class GitUtils(VCSUtils):
+    def is_available():
+        return pathlib.Path(".git").exists()
+
+    def rename(self, before, after):
+        cmd = ["git", "mv", before, after]
+        subprocess.run(cmd, check=True)
+
+    def find_jsms(self, path):
+        jsms = []
+
+        cmd = ["git", "ls-files", f"{path}/*.jsm"]
+        for line in self.run(cmd):
+            jsm = pathlib.Path(line)
+            if is_excluded_from_convert(jsm):
+                continue
+            jsms.append(jsm)
+
+        handled = {}
+        cmd = ["git", "grep", "EXPORTED_SYMBOLS = \[", f"{path}/*.js"]
+        for line in self.run(cmd):
+            m = re.search("^([^:]+):", line)
+            if not m:
+                continue
+            filename = m.group(1)
+            if filename in handled:
+                continue
+            handled[filename] = True
+            jsm = pathlib.Path(filename)
+            if is_excluded_from_convert(jsm):
+                continue
+            jsms.append(jsm)
+
+        return jsms
+
+    def find_all_jss(self, path):
+        jss = []
+
+        cmd = [
+            "git",
+            "ls-files",
+            f"{path}/*.jsm",
+            f"{path}/*.js",
+            f"{path}/*.mjs",
+            f"{path}/*.sjs",
+        ]
+        for line in self.run(cmd):
+            js = pathlib.Path(line)
+            if is_excluded_from_imports(js):
+                continue
+            jss.append(js)
+
+        return jss
+
+
+class Summary:
+    def __init__(self):
+        self.convert_errors = []
+        self.import_errors = []
+        self.rename_errors = []
+        self.no_refs = []
+
+
+def esmify(path=None, convert=False, imports=False, prefix=""):
+    class CommandContext:
+        def log(self, level, tag, _, message):
+            print(f"{tag}: {message}")
+    command_context = CommandContext()
+    
+    """
+    This command does the following 2 steps:
+      1. Convert the JSM file specified by `path` to ESM file, or the JSM files
+         inside the directory specified by `path` to ESM files, and also
+         fix references in build files and test definitions
+      2. Convert import calls inside file(s) specified by `path` for ESM-ified
+         files to use new APIs
+
+    Example 1:
+      # Convert all JSM files inside `browser/components/pagedata` directory,
+      # and replace all references for ESM-ified files in the entire tree to use
+      # new APIs
+
+      $ ./mach esmify --convert browser/components/pagedata
+      $ ./mach esmify --imports . --prefix=browser/components/pagedata
+
+    Example 2:
+      # Convert all JSM files inside `browser` directory, and replace all
+      # references for the JSM files inside `browser` directory to use
+      # new APIs
+
+      $ ./mach esmify browser
+    """
+
+    def error(text):
+        command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}")
+
+    def warn(text):
+        command_context.log(logging.WARN, "esmify", {}, f"[WARN] {text}")
+
+    def info(text):
+        command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+    # If no options is specified, perform both.
+    if not convert and not imports:
+        convert = True
+        imports = True
+
+    path = pathlib.Path(path[0])
+
+    if not verify_path(command_context, path):
+        return 1
+
+    if GitUtils.is_available():
+        vcs_utils = GitUtils()
+    else:
+        error(
+            "This script needs to be run inside mozilla-central "
+            "checkout of either mercurial or git."
+        )
+        return 1
+
+    load_exclusion_files()
+
+    info("Setting up jscodeshift...")
+    setup_jscodeshift()
+
+    is_single_file = path.is_file()
+
+    summary = Summary()
+
+    if convert:
+        info("Searching files to convert to ESM...")
+        if is_single_file:
+            jsms = [path]
+        else:
+            jsms = vcs_utils.find_jsms(path)
+
+        info(f"Found {len(jsms)} file(s) to convert to ESM.")
+
+        info("Converting to ESM...")
+        jsms = convert_module(jsms, summary)
+        if jsms is None:
+            error("Failed to rewrite exports.")
+            return 1
+
+        info("Renaming...")
+        esms = rename_jsms(command_context, vcs_utils, jsms, summary)
+
+    if imports:
+        info("Searching files to rewrite imports...")
+
+        if is_single_file:
+            if convert:
+                # Already converted above
+                jss = esms
+            else:
+                jss = [path]
+        else:
+            jss = vcs_utils.find_all_jss(path)
+
+        info(f"Checking {len(jss)} JS file(s). Rewriting any matching imports...")
+
+        result = rewrite_imports(jss, prefix, summary)
+        if result is None:
+            return 1
+
+        info(f"Rewritten {len(result)} file(s).")
+
+    def print_files(f, errors):
+        for [path, message] in errors:
+            f(f"  * {path}")
+            if message:
+                f(f"    {message}")
+
+    if len(summary.convert_errors):
+        error("========")
+        error("Following files are not converted into ESM due to error:")
+        print_files(error, summary.convert_errors)
+
+    if len(summary.import_errors):
+        warn("========")
+        warn("Following files are not rewritten to import ESMs due to error:")
+        warn(
+            "(NOTE: Errors related to 'private names' are mostly due to "
+            " preprocessor macros in the file):"
+        )
+        print_files(warn, summary.import_errors)
+
+    if len(summary.rename_errors):
+        error("========")
+        error("Following files are not renamed due to error:")
+        print_files(error, summary.rename_errors)
+
+    if len(summary.no_refs):
+        warn("========")
+        warn("Following files are not found in any build files.")
+        warn("Please update references to those files manually:")
+        print_files(warn, summary.rename_errors)
+
+    return 0
+
+
+def verify_path(command_context, path):
+    """Check if the path passed to the command is valid relative path."""
+
+    def error(text):
+        command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}")
+
+    if not path.exists():
+        error(f"{path} does not exist.")
+        return False
+
+    if path.is_absolute():
+        error("Path must be a relative path from mozilla-central checkout.")
+        return False
+
+    return True
+
+
+def find_file(path, target):
+    """Find `target` file in ancestor of path."""
+    target_path = path.parent / target
+    if not target_path.exists():
+        if path.parent == path:
+            return None
+
+        return find_file(path.parent, target)
+
+    return target_path
+
+
+def try_rename_in(command_context, path, target, jsm_name, esm_name, jsm_path):
+    """Replace the occurrences of `jsm_name` with `esm_name` in `target`
+    file."""
+
+    def info(text):
+        command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+    if type(target) is str:
+        # Target is specified by filename, that may exist somewhere in
+        # the jsm's directory or ancestor directories.
+        target_path = find_file(path, target)
+        if not target_path:
+            return False
+
+        # JSM should be specified with relative path in the file.
+        #
+        # Single moz.build or jar.mn can contain multiple files with same name.
+        # Search for relative path.
+        jsm_relative_path = jsm_path.relative_to(target_path.parent)
+        jsm_path_str = path_sep_from_native(str(jsm_relative_path))
+    else:
+        # Target is specified by full path.
+        target_path = target
+
+        # JSM should be specified with full path in the file.
+        jsm_path_str = path_sep_from_native(str(jsm_path))
+
+    jsm_path_re = re.compile(r"\b" + jsm_path_str.replace(".", r"\.") + r"\b")
+    jsm_name_re = re.compile(r"\b" + jsm_name.replace(".", r"\.") + r"\b")
+
+    modified = False
+    content = ""
+    with open(target_path, "r") as f:
+        for line in f:
+            if jsm_path_re.search(line):
+                modified = True
+                line = jsm_name_re.sub(esm_name, line)
+
+            content += line
+
+    if modified:
+        info(f"  {str(target_path)}")
+        info(f"    {jsm_name} => {esm_name}")
+        with open(target_path, "w", newline="\n") as f:
+            f.write(content)
+
+    return True
+
+
+def try_rename_uri_in(command_context, target, jsm_name, esm_name, jsm_uri, esm_uri):
+    """Replace the occurrences of `jsm_uri` with `esm_uri` in `target` file."""
+
+    def info(text):
+        command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+    modified = False
+    content = ""
+    with open(target, "r") as f:
+        for line in f:
+            if jsm_uri in line:
+                modified = True
+                line = line.replace(jsm_uri, esm_uri)
+
+            content += line
+
+    if modified:
+        info(f"  {str(target)}")
+        info(f"    {jsm_name} => {esm_name}")
+        with open(target, "w", newline="\n") as f:
+            f.write(content)
+
+    return True
+
+
+def try_rename_components_conf(command_context, path, jsm_name, esm_name):
+    """Replace the occurrences of `jsm_name` with `esm_name` in components.conf
+    file."""
+
+    def info(text):
+        command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+    target_path = find_file(path, "components.conf")
+    if not target_path:
+        return False
+
+    # Unlike try_rename_in, components.conf contains the URL instead of
+    # relative path, and also there are no known files with same name.
+    # Simply replace the filename.
+
+    with open(target_path, "r") as f:
+        content = f.read()
+
+    prop_re = re.compile(
+        "[\"']jsm[\"']:(.*)" + r"\b" + jsm_name.replace(".", r"\.") + r"\b"
+    )
+
+    if not prop_re.search(content):
+        return False
+
+    info(f"  {str(target_path)}")
+    info(f"    {jsm_name} => {esm_name}")
+
+    content = prop_re.sub(r"'esModule':\1" + esm_name, content)
+    with open(target_path, "w", newline="\n") as f:
+        f.write(content)
+
+    return True
+
+
+def esmify_name(name):
+    return re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", name)
+
+
+def esmify_path(jsm_path):
+    jsm_name = jsm_path.name
+    esm_name = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_name)
+    esm_path = jsm_path.parent / esm_name
+    return esm_path
+
+
+path_to_uri_map = None
+
+
+def load_path_to_uri_map():
+    global path_to_uri_map
+
+    if path_to_uri_map:
+        return
+
+    if "ESMIFY_MAP_JSON" in os.environ:
+        json_map = pathlib.Path(os.environ["ESMIFY_MAP_JSON"])
+    else:
+        json_map = pathlib.Path(__file__).parent / "map.json"
+
+    with open(json_map, "r") as f:
+        uri_to_path_map = json.loads(f.read())
+
+    path_to_uri_map = dict()
+
+    for uri, paths in uri_to_path_map.items():
+        if type(paths) is str:
+            paths = [paths]
+
+        for path in paths:
+            path_to_uri_map[path] = uri
+
+
+def find_jsm_uri(jsm_path):
+    load_path_to_uri_map()
+
+    path = path_sep_from_native(jsm_path)
+
+    if path in path_to_uri_map:
+        return path_to_uri_map[path]
+
+    return None
+
+
+def rename_single_file(command_context, vcs_utils, jsm_path, summary):
+    """Rename `jsm_path` to .sys.mjs, and fix references to the file in build
+    and test definitions."""
+
+    def info(text):
+        command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+    esm_path = esmify_path(jsm_path)
+
+    jsm_name = jsm_path.name
+    esm_name = esm_path.name
+
+    target_files = []
+
+    info(f"{jsm_path} => {esm_path}")
+
+    renamed = False
+    for target in target_files:
+        if try_rename_in(
+            command_context, jsm_path, target, jsm_name, esm_name, jsm_path
+        ):
+            renamed = True
+
+    if try_rename_components_conf(command_context, jsm_path, jsm_name, esm_name):
+        renamed = True
+
+    uri_target_files = [
+        pathlib.Path(
+            "browser", "base", "content", "test", "performance", "browser_startup.js"
+        ),
+        pathlib.Path(
+            "browser",
+            "base",
+            "content",
+            "test",
+            "performance",
+            "browser_startup_content.js",
+        ),
+        pathlib.Path(
+            "browser",
+            "base",
+            "content",
+            "test",
+            "performance",
+            "browser_startup_content_subframe.js",
+        ),
+        pathlib.Path(
+            "toolkit",
+            "components",
+            "backgroundtasks",
+            "tests",
+            "browser",
+            "browser_xpcom_graph_wait.js",
+        ),
+    ]
+
+    jsm_uri = find_jsm_uri(jsm_path)
+    if jsm_uri:
+        esm_uri = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_uri)
+
+        for target in uri_target_files:
+            if try_rename_uri_in(
+                command_context, target, jsm_uri, esm_uri, jsm_name, esm_name
+            ):
+                renamed = True
+
+    if not renamed:
+        summary.no_refs.append([jsm_path, None])
+
+    if not esm_path.exists():
+        vcs_utils.rename(jsm_path, esm_path)
+    else:
+        summary.rename_errors.append([jsm_path, f"{esm_path} already exists"])
+
+    return esm_path
+
+
+def rename_jsms(command_context, vcs_utils, jsms, summary):
+    esms = []
+    for jsm in jsms:
+        esm = rename_single_file(command_context, vcs_utils, jsm, summary)
+        esms.append(esm)
+
+    return esms
+
+
+npm_prefix = pathlib.Path("scripts") / "esmify"
+path_from_npm_prefix = pathlib.Path("..") / ".."
+
+
+def setup_jscodeshift():
+    """Install jscodeshift."""
+    cmd = [
+        "npm",
+        "install",
+        "jscodeshift",
+        "--save-dev",
+        "--prefix",
+        str(npm_prefix),
+    ]
+    subprocess.run(cmd, check=True)
+
+
+def run_npm_command(args, env, stdin):
+    cmd = [
+        "npm",
+        "run",
+    ] + args
+    p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    p.stdin.write(stdin)
+    p.stdin.close()
+
+    ok_files = []
+    errors = []
+    while True:
+        line = p.stdout.readline()
+        if not line:
+            break
+        line = line.rstrip().decode()
+
+        if line.startswith(" NOC "):
+            continue
+
+        print(line)
+
+        m = re.search(r"^ (OKK|ERR) ([^ ]+)(?: (.+))?", line)
+        if not m:
+            continue
+
+        result = m.group(1)
+        # NOTE: path is written from `tools/esmify`.
+        path = pathlib.Path(m.group(2)).relative_to(path_from_npm_prefix)
+        error = m.group(3)
+
+        if result == "OKK":
+            ok_files.append(path)
+
+        if result == "ERR":
+            errors.append([path, error])
+
+    if p.wait() != 0:
+        return [None, None]
+
+    return ok_files, errors
+
+
+def convert_module(jsms, summary):
+    """Replace EXPORTED_SYMBOLS with export declarations, and replace
+    ChromeUtils.importESModule with static import as much as possible,
+    and return the list of successfully rewritten files."""
+
+    if len(jsms) == 0:
+        return []
+
+    env = os.environ.copy()
+
+    stdin = "\n".join(map(str, paths_from_npm_prefix(jsms))).encode()
+
+    ok_files, errors = run_npm_command(
+        [
+            "convert_module",
+            "--prefix",
+            str(npm_prefix),
+        ],
+        env=env,
+        stdin=stdin,
+    )
+
+    if ok_files is None and errors is None:
+        return None
+
+    summary.convert_errors.extend(errors)
+
+    return ok_files
+
+
+def rewrite_imports(jss, prefix, summary):
+    """Replace import calls for JSM with import calls for ESM or static import
+    for ESM."""
+
+    if len(jss) == 0:
+        return []
+
+    env = os.environ.copy()
+    env["ESMIFY_TARGET_PREFIX"] = prefix
+
+    stdin = "\n".join(map(str, paths_from_npm_prefix(jss))).encode()
+
+    ok_files, errors = run_npm_command(
+        [
+            "rewrite_imports",
+            "--prefix",
+            str(npm_prefix),
+        ],
+        env=env,
+        stdin=stdin,
+    )
+
+    if ok_files is None and errors is None:
+        return None
+
+    summary.import_errors.extend(errors)
+
+    return ok_files
+
+
+def paths_from_npm_prefix(paths):
+    """Convert relative path from mozilla-central to relative path from
+    tools/esmify."""
+    return list(map(lambda path: path_from_npm_prefix / path, paths))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "path",
+        nargs=1,
+        help="Path to the JSM file to ESMify, or the directory that contains "
+             "JSM files and/or JS files that imports ESM-ified JSM.",
+    )
+    parser.add_argument(
+        "--convert",
+        action="store_true",
+        help="Only perform the step 1 = convert part",
+    )
+    parser.add_argument(
+        "--imports",
+        action="store_true",
+        help="Only perform the step 2 = import calls part",
+    )
+    parser.add_argument(
+        "--prefix",
+        default="",
+        help="Restrict the target of import in the step 2 to ESM-ified JSM, by the "
+             "prefix match for the JSM file's path.  e.g. 'browser/'.",
+    )
+    args = parser.parse_args()
+    esmify(path=args.path, convert=args.convert, imports=args.imports, prefix=args.prefix)

scripts/esmify/import-to-import_esmodule.js

@@ -0,0 +1,480 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// jscodeshift rule to replace import calls for JSM with import calls for ESM
+// or static import for ESM.
+
+/* eslint-env node */
+
+const _path = require("path");
+const { isESMified } = require(_path.resolve(__dirname, "./is-esmified.js"));
+const {
+  jsmExtPattern,
+  esmifyExtension,
+  isIdentifier,
+  isString,
+  warnForPath,
+  getPrevStatement,
+  getNextStatement,
+  isMemberExpressionWithIdentifiers,
+  rewriteMemberExpressionWithIdentifiers,
+  createMemberExpressionWithIdentifiers,
+} = require(_path.resolve(__dirname, "./utils.js"));
+const {
+  isImportESModuleCall,
+  replaceImportESModuleCall,
+  tryReplacingWithStaticImport,
+} = require(_path.resolve(__dirname, "./static-import.js"));
+
+module.exports = function (fileInfo, api) {
+  const { jscodeshift } = api;
+  const root = jscodeshift(fileInfo.source);
+  doTranslate(fileInfo.path, jscodeshift, root);
+  return root.toSource({ lineTerminator: "\n" });
+};
+
+module.exports.doTranslate = doTranslate;
+
+function isESMifiedAndTarget(resourceURI) {
+  const files = [];
+  if (!isESMified(resourceURI, files)) {
+    return false;
+  }
+
+  if ("ESMIFY_TARGET_PREFIX" in process.env) {
+    const targetPrefix = process.env.ESMIFY_TARGET_PREFIX;
+    for (const esm of files) {
+      if (esm.startsWith(targetPrefix)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
+const importCalls = [
+  {
+    from: ["Cu", "import"],
+    to: ["ChromeUtils", "importESModule"],
+  },
+  {
+    from: ["Components", "utils", "import"],
+    to: ["ChromeUtils", "importESModule"],
+  },
+  {
+    from: ["ChromeUtils", "import"],
+    to: ["ChromeUtils", "importESModule"],
+  },
+  {
+    from: ["SpecialPowers", "ChromeUtils", "import"],
+    to: ["SpecialPowers", "ChromeUtils", "importESModule"],
+  },
+];
+
+const singleLazyGetterCalls = [
+  {
+    from: ["XPCOMUtils", "defineLazyModuleGetter"],
+    to: ["ChromeUtils", "defineESModuleGetters"],
+  },
+  {
+    from: ["ChromeUtils", "defineModuleGetter"],
+    to: ["ChromeUtils", "defineESModuleGetters"],
+  },
+  {
+    from: ["SpecialPowers", "ChromeUtils", "defineModuleGetter"],
+    to: ["SpecialPowers", "ChromeUtils", "defineESModuleGetters"],
+  },
+];
+
+const multiLazyGettersCalls = [
+  {
+    from: ["XPCOMUtils", "defineLazyModuleGetters"],
+    to: ["ChromeUtils", "defineESModuleGetters"],
+  },
+];
+
+function isMemberExpressionMatchingPatterns(node, patterns) {
+  for (const item of patterns) {
+    if (isMemberExpressionWithIdentifiers(node, item.from)) {
+      return item;
+    }
+  }
+
+  return null;
+}
+
+function replaceImportCall(inputFile, jscodeshift, path, rewriteItem) {
+  if (path.node.arguments.length !== 1) {
+    warnForPath(inputFile, path, `import call should have only one argument`);
+    return;
+  }
+
+  const resourceURINode = path.node.arguments[0];
+  if (!isString(resourceURINode)) {
+    warnForPath(inputFile, path, `resource URI should be a string`);
+    return;
+  }
+
+  const resourceURI = resourceURINode.value;
+  if (!resourceURI.match(jsmExtPattern)) {
+    warnForPath(inputFile, path, `Non-jsm: ${resourceURI}`);
+    return;
+  }
+
+  if (!isESMifiedAndTarget(resourceURI)) {
+    return;
+  }
+
+  if (
+    !tryReplacingWithStaticImport(
+      jscodeshift,
+      inputFile,
+      path,
+      resourceURINode,
+      false
+    )
+  ) {
+    rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to);
+    resourceURINode.value = esmifyExtension(resourceURI);
+  }
+}
+
+// Find `ChromeUtils.defineESModuleGetters` or variant statement specified by
+// expectedIDs, adjacent to `path` which uses the same target object.
+function findDefineESModuleGettersStmt(path, expectedIDs) {
+  // `path` must be top-level.
+  if (path.parent.node.type !== "ExpressionStatement") {
+    return null;
+  }
+
+  if (path.parent.parent.node.type !== "Program") {
+    return null;
+  }
+
+  // Get previous or next statement with ChromeUtils.defineESModuleGetters.
+  let callStmt;
+  const prev = getPrevStatement(path.parent);
+  if (
+    prev &&
+    prev.type === "ExpressionStatement" &&
+    prev.expression.type === "CallExpression" &&
+    isMemberExpressionWithIdentifiers(prev.expression.callee, expectedIDs)
+  ) {
+    callStmt = prev;
+  } else {
+    const next = getNextStatement(path.parent);
+    if (
+      next &&
+      next.type === "ExpressionStatement" &&
+      next.expression.type === "CallExpression" &&
+      isMemberExpressionWithIdentifiers(next.expression.callee, expectedIDs)
+    ) {
+      callStmt = next;
+    } else {
+      return null;
+    }
+  }
+
+  const call = callStmt.expression;
+
+  if (call.arguments.length !== 2) {
+    return null;
+  }
+
+  const modulesNode = call.arguments[1];
+  if (modulesNode.type !== "ObjectExpression") {
+    return null;
+  }
+
+  // Check if the target object is same.
+  if (
+    path.node.arguments[0].type === "ThisExpression" &&
+    call.arguments[0].type === "ThisExpression"
+  ) {
+    return callStmt;
+  }
+
+  if (
+    path.node.arguments[0].type === "Identifier" &&
+    call.arguments[0].type === "Identifier" &&
+    path.node.arguments[0].name === call.arguments[0].name
+  ) {
+    return callStmt;
+  }
+
+  return null;
+}
+
+function getPropKeyString(prop) {
+  if (prop.key.type === "Identifier") {
+    return prop.key.name;
+  }
+
+  if (prop.key.type === "Literal") {
+    return prop.key.value.toString();
+  }
+
+  return "";
+}
+
+function sortProps(obj) {
+  obj.properties.sort((a, b) => {
+    return getPropKeyString(a) < getPropKeyString(b) ? -1 : 1;
+  });
+}
+
+// Move comments above `nodeFrom` before `nodeTo`.
+function moveComments(nodeTo, nodeFrom) {
+  if (!nodeFrom.comments) {
+    return;
+  }
+  if (nodeTo.comments) {
+    nodeTo.comments = [...nodeTo.comments, ...nodeFrom.comments];
+  } else {
+    nodeTo.comments = nodeFrom.comments;
+  }
+  nodeFrom.comments = [];
+}
+
+function replaceLazyGetterCall(inputFile, jscodeshift, path, rewriteItem) {
+  if (path.node.arguments.length !== 3) {
+    warnForPath(inputFile, path, `lazy getter call should have 3 arguments`);
+    return;
+  }
+
+  const nameNode = path.node.arguments[1];
+  if (!isString(nameNode)) {
+    warnForPath(inputFile, path, `name should be a string`);
+    return;
+  }
+
+  const resourceURINode = path.node.arguments[2];
+  if (!isString(resourceURINode)) {
+    warnForPath(inputFile, path, `resource URI should be a string`);
+    return;
+  }
+
+  const resourceURI = resourceURINode.value;
+  if (!resourceURI.match(jsmExtPattern)) {
+    warnForPath(inputFile, path, `Non-js/jsm: ${resourceURI}`);
+    return;
+  }
+
+  if (!isESMifiedAndTarget(resourceURI)) {
+    return;
+  }
+
+  resourceURINode.value = esmifyExtension(resourceURI);
+  const prop = jscodeshift.property(
+    "init",
+    jscodeshift.identifier(nameNode.value),
+    resourceURINode
+  );
+
+  const callStmt = findDefineESModuleGettersStmt(path, rewriteItem.to);
+  if (callStmt) {
+    // Move a property to existing ChromeUtils.defineESModuleGetters call.
+
+    moveComments(callStmt, path.parent.node);
+    path.parent.prune();
+
+    callStmt.expression.arguments[1].properties.push(prop);
+    sortProps(callStmt.expression.arguments[1]);
+  } else {
+    // Convert this call into ChromeUtils.defineESModuleGetters.
+
+    rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to);
+    path.node.arguments = [
+      path.node.arguments[0],
+      jscodeshift.objectExpression([prop]),
+    ];
+  }
+}
+
+function replaceLazyGettersCall(inputFile, jscodeshift, path, rewriteItem) {
+  if (path.node.arguments.length !== 2) {
+    warnForPath(inputFile, path, `lazy getters call should have 2 arguments`);
+    return;
+  }
+
+  const modulesNode = path.node.arguments[1];
+  if (modulesNode.type !== "ObjectExpression") {
+    warnForPath(inputFile, path, `modules parameter should be an object`);
+    return;
+  }
+
+  const esmProps = [];
+  const jsmProps = [];
+
+  for (const prop of modulesNode.properties) {
+    const resourceURINode = prop.value;
+    if (!isString(resourceURINode)) {
+      warnForPath(inputFile, path, `resource URI should be a string`);
+      jsmProps.push(prop);
+      continue;
+    }
+
+    const resourceURI = resourceURINode.value;
+    if (!resourceURI.match(jsmExtPattern)) {
+      warnForPath(inputFile, path, `Non-js/jsm: ${resourceURI}`);
+      jsmProps.push(prop);
+      continue;
+    }
+
+    if (!isESMifiedAndTarget(resourceURI)) {
+      jsmProps.push(prop);
+      continue;
+    }
+
+    esmProps.push(prop);
+  }
+
+  if (esmProps.length === 0) {
+    return;
+  }
+
+  let callStmt = findDefineESModuleGettersStmt(path, rewriteItem.to);
+  if (jsmProps.length === 0) {
+    if (callStmt) {
+      // Move all properties to existing ChromeUtils.defineESModuleGetters call.
+
+      moveComments(callStmt, path.parent.node);
+      path.parent.prune();
+
+      for (const prop of esmProps) {
+        const resourceURINode = prop.value;
+        resourceURINode.value = esmifyExtension(resourceURINode.value);
+        callStmt.expression.arguments[1].properties.push(prop);
+      }
+      sortProps(callStmt.expression.arguments[1]);
+    } else {
+      // Convert this call into ChromeUtils.defineESModuleGetters.
+
+      rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to);
+      for (const prop of esmProps) {
+        const resourceURINode = prop.value;
+        resourceURINode.value = esmifyExtension(resourceURINode.value);
+      }
+    }
+  } else {
+    // Move some properties to ChromeUtils.defineESModuleGetters.
+
+    if (path.parent.node.type !== "ExpressionStatement") {
+      warnForPath(inputFile, path, `lazy getters call in unexpected context`);
+      return;
+    }
+
+    if (!callStmt) {
+      callStmt = jscodeshift.expressionStatement(
+        jscodeshift.callExpression(
+          createMemberExpressionWithIdentifiers(jscodeshift, rewriteItem.to),
+          [path.node.arguments[0], jscodeshift.objectExpression([])]
+        )
+      );
+      path.parent.insertBefore(callStmt);
+    }
+
+    moveComments(callStmt, path.parent.node);
+
+    for (const prop of esmProps) {
+      const resourceURINode = prop.value;
+      resourceURINode.value = esmifyExtension(resourceURINode.value);
+      callStmt.expression.arguments[1].properties.push(prop);
+    }
+    sortProps(callStmt.expression.arguments[1]);
+
+    path.node.arguments[1].properties = jsmProps;
+  }
+}
+
+function getProp(obj, key) {
+  if (obj.type !== "ObjectExpression") {
+    return null;
+  }
+
+  for (const prop of obj.properties) {
+    if (prop.computed) {
+      continue;
+    }
+
+    if (!prop.key) {
+      continue;
+    }
+
+    if (isIdentifier(prop.key, key)) {
+      return prop;
+    }
+  }
+
+  return null;
+}
+
+function tryReplaceActorDefinition(inputFile, path, name) {
+  const obj = path.node;
+
+  const prop = getProp(obj, name);
+  if (!prop) {
+    return;
+  }
+
+  const moduleURIProp = getProp(prop.value, "moduleURI");
+  if (!moduleURIProp) {
+    return;
+  }
+
+  if (!isString(moduleURIProp.value)) {
+    warnForPath(inputFile, path, `${name} moduleURI should be a string`);
+    return;
+  }
+
+  const moduleURI = moduleURIProp.value.value;
+  if (!moduleURI.match(jsmExtPattern)) {
+    warnForPath(inputFile, path, `${name} Non-js/jsm: ${moduleURI}`);
+    return;
+  }
+
+  if (!isESMifiedAndTarget(moduleURI)) {
+    return;
+  }
+
+  moduleURIProp.key.name = "esModuleURI";
+  moduleURIProp.value.value = esmifyExtension(moduleURI);
+}
+
+function doTranslate(inputFile, jscodeshift, root) {
+  root.find(jscodeshift.CallExpression).forEach(path => {
+    if (isImportESModuleCall(path.node)) {
+      replaceImportESModuleCall(inputFile, jscodeshift, path, false);
+      return;
+    }
+
+    const callee = path.node.callee;
+
+    let item;
+    item = isMemberExpressionMatchingPatterns(callee, importCalls);
+    if (item) {
+      replaceImportCall(inputFile, jscodeshift, path, item);
+      return;
+    }
+
+    item = isMemberExpressionMatchingPatterns(callee, singleLazyGetterCalls);
+    if (item) {
+      replaceLazyGetterCall(inputFile, jscodeshift, path, item);
+      return;
+    }
+
+    item = isMemberExpressionMatchingPatterns(callee, multiLazyGettersCalls);
+    if (item) {
+      replaceLazyGettersCall(inputFile, jscodeshift, path, item);
+    }
+  });
+
+  root.find(jscodeshift.ObjectExpression).forEach(path => {
+    tryReplaceActorDefinition(inputFile, path, "parent");
+    tryReplaceActorDefinition(inputFile, path, "child");
+  });
+}

scripts/esmify/is-esmified.js

@@ -0,0 +1,81 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// A utility to check if given JSM is already ESM-ified.
+
+/* eslint-env node */
+
+const fs = require("fs");
+const _path = require("path");
+const { esmifyExtension } = require(_path.resolve(__dirname, "./utils.js"));
+
+let json_map;
+if (process.env.ESMIFY_MAP_JSON) {
+  json_map = _path.resolve(process.env.ESMIFY_MAP_JSON);
+} else {
+  json_map = _path.resolve(__dirname, "./map.json");
+}
+const uri_map = JSON.parse(fs.readFileSync(json_map));
+const esm_uri_map = generateESMURIMap(uri_map);
+
+function generateESMURIMap(jsm_map) {
+  const esm_map = {};
+
+  for (let [uri, jsms] of Object.entries(jsm_map)) {
+    if (typeof jsms === "string") {
+      jsms = [jsms];
+    }
+    esm_map[esmifyExtension(uri)] = jsms.map(esmifyExtension);
+  }
+
+  return esm_map;
+}
+
+function isESMifiedSlow(resourceURI) {
+  if (!(resourceURI in uri_map)) {
+    console.log(`WARNING: Unknown module: ${resourceURI}`);
+    return { result: false, jsms: [] };
+  }
+
+  let jsms = uri_map[resourceURI];
+  if (typeof jsms === "string") {
+    jsms = [jsms];
+  }
+
+  const prefix = "../../";
+  for (const jsm of jsms) {
+    if (fs.existsSync(prefix + jsm)) {
+      return { result: false, jsms };
+    }
+    const esm = esmifyExtension(jsm);
+    if (!fs.existsSync(prefix + esm)) {
+      return { result: false, jsms };
+    }
+  }
+
+  return { result: true, jsms };
+}
+
+const isESMified_memo = {};
+function isESMified(resourceURI, files) {
+  if (!(resourceURI in isESMified_memo)) {
+    isESMified_memo[resourceURI] = isESMifiedSlow(resourceURI);
+  }
+
+  for (const jsm of isESMified_memo[resourceURI].jsms) {
+    files.push(esmifyExtension(jsm));
+  }
+
+  return isESMified_memo[resourceURI].result;
+}
+
+function getESMFiles(resourceURI) {
+  if (resourceURI in esm_uri_map) {
+    return esm_uri_map[resourceURI];
+  }
+  return [];
+}
+
+exports.isESMified = isESMified;
+exports.getESMFiles = getESMFiles;

scripts/esmify/package.json

@@ -0,0 +1,9 @@
+{
+  "devDependencies": {
+    "jscodeshift": "^0.13.1"
+  },
+  "scripts": {
+    "convert_module": "jscodeshift -t use-import-export-declarations.js --stdin --verbose=2",
+    "rewrite_imports": "jscodeshift -t import-to-import_esmodule.js --stdin --verbose=2"
+  }
+}

scripts/esmify/static-import.js

@@ -0,0 +1,147 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+const _path = require("path");
+const { getESMFiles } = require(_path.resolve(__dirname, "./is-esmified.js"));
+const {
+  esmifyExtension,
+  isString,
+  warnForPath,
+  isMemberExpressionWithIdentifiers,
+} = require(_path.resolve(__dirname, "./utils.js"));
+
+function isTargetESM(resourceURI) {
+  if ("ESMIFY_TARGET_PREFIX" in process.env) {
+    const files = getESMFiles(resourceURI);
+    const targetPrefix = process.env.ESMIFY_TARGET_PREFIX;
+    for (const esm of files) {
+      if (esm.startsWith(targetPrefix)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
+function isImportESModuleCall(node) {
+  return isMemberExpressionWithIdentifiers(node.callee, [
+    "ChromeUtils",
+    "importESModule",
+  ]);
+}
+
+// Replace `ChromeUtils.import`, `Cu.import`, and `ChromeUtils.importESModule`
+// with static import if it's at the top-level of system ESM file.
+function tryReplacingWithStaticImport(
+  jscodeshift,
+  inputFile,
+  path,
+  resourceURINode,
+  alwaysReplace
+) {
+  if (!alwaysReplace && !inputFile.endsWith(".mjs")) {
+    // Static import is available only in system ESM.
+    return false;
+  }
+
+  // Check if it's at the top-level.
+  if (path.parent.node.type !== "VariableDeclarator") {
+    return false;
+  }
+
+  if (path.parent.parent.node.type !== "VariableDeclaration") {
+    return false;
+  }
+
+  const decls = path.parent.parent.node;
+  if (decls.declarations.length !== 1) {
+    return false;
+  }
+
+  if (path.parent.parent.parent.node.type !== "Program") {
+    return false;
+  }
+
+  if (path.node.arguments.length !== 1) {
+    return false;
+  }
+
+  const resourceURI = resourceURINode.value;
+
+  // Collect imported symbols.
+  const specs = [];
+  if (path.parent.node.id.type === "Identifier") {
+    specs.push(jscodeshift.importNamespaceSpecifier(path.parent.node.id));
+  } else if (path.parent.node.id.type === "ObjectPattern") {
+    for (const prop of path.parent.node.id.properties) {
+      if (prop.shorthand) {
+        specs.push(jscodeshift.importSpecifier(prop.key));
+      } else if (prop.value.type === "Identifier") {
+        specs.push(jscodeshift.importSpecifier(prop.key, prop.value));
+      } else {
+        return false;
+      }
+    }
+  } else {
+    return false;
+  }
+
+  // If this is `ChromeUtils.import` or `Cu.import`, replace the extension.
+  // no-op for `ChromeUtils.importESModule`.
+  resourceURINode.value = esmifyExtension(resourceURI);
+
+  const e = jscodeshift.importDeclaration(specs, resourceURINode);
+  e.comments = path.parent.parent.node.comments;
+  path.parent.parent.node.comments = [];
+  path.parent.parent.replace(e);
+
+  return true;
+}
+
+function replaceImportESModuleCall(
+  inputFile,
+  jscodeshift,
+  path,
+  alwaysReplace
+) {
+  if (path.node.arguments.length !== 1) {
+    warnForPath(
+      inputFile,
+      path,
+      `importESModule call should have only one argument`
+    );
+    return;
+  }
+
+  const resourceURINode = path.node.arguments[0];
+  if (!isString(resourceURINode)) {
+    warnForPath(inputFile, path, `resource URI should be a string`);
+    return;
+  }
+
+  if (!alwaysReplace) {
+    const resourceURI = resourceURINode.value;
+    if (!isTargetESM(resourceURI)) {
+      return;
+    }
+  }
+
+  // If this cannot be replaced with static import, do nothing.
+  tryReplacingWithStaticImport(
+    jscodeshift,
+    inputFile,
+    path,
+    resourceURINode,
+    alwaysReplace
+  );
+}
+
+exports.isImportESModuleCall = isImportESModuleCall;
+exports.tryReplacingWithStaticImport = tryReplacingWithStaticImport;
+exports.replaceImportESModuleCall = replaceImportESModuleCall;

scripts/esmify/use-import-export-declarations.js

@@ -0,0 +1,208 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// jscodeshift rule to replace EXPORTED_SYMBOLS with export declarations,
+// and also convert existing ChromeUtils.importESModule to static import.
+
+/* eslint-env node */
+
+const _path = require("path");
+const {
+  warnForPath,
+  getPrevStatement,
+  getNextStatement,
+} = require(_path.resolve(__dirname, "./utils.js"));
+const {
+  isImportESModuleCall,
+  replaceImportESModuleCall,
+} = require(_path.resolve(__dirname, "./static-import.js"));
+
+module.exports = function (fileInfo, api) {
+  const { jscodeshift } = api;
+  const root = jscodeshift(fileInfo.source);
+  doTranslate(fileInfo.path, jscodeshift, root);
+  return root.toSource({ lineTerminator: "\n" });
+};
+
+module.exports.doTranslate = doTranslate;
+
+// Move the comment for `path.node` to adjacent statement, keeping the position
+// as much as possible.
+function moveComments(inputFile, path) {
+  const next = getNextStatement(path);
+  if (next) {
+    if (next.comments) {
+      next.comments = [...path.node.comments, ...next.comments];
+    } else {
+      next.comments = path.node.comments;
+    }
+    path.node.comments = [];
+
+    return;
+  }
+
+  const prev = getPrevStatement(path);
+  if (prev) {
+    path.node.comments.forEach(c => {
+      c.leading = false;
+      c.trailing = true;
+    });
+
+    if (prev.comments) {
+      prev.comments = [...prev.comments, ...path.node.comments];
+    } else {
+      prev.comments = path.node.comments;
+    }
+    path.node.comments = [];
+
+    return;
+  }
+
+  warnForPath(
+    inputFile,
+    path,
+    `EXPORTED_SYMBOLS has comments and it cannot be preserved`
+  );
+}
+
+function collectAndRemoveExportedSymbols(inputFile, root) {
+  const nodes = root.findVariableDeclarators("EXPORTED_SYMBOLS");
+  if (!nodes.length) {
+    throw Error(`EXPORTED_SYMBOLS not found`);
+  }
+
+  let path = nodes.get(0);
+  const obj = nodes.get(0).node.init;
+  if (!obj) {
+    throw Error(`EXPORTED_SYMBOLS is not statically known`);
+  }
+
+  if (path.parent.node.declarations.length !== 1) {
+    throw Error(`EXPORTED_SYMBOLS shouldn't be declared with other variables`);
+  }
+
+  if (path.parent.node.comments && path.parent.node.comments.length) {
+    moveComments(inputFile, path.parent);
+  }
+
+  path.parent.prune();
+
+  const EXPORTED_SYMBOLS = new Set();
+  if (obj.type !== "ArrayExpression") {
+    throw Error(`EXPORTED_SYMBOLS is not statically known`);
+  }
+
+  for (const elem of obj.elements) {
+    if (elem.type !== "Literal") {
+      throw Error(`EXPORTED_SYMBOLS is not statically known`);
+    }
+    var name = elem.value;
+    if (typeof name !== "string") {
+      throw Error(`EXPORTED_SYMBOLS item must be a string`);
+    }
+    EXPORTED_SYMBOLS.add(name);
+  }
+
+  return EXPORTED_SYMBOLS;
+}
+
+function isTopLevel(path) {
+  return path.parent.node.type === "Program";
+}
+
+function convertToExport(jscodeshift, path, name) {
+  const e = jscodeshift.exportNamedDeclaration(path.node);
+  e.comments = [];
+  e.comments = path.node.comments;
+  path.node.comments = [];
+
+  path.replace(e);
+}
+
+function doTranslate(inputFile, jscodeshift, root) {
+  const EXPORTED_SYMBOLS = collectAndRemoveExportedSymbols(inputFile, root);
+
+  root.find(jscodeshift.FunctionDeclaration).forEach(path => {
+    if (!isTopLevel(path)) {
+      return;
+    }
+    const name = path.node.id.name;
+    if (!EXPORTED_SYMBOLS.has(name)) {
+      return;
+    }
+    EXPORTED_SYMBOLS.delete(name);
+    convertToExport(jscodeshift, path, name);
+  });
+
+  root.find(jscodeshift.ClassDeclaration).forEach(path => {
+    if (!isTopLevel(path)) {
+      return;
+    }
+    const name = path.node.id.name;
+    if (!EXPORTED_SYMBOLS.has(name)) {
+      return;
+    }
+    EXPORTED_SYMBOLS.delete(name);
+    convertToExport(jscodeshift, path, name);
+  });
+
+  root.find(jscodeshift.VariableDeclaration).forEach(path => {
+    if (!isTopLevel(path)) {
+      return;
+    }
+
+    let exists = false;
+    let name;
+    for (const decl of path.node.declarations) {
+      if (decl.id.type === "Identifier") {
+        name = decl.id.name;
+        if (EXPORTED_SYMBOLS.has(name)) {
+          exists = true;
+          break;
+        }
+      }
+
+      if (decl.id.type === "ObjectPattern") {
+        if (decl.id.properties.length === 1) {
+          const prop = decl.id.properties[0];
+          if (prop.shorthand) {
+            if (prop.key.type === "Identifier") {
+              name = prop.key.name;
+              if (EXPORTED_SYMBOLS.has(name)) {
+                exists = true;
+                break;
+              }
+            }
+          }
+        }
+      }
+    }
+    if (!exists) {
+      return;
+    }
+
+    if (path.node.declarations.length !== 1) {
+      throw Error(
+        `exported variable shouldn't be declared with other variables`
+      );
+    }
+
+    EXPORTED_SYMBOLS.delete(name);
+    convertToExport(jscodeshift, path, name);
+  });
+
+  if (EXPORTED_SYMBOLS.size !== 0) {
+    throw Error(
+      `exported symbols ${[...EXPORTED_SYMBOLS].join(", ")} not found`
+    );
+  }
+
+  root.find(jscodeshift.CallExpression).forEach(path => {
+    if (isImportESModuleCall(path.node)) {
+      // This file is not yet renamed. Skip the extension check.
+      // Also skip the isTargetESM.
+      replaceImportESModuleCall(inputFile, jscodeshift, path, true);
+    }
+  });
+}

scripts/esmify/utils.js

@@ -0,0 +1,165 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Shared utility functions.
+
+/* eslint-env node */
+
+function warnForPath(inputFile, path, message) {
+  const loc = path.node.loc;
+  console.log(
+    `WARNING: ${inputFile}:${loc.start.line}:${loc.start.column} : ${message}`
+  );
+}
+
+// Get the previous statement of `path.node` in `Program`.
+function getPrevStatement(path) {
+  const parent = path.parent;
+  if (parent.node.type !== "Program") {
+    return null;
+  }
+
+  const index = parent.node.body.findIndex(n => n == path.node);
+  if (index === -1) {
+    return null;
+  }
+
+  if (index === 0) {
+    return null;
+  }
+
+  return parent.node.body[index - 1];
+}
+
+// Get the next statement of `path.node` in `Program`.
+function getNextStatement(path) {
+  const parent = path.parent;
+  if (parent.node.type !== "Program") {
+    return null;
+  }
+
+  const index = parent.node.body.findIndex(n => n == path.node);
+  if (index === -1) {
+    return null;
+  }
+
+  if (index + 1 == parent.node.body.length) {
+    return null;
+  }
+
+  return parent.node.body[index + 1];
+}
+
+function isIdentifier(node, name) {
+  if (node.type !== "Identifier") {
+    return false;
+  }
+  if (node.name !== name) {
+    return false;
+  }
+  return true;
+}
+
+function isString(node) {
+  return node.type === "Literal" && typeof node.value === "string";
+}
+
+const jsmExtPattern = /\.(jsm|js|jsm\.js)$/;
+
+function esmifyExtension(path) {
+  return path.replace(jsmExtPattern, ".sys.mjs");
+}
+
+// Given possible member expression, return the list of Identifier nodes in
+// the source order.
+//
+// Returns an empty array if:
+//   * not a simple MemberExpression tree with Identifiers
+//   * there's computed property
+function memberExpressionsToIdentifiers(memberExpr) {
+  let ids = [];
+
+  function f(node) {
+    if (node.type !== "MemberExpression" || node.computed) {
+      return false;
+    }
+
+    if (node.object.type === "Identifier") {
+      ids.push(node.object);
+      ids.push(node.property);
+      return true;
+    }
+
+    if (!f(node.object)) {
+      return false;
+    }
+    ids.push(node.property);
+    return true;
+  }
+
+  if (!f(memberExpr)) {
+    return [];
+  }
+
+  return ids;
+}
+
+// Returns true if the node is a simple MemberExpression tree with Identifiers
+// matches expectedIDs.
+function isMemberExpressionWithIdentifiers(node, expectedIDs) {
+  const actualIDs = memberExpressionsToIdentifiers(node);
+  if (actualIDs.length !== expectedIDs.length) {
+    return false;
+  }
+
+  for (let i = 0; i < expectedIDs.length; i++) {
+    if (actualIDs[i].name !== expectedIDs[i]) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+// Rewrite the Identifiers of MemberExpression tree to toIDs.
+// `node` must be a simple MemberExpression tree with Identifiers, and
+// the length of Identifiers should match.
+function rewriteMemberExpressionWithIdentifiers(node, toIDs) {
+  const actualIDs = memberExpressionsToIdentifiers(node);
+  for (let i = 0; i < toIDs.length; i++) {
+    actualIDs[i].name = toIDs[i];
+  }
+}
+
+// Create a simple MemberExpression tree with given Identifiers.
+function createMemberExpressionWithIdentifiers(jscodeshift, ids) {
+  if (ids.length < 2) {
+    throw new Error("Unexpected length of ids for member expression");
+  }
+
+  if (ids.length > 2) {
+    return jscodeshift.memberExpression(
+      createMemberExpressionWithIdentifiers(jscodeshift, ids.slice(0, -1)),
+      jscodeshift.identifier(ids[ids.length - 1])
+    );
+  }
+
+  return jscodeshift.memberExpression(
+    jscodeshift.identifier(ids[0]),
+    jscodeshift.identifier(ids[1])
+  );
+}
+
+exports.warnForPath = warnForPath;
+exports.getPrevStatement = getPrevStatement;
+exports.getNextStatement = getNextStatement;
+exports.isIdentifier = isIdentifier;
+exports.isString = isString;
+exports.jsmExtPattern = jsmExtPattern;
+exports.esmifyExtension = esmifyExtension;
+exports.isMemberExpressionWithIdentifiers = isMemberExpressionWithIdentifiers;
+exports.rewriteMemberExpressionWithIdentifiers =
+  rewriteMemberExpressionWithIdentifiers;
+exports.createMemberExpressionWithIdentifiers =
+  createMemberExpressionWithIdentifiers;