Skip to content

Instantly share code, notes, and snippets.

@flodolo

flodolo/find_unreferenced_strings.py

Last active March 23, 2025 07:05
Show Gist options
  • Save flodolo/1e9a93a8579aefde3818f2f48dd9bfa2 to your computer and use it in GitHub Desktop.
Save flodolo/1e9a93a8579aefde3818f2f48dd9bfa2 to your computer and use it in GitHub Desktop.
Download ZIP
Find unrreferenced Fluent strings in mozilla-central
Raw
find_unreferenced_strings.py
#!/usr/bin/env python3
import json
import os
import signal
import subprocess
import sys
# Capture CTRL+C
unreferenced_entities = []
processed_files = []
def signal_handler(sig, frame):
print_output()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
def print_output():
if processed_files:
print("\nProcessed files:")
for f in processed_files:
print(f" - {f}")
if unreferenced_entities:
print("Unreference entities:")
for e in unreferenced_entities:
print(f" - {e}")
def grep_entity(entity, search_dir):
"""
Use grep to search for the literal string `entity` recursively in `search_dir`,
restricting the search to files with the given extensions.
Returns True if the entity is found, False otherwise.
"""
grep_cmd = [
"grep",
"-R",
"-F",
"-q",
"--include=*.cpp",
"--include=*.html",
"--include=*.inc",
"--include=*.js",
"--include=*.jsx",
"--include=*.mjs",
"--include=*.nsh",
"--include=*.nsi",
"--include=*.xhtml",
entity,
search_dir,
]
result = subprocess.run(grep_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return result.returncode == 0
def main():
# Path to master.json file in firefox-l10n-source clone
json_file = "/Users/flodolo/mozilla/mercurial/firefox-quarantine/_data/master.json"
# Path to mozilla-unified clone
search_directory = "/Users/flodolo/mozilla/mercurial/mozilla-unified"
# Excluded files
excluded_files = [
"browser/branding/official/brand.ftl",
"browser/branding/official/brand.properties",
# These are experiments?
"browser/browser/featureCallout.ftl",
# Used in a Python file for packaging
"browser/browser/linuxDesktopEntry.ftl",
# Experiments
"browser/browser/newtab/asrouter.ftl",
"browser/browser/newtab/onboarding.ftl",
# Run-time IDs
"browser/browser/policies/policies-descriptions.ftl",
# Should check this one
"browser/browser/preferences/containers.ftl",
# Used elsewhere, worth checking
# https://searchfox.org/mozilla-central/source/toolkit/mozapps/installer/windows/nsis/preprocess-locale.py#88
"browser/installer/override.properties",
]
# Already analyzed. Empty the array to do a new analysis from scratch.
analyzed = [
"browser/browser/aboutDialog.ftl",
"browser/browser/aboutLogins.ftl",
"browser/browser/aboutPocket.ftl",
"browser/browser/aboutPolicies.ftl",
"browser/browser/aboutPrivateBrowsing.ftl",
"browser/browser/aboutRestartRequired.ftl",
"browser/browser/aboutRobots.ftl",
"browser/browser/aboutSessionRestore.ftl",
"browser/browser/aboutTabCrashed.ftl",
"browser/browser/aboutUnloads.ftl",
"browser/browser/accounts.ftl",
"browser/browser/addonNotifications.ftl",
"browser/browser/addonNotifications.ftl",
"browser/browser/allTabsMenu.ftl",
"browser/browser/appExtensionFields.ftl",
"browser/browser/appmenu.ftl",
"browser/browser/appMenuNotifications.ftl",
"browser/browser/backgroundtasks/defaultagent.ftl",
"browser/browser/browser.ftl",
"browser/browser/browserContext.ftl",
"browser/browser/browserSets.ftl",
"browser/browser/colorways.ftl",
"browser/browser/confirmationHints.ftl",
"browser/browser/contentCrash.ftl",
"browser/browser/customizeMode.ftl",
"browser/browser/defaultBrowserNotification.ftl",
"browser/browser/downloads.ftl",
"browser/browser/editBookmarkOverlay.ftl",
"browser/browser/extensionsUI.ftl",
"browser/browser/firefoxRelay.ftl",
"browser/browser/firefoxView.ftl",
"browser/browser/fxviewTabList.ftl",
"browser/browser/genai.ftl",
"browser/browser/identityCredentialNotification.ftl",
"browser/browser/menubar.ftl",
"browser/browser/migrationWizard.ftl",
"browser/browser/newtab/newtab.ftl",
"browser/browser/originControls.ftl",
"browser/browser/pageInfo.ftl",
"browser/browser/panelUI.ftl",
"browser/browser/panicButton.ftl",
"browser/browser/places.ftl",
"browser/browser/placesPrompts.ftl",
"browser/browser/preferences/applicationManager.ftl",
"browser/browser/preferences/blocklists.ftl",
"browser/browser/preferences/clearSiteData.ftl",
"browser/browser/preferences/colors.ftl",
"browser/browser/preferences/connection.ftl",
"browser/browser/preferences/fonts.ftl",
"browser/browser/preferences/formAutofill.ftl",
"browser/browser/preferences/fxaPairDevice.ftl",
"browser/browser/preferences/languages.ftl",
"browser/browser/preferences/moreFromMozilla.ftl",
"browser/browser/preferences/permissions.ftl",
"browser/browser/preferences/preferences.ftl",
"browser/browser/preferences/selectBookmark.ftl",
"browser/browser/preferences/siteDataSettings.ftl",
"browser/browser/preferences/translation.ftl",
"browser/browser/preonboarding.ftl",
"browser/browser/profile/default-bookmarks.ftl",
"browser/browser/profiles.ftl",
"browser/browser/protections.ftl",
"browser/browser/protectionsPanel.ftl",
"browser/browser/recentlyClosed.ftl",
"browser/browser/reportBrokenSite.ftl",
"browser/browser/safebrowsing/blockedSite.ftl",
"browser/browser/safeMode.ftl",
"browser/browser/sanitize.ftl",
"browser/browser/screenshots.ftl",
"browser/browser/search.ftl",
"browser/browser/setDesktopBackground.ftl",
"browser/browser/shopping.ftl",
"browser/browser/sidebar.ftl",
"browser/browser/sidebarMenu.ftl",
"browser/browser/sitePermissions.ftl",
"browser/browser/siteProtections.ftl",
"browser/browser/speechDispatcher.ftl",
"browser/browser/spotlight.ftl",
"browser/safebrowsing/blockedSite.ftl",
"browser/browser/sync.ftl",
"browser/browser/syncedTabs.ftl",
"browser/browser/tabContextMenu.ftl",
"browser/browser/tabbrowser.ftl",
"browser/browser/textRecognition.ftl",
"browser/browser/toolbarContextMenu.ftl",
"browser/browser/touchbar/touchbar.ftl",
"browser/browser/translations.ftl",
"browser/browser/unifiedExtensions.ftl",
"browser/browser/webProtocolHandler.ftl",
"browser/browser/webauthnDialog.ftl",
"browser/browser/webrtcIndicator.ftl",
"browser/chrome/browser/browser.properties",
"browser/chrome/browser/customizableui/customizableWidgets.properties",
"browser/chrome/browser/downloads/downloads.properties",
"browser/chrome/browser/feeds/subscribe.properties",
"browser/chrome/browser/places/bookmarkProperties.properties",
"browser/chrome/browser/safebrowsing/safebrowsing.properties",
"browser/chrome/browser/search.properties",
"browser/chrome/browser/shellservice.properties",
"browser/chrome/browser/siteData.properties",
"browser/chrome/browser/sitePermissions.properties",
"browser/chrome/browser/taskbar.properties",
"browser/chrome/browser/uiDensity.properties",
"browser/chrome/overrides/appstrings.properties",
"browser/extensions/report-site-issue/webcompat.properties",
"browser/firefox-l10n.js",
"browser/browser/contextual-manager.ftl",
"browser/installer/custom.properties",
"browser/installer/mui.properties",
"browser/installer/nsisstrings.properties",
"browser/installer/override.properties",
"browser/langpack-metadata.ftl",
"browser/updater/updater.ini",
"devtools/client/aboutdebugging.ftl",
"devtools/client/accessibility.ftl",
"devtools/client/accessibility.properties",
"devtools/client/animationinspector.properties",
"devtools/client/application.ftl",
"devtools/client/boxmodel.properties",
"devtools/client/changes.properties",
"devtools/client/compatibility.ftl",
"devtools/client/components.properties",
"devtools/client/debugger.properties",
"devtools/client/device.properties",
"devtools/client/dom.properties",
"devtools/client/filterwidget.properties",
"devtools/client/font-inspector.properties",
"devtools/client/har.properties",
"devtools/client/inspector.properties",
"devtools/client/jsonview.properties",
"devtools/client/layout.properties",
"devtools/client/memory.properties",
"devtools/client/menus.properties",
"devtools/client/netmonitor.ftl",
"devtools/client/netmonitor.properties",
"devtools/client/network-throttling.properties",
"devtools/client/perftools.ftl",
"devtools/client/responsive.properties",
"devtools/client/shared.properties",
"devtools/client/sourceeditor.properties",
"devtools/client/startup.properties",
"devtools/client/storage.ftl",
"devtools/client/styleeditor.ftl",
"devtools/client/styleeditor.properties",
"devtools/client/toolbox-options.ftl",
"devtools/client/toolbox.ftl",
"devtools/client/toolbox.properties",
"devtools/client/tooltips.ftl",
"devtools/client/webconsole.properties",
"devtools/shared/accessibility.properties",
"devtools/shared/debugger-paused-reasons.ftl",
"devtools/shared/debugger.properties",
"devtools/shared/eyedropper.properties",
"devtools/shared/highlighters.ftl",
"devtools/shared/screenshot.properties",
"devtools/shared/shared.properties",
"devtools/shared/styleinspector.properties",
"devtools/shared/webconsole-commands.ftl",
"devtools/startup/key-shortcuts.ftl",
]
excluded_files.extend(analyzed)
# Excluded IDs
excluded_ids = [
"about-logins-copy-password-os-auth-dialog-message-macosx",
"about-logins-copy-password-os-auth-dialog-message-win",
"about-logins-edit-login-os-auth-dialog-message2-macosx",
"about-logins-edit-login-os-auth-dialog-message2-win",
"about-logins-reveal-password-os-auth-dialog-message-macosx",
"about-logins-reveal-password-os-auth-dialog-message-win",
"extension-colorways-balanced-name",
"extension-colorways-bold-name",
"extension-colorways-soft-name",
"extension-default-theme-description",
"extension-default-theme-name-auto",
"extension-firefox-alpenglow-description",
"extension-firefox-alpenglow-name",
"extension-firefox-compact-dark-description",
"extension-firefox-compact-dark-name",
"extension-firefox-compact-light-description",
"extension-firefox-compact-light-name",
"firefox-relay-and-fxa-popup-notification-first-sentence-with-domain-and-value-prop",
"firefox-relay-and-fxa-opt-in-confirmation-enable-button-with-domain-and-value-prop",
"firefox-relay-and-fxa-opt-in-confirmation-enable-button-basic-info",
"firefox-relay-and-fxa-popup-notification-header-with-domain-and-value-prop",
"firefox-relay-and-fxa-popup-notification-second-sentence-with-domain-and-value-prop",
"firefox-relay-and-fxa-opt-in-confirmation-enable-button-with-domain",
"newtab-wallpaper-celestial-black-hole",
"newtab-wallpaper-abstract-purple",
"newtab-wallpaper-brown",
"newtab-wallpaper-abstract-black-waves",
"newtab-wallpaper-light-purple",
"newtab-wallpaper-blue-flowers",
"newtab-wallpaper-light-landscape",
"newtab-wallpaper-sky-with-pink-clouds",
"newtab-wallpaper-yellow",
"newtab-wallpaper-abstract-blue",
"newtab-wallpaper-light-pink",
"newtab-wallpaper-celestial-earth-night",
"newtab-wallpaper-dark-color",
"newtab-wallpaper-dark-fox-anniversary",
"newtab-wallpaper-celestial-starry-sky",
"newtab-wallpaper-dark-mountain",
"newtab-wallpaper-starry-canyon",
"newtab-wallpaper-dark-aurora",
"newtab-wallpaper-light-color",
"newtab-wallpaper-abstract-white-curves",
"newtab-wallpaper-celestial-eclipse-time-lapse",
"newtab-wallpaper-dark-blue",
"newtab-wallpaper-celestial-lunar-eclipse",
"newtab-wallpaper-error-max-file-size",
"newtab-wallpaper-light-fox-anniversary",
"newtab-wallpaper-abstract-purple-green",
"newtab-wallpaper-palm-trees",
"newtab-wallpaper-dark-purple",
"newtab-wallpaper-red",
"newtab-wallpaper-light-mountain",
"newtab-wallpaper-green",
"newtab-wallpaper-pink",
"newtab-wallpaper-light-beach",
"newtab-wallpaper-abstract-blue-purple",
"newtab-wallpaper-dark-green",
"newtab-wallpaper-suspension-bridge",
"newtab-wallpaper-light-sky",
"newtab-wallpaper-dark-sky",
"newtab-wallpaper-error-file-type",
"newtab-wallpaper-dark-panda",
"newtab-wallpaper-gradient-orange",
"newtab-wallpaper-beige",
"newtab-wallpaper-red-panda-yawns-in-a-tree",
"newtab-wallpaper-blue",
"newtab-wallpaper-sand-dunes",
"newtab-wallpaper-abstract-blue-purple-waves",
"newtab-wallpaper-beach-at-sunrise",
"newtab-wallpaper-hot-air-balloons",
"newtab-wallpaper-abstract-green",
"newtab-wallpaper-storm-sky",
"newtab-wallpaper-abstract-orange",
"newtab-wallpaper-dark-city",
"newtab-wallpaper-light-green",
"newtab-wallpaper-orange",
"newtab-wallpaper-light-blue",
"newtab-topic-label-education-science",
"newtab-topic-label-business",
"newtab-topic-label-career",
"newtab-topic-label-education",
"newtab-topic-label-food",
"newtab-topic-label-society",
"newtab-topic-label-hobbies",
"newtab-topic-label-finance",
"newtab-topic-label-sports",
"newtab-topic-label-health",
"newtab-topic-label-tech",
"newtab-topic-label-society-parenting",
"newtab-weather-menu-temperature-units",
"newtab-weather-menu-weather-display-option-simple",
"newtab-weather-menu-temperature-option-fahrenheit",
"newtab-weather-menu-weather-display-option-detailed",
"newtab-weather-menu-weather-display",
"newtab-weather-menu-temperature-option-celsius",
"newtab-topic-label-arts",
"newtab-topic-label-government",
"newtab-wallpaper-light-red-panda",
"newtab-topic-label-home",
"newtab-wallpaper-celestial-river",
"newtab-wallpaper-beach-at-sunset",
"newtab-topic-label-travel",
"newtab-wallpaper-white-mountains",
"contextual-manager-passwords-reveal-password-os-auth-dialog-message-macosx",
"contextual-manager-passwords-reveal-password-os-auth-dialog-message-win",
"contextual-manager-passwords-export-os-auth-dialog-message-macosx",
"contextual-manager-passwords-export-os-auth-dialog-message-win",
"contextual-manager-passwords-copy-password-os-auth-dialog-message-macosx",
"contextual-manager-passwords-copy-password-os-auth-dialog-message-win",
"contextual-manager-passwords-edit-password-os-auth-dialog-message-macosx",
"contextual-manager-passwords-edit-password-os-auth-dialog-message-win",
# Run-time
# https://searchfox.org/mozilla-central/source/browser/components/preferences/main.js#3998
"applications-use-app-label",
"applications-use-plugin-in",
"applications-use-os-default-label",
"applications-use-app-default-label",
"applications-use-other-label",
"applications-use-plugin-in-label",
"applications-always-ask-label",
# How many strings??
# https://searchfox.org/mozilla-central/source/browser/components/preferences/main.js#4078
"applications-file-ending-with-type",
]
try:
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
except Exception as e:
print(f"Error loading JSON file: {e}")
sys.exit(1)
# Process each file entry in the JSON.
# Each key is a file ID like "browser/browser/aboutDialog.ftl" and each value is an array of entities.
for file_id, entities in data.items():
if file_id in excluded_files:
continue
# Determine the subfolder based on the first path component (e.g. "browser" or "toolkit").
prefix = file_id.split("/")[0]
current_search_dir = os.path.join(search_directory, prefix)
# Process entities:
# - Remove everything from the first dot onward (e.g. "update-updateButton.label" becomes "update-updateButton")
# - Use a set to remove duplicates.
processed_entities = {e.split(".")[0] for e in entities}
for entity in processed_entities:
# Skip if the entity is in the exclusion list.
if entity in excluded_ids:
continue
sys.stdout.write(".")
sys.stdout.flush()
# First search: in the subfolder based on file_id.
if not grep_entity(entity, current_search_dir):
# Then search in other known paths, arranged in order of use.
common_dirs = [
"toolkit",
"devtools",
"services",
"docshell",
]
if "accessibility" in file_id:
common_dirs.extend(["layout", "remote", "accessible"])
other_dirs = [os.path.join(search_directory, d) for d in common_dirs]
found = False
for dir in other_dirs:
if not found:
found = grep_entity(entity, dir)
if not found:
unreferenced_entities.append(f"{file_id}:{entity}")
sys.stdout.write("+")
sys.stdout.flush()
processed_files.append(file_id)
# Print output if execution wasn't interrupted
print_output()
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment