NFD / NFC Unicode 归一化编码记录

# macOS 默认已启用（Git 2.18+）
git config --global core.precomposeunicode true

$Ordinal = [System.StringComparison]::Ordinal

function Test-IsNfd([string]$Name) {
    $nfc = $Name.Normalize([System.Text.NormalizationForm]::FormC)
    $nfd = $Name.Normalize([System.Text.NormalizationForm]::FormD)
    $eqNfd = [System.String]::Equals($Name, $nfd, $Ordinal)
    $eqNfc = [System.String]::Equals($Name, $nfc, $Ordinal)
    return $eqNfd -and -not $eqNfc
}

function Test-IsNfc([string]$Name) {
    $nfc = $Name.Normalize([System.Text.NormalizationForm]::FormC)
    $nfd = $Name.Normalize([System.Text.NormalizationForm]::FormD)
    $eqNfd = [System.String]::Equals($Name, $nfd, $Ordinal)
    $eqNfc = [System.String]::Equals($Name, $nfc, $Ordinal)
    return $eqNfc -and -not $eqNfd
}

function isNfd(filename) {
  return filename === filename.normalize('NFD') && filename !== filename.normalize('NFC');
}

function isNfc(filename) {
  return filename === filename.normalize('NFC') && filename !== filename.normalize('NFD');
}

import unicodedata
import os


def is_nfd(filename: str) -> bool:
    """判断文件名是否为 NFD 编码"""
    nfc = unicodedata.normalize('NFC', filename)
    nfd = unicodedata.normalize('NFD', filename)
    return filename == nfd and filename != nfc


def is_nfc(filename: str) -> bool:
    """判断文件名是否为 NFC 编码"""
    nfc = unicodedata.normalize('NFC', filename)
    nfd = unicodedata.normalize('NFD', filename)
    return filename == nfc and filename != nfd


def scan_directory(path: str):
    """扫描目录，按 NFD/NFC/Normal 分类文件"""
    result = {"nfd": [], "nfc": [], "normal": []}
    for f in os.listdir(path):
        if is_nfd(f):
            result["nfd"].append(f)
        elif is_nfc(f):
            result["nfc"].append(f)
        else:
            result["normal"].append(f)
    return result

$nativeNames = [System.IO.Directory]::EnumerateFileSystemEntries($dir)

--- Default comparison (-eq) ---
NFD == NFC? True          ← 语义相等，但字节不同
NFD == NFD? True

--- Ordinal comparison ---
NFD == NFC? False         ← 字节不同，正确区分
NFD == NFD? True
NFD == NFD? True

# 预览（不删除）
.\normalize-filenames.ps1 "D:\downloads"

# 执行清理（保留 NFC）
.\normalize-filenames.ps1 "D:\downloads" -Delete

# 或保留 NFD
.\normalize-filenames.ps1 "D:\downloads" -Delete -KeepNfd

node normalize-filenames.js "G:\telegram\30\audio\se"           # 仅预览
node normalize-filenames.js "G:\telegram\30\audio\se" --delete  # 执行清理
node normalize-filenames.js "G:\telegram\30\audio\se" --keep-nfd --delete

import unicodedata
import os
import argparse
from collections import defaultdict


def normalize_classify(filename: str) -> str:
    """返回文件名归一化类型: nfd / nfc / normal"""
    nfc = unicodedata.normalize('NFC', filename)
    nfd = unicodedata.normalize('NFD', filename)
    is_nfc = (filename == nfc) and (filename != nfd)
    is_nfd = (filename == nfd) and (filename != nfc)
    return "nfd" if is_nfd else "nfc" if is_nfc else "normal"


def find_duplicates(files: list) -> list:
    """找到 NFD/NFC 配对重复文件"""
    nfc_map = {}
    nfd_files = []
    for f in files:
        nfc_normalized = unicodedata.normalize('NFC', f)
        cls = normalize_classify(f)
        if cls == "nfc":
            nfc_map[nfc_normalized] = f
        elif cls == "nfd":
            nfd_files.append((nfc_normalized, f))
    pairs = []
    for norm_name, nfd_name in nfd_files:
        if norm_name in nfc_map:
            pairs.append((nfd_name, nfc_map[norm_name]))
    return pairs


def main():
    parser = argparse.ArgumentParser(description="NFD/NFC 重复文件清理工具")
    parser.add_argument("directory", help="目标目录")
    parser.add_argument("--delete", action="store_true", help="执行删除（默认仅预览）")
    parser.add_argument("--keep-nfd", action="store_true", help="保留 NFD 版本（默认保留 NFC）")
    args = parser.parse_args()
    files = os.listdir(args.directory)
    classified = defaultdict(list)
    for f in files:
        classified[normalize_classify(f)].append(f)
    print(f"Total:      {len(files)}")
    print(f"NFD (macOS): {len(classified['nfd'])}")
    print(f"NFC (Win):   {len(classified['nfc'])}")
    print(f"Normal:      {len(classified['normal'])}")
    pairs = find_duplicates(files)
    print(f"Duplicates: {len(pairs)} pairs")
    keep = "nfd" if args.keep_nfd else "nfc"
    for nfd_name, nfc_name in pairs:
        to_delete, to_keep = (nfc_name, nfd_name) if keep == "nfd" else (nfd_name, nfc_name)
        if args.delete:
            os.remove(os.path.join(args.directory, to_delete))
            print(f"DELETED: {to_delete}")
        else:
            print(f"DELETE: {to_delete}")
            print(f"KEEP:   {to_keep}")


if __name__ == "__main__":
    main()

Total:      188
NFD (macOS): 25
NFC (Win):   25
Normal:     138
Pairs:      25
To free:  2069.1 KB

DELETE: スタジアムファンファーレ13.ogg   (NFD: ジ = U+30B7 + U+3099)
KEEP:   スタジアムファンファーレ13.ogg     (NFC: ジ = U+30B8)