Remove debug data normalization for span analysis (#13203)

* Remove debug data normalization for span analysis As a result of this normalization, `debug data` could show a user tokens that do not exist in their data. * Update spacy/cli/debug_data.py --------- Co-authored-by: svlandeg <[email protected]>
explosion · Feb 6, 2024 · afb22ad · afb22ad
1 parent 1052cba
commit afb22ad
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py
@@ -1073,8 +1073,7 @@ def _get_distribution(docs, normalize: bool = True) -> Counter:
     word_counts: Counter = Counter()
     for doc in docs:
         for token in doc:
-            # Normalize the text
-            t = token.text.lower().replace("``", '"').replace("''", '"')
+            t = token.text.lower()
             word_counts[t] += 1
     if normalize:
         total = sum(word_counts.values(), 0.0)