Update 2-Working-With-Data/08-data-preparation/notebook.ipynb

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
pull/684/head
Lee Stott 1 year ago committed by GitHub
parent 37aa09a160
commit 186327158e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -3815,24 +3815,29 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# rapidfuzz was already imported in an earlier cell\n", "try:\n",
"from rapidfuzz import process, fuzz\n", " from rapidfuzz import process, fuzz\n",
"\n", "except ImportError:\n",
"\n", " print(\"rapidfuzz is not installed. Please install it with 'pip install rapidfuzz' to use fuzzy matching.\")\n",
" process = None\n",
" fuzz = None\n",
"\n", "\n",
"# Get unique countries\n", "# Get unique countries\n",
"unique_countries = dirty_data['country'].unique()\n", "unique_countries = dirty_data['country'].unique()\n",
"\n", "\n",
"# For each country, find similar matches\n", "# For each country, find similar matches\n",
"print(\"Finding similar country names (similarity > 70%):\")\n", "if process is not None and fuzz is not None:\n",
"for country in unique_countries:\n", " print(\"Finding similar country names (similarity > 70%):\")\n",
" matches = process.extract(country, unique_countries, scorer=fuzz.ratio, limit=3)\n", " for country in unique_countries:\n",
" # Filter matches with similarity > 70 and not identical\n", " matches = process.extract(country, unique_countries, scorer=fuzz.ratio, limit=3)\n",
" similar = [m for m in matches if m[1] > 70 and m[0] != country]\n", " # Filter matches with similarity > 70 and not identical\n",
" if similar:\n", " similar = [m for m in matches if m[1] > 70 and m[0] != country]\n",
" print(f\"\\n'{country}' is similar to:\")\n", " if similar:\n",
" for match, score, _ in similar:\n", " print(f\"\\n'{country}' is similar to:\")\n",
" print(f\" - '{match}' (similarity: {score}%)\")" " for match, score, _ in similar:\n",
" print(f\" - '{match}' (similarity: {score}%)\")\n",
"else:\n",
" print(\"Skipping fuzzy matching because rapidfuzz is not available.\")"
] ]
}, },
{ {

Loading…
Cancel
Save