mirror of https://github.com/sveltejs/svelte
Simon Holthausen
3 years ago
parent
3a229bc0f6
commit
79104e7adc
@ -1,227 +1,268 @@
|
|||||||
export default function fuzzymatch(name: string, names: string[]) {
|
/**
|
||||||
const set = new FuzzySet(names);
|
* @param {string} name
|
||||||
const matches = set.get(name);
|
* @param {string[]} names
|
||||||
|
*/
|
||||||
return matches && matches[0] && matches[0][0] > 0.7 ? matches[0][1] : null;
|
export default function fuzzymatch(name, names) {
|
||||||
|
const set = new FuzzySet(names);
|
||||||
|
const matches = set.get(name);
|
||||||
|
return matches && matches[0] && matches[0][0] > 0.7 ? matches[0][1] : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// adapted from https://github.com/Glench/fuzzyset.js/blob/master/lib/fuzzyset.js
|
// adapted from https://github.com/Glench/fuzzyset.js/blob/master/lib/fuzzyset.js
|
||||||
// BSD Licensed
|
// BSD Licensed
|
||||||
|
|
||||||
const GRAM_SIZE_LOWER = 2;
|
const GRAM_SIZE_LOWER = 2;
|
||||||
const GRAM_SIZE_UPPER = 3;
|
const GRAM_SIZE_UPPER = 3;
|
||||||
|
|
||||||
// return an edit distance from 0 to 1
|
// return an edit distance from 0 to 1
|
||||||
function _distance(str1: string, str2: string) {
|
|
||||||
if (str1 === null && str2 === null) {
|
|
||||||
throw 'Trying to compare two null values';
|
|
||||||
}
|
|
||||||
if (str1 === null || str2 === null) return 0;
|
|
||||||
str1 = String(str1);
|
|
||||||
str2 = String(str2);
|
|
||||||
|
|
||||||
const distance = levenshtein(str1, str2);
|
|
||||||
if (str1.length > str2.length) {
|
|
||||||
return 1 - distance / str1.length;
|
|
||||||
} else {
|
|
||||||
return 1 - distance / str2.length;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// helper functions
|
/**
|
||||||
function levenshtein(str1: string, str2: string) {
|
* @param {string} str1
|
||||||
const current: number[] = [];
|
* @param {string} str2
|
||||||
let prev;
|
*/
|
||||||
let value;
|
function _distance(str1, str2) {
|
||||||
|
if (str1 === null && str2 === null) {
|
||||||
for (let i = 0; i <= str2.length; i++) {
|
throw 'Trying to compare two null values';
|
||||||
for (let j = 0; j <= str1.length; j++) {
|
}
|
||||||
if (i && j) {
|
if (str1 === null || str2 === null)
|
||||||
if (str1.charAt(j - 1) === str2.charAt(i - 1)) {
|
return 0;
|
||||||
value = prev;
|
str1 = String(str1);
|
||||||
} else {
|
str2 = String(str2);
|
||||||
value = Math.min(current[j], current[j - 1], prev) + 1;
|
const distance = levenshtein(str1, str2);
|
||||||
}
|
if (str1.length > str2.length) {
|
||||||
} else {
|
return 1 - distance / str1.length;
|
||||||
value = i + j;
|
}
|
||||||
}
|
else {
|
||||||
|
return 1 - distance / str2.length;
|
||||||
prev = current[j];
|
}
|
||||||
current[j] = value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return current.pop();
|
|
||||||
}
|
}
|
||||||
|
// helper functions
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} str1
|
||||||
|
* @param {string} str2
|
||||||
|
*/
|
||||||
|
function levenshtein(str1, str2) {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @type {number[]}
|
||||||
|
*/
|
||||||
|
const current = [];
|
||||||
|
let prev;
|
||||||
|
let value;
|
||||||
|
for (let i = 0; i <= str2.length; i++) {
|
||||||
|
for (let j = 0; j <= str1.length; j++) {
|
||||||
|
if (i && j) {
|
||||||
|
if (str1.charAt(j - 1) === str2.charAt(i - 1)) {
|
||||||
|
value = prev;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
value = Math.min(current[j], current[j - 1], prev) + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
value = i + j;
|
||||||
|
}
|
||||||
|
prev = current[j];
|
||||||
|
current[j] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return current.pop();
|
||||||
|
}
|
||||||
const non_word_regex = /[^\w, ]+/;
|
const non_word_regex = /[^\w, ]+/;
|
||||||
|
|
||||||
function iterate_grams(value: string, gram_size = 2) {
|
/**
|
||||||
const simplified = '-' + value.toLowerCase().replace(non_word_regex, '') + '-';
|
* @param {string} value
|
||||||
const len_diff = gram_size - simplified.length;
|
* @param {any} gram_size
|
||||||
const results = [];
|
*/
|
||||||
|
function iterate_grams(value, gram_size = 2) {
|
||||||
if (len_diff > 0) {
|
const simplified = '-' + value.toLowerCase().replace(non_word_regex, '') + '-';
|
||||||
for (let i = 0; i < len_diff; ++i) {
|
const len_diff = gram_size - simplified.length;
|
||||||
value += '-';
|
const results = [];
|
||||||
}
|
if (len_diff > 0) {
|
||||||
}
|
for (let i = 0; i < len_diff; ++i) {
|
||||||
for (let i = 0; i < simplified.length - gram_size + 1; ++i) {
|
value += '-';
|
||||||
results.push(simplified.slice(i, i + gram_size));
|
}
|
||||||
}
|
}
|
||||||
return results;
|
for (let i = 0; i < simplified.length - gram_size + 1; ++i) {
|
||||||
|
results.push(simplified.slice(i, i + gram_size));
|
||||||
|
}
|
||||||
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
function gram_counter(value: string, gram_size = 2) {
|
/**
|
||||||
// return an object where key=gram, value=number of occurrences
|
* @param {string} value
|
||||||
const result = {};
|
* @param {any} gram_size
|
||||||
const grams = iterate_grams(value, gram_size);
|
*/
|
||||||
let i = 0;
|
function gram_counter(value, gram_size = 2) {
|
||||||
|
// return an object where key=gram, value=number of occurrences
|
||||||
for (i; i < grams.length; ++i) {
|
const result = {};
|
||||||
if (grams[i] in result) {
|
const grams = iterate_grams(value, gram_size);
|
||||||
result[grams[i]] += 1;
|
let i = 0;
|
||||||
} else {
|
for (i; i < grams.length; ++i) {
|
||||||
result[grams[i]] = 1;
|
if (grams[i] in result) {
|
||||||
}
|
result[grams[i]] += 1;
|
||||||
}
|
}
|
||||||
return result;
|
else {
|
||||||
|
result[grams[i]] = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {any} a
|
||||||
|
* @param {any} b
|
||||||
|
*/
|
||||||
function sort_descending(a, b) {
|
function sort_descending(a, b) {
|
||||||
return b[0] - a[0];
|
return b[0] - a[0];
|
||||||
}
|
}
|
||||||
|
/** */
|
||||||
class FuzzySet {
|
class FuzzySet {
|
||||||
exact_set = {};
|
|
||||||
match_dict = {};
|
/**
|
||||||
items = {};
|
* @default {}
|
||||||
|
*/
|
||||||
constructor(arr: string[]) {
|
exact_set = {};
|
||||||
// initialization
|
|
||||||
for (let i = GRAM_SIZE_LOWER; i < GRAM_SIZE_UPPER + 1; ++i) {
|
/**
|
||||||
this.items[i] = [];
|
* @default {}
|
||||||
}
|
*/
|
||||||
|
match_dict = {};
|
||||||
// add all the items to the set
|
|
||||||
for (let i = 0; i < arr.length; ++i) {
|
/**
|
||||||
this.add(arr[i]);
|
* @default {}
|
||||||
}
|
*/
|
||||||
}
|
items = {};
|
||||||
|
constructor(arr) {
|
||||||
add(value: string) {
|
// initialization
|
||||||
const normalized_value = value.toLowerCase();
|
for (let i = GRAM_SIZE_LOWER; i < GRAM_SIZE_UPPER + 1; ++i) {
|
||||||
if (normalized_value in this.exact_set) {
|
this.items[i] = [];
|
||||||
return false;
|
}
|
||||||
}
|
// add all the items to the set
|
||||||
|
for (let i = 0; i < arr.length; ++i) {
|
||||||
let i = GRAM_SIZE_LOWER;
|
this.add(arr[i]);
|
||||||
for (i; i < GRAM_SIZE_UPPER + 1; ++i) {
|
}
|
||||||
this._add(value, i);
|
}
|
||||||
}
|
|
||||||
}
|
/**
|
||||||
|
* @param {string} value
|
||||||
_add(value: string, gram_size: number) {
|
*/
|
||||||
const normalized_value = value.toLowerCase();
|
add(value) {
|
||||||
const items = this.items[gram_size] || [];
|
const normalized_value = value.toLowerCase();
|
||||||
const index = items.length;
|
if (normalized_value in this.exact_set) {
|
||||||
|
return false;
|
||||||
items.push(0);
|
}
|
||||||
const gram_counts = gram_counter(normalized_value, gram_size);
|
let i = GRAM_SIZE_LOWER;
|
||||||
let sum_of_square_gram_counts = 0;
|
for (i; i < GRAM_SIZE_UPPER + 1; ++i) {
|
||||||
let gram;
|
this._add(value, i);
|
||||||
let gram_count;
|
}
|
||||||
|
}
|
||||||
for (gram in gram_counts) {
|
|
||||||
gram_count = gram_counts[gram];
|
/**
|
||||||
sum_of_square_gram_counts += Math.pow(gram_count, 2);
|
* @param {string} value
|
||||||
if (gram in this.match_dict) {
|
* @param {number} gram_size
|
||||||
this.match_dict[gram].push([index, gram_count]);
|
*/
|
||||||
} else {
|
_add(value, gram_size) {
|
||||||
this.match_dict[gram] = [[index, gram_count]];
|
const normalized_value = value.toLowerCase();
|
||||||
}
|
const items = this.items[gram_size] || [];
|
||||||
}
|
const index = items.length;
|
||||||
const vector_normal = Math.sqrt(sum_of_square_gram_counts);
|
items.push(0);
|
||||||
items[index] = [vector_normal, normalized_value];
|
const gram_counts = gram_counter(normalized_value, gram_size);
|
||||||
this.items[gram_size] = items;
|
let sum_of_square_gram_counts = 0;
|
||||||
this.exact_set[normalized_value] = value;
|
let gram;
|
||||||
}
|
let gram_count;
|
||||||
|
for (gram in gram_counts) {
|
||||||
get(value: string) {
|
gram_count = gram_counts[gram];
|
||||||
const normalized_value = value.toLowerCase();
|
sum_of_square_gram_counts += Math.pow(gram_count, 2);
|
||||||
const result = this.exact_set[normalized_value];
|
if (gram in this.match_dict) {
|
||||||
|
this.match_dict[gram].push([index, gram_count]);
|
||||||
if (result) {
|
}
|
||||||
return [[1, result]];
|
else {
|
||||||
}
|
this.match_dict[gram] = [[index, gram_count]];
|
||||||
|
}
|
||||||
let results = [];
|
}
|
||||||
// start with high gram size and if there are no results, go to lower gram sizes
|
const vector_normal = Math.sqrt(sum_of_square_gram_counts);
|
||||||
for (let gram_size = GRAM_SIZE_UPPER; gram_size >= GRAM_SIZE_LOWER; --gram_size) {
|
items[index] = [vector_normal, normalized_value];
|
||||||
results = this.__get(value, gram_size);
|
this.items[gram_size] = items;
|
||||||
if (results) {
|
this.exact_set[normalized_value] = value;
|
||||||
return results;
|
}
|
||||||
}
|
|
||||||
}
|
/**
|
||||||
return null;
|
* @param {string} value
|
||||||
}
|
*/
|
||||||
|
get(value) {
|
||||||
__get(value: string, gram_size: number) {
|
const normalized_value = value.toLowerCase();
|
||||||
const normalized_value = value.toLowerCase();
|
const result = this.exact_set[normalized_value];
|
||||||
const matches = {};
|
if (result) {
|
||||||
const gram_counts = gram_counter(normalized_value, gram_size);
|
return [[1, result]];
|
||||||
const items = this.items[gram_size];
|
}
|
||||||
let sum_of_square_gram_counts = 0;
|
let results = [];
|
||||||
let gram;
|
// start with high gram size and if there are no results, go to lower gram sizes
|
||||||
let gram_count;
|
for (let gram_size = GRAM_SIZE_UPPER; gram_size >= GRAM_SIZE_LOWER; --gram_size) {
|
||||||
let i;
|
results = this.__get(value, gram_size);
|
||||||
let index;
|
if (results) {
|
||||||
let other_gram_count;
|
return results;
|
||||||
|
}
|
||||||
for (gram in gram_counts) {
|
}
|
||||||
gram_count = gram_counts[gram];
|
return null;
|
||||||
sum_of_square_gram_counts += Math.pow(gram_count, 2);
|
}
|
||||||
if (gram in this.match_dict) {
|
|
||||||
for (i = 0; i < this.match_dict[gram].length; ++i) {
|
/**
|
||||||
index = this.match_dict[gram][i][0];
|
* @param {string} value
|
||||||
other_gram_count = this.match_dict[gram][i][1];
|
* @param {number} gram_size
|
||||||
if (index in matches) {
|
*/
|
||||||
matches[index] += gram_count * other_gram_count;
|
__get(value, gram_size) {
|
||||||
} else {
|
const normalized_value = value.toLowerCase();
|
||||||
matches[index] = gram_count * other_gram_count;
|
const matches = {};
|
||||||
}
|
const gram_counts = gram_counter(normalized_value, gram_size);
|
||||||
}
|
const items = this.items[gram_size];
|
||||||
}
|
let sum_of_square_gram_counts = 0;
|
||||||
}
|
let gram;
|
||||||
|
let gram_count;
|
||||||
const vector_normal = Math.sqrt(sum_of_square_gram_counts);
|
let i;
|
||||||
let results = [];
|
let index;
|
||||||
let match_score;
|
let other_gram_count;
|
||||||
|
for (gram in gram_counts) {
|
||||||
// build a results list of [score, str]
|
gram_count = gram_counts[gram];
|
||||||
for (const match_index in matches) {
|
sum_of_square_gram_counts += Math.pow(gram_count, 2);
|
||||||
match_score = matches[match_index];
|
if (gram in this.match_dict) {
|
||||||
results.push([match_score / (vector_normal * items[match_index][0]), items[match_index][1]]);
|
for (i = 0; i < this.match_dict[gram].length; ++i) {
|
||||||
}
|
index = this.match_dict[gram][i][0];
|
||||||
|
other_gram_count = this.match_dict[gram][i][1];
|
||||||
results.sort(sort_descending);
|
if (index in matches) {
|
||||||
|
matches[index] += gram_count * other_gram_count;
|
||||||
let new_results = [];
|
}
|
||||||
const end_index = Math.min(50, results.length);
|
else {
|
||||||
// truncate somewhat arbitrarily to 50
|
matches[index] = gram_count * other_gram_count;
|
||||||
for (let i = 0; i < end_index; ++i) {
|
}
|
||||||
new_results.push([_distance(results[i][1], normalized_value), results[i][1]]);
|
}
|
||||||
}
|
}
|
||||||
results = new_results;
|
}
|
||||||
results.sort(sort_descending);
|
const vector_normal = Math.sqrt(sum_of_square_gram_counts);
|
||||||
|
let results = [];
|
||||||
new_results = [];
|
let match_score;
|
||||||
for (let i = 0; i < results.length; ++i) {
|
// build a results list of [score, str]
|
||||||
if (results[i][0] == results[0][0]) {
|
for (const match_index in matches) {
|
||||||
new_results.push([results[i][0], this.exact_set[results[i][1]]]);
|
match_score = matches[match_index];
|
||||||
}
|
results.push([match_score / (vector_normal * items[match_index][0]), items[match_index][1]]);
|
||||||
}
|
}
|
||||||
|
results.sort(sort_descending);
|
||||||
return new_results;
|
let new_results = [];
|
||||||
}
|
const end_index = Math.min(50, results.length);
|
||||||
|
// truncate somewhat arbitrarily to 50
|
||||||
|
for (let i = 0; i < end_index; ++i) {
|
||||||
|
new_results.push([_distance(results[i][1], normalized_value), results[i][1]]);
|
||||||
|
}
|
||||||
|
results = new_results;
|
||||||
|
results.sort(sort_descending);
|
||||||
|
new_results = [];
|
||||||
|
for (let i = 0; i < results.length; ++i) {
|
||||||
|
if (results[i][0] == results[0][0]) {
|
||||||
|
new_results.push([results[i][0], this.exact_set[results[i][1]]]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new_results;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in new issue