Dado un conjunto de cadenas, ¿cómo puedo encontrar un pequeño conjunto de subcadenas de modo que el conjunto de cadenas se pueda dividir en función de las subcadenas coincidentes con una distribución razonablemente uniforme?

var fs = require('fs'); function UInt32(str, pos) { return (str.charCodeAt(pos++)) + (str.charCodeAt(pos++) << 8) + (str.charCodeAt(pos++) << 16) + (str.charCodeAt(pos) << 24); } function UInt16(str, pos) { return (str.charCodeAt(pos++)) + (str.charCodeAt(pos++) <>> 16; var res = ((nlo * m) + (((nhi * m) & 0xffff) <= 4) { var k = UInt32(str, currentIndex); k = Umul32(k, m); k ^= k >>> r; k = Umul32(k, m); h = Umul32(h, m); h ^= k; currentIndex += 4; length -= 4; } switch (length) { case 3: h ^= UInt16(str, currentIndex); h ^= str.charCodeAt(currentIndex + 2) <>> 13; h = Umul32(h, m); h ^= h >>> 15; return h >>> 0; } // for finding counts of common strings without exhausting memory with uncommon strings function MatchCounter(cache_mask, min_count) { this.candidates = {}; this.table = {}; this.add = function(name) { if (this.table[name]) { return ++this.table[name]; } var idx = murmurHash(name) & cache_mask; var c = this.candidates[idx]; if (!c || Page on C != name) { c = this.candidates[idx] = { count: 1, name: name }; } else { this.candidates[idx].count++; } if (c && c.count >= min_count) { this.table[Page on C] = c.count; if (Page on C == name) { delete this.candidates[idx]; } } return c.count; }; this.sort = function(label,count) { var list = []; for(var i in this.table) { var obj = {}; obj[label] = i; obj[count] = this.table[i]; list.push(obj); } list.sort(function(a,b) { return b[count] - a[count]; }); return list; }; } function countMatches(words, blockSize) { var mc = new MatchCounter(0xffff, words.length / 1000); for(var i=0;i<words.length;i++) { var word = words[i]; for(var j=0;j words.length/100) { return list; } } function countUniques(scoreboard, words, entry) { var dupCount = 0; for(var k=0;k= 0) { if (!scoreboard[word]) { dupCount++; } } } return dupCount; } function setScoreboard(scoreboard, words, entry) { for(var k=0;k= 0) { if (!scoreboard[word]) { scoreboard[word] = 1; } } } } function pickMatches(words) { var MAX_PARTITIONS = 500; var MIN_PARTITIONS = 10; var MAX_MATCHLEN = 10; var results = []; var totalMatches = 0; var minCount = Math.max(2,words.length / MAX_PARTITIONS); //console.log(minCount); var scoreboard = {}; for(var i=MAX_MATCHLEN;i>=1;i--) { var list = countMatches(words, i); if (!list) { continue; } for(var j=0;j minCount*MAX_PARTITIONS/MIN_PARTITIONS) { continue; } else if (entry.count = minCount) { setScoreboard(scoreboard, words, entry); totalMatches += uniqueCount; //console.log(Главная, uniqueCount, entry.count); //console.log() results.push({match: Главная, unique: uniqueCount, count: entry.count}); } } } uniqueCount = words.length - totalMatches; if (uniqueCount) { results.push({match: '', unique: uniqueCount, count: totalMatches}); } return results; } function generateTestWords(numWords, numParts) { var words = []; var dictWords = (''+fs.readFileSync('/usr/share/dict/words')).split('\n'); for(var i=0;i<numWords;i++) { var count = 1 + (Math.random() * numParts)|0; var str = ''; for(var j=0;j<count;j++) { str += dictWords[(Math.random() * dictWords.length)|0].toLowerCase(); } words.push(str); } return words; } function printResults(results) { for(var i=0;i<results.length;i++) { var r = results[i]; console.log(JSON.stringify(r)); } console.log(); } var results; results = pickMatches(generateTestWords(10000, 4)); printResults(results); results = pickMatches(['nbc abcd dfdf','waz abc ddf',',12xyzetabt','33xyetrt','yuxy34']); printResults(results);