libs/phplibrary
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Библиотека для cis, online, cms1

Browse source
This commit is contained in:
Фёдор Подлеснов 2016-06-29 18:51:32 +03:00
commit 3c2e614d87
269 changed files with 39854 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

181
core/search/stemmer.php Normal file
View file

@ -0,0 +1,181 @@
<?php
/*
* PHP5 implementation of Martin Porter's stemming algorithm for Russian language.
* Written on a cold winter evening close to the end of 2005 by Dennis Kreminsky (etranger at etranger dot ru)
* Use the code freely, but don't hold me responsible if it breaks whatever it might break.
*
*/
define ('CHAR_LENGTH', '1'); // all Russian characters take 2 bytes in UTF-8, so instead of using (not supported by default) mb_*
// string functions, we use the standard ones with a dirty char-length trick.
// Should you want to use WIN-1251 (or any other charset), convert this source file to that encoding
// and then change CHAR_LENGTH to the proper value, which is likely to be '1' then.
//
class Stemmer {
static public function russian($word)
{
$a = self::rv($word);
$start = $a[0];
$rv = $a[1];
$rv = self::step1($rv);
$rv = self::step2($rv);
$rv = self::step3($rv);
$rv = self::step4($rv);
return $start.$rv;
}
static private function rv($word)
{
$vowels = array('à','å','è','î','ó','û','ý','þ','ÿ');
$flag = 0;
$rv = '';
$start = '';
for ($i=0; $i<strlen($word); $i+=CHAR_LENGTH) {
if ($flag==1)
$rv .= substr($word, $i, CHAR_LENGTH);
else
$start .= substr($word, $i, CHAR_LENGTH);
if (array_search(substr($word,$i,CHAR_LENGTH), $vowels) !== false)
$flag=1;
}
return array($start,$rv);
}
static function substitute ($word, &$suffix_list)
{
foreach ($suffix_list as $suffix) {
if (self::has_suffix($word, $suffix)) {
$word = self::cut_suffix($word, $suffix);
}
}
return $word;
}
static function has_suffix ($word, $suffix)
{
return substr($word, -(strlen($suffix))) == $suffix;
}
static function has_aya ($word, $suffix)
{
return (substr($word,-strlen($suffix)-CHAR_LENGTH,CHAR_LENGTH)=='à' || substr($word,-strlen($suffix)-CHAR_LENGTH,CHAR_LENGTH)=='ÿ');
}
static function cut_suffix ($word, $suffix)
{
return substr($word, 0, strlen($word) - strlen($suffix));
}
static private function step1($word)
{
$perfective1 = array('â', 'âøè', 'âøèñü');
foreach ($perfective1 as $suffix) {
if (self::has_suffix($word, $suffix) && self::has_aya ($word, $suffix)) {
return self::cut_suffix($word, $suffix);
}
}
$perfective2 = array('èâ','èâøè','èâøèñü','ûâøè','ûâøèñü');
foreach ($perfective2 as $suffix) {
if (self::has_suffix($word, $suffix)) {
return self::cut_suffix($word, $suffix);
}
}
$reflexive = array('ñÿ', 'ñü');
$word = self::substitute($word, $reflexive);
$adjective = array('åå','èå','ûå','îå','èìè','ûìè','åé','èé','ûé','îé','åì','èì','ûì','îì','åãî','îãî','åìó','îìó','èõ','ûõ','óþ','þþ','àÿ','ÿÿ','îþ','åþ');
$participle2 = array('åì','íí','âø','þù','ù');
$participle1 = array('èâø','ûâø','óþù');
foreach ($adjective as $suffix) {
if (self::has_suffix($word, $suffix)) {
$word = self::cut_suffix($word, $suffix);
foreach ($participle1 as $suffix)
if (self::has_suffix($word, $suffix) && self::has_aya ($word, $suffix))
$word = self::cut_suffix($word, $suffix);
return self::substitute($word, $participle2);
}
}
$verb1 = array('ëà','íà','åòå','éòå','ëè','é','ë','åì','í','ëî','íî','åò','þò','íû','òü','åøü','ííî');
foreach ($verb1 as $suffix)
if (self::has_suffix($word, $suffix) && self::has_aya ($word, $suffix))
return self::cut_suffix($word, $suffix);
$verb2 = array('èëà','ûëà','åíà','åéòå','óéòå','èòå','èëè','ûëè','åé','óé','èë','ûë','èì','ûì','åí','èëî','ûëî','åíî','ÿò','óåò','óþò','èò','ûò','åíû','èòü','ûòü','èøü','óþ','þ');
foreach ($verb2 as $suffix)
if (self::has_suffix($word, $suffix))
return self::cut_suffix($word, $suffix);
$noun = array('à','åâ','îâ','èå','üå','å','èÿìè','ÿìè','àìè','åè','èè','è','èåé','åé','îé','èé','é','èÿì','ÿì','èåì','åì','àì','îì','î','ó','àõ','èÿõ','ÿõ','û','ü','èþ','üþ','þ','èÿ','üÿ','ÿ');
foreach ($noun as $suffix) {
if (self::has_suffix($word, $suffix))
return self::cut_suffix($word, $suffix);
}
return $word;
}
static private function step2($word)
{
if (substr($word,-CHAR_LENGTH,CHAR_LENGTH) == 'è')
$word = substr($word, 0, strlen($word)-CHAR_LENGTH);
return $word;
}
static private function step3($word)
{
$vowels = array('à','å','è','î','ó','û','ý','þ','ÿ');
$flag = 0;
$r1 = '';
$r2 = '';
for ($i=0; $i<strlen($word); $i+=CHAR_LENGTH)
{
if ($flag==2)
$r1.=substr($word, $i, CHAR_LENGTH);
if (array_search(substr($word,$i,CHAR_LENGTH), $vowels) !== false)
$flag=1;
if ($flag=1 && array_search(substr($word,$i,CHAR_LENGTH), $vowels) === false)
$flag=2;
}
$flag=0;
for ($i=0; $i<strlen($r1); $i+=CHAR_LENGTH)
{
if ($flag==2)
$r2.=substr($r1, $i, CHAR_LENGTH);
if (array_search(substr($r1,$i,CHAR_LENGTH), $vowels) !== false)
$flag=1;
if ($flag=1 && array_search(substr($r1,$i,CHAR_LENGTH), $vowels) === false)
$flag=2;
}
$derivational=array('îñò', 'îñòü');
foreach ($derivational as $suffix)
if (substr($r2,-(strlen($suffix))) == $suffix)
$word=substr($word, 0, strlen($r2)-strlen($suffix));
return $word;
}
static private function step4($word)
{
if (substr($word,-CHAR_LENGTH*2)=='íí')
$word=substr($word, 0, strlen($word)-CHAR_LENGTH);
else
{
$superlative=array('åéø', 'åéøå');
foreach ($superlative as $suffix)
if (substr($word,-(strlen($suffix))) == $suffix)
$word = substr($word, 0, strlen($word) - strlen($suffix));
if (substr($word,-CHAR_LENGTH*2) == 'íí')
$word = substr($word, 0, strlen($word) - CHAR_LENGTH);
}
// should there be a guard flag? can't think of a russian word that ends with åéøü or ííü anyways, though the algorithm states this is an "otherwise" case
if (substr($word,-CHAR_LENGTH,CHAR_LENGTH) == 'ü')
$word=substr($word, 0, strlen($word)-CHAR_LENGTH);
return $word;
}
}
?>