Skip to content

simple yet powerful enhancements #86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
melanholly wants to merge 3 commits into
base: L4
Choose a base branch
from
Open

simple yet powerful enhancements #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
16d9bf9
Adding Option for limiting the searchble fields of the model.
melanholly Feb 3, 2015
435a676
Added tokenizers and filters for bulgarian
melanholly May 22, 2015
efa6330
Added fields mapping and fixed parser error in for date fields
melanholly May 22, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions src/Iverberk/Larasearch/Index.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,12 @@ public function __construct(Proxy $proxy, $name = '')
public function import(Model $model, $relations = [], $batchSize = 750, Callable $callback = null)
{
$batch = 0;


//model searchble fields
$fields=isset($model::$searchable_fields)
?array_merge(['id','created_at','updated_at'],$model::$searchable_fields)
:false;

while (true)
{
// Increase the batch number
Expand All @@ -81,8 +86,17 @@ public function import(Model $model, $relations = [], $batchSize = 750, Callable
$records = $model
->with($relations)
->skip($batchSize * ($batch - 1))
->take($batchSize)
->get();
->take($batchSize);
if($fields)
{
$records = $records
->select($fields)
->get();
}
else
{
$records = $records->get();
}

// Break out of the loop if we are out of records
if (count($records) == 0) break;
Expand Down Expand Up @@ -167,10 +181,10 @@ public function getProxy()
*
* @param array $options
*/
public function create($options = [])
public function create($options = [],$additional = [])
{
$body = empty($options) ? $this->getDefaultIndexParams() : $options;

$body = array_merge_recursive($body, $additional);
self::getClient()->indices()->create(['index' => $this->getName(), 'body' => $body]);
}

Expand Down Expand Up @@ -299,7 +313,7 @@ public function bulk($records)
$errorItems[] = $item;
}
}

\Log::error($errorItems);
throw new ImportException('Bulk import with errors', 1, $errorItems);
}
}
Expand Down
29 changes: 26 additions & 3 deletions src/Iverberk/Larasearch/Proxy.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,14 +123,37 @@ public function reindex($relations = false, $batchSize = 750, $mapping = [], Cal
{
$model = $this->config['model'];
$name = $this->config['index']->getName();

$newName = $name . '_' . date("YmdHis");
$relations = $relations ? Config::get('larasearch::paths.' . get_class($model)) : [];

$type_mapper = ["Array"=>['type' => 'string'],"BigInt"=>['type'=> 'long'],"Object"=>['type' => 'string'],"SmallInt"=>['type'=> 'integer'],"String"=>['type' => 'string'],"Text"=>['type' => 'string'],"Time"=>['type' => 'date'],"Blob"=>['type'=> 'binary'],"Boolean"=>['type'=> 'boolean'],"DateTime"=>['type' => 'date','format' => 'yyyy-MM-dd HH:mm:ss',],"DateTimeTz"=>['type' => 'date','format' => 'yyyy-MM-dd HH:mm:ss',],"Date"=>['type' => 'date','format' => 'yyyy-MM-dd',],"Decimal"=>['type' => 'double'],"Float"=>['type' => 'float'],"Integer"=>['type'=> 'integer'],"VarDateTime"=>['type' => 'string'],];
$additional_mapping=['mappings'=>[$this->config['type']=> ['properties' => []]]];
//model searchble fields
$fields=isset($model::$searchable_fields)
?array_merge(['id','created_at','updated_at'],$model::$searchable_fields)
:false;

foreach ($model->getColunmTypes() as $field => $type) {
if($fields)
{
if(in_array($field, $fields))
{
$additional_mapping['mappings']
[$this->config['type']]
['properties'][$field]=$type_mapper[$type];
}
}
else
{
$additional_mapping['mappings']
[$this->config['type']]
['properties'][$field]=$type_mapper[$type];
}
}

Index::clean($name);

$index = App::make('iverberk.larasearch.index', array('name' => $newName, 'proxy' => $this));
$index->create($mapping);
$index->create($mapping,$additional_mapping);

if ($index->aliasExists($name))
{
Expand Down
2 changes: 1 addition & 1 deletion src/Iverberk/Larasearch/Query.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ public function execute()
$this->getAggregations();
$this->getSort();
$this->getPayload();

$params = [
'index' => Utils::findKey($this->options, 'index', false) ?: $this->proxy->getIndex()->getName(),
'type' => Utils::findKey($this->options, 'type', false) ?: $this->proxy->getType(),
Expand Down
31 changes: 31 additions & 0 deletions src/Iverberk/Larasearch/Traits/TransformableTrait.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,40 @@ public function transform($relations = false)
{
$relations = $relations ? Config::get('larasearch::paths.' . get_class($this)) : [];


foreach ($this->getColunmTypes('Date') as $field => $type) {
if(strpos(strtolower($this->{$field}),"0000")!==false)
{
$this->{$field}=null;
}
};
$doc = $this->load($relations)->toArray();

return $doc;
}

public function getColunmTypes($search = null)
{
$columns = [];
$table_columns = \DB::getDoctrineSchemaManager()
->listTableDetails(
$this->getTable())
->getColumns();
if (!is_null($search)) {
foreach ($table_columns as $key => $value)
{
$name = preg_replace('/(".*Doctrine.*\\\\)(.*?)(Type:.*)/', "$2", $value->toArray()['type']);
if(strpos(strtolower($name),strtolower($search))!== false)
$columns[$key]=$name;
}
}
else
{
foreach ($table_columns as $key => $value)
{
$columns[$key]=preg_replace('/(".*Doctrine.*\\\\)(.*?)(Type:.*)/', "$2", $value->toArray()['type']);
}
}
return $columns;
}
}
48 changes: 34 additions & 14 deletions src/config/config.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,72 +65,82 @@
'larasearch_keyword' => [
'type' => "custom",
'tokenizer' => "keyword",
'filter' => ["lowercase", "larasearch_stemmer"]
'filter' => ["lowercase", "larasearch_stemmer", "bulgarian_stop", "bulgarian_stemmer"]
],
'default_index' => [
'type' => "custom",
'tokenizer' => "standard",
'filter' => ["standard", "lowercase", "asciifolding", "larasearch_index_shingle", "larasearch_stemmer"]
'filter' => ["standard", "lowercase", "asciifolding", "larasearch_index_shingle", "larasearch_stemmer", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_search' => [
'type' => "custom",
'tokenizer' => "standard",
'filter' => ["standard", "lowercase", "asciifolding", "larasearch_search_shingle", "larasearch_stemmer"]
'filter' => ["standard", "lowercase", "asciifolding", "larasearch_search_shingle", "larasearch_stemmer", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_search2' => [
'type' => "custom",
'tokenizer' => "standard",
'filter' => ["standard", "lowercase", "asciifolding", "larasearch_stemmer"]
'filter' => ["standard", "lowercase", "asciifolding", "larasearch_stemmer", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_autocomplete_index' => [
'type' => "custom",
'tokenizer' => "larasearch_autocomplete_ngram",
'filter' => ["lowercase", "asciifolding"]
'filter' => ["lowercase", "asciifolding", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_autocomplete_search' => [
'type' => "custom",
'tokenizer' => "keyword",
'filter' => ["lowercase", "asciifolding"]
'filter' => ["lowercase", "asciifolding", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_word_search' => [
'type' => "custom",
'tokenizer' => "standard",
'filter' => ["lowercase", "asciifolding"]
'filter' => ["lowercase", "asciifolding", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_suggest_index' => [
'type' => "custom",
'tokenizer' => "standard",
'filter' => ["lowercase", "asciifolding", "larasearch_suggest_shingle"]
'filter' => ["lowercase", "asciifolding", "larasearch_suggest_shingle", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_text_start_index' => [
'type' => "custom",
'tokenizer' => "keyword",
'filter' => ["lowercase", "asciifolding", "larasearch_edge_ngram"]
'filter' => ["lowercase", "asciifolding", "larasearch_edge_ngram", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_text_middle_index' => [
'type' => "custom",
'tokenizer' => "keyword",
'filter' => ["lowercase", "asciifolding", "larasearch_ngram"]
'filter' => ["lowercase", "asciifolding", "larasearch_ngram", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_text_end_index' => [
'type' => "custom",
'tokenizer' => "keyword",
'filter' => ["lowercase", "asciifolding", "reverse", "larasearch_edge_ngram", "reverse"]
'filter' => ["lowercase", "asciifolding", "reverse", "larasearch_edge_ngram", "reverse", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_word_start_index' => [
'type' => "custom",
'tokenizer' => "standard",
'filter' => ["lowercase", "asciifolding", "larasearch_edge_ngram"]
'filter' => ["lowercase", "asciifolding", "larasearch_edge_ngram", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_word_middle_index' => [
'type' => "custom",
'tokenizer' => "standard",
'filter' => ["lowercase", "asciifolding", "larasearch_ngram"]
'filter' => ["lowercase", "asciifolding", "larasearch_ngram", "bulgarian_stop", "bulgarian_stemmer"]
],
'larasearch_word_end_index' => [
'type' => "custom",
'tokenizer' => "standard",
'filter' => ["lowercase", "asciifolding", "reverse", "larasearch_edge_ngram", "reverse"]
'filter' => ["lowercase", "asciifolding", "reverse", "larasearch_edge_ngram", "reverse", "bulgarian_stop", "bulgarian_stemmer"]
],
'bulgarian' =>
[
'tokenizer' => 'standard',
'filter' =>
[
'lowercase',
'bulgarian_stop',
'bulgarian_stemmer'
]
]
],
'filter' => [
Expand Down Expand Up @@ -161,6 +171,16 @@
'larasearch_stemmer' => [
'type' => "snowball",
'language' => "English"
],
'bulgarian_stop' =>
[
'type' => 'stop',
'stopwords' => 'а,аз,ако,ала,бе,без,беше,би,бил,била,били,било,близо,бъдат,бъде,бяха,в,вас,ваш,ваша,вероятно,вече,взема,ви,вие,винаги,все,всеки,всички,всичко,всяка,във,въпреки,върху,г,ги,главно,го,д,да,дали,до,докато,докога,дори,досега,доста,е,едва,един,ето,за,зад,заедно,заради,засега,затова,защо,защото,и,из,или,им,има,имат,иска,й,каза,как,каква,какво,както,какъв,като,кога,когато,което,които,кой,който,колко,която,къде,където,към,ли,м,ме,между,мен,ми,мнозина,мога,могат,може,моля,момента,му,н,на,над,назад,най,направи,напред,например,нас,не,него,нея,ни,ние,никой,нито,но,някои,някой,няма,обаче,около,освен,особено,от,отгоре,отново,още,пак,по,повече,повечето,под,поне,поради,после,почти,прави,пред,преди,през,при,пък,първо,с,са,само,се,сега,си,скоро,след,сме,според,сред,срещу,сте,съм,със,също,т,тази,така,такива,такъв,там,твой,те,тези,ти,тн,то,това,тогава,този,той,толкова,точно,трябва,тук,тъй,тя,тях,у,харесва,ч,че,често,чрез,ще,щом,я,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with',
],
'bulgarian_stemmer' =>
[
'type' => 'stemmer',
'language' => 'bulgarian'
]
],
'tokenizer' => [
Expand Down