diff --git a/src/Iverberk/Larasearch/Index.php b/src/Iverberk/Larasearch/Index.php index 49a370e..75c0adf 100644 --- a/src/Iverberk/Larasearch/Index.php +++ b/src/Iverberk/Larasearch/Index.php @@ -71,7 +71,12 @@ public function __construct(Proxy $proxy, $name = '') public function import(Model $model, $relations = [], $batchSize = 750, Callable $callback = null) { $batch = 0; - + + //model searchble fields + $fields=isset($model::$searchable_fields) + ?array_merge(['id','created_at','updated_at'],$model::$searchable_fields) + :false; + while (true) { // Increase the batch number @@ -81,8 +86,17 @@ public function import(Model $model, $relations = [], $batchSize = 750, Callable $records = $model ->with($relations) ->skip($batchSize * ($batch - 1)) - ->take($batchSize) - ->get(); + ->take($batchSize); + if($fields) + { + $records = $records + ->select($fields) + ->get(); + } + else + { + $records = $records->get(); + } // Break out of the loop if we are out of records if (count($records) == 0) break; @@ -167,10 +181,10 @@ public function getProxy() * * @param array $options */ - public function create($options = []) + public function create($options = [],$additional = []) { $body = empty($options) ? $this->getDefaultIndexParams() : $options; - + $body = array_merge_recursive($body, $additional); self::getClient()->indices()->create(['index' => $this->getName(), 'body' => $body]); } @@ -299,7 +313,7 @@ public function bulk($records) $errorItems[] = $item; } } - + \Log::error($errorItems); throw new ImportException('Bulk import with errors', 1, $errorItems); } } diff --git a/src/Iverberk/Larasearch/Proxy.php b/src/Iverberk/Larasearch/Proxy.php index b84fb9f..fcd1d6d 100644 --- a/src/Iverberk/Larasearch/Proxy.php +++ b/src/Iverberk/Larasearch/Proxy.php @@ -123,14 +123,37 @@ public function reindex($relations = false, $batchSize = 750, $mapping = [], Cal { $model = $this->config['model']; $name = $this->config['index']->getName(); - $newName = $name . '_' . date("YmdHis"); $relations = $relations ? Config::get('larasearch::paths.' . get_class($model)) : []; - + $type_mapper = ["Array"=>['type' => 'string'],"BigInt"=>['type'=> 'long'],"Object"=>['type' => 'string'],"SmallInt"=>['type'=> 'integer'],"String"=>['type' => 'string'],"Text"=>['type' => 'string'],"Time"=>['type' => 'date'],"Blob"=>['type'=> 'binary'],"Boolean"=>['type'=> 'boolean'],"DateTime"=>['type' => 'date','format' => 'yyyy-MM-dd HH:mm:ss',],"DateTimeTz"=>['type' => 'date','format' => 'yyyy-MM-dd HH:mm:ss',],"Date"=>['type' => 'date','format' => 'yyyy-MM-dd',],"Decimal"=>['type' => 'double'],"Float"=>['type' => 'float'],"Integer"=>['type'=> 'integer'],"VarDateTime"=>['type' => 'string'],]; + $additional_mapping=['mappings'=>[$this->config['type']=> ['properties' => []]]]; + //model searchble fields + $fields=isset($model::$searchable_fields) + ?array_merge(['id','created_at','updated_at'],$model::$searchable_fields) + :false; + + foreach ($model->getColunmTypes() as $field => $type) { + if($fields) + { + if(in_array($field, $fields)) + { + $additional_mapping['mappings'] + [$this->config['type']] + ['properties'][$field]=$type_mapper[$type]; + } + } + else + { + $additional_mapping['mappings'] + [$this->config['type']] + ['properties'][$field]=$type_mapper[$type]; + } + } + Index::clean($name); $index = App::make('iverberk.larasearch.index', array('name' => $newName, 'proxy' => $this)); - $index->create($mapping); + $index->create($mapping,$additional_mapping); if ($index->aliasExists($name)) { diff --git a/src/Iverberk/Larasearch/Query.php b/src/Iverberk/Larasearch/Query.php index ac492a5..d7f0abc 100644 --- a/src/Iverberk/Larasearch/Query.php +++ b/src/Iverberk/Larasearch/Query.php @@ -323,7 +323,7 @@ public function execute() $this->getAggregations(); $this->getSort(); $this->getPayload(); - + $params = [ 'index' => Utils::findKey($this->options, 'index', false) ?: $this->proxy->getIndex()->getName(), 'type' => Utils::findKey($this->options, 'type', false) ?: $this->proxy->getType(), diff --git a/src/Iverberk/Larasearch/Traits/TransformableTrait.php b/src/Iverberk/Larasearch/Traits/TransformableTrait.php index fe6f6af..ba83347 100644 --- a/src/Iverberk/Larasearch/Traits/TransformableTrait.php +++ b/src/Iverberk/Larasearch/Traits/TransformableTrait.php @@ -14,9 +14,40 @@ public function transform($relations = false) { $relations = $relations ? Config::get('larasearch::paths.' . get_class($this)) : []; + + foreach ($this->getColunmTypes('Date') as $field => $type) { + if(strpos(strtolower($this->{$field}),"0000")!==false) + { + $this->{$field}=null; + } + }; $doc = $this->load($relations)->toArray(); return $doc; } + public function getColunmTypes($search = null) + { + $columns = []; + $table_columns = \DB::getDoctrineSchemaManager() + ->listTableDetails( + $this->getTable()) + ->getColumns(); + if (!is_null($search)) { + foreach ($table_columns as $key => $value) + { + $name = preg_replace('/(".*Doctrine.*\\\\)(.*?)(Type:.*)/', "$2", $value->toArray()['type']); + if(strpos(strtolower($name),strtolower($search))!== false) + $columns[$key]=$name; + } + } + else + { + foreach ($table_columns as $key => $value) + { + $columns[$key]=preg_replace('/(".*Doctrine.*\\\\)(.*?)(Type:.*)/', "$2", $value->toArray()['type']); + } + } + return $columns; + } } \ No newline at end of file diff --git a/src/config/config.php b/src/config/config.php index b33b5fe..b539940 100644 --- a/src/config/config.php +++ b/src/config/config.php @@ -65,72 +65,82 @@ 'larasearch_keyword' => [ 'type' => "custom", 'tokenizer' => "keyword", - 'filter' => ["lowercase", "larasearch_stemmer"] + 'filter' => ["lowercase", "larasearch_stemmer", "bulgarian_stop", "bulgarian_stemmer"] ], 'default_index' => [ 'type' => "custom", 'tokenizer' => "standard", - 'filter' => ["standard", "lowercase", "asciifolding", "larasearch_index_shingle", "larasearch_stemmer"] + 'filter' => ["standard", "lowercase", "asciifolding", "larasearch_index_shingle", "larasearch_stemmer", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_search' => [ 'type' => "custom", 'tokenizer' => "standard", - 'filter' => ["standard", "lowercase", "asciifolding", "larasearch_search_shingle", "larasearch_stemmer"] + 'filter' => ["standard", "lowercase", "asciifolding", "larasearch_search_shingle", "larasearch_stemmer", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_search2' => [ 'type' => "custom", 'tokenizer' => "standard", - 'filter' => ["standard", "lowercase", "asciifolding", "larasearch_stemmer"] + 'filter' => ["standard", "lowercase", "asciifolding", "larasearch_stemmer", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_autocomplete_index' => [ 'type' => "custom", 'tokenizer' => "larasearch_autocomplete_ngram", - 'filter' => ["lowercase", "asciifolding"] + 'filter' => ["lowercase", "asciifolding", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_autocomplete_search' => [ 'type' => "custom", 'tokenizer' => "keyword", - 'filter' => ["lowercase", "asciifolding"] + 'filter' => ["lowercase", "asciifolding", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_word_search' => [ 'type' => "custom", 'tokenizer' => "standard", - 'filter' => ["lowercase", "asciifolding"] + 'filter' => ["lowercase", "asciifolding", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_suggest_index' => [ 'type' => "custom", 'tokenizer' => "standard", - 'filter' => ["lowercase", "asciifolding", "larasearch_suggest_shingle"] + 'filter' => ["lowercase", "asciifolding", "larasearch_suggest_shingle", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_text_start_index' => [ 'type' => "custom", 'tokenizer' => "keyword", - 'filter' => ["lowercase", "asciifolding", "larasearch_edge_ngram"] + 'filter' => ["lowercase", "asciifolding", "larasearch_edge_ngram", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_text_middle_index' => [ 'type' => "custom", 'tokenizer' => "keyword", - 'filter' => ["lowercase", "asciifolding", "larasearch_ngram"] + 'filter' => ["lowercase", "asciifolding", "larasearch_ngram", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_text_end_index' => [ 'type' => "custom", 'tokenizer' => "keyword", - 'filter' => ["lowercase", "asciifolding", "reverse", "larasearch_edge_ngram", "reverse"] + 'filter' => ["lowercase", "asciifolding", "reverse", "larasearch_edge_ngram", "reverse", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_word_start_index' => [ 'type' => "custom", 'tokenizer' => "standard", - 'filter' => ["lowercase", "asciifolding", "larasearch_edge_ngram"] + 'filter' => ["lowercase", "asciifolding", "larasearch_edge_ngram", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_word_middle_index' => [ 'type' => "custom", 'tokenizer' => "standard", - 'filter' => ["lowercase", "asciifolding", "larasearch_ngram"] + 'filter' => ["lowercase", "asciifolding", "larasearch_ngram", "bulgarian_stop", "bulgarian_stemmer"] ], 'larasearch_word_end_index' => [ 'type' => "custom", 'tokenizer' => "standard", - 'filter' => ["lowercase", "asciifolding", "reverse", "larasearch_edge_ngram", "reverse"] + 'filter' => ["lowercase", "asciifolding", "reverse", "larasearch_edge_ngram", "reverse", "bulgarian_stop", "bulgarian_stemmer"] + ], + 'bulgarian' => + [ + 'tokenizer' => 'standard', + 'filter' => + [ + 'lowercase', + 'bulgarian_stop', + 'bulgarian_stemmer' + ] ] ], 'filter' => [ @@ -161,6 +171,16 @@ 'larasearch_stemmer' => [ 'type' => "snowball", 'language' => "English" + ], + 'bulgarian_stop' => + [ + 'type' => 'stop', + 'stopwords' => 'а,аз,ако,ала,бе,без,беше,би,бил,била,били,било,близо,бъдат,бъде,бяха,в,вас,ваш,ваша,вероятно,вече,взема,ви,вие,винаги,все,всеки,всички,всичко,всяка,във,въпреки,върху,г,ги,главно,го,д,да,дали,до,докато,докога,дори,досега,доста,е,едва,един,ето,за,зад,заедно,заради,засега,затова,защо,защото,и,из,или,им,има,имат,иска,й,каза,как,каква,какво,както,какъв,като,кога,когато,което,които,кой,който,колко,която,къде,където,към,ли,м,ме,между,мен,ми,мнозина,мога,могат,може,моля,момента,му,н,на,над,назад,най,направи,напред,например,нас,не,него,нея,ни,ние,никой,нито,но,някои,някой,няма,обаче,около,освен,особено,от,отгоре,отново,още,пак,по,повече,повечето,под,поне,поради,после,почти,прави,пред,преди,през,при,пък,първо,с,са,само,се,сега,си,скоро,след,сме,според,сред,срещу,сте,съм,със,също,т,тази,така,такива,такъв,там,твой,те,тези,ти,тн,то,това,тогава,този,той,толкова,точно,трябва,тук,тъй,тя,тях,у,харесва,ч,че,често,чрез,ще,щом,я,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with', + ], + 'bulgarian_stemmer' => + [ + 'type' => 'stemmer', + 'language' => 'bulgarian' ] ], 'tokenizer' => [