add logs / unique keys for update

Author: numerogeek

README.md

@@ -9,6 +9,10 @@ Download and install the module through the Admin or Manually
 
 ## How to use
 
+### Quick Import
+
+You can choose to run a _Quick Import_ which will act as a one-time import. Simply follow the steps in the Admin to complete.
+
 ### Profiles
 
 Profiles can be used to setup a common import you will run multiple times. A Profile will store your destination Stream and field mapping to eliminate these steps in the future.
@@ -19,23 +23,11 @@ If you have not created a profile, do so now. A Profile will contain all of the
 
 Once your profile is created, you can click on __Run__ which will propose you to select a File you've uploaded in Files Module, and will import it instantly.
 
-#### Customization in mind
-
-Sometimes you will wish to handle a specific fields and preprocess it before running the insert. So the module create a for you 2 helpers.
-The first one is for preprocessing data. All the fields of the stream you want import into are listed here. 
-the second one is for post processing data. A unique function with the $data and $entry_id to make whatever you want.
-
-### Tehcnical detail
-
-The module will try to convert your file into a PHP array. 
-If for some reason, the root of the array is not the items you want to loop in, you can customize it in the Files settings of the profiles.
-You'll need to check the rights for the directory : helpers/profiles. 
-
 ## Improve it ! 
 
 Feel free to improve this Free and awesome module :)
 It's in beta for now, but we all need import system in our favorite CMS !
 
 ## Issues
 
-Please submit any issues back to the Github issue tracker: <https://github.com/bergeo-fr/streams_import>
+Please submit any issues back to the Github issue tracker: <https://github.com/bergeo-fr/streams_import>

controllers/admin_profiles.php

@@ -261,7 +261,7 @@ public function edit($id)
 			array(
 				'title' 	=> lang($this->namespace. ':tabs:' .'source_connection'  ),
 				'id'		=> 'general',
-				'fields'	=> array('xml_path_loop'),
+				'fields'	=> array('xml_path_loop','unique_keys'),
 				));
 
 		echo $this->streams->cp->entry_form($this->section, $this->namespace, $mode = 'edit', $entry = $id, $view_override = true, $extra, $skips = array(),$this->_tabs);

details.php

@@ -12,7 +12,7 @@
 class Module_Streams_import extends Module
 {
 
-	public $version = 0.6;
+	public $version = 0.7;
 	public $module_name = 'streams_import';
 
 
@@ -326,6 +326,25 @@ public function install()
 			);
 			$this->streams->fields->add_field($field);
 		}
+		$field_slug = "unique_keys";
+		if ( $this->db->where('field_namespace', $this->module_name)->where('field_slug', $field_slug)->limit(1)->get('data_fields')->num_rows() == null )
+		{
+			$field = array(
+				'name'            => 'lang:' . $this->module_name . ':fields:' . $field_slug,
+				'slug'            => $field_slug,
+				'namespace'       => $this->module_name,
+				'type'            => 'text',
+				'extra'			  =>array(
+					"max_length"=>555,
+					"default_value"=>"" ),
+				'assign'          => $stream_slug,
+				'title_column'    => false,
+				'required'        => false,
+				'unique'          => false
+			);
+			$this->streams->fields->add_field($field);
+		}	
+
 
 		// Add Streams - equalities
 		$stream_slug = "mapping";
@@ -453,7 +472,6 @@ public function install()
 			$this->streams->fields->add_field($field);
 		}	
 
-		// Add Fields profiles
 		$field_slug = "filename";
 		if ( $this->db->where('field_namespace', $this->module_name)->where('field_slug', $field_slug)->limit(1)->get('data_fields')->num_rows() == null )
 		{
@@ -468,7 +486,23 @@ public function install()
 				'unique'          => false
 			);
 			$this->streams->fields->add_field($field);
-		}	
+		}
+
+		$field_slug = "log_detail";
+		if ( $this->db->where('field_namespace', $this->module_name)->where('field_slug', $field_slug)->limit(1)->get('data_fields')->num_rows() == null )
+		{
+			$field = array(
+				'name'            => 'lang:' . $this->module_name . ':fields:' . $field_slug,
+				'slug'            => $field_slug,
+				'namespace'       => $this->module_name,
+				'type'            => 'textarea',
+				'assign'          => $stream_slug,
+				'title_column'    => false,
+				'required'        => false,
+				'unique'          => false
+			);
+			$this->streams->fields->add_field($field);
+		}		
 		return true;
 	}
 

events.php

@@ -15,7 +15,12 @@ class Events_Streams_import {
 	public function __construct()
 	{
 		$this->CI =& get_instance();
-		
+
+		// Load up our classes
+		$this->CI->load->driver('Streams');
+		$this->CI->load->helper(array('common_import','streams_import/streams_import'));
+		$this->CI->load->config('streams_import/streams_import_c');
+
 		// Register CRON module shtuff
 		Events::register('cron_process_test', array($this, 'import_pics'));		// cron/test
 		Events::register('cron_process_test', array($this, 'import'));		// cron/test
@@ -34,12 +39,14 @@ public function __construct()
 	public function import_pics()
 	{
 
-		$this->CI->load->helper('streams_import/streams_import');
-		$query=$this->CI->db->get_where('files', array('is_downloaded' => 0), $limit=10, $offset=0); //DL 10 files a time
+		$query=$this->CI->db->get_where('files', array('is_downloaded' => 0), $limit=50, $offset=0); //DL 10 files a time
 
 		//donwload them all
 		foreach ($query->result() as $single_file) {
 				$raw = file_get_contents($single_file->src);
+				if ($raw)
+				{
+				
 				$full_path="uploads/default/files/$single_file->filename";
 				write_file($full_path, $raw);
 
@@ -57,6 +64,10 @@ public function import_pics()
 					);
 					$this->CI->db->where('id',$single_file->id);				
 					$this->CI->db->update('files',$query);
+				}else{					
+					echo "src :".$single_file->src;
+					echo "<br/> no downloadé <br/>";
+				}
 
 
 		}			
@@ -68,10 +79,25 @@ public function import_pics()
 	public function import()
 	{
 
-		// Load up our classes
-		$this->CI->load->driver('Streams');
-		$this->CI->load->helper('common_import');
-		$this->CI->load->config('streams_import/streams_import_c');
+		//Steps for automatic import
+		// 0- Purge
+		// 1- check in the config the path for the profiles.
+		// 2- Load all the profiles where "auto = true"
+		// 3- check if a file is awaiting for import process
+		// 4- Import it :)  
+
+
+
+		//Purge
+
+		$sql =" SELECT id FROM default_listing_homes WHERE (updated is null and (TO_DAYS(NOW()) - TO_DAYS(created)) > 30 ) OR ( (TO_DAYS(NOW()) - TO_DAYS(updated)) >30 ) LIMIT 60"; 
+		echo $sql;
+		$entries = $this->CI->db->query($sql)->result();
+		foreach ($entries as $entry) {
+			$this->CI->streams->entries->delete_entry($entry->id, 'homes', 'listing');
+		}
+
+die();
 
 
 		/*

language/english/streams_import_lang.php

@@ -7,19 +7,14 @@
  * @author   PyroCMS Community
  * @website  https://github.com/bergeo-fr/streams_import
  */
- 
-// CodeIgniter Reference Object
-$ci = &get_instance();
-
+$ci =& get_instance();
 $lang = array(
 	# titles
 	'streams_import:title:profiles:index'                   => 'Profiles',
 	'streams_import:title:profiles:create'                  => 'Create a new profile',
 	'streams_import:title:profiles:edit'                    => 'Edit the profile',
-	'streams_import:title:profiles:quick_import'            => 'Quick Import',
-	'streams_import:title:profiles:quick_import_mapping'    => 'Quick Import: Mapping',
+	'streams_import:title:logs:index'               	  	=> 'Logs',
 	'streams_import:title:profiles:quick_import_success'    => 'Quick Import: Success',
-	'streams_import:title:logs:index'			=> 'Logs',
 
 	# messages
 	'streams_import:messages:profiles:create:success'       => 'New profile created with success.',
@@ -32,12 +27,14 @@
 	'streams_import:messages:import:failure'                => 'Import fail. Sorry about that :(',
 	'streams_import:messages:mapping:save:success'          => 'Mapping saved',
 	'streams_import:messages:mapping:save:failure'          => 'Mapping Failed',
+	'streams_import:messages:logs:delete:success'          	=> 'Logs deleted succesfully',
+	'streams_import:messages:logs:delete:failure'          	=> 'Logs could not be deleted succesfully',
 
 	# fields
 	'streams_import:fields:profile_name'                    => 'Profile Name',
 	'streams_import:fields:profile'                         => 'Profile ID',
 	'streams_import:fields:entry_number'                    => 'Entry',
-	'streams_import:fields:stream_field'                 => 'Stream Field ID',
+	'streams_import:fields:stream_field'                	 => 'Stream Field ID',
 	'streams_import:fields:equalities'                      => 'Associations',
 	'streams_import:fields:stream_identifier'               => 'Stream',
 	'streams_import:fields:eol'                             => 'EOL',
@@ -65,6 +62,9 @@
 	'streams_import:fields:password_instructions'      		=> '',
 	'streams_import:fields:xml_path_loop'      				=> 'Loop Path',
 	'streams_import:fields:xml_path_loop_instructions'		=> 'The file will be converted into an array. You have to provide the node where the system have to loop to import. <a  onClick="window.open(\'/admin/streams_import/profiles/raw_data/'.$ci->uri->segment(5).'\', \'NOM\', \'scrollbars=yes,width=550,height=600\')"target=\'_blank\'>See the raw data</a>',
+	'streams_import:fields:filename'						=> 'File\'s name/ID',
+	'streams_import:fields:profile_rel_logs'				=> 'Profile',
+	'streams_import:fields:profile_slug'					=> 'Profile Slug',
 
 	# buttons
 	'profiles:button:add'                                   => 'Add a profile',
@@ -85,4 +85,4 @@
 	'streams_import:misc_csv_file'                          => 'Pick a file',
 	'streams_import:misc_instructions_csv_file'             => 'or upload one using Files Manager.',
 
-);
+);

libraries/streams_import.php

@@ -74,12 +74,10 @@ public function process_import($profile_id, $file_id = 0)
 		$log_id= $this->register_logs($file_id, $profile_id);
 
 		$profile = $this->ci->streams_import_m->get_profile_object($profile_id);
-		//Import helper
-		$profile_slug=create_slug($profile->profile_name);
 
 		// _helper.php' Preproccess and post process
-		$this->ci->load->helper('profiles/'.$profile_slug . '_pre');
-		$this->ci->load->helper('profiles/'.$profile_slug . '_post');
+		$this->ci->load->helper('profiles/'.$profile->profile_slug . '_pre');
+		$this->ci->load->helper('profiles/'.$profile->profile_slug . '_post');
 
 
 		$data_array = $this->file_to_array($file_id, $profile);
@@ -105,13 +103,19 @@ public function process_import($profile_id, $file_id = 0)
 		foreach ($fields as $field)
 		{
 			$formated_fields[$field->field_id] = $field->field_slug; // ex : $formated_fields[59] = 'name'
-		}
+			//we save the folder field type because the update will be screwed if we don't unset it in the entry_data
+			if($field->field_type == 'folder')
+			{
+				$fields_folder[]=$field->field_slug;
+			}
+	}
 
 
 
 
 		// Soemtimes, the entries are not in the root of the array. You can deal with it in editing the profil and adding the path to the entries where we shoudl loop
 
+
 		if ($profile->xml_path_loop)
 		{
 			//Get the text between [] into array if there's more than 1 ! 
@@ -125,7 +129,6 @@ public function process_import($profile_id, $file_id = 0)
 		}
 
 
-
 		// Detect and get the top level of our data_array => DOESNT WORK.
 		//$data_array = $this->array_to_top_level($data_array);
 
@@ -140,23 +143,52 @@ public function process_import($profile_id, $file_id = 0)
 			{
 
 				// Build the function name
-				$preprocess=$profile_slug .'_'.$map['stream_field'].'_sim_preprocess';
+				$preprocess=$profile->profile_slug .'_'.$map['stream_field'].'_sim_preprocess';
 
 				// Process the value
-				//if the entry_number is 'preprocess' then we call the preprocessor with null because nodata is passed out and they should be hardcoded
-				$processed_value = ($map['entry_number']!='preprocess')?$preprocess($entry[$map['entry_number']]):$preprocess(null);
+				//if the entry_number is 'preprocess' then we call the preprocessor with the full entry set because nodata is passed out and they should be hardcoded
+				$processed_value = ($map['entry_number']!='preprocess')?$preprocess($entry[$map['entry_number']]):$preprocess($entry);
 				$insert_data[$map['stream_field']] = (empty($processed_value))?null:$processed_value;
 
 				// Debug
 				//print_r($insert_data);die;
 			}
-			if (($entry_id = $this->ci->streams->entries->insert_entry($insert_data, $stream->stream_slug, $stream->stream_namespace, $skips = array(), $extra = array())) === false)
+
+			//Ok now : INSERT OR UPDATE ? 
+			//Check if the profile fields "unique_keys" is setted up
+			if(!empty($profile->unique_keys))
+			{
+				//Oh... so you want to update ? explode the keys ! 
+				$keys = explode(',', str_replace(' ', '', $profile->unique_keys));
+				foreach ($keys as $key) {
+					# code...
+					$this->ci->db->where($key,$insert_data[$key]);
+				}
+
+				$update = $this->ci->db->limit(1)->get($stream->stream_namespace.'_'.$stream->stream_slug)->row();
+			}
+
+
+			if(!empty($update->id))
+			{
+				 $skips = array();
+				//unset the folder fields.
+				foreach ($fields_folder as $slug) {
+					# code...
+					unset($insert_data[$slug]);
+					$skips[] = $slug;
+				}
+				//var_dump($insert_data);
+
+				$this->ci->streams->entries->update_entry($update->id,$insert_data, $stream->stream_slug, $stream->stream_namespace,$skips, $extra = array());
+			}
+			elseif (($entry_id = $this->ci->streams->entries->insert_entry($insert_data, $stream->stream_slug, $stream->stream_namespace, $skips = array(), $extra = array())) === false)
 			{
 				continue;
 			}
 
 			//call the post process function
-			$post_process=$profile_slug .'_'.$stream->stream_slug.'_'.'sim_postprocess';
+			$post_process=$profile->profile_slug .'_'.$stream->stream_slug.'_'.'sim_postprocess';
 
 			$post_process($stream, $entry_id, $entry);
 			//die();
@@ -318,7 +350,7 @@ public function post_process_array($data = null, $key =null, $array = array())
 				   and isset($evaluate[2]) and is_array($evaluate[2])
 				)
 			{
-				return $this->post_process_array($value,$key, &$array);
+				return $this->post_process_array($value,$key, $array);
 			}
 			else
 			{