fixes

Author: numerogeek

config/routes.php

@@ -8,6 +8,7 @@
  * @website  https://github.com/bergeo-fr/streams_import
  */
 $route['streams_import/admin/profiles(/:any)?']     = 'admin_profiles$1';
+$route['streams_import/admin/logs(/:num)?']         = 'admin_logs/index/$1';
 $route['streams_import/admin/logs(/:any)?']         = 'admin_logs$1';
 $route['streams_import/admin(/:any)?'] 				= 'admin_profiles$1';
 

config/streams_import_c.php

@@ -3,5 +3,8 @@
 /*
  * We some security so we can call the importer on the front end
  */
-$config['streams_import:hash']				= 'a874de040c2439d710d1b8e6c7bd59a0';
-$config['streams_import:unzip_folder']      = "uploads/default/files/temp_unzip/";
+$config['streams_import:hash']                      = 'a874de040c2439d710d1b8e6c7bd59a0';
+$config['streams_import:unzip_folder']              = "uploads/default/files/temp_unzip/";
+$config['streams_import:archives_folder']           = "uploads/default/files/archives/";
+$config['streams_import:anomalies_folder']          = "uploads/default/files/anomalies/";
+$config['streams_import:profiles_directory']        = "uploads/default/ftp/profiles/";

controllers/admin_logs.php

@@ -41,19 +41,24 @@ public function __construct()
 
 	}
 
-    public function index()
+    public function index($offset)
     {
         $extra = 
          array(
          'title'                => lang($this->namespace.':title:'.$this->section.':index'),
          'buttons' => array(
-	        array(
-	            'label'     => lang('global:delete'),
-	            'url'       => 'admin/'.$this->namespace.'/'.$this->section.'/delete/-entry_id-',
-	            'confirm'   => true
-	        ))
+            array(
+                'label'     => lang('global:delete'),
+                'url'       => 'admin/'.$this->namespace.'/'.$this->section.'/delete/-entry_id-',
+                'confirm'   => true
+            ),
+            array(
+                'label'     => lang('global:view'),
+                'url'       => 'admin/'.$this->namespace.'/'.$this->section.'/view/-entry_id-',
+                'confirm'   => false
+            ))
          );
-         echo $this->streams->cp->entries_table($this->section, $this->namespace, $pagination = null, $pagination_uri = null, $view_override = true, $extra);
+         echo $this->streams->cp->entries_table($this->section, $this->namespace, $pagination = "50", $pagination_uri = "admin/streams_import/logs", $view_override = true, $extra);
     }
 
     public function create()
@@ -68,6 +73,15 @@ public function edit ($id)
         redirect('admin/'.$this->namespace.'/'.$this->section);
     }
 
+    public function view ($id)
+    {
+        $query = $this->streams->entries->get_entry($id,$this->section, $this->namespace,false);
+
+        echo "<pre>";
+        var_dump($query);
+    }
+
+
     public function delete($id)
     {
         if($this->streams->entries->delete_entry($id, $this->section, $this->namespace)){

events.php

@@ -75,37 +75,42 @@ public function import_pics()
 	}
 
 
+	public function purge()
+	{
+
+
+		//Purge
+		//SELECT listings older than 30 days (updated OR created.) Limit it. It's really slow. maximum should be 50
+		$sql =" SELECT id FROM default_listing_homes WHERE (updated is null and (TO_DAYS(NOW()) - TO_DAYS(created)) > 30 ) OR ( (TO_DAYS(NOW()) - TO_DAYS(updated)) >30 ) LIMIT 10"; 
+
+		$entries = $this->CI->db->query($sql)->result();
+		foreach ($entries as $entry) {
+			$this->CI->streams->entries->delete_entry($entry->id, 'homes', 'listing');
+		}
+	}
 
 	public function import()
 	{
-
+die();
 		//Steps for automatic import
-		// 0- Purge
 		// 1- check in the config the path for the profiles.
 		// 2- Load all the profiles where "auto = true"
 		// 3- check if a file is awaiting for import process
 		// 4- Import it :)  
 
 
 
-		//Purge
 
-		$sql =" SELECT id FROM default_listing_homes WHERE (updated is null and (TO_DAYS(NOW()) - TO_DAYS(created)) > 30 ) OR ( (TO_DAYS(NOW()) - TO_DAYS(updated)) >30 ) LIMIT 60"; 
-		echo $sql;
-		$entries = $this->CI->db->query($sql)->result();
-		foreach ($entries as $entry) {
-			$this->CI->streams->entries->delete_entry($entry->id, 'homes', 'listing');
-		}
 
-die();
 
 
 		/*
 		* Set up our root directory to scan
 		*/
 
-		$base = UPLOAD_PATH.'ftp/profiles/';
+		$base = $this->CI->config->item('streams_import:profiles_directory');
 
+		$exit_at_the_end = false;
 
 
 		/*
@@ -116,29 +121,20 @@ public function import()
 		$profiles = $this->CI->streams->entries->get_entries(
 			array(
 				'stream' => 'profiles',
-				'namespace' => 'streams_import',
+				'namespace' => 'streams_import'
 				)
 			);
 
-		foreach ($profiles['entries'] as &$profile)
-		{
-			$profile = array(
-				'slug' => slugify($profile['profile_name'], ''),
-				'id' => $profile['id'],
-				);
-		}
-
-
-
 		/*
 		 * Scan
 		 */
 
 		foreach ($profiles['entries'] as $profile)
 		{
-			$path = $base.$profile['slug'];
+			$path = $base.$profile['profile_slug'].'/data/';
 
 
+var_dump($path);
 			// Does the folder exist?
 			if (is_dir($path))
 			{
@@ -160,39 +156,55 @@ public function import()
 
 
 				// Check for the achive folder
-				if (!is_dir($path.'/archive')) mkdir($path.'/archive');
+				$archive = $this->CI->config->item('streams_import:archives_folder').$profile['profile_slug'];
+
+				
+				if (!is_dir($this->CI->config->item('streams_import:archives_folder')))
+					{ mkdir($this->CI->config->item('streams_import:archives_folder')); }				
+				if (!is_dir($archive))
+					{ mkdir($archive); }
 
 
 				// Move the files to archive
 				foreach ($files as $file)
 				{
-
+					$exit_at_the_end = true;
 					// Try importing the file
-					if(import_file($path.'/'.$file,$file,$folder_id = 18,$owner = 17,$description = ''))	// $folder_id = 18 is "import" and $owner = 17 is "[email protected]"
+					if(import_file($path.$file,$file,$folder_id = 18,$owner = 17,$description = 'file to import'))	// $folder_id = 18 is "import" and $owner = 17 is "[email protected]"
 					{
+						//echo "success ".$path.$file;
 						##
 						## There needs to be a public import method
 						##
 						## Something like website.com/streams_import/import/PROFILE/FILE/SECURITY_HASH
 						##
 
 						// File ID
-						$file = $this->CI->db->select()->where('filename', $file)->limit(1)->get('files')->row(0);
+						$file_id = $this->CI->db->select()->where('filename', $file)->limit(1)->get('files')->row(0);
 
 						// Try this
-						echo file_get_contents(site_url('streams_import/import/'.$this->CI->config->item('streams_import:hash').'/'.$profile['id'].'/'.$file->id));die;
+						$hit = site_url('streams_import/import/'.$this->CI->config->item('streams_import:hash').'/'.$profile['id'].'/'.$file_id->id);
+						//var_dump($hit);
+						echo file_get_contents($hit);
 						//echo site_url('streams_import/import/'.$this->CI->config->item('streams_import:hash').'/'.$profile['id'].'/'.$file->id);die;
 
 						// Clean that snatch like a dirty carpet
-						rename($path.'/'.$file, $path.'/archive/'.now().'.'.$file);
+
+						rename($path.$file, $archive.'/'.now().'.'.$file);
 					}
 				}
+				if ($exit_at_the_end)
+				{
+					//We want to process one profile per time.
+					die();
+				}
 			}
 			else
 			{
+				//echo "ano";
 				// Who knows?
 				// Maybe we'll need it next time.
-				mkdir($path);
+				rename($path.$file, $archive.'/'.$file);
 			}
 		}
 	}

libraries/streams_import.php

@@ -169,6 +169,7 @@ public function process_import($profile_id, $file_id = 0)
 			}
 
 
+			$action = "INSERT "; // this is for log
 			if(!empty($update->id))
 			{
 				 $skips = array();
@@ -179,6 +180,7 @@ public function process_import($profile_id, $file_id = 0)
 					$skips[] = $slug;
 				}
 				//var_dump($insert_data);
+				$action = "UPDATE "; // this is for log
 
 				$this->ci->streams->entries->update_entry($update->id,$insert_data, $stream->stream_slug, $stream->stream_namespace,$skips, $extra = array());
 			}
@@ -187,6 +189,14 @@ public function process_import($profile_id, $file_id = 0)
 				continue;
 			}
 
+			//Log that shit.
+			$data_log = array();
+			foreach($insert_data as $key=>$value) {
+			  $data_log[] = $key.','.$value;
+			}
+			;
+			$this->enrich_log($log_id,$action.implode('|',$data_log));
+
 			//call the post process function
 			$post_process=$profile->profile_slug .'_'.$stream->stream_slug.'_'.'sim_postprocess';
 
@@ -438,6 +448,18 @@ public function register_logs($filename, $profile_id, $id=null)
 
 	}
 
+	public function enrich_log($id,$content){
+
+		$log = $this->ci->db->get_where('streams_import_logs', array('id'=>$id))->row();
+		$entry_data = array(
+			'log_detail'			=>  $log->log_detail.br(2).$content
+			);
+		$skips = array('filename','profile_rel_logs');
+		return $this->ci->streams->entries->update_entry($id, $entry_data = array(), 'logs', 'streams_import',$skips, $extra = array());
+		
+
+	}
+
 
 }