diff --git a/.gitignore b/.gitignore index 0d20b6487..91e2b8a7e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,136 @@ *.pyc +.DS_Store +src/rb/Monitoring/log/* +src/rb/Monitoring/public/images/rs*.png +run/monitoring/copy_of*.txt +run/monitoring/rs*.rrd +run/monitoring/*stats.txt +/src/log/sinatra.log +/src/rb/HTMonitoring/app/views/graphs.erb.old +/src/rb/HTMonitoring/app/views/test.erb.old +/src/rb/HTMonitoring/app/public/graphimages/b_c_accesses_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_accesses_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_accesses_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_accesses_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_hits_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_hits_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_hits_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_hits_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_max_mem_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_max_mem_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_max_mem_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/b_c_max_mem_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/bytes_read_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/bytes_read_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/bytes_read_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/bytes_read_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/bytes_written_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/bytes_written_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/bytes_written_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/bytes_written_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/cells_read_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/cells_read_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/cells_read_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/cells_read_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/cells_written_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/cells_written_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/cells_written_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/cells_written_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/clock_mhz_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/clock_mhz_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/clock_mhz_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/clock_mhz_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/cpu_pct_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/cpu_pct_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/cpu_pct_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/cpu_pct_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/loadavg_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/loadavg_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/loadavg_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/loadavg_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/mem_total_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/mem_total_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/mem_total_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/mem_total_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/mem_used_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/mem_used_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/mem_used_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/mem_used_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/net_recv_KBps_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/net_recv_KBps_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/net_recv_KBps_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/net_sent_KBps_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/net_sent_KBps_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/net_sent_KBps_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/net_sent_KBps_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/num_cores_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/num_cores_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/num_cores_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/num_cores_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/num_ranges_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/num_ranges_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/num_ranges_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/num_ranges_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/percent_mem_used_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/percent_mem_used_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/percent_mem_used_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/percent_mem_used_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/percent_query_cache_memory_used_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/percent_query_cache_memory_used_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/percent_query_cache_memory_used_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/percent_query_cache_memory_used_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_accesses_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_accesses_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_accesses_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_accesses_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_avail_mem_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_avail_mem_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_avail_mem_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_avail_mem_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_hits_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_hits_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_hits_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_hits_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_max_mem_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_max_mem_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_max_mem_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/q_c_max_mem_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/scans_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/scans_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/scans_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/scans_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/syncs_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/syncs_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/syncs_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/syncs_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/vm_resident_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/vm_resident_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/vm_resident_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/vm_resident_rs4.png +/src/rb/HTMonitoring/app/public/graphimages/vm_size_rs1.png +/src/rb/HTMonitoring/app/public/graphimages/vm_size_rs2.png +/src/rb/HTMonitoring/app/public/graphimages/vm_size_rs3.png +/src/rb/HTMonitoring/app/public/graphimages/vm_size_rs4.png +/src/rb/HTMonitoring/log/sinatra.log +/src/rb/HTMonitoring/app/config/#config.yml# +/src/rb/HTMonitoring/MonitoringServer.pid +/src/rb/HTMonitoring/app/views/#index.erb.old# +/src/rb/HTMonitoring/app/views/.#index.erb.old +/src/rb/HTMonitoring/app/views/index.erb.old +/src/rb/HTMonitoring/app/lib/graph_flymake.rb +/src/rb/Monitoring/app/views/#index.erb.old# +/src/rb/Monitoring/app/views/graphs.erb.old +/src/rb/Monitoring/app/views/index.erb.old +/src/rb/Monitoring/app/views/test.erb.old +/src/rb/Monitoring/app/config/#config.yml# +/src/rb/HTMonitoring/htmonitoring.rb +/src/rb/Monitoring/MonitoringServer.pid +/src/rb/Monitoring/app/config/thin.yml +/src/rb/Monitoring/app/views/semantic.cache +/src/rb/Monitoring/sinatra.log +/src/rb/Monitoring/app/lib/data/rrd_stat_flymake.rb +/src/cc/Hypertable/RangeServer/#MergeScanner.cc# +/src/rb/Monitoring/app/config/config.yml.in +/src/rb/Monitoring/test.log +/src/cc/Hypertable/Master/#RangeServerStatistics.h# +/src/rb/Monitoring/Monitoring diff --git a/CHANGES b/CHANGES index beae1b921..9278fafd0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,511 @@ +Version 0.9.5.0.pre: +(2011-03-24) + + Master overhaul + MetaLog overhaul + Asynchronous Scanner API + Upgraded to Thrift 0.6.0 + Upgraded to CDH3B4 + Added sys/RS_METRICS + Fixed bug in monitoring system that was calculating buggy Cell and Byte read/write rates. + Added METADATA-split master failover tests + Added MasterClient-TransparentFailover test; fix bugs that turned up + Added VERSION_MISC_SUFFIX to version string for 0.9.5.0.pre release. + Added delete_count to CellStoreV5Trailer which stores the number of delete records in the CS. + Added list of replaced files to CellStoreV5. + Fixed soft_limit regression + Fixed intermittent test failues due to exit(); Got rid of valgrind warnings + Added two-phase master requests + Fixed warnings + Upgraded version number to 0.9.5.0 + Cleaned up prune threshold limits; Got rid of warnings + Fixed deadlock in ResponseManager + Fixed data loss bug - made CommitLog close synchronous; fixed async scanner bug + [Issue 579] metalog backup verification causing intermittent test failures. Fixed + Added regression tests for stopping synchronous and asynchronous scanners abruptly before scan completes. + Fixed deadlock in TableScannerAsync code. + Added needs_compaction flag to RangeServer::load_range method. + [Issue 578] Deadlock in async scanner. Fixed + Added ht_master_client shell program with shutdown command + Fixed hyperspace-reconnect test + Fixed monitoring server initialization problem + Minor fix to dot jpg file generation. + added start_time,end_time as http query params + Close cell store file before removing directory + Create /run folder if required + Fixed a bunch of minor issues. + [Issue 577] RangeServer::commit_log_sync should respect group commit. Fixed + Added regression tests and bug fixes for Future API. + Performance and functional bug fixes to TableScanner class. + Implemented changes to C++ and Thrift clients to support asynchronous scanners. -TODO: add tests for Php and Python + Fixed a bug that was causing the METADATA-split-recovery test to fail intermittently. + issue 552: Ensure Hyperspace handles get closed; Naming cleanup + Fixed incorrect CellStoreTrailerV5 version check caught by assert + Added CellStoreV5; Monitoring system improvements (avg. key & value size) + added new column to table stats + Added file_count to StatsTable; Fixed compression ratio computation + Got rid of read_ids flag in Schema parse API + "changes to header labels" + Monitoring UI Changes, Sorting options for stats summary + Updated clean-database.sh script to reflect new rsml backup location + added invalidate methods for table name changes + bunch of changes to Monitoring UI changes (reading from json and got rid google graphs which gives summary) Added Ta + Fixes to monitoring & stats gathering + Fixed bugs caught by Andy Thalmann (ScanContext copy ctor bug) + added table names to json , removing unnecessary code + Fixed minor monitoring/stats gathering bugs + issue 563: fixed METADATA split test + bunch of changes to Monitoring UI changes (reading from json and got rid google graphs which gives summary) Added Ta + Added Hypertable.RangeServer.CellStore.SkipNotFound + issue 559: Prevent transfer log from getting linked in twice + issue 552: Ensure Hyperspace handles get closed; Naming cleanup + [Issue 505] Client-no-log-sync regression failure + issue 537: Fixed RangeServer shutdown hang + issue 542: Only write last-dfs file if it doesn't exist + issue 544: Set default RangeServer memory limit to 50% + issue 553: Schema HQL render wrap table name in quotes if necessary + issue 545: Reduced random-write-read test to 1/5 the size + [Issue 551] Upgraded to QuickLZ 1.5 + Fixed bugs related to MasterGc and live file tracking + Renamed MoveStart and MoveDone RSML operations to RelinquishStart and RelinquishDone. + Made changes to BalanceStarted/Done and RangeMoveLoaded/Acknowledged MML entries. + Added code for additional MML entries. + Changed MasterMetaLog to garbage collect entries during recovery. + Added MoveStart and MoveDone RSML entries. + Changed RangeServerMetaLog to garbage collect entries during recovery. + Fixed rare bug that caused root range corruption + issue 547: Found and fixed more race conditions + Fixed excessive maintenance scheduling during low memory condition + Fixed deadlocks uncovered recently + Fixed race cond in drop_table; Fixed RangeServer::update bug + Changed DfsBroker.Local.DirectIO default to false + Fixed some stats gathering issues discovered in sys/RS_METRICS + issue 531: Fixed bug in load_generator that caused intermittent drop of last cells + Added RangeServer::relinquish_range() (with RSML update code stubbed out) + Added Hyperspace::Session::open() method with no callback; code cleanup + Use "scanned" cells/bytes for load metrics; Added paging info to load metrics + Fixed recently introduced "bad ProxyName" problem + Do not reduce the limit below the minimum + Scan and filter rows fix and optimization + Config property "Hypertable.RangeServer.LowMemoryLimit.Percentage" has been added + Added disk_used, disk_estimate, and compression_ratio to StatsTable + Fixed revision number problem in CommitLogReader with link entries + Renamed Master::report_split() to Master::move_range() + Added MML + Fixed monitoring stats; fixed JSON output for RS summary + Monitoring overhaul part 1 + new rangeserver stats + changes to use new rangeserver summary data + Destroy comm has been fixed + HQL scan and filter rows option added + Logic changed to setup row intervals in case of scan and filter rows + Optimization for scan and filter rows + Scan and filter rows has been implemented (Issue 525) + check for negative resolution + issues with config + Added resolution param to rrd page + Fixed monitoring stats; fixed JSON output for RS summary + Monitoring overhaul part 1 + Check for zero-lengthed row and skip in LoadDataSource + Allow NULL strings to be passed into FlyweightString + Fixed bad memory reference in RangeServer::FillScanBlock + cleanup has been added + Added prepend_md5 tool + Redirect thrift output to HT logger + Assignment operator added + Support empty qualifier filtering + Recursive option added to the hyperspace readdirattr command + Recursivly option added to the hyperspace readdirattr command + Include sub entries for get_listing/NamespaceListing, readdir_attr/DirEntryAttr + Fixed syntax error recently intoduced into Ceph broker code + Added StrictHostKeyChecking=no to rsync + Added regexp filtering to DUMP TABLE command. + Added optimization for row and qualifier regex matching. + Added script to compare test runs times and detect potential performance regressions. + Cleaned up SELECT [CELLS] Hql command. + Removed DfsBroker.Host from default hypertable.cfg; Cleaned up DFS Port properties + Fixed bug in Hyperspace caused by Reactor thread directly calling BerkeleyDbFilesystem on disconnect. + Improved Master handling of already assigned location in register_server + Fixed performance regression in ScanContext by using set instead of hash_set for exact qualifiers + + +Version 0.9.4.3: +(2010-11-15) + + issue 534: Made RangeLocator thread-safe + MetaKeyBuilder crash with nested namespces fixed + Got rid of spurious "Connection refused" error messages + Decreased default bucket count to 20 for DUMP TABLE + Added group commit + Added regex filtering on ROW, COLUMN FAMILY, and VALUE + Modified Capfiles to start ThriftBroker on master + Added code to catch BAD HEADER LENGTH exception in IOHandlerDatagram + Fixed unprotected access to Range stats data + Changed regexp members of ScanSpec to use memory arena + Added more documentation about how to LOAD DATA INFILE from stdin + Added support for filtering by qualified column and remove restriction of HT rowkey==first col in Hi + Added the ability to set multiple column qualifier filters for a given column family. + Added retry logic to LiveFileTracker::update_file_column. -Added regression test, commented out for + issue 247: Added methods to RangeServerClient that accept Timer + Fixed bug in AsyncComm poll() mode + Evaluate --hyperspace host:port argument + Fixed problems in MetalogRangeServer and FileBlockCache tests (Andy Thalmann) + Fixed bug in SerializedCellsReader; Added more test scripts + Toplevel directory added + Added parallel INCR test to PerformanceTest; Increased ThriftBroker timeouts + Added instrumentation to group-commit tests; Fixed TestHarness + Lazy fix for issue 530 + Crash fixed (happens for MutatorNoLogSyncTest) + Close open file handles before delete file + Hypertable.DataDirectory (Hypertable data directory root) configuration parameter added to the Defau + Added RE2 dependency to PackageHelper + Added API support for row, value and column qualifier regexps. -Added API support for row, value and + Added dependency & cmake check for RE2 regex library. + Fixed error in CREATE NAMESPACE documentation + + +Version 0.9.4.2: +(2010-10-26) + + Upgraded to Thrift 0.5.0 + Added COUNTER type column family. + Fixed potential deadlock in periodic mutator code path. + Added missing rrd dependencies to packages + Pulled in Trent's fix for null values in TextTableInputFormat and InputFormat. + Support for cell flags (FLAG_INSERT, FLAG_DELETE_*) in thrift serialized cells + Make use of existing Common/endian-c.h + Read and set timeout for thriftbroker + Added optimization to increment CellCache counter in place + Updated PerformanceTest added support for increment + Fixed problem with IF EXISTS option to DROP NAMESPACE + Added clean task as dependency to jar task. + issue 518: Trim whitespace to avoid HqlParser error + issue 517: compilation problem with VERSION constant in CommHeader + issue 515: prevent PHP error reporting from being enabled unconditionally + Fixed ThriftBroker crash when opening bad namespace + Added resolution param to rrd page + + +Version 0.9.4.1: +(2010-09-23) + + issue 516: fixed Table::get_name() + Monitoring README fix. + shell script changes + Removed config.yml.in Changes to start script. + changes to config.yml + using thin server now, fixed config.yml data dir issue, added progress bar on graphs page + Modified Monitoring UI start/stop scripts. + Fixed bugs in Hive extension introduced by thrift 0.4.0 byte[] -> ByteBuffer change. + + +Version 0.9.4.0: +(2010-09-08) + + Added Namespaces + Added AG garbage collection logic for MAX_VERSIONS, deletes, and TTL + Hypertable Monitoring web interface using sinatra + Made changes to garbage collect unused Hyperspace BerkeleyDB logs. + Upgraded to Thrift 0.4.0 + issue 501: Made Capistrano "upgrade" task first verify that upgrade is OK + issue 440: Renamed CellFlag to KeyFlag; Renamed put_ methods to offer_ + issue 504: Store relative filenames in METADATA + Added RENAME TABLE feature. + issue 479: Added support for configurable toplevel directory name + issue 174: Changed all state file/dir names to include table id instead of name + Modified capfile to start and stop monitoring server on master. + Updated location of monitoring UI code. + Modified start and stop scripts for Monitoring UI + Made changes to print out ProxyMap to text file to be used for the Monitoring UI. + Fixed string encoding/decoding problem in Java code + Fixed stats gathering logic to handle variable length string table ID + issue 316: Moved CREATE TABLE options to the end after column defs + issue 487: Fixed Comm layer core dump with many client connections + issue 496: Fixed bad logic in EINTR handling + Changed namespace "SYS" to "sys" + issue 497: fixed problem of column qualifier not getting cleared in Thrift API + issue 11: Use shorter cellstore filenames. Fixed + Added hypertable shell regression test for namespaces. Updated some documentation. + Namespace Client API changes. + issue 174: Changed all state file/dir names to include table id instead of name + issue 234: Propagate better error messages from DfsBroker to RangeServer log + issue 106: Use JAVA_HOME to find java binary, if set + Got rid of verbose logging from recent AG garbage collection change - Also changed Hypertable.RangeServer.AccessGrou + issue 479: Added dual-instance regression test + Changed semantics of readdirattr and readpathattr Hyperspace commands + Added NameIdMapper class. + Made changes to store next table id in Hyperspace.. + Added Hyperspace readpathattr API to list the value of an attribute for all components in a path + Added Hyperspace attr_incr API to atomically increment an attribute. + [Issue 148] hql parser doesn't like quotes inside a string even when escaped. Fixed. + [Issue 481] ThriftClient-reconnect-hyperspace failure. Fixed. + [Issue 491] core dump at get_ts64() of Time.cc. Fixed + [Issue 488] Hyperspace crash. Fixed + [Issue 492] RangeServer crash. Fixed + Fixed bug in Master stats collection. + Made changes to checkpoint BDB every time there is 1M worth on uncheckpointed log data. + Minor fix in FindRRDtool.cmake + Don't link Hyperspace.Master to tcmalloc for 32-bit + +Version 0.9.3.4: +(2010-07-15) + + Switched to CDH3; Made Ant and Thrift required + Updated some auto-generated files for new version of thrift + Updated HqlHelpText with CELL_LIMIT feature info. + Don't link ThriftBroker with tcmalloc on 32-bit + Added support for newer versions of Boost + Minor fixes to monitoring code. + Added cell_limit feature to scan/select queries. + Fixed link problem with serverup on Mac OSX + Added dependent RRD libs to install + +Version 0.9.3.3: +(2010-06-23) + + Fixed bug in fhsize.sh; Added fhsize task to Capfiles - Load libthrift jar file to CLASSPATH first + Minor fixes to namespace violations and use Table name in place of ID for monitoring. + Fixes to mapred Input/Output formats + Added upgrade task to Capfiles + Made changes to monitoring system to print Table name as well as ID. + Hypertable-Hive connector. + More tuning based on perfeval test1 results + Changed ThriftBroker threading model to threaded instead of thread pool + + +Version 0.9.3.2: +(2010-06-12) + + Changed some defaults based on perf tests + Added range readahead in IntervalScanner - Report info about actual data read in Performance test driver + Upgraded to thrift-0.3.0-rc4 from http://people.apache.org/~bryanduxbury/thrift-0.3.0-rc4.tar.gz + Initial version of Monitoring application + Fixed Sigar library name for OSX. + Modified Thrift get_schema API to return Schema object and added get_schema_str API + Fixed TableSplit.java to lookup hostname for location since Hadoop doesn't understand IP addr + issue 356: Record DFS is run/last-dfs file and report error if mismatch + Fixed race condition in Master::get_statistics() + Fixed timeout problems with ThriftClient + Fixed compile problem in ThriftBroker; Added run-benchmark.sh script + increased transfer buffer + Added support for Zipf, report output file, etc. + Added rrd library to install; Fixed race condition in Client table caching + Increased interval scanner readhead buffer size 5x512K + Fixed bugs in ThriftBroker next_cells_serialized and next_row_serialized APIs + Added next_row_serialized API to ThriftBroker. + Added system wide stats like #cores, disk , mem utilization, load avg to monitoring stats. + Added code to blow away monitoring dir when clean-database.sh is run. + Added PerformanceTest benchmark framework - Switch ThriftBroker to TThreadPoolServer on linux - Added _serialized me + Added support for DIRECT I/O + [Issue 464] Add refresh_mutator API. Added refresh_shared_mutator API to Thrift interface + [Issue 86] Monitoring. + Added Master maintenance timer + Fixed circular dependency in build. + issue 414: added thriftbroker and spare Capistrano roles - Also included DfsBroker in the hypertable-thriftbroker pa + added the ability to specify start/end row with streaming fixed the handling of splits with regards to maps + Added Hypertable.RangeServer.Scanner.BufferSize + Set key_compression_scheme propery in CellStoreV3 trailer + issue 427: Added expiration_time and expirable_data to CellStoreTrailerV3 + issue 149: Added prefix key compression to CellStore (v3) + [Issue 259] LDI should be able to load data from DFS. Updated documentation. + + +Version 0.9.3.1: +(2010-04-14) + + added support for map reduce streaming + Fixed prune_tsv to work with DUMP TABLE format, added regression test + issue 449: Remove fhsize from RPM and DEB post-install + [Issue 448] Hypertable shell trying to connect to DFSBroker. Fixed. + issue 439: Strip duplicate cells during scan + issue 443: Check for invalid ':' character in column family names + issue 442: Escape/unescape row key and column qualifier + Added Dfs support to DUMP TABLE command. + [Issue 416] Commands taking long time to run. Fixed + [Issue 420] Incorrect checking of errno after calling FileUtils:: function. Fixed + issue 436: Fixed problem causing full table scan for each split + issue 408: Fixed problem where scanner didn't propagate revision number + issue 430: Fixed STL comparator semantics + issue 385: Made REPLICATION factor configurable per-table and access group + [Issue 175] HQL parser getting confused on INSERT of 4-digit row key. Fixed. + issue 124: Documented LDI options IGNORE_UNKNOWN_COLUMNS and DUPLICATE_KEY_COLUMNS + issue 260: Display useful error message when LDI HEADER_FILE not found + issue 419: Fixed race condition in Comm::connect HandlerMap access + issue 421: Fixed Free Memory Read bug in ProxyMap + [Issue 395] Client-no-log-sync test fails intermittently. Fixed + [Issue 381] HqlInterpreter::execute(...) pass reference to output parameter. + issue 378: Added escaping of '\' in addition to \n and \t + issue 423: Fixed MetaLogDfsBase::get_filename() + issue 424: Added destructor to MetaLogReaderDfsBase to close file handle + issue 402: Lazily load CellStore block index for faster startup time + Fixed warnings on Linux x86_64 gcc 4.1.2 + [Issue 259] Checked in changes to allow LDI and SELECT to read/write from/to DFS. + issue 136: Fixed bug in DfsBroker ClientBufferReaderHandler + issue 403: fixed Coverity bugs + issue 417: Allow quoted table names in INSERT command - Added regression test (verifies issue 34 as well) + issue 34: Fixed bug in ScanContext causing minimum timestamp to be 1970-01-01 + issue 422: Fixed bad memory reference after copying Managed objects + Fixed minor (innocuous) problems in Comm layer + issue 411: fixed priority_queue less comparator + issue 410: fixed reverse proxy map invalidation + Fixed double file handle close in HQL interpreter + issue 406: Check iterator against appropriate map structure + issue 404: close file handle in FileUtils::file_to_buffer + Fixed FindThrift.cmake to look in /usr/lib64 + Added libstacktrace to package + Fixed Capfile rsync_installation to work with FHSized install + + +Version 0.9.3.0: +(2010-03-22) + + Added Hyperspace replication + Added MapReduce connector + Added Query Cache + FreeBSD port + Solaris port + Added DUMP TABLE + Added LOAD DATA FORMAT input file format auto-detect + Fixed memory leak + Added shadow CellCache + Added poll() support to Comm layer for Linux 2.4 kernels + Exclude CellStores that are outside of scan spec timestamp range + Improved efficiency of KeyWritable; added comments + Fixed race condition with proxy map propagation + Updated htbuild + Added missing ssl library to CephBroker build + Added --thrift flag to ht_load_generator tool to test loads via ThriftBroker. + Changed PerformanceEvaluation tool to generate random buffer once and re-use it to lower CPU load + Increased ThriftClient timeout to 10min for MR PerformanceEvaluation task + Changed Hyperspace lock operation result log from debug to info. + Fixed recursive includes in hypertable jar files that were bloating the file size + Modified path for Java Client examples and build.xml paths for examples jar file + Added Hypertable PerformanceEvaluation class. + Fixed duplicate server location assignment problem + Added Key class to Thrift, make Cell a composition of Key and value + Plumbed the Client::get_table_splits API through the ThriftBroker + Checked in OutputFormat for Hypertable MapReduce connector. + Added 'exists_table' API to Hypertable client, Thrift interface and HQL. + Added INSTALL_EXCLUDE_DEPENDENT_LIBS cmake variable + More FreeBSD porting work + Turned off ShadowCache by default + Made poll error handling more graceful (return error instead of exit) + Added Hypertable.RangeServer.AccessGroup.ShadowCache property to enable/disable shadow cache + Workaround for xen timestamp problem; Fixed Master deadlocks + Fixed memory leak in maintenance queue + Fixed deadlock in Master + Added VERSION_ADD_COMMIT_SUFFIX cmake variable + Added address proxy + Got rid of boost spirit warnings + Fix to allow 0 column family for retured cells that are DELETE_ROW + Auto-adjust blockcache size based on workload - Maintenance scheduler re-work + Fixed progress meter for large load files and generated data sets + Fixed --max-keys handling for ht_load_generator - Also respect min_pool and max_pool for Uniform random number generator + Got bloom filter working again - Added new CellStoreV2 format with all necessary bloom filter state in trailer - Added --num-hashes and --bits-per-item bloom filter options + Cleaned up FailureInducer interface + Fixed METADATA split recovery race conditions + Cleaned up regression tests; Updated documentation + Made Master::server_left() close connection to avoid race condition + [Issue 388] Fixed issue by checking for session existence within BDB txns. + Setting replication site priority according to order in which replicas are listed in config file + Fixed bug in Hyperspace logic which was causing Hyperspace to delete a handle for an expired session. + Made changes to avoid duplicate masters among BDB replication sites. + Modified Hyperspace replication logic to wait for permanent message acks from all replicas. + Modified Capfiles with new hyperspace replication stuff + [Issue 372] Minor cleanup, changed error message to info. -This is an expected case and not really an error + Upgrading BerkeleyDB requirement to >= 4.8.x + Replaced HT_EXPECTs in BerkeleyDbFilesystem code with HT_ASSERTs + [Issue 372] Added scope guards to prevent BerkeleyDB cursor leaks. -Also changed a few asserts to HT_ASSERT + Made changes to allow Hyperspace clients to connect to any Hyperspace replica and get redirected to master. + Made changes to BerkeleyDbFilesystem so that each txn opens its own DB handle to avoid DB_REP_HANDLE_DEAD errors. + Changes to implement basic replication + Fixed problem preventing CellStore indexes from getting GC'd + Leave AccessGroup in consistent state after compaction failure + Fixed Memory Corruption in MetaLogDfsBase + Fixed infinite loop on ROW_OVERFLOW + Fixed deadlock + Apply MAX_VERSIONS unconditionally + Added RSML backups in $INSTALL/run/rsml_backup/ + Fixed problem with "LOAD DATA INFILE" from STDIN where incorrect progress bar update was the load down. + Updated version number to 0.9.2.8 + Fix for Hyperspace race condition; Added compaction asserts + Fixed delayed connect problem + Updated system tests; Increased BlockCache default size to 150M + Default dfsclient timeout to Hypertable.Request.Timeout + Fixed memory leak; Added support for tcmalloc HEAPCHECK + Added shadow CellCache; Re-wrote maintenance scheduler + Allow .tsv files with empty values + Issue 358: be a little smarter about default config file path. + Fixed problem with soname.sh and libsigar + Workaround for clock()/poll() incompatibility + Added support for cmake variable HT_DEPENDENCY_DIR + Modified BerkeleyDb search path to look for 4.8 release if available + Changing BDB_DEADLOCK messages to info instead of warning. + Added keys_only to Thrift ScanSpec interface + Issue 361: Allow RangeServer to reconnect with Master after connection reset + Upgraded to Thrift 0.2.0 + Added put_cell* API to Thrift interface to give easy access to shared periodic flushing mutators + Removing BDB deadlock error messages. Use warnings instead. + Fixed compile error on mac + Updated htbuild for thrift r830673, kfs 0.4 and ceph 0.17 - Also added fix for python thrift installation on ubuntu + Made Cells and ScanSpec fully take advantage of custom allocator - Made clear() semantics for the containers consistent as well. + Consolidate pool allocators and their usage. + Updated doxygen doc version + Cleaned up some compiler warnings under gcc 4.3. + Fixed an fmr for valgrind + Added some test helper functions/classes/templates. + Made property accessors const where appropriate - also tweaked the default config file to use relative path if possible. + Temporarily added back thrift perl dependencies to the rpm package - probably better off zip the perl examples in the distribution. + Added thrift version ht -v + Added help/usage info for the ht wrapper + Added basic regression test init_* function. - Caught int32_t masking issues on Solaris + htbuild: update for fedora 11 + Fixed unknown option error from serverup for non-default options. + Fixed hql result logging in ThriftBroker + Upgraded thrift to r830673 to fix stack smashing in ThriftBroker - added slf4j jars for the new thrift - added java-thrift.sh for easy launching jhava thrift clients + Added support for Hypertable.Network.Interface config property + Added tsv-format and column-id-map options to csdump + Added Query Cache - Added .values property to load generator spec file to indicate number of distinct values to generate + Fixed RangeServer::load_range RSML write error handling to leave system consistent + [Issue 355] Fixed and added regression for LDI timestamp issue. + Fixed load_generator to support MaxKeys for load type 'update' + Added CellCacheScanner entry cache - Also fixed some build issues on Snow Leopard + Fixed problem where load_generator not respecting MaxBytes for query + Fixed TableMutator::set_cells to throw BAD_KEY exception for cf/cell deletes without a column family. + Replaced sleep with perl select() command in random-wait.sh + Fixed bug in ConnectionManager::remove - Fixed broken regressions + Fixed trailing '\n' problem in prune_tsv + Fixed CLOSED_WAIT problem with serverup and HdfsBroker + [Issue 340] Added Thrift MutatorFlag IGNORE_UNKNOWN_CFS. + Added "--exec/-e" and "--command-file" switches to CommandShell to execute commands. + Fixed memory corruption issue triggered when the interval mutator fails on successive writes. + Added --newer switch to prune_tsv + Fixed bug in Schema rename method + Fixed SIGAR library detection on mac Snow Leopard + Fixed connection/retry problems surfaced on Solaris + Fixed int32_t overloading issue on Solaris i386. + Change all tests/integration scripts to explicitly use bash + Worked around RAND_MAX quirk on Solaris + Worked around quirks in cmake on Solaris + Added table name to error message and modified Table & TableIdentifier classes to expose name. + [Issue 340] Added "ALTER TABLE RENAME COLUMN FAMILY" feature to rename column family names. + [Issue 340] Added ignore unknown cfs feature to LDI. + Changed Hypertable client Hyperspace connection check interval to 5s instead of 3s. + Fixed CellStoreScanner to turn off readahead mode in case of single row scans. + Commited changes to allow a client to automatically refresh stale schema during certain ops. + [Issue 339] Added instrumentation to debug this issue and fixed empty bloom filter problem. + Got rid of Hypertable.Request.Timeout from default hypertable.cfg + Got Master::close() plumbed through properly - Got rid of some warnings in Hyperspace - Made performance improvements to prune_tsv + Commited changes to allow a client to automatically refresh stale schema during certain ops. + Added flag to allow Sessions to reconnect to Hyperspace instead of expiring. + Major Hyperspace overhaul to store Session, Handle, Node and Event data mostly in BerkeleyDb. + Added prune_tsv tool + Fixed sequential-load test script + htbuild: exit ASAP in case of errors + Fixed tcmalloc version check (should run every time) + Fixed FindCeph.cmake for the new location of libceph.h in 0.14 + Updated boost to 1.40 in package build - added ntpdate to htbuild to avoid make timestamp issues - fixed tcmalloc build problems introduced by last refactor + Version 0.9.2.7: (2009-09-18) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1237b7c2b..ab2f39895 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,8 +22,9 @@ project(HYPERTABLE) # Hypertable version components set(VERSION_MAJOR "0") set(VERSION_MINOR "9") -set(VERSION_MICRO "2") -set(VERSION_PATCH "7") +set(VERSION_MICRO "5") +set(VERSION_PATCH "0") +set(VERSION_MISC_SUFFIX "pre") # cmake behavior compatibility cmake_minimum_required(VERSION 2.6.0) @@ -50,6 +51,11 @@ if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) CACHE PATH "installation directory prefix" FORCE) endif () +if (HT_DEPENDENCY_DIR) + set(HT_DEPENDENCY_LIB_DIR ${HT_DEPENDENCY_DIR}/lib) + set(HT_DEPENDENCY_INCLUDE_DIR ${HT_DEPENDENCY_DIR}/include) +endif () + # Enable testing enable_testing() @@ -58,7 +64,11 @@ set(TEST_DIFF ${HYPERTABLE_SOURCE_DIR}/bin/src-utils/testdiff) # gcc warning settings if (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) - add_definitions(-Wall -pedantic -Wno-long-long) + if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") + add_definitions(-Wall -pedantic -Wno-long-long -pthreads) + else (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") + add_definitions(-Wall -pedantic -Wno-long-long) + endif (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") endif () add_definitions(-D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -D_REENTRANT) @@ -81,29 +91,36 @@ find_package(Threads REQUIRED) message(STATUS "Use thread library: ${CMAKE_THREAD_LIBS_INIT}") find_package(Boost REQUIRED) find_package(BerkeleyDB REQUIRED) +find_package(RRDtool REQUIRED) find_package(EXPAT REQUIRED) find_package(Log4cpp REQUIRED) find_package(Readline REQUIRED) find_package(SIGAR REQUIRED) find_package(ZLIB REQUIRED) find_package(BZip2 REQUIRED) +find_package(RE2 REQUIRED) find_package(Doxygen) find_package(Tcmalloc) +find_package(Hoard) find_package(Kfs) find_package(Ceph) -find_package(Ant) +find_package(Ant REQUIRED) find_package(JNI) find_package(PythonLibs) -find_package(LibEvent) -find_package(Thrift) +find_package(LibEvent REQUIRED) +find_package(Thrift REQUIRED) find_package(RubyThrift) find_package(PHP5Thrift) find_package(PerlThrift) find_package(PythonThrift) -# set malloc library except in Debug mode -if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - set(MALLOC_LIBRARY ${Tcmalloc_LIBRARIES}) +# set malloc library (maybe) +if (Hoard_FOUND AND USE_HOARD) + set(MALLOC_LIBRARY ${Hoard_LIBRARIES}) +elseif (NOT USE_GLIBC_MALLOC) + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" OR USE_TCMALLOC) + set(MALLOC_LIBRARY ${Tcmalloc_LIBRARIES}) + endif () endif () # include directories @@ -139,7 +156,7 @@ include(DocHelper) # sub-project directories add_subdirectory(src) -add_subdirectory(examples) +#add_subdirectory(examples) add_subdirectory(contrib) add_subdirectory(tests/integration) @@ -155,13 +172,16 @@ configure_file(bin/prerm.in prerm @ONLY) file(GLOB CONFS conf/[^.]*) install(FILES ${CONFS} DESTINATION conf) install(PROGRAMS ${HYPERTABLE_BINARY_DIR}/ht DESTINATION bin) +file(GLOB JARS lib/*.jar lib/*.txt) +install(FILES ${JARS} DESTINATION lib/java) +install(FILES ${HYPERTABLE_BINARY_DIR}/java/hypertable-${VERSION}.jar + DESTINATION lib/java OPTIONAL) +install(PROGRAMS bin/jrun DESTINATION bin) if (NOT HT_COMPONENT_INSTALL) - file(GLOB JARS lib/*.jar lib/*.txt) - install(FILES ${JARS} DESTINATION lib/java) - install(FILES ${HYPERTABLE_BINARY_DIR}/java/hypertable-${VERSION}.jar + install(FILES ${HYPERTABLE_BINARY_DIR}/java/hypertable-${VERSION}-examples.jar DESTINATION lib/java OPTIONAL) - file(GLOB SCRIPTS bin/jrun bin/*.sh) + file(GLOB SCRIPTS bin/*.sh) install(PROGRAMS ${SCRIPTS} DESTINATION bin) install(DIRECTORY examples/apache_log DESTINATION examples USE_SOURCE_PERMISSIONS) @@ -170,7 +190,7 @@ if (NOT HT_COMPONENT_INSTALL) install(DIRECTORY examples/freebase DESTINATION examples USE_SOURCE_PERMISSIONS) - include(PackageHelper) +include(PackageHelper) else () install(PROGRAMS bin/ht-env.sh bin/fhsize.sh bin/prerm.sh DESTINATION bin) endif () diff --git a/README.md b/README.md index 3a31317d6..b4fc7e334 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,11 @@ HOW TO BUILD FROM SOURCE cmake -DCMAKE_BUILD_TYPE=Release ~/src/hypertable + If you would like to install the build in a directory that contains + a version suffix (e.g. 0.9.3.0.1d45f8d), you can configure as follows: + + cmake -DCMAKE_BUILD_TYPE=Release -DVERSION_ADD_COMMIT_SUFFIX=1 ~/src/hypertable + Note, you can also use: ccmake ~/src/hypertable @@ -72,8 +77,7 @@ HOW TO BUILD FROM SOURCE Since PHP has no builtin package system, its thrift installation needs to be manually specified for ThriftBroker support: - - cmake -DPHPTHRIFT_ROOT=~/thrift/lib/php/src ~/src/hypertable + cmake -DPHPTHRIFT_ROOT:STRING=/home/user/src/thrift/lib/php/src 4. Build Hypertable binaries. diff --git a/bin/clean-database.sh b/bin/clean-database.sh index 056b2bec8..9fc7997ea 100755 --- a/bin/clean-database.sh +++ b/bin/clean-database.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2008 Doug Judd (Zvents, Inc.) # @@ -19,6 +19,10 @@ export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) . $HYPERTABLE_HOME/bin/ht-env.sh +if [ "e$RUNTIME_ROOT" == "e" ]; then + RUNTIME_ROOT=$HYPERTABLE_HOME +fi + confirm=y if tty > /dev/null && [ $# == 1 ]; then @@ -45,20 +49,28 @@ case $confirm in # # Clear state # - check_server dfsbroker "$@" + check_server "$@" dfsbroker if [ $? != 0 ] ; then echo "ERROR: DfsBroker not running, database not cleaned" # remove local stuff anyway. - rm -rf $HYPERTABLE_HOME/hyperspace/* $HYPERTABLE_HOME/fs/* + rm -rf $RUNTIME_ROOT/hyperspace/* $RUNTIME_ROOT/fs/* $RUNTIME_ROOT/run/log_backup/rsml/* exit 1 fi - $HYPERTABLE_HOME/bin/dfsclient --eval "rmdir /hypertable/servers" "$@" - $HYPERTABLE_HOME/bin/dfsclient --eval "rmdir /hypertable/tables" "$@" - echo "Removed /hypertable/servers in DFS" - echo "Removed /hypertable/tables in DFS" - /bin/rm -rf $HYPERTABLE_HOME/hyperspace/* + TOPLEVEL="/"`$HYPERTABLE_HOME/bin/get_property $@ Hypertable.Directory`"/" + TOPLEVEL=`echo $TOPLEVEL | tr -s "/" | sed 's/.$//g'` + + $HYPERTABLE_HOME/bin/dfsclient --timeout 60000 --eval "rmdir $TOPLEVEL/servers" "$@" + $HYPERTABLE_HOME/bin/dfsclient --timeout 60000 --eval "rmdir $TOPLEVEL/tables" "$@" + echo "Removed $TOPLEVEL/servers in DFS" + echo "Removed $TOPLEVEL/tables in DFS" + /bin/rm -rf $RUNTIME_ROOT/hyperspace/* + /bin/rm -rf $RUNTIME_ROOT/run/log_backup/rsml/* + /bin/rm -rf $RUNTIME_ROOT/run/location + /bin/rm -rf $RUNTIME_ROOT/run/last-dfs echo "Cleared hyperspace" + /bin/rm -rf $RUNTIME_ROOT/run/monitoring/* + echo "Cleared monitoring data" ;; *) echo "Database not cleared";; esac diff --git a/bin/clean-hyperspace.sh b/bin/clean-hyperspace.sh new file mode 100755 index 000000000..30378e6eb --- /dev/null +++ b/bin/clean-hyperspace.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Copyright 2010 Sanjit Jhala(Hypertable Inc.) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The installation directory +export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) +. $HYPERTABLE_HOME/bin/ht-env.sh + +confirm=y + +if tty > /dev/null && [ $# == 1 ]; then + case $1 in + -i|--interactive) + echo "Are you sure you want to clear the database (on default ports)?" + echo "Will proceed in 10 seconds, (Ctrl-C to quit)" + shift + sleep 10 + ;; + esac +fi + +# Stop servers other than dfsbroker +stop_server hyperspace +sleep 1 +wait_for_server_shutdown hyperspace "hyperspace" "$@" & + +case $confirm in + y|Y) + # + # Clear state + # + /bin/rm -rf $HYPERTABLE_HOME/hyperspace/* + echo "Cleared hyperspace" + ;; + *) echo "Hyperspace not cleared";; +esac + +wait diff --git a/bin/fhsize.sh b/bin/fhsize.sh index ee980ac64..c27c2bcf9 100755 --- a/bin/fhsize.sh +++ b/bin/fhsize.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # @@ -21,33 +21,37 @@ # Post install script to FHSize the running layout according FHS # currently FHS 2.3: http://www.pathname.com/fhs/pub/fhs-2.3.html +set -e + export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) version=`basename $HYPERTABLE_HOME` # Variable/runtime data in /var/opt -varhome=/var/opt/hypertable/$version -varlink=/var/opt/hypertable/current +varhome=/var/opt/hypertable echo "Setting up $varhome" mkdir -p $varhome/hyperspace $varhome/fs $varhome/run $varhome/log -rm -f $varlink && ln -s $varhome $varlink -rm -f $HYPERTABLE_HOME/{hyperspace,fs,run,log} && +rm -rf $HYPERTABLE_HOME/{hyperspace,fs,run,log} && ln -s $varhome/{hyperspace,fs,run,log} $HYPERTABLE_HOME # Config files in /etc/opt -etchome=/etc/opt/hypertable/$version -etclink=/etc/opt/hypertable/current +etchome=/etc/opt/hypertable echo "Setting up $etchome" mkdir -p $etchome -cp $HYPERTABLE_HOME/conf/* $etchome && - rm -rf $HYPERTABLE_HOME/conf && ln -s $etchome $HYPERTABLE_HOME/conf -rm -f $etclink && ln -s $etchome $etclink +nfiles=`ls -1 $etchome | wc -l` +if [ $nfiles -eq 0 ] ; then + cp $HYPERTABLE_HOME/conf/* $etchome +else + cp $HYPERTABLE_HOME/conf/METADATA.xml $etchome + cp $HYPERTABLE_HOME/conf/RS_METRICS.xml $etchome +fi +rm -rf $HYPERTABLE_HOME/conf && ln -s $etchome $HYPERTABLE_HOME/conf echo "" echo "To run Hypertable under user , do something like the following:" -echo "sudo chown -R $varhome" +echo "sudo chown -R $varhome $etchome" echo "" echo "For log rotation, install cronolog and (re)start Hypertable." diff --git a/bin/ht-env.sh b/bin/ht-env.sh index b39097dc4..198d90205 100755 --- a/bin/ht-env.sh +++ b/bin/ht-env.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # @@ -20,6 +20,11 @@ ulimit -c unlimited +if [ "e$RUNTIME_ROOT" == "e" ]; then + RUNTIME_ROOT=$HYPERTABLE_HOME +fi + + die() { echo "$@" exit 1 @@ -27,12 +32,14 @@ die() { server_pidfile() { case $1 in - hyperspace) echo $HYPERTABLE_HOME/run/Hyperspace.pid;; - dfsbroker) echo $HYPERTABLE_HOME/run/DfsBroker.*.pid;; - master) echo $HYPERTABLE_HOME/run/Hypertable.Master.pid;; - rangeserver) echo $HYPERTABLE_HOME/run/Hypertable.RangeServer.pid;; - thriftbroker) echo $HYPERTABLE_HOME/run/ThriftBroker.pid;; - *) echo "ERROR: unknown service: $1"; return 1 + hyperspace) echo $RUNTIME_ROOT/run/Hyperspace.pid;; + dfsbroker) echo $RUNTIME_ROOT/run/DfsBroker.*.pid | grep -v "*";; + master) echo $RUNTIME_ROOT/run/Hypertable.Master.pid;; + rangeserver) echo $RUNTIME_ROOT/run/Hypertable.RangeServer.pid;; + thriftbroker) echo $RUNTIME_ROOT/run/ThriftBroker.pid;; + testclient) echo $RUNTIME_ROOT/run/Hypertable.TestClient*.pid | grep -v "*";; + testdispatcher) echo $RUNTIME_ROOT/run/Hypertable.TestDispatcher.pid;; + *) echo "unknown"; echo "ERROR: unknown service: $1" >&2; return 1 esac } @@ -45,29 +52,32 @@ check_pidfile() { [ $? != 0 ] && return 1 echo "$service appears to be running ($pid):" else - name=`basename $f` - ret=`ps -ef | grep -v grep | grep $name` - [ $? != 0 ] && return 1 - echo "$service seems to be running:" + return 1 fi echo $ret done } +kill_from_pidfiles() { + for i do + if [ -f "$i" ] ; then + pid=`cat $i` + echo "Killing `basename $i` $pid" + kill -9 $pid + rm $i + fi + done +} + stop_server() { pidre='.pid$' for server in $@; do if [[ $server =~ $pidre ]]; then - pidfile=$server + pidfiles=$server else - pidfile=`server_pidfile $server` - fi - if [ -f $pidfile ] ; then - pid=`cat $pidfile` - echo "Killing `basename $pidfile` $pid" - kill -9 $pid - rm $pidfile + pidfiles=`server_pidfile $server` fi + kill_from_pidfiles $pidfiles done } @@ -93,17 +103,17 @@ show_success() { } check_server() { - $HYPERTABLE_HOME/bin/serverup --silent "$@" + $HYPERTABLE_HOME/bin/serverup --silent "$@" >& /dev/null } wait_for_server() { become=$1; shift server=$1; shift server_desc=$1; shift - max_retries=${max_retries:-20} - report_interval=${report_interval:-3} + max_retries=${max_retries:-40} + report_interval=${report_interval:-5} - check_server $server "$@" + check_server "$@" $server ret=$? retries=0 while should_wait $ret "$become" && [ $retries -lt $max_retries ]; do @@ -111,7 +121,7 @@ wait_for_server() { let report=retries%$report_interval [ $report == 0 ] && echo "Waiting for $server_desc to $become..." sleep 1 - check_server $server "$@" + check_server "$@" $server ret=$? done if should_wait $ret "$become"; then @@ -133,12 +143,12 @@ wait_for_server_shutdown() { } set_start_vars() { - pidfile=$HYPERTABLE_HOME/run/$1.pid - logfile=$HYPERTABLE_HOME/log/$1.log + pidfile=$RUNTIME_ROOT/run/$1.pid + logfile=$RUNTIME_ROOT/log/$1.log startlog=/tmp/start-$1$$.log if type cronolog > /dev/null 2>&1; then logger="cronolog --link $logfile \ - $HYPERTABLE_HOME/log/archive/%Y-%m/%d/$1.log" + $RUNTIME_ROOT/log/archive/%Y-%m/%d/$1.log" else logger= fi @@ -153,10 +163,10 @@ check_startlog() { exec_server() { servercmd=$1; shift if [ "$logger" ]; then - ($VALGRIND $HYPERTABLE_HOME/bin/$servercmd --pidfile $pidfile "$@" 2>&1 | + ($HEAPCHECK $VALGRIND $HYPERTABLE_HOME/bin/$servercmd --pidfile $pidfile "$@" 2>&1 | $logger) &> $startlog & else - $VALGRIND $HYPERTABLE_HOME/bin/$servercmd --pidfile $pidfile "$@" \ + $HEAPCHECK $VALGRIND $HYPERTABLE_HOME/bin/$servercmd --pidfile $pidfile "$@" \ &> $logfile & fi } @@ -168,24 +178,33 @@ start_server() { set_start_vars $pidname check_pidfile $pidfile && return 0 - check_server $server "$@" + check_server "$@" $server if [ $? != 0 ] ; then exec_server $servercmd --verbose "$@" - wait_for_server_up $server "$pidname" "$@" + wait_for_server_up $server "$pidname" $@ else echo "WARNING: $pidname already running." fi } +start_server_no_check() { + server=$1; shift + servercmd=$1; shift + pidname=$1; shift + set_start_vars $pidname + check_pidfile $pidfile && return 0 + exec_server $servercmd --verbose "$@" +} + # Sanity check [ "$HYPERTABLE_HOME" ] || die "ERROR: HYPERTABLE_HOME is not set" -versionre='/([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+|current)$' +versionre='/([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(\.([a-fA-F0-9]+|pre))?|current)$' [[ $HYPERTABLE_HOME =~ $versionre ]] || - die "ERROR: Invalid HYPERTABLE_HOME: $HYPERTABLE_HOME" + die "ERROR: Invalid HYPERTABLE_HOME: $HYPERTABLE_HOME doesnt match regex ${versionre}" # Make sure log and run directories exist -[ -d $HYPERTABLE_HOME/run ] || mkdir $HYPERTABLE_HOME/run -[ -d $HYPERTABLE_HOME/log ] || mkdir $HYPERTABLE_HOME/log +[ -d $RUNTIME_ROOT/run ] || mkdir $RUNTIME_ROOT/run +[ -d $RUNTIME_ROOT/log ] || mkdir $RUNTIME_ROOT/log # Runtime libraries export LD_LIBRARY_PATH="$HYPERTABLE_HOME/lib:$LD_LIBRARY_PATH" diff --git a/bin/ht-monitoring.sh b/bin/ht-monitoring.sh new file mode 100644 index 000000000..495b13bc4 --- /dev/null +++ b/bin/ht-monitoring.sh @@ -0,0 +1,21 @@ +export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) + +port=38090 +pidfile="$HYPERTABLE_HOME/run/MonitoringServer.pid" +log="$HYPERTABLE_HOME/log/MonitoringServer.log" +monitoring_script_dir="$HYPERTABLE_HOME/Monitoring" +config_file="${monitoring_script_dir}/app/config/config.yml" +rack_file="${monitoring_script_dir}/config.ru" + +start_monitoring() { + cd ${monitoring_script_dir} + command="thin -p ${port} -e production -P ${pidfile} -l ${log} -R ${rack_file} -d start" + $command +} + +stop_monitoring() { + cd ${monitoring_script_dir} + command="thin -p ${port} -e production -P ${pidfile} -l ${log} -R ${rack_file} -d stop" + $command +} + diff --git a/bin/ht.in b/bin/ht.in index 407994202..17a4d8e72 100755 --- a/bin/ht.in +++ b/bin/ht.in @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # @@ -20,6 +20,26 @@ export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) . $HYPERTABLE_HOME/bin/ht-env.sh +print_help() { + echo "Usage: $0 [Command] [Options]" + echo "Commands:" + echo " start [ServiceOptions] See start --help for" + echo " for service options" + echo " valid : all-servers, dfsbroker, hyperspace, master" + echo " rangeserver, thriftbroker, test-servers" + echo " restart [ServiceOptions] Restart " + echo " valid : dfsbroker, hyperspace, master, rangeserver, " + echo " thriftbroker" + echo " stop Stop any running " + echo " valid : dfsbroker, hyperspace, master, rangeserver, " + echo " thriftbroker, servers" + echo " clean-databases Remove all existing databases" + echo " ldd Print dynamic linking info" + echo " shell [ShellOptions] See shell --help for options" + echo " version Show version info" + echo " help Show this help" +} + do_start() { service=$1; shift exec $HYPERTABLE_HOME/bin/start-$service.sh "$@" @@ -48,11 +68,13 @@ do_restart() { do_cmd() { cmd=$1; shift - [ "$cmd" ] || exit + [ "$cmd" ] || { print_help; exit; } if [ -x $HYPERTABLE_HOME/bin/$cmd ]; then exec $HYPERTABLE_HOME/bin/$cmd "$@" elif [ -x $HYPERTABLE_HOME/bin/$cmd.sh ]; then exec $HYPERTABLE_HOME/bin/$cmd.sh "$@" + elif [ -x $HYPERTABLE_HOME/bin/ht_$cmd ]; then + exec $HYPERTABLE_HOME/bin/ht_$cmd "$@" fi exec $cmd "$@"; } @@ -69,9 +91,10 @@ print_version() { fi echo " BerkeleyDB:" @BDB_VERSION@ echo " Boost:" @BOOST_VERSION@ - [ "@Tcmalloc_LIBRARIES@" ] && echo " Tcmalloc:" @TC_VERSION@ - [ "@JAVAC_VERSION@" ] && echo " Javac:" @JAVAC_VERSION@ - [ "@GCC_VERSION@" ] && echo " GCC/G++:" @GCC_VERSION@ + echo " Thrift:" @Thrift_VERSION@ + [ "@Tcmalloc_LIBRARIES@" ] && echo " Tcmalloc: @TC_VERSION@" + [ "@JAVAC_VERSION@" ] && echo " Javac: @JAVAC_VERSION@" + [ "@GCC_VERSION@" ] && echo " GCC/G++: @GCC_VERSION@" } cmd=$1 @@ -83,6 +106,7 @@ case $cmd in restart) do_restart "$@";; shell) exec $HYPERTABLE_HOME/bin/hypertable "$@";; version|-v|--version) print_version;; + help|-h|--help) print_help;; /*) exec $cmd "$@";; *) do_cmd "$cmd" "$@";; esac diff --git a/bin/htbt.sh b/bin/htbt.sh index 7a9d9ddc3..518989a15 100755 --- a/bin/htbt.sh +++ b/bin/htbt.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # diff --git a/bin/java-thrift.sh b/bin/java-thrift.sh new file mode 100755 index 000000000..b5b929cd6 --- /dev/null +++ b/bin/java-thrift.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2009 Luke Lu (llu@hypertable.org) +# +# This file is part of Hypertable. +# +# Hypertable is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or any later version. +# +# Hypertable is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Hypertable. If not, see +# + +# This script invokes java with class path for thrift clients + +# The installation directory +export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) +. $HYPERTABLE_HOME/bin/ht-env.sh + +javalib=$HYPERTABLE_HOME/lib/java +CLASSPATH=`echo $javalib/libthrift-*.jar` +for j in $javalib/hypertable*.jar $javalib/slf4j*.jar $javalib/log4j*.jar +do CLASSPATH=$CLASSPATH:$j +done + +exec java -cp "$CLASSPATH" "$@" diff --git a/bin/jrun b/bin/jrun index d41a4e4da..fefa9f21a 100755 --- a/bin/jrun +++ b/bin/jrun @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2007 Doug Judd (Zvents, Inc.) # @@ -22,6 +22,9 @@ export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) # Parse and remove jrun specific arguments DEBUG_ARGS= +# Setup CLASSPATH +CLASSPATH="${HYPERTABLE_HOME}" + while [ $# -gt 1 ] ; do if [ "--pidfile" = "$1" ] ; then shift @@ -31,16 +34,17 @@ while [ $# -gt 1 ] ; do DEBUG_ARGS="-Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n\ -Xdebug" shift + elif [ "--add-to-classpath" = "$1" ] ; then + shift + CLASSPATH=${CLASSPATH}:$1 + shift + elif [ "--verbose" = "$1" ] ; then + shift else break fi done - -# Setup CLASSPATH -CLASSPATH="${HYPERTABLE_HOME}" -#CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar - # for developers, add Hadoop classes to CLASSPATH if [ -d "$HYPERTABLE_HOME/build/classes" ]; then CLASSPATH=${CLASSPATH}:$HYPERTABLE_HOME/build/classes @@ -60,10 +64,11 @@ if [ -d "$HYPERTABLE_HOME/webapps" ]; then CLASSPATH=${CLASSPATH}:$HYPERTABLE_HOME fi -# add libs to CLASSPATH -for f in $HYPERTABLE_HOME/lib/*.jar; do +# add thrift to classpath +for f in $HYPERTABLE_HOME/lib/java/libthrift*.jar; do CLASSPATH=${CLASSPATH}:$f; done + # add lib/java to CLASSPATH for f in $HYPERTABLE_HOME/lib/java/*.jar; do CLASSPATH=${CLASSPATH}:$f; @@ -79,4 +84,8 @@ unset IFS # # run it # -exec java $DEBUG_ARGS -classpath "$CLASSPATH" "$@" +if [ "$JAVA_HOME" != "" ] ; then + exec $JAVA_HOME/bin/java $DEBUG_ARGS -classpath "$CLASSPATH" "$@" +else + exec java $DEBUG_ARGS -classpath "$CLASSPATH" "$@" +fi diff --git a/bin/ldd.sh b/bin/ldd.sh index 490809fd0..609b5422d 100755 --- a/bin/ldd.sh +++ b/bin/ldd.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # diff --git a/bin/postinst.in b/bin/postinst.in index 12ef7a990..619a47450 100755 --- a/bin/postinst.in +++ b/bin/postinst.in @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # diff --git a/bin/prerm.in b/bin/prerm.in index f352a5129..846b9dead 100755 --- a/bin/prerm.in +++ b/bin/prerm.in @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # diff --git a/bin/prerm.sh b/bin/prerm.sh index 591307b50..6b50545d3 100755 --- a/bin/prerm.sh +++ b/bin/prerm.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # diff --git a/bin/random-wait.sh b/bin/random-wait.sh index 0e0522dcd..3b8cd2cea 100755 --- a/bin/random-wait.sh +++ b/bin/random-wait.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash # randomly wait up to specified seconds [ $# -gt 0 ] || { echo "Usage: $0 "; exit 1; } let l=$RANDOM%$1 r=$RANDOM%1000 -sleep $l.$r +perl -e "select(undef,undef,undef,$l.$r);" diff --git a/bin/soname.sh b/bin/soname.sh index 870ce7f48..fe05e1a47 100755 --- a/bin/soname.sh +++ b/bin/soname.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # @@ -26,7 +26,9 @@ lib=$1 if o=`(objdump -p $lib | grep SONAME) 2>/dev/null` || o=`(elfdump -d $lib | grep SONAME) 2>/dev/null` then - echo $o | sed 's/.* \([^ ][^ ]*\)/\1/' -elif o=`otool -D $lib 2>/dev/null | grep -v ':$'`; then - echo $o | sed 's/.*\/\([^\/][^\/]*\)/\1/' + echo $o | sed 's/.* \([^ ][^ ]*\)/\1/' | cut -f 1 -d' ' +elif o=`otool -D $lib 2>/dev/null`; then + echo $o | tail -1 | sed 's/.*\/\([^\/][^\/]*\)/\1/' | cut -f 1 -d' ' | sed 's/:$//' +else + echo $lib | awk 'BEGIN {FS="/"} { print $NF; }' | cut -f 1 -d' ' fi diff --git a/bin/src-utils/gpp b/bin/src-utils/gpp index 88ad3eb9d..b52c7a9d0 100755 --- a/bin/src-utils/gpp +++ b/bin/src-utils/gpp @@ -20,7 +20,6 @@ if (defined $ENV{'GPP_DEBUG'}) { sub cleanup { local($_) = shift; s/^#.*//mg; # remove preprocessing directives - s/;/;\n/g; # add a newline after ; to workaround a problem in bcpp return $_; } diff --git a/bin/src-utils/htbuild b/bin/src-utils/htbuild index 300b93858..94a399e14 100755 --- a/bin/src-utils/htbuild +++ b/bin/src-utils/htbuild @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (C) 2009 Luke Lu (llu@hypertable.org) # @@ -84,23 +84,23 @@ _install_libunwind() { } _install_tcmalloc() { - install_libunwind + _install_libunwind || return 1 get http://google-perftools.googlecode.com/files/google-perftools-1.4.tar.gz tcmalloc.tgz tar zxf tcmalloc.tgz (cd google-perftools-1.4 && ./configure && make && make install) } _install_boost() { - get http://sourceforge.net/projects/boost/files/boost/1.39.0/boost_1_39_0.tar.bz2/download boost.tbz2 + get http://sourceforge.net/projects/boost/files/boost/1.40.0/boost_1_40_0.tar.bz2/download boost.tbz2 tar jxf boost.tbz2 - (cd boost_1_39_0 && ./bootstrap.sh --with-libraries=filesystem,iostreams,program_options,system,thread && ./bjam install) + (cd boost_1_40_0 && ./bootstrap.sh --with-libraries=filesystem,iostreams,program_options,system,thread && ./bjam install) } _install_bdb() { - get http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz bdb.tgz + get http://download.oracle.com/berkeley-db/db-4.8.26.tar.gz bdb.tgz tar zxf bdb.tgz - (cd db-4.7.25/build_unix && ../dist/configure --enable-cxx && make && - make install && echo /usr/local/BerkeleyDB.4.7/lib \ + (cd db-4.8.26/build_unix && ../dist/configure --enable-cxx && make && + make install && echo /usr/local/BerkeleyDB.4.8/lib \ > /etc/ld.so.conf.d/bdb.conf) } @@ -118,11 +118,15 @@ _install_sigar() { } _install_thrift() { - get http://hypertable.org/pub/thrift-r796538.tgz thrift.tgz + get http://hypertable.org/pub/thrift-0.2.0-incubating.tar.gz thrift.tgz tar zxf thrift.tgz if [ "$SETUP_DEV_ENV" ]; then confargs= + # handle ubuntu python packaging + pylib=`python -c 'import sys; print sys.path[1]'` + [ -d $pylib/dist-packages ] && + ln -sf $pylib/dist-packages $pylib/site-packages else confargs="--without-py --without-perl --without-ruby" fi @@ -131,31 +135,36 @@ _install_thrift() { } _install_kfs() { - svn co http://kosmosfs.svn.sourceforge.net/svnroot/kosmosfs/trunk@364 kfs - (cd kfs && mv CMakeLists.txt CMakeLists.txt.orig && - sed -e 's/_VERSIONS "1.37" "1.38")/_VERSIONS "1.37" "1.38" "1.39")/' \ - -e 's/NOT ${JAVA_INCLUDE_PATH} STREQUAL ""/JAVA_INCLUDE_PATH/' \ - CMakeLists.txt.orig > CMakeLists.txt && + get http://downloads.sourceforge.net/project/kosmosfs/kosmosfs/kfs-0.4/kfs-0.4.tar.gz kfs.tgz + tar zxf kfs.tgz + (cd kfs-0.4 && mv CMakeLists.txt CMakeLists.txt.orig && sed \ + -e 's/"1.37" "1.38")/"1.37" "1.38" "1.39" "1.40")/' \ + -e 's/NOT ${JAVA_INCLUDE_PATH} STREQUAL ""/JAVA_INCLUDE_PATH/' \ + CMakeLists.txt.orig > CMakeLists.txt && mkdir -p build && cd build && cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=/opt/kfs .. && make install) } -_install_ceph() { - get http://ceph.newdream.net/download/ceph-0.14.tar.gz ceph.tgz +e_install_ceph() { + get http://ceph.newdream.net/download/ceph-0.19.tar.gz ceph.tgz tar zxf ceph.tgz - (cd ceph-0.14 && ./autogen.sh && ./configure && make && make install) - # fix permission problems for 0.14 + (cd ceph-0.19 && ./autogen.sh && ./configure && make && make install) chmod 644 /usr/local/include/ceph/*.h } +has_command() { + type $1 > /dev/null 2>&1 +} + do_install() { for i in "$@"; do - if type install_$i > /dev/null; then - install_$i - elif type _install_$i > /dev/null; then + if has_command install_$i; then + install_$i || exit 1 + elif has_command _install_$i; then installed $i && continue - _install_$i && set_installed $i + _install_$i || exit 1 + set_installed $i else echo "Unknown install target: $i" fi @@ -200,7 +209,8 @@ disable_prelink() { rpm_setup_repos() { disable_prelink - yum install -y curl curl-devel || exit 1 + yum install -y ntpdate curl curl-devel || exit 1 + ntpdate -b -u pool.ntp.org } install_dev_env_from_src() { @@ -210,7 +220,7 @@ install_dev_env_from_src() { # in a non-standard directory if compiled from source. This is and is # required for regression tests to finish properly. The normal usage of # binary packages themselves doesn't require selinux to be off. - type setenforce 2>/dev/null && setenforce 0 + has_command setenforce && setenforce 0 ldconfig } @@ -219,8 +229,15 @@ rpm_install_dev_env() { libevent-devel readline-devel ncurses-devel bison flex pkgconfig \ subversion rpm-build xfsprogs-devel openssl-devel - case `yum info cmake | grep Version:` in - Version:*2.6*) yum install -y cmake;; + yum install -y log4cpp-devel && set_installed log4cpp + + case `yum info cmake | grep Version` in + Version*:*2.6*) yum install -y cmake && set_installed cmake;; + esac + + case `yum info boost-devel | grep Version` in + Version*:*1.3[5-9]*|Version*:*1.4*) + yum install -y boost-devel && set_installed boost;; esac if [ "$SETUP_DEV_ENV" ]; then @@ -244,7 +261,8 @@ deb_setup_repos() { >> /etc/apt/sources.list apt-get update || exit 1 - apt-get install -y curl + apt-get install -y curl ntpdate + ntpdate -b -u pool.ntp.org } deb_install_dev_env() { @@ -362,7 +380,7 @@ do_ami() { } sanity_check() { - type do_$TARGET > /dev/null || { echo "$TARGET not supported"; exit 1; } + has_command do_$TARGET || { echo "$TARGET not supported"; exit 1; } if [ ${#INSTALLS} -eq 0 -a ${#GENERATORS} -eq 0 ]; then echo "one or more generators required" diff --git a/bin/src-utils/htpkg b/bin/src-utils/htpkg index ce0d7c253..0763b4439 100755 --- a/bin/src-utils/htpkg +++ b/bin/src-utils/htpkg @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash build=Release srcdir=$HOME/Source/hypertable diff --git a/bin/src-utils/ver b/bin/src-utils/ver index 7521af0c8..e4daaca1d 100755 --- a/bin/src-utils/ver +++ b/bin/src-utils/ver @@ -1,13 +1,22 @@ -#!/bin/sh +#!/usr/bin/env bash # show a version string of the repo vstr=`git describe 2> /dev/null` -case "$vstr" in -v[0-9]*) - git update-index -q --refresh 2> /dev/null - test -z "`git diff-index --name-only HEAD -- 2>&1`" || vstr="$vstr-dirty" - ;; -*) vstr="tarball" -esac +if [ "$1" == "--commit-only" ] ; then + case "$vstr" in + v[0-9]*) + vstr=`echo $vstr | cut -f3 -d- | cut -b2-` + ;; + *) vstr="tarball" + esac +else + case "$vstr" in + v[0-9]*) + git update-index -q --refresh 2> /dev/null + test -z "`git diff-index --name-only HEAD -- 2>&1`" || vstr="$vstr-dirty" + ;; + *) vstr="tarball" + esac +fi echo $vstr diff --git a/bin/start-all-servers.sh b/bin/start-all-servers.sh index 3d2d11738..d381ede7c 100755 --- a/bin/start-all-servers.sh +++ b/bin/start-all-servers.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2008 Doug Judd (Zvents, Inc.) # @@ -32,6 +32,7 @@ usage() { echo "usage: start-all-servers.sh [OPTIONS] []" echo "" echo "OPTIONS:" + echo " --heapcheck-rangeserver run Hyperspace.RangeServer with tcmalloc heapcheck" echo " --valgrind-hyperspace run Hyperspace.Master with valgrind" echo " --valgrind-master run Hypertable.Master with valgrind" echo " --valgrind-rangeserver run Hypertable.RangeServer with valgrind" @@ -44,16 +45,23 @@ usage() { echo "" } - while [ "$1" != "${1##[-+]}" ]; do case $1 in '') usage exit 1;; + --heapcheck-rangeserver) + RANGESERVER_OPTS="--heapcheck " + shift + ;; --valgrind-rangeserver) RANGESERVER_OPTS="--valgrind " shift ;; + --heapcheck-master) + MASTER_OPTS="--heapcheck " + shift + ;; --valgrind-master) MASTER_OPTS="--valgrind " shift diff --git a/bin/start-dfsbroker.sh b/bin/start-dfsbroker.sh index 04f02e980..beb176004 100755 --- a/bin/start-dfsbroker.sh +++ b/bin/start-dfsbroker.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2008 Doug Judd (Zvents, Inc.) # @@ -30,6 +30,39 @@ usage() { echo "" } +dfs_conflict_error() { + OLD_DFS=$1 + shift + NEW_DFS=$1 + echo "" + echo "ERROR: DFS conflict" + echo "" + echo "You are trying to run Hypertable with the '$NEW_DFS' broker" + echo "on a system that was previously run with the '$OLD_DFS' broker." + echo "" + if [ "$OLD_DFS" == "local" ] ; then + echo "Run the following command to remove the previous database," + echo "and all of its associated state, before launching with the" + echo "'$NEW_DFS' broker:" + echo "" + echo "$HYPERTABLE_HOME/bin/stop-servers.sh ; $HYPERTABLE_HOME/bin/start-dfsbroker.sh $OLD_DFS ; $HYPERTABLE_HOME/bin/clean-database.sh" + echo "" + else + echo "To remove the previous database, and all it's associated state," + echo "in order to launch with the '$NEW_DFS' broker, start the system" + echo "on the old DFS and then clean the database. For example, with" + echo "Capistrano:" + echo "" + echo "cap stop ; cap -S dfs=$OLD_DFS cleandb" + echo "" + fi + echo "Alternatively, you can manually purge the database state by issuing" + echo "the following command on each Master and Hyperspace replica machine:" + echo "" + echo "rm -rf $HYPERTABLE_HOME/hyperspace/* $HYPERTABLE_HOME/fs/* $HYPERTABLE_HOME/run/rsml_backup/* $HYPERTABLE_HOME/run/last-dfs" + echo "" +} + while [ "$1" != "${1##[-+]}" ]; do case $1 in --valgrind) @@ -50,10 +83,21 @@ fi DFS=$1 shift +if [ -e $HYPERTABLE_HOME/run/last-dfs ] ; then + LAST_DFS=`cat $HYPERTABLE_HOME/run/last-dfs` + if [ "$DFS" != "$LAST_DFS" ] ; then + dfs_conflict_error $LAST_DFS $DFS + exit 1 + fi +else + # record last DFS + echo $DFS > $HYPERTABLE_HOME/run/last-dfs +fi + set_start_vars DfsBroker.$DFS check_pidfile $pidfile && exit 0 -check_server dfsbroker "$@" +check_server "$@" dfsbroker if [ $? != 0 ] ; then if [ "$DFS" == "hadoop" ] ; then if [ "n$VALGRIND" != "n" ] ; then diff --git a/bin/start-hyperspace.sh b/bin/start-hyperspace.sh index ccaaab036..d7ee5f199 100755 --- a/bin/start-hyperspace.sh +++ b/bin/start-hyperspace.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2008 Doug Judd (Zvents, Inc.) # diff --git a/bin/start-master.sh b/bin/start-master.sh index 5ae3e8dae..14d76f673 100755 --- a/bin/start-master.sh +++ b/bin/start-master.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2008 Doug Judd (Zvents, Inc.) # @@ -25,6 +25,7 @@ usage() { echo "" echo "OPTIONS:" echo " --valgrind run master with valgrind" + echo " --heapcheck run master with google-perf-tools Heapcheck" echo "" } @@ -34,10 +35,15 @@ while [ "$1" != "${1##[-+]}" ]; do VALGRIND="valgrind -v --log-file=vg --leak-check=full --num-callers=20 " shift ;; + --heapcheck) + HEAPCHECK="env HEAPCHECK=normal " + shift + ;; *) break ;; esac done + start_server master Hypertable.Master Hypertable.Master "$@" diff --git a/bin/start-monitoring.sh b/bin/start-monitoring.sh new file mode 100755 index 000000000..19056a7b7 --- /dev/null +++ b/bin/start-monitoring.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Copyright 2010 Sanjit Jhala (Hypertable, Inc.) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The installation directory +export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) +. $HYPERTABLE_HOME/bin/ht-monitoring.sh + +usage() { + echo "" + echo "usage: start-monitoring.sh" + echo "" +} + +start_monitoring diff --git a/bin/start-rangeserver.sh b/bin/start-rangeserver.sh index ef48dec09..a41a4a02a 100755 --- a/bin/start-rangeserver.sh +++ b/bin/start-rangeserver.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2008 Doug Judd (Zvents, Inc.) # @@ -25,6 +25,7 @@ usage() { echo "" echo "OPTIONS:" echo " --valgrind run range server with valgrind" + echo " --heapcheck run range server with google-perf-tools Heapcheck" echo "" } @@ -34,6 +35,10 @@ while [ "$1" != "${1##[-+]}" ]; do VALGRIND="valgrind -v --log-file=vg --leak-check=full --num-callers=20 " shift ;; + --heapcheck) + HEAPCHECK="env HEAPCHECK=normal" + shift + ;; *) break ;; diff --git a/bin/start-test-client.sh b/bin/start-test-client.sh new file mode 100755 index 000000000..defb8bbdd --- /dev/null +++ b/bin/start-test-client.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Copyright 2010 Doug Judd (Hypertable, Inc.) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The installation directory +export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) +. $HYPERTABLE_HOME/bin/ht-env.sh + +JRUN_OPTS= +COUNT=1 + +usage() { + echo "" + echo "usage: start-test-client.sh [options] [:]" + echo "" + echo "options:" + echo " --jrun-opts Supplies to the jrun command" + echo "" + echo "Launches the java test client. Client will connect to the" + echo "dispatcher running at [:] to receive instructions" + echo "" +} + +while [ $# -gt 1 ] ; do + if [ "--jrun-opts" = "$1" ] ; then + shift + JRUN_OPTS="$JRUN_OPTS $1" + shift + elif [ "--count" = "$1" ] ; then + shift + COUNT=$1 + shift + fi +done + +if [ "$#" -eq 0 ]; then + usage + exit 1 +fi + +let j=$COUNT + +while [ $j -gt 0 ] ; do + start_server_no_check test_client jrun Hypertable.TestClient-$j $JRUN_OPTS org.hypertable.examples.PerformanceTest.Client "$@" + let j-- +done + + diff --git a/bin/start-test-servers.sh b/bin/start-test-servers.sh index 1d8385120..a95010a7d 100755 --- a/bin/start-test-servers.sh +++ b/bin/start-test-servers.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash INSTALL_DIR=${INSTALL_DIR:-$(cd `dirname $0`/.. && pwd)} HT_TEST_DFS=${HT_TEST_DFS:-local} @@ -13,18 +13,18 @@ usage_exit() { while [ $# -gt 0 ]; do case $1 in - --clear) clear=1;; - -h|--help) usage_exit;; - --val*|--no*) opts[${#opts[*]}]=$1;; - *) break;; + --clear) clear=1;; + -h|--help) usage_exit;; + --val*|--no*|--heap*) opts[${#opts[*]}]=$1;; + *) break;; esac shift done if [ "$clear" ]; then - $INSTALL_DIR/bin/ht clean-database + $INSTALL_DIR/bin/ht clean-database $@ else $INSTALL_DIR/bin/ht stop servers fi -$INSTALL_DIR/bin/ht start all-servers "${opts[@]}" $HT_TEST_DFS "$@" +$INSTALL_DIR/bin/ht start all-servers "${opts[@]}" $HT_TEST_DFS $@ diff --git a/bin/start-thriftbroker.sh b/bin/start-thriftbroker.sh index be2ac0609..5dcac0e22 100755 --- a/bin/start-thriftbroker.sh +++ b/bin/start-thriftbroker.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2008 Doug Judd (Zvents, Inc.) # diff --git a/bin/stop-hyperspace.sh b/bin/stop-hyperspace.sh new file mode 100755 index 000000000..ee6f44a36 --- /dev/null +++ b/bin/stop-hyperspace.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Copyright 2009 Sanjit Jhala(Hypertable, Inc.) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The installation directory +export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) +. $HYPERTABLE_HOME/bin/ht-env.sh + +# +# Stop Hyperspace +# +stop_server hyperspace + +# wait for hyperspace shutdown +wait_for_server_shutdown hyperspace "hyperspace" "$@" & + +wait diff --git a/bin/stop-monitoring.sh b/bin/stop-monitoring.sh new file mode 100644 index 000000000..5a3f6e0bd --- /dev/null +++ b/bin/stop-monitoring.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Copyright 2010 Sanjit Jhala (Hypertable, Inc.) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The installation directory +export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) +. $HYPERTABLE_HOME/bin/ht-monitoring.sh + +usage() { + echo "" + echo "usage: stop-monitoring.sh" + echo "" +} + +stop_monitoring diff --git a/bin/stop-servers.sh b/bin/stop-servers.sh index 533f709cc..ffd35a199 100755 --- a/bin/stop-servers.sh +++ b/bin/stop-servers.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright 2008 Doug Judd (Zvents, Inc.) # @@ -19,14 +19,149 @@ export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) . $HYPERTABLE_HOME/bin/ht-env.sh -echo "Sending shutdown command" -echo 'shutdown;quit' | $HYPERTABLE_HOME/bin/ht_rsclient --batch --no-hyperspace -stop_server thriftbroker -stop_server master hyperspace dfsbroker +FORCE="false" + +STOP_DFSBROKER="true" +STOP_MASTER="true" +STOP_RANGESERVER="true" +STOP_THRIFTBROKER="true" +STOP_HYPERSPACE="true" +STOP_TESTCLIENT="true" +STOP_TESTDISPATCHER="true" + +usage() { + echo "" + echo "usage: stop-servers.sh [OPTIONS]" + echo "" + echo "OPTIONS:" + echo " --force kill processes to ensure they're stopped" + echo " --no-dfsbroker do not stop the DFS broker" + echo " --no-master do not stop the Hypertable master" + echo " --no-rangeserver do not stop the RangeServer" + echo " --no-hyperspace do not stop the Hyperspace master" + echo " --no-thriftbroker do not stop the ThriftBroker" + echo "" + echo "DFS choices: kfs, hadoop, local" + echo "" +} + +while [ "$1" != "${1##[-+]}" ]; do + case $1 in + '') + usage + exit 1;; + --force) + FORCE="true" + shift + ;; + --no-dfsbroker) + STOP_DFSBROKER="false" + shift + ;; + --no-master) + STOP_MASTER="false" + shift + ;; + --no-rangeserver) + STOP_RANGESERVER="false" + shift + ;; + --no-hyperspace) + STOP_HYPERSPACE="false" + shift + ;; + --no-thriftbroker) + STOP_THRIFTBROKER="false" + shift + ;; + *) + usage + exit 1;; + esac +done + + + +# +# Stop TestClient +# +if [ $STOP_TESTCLIENT == "true" ] ; then + stop_server testclient +fi + +# +# Stop TestDispatcher +# +if [ $STOP_TESTDISPATCHER == "true" ] ; then + stop_server testdispatcher +fi + +# +# Stop Thriftbroker +# +if [ $STOP_THRIFTBROKER == "true" ] ; then + stop_server thriftbroker +fi + +# +# Stop Master +# +if [ $STOP_MASTER == "true" ] ; then + echo 'shutdown;quit;' | $HYPERTABLE_HOME/bin/ht master_client --batch + # wait for master shutdown + wait_for_server_shutdown master "master" "$@" + if [ $FORCE == "true" ] ; then + stop_server master + fi +fi + +# +# Stop RangeServer +# +if [ $STOP_RANGESERVER == "true" ] ; then + echo "Sending shutdown command" + echo 'shutdown;quit' | $HYPERTABLE_HOME/bin/ht rsclient --batch --no-hyperspace + # wait for rangeserver shutdown + wait_for_server_shutdown rangeserver "range server" "$@" + if [ $FORCE == "true" ] ; then + stop_server rangeserver + fi +fi + + +# +# Stop DFSBroker +# +if [ $STOP_DFSBROKER == "true" ] ; then + stop_server dfsbroker +fi + + +# +# Stop Hyperspace +# +if [ $STOP_HYPERSPACE == "true" ] ; then + stop_server hyperspace +fi + sleep 1 -wait_for_server_shutdown thriftbroker "thrift broker" "$@" & -wait_for_server_shutdown dfsbroker "DFS broker" "$@" & -wait_for_server_shutdown rangeserver "range server" "$@" & -wait_for_server_shutdown master "hypertable master" "$@" & -wait_for_server_shutdown hyperspace "hyperspace" "$@" & +# wait for thriftbroker shutdown +if [ $STOP_THRIFTBROKER == "true" ] ; then + wait_for_server_shutdown thriftbroker "thrift broker" "$@" & +fi +# wait for dfsbroker shutdown +if [ $STOP_DFSBROKER == "true" ] ; then + wait_for_server_shutdown dfsbroker "DFS broker" "$@" & +fi + +# wait for master shutdown +if [ $STOP_MASTER == "true" ] ; then + wait_for_server_shutdown master "hypertable master" "$@" & +fi + +# wait for hyperspace shutdown +if [ $STOP_HYPERSPACE == "true" ] ; then + wait_for_server_shutdown hyperspace "hyperspace" "$@" & +fi + wait diff --git a/bin/upgrade-ok.sh b/bin/upgrade-ok.sh new file mode 100755 index 000000000..50a04742f --- /dev/null +++ b/bin/upgrade-ok.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# +# Copyright 2010 Doug Judd (Hypertable, Inc.) +# +# This file is part of Hypertable. +# +# Hypertable is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 of the +# License, or any later version. +# +# Hypertable is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + +# The installation directory +export HYPERTABLE_HOME=$(cd `dirname "$0"`/.. && pwd) +. $HYPERTABLE_HOME/bin/ht-env.sh + +usage() { + echo "" + echo "usage: upgrade-ok.sh " + echo "" + echo "description:" + echo " Determines whether or not the upgrade from Hypertable" + echo " version to version is valid. and " + echo " are assumed to be Hypertable installation directories" + echo " whos last path component is either a version number or" + echo " the symbolic link \"current\" which points to a Hypertable" + echo " installation directory whos last path component is a" + echo " version number." + echo "" + echo "return:" + echo " Zero if upgrade is OK, non-zero otherwise" + echo "" +} + +if [ $# != 2 ] ; then + usage + exit 1 +fi + +FROM=`echo $1 | awk -F'/' '{ print $NF; }'` +TO=`echo $2 | awk -F'/' '{ print $NF; }'` + +if [ "$FROM" == "current" ] ; then + FROM=`/bin/ls -l $1 | tr -s " " | awk '{ print $NF; }' | awk -F'/' '{ print $NF; }'` +fi + +if [ "$TO" == "current" ] ; then + TO=`/bin/ls -l $2 | tr -s " " | awk '{ print $NF; }' | awk -F'/' '{ print $NF; }'` +fi + +if [ "$FROM" == "$TO" ] ; then + echo "Can't upgrade to identical version: $FROM -> $TO" + exit 1 +fi + +MAJOR=`echo $FROM | cut -d'.' -f1` +MINOR=`echo $FROM | cut -d'.' -f2` +MICRO=`echo $FROM | cut -d'.' -f3` +PATCH=`echo $FROM | cut -d'.' -f4` + +if [ "$MAJOR" == "" ] || + [ "$MINOR" == "" ] || + [ "$MICRO" == "" ] || + [ "$PATCH" == "" ] ; then + echo "Unable to extract version number from argument: $1" + exit 1 +fi + +let FROM_PATCH=$PATCH +let FROM_MICRO=($MICRO*1000)+$FROM_PATCH +let FROM_MINOR=($MINOR*100000)+$FROM_MICRO +let FROM_MAJOR=($MAJOR*10000000)+$FROM_MINOR + +MAJOR=`echo $TO | cut -d'.' -f1` +MINOR=`echo $TO | cut -d'.' -f2` +MICRO=`echo $TO | cut -d'.' -f3` +PATCH=`echo $TO | cut -d'.' -f4` + +if [ "$MAJOR" == "" ] || + [ "$MINOR" == "" ] || + [ "$MICRO" == "" ] || + [ "$PATCH" == "" ] ; then + echo "Unable to extract version number from argument: $1" + exit 1 +fi + +let TO_PATCH=$PATCH +let TO_MICRO=($MICRO*1000)+$TO_PATCH +let TO_MINOR=($MINOR*100000)+$TO_MICRO +let TO_MAJOR=($MAJOR*10000000)+$TO_MINOR + +if [ $TO_MINOR -le 1000000 ] && [ $TO_MINOR -ge 904000 ] ; then + if [ $FROM_MINOR -le 1000000 ] && [ $FROM_MINOR -ge 904000 ] ; then + exit 0 + fi + echo "Incompatible upgrade a: $FROM -> $TO" + exit 1 +elif [ $TO_MAJOR -ge 1000000 ] ; then + if [ $FROM_MAJOR -ge 1000000 ] ; then + if [ $FROM_MINOR -eq $TO_MINOR ] ; then + exit 0 + fi + fi + echo "Incompatible upgrade b: $FROM -> $TO" + exit 1 +elif [ $TO_MAJOR -lt 1000000 ] ; then + if [ $FROM_MAJOR -lt 1000000 ] ; then + if [ $FROM_MICRO -eq $TO_MICRO ] ; then + exit 0 + fi + fi + echo "Incompatible upgrade c: $FROM -> $TO" + exit 1 +fi + +exit 0 diff --git a/build.xml.in b/build.xml.in index 9e07b90e3..616b527a8 100644 --- a/build.xml.in +++ b/build.xml.in @@ -5,12 +5,16 @@ + + + + @@ -28,10 +32,34 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -40,6 +68,7 @@ + @@ -60,10 +89,22 @@ debug="true"> + + + + + + + + + @@ -71,8 +112,8 @@ - - + +
@@ -96,4 +137,21 @@ + + + + + + + + +
+ + +
+ + + + + diff --git a/cmake/CheckRE2.cc b/cmake/CheckRE2.cc new file mode 100644 index 000000000..30a036973 --- /dev/null +++ b/cmake/CheckRE2.cc @@ -0,0 +1,41 @@ +/** -*- C++ -*- + * Copyright (C) 2010 Sanjit Jhala (sanjit@hypertable.org) + * + * This file is part of Hypertable. + * + * Hypertable is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or any later version. + * + * Hypertable is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Hypertable. If not, see + */ + +#include +#include +#include + + +int main() { + using namespace std; + + string hello("hello"); + string re1("h.*o"); + string re2("h.*0"); + if (!RE2::FullMatch(hello.c_str(), re1.c_str())) { + fprintf (stderr, "'%s' should match pattern '%s'", hello.c_str(), re1.c_str()); + return 1; + } + if (RE2::FullMatch(hello.c_str(), re2.c_str())) { + fprintf (stderr, "'%s' should not match pattern '%s'", hello.c_str(), re2.c_str()); + return 1; + } + printf("0.0.0\n"); + return 0; +} diff --git a/cmake/CheckRRDtool.cc b/cmake/CheckRRDtool.cc new file mode 100644 index 000000000..dea599a56 --- /dev/null +++ b/cmake/CheckRRDtool.cc @@ -0,0 +1,9 @@ +#include +#include + +int main() { + + char *version = rrd_strversion(); + printf("%s\n", version); + return 0; +} diff --git a/cmake/DocHelper.cmake b/cmake/DocHelper.cmake index 87854660a..a6bde34f9 100644 --- a/cmake/DocHelper.cmake +++ b/cmake/DocHelper.cmake @@ -10,7 +10,8 @@ if (DOXYGEN_FOUND) ${HYPERTABLE_BINARY_DIR}/Doxyfile) add_custom_target(doc ${DOXYGEN_EXECUTABLE} ${HYPERTABLE_BINARY_DIR}/Doxyfile) - add_custom_command(TARGET doc POST_BUILD COMMAND make hqldoc) + add_custom_command(TARGET doc PRE_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_SOURCE_DIR}/doc/markdown markdown POST_BUILD COMMAND make hqldoc) if (Thrift_FOUND) add_custom_command(TARGET doc POST_BUILD COMMAND make thriftdoc) diff --git a/cmake/FindAnt.cmake b/cmake/FindAnt.cmake index 1e10033f9..0de8ad1e8 100644 --- a/cmake/FindAnt.cmake +++ b/cmake/FindAnt.cmake @@ -3,7 +3,7 @@ # ANT_VERSION version string of ant if found # ANT_FOUND, If false, do not try to use ant -exec_program(ant ARGS -version OUTPUT_VARIABLE ANT_VERSION +exec_program(env ARGS ant -version OUTPUT_VARIABLE ANT_VERSION RETURN_VALUE ANT_RETURN) if (ANT_RETURN STREQUAL "0") @@ -17,7 +17,7 @@ else () set(SKIP_JAVA_BUILD TRUE) endif () -exec_program(javac ARGS -version OUTPUT_VARIABLE JAVAC_OUT +exec_program(env ARGS javac -version OUTPUT_VARIABLE JAVAC_OUT RETURN_VALUE JAVAC_RETURN) if (JAVAC_RETURN STREQUAL "0") diff --git a/cmake/FindBerkeleyDB.cmake b/cmake/FindBerkeleyDB.cmake index 367b59ec4..03291de6b 100644 --- a/cmake/FindBerkeleyDB.cmake +++ b/cmake/FindBerkeleyDB.cmake @@ -10,10 +10,10 @@ # BDB_LIBRARY, where to find the BerkeleyDB library. find_path(BDB_INCLUDE_DIR db_cxx.h NO_DEFAULT_PATH PATHS - /usr/local/BerkeleyDB.4.7/include - /opt/local/include/db47 - /opt/local/include/db46 # introduced key_exists - /usr/local/include/db4 + ${HT_DEPENDENCY_INCLUDE_DIR} + /usr/local/BerkeleyDB.4.8/include + /usr/local/include/db48 + /opt/local/include/db48 /usr/local/include /usr/include/db4 /usr/include @@ -21,10 +21,10 @@ find_path(BDB_INCLUDE_DIR db_cxx.h NO_DEFAULT_PATH PATHS set(BDB_NAMES ${BDB_NAMES} db_cxx) find_library(BDB_LIBRARY NAMES ${BDB_NAMES} NO_DEFAULT_PATH PATHS - /usr/local/BerkeleyDB.4.7/lib - /opt/local/lib/db47 - /opt/local/lib/db46 # ditto - /usr/local/lib/db4 + ${HT_DEPENDENCY_LIB_DIR} + /usr/local/BerkeleyDB.4.8/lib + /usr/local/lib/db48 + /opt/local/lib/db48 /usr/local/lib /usr/lib ) @@ -62,11 +62,30 @@ if (NOT BDB_CHECK STREQUAL "0") "remove CMakeCache.txt and try again.") endif () -if (NOT BDB_VERSION MATCHES "^([4-9]|[1-9][0-9]+)\\.([6-9]|[1-9][0-9]+)") - message(FATAL_ERROR "At least 4.6.x of BerkeleyDB is required. " - "Please fix the installation, remove CMakeCache.txt and try again.") -endif () +STRING(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\1" MAJOR "${BDB_VERSION}") +STRING(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\2" MINOR "${BDB_VERSION}") +STRING(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\3" PATCH "${BDB_VERSION}") +set(BDB_COMPATIBLE "NO") +if (MAJOR MATCHES "^([5-9]|[1-9][0-9]+)" ) + set(BDB_COMPATIBLE "YES") +elseif(MAJOR MATCHES "^4$") + if(MINOR MATCHES "^([9]|[1-9][0-9]+)") + set(BDB_COMPATIBLE "YES") + elseif(MINOR MATCHES "^8$") + if(PATCH MATCHES "^([3-9][0-9]+)") + set(BDB_COMPATIBLE "YES") + elseif(PATCH MATCHES "^([2][4-9])") + set(BDB_COMPATIBLE "YES") + endif() + endif() +endif() + +if (NOT BDB_COMPATIBLE) + message(FATAL_ERROR "BerkeleyDB version >= 4.8.24 required." + "Found version ${MAJOR}.${MINOR}.${PATCH}" + "Please fix the installation, remove CMakeCache.txt and try again.") +endif() mark_as_advanced( BDB_LIBRARY diff --git a/cmake/FindBoost.cmake b/cmake/FindBoost.cmake index e1ff24bf6..3be1b6b2c 100644 --- a/cmake/FindBoost.cmake +++ b/cmake/FindBoost.cmake @@ -33,7 +33,13 @@ if (WIN32) set(Boost_LIB_DIAGNOSTIC_DEFINITIONS "-DBOOST_LIB_DIAGNOSTIC") endif () -set(BOOST_DIR_SEARCH $ENV{BOOST_ROOT}) +# Hypertable Dependency Directory +if (NOT HT_DEPENDENCY_DIR) + set(BOOST_DIR_SEARCH $ENV{BOOST_ROOT}) +else () + set(BOOST_DIR_SEARCH ${HT_DEPENDENCY_DIR}) + set(Boost_INCLUDE_DIR ${HT_DEPENDENCY_DIR}/include) +endif () if (BOOST_DIR_SEARCH) file(TO_CMAKE_PATH ${BOOST_DIR_SEARCH} BOOST_DIR_SEARCH) @@ -51,6 +57,10 @@ endif () # Add in some path suffixes. These will have to be updated whenever a new # Boost version comes out. set(SUFFIX_FOR_PATH + boost-1_44 + boost-1_43 + boost-1_42 + boost-1_41 boost-1_40 boost-1_39 boost-1_38 @@ -67,14 +77,16 @@ set(SUFFIX_FOR_PATH # # Look for an installation. # -find_path(Boost_INCLUDE_DIR - NAMES boost/config.hpp - PATH_SUFFIXES ${SUFFIX_FOR_PATH} - # Look in other places. - PATHS ${BOOST_DIR_SEARCH} - # Help the user find it if we cannot. - DOC "The ${BOOST_INCLUDE_PATH_DESCRIPTION}" -) +if (NOT Boost_INCLUDE_DIR) + find_path(Boost_INCLUDE_DIR + NAMES boost/config.hpp + PATH_SUFFIXES ${SUFFIX_FOR_PATH} + # Look in other places. + PATHS ${BOOST_DIR_SEARCH} /usr/include /usr/local/include /opt/local/include + # Help the user find it if we cannot. + DOC "The ${BOOST_INCLUDE_PATH_DESCRIPTION}" + ) +endif () macro(FIND_BOOST_PARENT root includedir) # Look for the boost library path. @@ -107,9 +119,9 @@ macro(FIND_BOOST_LIBRARY lib libname libroot required) set(${lib}_NAMES ${${lib}_NAMES} "boost_${libname}") endif () - find_library(${lib} + find_library(${lib} NO_DEFAULT_PATH NAMES ${${lib}_NAMES} - PATHS "${libroot}/lib" "${libroot}/lib64" + PATHS "${libroot}/lib" "${libroot}/lib64" /lib /usr/lib /usr/local/lib /opt/local/lib ) if (required AND ${lib} MATCHES "NOTFOUND$") message(FATAL_ERROR "required boost library: ${lib} not found") @@ -149,7 +161,6 @@ if (Boost_INCLUDE_DIR) set(Boost_HAS_SYSTEM_LIB true) endif() - message(STATUS "Boost include dir: ${Boost_INCLUDE_DIR}") FIND_BOOST_PARENT(Boost_PARENT ${Boost_INCLUDE_DIR}) # Add boost libraries here diff --git a/cmake/FindCeph.cmake b/cmake/FindCeph.cmake index efa72c220..6749bab33 100644 --- a/cmake/FindCeph.cmake +++ b/cmake/FindCeph.cmake @@ -11,9 +11,8 @@ if (Ceph_INCLUDE) set(Ceph_FIND_QUIETLY TRUE) endif () -find_path(Ceph_INCLUDE libceph.h +find_path(Ceph_INCLUDE ceph/libceph.h /usr/local/include - /usr/include $ENV{HOME}/ceph/src/client ) mark_as_advanced(Ceph_INCLUDE) @@ -24,9 +23,13 @@ find_library(Ceph_LIB $ENV{HOME}/ceph/src/.libs) mark_as_advanced(Ceph_LIB) -if (Ceph_INCLUDE AND Ceph_LIB) +find_library(SSL_LIB NAMES ssl + PATHS /usr/local/lib) +mark_as_advanced(SSL_LIB) + +if (Ceph_INCLUDE AND Ceph_LIB AND SSL_LIB) set(Ceph_FOUND TRUE) - set(Ceph_LIBRARIES ${Ceph_LIB}) + set(Ceph_LIBRARIES ${Ceph_LIB} ${SSL_LIB}) else () set(Ceph_FOUND FALSE) set(Ceph_LIBRARIES) @@ -37,7 +40,7 @@ if (Ceph_FOUND) else () message(STATUS "Did not find ceph libraries") if (Ceph_FIND_REQUIRED) - message(FATAL_ERROR "Could NOT find ceph libraries") + message(FATAL_ERROR "Could NOT find ceph libraries (libceph & libssl)") endif () endif () diff --git a/cmake/FindHoard.cmake b/cmake/FindHoard.cmake new file mode 100644 index 000000000..b160d33f7 --- /dev/null +++ b/cmake/FindHoard.cmake @@ -0,0 +1,30 @@ +# - Find Hoard +# Find the native Hoard includes and library +# +# Hoard_LIBRARIES - List of libraries when using Hoard. +# Hoard_FOUND - True if Hoard found. + +set(Hoard_NAMES hoard) + +find_library(Hoard_LIBRARY NO_DEFAULT_PATH + NAMES ${Hoard_NAMES} + PATHS ${HT_DEPENDENCY_LIB_DIR} /lib /usr/lib /usr/local/lib /opt/local/lib +) + +if (Hoard_LIBRARY) + set(Hoard_FOUND TRUE) + set( Hoard_LIBRARIES ${Hoard_LIBRARY} ) +else () + set(Hoard_FOUND FALSE) + set( Hoard_LIBRARIES ) +endif () + +if (Hoard_FOUND) + message(STATUS "Found Hoard: ${Hoard_LIBRARY}") +else () + message(STATUS "Not Found Hoard: ${Hoard_LIBRARY}") +endif () + +mark_as_advanced( + Hoard_LIBRARY + ) diff --git a/cmake/FindKfs.cmake b/cmake/FindKfs.cmake index 938b9acd6..113f654c8 100644 --- a/cmake/FindKfs.cmake +++ b/cmake/FindKfs.cmake @@ -31,9 +31,12 @@ FIND_KFS_LIB(kfsIO) FIND_KFS_LIB(kfsCommon) FIND_KFS_LIB(qcdio) +find_library(Crypto_LIB NAMES crypto PATHS /opt/local/lib /usr/local/lib) + if (Kfs_INCLUDE_DIR AND kfsClient_LIB) set(Kfs_FOUND TRUE) - set( Kfs_LIBRARIES ${kfsClient_LIB} ${kfsIO_LIB} ${kfsCommon_LIB} ${qcdio_LIB}) + set( Kfs_LIBRARIES ${kfsClient_LIB} ${kfsIO_LIB} ${kfsCommon_LIB} + ${qcdio_LIB} ${Crypto_LIB}) else () set(Kfs_FOUND FALSE) set( Kfs_LIBRARIES) diff --git a/cmake/FindLog4cpp.cmake b/cmake/FindLog4cpp.cmake index 138961960..82cfb6ee9 100644 --- a/cmake/FindLog4cpp.cmake +++ b/cmake/FindLog4cpp.cmake @@ -5,22 +5,22 @@ # Log4cpp_LIBRARIES - List of libraries when using Log4cpp. # Log4cpp_FOUND - True if Log4cpp found. - if (Log4cpp_INCLUDE_DIR) # Already in cache, be silent set(Log4cpp_FIND_QUIETLY TRUE) endif () -find_path(Log4cpp_INCLUDE_DIR log4cpp/Category.hh +find_path(Log4cpp_INCLUDE_DIR log4cpp/Category.hh NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_INCLUDE_DIR} /opt/local/include /usr/local/include /usr/include ) set(Log4cpp_NAMES log4cpp) -find_library(Log4cpp_LIBRARY +find_library(Log4cpp_LIBRARY NO_DEFAULT_PATH NAMES ${Log4cpp_NAMES} - PATHS /usr/lib /usr/local/lib /opt/local/lib + PATHS ${HT_DEPENDENCY_LIB_DIR} /usr/lib /usr/local/lib /opt/local/lib ) if (Log4cpp_INCLUDE_DIR AND Log4cpp_LIBRARY) diff --git a/cmake/FindPerlThrift.cmake b/cmake/FindPerlThrift.cmake index fd5a0af9e..e4d5981e9 100644 --- a/cmake/FindPerlThrift.cmake +++ b/cmake/FindPerlThrift.cmake @@ -2,7 +2,7 @@ # This module defines # PERLTHRIFT_FOUND, If false, do not try to use ant -exec_program(perl ARGS -MThrift -e 0 OUTPUT_VARIABLE PERLTHRIFT_OUT +exec_program(env ARGS perl -MThrift -e 0 OUTPUT_VARIABLE PERLTHRIFT_OUT RETURN_VALUE PERLTHRIFT_RETURN) if (PERLTHRIFT_RETURN STREQUAL "0") diff --git a/cmake/FindPythonThrift.cmake b/cmake/FindPythonThrift.cmake index d08d09b25..a79f4bad0 100644 --- a/cmake/FindPythonThrift.cmake +++ b/cmake/FindPythonThrift.cmake @@ -2,7 +2,7 @@ # This module defines # PYTHONTHRIFT_FOUND, If false, do not try to use ant -exec_program(python ARGS -c 'import thrift' OUTPUT_VARIABLE PYTHONTHRIFT_OUT +exec_program(env ARGS python -c'import thrift' OUTPUT_VARIABLE PYTHONTHRIFT_OUT RETURN_VALUE PYTHONTHRIFT_RETURN) if (PYTHONTHRIFT_RETURN STREQUAL "0") diff --git a/cmake/FindRE2.cmake b/cmake/FindRE2.cmake new file mode 100644 index 000000000..e7b2cd3cf --- /dev/null +++ b/cmake/FindRE2.cmake @@ -0,0 +1,62 @@ +# - Find RE2 +# Find the native RE2 includes and library +# +# RE2_INCLUDE_DIR - where to find re2.h, etc. +# RE2_LIBRARIES - List of libraries when using RE2. +# RE2_FOUND - True if RE2 found. + +find_path(RE2_INCLUDE_DIR re2/re2.h NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_INCLUDE_DIR} + /usr/include + /opt/local/include + /usr/local/include +) + +set(RE2_NAMES ${RE2_NAMES} re2) +find_library(RE2_LIBRARY NAMES ${RE2_NAMES} NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_LIB_DIR} + /usr/local/lib + /usr/local/re2/lib + /opt/local/lib + /usr/lib + ) + +if (RE2_INCLUDE_DIR AND RE2_LIBRARY) + set(RE2_FOUND TRUE) + set( RE2_LIBRARIES ${RE2_LIBRARY} ) +else () + set(RE2_FOUND FALSE) + set( RE2_LIBRARIES ) +endif () + +if (RE2_FOUND) + message(STATUS "Found RE2: ${RE2_LIBRARY}") + try_run(RE2_CHECK RE2_CHECK_BUILD + ${HYPERTABLE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp + ${HYPERTABLE_SOURCE_DIR}/cmake/CheckRE2.cc + CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${RE2_INCLUDE_DIR} + -DLINK_LIBRARIES=${RE2_LIBRARIES} + OUTPUT_VARIABLE RE2_TRY_OUT) + if (RE2_CHECK_BUILD AND NOT RE2_CHECK STREQUAL "0") + string(REGEX REPLACE ".*\n(RE2 .*)" "\\1" RE2_TRY_OUT ${RE2_TRY_OUT}) + message(STATUS "${RE2_TRY_OUT}") + message(FATAL_ERROR "Please fix the re2 installation and try again.") + set(RE2_LIBRARIES) + endif () + string(REGEX REPLACE ".*\n([0-9]+[^\n]+).*" "\\1" RE2_VERSION ${RE2_TRY_OUT}) + if (NOT RE2_VERSION MATCHES "^[0-9]+.*") + set(RE2_VERSION "unknown") + endif () + message(STATUS " version: ${RE2_VERSION}") +else () + message(STATUS "Not Found RE2: ${RE2_LIBRARY}") + if (RE2_FIND_REQUIRED) + message(STATUS "Looked for RE2 libraries named ${RE2_NAMES}.") + message(FATAL_ERROR "Could NOT find RE2 library") + endif () +endif () + +mark_as_advanced( + RE2_LIBRARY + RE2_INCLUDE_DIR + ) diff --git a/cmake/FindRRDtool.cmake b/cmake/FindRRDtool.cmake new file mode 100644 index 000000000..9817812b0 --- /dev/null +++ b/cmake/FindRRDtool.cmake @@ -0,0 +1,103 @@ +# -*- cmake -*- + +# - Find RRDtool +# Find the RRDtool includes and library +# This module defines +# RRD_INCLUDE_DIR, where to find rrd.h, etc. +# RRD_LIBRARIES, the libraries needed to use RRDtool. +# RRD_FOUND, If false, do not try to use RRDtool. +# also defined, RD_LIBRARY, where to find the RRDtool library. + +find_path(RRD_INCLUDE_DIR rrd.h NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_INCLUDE_DIR} + /opt/local/include + /usr/local/include + /usr/include + ) + +set(RRD_NAMES ${RRD_NAMES} rrd) +find_library(RRD_LIBRARY NAMES ${RRD_NAMES} NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_LIB_DIR} + /opt/local/lib + /usr/local/lib + /usr/lib + /usr/lib64 + ) + +find_library(FREETYPE_LIBRARY freetype PATHS + ${HT_DEPENDENCY_LIB_DIR} + /opt/local/lib + /usr/local/lib + /usr/lib + /usr/lib64 + ) + +set(PNG_NAMES ${PNG_NAMES} png12 png14) + find_library(PNG_LIBRARY NAMES ${PNG_NAMES} NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_LIB_DIR} + /opt/local/lib + /usr/local/lib + /usr/lib + /usr/lib64 + ) + +find_library(ART_LGPL_2_LIBRARY NAMES art_lgpl_2 NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_LIB_DIR} + /opt/local/lib + /usr/local/lib + /usr/lib + /usr/lib64 + ) + +message(STATUS "${RRD_LIBRARY} ${RRD_INCLUDE_DIR} ${FREETYPE_LIBRARY} ${PNG_LIBRARY} ${ART_LGPL_2_LIBRARY}") + +if (RRD_LIBRARY AND RRD_INCLUDE_DIR AND FREETYPE_LIBRARY AND PNG_LIBRARY AND ART_LGPL_2_LIBRARY) + set(RRD_LIBRARIES ${RRD_LIBRARY} ${FREETYPE_LIBRARY} ${PNG_LIBRARY} ${ART_LGPL_2_LIBRARY}) + set(RRD_FOUND TRUE) + message(STATUS "Found RRDtool: ${RRD_LIBRARIES}") +else () + set(RRD_FOUND FALSE) +endif () + + +if (RRD_FOUND) + if (NOT RRD_FIND_QUIETLY) + message(STATUS "Found RRDtool: ${RRD_LIBRARIES}") + endif () +else () + if (RRDtool_FIND_REQUIRED) + message(FATAL_ERROR "Could not find RRDtool library") + endif () +endif () + +try_run(RRD_CHECK SHOULD_COMPILE + ${HYPERTABLE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp + ${HYPERTABLE_SOURCE_DIR}/cmake/CheckRRDtool.cc + CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${RRD_INCLUDE_DIR} + -DLINK_LIBRARIES=${RRD_LIBRARY} + OUTPUT_VARIABLE RRD_TRY_OUT) +string(REGEX REPLACE ".*\n([0-9.]+).*" "\\1" RRD_VERSION ${RRD_TRY_OUT}) +string(REGEX REPLACE ".*\n(RRDtool .*)" "\\1" RRD_VERSION ${RRD_VERSION}) +message(STATUS "RRDtool version: ${RRD_VERSION}") + +if (NOT RRD_CHECK STREQUAL "0") + message(FATAL_ERROR "Please fix the RRDtool installation, " + "remove CMakeCache.txt and try again.") +endif () + +STRING(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\1" MAJOR "${RRD_VERSION}") +STRING(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\2" MINOR "${RRD_VERSION}") +STRING(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\3" PATCH "${RRD_VERSION}") + +set(RRD_COMPATIBLE "YES") + +if (NOT RRD_COMPATIBLE) + message(FATAL_ERROR "RRD tool version >= required." + "Found version ${MAJOR}.${MINOR}.${PATCH}" + "Please fix the installation, remove CMakeCache.txt and try again.") +endif() + +mark_as_advanced( + RRD_LIBRARIES + RRD_INCLUDE_DIR + ) diff --git a/cmake/FindRubyThrift.cmake b/cmake/FindRubyThrift.cmake index 032ed4999..3868abef8 100644 --- a/cmake/FindRubyThrift.cmake +++ b/cmake/FindRubyThrift.cmake @@ -2,7 +2,7 @@ # This module defines # RUBYTHRIFT_FOUND, If false, do not try to use ant -exec_program(ruby ARGS -r thrift -e 0 OUTPUT_VARIABLE RUBYTHRIFT_OUT +exec_program(env ARGS ruby -r thrift -e 0 OUTPUT_VARIABLE RUBYTHRIFT_OUT RETURN_VALUE RUBYTHRIFT_RETURN) if (RUBYTHRIFT_RETURN STREQUAL "0") diff --git a/cmake/FindSIGAR.cmake b/cmake/FindSIGAR.cmake index 6d5527009..6b7d83c44 100644 --- a/cmake/FindSIGAR.cmake +++ b/cmake/FindSIGAR.cmake @@ -22,7 +22,10 @@ find_path(SIGAR_INCLUDE_DIR sigar.h set(SIGAR_NAMES sigar-x86-linux sigar-x86_64-linux sigar-amd64-linux - sigar-universal-macosx) + sigar-universal64-macosx + sigar-universal-macosx + sigar-x86-solaris + sigar) find_library(SIGAR_LIBRARY NAMES ${SIGAR_NAMES} PATHS /usr/lib /usr/local/lib /opt/local/lib diff --git a/cmake/FindTcmalloc.cmake b/cmake/FindTcmalloc.cmake index 528769173..f16b34eb8 100644 --- a/cmake/FindTcmalloc.cmake +++ b/cmake/FindTcmalloc.cmake @@ -5,21 +5,22 @@ # Tcmalloc_LIBRARIES - List of libraries when using Tcmalloc. # Tcmalloc_FOUND - True if Tcmalloc found. - -if (Tcmalloc_INCLUDE_DIR) - # Already in cache, be silent - set(Tcmalloc_FIND_QUIETLY TRUE) -endif () - -find_path(Tcmalloc_INCLUDE_DIR google/tcmalloc.h +find_path(Tcmalloc_INCLUDE_DIR google/tcmalloc.h NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_INCLUDE_DIR} + /usr/include /opt/local/include /usr/local/include ) -set(Tcmalloc_NAMES tcmalloc_minimal tcmalloc) -find_library(Tcmalloc_LIBRARY +if (USE_TCMALLOC) + set(Tcmalloc_NAMES tcmalloc) +else () + set(Tcmalloc_NAMES tcmalloc_minimal tcmalloc) +endif () + +find_library(Tcmalloc_LIBRARY NO_DEFAULT_PATH NAMES ${Tcmalloc_NAMES} - PATHS /usr/local/lib /opt/local/lib + PATHS ${HT_DEPENDENCY_LIB_DIR} /lib /usr/lib /usr/local/lib /opt/local/lib ) if (Tcmalloc_INCLUDE_DIR AND Tcmalloc_LIBRARY) @@ -31,29 +32,27 @@ else () endif () if (Tcmalloc_FOUND) - if (NOT Tcmalloc_FIND_QUIETLY) - message(STATUS "Found Tcmalloc: ${Tcmalloc_LIBRARY}") - try_run(TC_CHECK TC_CHECK_BUILD - ${HYPERTABLE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp - ${HYPERTABLE_SOURCE_DIR}/cmake/CheckTcmalloc.cc - CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${Tcmalloc_INCLUDE_DIR} - -DLINK_LIBRARIES=${Tcmalloc_LIBRARIES} - OUTPUT_VARIABLE TC_TRY_OUT) - #message("tc_check build: ${TC_CHECK_BUILD}") - #message("tc_check: ${TC_CHECK}") - #message("tc_version: ${TC_TRY_OUT}") - if (TC_CHECK_BUILD AND NOT TC_CHECK STREQUAL "0") - string(REGEX REPLACE ".*\n(Tcmalloc .*)" "\\1" TC_TRY_OUT ${TC_TRY_OUT}) - message(STATUS "${TC_TRY_OUT}") - message(FATAL_ERROR "Please fix the tcmalloc installation and try again.") - set(Tcmalloc_LIBRARIES) - endif () - string(REGEX REPLACE ".*\n([0-9]+[^\n]+).*" "\\1" TC_VERSION ${TC_TRY_OUT}) - if (NOT TC_VERSION MATCHES "^[0-9]+.*") - set(TC_VERSION "unknown -- make sure it's 1.1+") - endif () - message(STATUS " version: ${TC_VERSION}") + message(STATUS "Found Tcmalloc: ${Tcmalloc_LIBRARY}") + try_run(TC_CHECK TC_CHECK_BUILD + ${HYPERTABLE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp + ${HYPERTABLE_SOURCE_DIR}/cmake/CheckTcmalloc.cc + CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${Tcmalloc_INCLUDE_DIR} + -DLINK_LIBRARIES=${Tcmalloc_LIBRARIES} + OUTPUT_VARIABLE TC_TRY_OUT) + #message("tc_check build: ${TC_CHECK_BUILD}") + #message("tc_check: ${TC_CHECK}") + #message("tc_version: ${TC_TRY_OUT}") + if (TC_CHECK_BUILD AND NOT TC_CHECK STREQUAL "0") + string(REGEX REPLACE ".*\n(Tcmalloc .*)" "\\1" TC_TRY_OUT ${TC_TRY_OUT}) + message(STATUS "${TC_TRY_OUT}") + message(FATAL_ERROR "Please fix the tcmalloc installation and try again.") + set(Tcmalloc_LIBRARIES) + endif () + string(REGEX REPLACE ".*\n([0-9]+[^\n]+).*" "\\1" TC_VERSION ${TC_TRY_OUT}) + if (NOT TC_VERSION MATCHES "^[0-9]+.*") + set(TC_VERSION "unknown -- make sure it's 1.1+") endif () + message(STATUS " version: ${TC_VERSION}") else () message(STATUS "Not Found Tcmalloc: ${Tcmalloc_LIBRARY}") if (Tcmalloc_FIND_REQUIRED) diff --git a/cmake/FindThrift.cmake b/cmake/FindThrift.cmake index 8cd38cc9a..105156e39 100644 --- a/cmake/FindThrift.cmake +++ b/cmake/FindThrift.cmake @@ -5,17 +5,19 @@ # Thrift_LIBS, Thrift libraries # Thrift_FOUND, If false, do not try to use ant -exec_program(thrift ARGS -version OUTPUT_VARIABLE Thrift_VERSION +exec_program(env ARGS thrift -version OUTPUT_VARIABLE Thrift_VERSION RETURN_VALUE Thrift_RETURN) -find_path(Thrift_INCLUDE_DIR Thrift.h PATHS +find_path(Thrift_INCLUDE_DIR Thrift.h NO_DEFAULT_PATH PATHS + ${HT_DEPENDENCY_INCLUDE_DIR}/thrift /usr/local/include/thrift /opt/local/include/thrift + /usr/include/thrift ) -set(Thrift_LIB_PATHS /usr/local/lib /opt/local/lib) +set(Thrift_LIB_PATHS ${HT_DEPENDENCY_LIB_DIR} /usr/local/lib /opt/local/lib /usr/lib64) -find_library(Thrift_LIB NAMES thrift PATHS ${Thrift_LIB_PATHS}) +find_library(Thrift_LIB NAMES thrift NO_DEFAULT_PATH PATHS ${Thrift_LIB_PATHS}) find_library(Thrift_NB_LIB NAMES thriftnb PATHS ${Thrift_LIB_PATHS}) if (Thrift_VERSION MATCHES "^Thrift version" AND LibEvent_LIBS diff --git a/cmake/JavaHelper.cmake b/cmake/JavaHelper.cmake index 6738a232a..2d296d28e 100644 --- a/cmake/JavaHelper.cmake +++ b/cmake/JavaHelper.cmake @@ -16,3 +16,10 @@ configure_file(${HYPERTABLE_SOURCE_DIR}/build.xml.in build.xml @ONLY) add_custom_target(HypertableJavaComponents ALL ant -f build.xml -Dbuild.dir=${HYPERTABLE_BINARY_DIR}/java jar DEPENDS ${ThriftGenJava_SRCS}) +add_custom_target(HypertableJavaExamples ALL ant -f build.xml + -Dbuild.dir=${HYPERTABLE_BINARY_DIR}/java examples + DEPENDS ${ThriftGenJava_SRCS}) + +add_custom_target(java) +add_dependencies(java HypertableJavaExamples HypertableJavaComponents) + diff --git a/cmake/PackageHelper.cmake b/cmake/PackageHelper.cmake index cbc275962..20ebbf90b 100644 --- a/cmake/PackageHelper.cmake +++ b/cmake/PackageHelper.cmake @@ -16,7 +16,6 @@ # along with Hypertable. If not, see # -# Obtain the soname via objdump macro(HT_GET_SONAME var fpath) exec_program(${CMAKE_SOURCE_DIR}/bin/soname.sh ARGS ${fpath} OUTPUT_VARIABLE SONAME_OUT RETURN_VALUE SONAME_RETURN) @@ -35,7 +34,7 @@ macro(HT_GET_SONAME var fpath) endif () # check if the library is prelinked, if so, warn - exec_program(objdump ARGS -h ${fpath} OUTPUT_VARIABLE ODH_OUT + exec_program(env ARGS objdump -h ${fpath} OUTPUT_VARIABLE ODH_OUT RETURN_VALUE ODH_RETURN) if (ODH_RETURN STREQUAL "0") string(REGEX MATCH "prelink_undo" match ${ODH_OUT}) @@ -47,14 +46,20 @@ endmacro() # This is a workaround for install() which always preserves symlinks macro(HT_INSTALL_LIBS dest) - foreach(fpath ${ARGN}) - if (NOT ${fpath} MATCHES "(NOTFOUND|\\.a)$") - #message(STATUS "install copy: ${fpath}") - HT_GET_SONAME(soname ${fpath}) - configure_file(${fpath} "${dest}/${soname}" COPYONLY) - install(FILES "${CMAKE_BINARY_DIR}/${dest}/${soname}" DESTINATION ${dest}) - endif () - endforeach() + if (INSTALL_EXCLUDE_DEPENDENT_LIBS) + message(STATUS "Not installing dependent libraries") + else () + foreach(fpath ${ARGN}) + if (NOT ${fpath} MATCHES "(NOTFOUND|\\.a)$") + if (HT_CMAKE_DEBUG) + message(STATUS "install copy: ${fpath}") + endif () + HT_GET_SONAME(soname ${fpath}) + configure_file(${fpath} "${dest}/${soname}" COPYONLY) + install(FILES "${CMAKE_BINARY_DIR}/${dest}/${soname}" DESTINATION ${dest}) + endif () + endforeach() + endif () endmacro() # Dependent libraries @@ -62,7 +67,7 @@ HT_INSTALL_LIBS(lib ${BOOST_LIBS} ${BDB_LIBRARIES} ${Thrift_LIBS} ${Kfs_LIBRARIES} ${LibEvent_LIB} ${Log4cpp_LIBRARIES} ${READLINE_LIBRARIES} ${EXPAT_LIBRARIES} ${BZIP2_LIBRARIES} ${ZLIB_LIBRARIES} ${SIGAR_LIBRARY} ${Tcmalloc_LIBRARIES} - ${Ceph_LIBRARIES}) + ${Ceph_LIBRARIES} ${RRD_LIBRARIES} ${RE2_LIBRARIES}) # Need to include some "system" libraries as well exec_program(${CMAKE_SOURCE_DIR}/bin/ldd.sh @@ -74,20 +79,74 @@ if (HT_CMAKE_DEBUG) endif () if (LDD_RETURN STREQUAL "0") - string(REGEX MATCH "[ \t](/[^ ]+/libgcc_s\\.[^ ]+)" dummy ${LDD_OUT}) + string(REGEX MATCH "[ \t](/[^ ]+/libgcc_s\\.[^ \n]+)" dummy ${LDD_OUT}) set(gcc_s_lib ${CMAKE_MATCH_1}) - string(REGEX MATCH "[ \t](/[^ ]+/libstdc\\+\\+\\.[^ ]+)" dummy ${LDD_OUT}) + string(REGEX MATCH "[ \t](/[^ ]+/libstdc\\+\\+\\.[^ \n]+)" dummy ${LDD_OUT}) set(stdcxx_lib ${CMAKE_MATCH_1}) - HT_INSTALL_LIBS(lib ${gcc_s_lib} ${stdcxx_lib}) + string(REGEX MATCH "[ \t](/[^ ]+/libstacktrace\\.[^ \n]+)" dummy ${LDD_OUT}) + set(stacktrace_lib ${CMAKE_MATCH_1}) + HT_INSTALL_LIBS(lib ${gcc_s_lib} ${stdcxx_lib} ${stacktrace_lib}) +endif () + +# Include other RRDTool dependencies found in Hypertable.Master +exec_program(${CMAKE_SOURCE_DIR}/bin/ldd.sh + ARGS ${CMAKE_BINARY_DIR}/src/cc/Hypertable/Master/Hypertable.Master + OUTPUT_VARIABLE LDD_OUT RETURN_VALUE LDD_RETURN) + +if (HT_CMAKE_DEBUG) + message("ldd.sh output: ${LDD_OUT}") endif () +if (LDD_RETURN STREQUAL "0") + string(REGEX MATCH "[ \t](/[^ ]+/libpangocairo-[^ \n]+)" dummy ${LDD_OUT}) + set(pangocairo_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libpango-[^ \n]+)" dummy ${LDD_OUT}) + set(pango_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libcairo\\.[^ \n]+)" dummy ${LDD_OUT}) + set(cairo_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libfontconfig\\.[^ \n]+)" dummy ${LDD_OUT}) + set(fontconfig_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libXrender\\.[^ \n]+)" dummy ${LDD_OUT}) + set(Xrender_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libX11\\.[^ \n]+)" dummy ${LDD_OUT}) + set(X11_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libxml2\\.[^ \n]+)" dummy ${LDD_OUT}) + set(xml2_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libpixman-[^ \n]+)" dummy ${LDD_OUT}) + set(pixman_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libgobject-[^ \n]+)" dummy ${LDD_OUT}) + set(gobject_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libgmodule-[^ \n]+)" dummy ${LDD_OUT}) + set(gmodule_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libglib-[^ \n]+)" dummy ${LDD_OUT}) + set(glib_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libpangoft2-[^ \n]+)" dummy ${LDD_OUT}) + set(pangoft2_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libxcb-xlib\\.[^ \n]+)" dummy ${LDD_OUT}) + set(xcb_xlib_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libxcb\\.[^ \n]+)" dummy ${LDD_OUT}) + set(xcb_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libpcre\\.[^ \n]+)" dummy ${LDD_OUT}) + set(pcre_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libXau\\.[^ \n]+)" dummy ${LDD_OUT}) + set(Xau_lib ${CMAKE_MATCH_1}) + string(REGEX MATCH "[ \t](/[^ ]+/libXdmcp\\.[^ \n]+)" dummy ${LDD_OUT}) + set(Xdmcp_lib ${CMAKE_MATCH_1}) + HT_INSTALL_LIBS(lib ${pangocairo_lib} ${pango_lib} ${cairo_lib} + ${fontconfig_lib} ${Xrender_lib} ${X11_lib} ${xml2_lib} + ${pixman_lib} ${gobject_lib} ${gmodule_lib} ${glib_lib} + ${pangoft2_lib} ${xcb_xlib_lib} ${xcb_lib} ${pcre_lib} + ${Xau_lib} ${Xdmcp_lib}) +endif () + + # General package variables if (NOT CPACK_PACKAGE_NAME) set(CPACK_PACKAGE_NAME "hypertable") endif () if (NOT CPACK_PACKAGE_CONTACT) - set(CPACK_PACKAGE_CONTACT "llu@hypertable.org") + set(CPACK_PACKAGE_CONTACT "doug@hypertable.org") endif () if (NOT CPACK_PACKAGE_DESCRIPTION_SUMMARY) @@ -127,7 +186,7 @@ set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) # Debian pakcage variables set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA - "${CMAKE_BINARY_DIR}/postinst;${CMAKE_BINARY_DIR}/prerm") + "${CMAKE_BINARY_DIR}/prerm") # RPM package variables set(CPACK_RPM_PACKAGE_LICENSE "GPLv2+") @@ -135,14 +194,15 @@ set(CPACK_RPM_PACKAGE_GROUP "Applications/Databases") # rpm perl dependencies stuff is dumb set(CPACK_RPM_SPEC_MORE_DEFINE " +Provides: perl(Thrift) +Provides: perl(Thrift::BinaryProtocol) +Provides: perl(Thrift::FramedTransport) +Provides: perl(Thrift::Socket) Provides: perl(Hypertable::ThriftGen2::HqlService) Provides: perl(Hypertable::ThriftGen2::Types) Provides: perl(Hypertable::ThriftGen::ClientService) Provides: perl(Hypertable::ThriftGen::Types) -%post -${CMAKE_INSTALL_PREFIX}/bin/fhsize.sh - %preun ${CMAKE_INSTALL_PREFIX}/bin/prerm.sh") diff --git a/cmake/TestHelper.cmake b/cmake/TestHelper.cmake index 3d181da8b..a6383016a 100644 --- a/cmake/TestHelper.cmake +++ b/cmake/TestHelper.cmake @@ -32,7 +32,7 @@ add_custom_target(runtestservers DEPENDS ${TEST_SERVERS_STARTED}) macro(add_test_target target dir) add_custom_target(${target}) add_dependencies(${target} runtestservers) - add_custom_command(TARGET ${target} POST_BUILD COMMAND make test + add_custom_command(TARGET ${target} POST_BUILD COMMAND ${INSTALL_DIR}/bin/ht $(MAKE) test WORKING_DIRECTORY ${dir}) endmacro() diff --git a/cmake/VersionHelper.cmake b/cmake/VersionHelper.cmake index cab689750..c9ed9ef96 100644 --- a/cmake/VersionHelper.cmake +++ b/cmake/VersionHelper.cmake @@ -16,7 +16,24 @@ # along with Hypertable. If not, see # # figure out version info from the repository -set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_MICRO}.${VERSION_PATCH}) + +if (VERSION_ADD_COMMIT_SUFFIX) + exec_program(${HYPERTABLE_SOURCE_DIR}/bin/src-utils/ver ${HYPERTABLE_SOURCE_DIR} + ARGS "--commit-only" OUTPUT_VARIABLE HT_COMMIT_STRING) + if (VERSION_MISC_SUFFIX) + set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_MICRO}.${VERSION_PATCH}.${VERSION_MISC_SUFFIX.${HT_COMMIT_STRING}) + else () + set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_MICRO}.${VERSION_PATCH}}.${HT_COMMIT_STRING}) + endif () +else () + if (VERSION_MISC_SUFFIX) + set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_MICRO}.${VERSION_PATCH}.${VERSION_MISC_SUFFIX}) + else () + set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_MICRO}.${VERSION_PATCH}) + endif () +endif () + + exec_program(${HYPERTABLE_SOURCE_DIR}/bin/src-utils/ver ${HYPERTABLE_SOURCE_DIR} OUTPUT_VARIABLE HT_GIT_VERSION RETURN_VALUE GIT_RETURN) diff --git a/conf/Capfile.cluster b/conf/Capfile.cluster index 853b44bab..bd9ead832 100644 --- a/conf/Capfile.cluster +++ b/conf/Capfile.cluster @@ -1,17 +1,40 @@ set :source_machine, "motherlode010" set :install_dir, "/opt/hypertable" -set :hypertable_version, "0.9.2.7" +set :hypertable_version, "0.9.5.0" set :default_dfs, "hadoop" set :default_config, "/opt/hypertable/production-hypertable.cfg" +set :default_additional_args, "" +set :hbase_home, "/opt/hbase/current" +set :default_client_multiplier, 1 +set :default_test_driver, "hypertable" +set :default_test_args, "" +role :source, "motherlode010" role :master, "motherlode010" +role :hyperspace, "motherlode010", "motherlode011", "motherlode012" role :slave, "motherlode010", "motherlode011", "motherlode012", \ "motherlode013", "motherlode014", "motherlode015", \ "motherlode016", "motherlode017", "motherlode019" +role :thriftbroker, "www001", "www002", "www003", "www004", "www005" +role :spare, "motherlode018", "motherlode020" role :localhost, "motherlode010" +role :test_client, "test001", "test002", "test003", "test004" +role :test_dispatcher, "test000" ######################### END OF USER CONFIGURATION ############################ +def install_machines + (roles[:master].servers | \ + roles[:hyperspace].servers | \ + roles[:slave].servers | \ + roles[:thriftbroker].servers | \ + roles[:spare].servers | \ + roles[:test_client].servers | \ + roles[:test_dispatcher].servers) - roles[:source].servers +end + +role(:install) { install_machines } + set(:dfs) do "#{default_dfs}" end unless exists?(:dfs) @@ -20,6 +43,22 @@ set(:config) do "#{default_config}" end unless exists?(:config) +set(:additional_args) do + "#{default_additional_args}" +end unless exists?(:additional_args) + +set(:test_driver) do + "#{default_test_driver}" +end unless exists?(:test_driver) + +set(:test_args) do + "#{default_test_args}" +end unless exists?(:test_args) + +set(:client_multiplier) do + "#{default_client_multiplier}".to_i +end unless exists?(:client_multiplier) + set :config_file, "#{config}".split('/')[-1] set :config_option, \ "--config=#{install_dir}/#{hypertable_version}/conf/#{config_file}" @@ -32,7 +71,7 @@ set :config_option, \ (default=#{install_dir}) DESC task :copy_config, :roles => :localhost do - run("rsync #{config} #{install_dir}/#{hypertable_version}/conf") + run("rsync -e \"ssh -o StrictHostKeyChecking=no\" #{config} #{install_dir}/#{hypertable_version}/conf/") end desc <<-DESC @@ -41,10 +80,11 @@ end installation machine specified by the variable 'source_machine' \ (default=#{source_machine}) DESC -task :rsync_installation, :roles => [:master, :slave] do - run "rsync -av --exclude=log --exclude=run --exclude=demo --exclude=fs " \ - "--exclude=hyperspace " \ - "#{source_machine}:#{install_dir}/#{hypertable_version} #{install_dir}" +task :rsync, :roles => :install do + run <<-CMD + rsync -av -e "ssh -o StrictHostKeyChecking=no" --exclude=log --exclude=run --exclude=demo --exclude=fs --exclude=hyperspace/ #{source_machine}:#{install_dir}/#{hypertable_version} #{install_dir} && + rsync -av -e "ssh -o StrictHostKeyChecking=no" --exclude=log --exclude=run --exclude=demo --exclude=fs --exclude=hyperspace/ #{source_machine}:#{install_dir}/#{hypertable_version}/conf/ #{install_dir}/#{hypertable_version}/conf + CMD end desc <<-DESC @@ -52,7 +92,7 @@ end to point to the directory of the current version (default=#{hypertable_version}) DESC -task :set_current, :roles => [:master, :slave] do +task :set_current, :roles => [:install, :source] do run <<-CMD cd #{install_dir} && rm -f current && @@ -67,33 +107,116 @@ end task :dist do transaction do copy_config - rsync_installation + rsync + end +end + + desc <<-DESC + Distributes and fhsizes the installation, then copies + config and rsyncs + DESC +task :fhsize do + transaction do + rsync + fhsize_install + copy_config + rsync + end +end + + + desc <<-DESC + fhsize's the installations + DESC +task :fhsize_install, :roles => [:install, :source] do + run <<-CMD + #{install_dir}/#{hypertable_version}/bin/fhsize.sh + CMD +end + +desc "Verify that upgrade is OK." +task :qualify_upgrade, :roles => :source do + run <<-CMD + #{install_dir}/#{hypertable_version}/bin/upgrade-ok.sh #{install_dir}/current #{hypertable_version} + CMD +end + + desc <<-DESC + Upgrades installation. Stops servers, copies config, rsyncs + the installation, then copies hyperspace and the rangeserver + state in the run/ directory to new installation + DESC +task :upgrade do + transaction do + qualify_upgrade + stop + copy_config + rsync + upgrade_hyperspace + upgrade_rangeservers set_current end end + desc <<-DESC + Upgrades (copies) the Hyperspace database from the current + installation to the new installation specified by the + hypertable_version (#{hypertable_version}) + DESC +task :upgrade_hyperspace, :roles => :hyperspace do + run <<-CMD + cp -dpR #{install_dir}/current/hyperspace \ + #{install_dir}/#{hypertable_version} + CMD +end + + desc <<-DESC + Upgrades (copies) the RangeServers by copying the contents + of the run directory from the current installation to + installation specified by the hypertable_version + (#{hypertable_version}) + DESC +task :upgrade_rangeservers, :roles => :slave do + run <<-CMD + cp -dpR #{install_dir}/current/run \ + #{install_dir}/#{hypertable_version} + CMD +end + desc "Starts all processes." task :start do transaction do + start_hyperspace start_master start_slaves + start_master_thriftbroker end end +desc "Starts hyperspace processes." +task :start_hyperspace, :roles => :hyperspace do + run <<-CMD + #{install_dir}/current/bin/start-hyperspace.sh \ + #{config_option} + CMD +end + desc "Starts master processes." task :start_master, :roles => :master do run <<-CMD #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ #{config_option} && - #{install_dir}/current/bin/start-hyperspace.sh \ - #{config_option} && - #{install_dir}/current/bin/start-master.sh #{config_option}; + #{install_dir}/current/bin/start-master.sh #{config_option} && + #{install_dir}/current/bin/start-monitoring.sh CMD end -desc "Stops master processes." -task :stop_master, :roles => :master do - run "#{install_dir}/current/bin/stop-servers.sh" +desc "Starts ThriftBroker on master." +task :start_master_thriftbroker, :roles => :master do + run <<-CMD + #{install_dir}/current/bin/start-thriftbroker.sh \ + #{config_option} + CMD end desc "Starts slave processes." @@ -109,31 +232,101 @@ task :start_slaves, :roles => :slave do CMD end -desc "Stops slave processes." -task :stop_slaves, :roles => :slave do - run "#{install_dir}/current/bin/stop-servers.sh" +desc "Starts ThriftBroker processes." +task :start_thriftbrokers, :roles => :thriftbroker do + run <<-CMD + #{install_dir}/current/bin/random-wait.sh 5 && + #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ + #{config_option} && + #{install_dir}/current/bin/start-thriftbroker.sh \ + #{config_option} + CMD +end + + +desc "Starts DFS brokers." +task :start_dfsbrokers, :roles => [:master, :slave] do + run "#{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ + #{config_option}" end desc "Stops all servers." -task :stop, :roles => [:master, :slave] do +task :stop do transaction do - stop_slaves stop_master + stop_slaves + stop_hyperspace + stop_dfsbrokers end end -desc "Starts DFS brokers." -task :start_dfsbrokers, :roles => [:master, :slave] do - run "#{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ - #{config_option}" +desc "Stops DFS brokers." +task :stop_dfsbrokers, :roles => [:master, :slave] do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh #{additional_args} + CMD +end + + +desc "Stops slave processes." +task :stop_slaves, :roles => :slave do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master --no-dfsbroker #{additional_args} + CMD +end + +desc "Stops master processes." +task :stop_master, :roles => :master do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-rangeserver --no-dfsbroker #{additional_args} && + #{install_dir}/current/bin/stop-monitoring.sh + CMD +end + +desc "Stops hyperspace processes." +task :stop_hyperspace, :roles => :hyperspace do + run <<-CMD + #{install_dir}/current/bin/stop-hyperspace.sh + CMD +end + +desc "Stops ThriftBroker processes." +task :stop_thriftbrokers, :roles => :thriftbroker do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master --no-rangeserver + CMD end -desc "Cleans database, removing all tables." -task :cleandb, :roles => [:master, :slave] do +desc "Cleans hyperspace & rangeservers, removing all tables." +task :cleandb do + transaction do + clean_master + clean_hyperspace + clean_slaves + end +end + +desc "Cleans master state but not hyperspace." +task :clean_master, :roles => :master do run <<-CMD #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ #{config_option} && \ - #{install_dir}/current/bin/clean-database.sh + #{install_dir}/current/bin/clean-database.sh #{config_option} ; + CMD +end + +desc "Cleans hyperspace." +task :clean_hyperspace, :roles => :hyperspace do + run <<-CMD + #{install_dir}/current/bin/clean-hyperspace.sh + CMD +end + +desc "Cleans rangeservers and master state but not hyperspace." +task :clean_slaves, :roles => :slave do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master && + rm -rf #{install_dir}/current/run/* CMD end @@ -162,7 +355,7 @@ task :master_status, :roles => [:master] do end desc "Get status for Hyperspace.Master process." -task :hyperspace_status, :roles => [:master] do +task :hyperspace_status, :roles => [:hyperspace] do run <<-CMD #{install_dir}/current/bin/ht serverup hyperspace CMD @@ -184,6 +377,59 @@ end unless exists?(:dumpfile) desc "Run dump command on each rangeserver" task :rangeserver_dump, :roles => [:slave] do run <<-CMD - echo "dump '#{dumpfile}';" | #{install_dir}/current/bin/ht_rsclient --batch #{config_option} + echo "dump NOKEYS '#{dumpfile}';" | #{install_dir}/current/bin/ht ht_rsclient --batch #{config_option} + CMD +end + + +if "#{test_driver}" == "hypertable" + set :thrift_broker_command, "#{install_dir}/current/bin/start-thriftbroker.sh #{config_option}" + set :start_test_client_command, "#{install_dir}/current/bin/start-test-client.sh --count #{client_multiplier} #{roles[:test_dispatcher].servers[0]}" + set :run_test_dispatcher_command, "#{install_dir}/current/bin/jrun --pidfile #{install_dir}/#{hypertable_version}/run/Hypertable.TestDispatcher.pid org.hypertable.examples.PerformanceTest.Dispatcher --driver=#{test_driver} --clients=#{roles[:test_client].servers.length*client_multiplier} #{test_args}" + set :stop_test_args, "" +elsif "#{test_driver}" == "hbase" + set :thrift_broker_command, "echo -n" + set :start_test_client_command, "#{install_dir}/current/bin/start-test-client.sh --jrun-opts \"--add-to-classpath #{hbase_home}/conf\" --count #{client_multiplier} #{roles[:test_dispatcher].servers[0]}" + set :run_test_dispatcher_command, "#{install_dir}/current/bin/jrun --pidfile #{install_dir}/#{hypertable_version}/run/Hypertable.TestDispatcher.pid --add-to-classpath #{hbase_home}/conf org.hypertable.examples.PerformanceTest.Dispatcher --driver=#{test_driver} --clients=#{roles[:test_client].servers.length*client_multiplier} #{test_args}" + set :stop_test_args, "--no-thriftbroker --no-dfsbroker" +else + set :thrift_broker_command, "echo Invalid test driver - #{test_driver}" + set :start_test_client_command, "echo Invalid test driver - #{test_driver}" + set :run_test_dispatcher_command, "echo Invalid test driver - #{test_driver}" + set :stop_test_args, "--no-thriftbroker --no-dfsbroker" +end + +desc "Starts test clients." +task :start_test_clients, :roles => :test_client do + run <<-CMD + #{install_dir}/current/bin/random-wait.sh 5 && + #{thrift_broker_command} && + #{start_test_client_command} CMD end + +desc "Run test dispatcher." +task :run_test_dispatcher, :roles => :test_dispatcher do + run <<-CMD + #{thrift_broker_command} && + #{run_test_dispatcher_command} + CMD +end + +desc "Stops test." +task :stop_test, :roles => [:test_client, :test_dispatcher] do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master --no-rangeserver #{stop_test_args} + CMD +end + +desc "Run test" +task :run_test do + transaction do + stop_test + start_test_clients + run_test_dispatcher + end +end + + diff --git a/conf/Capfile.localhost b/conf/Capfile.localhost index 6583324f4..5b6fc23b2 100644 --- a/conf/Capfile.localhost +++ b/conf/Capfile.localhost @@ -1,15 +1,38 @@ set :source_machine, "localhost" set :install_dir, "/opt/hypertable" -set :hypertable_version, "0.9.2.7" +set :hypertable_version, "0.9.5.0" set :default_dfs, "local" set :default_config, "/opt/hypertable/#{hypertable_version}/conf/hypertable.cfg" +set :default_additional_args, "" +set :hbase_home, "/opt/hbase/current" +set :default_client_multiplier, 1 +set :default_test_driver, "hypertable" +set :default_test_args, "" +role :source, "localhost" role :master, "localhost" +role :hyperspace, "localhost" role :slave, "localhost" +role :thriftbroker +role :spare role :localhost, "localhost" +role :test_client, "localhost" +role :test_dispatcher, "localhost" ######################### END OF USER CONFIGURATION ############################ +def install_machines + (roles[:master].servers | \ + roles[:hyperspace].servers | \ + roles[:slave].servers | \ + roles[:thriftbroker].servers | \ + roles[:spare].servers | \ + roles[:test_client].servers | \ + roles[:test_dispatcher].servers) - roles[:source].servers +end + +role(:install) { install_machines } + set(:dfs) do "#{default_dfs}" end unless exists?(:dfs) @@ -18,6 +41,22 @@ set(:config) do "#{default_config}" end unless exists?(:config) +set(:additional_args) do + "#{default_additional_args}" +end unless exists?(:additional_args) + +set(:test_driver) do + "#{default_test_driver}" +end unless exists?(:test_driver) + +set(:test_args) do + "#{default_test_args}" +end unless exists?(:test_args) + +set(:client_multiplier) do + "#{default_client_multiplier}".to_i +end unless exists?(:client_multiplier) + set :config_file, "#{config}".split('/')[-1] set :config_option, \ "--config=#{install_dir}/#{hypertable_version}/conf/#{config_file}" @@ -30,7 +69,7 @@ set :config_option, \ (default=#{install_dir}) DESC task :copy_config, :roles => :localhost do - run("rsync #{config} #{install_dir}/#{hypertable_version}/conf") + run("rsync -e \"ssh -o StrictHostKeyChecking=no\" #{config} #{install_dir}/#{hypertable_version}/conf/") end desc <<-DESC @@ -39,10 +78,11 @@ end installation machine specified by the variable 'source_machine' \ (default=#{source_machine}) DESC -task :rsync_installation, :roles => [:master, :slave] do - run "rsync -av --exclude=log --exclude=run --exclude=demo --exclude=fs " \ - "--exclude=hyperspace " \ - "#{source_machine}:#{install_dir}/#{hypertable_version} #{install_dir}" +task :rsync, :roles => :install do + run <<-CMD + rsync -av -e "ssh -o StrictHostKeyChecking=no" --exclude=log --exclude=run --exclude=demo --exclude=fs --exclude=hyperspace/ #{source_machine}:#{install_dir}/#{hypertable_version} #{install_dir} && + rsync -av -e "ssh -o StrictHostKeyChecking=no" --exclude=log --exclude=run --exclude=demo --exclude=fs --exclude=hyperspace/ #{source_machine}:#{install_dir}/#{hypertable_version}/conf/ #{install_dir}/#{hypertable_version}/conf + CMD end desc <<-DESC @@ -50,7 +90,7 @@ end to point to the directory of the current version (default=#{hypertable_version}) DESC -task :set_current, :roles => [:master, :slave] do +task :set_current, :roles => [:install, :source] do run <<-CMD cd #{install_dir} && rm -f current && @@ -65,33 +105,116 @@ end task :dist do transaction do copy_config - rsync_installation + rsync + end +end + + desc <<-DESC + Distributes and fhsizes the installation, then copies + config and rsyncs + DESC +task :fhsize do + transaction do + rsync + fhsize_install + copy_config + rsync + end +end + + + desc <<-DESC + fhsize's the installations + DESC +task :fhsize_install, :roles => [:install, :source] do + run <<-CMD + #{install_dir}/#{hypertable_version}/bin/fhsize.sh + CMD +end + +desc "Verify that upgrade is OK." +task :qualify_upgrade, :roles => :source do + run <<-CMD + #{install_dir}/#{hypertable_version}/bin/upgrade-ok.sh #{install_dir}/current #{hypertable_version} + CMD +end + + desc <<-DESC + Upgrades installation. Stops servers, copies config, rsyncs + the installation, then copies hyperspace and the rangeserver + state in the run/ directory to new installation + DESC +task :upgrade do + transaction do + qualify_upgrade + stop + copy_config + rsync + upgrade_hyperspace + upgrade_rangeservers set_current end end + desc <<-DESC + Upgrades (copies) the Hyperspace database from the current + installation to the new installation specified by the + hypertable_version (#{hypertable_version}) + DESC +task :upgrade_hyperspace, :roles => :hyperspace do + run <<-CMD + cp -dpR #{install_dir}/current/hyperspace \ + #{install_dir}/#{hypertable_version} + CMD +end + + desc <<-DESC + Upgrades (copies) the RangeServers by copying the contents + of the run directory from the current installation to + installation specified by the hypertable_version + (#{hypertable_version}) + DESC +task :upgrade_rangeservers, :roles => :slave do + run <<-CMD + cp -dpR #{install_dir}/current/run \ + #{install_dir}/#{hypertable_version} + CMD +end + desc "Starts all processes." task :start do transaction do + start_hyperspace start_master start_slaves + start_master_thriftbroker end end +desc "Starts hyperspace processes." +task :start_hyperspace, :roles => :hyperspace do + run <<-CMD + #{install_dir}/current/bin/start-hyperspace.sh \ + #{config_option} + CMD +end + desc "Starts master processes." task :start_master, :roles => :master do run <<-CMD #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ #{config_option} && - #{install_dir}/current/bin/start-hyperspace.sh \ - #{config_option} && - #{install_dir}/current/bin/start-master.sh #{config_option}; + #{install_dir}/current/bin/start-master.sh #{config_option} && + #{install_dir}/current/bin/start-monitoring.sh CMD end -desc "Stops master processes." -task :stop_master, :roles => :master do - run "#{install_dir}/current/bin/stop-servers.sh" +desc "Starts ThriftBroker on master." +task :start_master_thriftbroker, :roles => :master do + run <<-CMD + #{install_dir}/current/bin/start-thriftbroker.sh \ + #{config_option} + CMD end desc "Starts slave processes." @@ -107,31 +230,100 @@ task :start_slaves, :roles => :slave do CMD end -desc "Stops slave processes." -task :stop_slaves, :roles => :slave do - run "#{install_dir}/current/bin/stop-servers.sh" +desc "Starts ThriftBroker processes." +task :start_thriftbrokers, :roles => :thriftbroker do + run <<-CMD + #{install_dir}/current/bin/random-wait.sh 5 && + #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ + #{config_option} && + #{install_dir}/current/bin/start-thriftbroker.sh \ + #{config_option} + CMD +end + + +desc "Starts DFS brokers." +task :start_dfsbrokers, :roles => [:master, :slave] do + run "#{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ + #{config_option}" end desc "Stops all servers." -task :stop, :roles => [:master, :slave] do +task :stop do transaction do - stop_slaves stop_master + stop_slaves + stop_hyperspace + stop_dfsbrokers end end -desc "Starts DFS brokers." -task :start_dfsbrokers, :roles => [:master, :slave] do - run "#{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ - #{config_option}" +desc "Stops DFS brokers." +task :stop_dfsbrokers, :roles => [:master, :slave] do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh #{additional_args} + CMD +end + +desc "Stops slave processes." +task :stop_slaves, :roles => :slave do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master --no-dfsbroker #{additional_args} + CMD end -desc "Cleans database, removing all tables." -task :cleandb, :roles => [:master, :slave] do +desc "Stops master processes." +task :stop_master, :roles => :master do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-rangeserver --no-dfsbroker #{additional_args} && + #{install_dir}/current/bin/stop-monitoring.sh + CMD +end + +desc "Stops hyperspace processes." +task :stop_hyperspace, :roles => :hyperspace do + run <<-CMD + #{install_dir}/current/bin/stop-hyperspace.sh + CMD +end + +desc "Stops ThriftBroker processes." +task :stop_thriftbrokers, :roles => :thriftbroker do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master --no-rangeserver + CMD +end + +desc "Cleans hyperspace & rangeservers, removing all tables." +task :cleandb do + transaction do + clean_master + clean_hyperspace + clean_slaves + end +end + +desc "Cleans master state but not hyperspace." +task :clean_master, :roles => :master do run <<-CMD #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \ #{config_option} && \ - #{install_dir}/current/bin/clean-database.sh + #{install_dir}/current/bin/clean-database.sh #{config_option} ; + CMD +end + +desc "Cleans hyperspace." +task :clean_hyperspace, :roles => :hyperspace do + run <<-CMD + #{install_dir}/current/bin/clean-hyperspace.sh + CMD +end + +desc "Cleans rangeservers and master state but not hyperspace." +task :clean_slaves, :roles => :slave do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master && + rm -rf #{install_dir}/current/run/* CMD end @@ -160,7 +352,7 @@ task :master_status, :roles => [:master] do end desc "Get status for Hyperspace.Master process." -task :hyperspace_status, :roles => [:master] do +task :hyperspace_status, :roles => [:hyperspace] do run <<-CMD #{install_dir}/current/bin/ht serverup hyperspace CMD @@ -182,6 +374,59 @@ end unless exists?(:dumpfile) desc "Run dump command on each rangeserver" task :rangeserver_dump, :roles => [:slave] do run <<-CMD - echo "dump '#{dumpfile}';" | #{install_dir}/current/bin/ht_rsclient --batch #{config_option} + echo "dump NOKEYS '#{dumpfile}';" | #{install_dir}/current/bin/ht ht_rsclient --batch #{config_option} CMD end + + +if "#{test_driver}" == "hypertable" + set :thrift_broker_command, "#{install_dir}/current/bin/start-thriftbroker.sh #{config_option}" + set :start_test_client_command, "#{install_dir}/current/bin/start-test-client.sh --count #{client_multiplier} #{roles[:test_dispatcher].servers[0]}" + set :run_test_dispatcher_command, "#{install_dir}/current/bin/jrun --pidfile #{install_dir}/#{hypertable_version}/run/Hypertable.TestDispatcher.pid org.hypertable.examples.PerformanceTest.Dispatcher --driver=#{test_driver} --clients=#{roles[:test_client].servers.length*client_multiplier} #{test_args}" + set :stop_test_args, "" +elsif "#{test_driver}" == "hbase" + set :thrift_broker_command, "echo -n" + set :start_test_client_command, "#{install_dir}/current/bin/start-test-client.sh --jrun-opts \"--add-to-classpath #{hbase_home}/conf\" --count #{client_multiplier} #{roles[:test_dispatcher].servers[0]}" + set :run_test_dispatcher_command, "#{install_dir}/current/bin/jrun --pidfile #{install_dir}/#{hypertable_version}/run/Hypertable.TestDispatcher.pid --add-to-classpath #{hbase_home}/conf org.hypertable.examples.PerformanceTest.Dispatcher --driver=#{test_driver} --clients=#{roles[:test_client].servers.length*client_multiplier} #{test_args}" + set :stop_test_args, "--no-thriftbroker --no-dfsbroker" +else + set :thrift_broker_command, "echo Invalid test driver - #{test_driver}" + set :start_test_client_command, "echo Invalid test driver - #{test_driver}" + set :run_test_dispatcher_command, "echo Invalid test driver - #{test_driver}" + set :stop_test_args, "--no-thriftbroker --no-dfsbroker" +end + +desc "Starts test clients." +task :start_test_clients, :roles => :test_client do + run <<-CMD + #{install_dir}/current/bin/random-wait.sh 5 && + #{thrift_broker_command} && + #{start_test_client_command} + CMD +end + +desc "Run test dispatcher." +task :run_test_dispatcher, :roles => :test_dispatcher do + run <<-CMD + #{thrift_broker_command} && + #{run_test_dispatcher_command} + CMD +end + +desc "Stops test." +task :stop_test, :roles => [:test_client, :test_dispatcher] do + run <<-CMD + #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master --no-rangeserver #{stop_test_args} + CMD +end + +desc "Run test" +task :run_test do + transaction do + stop_test + start_test_clients + run_test_dispatcher + end +end + + diff --git a/conf/METADATA.xml b/conf/METADATA.xml index 8a6e31528..bc9a65929 100644 --- a/conf/METADATA.xml +++ b/conf/METADATA.xml @@ -1,25 +1,47 @@ - + - + + 1 LogDir + false + false - + + 1 Files + false + false + + + 2 + NextCSID + 1 + false + false - + + 1 StartRow + false 5 + false - + + 1 Location + false 5 + false - + + 1 Event + false + false diff --git a/conf/RS_METRICS.xml b/conf/RS_METRICS.xml new file mode 100644 index 000000000..0628b9585 --- /dev/null +++ b/conf/RS_METRICS.xml @@ -0,0 +1,34 @@ + + + + 1 + server + false + 336 + false + + + + + 1 + range + false + 24 + false + + + 1 + range_start_row + false + 1 + false + + + 1 + range_move + false + 1 + false + + + diff --git a/conf/hypertable.cfg b/conf/hypertable.cfg index 55d86b9d4..0ae7a12b2 100644 --- a/conf/hypertable.cfg +++ b/conf/hypertable.cfg @@ -2,35 +2,25 @@ # hypertable.cfg # -# Global properties -Hypertable.Request.Timeout=180000 - # HDFS Broker -HdfsBroker.Port=38030 HdfsBroker.fs.default.name=hdfs://localhost:9000 -HdfsBroker.Workers=20 # Ceph Broker -CephBroker.Port=38030 -CephBroker.Workers=20 CephBroker.MonAddr=10.0.1.245:6789 # Local Broker -DfsBroker.Local.Port=38030 DfsBroker.Local.Root=fs/local # DFS Broker - for clients -DfsBroker.Host=localhost DfsBroker.Port=38030 # Hyperspace -Hyperspace.Master.Host=localhost -Hyperspace.Master.Port=38040 -Hyperspace.Master.Dir=hyperspace -Hyperspace.Master.Workers=20 +Hyperspace.Replica.Host=localhost +Hyperspace.Replica.Port=38040 +Hyperspace.Replica.Dir=hyperspace +Hyperspace.Replica.Workers=20 # Hypertable.Master -Hypertable.Master.Host=localhost Hypertable.Master.Port=38050 Hypertable.Master.Workers=20 diff --git a/contrib/rb/Monitoring/README.mdwn b/contrib/rb/Monitoring/README.mdwn new file mode 100644 index 000000000..db8c5703c --- /dev/null +++ b/contrib/rb/Monitoring/README.mdwn @@ -0,0 +1,46 @@ +Hypertable Monitoring +===================== + +in development + + +# to run + +* install required tools + + sudo port install rrdtool + + + * OS X install error encountered: + + ... + Image error: /opt/local/lib/perl5/5.8.9/darwin-2level/perllocal.pod already exists and does not belong to a registered port. Unable to activate port + ... + + resolved with: + + mv /opt/local/lib/perl5/5.8.9/darwin-2level/perllocal.pod /opt/local/lib/perl5/5.8.9/darwin-2level/perllocal.old.pod + + +* install rrdtool ruby bindings OS X notes + + * [http://oss.oetiker.ch/rrdtool/prog/rrdruby.en.html](http://oss.oetiker.ch/rrdtool/prog/rrdruby.en.html) + + * Find the ruby bindings in the bindings directory of the rrdtool source. [http://oss.oetiker.ch/rrdtool/download.en.html](http://oss.oetiker.ch/rrdtool/download.en.html) + + * Then follow the readme instructions, with the following change: + + dir_config("rrd","../../src","../../src/.libs") + + to + + dir_config("rrd","../../src","/opt/local/lib") # or your location + +* run app + + script/server + +No database or migrations currently required + + + diff --git a/contrib/rb/Monitoring/Rakefile b/contrib/rb/Monitoring/Rakefile new file mode 100644 index 000000000..3bb0e8592 --- /dev/null +++ b/contrib/rb/Monitoring/Rakefile @@ -0,0 +1,10 @@ +# Add your own tasks in files placed in lib/tasks ending in .rake, +# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. + +require(File.join(File.dirname(__FILE__), 'config', 'boot')) + +require 'rake' +require 'rake/testtask' +require 'rake/rdoctask' + +require 'tasks/rails' diff --git a/contrib/rb/Monitoring/app/controllers/application_controller.rb b/contrib/rb/Monitoring/app/controllers/application_controller.rb new file mode 100644 index 000000000..a3a46784f --- /dev/null +++ b/contrib/rb/Monitoring/app/controllers/application_controller.rb @@ -0,0 +1,66 @@ +# Filters added to this controller apply to all controllers in the application. +# Likewise, all the methods added will be available for all controllers. + +class ApplicationController < ActionController::Base + helper :all # include all helpers, all the time + protect_from_forgery # See ActionController::RequestForgeryProtection for details + + # Scrub sensitive parameters from your log + # filter_parameter_logging :password + + def find_largest(array) + array.flatten.sort.last + end + + def find_smallest(array) + array.flatten.sort.first + end + + #to use helper methods in controller + def help + Helper.instance + end + + class Helper + include Singleton + include ActionView::Helpers::NumberHelper + end + + # retrieval and graph generation from RRDtool dbs + PATH_TO_FILE = ENV['HYPERTABLE_HOME'] + "/run/monitoring/" + VERSION_NUMBER = 0 + + def get_all_rrd_rs_graphs range_server, stat_types + rrd_name = range_server.name + + rrd = "#{PATH_TO_FILE}#{rrd_name}_stats_v#{VERSION_NUMBER}.rrd" + + #todo: get proper times + # start_time = range_server.timestamps[2] / 10 ** 9 + # end_time = range_server.timestamps[0] / 10 ** 9 + now = Time.now + end_time = now.to_i + start_time = (now - 24.hours).to_i + + graphs = [] + + stat_types.each do |stat_name| + file_name = "#{rrd_name}_#{stat_name.to_s}" + (fstart, fend, data) = RRD.fetch(rrd, "--start", start_time, "--end", end_time, "AVERAGE") + graph_key = RangeServer.get_chart_type stat_name + if RangeServer::STAT_TO_RRD_KEY[:"#{stat_name}"] + stat_a = RangeServer::STAT_TO_RRD_KEY[graph_key[:stats][0]] + RRD.graph( + "../../../src/rb/Monitoring/public/images/#{file_name}.png", + "--title", "#{RangeServer.pretty_titleize stat_name}", + "--start", start_time, + "--end", end_time, + "DEF:#{stat_a}=#{rrd}:#{stat_a}:AVERAGE", + "LINE2:#{stat_a}#FF0000") + end + graphs.push "#{file_name}.png" + end + return graphs + end + +end diff --git a/contrib/rb/Monitoring/app/controllers/main_controller.rb b/contrib/rb/Monitoring/app/controllers/main_controller.rb new file mode 100644 index 000000000..15a42589f --- /dev/null +++ b/contrib/rb/Monitoring/app/controllers/main_controller.rb @@ -0,0 +1,16 @@ +class MainController < ApplicationController + def index + @time_interval_summary = FileReader::TIME_INTERVAL_SUMMARY + @time_interval_tables = FileReader::TIME_INTERVAL_TABLES + @time_interval_rs = FileReader::TIME_INTERVAL_RS + @tables = Table.get_stats + @range_servers = RangeServer.get_stats + + @table_timestamps, @table_system_totals = Table.get_system_totals @tables + @rs_timestamps, @rs_system_totals = RangeServer.get_system_totals @range_servers + + respond_to do |format| + format.html # index.html.erb + end + end +end diff --git a/contrib/rb/Monitoring/app/controllers/range_servers_controller.rb b/contrib/rb/Monitoring/app/controllers/range_servers_controller.rb new file mode 100644 index 000000000..b0f5623b3 --- /dev/null +++ b/contrib/rb/Monitoring/app/controllers/range_servers_controller.rb @@ -0,0 +1,56 @@ +class RangeServersController < ApplicationController + include GoogleChart + + def index + # XXX: sanjit remove this later + debugger + @time_intervals = FileReader::TIME_INTERVAL_RS + range_servers = RangeServer.get_stats + + @stat_types = RangeServer.get_stat_types + pp @stat_types + + @sort_types = ["data", "name"] + + @selected_sort = params[:sort_by] || @sort_types[0] # default if no params in url + @selected_stat = params[:data_type] || @stat_types[0] + @timestamp_index = params[:time_interval].blank? ? 2 : params[:time_interval].to_i # default interval at index 2 (10 minutes has interesting test data) + + @chart_type = RangeServer.get_chart_type @selected_stat + sorted_range_servers = RangeServer.sort(@chart_type, range_servers, @selected_sort, @selected_stat, @timestamp_index) + + #todo: handle large charts better. currently will only show top ones (that fit in 1 graph) + max_size = max_elements_in_chart(@chart_type) + + #temp, throws away elements that won't fit on the graph + sorted_tables = sorted_range_servers.slice(0..(max_size - 1)) + + #todo: give unique ids when creating multiple graphs on one page + @graph_id = 0 + + # stats_array = RangeServer.get_all_stats(sorted_range_servers, @selected_stat, @timestamp_index) + @chart = generate_chart(@chart_type, sorted_range_servers, @selected_sort, @timestamp_index, @selected_stat, @time_intervals) + + @json_map = json_map(@chart) + @html_map = generate_html_map(@json_map, sorted_range_servers, @selected_stat, @timestamp_index) + + #todo: this selects the first table's timestamp. + @time = Time.at sorted_range_servers.first.timestamps[@timestamp_index] / 10 ** 9 + + # if request is ajax render only + + respond_to do |format| + format.html # index.html.erb + format.js {render :partial => "ajax_data"} + end + end + + def show + @time_intervals = FileReader::TIME_INTERVAL_RS + @range_server = RangeServer.get_stat params[:id] + @stat_types = RangeServer.get_stat_types #array of symbols + + @rs_rrd_graphs = get_all_rrd_rs_graphs @range_server, @stat_types + end + +end diff --git a/contrib/rb/Monitoring/app/controllers/tables_controller.rb b/contrib/rb/Monitoring/app/controllers/tables_controller.rb new file mode 100644 index 000000000..5cd34cca5 --- /dev/null +++ b/contrib/rb/Monitoring/app/controllers/tables_controller.rb @@ -0,0 +1,48 @@ +class TablesController < ApplicationController + include GoogleChart + + def index + #XXX: sanjit remove this later + debugger + @time_intervals = FileReader::TIME_INTERVAL_TABLES + tables = Table.get_stats + + @stat_types = Table.get_stat_types + @sort_types = ["data", "name"] + + @selected_sort = params[:sort_by] || @sort_types[0] # default if no params in url + @selected_stat = params[:data_type] || @stat_types[0] + @timestamp_index = params[:time_interval].blank? ? 2 : params[:time_interval].to_i # default interval at index 2 (10 minutes has interesting test data) + + @chart_type = Table.get_chart_type @selected_stat + sorted_tables = Table.sort(@chart_type, tables, @selected_sort, @selected_stat, @timestamp_index) + + #todo: handle large charts better. currently will only show top ones (that fit in 1 graph) + max_size = max_elements_in_chart(@chart_type) + + #temp, throws away elements that won't fit on the graph + sorted_tables = sorted_tables.slice(0..(max_size - 1)) + + # dynamic charts + @chart = generate_chart(@chart_type, sorted_tables, @selected_sort, @timestamp_index, @selected_stat, @time_intervals) + + @json_map = json_map(@chart) + @html_map = generate_html_map(@json_map, sorted_tables, @selected_stat, @timestamp_index) + + #todo: this selects the first table's timestamp. + @time = Time.at sorted_tables.first.timestamps[@timestamp_index] / 10 ** 9 + + respond_to do |format| + format.html # index.html.erb + end + end + + def show + #XXX: sanjit remove this later + debugger + @time_intervals = FileReader::TIME_INTERVAL_TABLES + @table = Table.get_stat params[:id] + @stat_types = Table.get_stat_types #array of symbols + end + +end diff --git a/contrib/rb/Monitoring/app/helpers/application_helper.rb b/contrib/rb/Monitoring/app/helpers/application_helper.rb new file mode 100644 index 000000000..6df29deea --- /dev/null +++ b/contrib/rb/Monitoring/app/helpers/application_helper.rb @@ -0,0 +1,9 @@ +# Methods added to this helper will be available to all templates in the application. +module ApplicationHelper + include GoogleChart + + def sort_data_hash(data) + data.sort {|a,b| a.to_s <=> b.to_s } + end + +end diff --git a/contrib/rb/Monitoring/app/helpers/main_helper.rb b/contrib/rb/Monitoring/app/helpers/main_helper.rb new file mode 100644 index 000000000..e99854a01 --- /dev/null +++ b/contrib/rb/Monitoring/app/helpers/main_helper.rb @@ -0,0 +1,2 @@ +module MainHelper +end diff --git a/contrib/rb/Monitoring/app/helpers/range_servers_helper.rb b/contrib/rb/Monitoring/app/helpers/range_servers_helper.rb new file mode 100644 index 000000000..fa5db5dd0 --- /dev/null +++ b/contrib/rb/Monitoring/app/helpers/range_servers_helper.rb @@ -0,0 +1,10 @@ +module RangeServersHelper + + def link_to_if_rrd_chart_exists stat + if RangeServer::STAT_TO_RRD_KEY[stat.to_sym] + link_to RangeServer.pretty_titleize(stat), "##{stat}" + else + RangeServer.pretty_titleize(stat) + end + end +end diff --git a/contrib/rb/Monitoring/app/helpers/tables_helper.rb b/contrib/rb/Monitoring/app/helpers/tables_helper.rb new file mode 100644 index 000000000..35826e3fc --- /dev/null +++ b/contrib/rb/Monitoring/app/helpers/tables_helper.rb @@ -0,0 +1,2 @@ +module TablesHelper +end diff --git a/contrib/rb/Monitoring/app/models/range_server.rb b/contrib/rb/Monitoring/app/models/range_server.rb new file mode 100644 index 000000000..7723626f0 --- /dev/null +++ b/contrib/rb/Monitoring/app/models/range_server.rb @@ -0,0 +1,174 @@ +# RRDtool backed model +class RangeServer + #class methods from module + extend FileReader + @time_intervals = FileReader::TIME_INTERVAL_SUMMARY + HYPERTABLE_HOME = ENV["HYPERTABLE_HOME"] + PATH_TO_FILE = HYPERTABLE_HOME + "/run/monitoring/" + ORIGINAL_FILE_NAME = "rs_stats.txt" + COPY_FILE_NAME = "copy_of_#{ORIGINAL_FILE_NAME}" + UNIT = FileReader::UNIT + CHART_A_OPTIONS = FileReader::CHART_A_OPTIONS + CHART_B_OPTIONS = FileReader::CHART_B_OPTIONS + CHART_C_OPTIONS = FileReader::CHART_C_OPTIONS + + disk_read_KBps = {:type => :B, :stats => [:disk_read_KBps, :disk_write_KBps], :chart_options => CHART_B_OPTIONS, :units => UNIT[:kbps]} + disk_read_rate = {:type => :B, :stats => [:disk_read_rate, :disk_write_rate], :chart_options => CHART_B_OPTIONS, :units => UNIT[:rwps]} + net_recv_KBps = {:type => :B, :stats => [:net_recv_KBps, :net_send_KBps], :chart_options => CHART_B_OPTIONS, :units => UNIT[:kbps]} + disk_read_KBps = {:type => :B, :stats => [:disk_read_KBps, :disk_write_KBps], :chart_options => CHART_B_OPTIONS, :units => UNIT[:kbps]} + + bytes_read = {:type => :B, :stats => [:bytes_read, :bytes_written], :chart_options => CHART_B_OPTIONS, :units => UNIT[:bytes]} + query_cache_accesses = {:type => :B, :stats => [:query_cache_accesses, :query_cache_hits], :chart_options => CHART_B_OPTIONS, :units => UNIT[:abs]} + loadavg_0 = {:type => :B, :stats => [:loadavg_0, :loadavg_1, :loadavg_2], :chart_options => CHART_B_OPTIONS, :units => UNIT[:loadave]} + cells_read = {:type => :B, :stats => [:cells_read, :cells_written], :chart_options => CHART_B_OPTIONS, :units => UNIT[:abs]} + scans = {:type => :B, :stats => [:scans, :syncs], :chart_options => CHART_B_OPTIONS, :units => UNIT[:abs]} + block_cache_accesses = {:type => :B, :stats => [:block_cache_accesses, :block_cache_hits], :chart_options => CHART_B_OPTIONS, :units => UNIT[:abs]} + + #data structure to determine graph types, and meta data about each element + STATS_KEY = { + #type A + :percent_disk_used => {:type => :A, :stats => [:disk_used, :disk_available], :chart_options => CHART_A_OPTIONS, :units => UNIT[:percent]}, + :percent_mem_used => {:type => :A, :stats => [:mem_used, :mem_total], :chart_options => CHART_A_OPTIONS, :units => UNIT[:percent]}, + :percent_query_cache_memory_used => {:type => :A, :stats => [:query_cache_available_memory, :query_cache_max_memory], :chart_options => CHART_A_OPTIONS, :units => UNIT[:percent]}, + :percent_block_cache_memory_used => {:type => :A, :stats => [:block_cache_available_memory, :block_cache_max_memory], :chart_options => CHART_A_OPTIONS, :units => UNIT[:percent]}, + + #type B + :disk_read_KBps => disk_read_KBps, + :disk_write_KBps => disk_read_KBps, + + :disk_read_rate => disk_read_rate, + :disk_write_rate => disk_read_rate, + + :net_recv_KBps => net_recv_KBps, + :net_send_KBps => net_recv_KBps, + + :bytes_read => bytes_read, + :bytes_written => bytes_read, + + :query_cache_accesses => query_cache_accesses, + :query_cache_hits => query_cache_accesses, + + :loadavg_0 => loadavg_0, + :loadavg_1 => loadavg_0, + :loadavg_2 => loadavg_0, + + :cells_read => cells_read, + :cells_written => cells_read, + + :scans => scans, + :syncs => scans, + + :block_cache_accesses => block_cache_accesses, + :block_cache_hits => block_cache_accesses, + + # TYPE C + :virtual_machine_size => {:type => :C, :stats => [:vm_size], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb]}, + :virtual_machine_resident => {:type => :C, :stats => [:vm_resident], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb]}, + :updates => {:type => :C, :stats => [:updates], :chart_options => CHART_C_OPTIONS, :units => UNIT[:ab]}, + :cpu_percent => {:type => :C, :stats => [:cpu_pct], :chart_options => CHART_C_OPTIONS, :units => UNIT[:percent]}, + :block_cache_max_memory => {:type => :C, :stats => [:block_cache_max_memory], :chart_options => CHART_C_OPTIONS, :units => UNIT[:bytes]}, + + # also have type A percent from these + :disk_used => {:type => :C, :stats => [:disk_used], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb]}, + :mem_used => {:type => :C, :stats => [:mem_used], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb]}, + :query_cache_available_memory => {:type => :C, :stats => [:query_cache_available_memory], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb]}, + :query_cache_max_memory => {:type => :C, :stats => [:query_cache_max_memory], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb]}, + :block_cache_available_memory => {:type => :C, :stats => [:block_cache_available_memory], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb]}, + :block_cache_max_memory => {:type => :C, :stats => [:block_cache_max_memory], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb]}, + + #todo: immutable + :disk_available => {:type => :C, :stats => [:disk_available], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb], :immutable => true}, + :mem_total => {:type => :C, :stats => [:mem_total], :chart_options => CHART_C_OPTIONS, :units => UNIT[:kb], :immutable => true}, + :clock_mhz => {:type => :C, :stats => [:clock_mhz], :chart_options => CHART_C_OPTIONS, :units => UNIT[:mhz], :immutable => true}, + :num_cores => {:type => :C, :stats => [:num_cores], :chart_options => CHART_C_OPTIONS, :units => UNIT[:ab], :immutable => true}, + :num_ranges => {:type => :C, :stats => [:num_ranges], :chart_options => CHART_C_OPTIONS, :units => UNIT[:ab], :immutable => true} + + } + + STAT_TO_RRD_KEY = { + :num_ranges => "num_ranges", + :syncs => "syncs", + :scans => "scans", + + :cells_read => "cells_read", + :bytes_read => "bytes_read", + :cells_written => "cells_written", + :bytes_written => "bytes_written", + + :query_cache_accesses => "q_c_accesses", + :query_cache_hits => "q_c_hits", + :block_cache_accesses => "b_c_accesses", + :block_cache_hits => "b_c_hits", + + :query_cache_max_memory => "q_c_max_mem", + :query_cache_available_memory => "q_c_avail_mem", + + :block_cache_available_memory => "b_c_avail_mem", + :block_cache_max_memory => "b_c_max_mem" + } + + def self.get_stat_types + STATS_KEY.keys.sort {|a,b| a.to_s <=> b.to_s}.map {|d| d.to_s} + end + + def self.get_chart_type stat + stat = stat.to_sym + STATS_KEY[stat] + end + + def initialize (id=nil) + @id = id + @data = {} + end + + attr_accessor :id, + :timestamps, + :data + + alias name id + + # RangeServer.get_stats.first.get_value("num_ranges", 0, true) + def get_value(data_name, time_index, show_units) + stat_key = STATS_KEY[:"#{data_name}"] + value = nil + + case stat_key[:type] + when :A + #handle nil or of div by zero + value = self.data[stat_key[:stats][0]][time_index] / (self.data[stat_key[:stats][1]][time_index] * 1.0) + value = round_to(value, 4) * 100 + when :B + value = self.data[stat_key[:stats][0]][time_index] + when :C + value = self.data[stat_key[:stats][0]][time_index] + else + value = "" + end + + if show_units + unit = stat_key[:units] + if unit == "%" + value.to_s + "#{stat_key[:units]}" + else + value.to_s + " #{stat_key[:units]}" + end + else + value + end + end + + def self.get_units(stat_name) + STATS_KEY[:"#{stat_name}"][:units] + end + + #utiliity + def self.round_to(val, x) + # This throws NaNs disable for now + #(val * 10**x).round.to_f / 10**x + if !val.nan? + val_str = "%#{x}f" % [val] + val = Float(val_str) + end + return val + end +end diff --git a/contrib/rb/Monitoring/app/models/table.rb b/contrib/rb/Monitoring/app/models/table.rb new file mode 100644 index 000000000..51b598b19 --- /dev/null +++ b/contrib/rb/Monitoring/app/models/table.rb @@ -0,0 +1,126 @@ +class Table + # require 'pp' + # require 'ftools' + extend FileReader + + @time_intervals = FileReader::TIME_INTERVAL_SUMMARY + HYPERTABLE_HOME = ENV["HYPERTABLE_HOME"] + PATH_TO_FILE = HYPERTABLE_HOME + "/run/monitoring/" + ORIGINAL_FILE_NAME = "table_stats.txt" + COPY_FILE_NAME = "copy_of_#{ORIGINAL_FILE_NAME}" + UNIT = FileReader::UNIT + CHART_A_OPTIONS = FileReader::CHART_A_OPTIONS + CHART_B_OPTIONS = FileReader::CHART_B_OPTIONS + CHART_C_OPTIONS = FileReader::CHART_C_OPTIONS + + cells_read = {:type => :B, :stats => [:cells_read, :cells_written], :chart_options => CHART_B_OPTIONS, :units => UNIT[:abs]} + bloom_filter_accesses = {:type => :B, :stats => [:bloom_filter_accesses, :bloom_filter_maybes], :chart_options => CHART_B_OPTIONS, :units => UNIT[:abs]} + bloom_filter_memory = {:type => :B, :stats => [:bloom_filter_memory, :block_index_memory], :chart_options => CHART_B_OPTIONS, :units => UNIT[:bytes]} + #data structure to determine graph types, and what graphs to display. + STATS_KEY = { + # buggy calculation in ruby code (unweighted average % across tables) disable for now + #:percent_memory_used => {:type => :A, :stats => [:memory_used, :memory_allocated], :chart_options => CHART_A_OPTIONS, :units => UNIT[:percent]}, + + :cells_read => cells_read, + :cells_written => cells_read, + + :bloom_filter_accesses => bloom_filter_accesses, + :bloom_filter_maybes => bloom_filter_accesses, + + :bloom_filter_memory => bloom_filter_memory, + :block_index_memory => bloom_filter_memory, + + :scans => {:type => :C, :stats => [:scans], :chart_options => CHART_C_OPTIONS, :units => UNIT[:ab]}, + :disk_used => {:type => :C, :stats => [:disk_used], :chart_options => CHART_C_OPTIONS, :units => UNIT[:bytes]}, + :memory_used => {:type => :C, :stats => [:memory_used], :chart_options => CHART_C_OPTIONS, :units => UNIT[:bytes]}, + + #todo: immutable + :memory_alocated => {:type => :C, :stats => [:memory_allocated], :chart_options => CHART_C_OPTIONS, :units => UNIT[:bytes], :immutable => true} + } + + def self.get_stat_types + STATS_KEY.keys.sort {|a,b| a.to_s <=> b.to_s}.map {|d| d.to_s} + end + + def self.get_chart_type stat + stat = stat.to_sym + STATS_KEY[stat] + end + + def initialize (id=nil) + @id = id + @data = {} + end + + attr_accessor :id, + :timestamps, + :data #data is a hash containing current stats collected. + # :scans, + # :cells_read, + # :bytes_read, + # :cells_written, + # :bytes_written, + # :bloom_filter_accesses, + # :bloom_filter_maybes, + # :bloom_filter_memory, + # :block_index_memory, + # :memory_used, + # :memory_allocated, + # :disk_used + + #todo: temp until we have a real table name + alias name id + + #todo: RS and Table get_value methods are identical + def get_value(data_name, time_index, show_units) + stat_key = STATS_KEY[:"#{data_name}"] + value = nil + + case stat_key[:type] + when :A + #handle nil or of div by zero + if (self.data[stat_key[:stats][1]][time_index] == 0) + value = 100 + else + numerator = self.data[stat_key[:stats][0]][time_index] * 1.0 + denominator = self.data[stat_key[:stats][1]][time_index] * 1.0 + value = numerator/denominator + value = round_to(value, 4) * 100 + end + when :B + value = self.data[stat_key[:stats][0]][time_index] + when :C + value = self.data[stat_key[:stats][0]][time_index] + else + value = "" + end + + if show_units + unit = stat_key[:units] + if unit == "%" + value.to_s + "#{stat_key[:units]}" + else + value.to_s + " #{stat_key[:units]}" + end + else + value + end + end + + def self.get_units(stat_name) + STATS_KEY[:"#{stat_name}"][:units] + end + + #utiliity + def self.round_to(val, x) + # This throws NaNs disable for now + #(val * 10**x).round.to_f / 10**x + if !val.nan? + val_str = "%#{x}f" % [val] + val = Float(val_str) + end + return val + end + +end + diff --git a/contrib/rb/Monitoring/app/views/layouts/_footer.html.erb b/contrib/rb/Monitoring/app/views/layouts/_footer.html.erb new file mode 100644 index 000000000..a8f1fa76b --- /dev/null +++ b/contrib/rb/Monitoring/app/views/layouts/_footer.html.erb @@ -0,0 +1,5 @@ +
+
+
+
+

hypertable inc © <%= Time.now.strftime("%Y-%m-%d %T") %>

diff --git a/contrib/rb/Monitoring/app/views/layouts/_header.html.erb b/contrib/rb/Monitoring/app/views/layouts/_header.html.erb new file mode 100644 index 000000000..73f58a7d8 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/layouts/_header.html.erb @@ -0,0 +1,4 @@ +<%= link_to "Main", root_path %> +<%= link_to "Tables", tables_path %> +<%= link_to "Range Servers", range_servers_path %> +

\ No newline at end of file diff --git a/contrib/rb/Monitoring/app/views/layouts/application.html.erb b/contrib/rb/Monitoring/app/views/layouts/application.html.erb new file mode 100644 index 000000000..b9ba69d7a --- /dev/null +++ b/contrib/rb/Monitoring/app/views/layouts/application.html.erb @@ -0,0 +1,24 @@ + + + + + + Hypertable Metrics: <%= controller.controller_name %> - <%= controller.action_name %> + <%= stylesheet_link_tag 'stock' %> + <%#= javascript_include_tag 'jquery-1.4.2.min.js', 'jquery.form.js' %> + <%#= javascript_include_tag 'application' %> + + + +
+ + <%= render :partial => "layouts/header"%> + +
+ <%= yield %> +
+ <%= render :partial => "layouts/footer" %> + + + diff --git a/contrib/rb/Monitoring/app/views/main/_rs_data_list.html.erb b/contrib/rb/Monitoring/app/views/main/_rs_data_list.html.erb new file mode 100644 index 000000000..affcce79d --- /dev/null +++ b/contrib/rb/Monitoring/app/views/main/_rs_data_list.html.erb @@ -0,0 +1,25 @@ +

Number of RangeServers <%= @range_servers.length %>

+ + + + + <% @rs_timestamps.each_with_index do |ts, i| %> + + <% end %> + + + + <% sort_data_hash(@rs_system_totals).each do |key, value| %> + + <% value.each do |v| %> + + <% end %> + + <% end %> +
Sums of All <%= link_to "Range Servers", range_servers_path %> + <%= @time_interval_summary[i].to_s + " minute" + (@time_interval_summary[i].to_i > 1 ? "s" : "")%> +
<%= (Time.at ts / 10 ** 9).strftime("%Y-%m-%d %H:%M") %> +
<%= RangeServer.pretty_titleize key.to_s.titleize %> + <%= number_with_delimiter v %> + <%= RangeServer.get_units(key)%> +
diff --git a/contrib/rb/Monitoring/app/views/main/_table_data_list.html.erb b/contrib/rb/Monitoring/app/views/main/_table_data_list.html.erb new file mode 100644 index 000000000..a1da11c23 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/main/_table_data_list.html.erb @@ -0,0 +1,25 @@ +

Number of Tables: <%= @tables.length %>

+ + + + + <% @table_timestamps.each_with_index do |ts, i| %> + + <% end %> + + + + <% sort_data_hash(@table_system_totals).each do |key, value| %> + + <% value.each do |v| %> + + <% end %> + + <% end %> +
Sums of All <%= link_to "Tables", tables_path %> + <%= @time_interval_tables[i].to_s + " minute" + (@time_interval_tables[i].to_i > 1 ? "s" : "")%> +
<%= (Time.at ts / 10 ** 9).strftime("%Y-%m-%d %H:%M") %> +
<%= Table.pretty_titleize key.to_s.titleize %> + <%= number_with_delimiter v %> + <%= Table.get_units(key)%> +
diff --git a/contrib/rb/Monitoring/app/views/main/index.html.erb b/contrib/rb/Monitoring/app/views/main/index.html.erb new file mode 100644 index 000000000..541761b56 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/main/index.html.erb @@ -0,0 +1,8 @@ +

Global Stats

+ +<%= render :partial => "rs_data_list" %> + +
+ +<%= render :partial => "table_data_list" %> + diff --git a/contrib/rb/Monitoring/app/views/range_servers/_ajax_data.js.erb b/contrib/rb/Monitoring/app/views/range_servers/_ajax_data.js.erb new file mode 100644 index 000000000..69add628f --- /dev/null +++ b/contrib/rb/Monitoring/app/views/range_servers/_ajax_data.js.erb @@ -0,0 +1,3 @@ +//bug needs to work with multi lines... +//also this should return only a raw string that's then handled w/in the js file locally. +$("#graph_0").replaceWith("<%= render :partial => 'data' %>"); \ No newline at end of file diff --git a/contrib/rb/Monitoring/app/views/range_servers/_data.html.erb b/contrib/rb/Monitoring/app/views/range_servers/_data.html.erb new file mode 100644 index 000000000..85df903f3 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/range_servers/_data.html.erb @@ -0,0 +1,8 @@ +
> + "##{map_name}"%>

+ + around <%= @time.strftime("%Y-%m-%d %H:%M") %> + + <%= @html_map %> + +
diff --git a/contrib/rb/Monitoring/app/views/range_servers/_list_data.html.erb b/contrib/rb/Monitoring/app/views/range_servers/_list_data.html.erb new file mode 100644 index 000000000..4eb601282 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/range_servers/_list_data.html.erb @@ -0,0 +1,27 @@ + + + + <% @range_server.timestamps.each_with_index do |ts, i| %> + + <% end %> + + + + <% @stat_types.each do |stat| %> + + + <% @time_intervals.each_with_index do |interval, index| %> + + + <% end %> + + <% end %> +
<%= @range_server.name %> +
+ 		+ <%= @time_intervals[i].to_s + " minute average" %>
at <%= (Time.at ts / 10 ** 9).strftime("%T") %> +
+ <%= link_to_if_rrd_chart_exists stat %> + + <%= number_with_delimiter @range_server.get_value(stat, index, true) %> +
\ No newline at end of file diff --git a/contrib/rb/Monitoring/app/views/range_servers/_rrd_graphs.html.erb b/contrib/rb/Monitoring/app/views/range_servers/_rrd_graphs.html.erb new file mode 100644 index 000000000..0253f2785 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/range_servers/_rrd_graphs.html.erb @@ -0,0 +1,8 @@ +<% @stat_types.each_with_index do |stat_name, index| %> + <% if RangeServer::STAT_TO_RRD_KEY[:"#{stat_name}"] %> +
<%= image_tag @rs_rrd_graphs[index] %>
+
<%= link_to "top", "#top"%> +

+ <% end %> + +<% end %> \ No newline at end of file diff --git a/contrib/rb/Monitoring/app/views/range_servers/index.html.erb b/contrib/rb/Monitoring/app/views/range_servers/index.html.erb new file mode 100644 index 000000000..32398762c --- /dev/null +++ b/contrib/rb/Monitoring/app/views/range_servers/index.html.erb @@ -0,0 +1,38 @@ +

Range Servers

+ +
+
+ Sort by +
    + <% @sort_types.each do |type| %> +
  • + <%= radio_button_tag "sort_by", type, (@selected_sort == type) %><%= type.titleize %> +
    • + <% end %> +
+ + Time interval + <%= @chart_type[:immutable] ? "
    " : "
      " %> + <%= pp @chart_type %> + <% @time_intervals.each_with_index do |interval, i| %> +
    • + <%= radio_button_tag "time_interval", i, (@timestamp_index == i) %><%= interval %>     +
      • + <% end %> +
    + Data types +
      + <% @stat_types.each do |type| %> +
    • + <%= radio_button_tag "data_type", type, (@selected_stat == type) %><%= RangeServer.pretty_titleize type %> +
      • + <% end %> +
    + + <%= submit_tag "update view" %> + +
+ +
+ <%= render :partial => "data" %> +
diff --git a/contrib/rb/Monitoring/app/views/range_servers/show.html.erb b/contrib/rb/Monitoring/app/views/range_servers/show.html.erb new file mode 100644 index 000000000..5fbbc646d --- /dev/null +++ b/contrib/rb/Monitoring/app/views/range_servers/show.html.erb @@ -0,0 +1,4 @@ +<%= render :partial => "list_data" %> + +
+<%= render :partial => "rrd_graphs" %> diff --git a/contrib/rb/Monitoring/app/views/tables/_data.html.erb b/contrib/rb/Monitoring/app/views/tables/_data.html.erb new file mode 100644 index 000000000..57baa249f --- /dev/null +++ b/contrib/rb/Monitoring/app/views/tables/_data.html.erb @@ -0,0 +1,8 @@ +
+ "##{map_name}"%>

+ + around <%= @time.strftime("%Y-%m-%d %H:%M") %> + + <%= @html_map %> + +
diff --git a/contrib/rb/Monitoring/app/views/tables/_list_data.html.erb b/contrib/rb/Monitoring/app/views/tables/_list_data.html.erb new file mode 100644 index 000000000..2319b4a09 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/tables/_list_data.html.erb @@ -0,0 +1,27 @@ + + + + <% @table.timestamps.each_with_index do |ts, i| %> + + <% end %> + + + + <% @stat_types.each do |stat| %> + + + <% @time_intervals.each_with_index do |interval, index| %> + + + <% end %> + + <% end %> +
<%= @table.name %> +
+ 		+ <%= @time_intervals[i].to_s + " minute average" %>
at <%= (Time.at ts / 10 ** 9).strftime("%T") %> +
+ <%= Table.pretty_titleize stat %> + + <%= number_with_delimiter @table.get_value(stat, index, true) %> +
\ No newline at end of file diff --git a/contrib/rb/Monitoring/app/views/tables/index.html.erb b/contrib/rb/Monitoring/app/views/tables/index.html.erb new file mode 100644 index 000000000..9b8931393 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/tables/index.html.erb @@ -0,0 +1,37 @@ +

Tables

+ +
+
+ Sort by +
    + <% @sort_types.each do |type| %> +
  • + <%= radio_button_tag "sort_by", type, (@selected_sort == type) %><%= type.titleize %> +
    • + <% end %> +
+ Time interval + <%= @chart_type[:immutable] ? "
    " : "
      " %> + <% @time_intervals.each_with_index do |interval, i| %> +
    • + <%= radio_button_tag "time_interval", i, (@timestamp_index == i) %><%= interval %> +
      • + <% end %> +
    + + Data types +
      + <% @stat_types.each do |type| %> +
    • + <%= radio_button_tag "data_type", type, (@selected_stat == type) %><%= Table.pretty_titleize type %> +
      • + <% end %> +
    + + <%= submit_tag "update view" %> + +
+ +
+ <%= render :partial => "data" %> +
\ No newline at end of file diff --git a/contrib/rb/Monitoring/app/views/tables/show.html.erb b/contrib/rb/Monitoring/app/views/tables/show.html.erb new file mode 100644 index 000000000..f1cca7a19 --- /dev/null +++ b/contrib/rb/Monitoring/app/views/tables/show.html.erb @@ -0,0 +1 @@ +<%= render :partial => "list_data" %> diff --git a/contrib/rb/Monitoring/config/boot.rb b/contrib/rb/Monitoring/config/boot.rb new file mode 100644 index 000000000..dd5e3b691 --- /dev/null +++ b/contrib/rb/Monitoring/config/boot.rb @@ -0,0 +1,110 @@ +# Don't change this file! +# Configure your app in config/environment.rb and config/environments/*.rb + +RAILS_ROOT = "#{File.dirname(__FILE__)}/.." unless defined?(RAILS_ROOT) + +module Rails + class << self + def boot! + unless booted? + preinitialize + pick_boot.run + end + end + + def booted? + defined? Rails::Initializer + end + + def pick_boot + (vendor_rails? ? VendorBoot : GemBoot).new + end + + def vendor_rails? + File.exist?("#{RAILS_ROOT}/vendor/rails") + end + + def preinitialize + load(preinitializer_path) if File.exist?(preinitializer_path) + end + + def preinitializer_path + "#{RAILS_ROOT}/config/preinitializer.rb" + end + end + + class Boot + def run + load_initializer + Rails::Initializer.run(:set_load_path) + end + end + + class VendorBoot < Boot + def load_initializer + require "#{RAILS_ROOT}/vendor/rails/railties/lib/initializer" + Rails::Initializer.run(:install_gem_spec_stubs) + Rails::GemDependency.add_frozen_gem_path + end + end + + class GemBoot < Boot + def load_initializer + self.class.load_rubygems + load_rails_gem + require 'initializer' + end + + def load_rails_gem + if version = self.class.gem_version + gem 'rails', version + else + gem 'rails' + end + rescue Gem::LoadError => load_error + $stderr.puts %(Missing the Rails #{version} gem. Please `gem install -v=#{version} rails`, update your RAILS_GEM_VERSION setting in config/environment.rb for the Rails version you do have installed, or comment out RAILS_GEM_VERSION to use the latest version installed.) + exit 1 + end + + class << self + def rubygems_version + Gem::RubyGemsVersion rescue nil + end + + def gem_version + if defined? RAILS_GEM_VERSION + RAILS_GEM_VERSION + elsif ENV.include?('RAILS_GEM_VERSION') + ENV['RAILS_GEM_VERSION'] + else + parse_gem_version(read_environment_rb) + end + end + + def load_rubygems + min_version = '1.3.2' + require 'rubygems' + unless rubygems_version >= min_version + $stderr.puts %Q(Rails requires RubyGems >= #{min_version} (you have #{rubygems_version}). Please `gem update --system` and try again.) + exit 1 + end + + rescue LoadError + $stderr.puts %Q(Rails requires RubyGems >= #{min_version}. Please install RubyGems and try again: http://rubygems.rubyforge.org) + exit 1 + end + + def parse_gem_version(text) + $1 if text =~ /^[^#]*RAILS_GEM_VERSION\s*=\s*["']([!~<>=]*\s*[\d.]+)["']/ + end + + private + def read_environment_rb + File.read("#{RAILS_ROOT}/config/environment.rb") + end + end + end +end + +# All that for this: +Rails.boot! diff --git a/contrib/rb/Monitoring/config/database.yml b/contrib/rb/Monitoring/config/database.yml new file mode 100644 index 000000000..998d8f630 --- /dev/null +++ b/contrib/rb/Monitoring/config/database.yml @@ -0,0 +1,32 @@ +# development: +# adapter: mysql +# database: hypertable_metrics_test +# encoding: utf8 +# reconnect: false +# pool: 5 +# username: root +# password: xxxxxxxx +# socket: /tmp/mysql.sock + +# # SQLite version 3.x +# # gem install sqlite3-ruby (not necessary on OS X Leopard) +# development: +# adapter: sqlite3 +# database: db/development.sqlite3 +# pool: 5 +# timeout: 5000 + +# Warning: The database defined as "test" will be erased and +# re-generated from your development database when you run "rake". +# Do not set this db to the same as development or production. +# test: +# adapter: sqlite3 +# database: db/test.sqlite3 +# pool: 5 +# timeout: 5000 +# +# production: +# adapter: sqlite3 +# database: db/production.sqlite3 +# pool: 5 +# timeout: 5000 diff --git a/contrib/rb/Monitoring/config/environment.rb b/contrib/rb/Monitoring/config/environment.rb new file mode 100644 index 000000000..f032b9eea --- /dev/null +++ b/contrib/rb/Monitoring/config/environment.rb @@ -0,0 +1,43 @@ +# Be sure to restart your server when you modify this file + +# Specifies gem version of Rails to use when vendor/rails is not present +RAILS_GEM_VERSION = '2.3.7' unless defined? RAILS_GEM_VERSION + +# Bootstrap the Rails environment, frameworks, and default configuration +require File.join(File.dirname(__FILE__), 'boot') + +Rails::Initializer.run do |config| + # Settings in config/environments/* take precedence over those specified here. + # Application configuration should go into files in config/initializers + # -- all .rb files in that directory are automatically loaded. + + # Add additional load paths for your own custom dirs + # config.load_paths += %W( #{RAILS_ROOT}/extras ) + + # Specify gems that this application depends on and have them installed with rake gems:install + # config.gem "bj" + # config.gem "hpricot", :version => '0.6', :source => "http://code.whytheluckystiff.net" + # config.gem "sqlite3-ruby", :lib => "sqlite3" + # config.gem "aws-s3", :lib => "aws/s3" + config.gem "json" + + # Only load the plugins named here, in the order given (default is alphabetical). + # :all can be used as a placeholder for all plugins not explicitly named + # config.plugins = [ :exception_notification, :ssl_requirement, :all ] + + # Skip frameworks you're not going to use. To use Rails without a database, + # you must remove the Active Record framework. + config.frameworks -= [ :active_record, :active_resource, :action_mailer ] #none currently used + + + # Activate observers that should always be running + # config.active_record.observers = :cacher, :garbage_collector, :forum_observer + + # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone. + # Run "rake -D time" for a list of tasks for finding time zone names. + config.time_zone = 'UTC' + + # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. + # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}')] + # config.i18n.default_locale = :de +end \ No newline at end of file diff --git a/contrib/rb/Monitoring/config/environments/development.rb b/contrib/rb/Monitoring/config/environments/development.rb new file mode 100644 index 000000000..85c9a6080 --- /dev/null +++ b/contrib/rb/Monitoring/config/environments/development.rb @@ -0,0 +1,17 @@ +# Settings specified here will take precedence over those in config/environment.rb + +# In the development environment your application's code is reloaded on +# every request. This slows down response time but is perfect for development +# since you don't have to restart the webserver when you make code changes. +config.cache_classes = false + +# Log error messages when you accidentally call methods on nil. +config.whiny_nils = true + +# Show full error reports and disable caching +config.action_controller.consider_all_requests_local = true +config.action_view.debug_rjs = true +config.action_controller.perform_caching = false + +# Don't care if the mailer can't send +config.action_mailer.raise_delivery_errors = false \ No newline at end of file diff --git a/contrib/rb/Monitoring/config/environments/production.rb b/contrib/rb/Monitoring/config/environments/production.rb new file mode 100644 index 000000000..27119d2d1 --- /dev/null +++ b/contrib/rb/Monitoring/config/environments/production.rb @@ -0,0 +1,28 @@ +# Settings specified here will take precedence over those in config/environment.rb + +# The production environment is meant for finished, "live" apps. +# Code is not reloaded between requests +config.cache_classes = true + +# Full error reports are disabled and caching is turned on +config.action_controller.consider_all_requests_local = false +config.action_controller.perform_caching = true +config.action_view.cache_template_loading = true + +# See everything in the log (default is :info) +# config.log_level = :debug + +# Use a different logger for distributed setups +# config.logger = SyslogLogger.new + +# Use a different cache store in production +# config.cache_store = :mem_cache_store + +# Enable serving of images, stylesheets, and javascripts from an asset server +# config.action_controller.asset_host = "http://assets.example.com" + +# Disable delivery errors, bad email addresses will be ignored +# config.action_mailer.raise_delivery_errors = false + +# Enable threaded mode +# config.threadsafe! \ No newline at end of file diff --git a/contrib/rb/Monitoring/config/environments/test.rb b/contrib/rb/Monitoring/config/environments/test.rb new file mode 100644 index 000000000..d6f80a408 --- /dev/null +++ b/contrib/rb/Monitoring/config/environments/test.rb @@ -0,0 +1,28 @@ +# Settings specified here will take precedence over those in config/environment.rb + +# The test environment is used exclusively to run your application's +# test suite. You never need to work with it otherwise. Remember that +# your test database is "scratch space" for the test suite and is wiped +# and recreated between test runs. Don't rely on the data there! +config.cache_classes = true + +# Log error messages when you accidentally call methods on nil. +config.whiny_nils = true + +# Show full error reports and disable caching +config.action_controller.consider_all_requests_local = true +config.action_controller.perform_caching = false +config.action_view.cache_template_loading = true + +# Disable request forgery protection in test environment +config.action_controller.allow_forgery_protection = false + +# Tell Action Mailer not to deliver emails to the real world. +# The :test delivery method accumulates sent emails in the +# ActionMailer::Base.deliveries array. +config.action_mailer.delivery_method = :test + +# Use SQL instead of Active Record's schema dumper when creating the test database. +# This is necessary if your schema can't be completely dumped by the schema dumper, +# like if you have constraints or database-specific column types +# config.active_record.schema_format = :sql \ No newline at end of file diff --git a/contrib/rb/Monitoring/config/initializers/0_my_requires.rb b/contrib/rb/Monitoring/config/initializers/0_my_requires.rb new file mode 100644 index 000000000..1489a8aff --- /dev/null +++ b/contrib/rb/Monitoring/config/initializers/0_my_requires.rb @@ -0,0 +1,8 @@ +# ruby libraries +require 'pp' +require 'uri' +require 'ftools' + +# gems +require 'json' +require "RRD" diff --git a/contrib/rb/Monitoring/config/initializers/backtrace_silencers.rb b/contrib/rb/Monitoring/config/initializers/backtrace_silencers.rb new file mode 100644 index 000000000..c2169ed01 --- /dev/null +++ b/contrib/rb/Monitoring/config/initializers/backtrace_silencers.rb @@ -0,0 +1,7 @@ +# Be sure to restart your server when you modify this file. + +# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. +# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } + +# You can also remove all the silencers if you're trying do debug a problem that might steem from framework code. +# Rails.backtrace_cleaner.remove_silencers! \ No newline at end of file diff --git a/contrib/rb/Monitoring/config/initializers/inflections.rb b/contrib/rb/Monitoring/config/initializers/inflections.rb new file mode 100644 index 000000000..d531b8bb8 --- /dev/null +++ b/contrib/rb/Monitoring/config/initializers/inflections.rb @@ -0,0 +1,10 @@ +# Be sure to restart your server when you modify this file. + +# Add new inflection rules using the following format +# (all these examples are active by default): +# ActiveSupport::Inflector.inflections do |inflect| +# inflect.plural /^(ox)$/i, '\1en' +# inflect.singular /^(ox)en/i, '\1' +# inflect.irregular 'person', 'people' +# inflect.uncountable %w( fish sheep ) +# end diff --git a/contrib/rb/Monitoring/config/initializers/mime_types.rb b/contrib/rb/Monitoring/config/initializers/mime_types.rb new file mode 100644 index 000000000..72aca7e44 --- /dev/null +++ b/contrib/rb/Monitoring/config/initializers/mime_types.rb @@ -0,0 +1,5 @@ +# Be sure to restart your server when you modify this file. + +# Add new mime types for use in respond_to blocks: +# Mime::Type.register "text/richtext", :rtf +# Mime::Type.register_alias "text/html", :iphone diff --git a/contrib/rb/Monitoring/config/initializers/new_rails_defaults.rb b/contrib/rb/Monitoring/config/initializers/new_rails_defaults.rb new file mode 100644 index 000000000..c94db0a66 --- /dev/null +++ b/contrib/rb/Monitoring/config/initializers/new_rails_defaults.rb @@ -0,0 +1,21 @@ +# Be sure to restart your server when you modify this file. + +# These settings change the behavior of Rails 2 apps and will be defaults +# for Rails 3. You can remove this initializer when Rails 3 is released. + +if defined?(ActiveRecord) + # Include Active Record class name as root for JSON serialized output. + ActiveRecord::Base.include_root_in_json = true + + # Store the full class name (including module namespace) in STI type column. + ActiveRecord::Base.store_full_sti_class = true +end + +ActionController::Routing.generate_best_match = false + +# Use ISO 8601 format for JSON serialized times and dates. +ActiveSupport.use_standard_json_time_format = true + +# Don't escape HTML entities in JSON, leave that for the #json_escape helper. +# if you're including raw json in an HTML page. +ActiveSupport.escape_html_entities_in_json = false \ No newline at end of file diff --git a/contrib/rb/Monitoring/config/initializers/session_store.rb b/contrib/rb/Monitoring/config/initializers/session_store.rb new file mode 100644 index 000000000..ff416325d --- /dev/null +++ b/contrib/rb/Monitoring/config/initializers/session_store.rb @@ -0,0 +1,15 @@ +# Be sure to restart your server when you modify this file. + +# Your secret key for verifying cookie session data integrity. +# If you change this key, all old sessions will become invalid! +# Make sure the secret is at least 30 characters and all random, +# no regular words or you'll be exposed to dictionary attacks. +ActionController::Base.session = { + :key => '_hyptertable_metrics_session', + :secret => '33a8cf5f27d225da9a4f71465f579257ab865b795cb054ed0e0cac0cc5560c01cd84b18342208b67477c54923f7526bd9d74beb94c1f6271c67d5ffae0d26f55' +} + +# Use the database for sessions instead of the cookie-based default, +# which shouldn't be used to store highly confidential information +# (create the session table with "rake db:sessions:create") +# ActionController::Base.session_store = :active_record_store diff --git a/contrib/rb/Monitoring/config/locales/en.yml b/contrib/rb/Monitoring/config/locales/en.yml new file mode 100644 index 000000000..f265c068d --- /dev/null +++ b/contrib/rb/Monitoring/config/locales/en.yml @@ -0,0 +1,5 @@ +# Sample localization file for English. Add more files in this directory for other locales. +# See http://github.com/svenfuchs/rails-i18n/tree/master/rails%2Flocale for starting points. + +en: + hello: "Hello world" \ No newline at end of file diff --git a/contrib/rb/Monitoring/config/routes.rb b/contrib/rb/Monitoring/config/routes.rb new file mode 100644 index 000000000..1ffa28025 --- /dev/null +++ b/contrib/rb/Monitoring/config/routes.rb @@ -0,0 +1,46 @@ +ActionController::Routing::Routes.draw do |map| + map.resources :tables + map.resources :range_servers + + # The priority is based upon order of creation: first created -> highest priority. + + # Sample of regular route: + # map.connect 'products/:id', :controller => 'catalog', :action => 'view' + # Keep in mind you can assign values other than :controller and :action + + # Sample of named route: + # map.purchase 'products/:id/purchase', :controller => 'catalog', :action => 'purchase' + # This route can be invoked with purchase_url(:id => product.id) + + # Sample resource route (maps HTTP verbs to controller actions automatically): + # map.resources :products + + # Sample resource route with options: + # map.resources :products, :member => { :short => :get, :toggle => :post }, :collection => { :sold => :get } + + # Sample resource route with sub-resources: + # map.resources :products, :has_many => [ :comments, :sales ], :has_one => :seller + + # Sample resource route with more complex sub-resources + # map.resources :products do |products| + # products.resources :comments + # products.resources :sales, :collection => { :recent => :get } + # end + + # Sample resource route within a namespace: + # map.namespace :admin do |admin| + # # Directs /admin/products/* to Admin::ProductsController (app/controllers/admin/products_controller.rb) + # admin.resources :products + # end + + # You can have the root of your site routed with map.root -- just remember to delete public/index.html. + map.root :controller => "main", :action => "index" + + # See how all your routes lay out with "rake routes" + + # Install the default routes as the lowest priority. + # Note: These default routes make all actions in every controller accessible via GET requests. You should + # consider removing or commenting them out if you're using named routes and resources. + map.connect ':controller/:action/:id' + map.connect ':controller/:action/:id.:format' +end diff --git a/contrib/rb/Monitoring/db/seeds.rb b/contrib/rb/Monitoring/db/seeds.rb new file mode 100644 index 000000000..3174d0cb8 --- /dev/null +++ b/contrib/rb/Monitoring/db/seeds.rb @@ -0,0 +1,7 @@ +# This file should contain all the record creation needed to seed the database with its default values. +# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup). +# +# Examples: +# +# cities = City.create([{ :name => 'Chicago' }, { :name => 'Copenhagen' }]) +# Major.create(:name => 'Daley', :city => cities.first) diff --git a/contrib/rb/Monitoring/doc/README_FOR_APP b/contrib/rb/Monitoring/doc/README_FOR_APP new file mode 100644 index 000000000..fe41f5cc2 --- /dev/null +++ b/contrib/rb/Monitoring/doc/README_FOR_APP @@ -0,0 +1,2 @@ +Use this README file to introduce your application and point to useful places in the API for learning more. +Run "rake doc:app" to generate API documentation for your models, controllers, helpers, and libraries. diff --git a/contrib/rb/Monitoring/lib/file_reader.rb b/contrib/rb/Monitoring/lib/file_reader.rb new file mode 100644 index 000000000..5e6af24de --- /dev/null +++ b/contrib/rb/Monitoring/lib/file_reader.rb @@ -0,0 +1,184 @@ +module FileReader + @time_intervals=[] + TIME_INTERVAL_SUMMARY = [1, 5, 10] + TIME_INTERVAL_TABLES = [1, 5, 10] + TIME_INTERVAL_RS = [1, 5, 10, 30, 60, 360, 720, 1440, 2880, 10080, 44640, 133920, 267840, 535680] + UNIT = { + :kbps => "KBps", + :rwps => "per second", + :bytes => "Bytes", + :kb => "KB", + :loadave => "",# "measure of waiting proc in proc queue", + :abs => "", #"absolute numbers", + :ab => "", #absolute number", + :percent => "%", + :mhz => "Mhz" + } + + CHART_A_OPTIONS = {:padding => 95, :legend_height => 8, :bar_width_or_scale => 8, :space_between_bars => 1, :space_between_groups => 2} + CHART_B_OPTIONS = {:padding => 95, :legend_height => 8, :bar_width_or_scale => 4, :space_between_bars => 1, :space_between_groups => 2} + CHART_C_OPTIONS = {:padding => 95, :legend_height => 8, :bar_width_or_scale => 8, :space_between_bars => 1, :space_between_groups => 2} + + def get_system_totals list=nil#, show_units=true + list = list || self.get_stats + + data = {} + stat_types = self.get_stat_types + list.each do |item| + stat_types.each do |stat| + data[:"#{stat}"] = Array.new(@time_intervals.length) unless data[:"#{stat}"] + @time_intervals.length.times do |i| + #XXX: sanjit remove this later + debugger + current = data[:"#{stat}"][i] + RAILS_DEFAULT_LOGGER.info("\n stat=#{stat} current=#{current} max_index=#{@time_intervals.length} type=#{self.name}\n") + running_total = data[:"#{stat}"][i] || 0 + data[:"#{stat}"][i] = running_total + item.get_value(stat, i, false) + end + if item == list.last && self::STATS_KEY[:"#{stat}"][:units] == "%" + @time_intervals.length.times do |i| + data[:"#{stat}"][i] = data[:"#{stat}"][i] / list.length # this assumes there are values for each stat + data[:"#{stat}"][i] = Table.round_to data[:"#{stat}"][i], 2 + end + end + end + + end + [list.first.timestamps, data] + end + + + + + # find just one. this could be optimized + def get_stat uid + list = self.get_stats + list.each do |item| + return item if item.id == uid + end + return nil + end + + def get_stats(wait_time=2) + list = [] + using_new_file = true + + # repeats the copy for some given time. + time_spent = 0 + start_time = Time.now + elapsed_time = Time.now + begin + elapsed_time = Time.now + File.copy("#{self::PATH_TO_FILE}#{self::ORIGINAL_FILE_NAME}", "#{self::PATH_TO_FILE}#{self::COPY_FILE_NAME}", false) + rescue => err + time_spent = elapsed_time - start_time + if time_spent <= wait_time + retry + else + #use old file if possible + if File.exists?("#{self::PATH_TO_FILE}#{self::COPY_FILE_NAME}") + using_new_file = false + else + raise + end + end + end + + begin + #parse copied file here + file = File.open("#{self::PATH_TO_FILE}#{self::COPY_FILE_NAME}", "r") + current_stat = self.new + file.each do |line| + #start parsing... + #XXX:sanjit remove this later + #debugger + + if line =~ /^(#{self.name.to_s}).*=\s*(\w+)/ + current_stat = self.new($2) + list.push current_stat + elsif line =~ /^\t(.+)=(.+)/ + key = :"#{$1}" + values = $2.split(",").map! do |v| + if v =~ /\./ + v.to_f #data can be floats + else + v.to_i + end + end + + # values = $2.split(",") #data can be floats + if key == :Timestamps + current_stat.timestamps = values + else + current_stat.data[key] = values + end + end + end + file.close + #Uses old copied file if necessary + # File.delete("#{self::PATH_TO_FILE}#{self::COPY_FILE_NAME}") + rescue + raise + end + # return the array of list populated with data. + return list + end + + def sort(chart_key, list, sort_type, selected_stat, interval_index) + data_type = selected_stat.to_sym + puts data_type + sorted = list.sort { |x, y| + if sort_type == "name" + x.id <=> y.id + elsif sort_type == "data" + case chart_key[:type] + + when :A + a = y.data[chart_key[:stats][0]][interval_index] + b = y.data[chart_key[:stats][1]][interval_index] + + c = x.data[chart_key[:stats][0]][interval_index] + d = x.data[chart_key[:stats][1]][interval_index] + #todo: handle divide by zero? doesn't blow up with + + # special case for :disk_available + if data_type == :disk_used + (b - a)/(b * 1.0) <=> (d - c)/(d * 1.0) + else + a/(b * 1.0) <=> c/(d * 1.0) + end + + when :B + y.data[data_type][interval_index] <=> x.data[data_type][interval_index] + when :C + data_type = chart_key[:stats][0] + y.data[data_type][interval_index] <=> x.data[data_type][interval_index] + end + end + } + # pp sorted.map{|s| s.data[data_type]}, chart_key[:type], chart_key, sort_type, data_type, interval_index + sorted + end + + #todo: doesn't handle if interval_index is there. (it pushes nil) + def get_all_stats(list, data_type, interval_index) + data = [] + list.each do |item| + #todo: if data doesn't exist for the selected index, push a nil value or -1? No. All values will be present + data.push item.data[:"#{data_type}"][interval_index] + end + data + end + + def pretty_titleize(title) + t = title.to_s.titleize + if t =~ /K Bps/ + return t.titleize.gsub!(/K Bps/,"KBps") + elsif t =~ /Cpu/ + return t.titleize.gsub!(/Cpu/,"CPU") + else + return t.titleize + end + end + +end diff --git a/contrib/rb/Monitoring/lib/google_chart.rb b/contrib/rb/Monitoring/lib/google_chart.rb new file mode 100644 index 000000000..3f927ab6b --- /dev/null +++ b/contrib/rb/Monitoring/lib/google_chart.rb @@ -0,0 +1,287 @@ +#All google chart generation methods live here +module GoogleChart + DEFAULT_COLOR = "00DD00" + DEFAULT_COLOR2 = "00BB00" + DEFAULT_COLOR3 = "00AA00" + + CRITICAL_COLOR = "FF0000" + MODERATE_COLOR = "FFB90F" + + #todo: 2nd version of chart url creation. + def generate_chart(chart_type, sorted_stats, selected_sort, timestamp_index, selected_stat, time_interval) + chart = ChartURL.new + + bar_width_or_scale = chart_type[:chart_options][:bar_width_or_scale] || 10 + space_between_bars = chart_type[:chart_options][:space_between_bars] || 2 + space_between_groups = chart_type[:chart_options][:space_between_groups] || 8 + + bar_width = "#{bar_width_or_scale},#{space_between_bars},#{space_between_groups}" + + chart_height = calculate_chart_height chart_type, sorted_stats.length + + chart_width = 400 + + options = { + :chts =>"FF0000,15", + :chxt => "x,x,y", + :chco => DEFAULT_COLOR, + :chbh => "#{bar_width}", #bar width.x 23px is default + :chs => "#{chart_width}x#{chart_height}" # size + } + + if selected_sort == "name" + # options[:chtt] = "#{selected_stat.titleize}, sorted by #{selected_sort.titleize}|#{time_interval[timestamp_index]} minute average" #title + options[:chtt] = "#{time_interval[timestamp_index]} minute average" #title + else + # options[:chtt] = "Sorted by #{selected_stat.titleize}|#{time_interval[timestamp_index]} minute average" #title + options[:chtt] = "#{time_interval[timestamp_index]} minute average" #title + end + + options[:chxp] = "1,50" + options[:chxl] = "1:|#{chart_type[:units]}|" + options[:chxl] += "2:|#{sorted_stats.map {|t| t.id }.reverse.map{|n| n.titleize}.join('|')}" #notice the order is reversed, put stat label here + # options[:chdl] = "#{(chart_type[:stats].map {|stat| stat.to_s.titleize}).join '|'}" + options[:chdl] = "#{(chart_type[:stats].map {|stat| RangeServer.pretty_titleize stat}).join '|'}" + options[:chdlp] = "tv" + + case chart_type[:type] + + #todo: use get_value method here instead + when :A + puts "A" + #for percents only + # chd = chart data + x_stats = Table.get_all_stats(sorted_stats, chart_type[:stats][0], timestamp_index) + y_stats = Table.get_all_stats(sorted_stats, chart_type[:stats][1], timestamp_index) + # smallest = find_smallest([x_stats, y_stats]) + # largest = find_largest([x_stats, y_stats]) + stats = ChartValue.new([x_stats, y_stats]) + percents = Array.new(x_stats.length) + x_stats.each_with_index { |x, i| + percents[i] = round_to(x / (y_stats[i] * 1.0), 4) + } + # pp stats, "percents", percents + largest = 1 + smallest = 0 + options[:chd] = "t:#{percents.join ','}" + + options[:chds] = "#{smallest},#{largest}" #todo: this breaks with 1 data point, or when all are same value. + options[:chxr] = "0,#{smallest},#{largest}" # values to be listed (high and low) + + chart_height = calculate_chart_height chart_type, sorted_stats.length + + options[:chco] = "#{DEFAULT_COLOR}" + + options[:chdl] = "#{selected_stat.titleize}" + options[:chdlp] = "tv" + options[:chs] = "#{chart_width}x#{chart_height}" # size + + chart = ChartURL.new("http://chart.apis.google.com/chart", "bho", options) + + + when :B + puts "B" + all_stats = [] + chart_type[:stats].each do |stat| + #todo: hack, converting all data to ints. need to scale data instead + all_stats.push Table.get_all_stats(sorted_stats, stat, timestamp_index).map {|stat| stat = stat.to_i} + end + + smallest = find_smallest(all_stats) + largest = find_largest(all_stats) + + stats = ChartValue.new(all_stats) + + + options[:chd] = "t:#{stats}" + + options[:chds] = "#{smallest},#{largest}" # scale #TODO: this breaks with 1 data point + options[:chxr] = "0,#{smallest},#{largest}" # values to be listed (high and low) + + chart_height = calculate_chart_height chart_type, sorted_stats.length + + options[:chco] = "#{DEFAULT_COLOR},#{DEFAULT_COLOR2},#{DEFAULT_COLOR3}" + options[:chs] = "#{chart_width}x#{chart_height}" # size + + chart = ChartURL.new("http://chart.apis.google.com/chart", "bhg", options) + + when :C + puts "C" + stats_array = Table.get_all_stats(sorted_stats, chart_type[:stats][0], timestamp_index) + + options[:chdl] = "#{selected_stat.titleize}" + options[:chdlp] = "tv" + + largest = find_largest(stats_array).to_i + smallest = find_smallest(stats_array).to_i + + # smallest = find_smallest(stats_array) + largest = find_largest(stats_array) + if (selected_stat.to_s.include? "percent") + # largest = find_largest(stats_array) * 100 + largest = 100 + smallest = 0 + stats_array.map! {|stat| stat *= 100 } #values are floats + elsif largest < 100 #when do we want 0 as the starting point? + smallest = 0 + stats_array.map! {|stat| stat = stat.to_i} + end + options[:chds] = "#{smallest},#{largest}" # scale #TODO: this breaks with 1 data point + options[:chxr] = "0,#{smallest},#{largest}" # values to be listed (high and low) + + options[:chd] = "t:#{stats_array.join(',')}" + + options[:chs] = "#{chart_width}x#{chart_height}" # size + + chart = ChartURL.new("http://chart.apis.google.com/chart", "bhs", options) + + end + # puts chart + chart.to_s + end + + + def json_map(chart) + chart_map = URI.encode(chart + "&chof=json") + chart_map = `curl "#{chart_map}"` + chart_map = JSON.parse(chart_map) + return chart_map + end + + def generate_html_map(json_map, sorted_stats, selected_stat, timestamp_index) + map = "\n" + json_map["chartshape"].each do |area| + #axes and bars: title and href + title = "" + href = "" + item = "" + if area["name"] =~ /axis2_(.+)/ + index = $1 + item = sorted_stats.reverse[index.to_i] + + value = help.number_with_delimiter item.get_value selected_stat, timestamp_index, true + title = item.id #this may be an actual name later + href = item.is_a?(RangeServer) ? range_server_path(title) : table_path(title) #title is also id right now. todo: better way to determine the path? + map += "\t\n" + elsif (area["name"] =~ /bar(.+)_(.+)/) + index_of_data = $1.to_i + index = $2.to_i + item = sorted_stats[index.to_i] + value = help.number_with_delimiter item.get_value selected_stat, timestamp_index, true + title = item.id + href = item.is_a?(RangeServer) ? range_server_path(title) : table_path(title) #todo: better way to determine path? + map += "\t\n" + end + # map += "\t\n" + end + map += "\n" + end + + # used in view template and controller + def map_name + return "generic_map_name" unless @selected_sort && @selected_stat && @timestamp_index + return "#{@selected_sort}_#{@selected_stat}_#{@timestamp_index}" + end + + def max_elements_in_chart chart_type, height=750 + padding = chart_type[:chart_options][:padding] + legend_height = chart_type[:chart_options][:legend_height] + unless chart_type[:type] == :A || chart_type[:type] == :C + legend_height *= chart_type[:stats].length + end + bar_group_height = get_bar_group_height chart_type + area_for_bars = height - padding - legend_height + max_elements = (area_for_bars / bar_group_height).floor + end + + def calculate_chart_height chart_type, list_size + padding = chart_type[:chart_options][:padding] + legend_height = chart_type[:chart_options][:legend_height] + legend_height *= chart_type[:stats].length + bar_group_height = get_bar_group_height chart_type + chart_height = bar_group_height * list_size + padding + legend_height + + chart_height > 750 ? 750 : chart_height + end + + private + def get_bar_group_height chart_type + bar_width_or_scale = chart_type[:chart_options][:bar_width_or_scale] + space_between_bars = chart_type[:chart_options][:space_between_bars] + space_between_groups = chart_type[:chart_options][:space_between_groups] + + bars_per_stat = 0 + + if chart_type[:type] == :A || chart_type[:type] == :C + bars_per_stat = 1 #1 element per stat + space_between_groups = 0 #because not grouped + + elsif chart_type[:type] == :B + bars_per_stat = chart_type[:stats].length + space_between_bars *= bars_per_stat - 1 + end + bar_group_height = bars_per_stat * bar_width_or_scale + space_between_groups + space_between_bars + + end + +end + +#options is only 1 level deep of concatination +# when giving values to various chart parameters the data needs to be formatted properly for that param ("|", or "," delimited ) +class ChartURL + def initialize (base_url=nil, chart_type=nil, options={}) + @base_url = base_url + @chart_type = chart_type + @options = options + end + + attr_accessor :base_url, :chart_type, :options + + def to_s + @base_url + "?" + "cht=" + @chart_type + "&" + options_to_s + end + + private + def options_to_s + opt = @options.to_a + opt = opt.map do |o| + if o[1].is_a?(Array) + o = o[0].to_s + "=" + o[1].join(",") + else #works for Strings, Symbols and ChartValues + o = o[0].to_s + "=" + o[1].to_s + end + end + opt.join "&" + end +end + +#A chart value is a nested array which with proper chart delimiter for gcharts +class ChartValue + def initialize(values) + # shuffled = Array.new(values[0].length) + # shuffled.map! {|a| a = Array.new(values.length)} + # shuffled.each_with_index do |element, outer_index| + # element.each_with_index do |inner_element, inner_index| + # shuffled[outer_index][inner_index] = values[inner_index][outer_index] + # end + # end + # @values = shuffled + @values = values + end + + def to_s + @values.map{|a| a = a.join ","}.join "|" + end +end + + +##random utilities +def round_to(val, x) + # This throws NaNs disable for now + #(val * 10**x).round.to_f / 10**x + if !val.nan? + val_str = "%#{x}f" % [val] + val = Float(val_str) + end + return val +end diff --git a/contrib/rb/Monitoring/public/404.html b/contrib/rb/Monitoring/public/404.html new file mode 100644 index 000000000..eff660b90 --- /dev/null +++ b/contrib/rb/Monitoring/public/404.html @@ -0,0 +1,30 @@ + + + + + + + The page you were looking for doesn't exist (404) + + + + + +
+

The page you were looking for doesn't exist.

+

You may have mistyped the address or the page may have moved.

+
+ + \ No newline at end of file diff --git a/contrib/rb/Monitoring/public/422.html b/contrib/rb/Monitoring/public/422.html new file mode 100644 index 000000000..b54e4a3ca --- /dev/null +++ b/contrib/rb/Monitoring/public/422.html @@ -0,0 +1,30 @@ + + + + + + + The change you wanted was rejected (422) + + + + + +
+

The change you wanted was rejected.

+

Maybe you tried to change something you didn't have access to.

+
+ + \ No newline at end of file diff --git a/contrib/rb/Monitoring/public/500.html b/contrib/rb/Monitoring/public/500.html new file mode 100644 index 000000000..ec3bbf02c --- /dev/null +++ b/contrib/rb/Monitoring/public/500.html @@ -0,0 +1,30 @@ + + + + + + + We're sorry, but something went wrong (500) + + + + + +
+

We're sorry, but something went wrong.

+

We've been notified about this issue and we'll take a look at it shortly.

+
+ + diff --git a/contrib/rb/Monitoring/public/favicon.ico b/contrib/rb/Monitoring/public/favicon.ico new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/rb/Monitoring/public/javascripts/application.js b/contrib/rb/Monitoring/public/javascripts/application.js new file mode 100644 index 000000000..ac7c98c29 --- /dev/null +++ b/contrib/rb/Monitoring/public/javascripts/application.js @@ -0,0 +1,29 @@ +//js autoupdate +$(document).ready(function() { + // $('#graph_form').ajaxForm({dataType: 'script'}); + + // //for jqueryForm plugin + // var options = { + // success: showResponse, // post-submit callback + // dataType: 'json' // 'xml', 'script', or 'json' (expected server response type) + // } + // bind to the form's submit event + // $('#graph_form').submit(function() { + // // inside event callbacks 'this' is the DOM element so we first + // // wrap it in a jQuery object and then invoke ajaxSubmit + // alert("in here") + // $(this).ajaxSubmit(options); + // // always return false to prevent standard browser submit and page navigation + // return false; + // }); + // + // function showResponse(responseText, statusText, xhr, $form) { + // alert("in here too?") + // + // alert(responseText); + // } +}); + + + + diff --git a/contrib/rb/Monitoring/public/javascripts/jquery-1.4.2.min.js b/contrib/rb/Monitoring/public/javascripts/jquery-1.4.2.min.js new file mode 100644 index 000000000..7c2430802 --- /dev/null +++ b/contrib/rb/Monitoring/public/javascripts/jquery-1.4.2.min.js @@ -0,0 +1,154 @@ +/*! + * jQuery JavaScript Library v1.4.2 + * http://jquery.com/ + * + * Copyright 2010, John Resig + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * Copyright 2010, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * + * Date: Sat Feb 13 22:33:48 2010 -0500 + */ +(function(A,w){function ma(){if(!c.isReady){try{s.documentElement.doScroll("left")}catch(a){setTimeout(ma,1);return}c.ready()}}function Qa(a,b){b.src?c.ajax({url:b.src,async:false,dataType:"script"}):c.globalEval(b.text||b.textContent||b.innerHTML||"");b.parentNode&&b.parentNode.removeChild(b)}function X(a,b,d,f,e,j){var i=a.length;if(typeof b==="object"){for(var o in b)X(a,o,b[o],f,e,d);return a}if(d!==w){f=!j&&f&&c.isFunction(d);for(o=0;o)[^>]*$|^#([\w-]+)$/,Ua=/^.[^:#\[\.,]*$/,Va=/\S/, +Wa=/^(\s|\u00A0)+|(\s|\u00A0)+$/g,Xa=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,P=navigator.userAgent,xa=false,Q=[],L,$=Object.prototype.toString,aa=Object.prototype.hasOwnProperty,ba=Array.prototype.push,R=Array.prototype.slice,ya=Array.prototype.indexOf;c.fn=c.prototype={init:function(a,b){var d,f;if(!a)return this;if(a.nodeType){this.context=this[0]=a;this.length=1;return this}if(a==="body"&&!b){this.context=s;this[0]=s.body;this.selector="body";this.length=1;return this}if(typeof a==="string")if((d=Ta.exec(a))&& +(d[1]||!b))if(d[1]){f=b?b.ownerDocument||b:s;if(a=Xa.exec(a))if(c.isPlainObject(b)){a=[s.createElement(a[1])];c.fn.attr.call(a,b,true)}else a=[f.createElement(a[1])];else{a=sa([d[1]],[f]);a=(a.cacheable?a.fragment.cloneNode(true):a.fragment).childNodes}return c.merge(this,a)}else{if(b=s.getElementById(d[2])){if(b.id!==d[2])return T.find(a);this.length=1;this[0]=b}this.context=s;this.selector=a;return this}else if(!b&&/^\w+$/.test(a)){this.selector=a;this.context=s;a=s.getElementsByTagName(a);return c.merge(this, +a)}else return!b||b.jquery?(b||T).find(a):c(b).find(a);else if(c.isFunction(a))return T.ready(a);if(a.selector!==w){this.selector=a.selector;this.context=a.context}return c.makeArray(a,this)},selector:"",jquery:"1.4.2",length:0,size:function(){return this.length},toArray:function(){return R.call(this,0)},get:function(a){return a==null?this.toArray():a<0?this.slice(a)[0]:this[a]},pushStack:function(a,b,d){var f=c();c.isArray(a)?ba.apply(f,a):c.merge(f,a);f.prevObject=this;f.context=this.context;if(b=== +"find")f.selector=this.selector+(this.selector?" ":"")+d;else if(b)f.selector=this.selector+"."+b+"("+d+")";return f},each:function(a,b){return c.each(this,a,b)},ready:function(a){c.bindReady();if(c.isReady)a.call(s,c);else Q&&Q.push(a);return this},eq:function(a){return a===-1?this.slice(a):this.slice(a,+a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(R.apply(this,arguments),"slice",R.call(arguments).join(","))},map:function(a){return this.pushStack(c.map(this, +function(b,d){return a.call(b,d,b)}))},end:function(){return this.prevObject||c(null)},push:ba,sort:[].sort,splice:[].splice};c.fn.init.prototype=c.fn;c.extend=c.fn.extend=function(){var a=arguments[0]||{},b=1,d=arguments.length,f=false,e,j,i,o;if(typeof a==="boolean"){f=a;a=arguments[1]||{};b=2}if(typeof a!=="object"&&!c.isFunction(a))a={};if(d===b){a=this;--b}for(;b
a"; +var e=d.getElementsByTagName("*"),j=d.getElementsByTagName("a")[0];if(!(!e||!e.length||!j)){c.support={leadingWhitespace:d.firstChild.nodeType===3,tbody:!d.getElementsByTagName("tbody").length,htmlSerialize:!!d.getElementsByTagName("link").length,style:/red/.test(j.getAttribute("style")),hrefNormalized:j.getAttribute("href")==="/a",opacity:/^0.55$/.test(j.style.opacity),cssFloat:!!j.style.cssFloat,checkOn:d.getElementsByTagName("input")[0].value==="on",optSelected:s.createElement("select").appendChild(s.createElement("option")).selected, +parentNode:d.removeChild(d.appendChild(s.createElement("div"))).parentNode===null,deleteExpando:true,checkClone:false,scriptEval:false,noCloneEvent:true,boxModel:null};b.type="text/javascript";try{b.appendChild(s.createTextNode("window."+f+"=1;"))}catch(i){}a.insertBefore(b,a.firstChild);if(A[f]){c.support.scriptEval=true;delete A[f]}try{delete b.test}catch(o){c.support.deleteExpando=false}a.removeChild(b);if(d.attachEvent&&d.fireEvent){d.attachEvent("onclick",function k(){c.support.noCloneEvent= +false;d.detachEvent("onclick",k)});d.cloneNode(true).fireEvent("onclick")}d=s.createElement("div");d.innerHTML="";a=s.createDocumentFragment();a.appendChild(d.firstChild);c.support.checkClone=a.cloneNode(true).cloneNode(true).lastChild.checked;c(function(){var k=s.createElement("div");k.style.width=k.style.paddingLeft="1px";s.body.appendChild(k);c.boxModel=c.support.boxModel=k.offsetWidth===2;s.body.removeChild(k).style.display="none"});a=function(k){var n= +s.createElement("div");k="on"+k;var r=k in n;if(!r){n.setAttribute(k,"return;");r=typeof n[k]==="function"}return r};c.support.submitBubbles=a("submit");c.support.changeBubbles=a("change");a=b=d=e=j=null}})();c.props={"for":"htmlFor","class":"className",readonly:"readOnly",maxlength:"maxLength",cellspacing:"cellSpacing",rowspan:"rowSpan",colspan:"colSpan",tabindex:"tabIndex",usemap:"useMap",frameborder:"frameBorder"};var G="jQuery"+J(),Ya=0,za={};c.extend({cache:{},expando:G,noData:{embed:true,object:true, +applet:true},data:function(a,b,d){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==A?za:a;var f=a[G],e=c.cache;if(!f&&typeof b==="string"&&d===w)return null;f||(f=++Ya);if(typeof b==="object"){a[G]=f;e[f]=c.extend(true,{},b)}else if(!e[f]){a[G]=f;e[f]={}}a=e[f];if(d!==w)a[b]=d;return typeof b==="string"?a[b]:a}},removeData:function(a,b){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==A?za:a;var d=a[G],f=c.cache,e=f[d];if(b){if(e){delete e[b];c.isEmptyObject(e)&&c.removeData(a)}}else{if(c.support.deleteExpando)delete a[c.expando]; +else a.removeAttribute&&a.removeAttribute(c.expando);delete f[d]}}}});c.fn.extend({data:function(a,b){if(typeof a==="undefined"&&this.length)return c.data(this[0]);else if(typeof a==="object")return this.each(function(){c.data(this,a)});var d=a.split(".");d[1]=d[1]?"."+d[1]:"";if(b===w){var f=this.triggerHandler("getData"+d[1]+"!",[d[0]]);if(f===w&&this.length)f=c.data(this[0],a);return f===w&&d[1]?this.data(d[0]):f}else return this.trigger("setData"+d[1]+"!",[d[0],b]).each(function(){c.data(this, +a,b)})},removeData:function(a){return this.each(function(){c.removeData(this,a)})}});c.extend({queue:function(a,b,d){if(a){b=(b||"fx")+"queue";var f=c.data(a,b);if(!d)return f||[];if(!f||c.isArray(d))f=c.data(a,b,c.makeArray(d));else f.push(d);return f}},dequeue:function(a,b){b=b||"fx";var d=c.queue(a,b),f=d.shift();if(f==="inprogress")f=d.shift();if(f){b==="fx"&&d.unshift("inprogress");f.call(a,function(){c.dequeue(a,b)})}}});c.fn.extend({queue:function(a,b){if(typeof a!=="string"){b=a;a="fx"}if(b=== +w)return c.queue(this[0],a);return this.each(function(){var d=c.queue(this,a,b);a==="fx"&&d[0]!=="inprogress"&&c.dequeue(this,a)})},dequeue:function(a){return this.each(function(){c.dequeue(this,a)})},delay:function(a,b){a=c.fx?c.fx.speeds[a]||a:a;b=b||"fx";return this.queue(b,function(){var d=this;setTimeout(function(){c.dequeue(d,b)},a)})},clearQueue:function(a){return this.queue(a||"fx",[])}});var Aa=/[\n\t]/g,ca=/\s+/,Za=/\r/g,$a=/href|src|style/,ab=/(button|input)/i,bb=/(button|input|object|select|textarea)/i, +cb=/^(a|area)$/i,Ba=/radio|checkbox/;c.fn.extend({attr:function(a,b){return X(this,a,b,true,c.attr)},removeAttr:function(a){return this.each(function(){c.attr(this,a,"");this.nodeType===1&&this.removeAttribute(a)})},addClass:function(a){if(c.isFunction(a))return this.each(function(n){var r=c(this);r.addClass(a.call(this,n,r.attr("class")))});if(a&&typeof a==="string")for(var b=(a||"").split(ca),d=0,f=this.length;d-1)return true;return false},val:function(a){if(a===w){var b=this[0];if(b){if(c.nodeName(b,"option"))return(b.attributes.value||{}).specified?b.value:b.text;if(c.nodeName(b,"select")){var d=b.selectedIndex,f=[],e=b.options;b=b.type==="select-one";if(d<0)return null;var j=b?d:0;for(d=b?d+1:e.length;j=0;else if(c.nodeName(this,"select")){var u=c.makeArray(r);c("option",this).each(function(){this.selected= +c.inArray(c(this).val(),u)>=0});if(!u.length)this.selectedIndex=-1}else this.value=r}})}});c.extend({attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true},attr:function(a,b,d,f){if(!a||a.nodeType===3||a.nodeType===8)return w;if(f&&b in c.attrFn)return c(a)[b](d);f=a.nodeType!==1||!c.isXMLDoc(a);var e=d!==w;b=f&&c.props[b]||b;if(a.nodeType===1){var j=$a.test(b);if(b in a&&f&&!j){if(e){b==="type"&&ab.test(a.nodeName)&&a.parentNode&&c.error("type property can't be changed"); +a[b]=d}if(c.nodeName(a,"form")&&a.getAttributeNode(b))return a.getAttributeNode(b).nodeValue;if(b==="tabIndex")return(b=a.getAttributeNode("tabIndex"))&&b.specified?b.value:bb.test(a.nodeName)||cb.test(a.nodeName)&&a.href?0:w;return a[b]}if(!c.support.style&&f&&b==="style"){if(e)a.style.cssText=""+d;return a.style.cssText}e&&a.setAttribute(b,""+d);a=!c.support.hrefNormalized&&f&&j?a.getAttribute(b,2):a.getAttribute(b);return a===null?w:a}return c.style(a,b,d)}});var O=/\.(.*)$/,db=function(a){return a.replace(/[^\w\s\.\|`]/g, +function(b){return"\\"+b})};c.event={add:function(a,b,d,f){if(!(a.nodeType===3||a.nodeType===8)){if(a.setInterval&&a!==A&&!a.frameElement)a=A;var e,j;if(d.handler){e=d;d=e.handler}if(!d.guid)d.guid=c.guid++;if(j=c.data(a)){var i=j.events=j.events||{},o=j.handle;if(!o)j.handle=o=function(){return typeof c!=="undefined"&&!c.event.triggered?c.event.handle.apply(o.elem,arguments):w};o.elem=a;b=b.split(" ");for(var k,n=0,r;k=b[n++];){j=e?c.extend({},e):{handler:d,data:f};if(k.indexOf(".")>-1){r=k.split("."); +k=r.shift();j.namespace=r.slice(0).sort().join(".")}else{r=[];j.namespace=""}j.type=k;j.guid=d.guid;var u=i[k],z=c.event.special[k]||{};if(!u){u=i[k]=[];if(!z.setup||z.setup.call(a,f,r,o)===false)if(a.addEventListener)a.addEventListener(k,o,false);else a.attachEvent&&a.attachEvent("on"+k,o)}if(z.add){z.add.call(a,j);if(!j.handler.guid)j.handler.guid=d.guid}u.push(j);c.event.global[k]=true}a=null}}},global:{},remove:function(a,b,d,f){if(!(a.nodeType===3||a.nodeType===8)){var e,j=0,i,o,k,n,r,u,z=c.data(a), +C=z&&z.events;if(z&&C){if(b&&b.type){d=b.handler;b=b.type}if(!b||typeof b==="string"&&b.charAt(0)==="."){b=b||"";for(e in C)c.event.remove(a,e+b)}else{for(b=b.split(" ");e=b[j++];){n=e;i=e.indexOf(".")<0;o=[];if(!i){o=e.split(".");e=o.shift();k=new RegExp("(^|\\.)"+c.map(o.slice(0).sort(),db).join("\\.(?:.*\\.)?")+"(\\.|$)")}if(r=C[e])if(d){n=c.event.special[e]||{};for(B=f||0;B=0){a.type= +e=e.slice(0,-1);a.exclusive=true}if(!d){a.stopPropagation();c.event.global[e]&&c.each(c.cache,function(){this.events&&this.events[e]&&c.event.trigger(a,b,this.handle.elem)})}if(!d||d.nodeType===3||d.nodeType===8)return w;a.result=w;a.target=d;b=c.makeArray(b);b.unshift(a)}a.currentTarget=d;(f=c.data(d,"handle"))&&f.apply(d,b);f=d.parentNode||d.ownerDocument;try{if(!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()]))if(d["on"+e]&&d["on"+e].apply(d,b)===false)a.result=false}catch(j){}if(!a.isPropagationStopped()&& +f)c.event.trigger(a,b,f,true);else if(!a.isDefaultPrevented()){f=a.target;var i,o=c.nodeName(f,"a")&&e==="click",k=c.event.special[e]||{};if((!k._default||k._default.call(d,a)===false)&&!o&&!(f&&f.nodeName&&c.noData[f.nodeName.toLowerCase()])){try{if(f[e]){if(i=f["on"+e])f["on"+e]=null;c.event.triggered=true;f[e]()}}catch(n){}if(i)f["on"+e]=i;c.event.triggered=false}}},handle:function(a){var b,d,f,e;a=arguments[0]=c.event.fix(a||A.event);a.currentTarget=this;b=a.type.indexOf(".")<0&&!a.exclusive; +if(!b){d=a.type.split(".");a.type=d.shift();f=new RegExp("(^|\\.)"+d.slice(0).sort().join("\\.(?:.*\\.)?")+"(\\.|$)")}e=c.data(this,"events");d=e[a.type];if(e&&d){d=d.slice(0);e=0;for(var j=d.length;e-1?c.map(a.options,function(f){return f.selected}).join("-"):"";else if(a.nodeName.toLowerCase()==="select")d=a.selectedIndex;return d},fa=function(a,b){var d=a.target,f,e;if(!(!da.test(d.nodeName)||d.readOnly)){f=c.data(d,"_change_data");e=Fa(d);if(a.type!=="focusout"||d.type!=="radio")c.data(d,"_change_data", +e);if(!(f===w||e===f))if(f!=null||e){a.type="change";return c.event.trigger(a,b,d)}}};c.event.special.change={filters:{focusout:fa,click:function(a){var b=a.target,d=b.type;if(d==="radio"||d==="checkbox"||b.nodeName.toLowerCase()==="select")return fa.call(this,a)},keydown:function(a){var b=a.target,d=b.type;if(a.keyCode===13&&b.nodeName.toLowerCase()!=="textarea"||a.keyCode===32&&(d==="checkbox"||d==="radio")||d==="select-multiple")return fa.call(this,a)},beforeactivate:function(a){a=a.target;c.data(a, +"_change_data",Fa(a))}},setup:function(){if(this.type==="file")return false;for(var a in ea)c.event.add(this,a+".specialChange",ea[a]);return da.test(this.nodeName)},teardown:function(){c.event.remove(this,".specialChange");return da.test(this.nodeName)}};ea=c.event.special.change.filters}s.addEventListener&&c.each({focus:"focusin",blur:"focusout"},function(a,b){function d(f){f=c.event.fix(f);f.type=b;return c.event.handle.call(this,f)}c.event.special[b]={setup:function(){this.addEventListener(a, +d,true)},teardown:function(){this.removeEventListener(a,d,true)}}});c.each(["bind","one"],function(a,b){c.fn[b]=function(d,f,e){if(typeof d==="object"){for(var j in d)this[b](j,f,d[j],e);return this}if(c.isFunction(f)){e=f;f=w}var i=b==="one"?c.proxy(e,function(k){c(this).unbind(k,i);return e.apply(this,arguments)}):e;if(d==="unload"&&b!=="one")this.one(d,f,e);else{j=0;for(var o=this.length;j0){y=t;break}}t=t[g]}m[q]=y}}}var f=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^[\]]*\]|['"][^'"]*['"]|[^[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g, +e=0,j=Object.prototype.toString,i=false,o=true;[0,0].sort(function(){o=false;return 0});var k=function(g,h,l,m){l=l||[];var q=h=h||s;if(h.nodeType!==1&&h.nodeType!==9)return[];if(!g||typeof g!=="string")return l;for(var p=[],v,t,y,S,H=true,M=x(h),I=g;(f.exec(""),v=f.exec(I))!==null;){I=v[3];p.push(v[1]);if(v[2]){S=v[3];break}}if(p.length>1&&r.exec(g))if(p.length===2&&n.relative[p[0]])t=ga(p[0]+p[1],h);else for(t=n.relative[p[0]]?[h]:k(p.shift(),h);p.length;){g=p.shift();if(n.relative[g])g+=p.shift(); +t=ga(g,t)}else{if(!m&&p.length>1&&h.nodeType===9&&!M&&n.match.ID.test(p[0])&&!n.match.ID.test(p[p.length-1])){v=k.find(p.shift(),h,M);h=v.expr?k.filter(v.expr,v.set)[0]:v.set[0]}if(h){v=m?{expr:p.pop(),set:z(m)}:k.find(p.pop(),p.length===1&&(p[0]==="~"||p[0]==="+")&&h.parentNode?h.parentNode:h,M);t=v.expr?k.filter(v.expr,v.set):v.set;if(p.length>0)y=z(t);else H=false;for(;p.length;){var D=p.pop();v=D;if(n.relative[D])v=p.pop();else D="";if(v==null)v=h;n.relative[D](y,v,M)}}else y=[]}y||(y=t);y||k.error(D|| +g);if(j.call(y)==="[object Array]")if(H)if(h&&h.nodeType===1)for(g=0;y[g]!=null;g++){if(y[g]&&(y[g]===true||y[g].nodeType===1&&E(h,y[g])))l.push(t[g])}else for(g=0;y[g]!=null;g++)y[g]&&y[g].nodeType===1&&l.push(t[g]);else l.push.apply(l,y);else z(y,l);if(S){k(S,q,l,m);k.uniqueSort(l)}return l};k.uniqueSort=function(g){if(B){i=o;g.sort(B);if(i)for(var h=1;h":function(g,h){var l=typeof h==="string";if(l&&!/\W/.test(h)){h=h.toLowerCase();for(var m=0,q=g.length;m=0))l||m.push(v);else if(l)h[p]=false;return false},ID:function(g){return g[1].replace(/\\/g,"")},TAG:function(g){return g[1].toLowerCase()}, +CHILD:function(g){if(g[1]==="nth"){var h=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(g[2]==="even"&&"2n"||g[2]==="odd"&&"2n+1"||!/\D/.test(g[2])&&"0n+"+g[2]||g[2]);g[2]=h[1]+(h[2]||1)-0;g[3]=h[3]-0}g[0]=e++;return g},ATTR:function(g,h,l,m,q,p){h=g[1].replace(/\\/g,"");if(!p&&n.attrMap[h])g[1]=n.attrMap[h];if(g[2]==="~=")g[4]=" "+g[4]+" ";return g},PSEUDO:function(g,h,l,m,q){if(g[1]==="not")if((f.exec(g[3])||"").length>1||/^\w/.test(g[3]))g[3]=k(g[3],null,null,h);else{g=k.filter(g[3],h,l,true^q);l||m.push.apply(m, +g);return false}else if(n.match.POS.test(g[0])||n.match.CHILD.test(g[0]))return true;return g},POS:function(g){g.unshift(true);return g}},filters:{enabled:function(g){return g.disabled===false&&g.type!=="hidden"},disabled:function(g){return g.disabled===true},checked:function(g){return g.checked===true},selected:function(g){return g.selected===true},parent:function(g){return!!g.firstChild},empty:function(g){return!g.firstChild},has:function(g,h,l){return!!k(l[3],g).length},header:function(g){return/h\d/i.test(g.nodeName)}, +text:function(g){return"text"===g.type},radio:function(g){return"radio"===g.type},checkbox:function(g){return"checkbox"===g.type},file:function(g){return"file"===g.type},password:function(g){return"password"===g.type},submit:function(g){return"submit"===g.type},image:function(g){return"image"===g.type},reset:function(g){return"reset"===g.type},button:function(g){return"button"===g.type||g.nodeName.toLowerCase()==="button"},input:function(g){return/input|select|textarea|button/i.test(g.nodeName)}}, +setFilters:{first:function(g,h){return h===0},last:function(g,h,l,m){return h===m.length-1},even:function(g,h){return h%2===0},odd:function(g,h){return h%2===1},lt:function(g,h,l){return hl[3]-0},nth:function(g,h,l){return l[3]-0===h},eq:function(g,h,l){return l[3]-0===h}},filter:{PSEUDO:function(g,h,l,m){var q=h[1],p=n.filters[q];if(p)return p(g,l,h,m);else if(q==="contains")return(g.textContent||g.innerText||a([g])||"").indexOf(h[3])>=0;else if(q==="not"){h= +h[3];l=0;for(m=h.length;l=0}},ID:function(g,h){return g.nodeType===1&&g.getAttribute("id")===h},TAG:function(g,h){return h==="*"&&g.nodeType===1||g.nodeName.toLowerCase()===h},CLASS:function(g,h){return(" "+(g.className||g.getAttribute("class"))+" ").indexOf(h)>-1},ATTR:function(g,h){var l=h[1];g=n.attrHandle[l]?n.attrHandle[l](g):g[l]!=null?g[l]:g.getAttribute(l);l=g+"";var m=h[2];h=h[4];return g==null?m==="!=":m=== +"="?l===h:m==="*="?l.indexOf(h)>=0:m==="~="?(" "+l+" ").indexOf(h)>=0:!h?l&&g!==false:m==="!="?l!==h:m==="^="?l.indexOf(h)===0:m==="$="?l.substr(l.length-h.length)===h:m==="|="?l===h||l.substr(0,h.length+1)===h+"-":false},POS:function(g,h,l,m){var q=n.setFilters[h[2]];if(q)return q(g,l,h,m)}}},r=n.match.POS;for(var u in n.match){n.match[u]=new RegExp(n.match[u].source+/(?![^\[]*\])(?![^\(]*\))/.source);n.leftMatch[u]=new RegExp(/(^(?:.|\r|\n)*?)/.source+n.match[u].source.replace(/\\(\d+)/g,function(g, +h){return"\\"+(h-0+1)}))}var z=function(g,h){g=Array.prototype.slice.call(g,0);if(h){h.push.apply(h,g);return h}return g};try{Array.prototype.slice.call(s.documentElement.childNodes,0)}catch(C){z=function(g,h){h=h||[];if(j.call(g)==="[object Array]")Array.prototype.push.apply(h,g);else if(typeof g.length==="number")for(var l=0,m=g.length;l";var l=s.documentElement;l.insertBefore(g,l.firstChild);if(s.getElementById(h)){n.find.ID=function(m,q,p){if(typeof q.getElementById!=="undefined"&&!p)return(q=q.getElementById(m[1]))?q.id===m[1]||typeof q.getAttributeNode!=="undefined"&& +q.getAttributeNode("id").nodeValue===m[1]?[q]:w:[]};n.filter.ID=function(m,q){var p=typeof m.getAttributeNode!=="undefined"&&m.getAttributeNode("id");return m.nodeType===1&&p&&p.nodeValue===q}}l.removeChild(g);l=g=null})();(function(){var g=s.createElement("div");g.appendChild(s.createComment(""));if(g.getElementsByTagName("*").length>0)n.find.TAG=function(h,l){l=l.getElementsByTagName(h[1]);if(h[1]==="*"){h=[];for(var m=0;l[m];m++)l[m].nodeType===1&&h.push(l[m]);l=h}return l};g.innerHTML=""; +if(g.firstChild&&typeof g.firstChild.getAttribute!=="undefined"&&g.firstChild.getAttribute("href")!=="#")n.attrHandle.href=function(h){return h.getAttribute("href",2)};g=null})();s.querySelectorAll&&function(){var g=k,h=s.createElement("div");h.innerHTML="

";if(!(h.querySelectorAll&&h.querySelectorAll(".TEST").length===0)){k=function(m,q,p,v){q=q||s;if(!v&&q.nodeType===9&&!x(q))try{return z(q.querySelectorAll(m),p)}catch(t){}return g(m,q,p,v)};for(var l in g)k[l]=g[l];h=null}}(); +(function(){var g=s.createElement("div");g.innerHTML="
";if(!(!g.getElementsByClassName||g.getElementsByClassName("e").length===0)){g.lastChild.className="e";if(g.getElementsByClassName("e").length!==1){n.order.splice(1,0,"CLASS");n.find.CLASS=function(h,l,m){if(typeof l.getElementsByClassName!=="undefined"&&!m)return l.getElementsByClassName(h[1])};g=null}}})();var E=s.compareDocumentPosition?function(g,h){return!!(g.compareDocumentPosition(h)&16)}: +function(g,h){return g!==h&&(g.contains?g.contains(h):true)},x=function(g){return(g=(g?g.ownerDocument||g:0).documentElement)?g.nodeName!=="HTML":false},ga=function(g,h){var l=[],m="",q;for(h=h.nodeType?[h]:h;q=n.match.PSEUDO.exec(g);){m+=q[0];g=g.replace(n.match.PSEUDO,"")}g=n.relative[g]?g+"*":g;q=0;for(var p=h.length;q=0===d})};c.fn.extend({find:function(a){for(var b=this.pushStack("","find",a),d=0,f=0,e=this.length;f0)for(var j=d;j0},closest:function(a,b){if(c.isArray(a)){var d=[],f=this[0],e,j= +{},i;if(f&&a.length){e=0;for(var o=a.length;e-1:c(f).is(e)){d.push({selector:i,elem:f});delete j[i]}}f=f.parentNode}}return d}var k=c.expr.match.POS.test(a)?c(a,b||this.context):null;return this.map(function(n,r){for(;r&&r.ownerDocument&&r!==b;){if(k?k.index(r)>-1:c(r).is(a))return r;r=r.parentNode}return null})},index:function(a){if(!a||typeof a=== +"string")return c.inArray(this[0],a?c(a):this.parent().children());return c.inArray(a.jquery?a[0]:a,this)},add:function(a,b){a=typeof a==="string"?c(a,b||this.context):c.makeArray(a);b=c.merge(this.get(),a);return this.pushStack(qa(a[0])||qa(b[0])?b:c.unique(b))},andSelf:function(){return this.add(this.prevObject)}});c.each({parent:function(a){return(a=a.parentNode)&&a.nodeType!==11?a:null},parents:function(a){return c.dir(a,"parentNode")},parentsUntil:function(a,b,d){return c.dir(a,"parentNode", +d)},next:function(a){return c.nth(a,2,"nextSibling")},prev:function(a){return c.nth(a,2,"previousSibling")},nextAll:function(a){return c.dir(a,"nextSibling")},prevAll:function(a){return c.dir(a,"previousSibling")},nextUntil:function(a,b,d){return c.dir(a,"nextSibling",d)},prevUntil:function(a,b,d){return c.dir(a,"previousSibling",d)},siblings:function(a){return c.sibling(a.parentNode.firstChild,a)},children:function(a){return c.sibling(a.firstChild)},contents:function(a){return c.nodeName(a,"iframe")? +a.contentDocument||a.contentWindow.document:c.makeArray(a.childNodes)}},function(a,b){c.fn[a]=function(d,f){var e=c.map(this,b,d);eb.test(a)||(f=d);if(f&&typeof f==="string")e=c.filter(f,e);e=this.length>1?c.unique(e):e;if((this.length>1||gb.test(f))&&fb.test(a))e=e.reverse();return this.pushStack(e,a,R.call(arguments).join(","))}});c.extend({filter:function(a,b,d){if(d)a=":not("+a+")";return c.find.matches(a,b)},dir:function(a,b,d){var f=[];for(a=a[b];a&&a.nodeType!==9&&(d===w||a.nodeType!==1||!c(a).is(d));){a.nodeType=== +1&&f.push(a);a=a[b]}return f},nth:function(a,b,d){b=b||1;for(var f=0;a;a=a[d])if(a.nodeType===1&&++f===b)break;return a},sibling:function(a,b){for(var d=[];a;a=a.nextSibling)a.nodeType===1&&a!==b&&d.push(a);return d}});var Ja=/ jQuery\d+="(?:\d+|null)"/g,V=/^\s+/,Ka=/(<([\w:]+)[^>]*?)\/>/g,hb=/^(?:area|br|col|embed|hr|img|input|link|meta|param)$/i,La=/<([\w:]+)/,ib=/"},F={option:[1,""],legend:[1,"
","
"],thead:[1,"","
"],tr:[2,"","
"],td:[3,"","
"],col:[2,"","
"],area:[1,"",""],_default:[0,"",""]};F.optgroup=F.option;F.tbody=F.tfoot=F.colgroup=F.caption=F.thead;F.th=F.td;if(!c.support.htmlSerialize)F._default=[1,"div
","
"];c.fn.extend({text:function(a){if(c.isFunction(a))return this.each(function(b){var d= +c(this);d.text(a.call(this,b,d.text()))});if(typeof a!=="object"&&a!==w)return this.empty().append((this[0]&&this[0].ownerDocument||s).createTextNode(a));return c.text(this)},wrapAll:function(a){if(c.isFunction(a))return this.each(function(d){c(this).wrapAll(a.call(this,d))});if(this[0]){var b=c(a,this[0].ownerDocument).eq(0).clone(true);this[0].parentNode&&b.insertBefore(this[0]);b.map(function(){for(var d=this;d.firstChild&&d.firstChild.nodeType===1;)d=d.firstChild;return d}).append(this)}return this}, +wrapInner:function(a){if(c.isFunction(a))return this.each(function(b){c(this).wrapInner(a.call(this,b))});return this.each(function(){var b=c(this),d=b.contents();d.length?d.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){c(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){c.nodeName(this,"body")||c(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.appendChild(a)})}, +prepend:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this)});else if(arguments.length){var a=c(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b, +this.nextSibling)});else if(arguments.length){var a=this.pushStack(this,"after",arguments);a.push.apply(a,c(arguments[0]).toArray());return a}},remove:function(a,b){for(var d=0,f;(f=this[d])!=null;d++)if(!a||c.filter(a,[f]).length){if(!b&&f.nodeType===1){c.cleanData(f.getElementsByTagName("*"));c.cleanData([f])}f.parentNode&&f.parentNode.removeChild(f)}return this},empty:function(){for(var a=0,b;(b=this[a])!=null;a++)for(b.nodeType===1&&c.cleanData(b.getElementsByTagName("*"));b.firstChild;)b.removeChild(b.firstChild); +return this},clone:function(a){var b=this.map(function(){if(!c.support.noCloneEvent&&!c.isXMLDoc(this)){var d=this.outerHTML,f=this.ownerDocument;if(!d){d=f.createElement("div");d.appendChild(this.cloneNode(true));d=d.innerHTML}return c.clean([d.replace(Ja,"").replace(/=([^="'>\s]+\/)>/g,'="$1">').replace(V,"")],f)[0]}else return this.cloneNode(true)});if(a===true){ra(this,b);ra(this.find("*"),b.find("*"))}return b},html:function(a){if(a===w)return this[0]&&this[0].nodeType===1?this[0].innerHTML.replace(Ja, +""):null;else if(typeof a==="string"&&!ta.test(a)&&(c.support.leadingWhitespace||!V.test(a))&&!F[(La.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(Ka,Ma);try{for(var b=0,d=this.length;b0||e.cacheable||this.length>1?k.cloneNode(true):k)}o.length&&c.each(o,Qa)}return this}});c.fragments={};c.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){c.fn[a]=function(d){var f=[];d=c(d);var e=this.length===1&&this[0].parentNode;if(e&&e.nodeType===11&&e.childNodes.length===1&&d.length===1){d[b](this[0]); +return this}else{e=0;for(var j=d.length;e0?this.clone(true):this).get();c.fn[b].apply(c(d[e]),i);f=f.concat(i)}return this.pushStack(f,a,d.selector)}}});c.extend({clean:function(a,b,d,f){b=b||s;if(typeof b.createElement==="undefined")b=b.ownerDocument||b[0]&&b[0].ownerDocument||s;for(var e=[],j=0,i;(i=a[j])!=null;j++){if(typeof i==="number")i+="";if(i){if(typeof i==="string"&&!jb.test(i))i=b.createTextNode(i);else if(typeof i==="string"){i=i.replace(Ka,Ma);var o=(La.exec(i)||["", +""])[1].toLowerCase(),k=F[o]||F._default,n=k[0],r=b.createElement("div");for(r.innerHTML=k[1]+i+k[2];n--;)r=r.lastChild;if(!c.support.tbody){n=ib.test(i);o=o==="table"&&!n?r.firstChild&&r.firstChild.childNodes:k[1]===""&&!n?r.childNodes:[];for(k=o.length-1;k>=0;--k)c.nodeName(o[k],"tbody")&&!o[k].childNodes.length&&o[k].parentNode.removeChild(o[k])}!c.support.leadingWhitespace&&V.test(i)&&r.insertBefore(b.createTextNode(V.exec(i)[0]),r.firstChild);i=r.childNodes}if(i.nodeType)e.push(i);else e= +c.merge(e,i)}}if(d)for(j=0;e[j];j++)if(f&&c.nodeName(e[j],"script")&&(!e[j].type||e[j].type.toLowerCase()==="text/javascript"))f.push(e[j].parentNode?e[j].parentNode.removeChild(e[j]):e[j]);else{e[j].nodeType===1&&e.splice.apply(e,[j+1,0].concat(c.makeArray(e[j].getElementsByTagName("script"))));d.appendChild(e[j])}return e},cleanData:function(a){for(var b,d,f=c.cache,e=c.event.special,j=c.support.deleteExpando,i=0,o;(o=a[i])!=null;i++)if(d=o[c.expando]){b=f[d];if(b.events)for(var k in b.events)e[k]? +c.event.remove(o,k):Ca(o,k,b.handle);if(j)delete o[c.expando];else o.removeAttribute&&o.removeAttribute(c.expando);delete f[d]}}});var kb=/z-?index|font-?weight|opacity|zoom|line-?height/i,Na=/alpha\([^)]*\)/,Oa=/opacity=([^)]*)/,ha=/float/i,ia=/-([a-z])/ig,lb=/([A-Z])/g,mb=/^-?\d+(?:px)?$/i,nb=/^-?\d/,ob={position:"absolute",visibility:"hidden",display:"block"},pb=["Left","Right"],qb=["Top","Bottom"],rb=s.defaultView&&s.defaultView.getComputedStyle,Pa=c.support.cssFloat?"cssFloat":"styleFloat",ja= +function(a,b){return b.toUpperCase()};c.fn.css=function(a,b){return X(this,a,b,true,function(d,f,e){if(e===w)return c.curCSS(d,f);if(typeof e==="number"&&!kb.test(f))e+="px";c.style(d,f,e)})};c.extend({style:function(a,b,d){if(!a||a.nodeType===3||a.nodeType===8)return w;if((b==="width"||b==="height")&&parseFloat(d)<0)d=w;var f=a.style||a,e=d!==w;if(!c.support.opacity&&b==="opacity"){if(e){f.zoom=1;b=parseInt(d,10)+""==="NaN"?"":"alpha(opacity="+d*100+")";a=f.filter||c.curCSS(a,"filter")||"";f.filter= +Na.test(a)?a.replace(Na,b):b}return f.filter&&f.filter.indexOf("opacity=")>=0?parseFloat(Oa.exec(f.filter)[1])/100+"":""}if(ha.test(b))b=Pa;b=b.replace(ia,ja);if(e)f[b]=d;return f[b]},css:function(a,b,d,f){if(b==="width"||b==="height"){var e,j=b==="width"?pb:qb;function i(){e=b==="width"?a.offsetWidth:a.offsetHeight;f!=="border"&&c.each(j,function(){f||(e-=parseFloat(c.curCSS(a,"padding"+this,true))||0);if(f==="margin")e+=parseFloat(c.curCSS(a,"margin"+this,true))||0;else e-=parseFloat(c.curCSS(a, +"border"+this+"Width",true))||0})}a.offsetWidth!==0?i():c.swap(a,ob,i);return Math.max(0,Math.round(e))}return c.curCSS(a,b,d)},curCSS:function(a,b,d){var f,e=a.style;if(!c.support.opacity&&b==="opacity"&&a.currentStyle){f=Oa.test(a.currentStyle.filter||"")?parseFloat(RegExp.$1)/100+"":"";return f===""?"1":f}if(ha.test(b))b=Pa;if(!d&&e&&e[b])f=e[b];else if(rb){if(ha.test(b))b="float";b=b.replace(lb,"-$1").toLowerCase();e=a.ownerDocument.defaultView;if(!e)return null;if(a=e.getComputedStyle(a,null))f= +a.getPropertyValue(b);if(b==="opacity"&&f==="")f="1"}else if(a.currentStyle){d=b.replace(ia,ja);f=a.currentStyle[b]||a.currentStyle[d];if(!mb.test(f)&&nb.test(f)){b=e.left;var j=a.runtimeStyle.left;a.runtimeStyle.left=a.currentStyle.left;e.left=d==="fontSize"?"1em":f||0;f=e.pixelLeft+"px";e.left=b;a.runtimeStyle.left=j}}return f},swap:function(a,b,d){var f={};for(var e in b){f[e]=a.style[e];a.style[e]=b[e]}d.call(a);for(e in b)a.style[e]=f[e]}});if(c.expr&&c.expr.filters){c.expr.filters.hidden=function(a){var b= +a.offsetWidth,d=a.offsetHeight,f=a.nodeName.toLowerCase()==="tr";return b===0&&d===0&&!f?true:b>0&&d>0&&!f?false:c.curCSS(a,"display")==="none"};c.expr.filters.visible=function(a){return!c.expr.filters.hidden(a)}}var sb=J(),tb=//gi,ub=/select|textarea/i,vb=/color|date|datetime|email|hidden|month|number|password|range|search|tel|text|time|url|week/i,N=/=\?(&|$)/,ka=/\?/,wb=/(\?|&)_=.*?(&|$)/,xb=/^(\w+:)?\/\/([^\/?#]+)/,yb=/%20/g,zb=c.fn.load;c.fn.extend({load:function(a,b,d){if(typeof a!== +"string")return zb.call(this,a);else if(!this.length)return this;var f=a.indexOf(" ");if(f>=0){var e=a.slice(f,a.length);a=a.slice(0,f)}f="GET";if(b)if(c.isFunction(b)){d=b;b=null}else if(typeof b==="object"){b=c.param(b,c.ajaxSettings.traditional);f="POST"}var j=this;c.ajax({url:a,type:f,dataType:"html",data:b,complete:function(i,o){if(o==="success"||o==="notmodified")j.html(e?c("
").append(i.responseText.replace(tb,"")).find(e):i.responseText);d&&j.each(d,[i.responseText,o,i])}});return this}, +serialize:function(){return c.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?c.makeArray(this.elements):this}).filter(function(){return this.name&&!this.disabled&&(this.checked||ub.test(this.nodeName)||vb.test(this.type))}).map(function(a,b){a=c(this).val();return a==null?null:c.isArray(a)?c.map(a,function(d){return{name:b.name,value:d}}):{name:b.name,value:a}}).get()}});c.each("ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split(" "), +function(a,b){c.fn[b]=function(d){return this.bind(b,d)}});c.extend({get:function(a,b,d,f){if(c.isFunction(b)){f=f||d;d=b;b=null}return c.ajax({type:"GET",url:a,data:b,success:d,dataType:f})},getScript:function(a,b){return c.get(a,null,b,"script")},getJSON:function(a,b,d){return c.get(a,b,d,"json")},post:function(a,b,d,f){if(c.isFunction(b)){f=f||d;d=b;b={}}return c.ajax({type:"POST",url:a,data:b,success:d,dataType:f})},ajaxSetup:function(a){c.extend(c.ajaxSettings,a)},ajaxSettings:{url:location.href, +global:true,type:"GET",contentType:"application/x-www-form-urlencoded",processData:true,async:true,xhr:A.XMLHttpRequest&&(A.location.protocol!=="file:"||!A.ActiveXObject)?function(){return new A.XMLHttpRequest}:function(){try{return new A.ActiveXObject("Microsoft.XMLHTTP")}catch(a){}},accepts:{xml:"application/xml, text/xml",html:"text/html",script:"text/javascript, application/javascript",json:"application/json, text/javascript",text:"text/plain",_default:"*/*"}},lastModified:{},etag:{},ajax:function(a){function b(){e.success&& +e.success.call(k,o,i,x);e.global&&f("ajaxSuccess",[x,e])}function d(){e.complete&&e.complete.call(k,x,i);e.global&&f("ajaxComplete",[x,e]);e.global&&!--c.active&&c.event.trigger("ajaxStop")}function f(q,p){(e.context?c(e.context):c.event).trigger(q,p)}var e=c.extend(true,{},c.ajaxSettings,a),j,i,o,k=a&&a.context||e,n=e.type.toUpperCase();if(e.data&&e.processData&&typeof e.data!=="string")e.data=c.param(e.data,e.traditional);if(e.dataType==="jsonp"){if(n==="GET")N.test(e.url)||(e.url+=(ka.test(e.url)? +"&":"?")+(e.jsonp||"callback")+"=?");else if(!e.data||!N.test(e.data))e.data=(e.data?e.data+"&":"")+(e.jsonp||"callback")+"=?";e.dataType="json"}if(e.dataType==="json"&&(e.data&&N.test(e.data)||N.test(e.url))){j=e.jsonpCallback||"jsonp"+sb++;if(e.data)e.data=(e.data+"").replace(N,"="+j+"$1");e.url=e.url.replace(N,"="+j+"$1");e.dataType="script";A[j]=A[j]||function(q){o=q;b();d();A[j]=w;try{delete A[j]}catch(p){}z&&z.removeChild(C)}}if(e.dataType==="script"&&e.cache===null)e.cache=false;if(e.cache=== +false&&n==="GET"){var r=J(),u=e.url.replace(wb,"$1_="+r+"$2");e.url=u+(u===e.url?(ka.test(e.url)?"&":"?")+"_="+r:"")}if(e.data&&n==="GET")e.url+=(ka.test(e.url)?"&":"?")+e.data;e.global&&!c.active++&&c.event.trigger("ajaxStart");r=(r=xb.exec(e.url))&&(r[1]&&r[1]!==location.protocol||r[2]!==location.host);if(e.dataType==="script"&&n==="GET"&&r){var z=s.getElementsByTagName("head")[0]||s.documentElement,C=s.createElement("script");C.src=e.url;if(e.scriptCharset)C.charset=e.scriptCharset;if(!j){var B= +false;C.onload=C.onreadystatechange=function(){if(!B&&(!this.readyState||this.readyState==="loaded"||this.readyState==="complete")){B=true;b();d();C.onload=C.onreadystatechange=null;z&&C.parentNode&&z.removeChild(C)}}}z.insertBefore(C,z.firstChild);return w}var E=false,x=e.xhr();if(x){e.username?x.open(n,e.url,e.async,e.username,e.password):x.open(n,e.url,e.async);try{if(e.data||a&&a.contentType)x.setRequestHeader("Content-Type",e.contentType);if(e.ifModified){c.lastModified[e.url]&&x.setRequestHeader("If-Modified-Since", +c.lastModified[e.url]);c.etag[e.url]&&x.setRequestHeader("If-None-Match",c.etag[e.url])}r||x.setRequestHeader("X-Requested-With","XMLHttpRequest");x.setRequestHeader("Accept",e.dataType&&e.accepts[e.dataType]?e.accepts[e.dataType]+", */*":e.accepts._default)}catch(ga){}if(e.beforeSend&&e.beforeSend.call(k,x,e)===false){e.global&&!--c.active&&c.event.trigger("ajaxStop");x.abort();return false}e.global&&f("ajaxSend",[x,e]);var g=x.onreadystatechange=function(q){if(!x||x.readyState===0||q==="abort"){E|| +d();E=true;if(x)x.onreadystatechange=c.noop}else if(!E&&x&&(x.readyState===4||q==="timeout")){E=true;x.onreadystatechange=c.noop;i=q==="timeout"?"timeout":!c.httpSuccess(x)?"error":e.ifModified&&c.httpNotModified(x,e.url)?"notmodified":"success";var p;if(i==="success")try{o=c.httpData(x,e.dataType,e)}catch(v){i="parsererror";p=v}if(i==="success"||i==="notmodified")j||b();else c.handleError(e,x,i,p);d();q==="timeout"&&x.abort();if(e.async)x=null}};try{var h=x.abort;x.abort=function(){x&&h.call(x); +g("abort")}}catch(l){}e.async&&e.timeout>0&&setTimeout(function(){x&&!E&&g("timeout")},e.timeout);try{x.send(n==="POST"||n==="PUT"||n==="DELETE"?e.data:null)}catch(m){c.handleError(e,x,null,m);d()}e.async||g();return x}},handleError:function(a,b,d,f){if(a.error)a.error.call(a.context||a,b,d,f);if(a.global)(a.context?c(a.context):c.event).trigger("ajaxError",[b,a,f])},active:0,httpSuccess:function(a){try{return!a.status&&location.protocol==="file:"||a.status>=200&&a.status<300||a.status===304||a.status=== +1223||a.status===0}catch(b){}return false},httpNotModified:function(a,b){var d=a.getResponseHeader("Last-Modified"),f=a.getResponseHeader("Etag");if(d)c.lastModified[b]=d;if(f)c.etag[b]=f;return a.status===304||a.status===0},httpData:function(a,b,d){var f=a.getResponseHeader("content-type")||"",e=b==="xml"||!b&&f.indexOf("xml")>=0;a=e?a.responseXML:a.responseText;e&&a.documentElement.nodeName==="parsererror"&&c.error("parsererror");if(d&&d.dataFilter)a=d.dataFilter(a,b);if(typeof a==="string")if(b=== +"json"||!b&&f.indexOf("json")>=0)a=c.parseJSON(a);else if(b==="script"||!b&&f.indexOf("javascript")>=0)c.globalEval(a);return a},param:function(a,b){function d(i,o){if(c.isArray(o))c.each(o,function(k,n){b||/\[\]$/.test(i)?f(i,n):d(i+"["+(typeof n==="object"||c.isArray(n)?k:"")+"]",n)});else!b&&o!=null&&typeof o==="object"?c.each(o,function(k,n){d(i+"["+k+"]",n)}):f(i,o)}function f(i,o){o=c.isFunction(o)?o():o;e[e.length]=encodeURIComponent(i)+"="+encodeURIComponent(o)}var e=[];if(b===w)b=c.ajaxSettings.traditional; +if(c.isArray(a)||a.jquery)c.each(a,function(){f(this.name,this.value)});else for(var j in a)d(j,a[j]);return e.join("&").replace(yb,"+")}});var la={},Ab=/toggle|show|hide/,Bb=/^([+-]=)?([\d+-.]+)(.*)$/,W,va=[["height","marginTop","marginBottom","paddingTop","paddingBottom"],["width","marginLeft","marginRight","paddingLeft","paddingRight"],["opacity"]];c.fn.extend({show:function(a,b){if(a||a===0)return this.animate(K("show",3),a,b);else{a=0;for(b=this.length;a").appendTo("body");f=e.css("display");if(f==="none")f="block";e.remove();la[d]=f}c.data(this[a],"olddisplay",f)}}a=0;for(b=this.length;a=0;f--)if(d[f].elem===this){b&&d[f](true);d.splice(f,1)}});b||this.dequeue();return this}});c.each({slideDown:K("show",1),slideUp:K("hide",1),slideToggle:K("toggle",1),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"}},function(a,b){c.fn[a]=function(d,f){return this.animate(b,d,f)}});c.extend({speed:function(a,b,d){var f=a&&typeof a==="object"?a:{complete:d||!d&&b||c.isFunction(a)&&a,duration:a,easing:d&&b||b&&!c.isFunction(b)&&b};f.duration=c.fx.off?0:typeof f.duration=== +"number"?f.duration:c.fx.speeds[f.duration]||c.fx.speeds._default;f.old=f.complete;f.complete=function(){f.queue!==false&&c(this).dequeue();c.isFunction(f.old)&&f.old.call(this)};return f},easing:{linear:function(a,b,d,f){return d+f*a},swing:function(a,b,d,f){return(-Math.cos(a*Math.PI)/2+0.5)*f+d}},timers:[],fx:function(a,b,d){this.options=b;this.elem=a;this.prop=d;if(!b.orig)b.orig={}}});c.fx.prototype={update:function(){this.options.step&&this.options.step.call(this.elem,this.now,this);(c.fx.step[this.prop]|| +c.fx.step._default)(this);if((this.prop==="height"||this.prop==="width")&&this.elem.style)this.elem.style.display="block"},cur:function(a){if(this.elem[this.prop]!=null&&(!this.elem.style||this.elem.style[this.prop]==null))return this.elem[this.prop];return(a=parseFloat(c.css(this.elem,this.prop,a)))&&a>-10000?a:parseFloat(c.curCSS(this.elem,this.prop))||0},custom:function(a,b,d){function f(j){return e.step(j)}this.startTime=J();this.start=a;this.end=b;this.unit=d||this.unit||"px";this.now=this.start; +this.pos=this.state=0;var e=this;f.elem=this.elem;if(f()&&c.timers.push(f)&&!W)W=setInterval(c.fx.tick,13)},show:function(){this.options.orig[this.prop]=c.style(this.elem,this.prop);this.options.show=true;this.custom(this.prop==="width"||this.prop==="height"?1:0,this.cur());c(this.elem).show()},hide:function(){this.options.orig[this.prop]=c.style(this.elem,this.prop);this.options.hide=true;this.custom(this.cur(),0)},step:function(a){var b=J(),d=true;if(a||b>=this.options.duration+this.startTime){this.now= +this.end;this.pos=this.state=1;this.update();this.options.curAnim[this.prop]=true;for(var f in this.options.curAnim)if(this.options.curAnim[f]!==true)d=false;if(d){if(this.options.display!=null){this.elem.style.overflow=this.options.overflow;a=c.data(this.elem,"olddisplay");this.elem.style.display=a?a:this.options.display;if(c.css(this.elem,"display")==="none")this.elem.style.display="block"}this.options.hide&&c(this.elem).hide();if(this.options.hide||this.options.show)for(var e in this.options.curAnim)c.style(this.elem, +e,this.options.orig[e]);this.options.complete.call(this.elem)}return false}else{e=b-this.startTime;this.state=e/this.options.duration;a=this.options.easing||(c.easing.swing?"swing":"linear");this.pos=c.easing[this.options.specialEasing&&this.options.specialEasing[this.prop]||a](this.state,e,0,1,this.options.duration);this.now=this.start+(this.end-this.start)*this.pos;this.update()}return true}};c.extend(c.fx,{tick:function(){for(var a=c.timers,b=0;b
"; +a.insertBefore(b,a.firstChild);d=b.firstChild;f=d.firstChild;e=d.nextSibling.firstChild.firstChild;this.doesNotAddBorder=f.offsetTop!==5;this.doesAddBorderForTableAndCells=e.offsetTop===5;f.style.position="fixed";f.style.top="20px";this.supportsFixedPosition=f.offsetTop===20||f.offsetTop===15;f.style.position=f.style.top="";d.style.overflow="hidden";d.style.position="relative";this.subtractsBorderForOverflowNotVisible=f.offsetTop===-5;this.doesNotIncludeMarginInBodyOffset=a.offsetTop!==j;a.removeChild(b); +c.offset.initialize=c.noop},bodyOffset:function(a){var b=a.offsetTop,d=a.offsetLeft;c.offset.initialize();if(c.offset.doesNotIncludeMarginInBodyOffset){b+=parseFloat(c.curCSS(a,"marginTop",true))||0;d+=parseFloat(c.curCSS(a,"marginLeft",true))||0}return{top:b,left:d}},setOffset:function(a,b,d){if(/static/.test(c.curCSS(a,"position")))a.style.position="relative";var f=c(a),e=f.offset(),j=parseInt(c.curCSS(a,"top",true),10)||0,i=parseInt(c.curCSS(a,"left",true),10)||0;if(c.isFunction(b))b=b.call(a, +d,e);d={top:b.top-e.top+j,left:b.left-e.left+i};"using"in b?b.using.call(a,d):f.css(d)}};c.fn.extend({position:function(){if(!this[0])return null;var a=this[0],b=this.offsetParent(),d=this.offset(),f=/^body|html$/i.test(b[0].nodeName)?{top:0,left:0}:b.offset();d.top-=parseFloat(c.curCSS(a,"marginTop",true))||0;d.left-=parseFloat(c.curCSS(a,"marginLeft",true))||0;f.top+=parseFloat(c.curCSS(b[0],"borderTopWidth",true))||0;f.left+=parseFloat(c.curCSS(b[0],"borderLeftWidth",true))||0;return{top:d.top- +f.top,left:d.left-f.left}},offsetParent:function(){return this.map(function(){for(var a=this.offsetParent||s.body;a&&!/^body|html$/i.test(a.nodeName)&&c.css(a,"position")==="static";)a=a.offsetParent;return a})}});c.each(["Left","Top"],function(a,b){var d="scroll"+b;c.fn[d]=function(f){var e=this[0],j;if(!e)return null;if(f!==w)return this.each(function(){if(j=wa(this))j.scrollTo(!a?f:c(j).scrollLeft(),a?f:c(j).scrollTop());else this[d]=f});else return(j=wa(e))?"pageXOffset"in j?j[a?"pageYOffset": +"pageXOffset"]:c.support.boxModel&&j.document.documentElement[d]||j.document.body[d]:e[d]}});c.each(["Height","Width"],function(a,b){var d=b.toLowerCase();c.fn["inner"+b]=function(){return this[0]?c.css(this[0],d,false,"padding"):null};c.fn["outer"+b]=function(f){return this[0]?c.css(this[0],d,false,f?"margin":"border"):null};c.fn[d]=function(f){var e=this[0];if(!e)return f==null?null:this;if(c.isFunction(f))return this.each(function(j){var i=c(this);i[d](f.call(this,j,i[d]()))});return"scrollTo"in +e&&e.document?e.document.compatMode==="CSS1Compat"&&e.document.documentElement["client"+b]||e.document.body["client"+b]:e.nodeType===9?Math.max(e.documentElement["client"+b],e.body["scroll"+b],e.documentElement["scroll"+b],e.body["offset"+b],e.documentElement["offset"+b]):f===w?c.css(e,d):this.css(d,typeof f==="string"?f:f+"px")}});A.jQuery=A.$=c})(window); diff --git a/contrib/rb/Monitoring/public/javascripts/jquery.form.js b/contrib/rb/Monitoring/public/javascripts/jquery.form.js new file mode 100755 index 000000000..be8c0b6bf --- /dev/null +++ b/contrib/rb/Monitoring/public/javascripts/jquery.form.js @@ -0,0 +1,675 @@ +/*! + * jQuery Form Plugin + * version: 2.43 (12-MAR-2010) + * @requires jQuery v1.3.2 or later + * + * Examples and documentation at: http://malsup.com/jquery/form/ + * Dual licensed under the MIT and GPL licenses: + * http://www.opensource.org/licenses/mit-license.php + * http://www.gnu.org/licenses/gpl.html + */ +;(function($) { + +/* + Usage Note: + ----------- + Do not use both ajaxSubmit and ajaxForm on the same form. These + functions are intended to be exclusive. Use ajaxSubmit if you want + to bind your own submit handler to the form. For example, + + $(document).ready(function() { + $('#myForm').bind('submit', function() { + $(this).ajaxSubmit({ + target: '#output' + }); + return false; // <-- important! + }); + }); + + Use ajaxForm when you want the plugin to manage all the event binding + for you. For example, + + $(document).ready(function() { + $('#myForm').ajaxForm({ + target: '#output' + }); + }); + + When using ajaxForm, the ajaxSubmit function will be invoked for you + at the appropriate time. +*/ + +/** + * ajaxSubmit() provides a mechanism for immediately submitting + * an HTML form using AJAX. + */ +$.fn.ajaxSubmit = function(options) { + // fast fail if nothing selected (http://dev.jquery.com/ticket/2752) + if (!this.length) { + log('ajaxSubmit: skipping submit process - no element selected'); + return this; + } + + if (typeof options == 'function') + options = { success: options }; + + var url = $.trim(this.attr('action')); + if (url) { + // clean url (don't include hash vaue) + url = (url.match(/^([^#]+)/)||[])[1]; + } + url = url || window.location.href || ''; + + options = $.extend({ + url: url, + type: this.attr('method') || 'GET', + iframeSrc: /^https/i.test(window.location.href || '') ? 'javascript:false' : 'about:blank' + }, options || {}); + + // hook for manipulating the form data before it is extracted; + // convenient for use with rich editors like tinyMCE or FCKEditor + var veto = {}; + this.trigger('form-pre-serialize', [this, options, veto]); + if (veto.veto) { + log('ajaxSubmit: submit vetoed via form-pre-serialize trigger'); + return this; + } + + // provide opportunity to alter form data before it is serialized + if (options.beforeSerialize && options.beforeSerialize(this, options) === false) { + log('ajaxSubmit: submit aborted via beforeSerialize callback'); + return this; + } + + var a = this.formToArray(options.semantic); + if (options.data) { + options.extraData = options.data; + for (var n in options.data) { + if(options.data[n] instanceof Array) { + for (var k in options.data[n]) + a.push( { name: n, value: options.data[n][k] } ); + } + else + a.push( { name: n, value: options.data[n] } ); + } + } + + // give pre-submit callback an opportunity to abort the submit + if (options.beforeSubmit && options.beforeSubmit(a, this, options) === false) { + log('ajaxSubmit: submit aborted via beforeSubmit callback'); + return this; + } + + // fire vetoable 'validate' event + this.trigger('form-submit-validate', [a, this, options, veto]); + if (veto.veto) { + log('ajaxSubmit: submit vetoed via form-submit-validate trigger'); + return this; + } + + var q = $.param(a); + + if (options.type.toUpperCase() == 'GET') { + options.url += (options.url.indexOf('?') >= 0 ? '&' : '?') + q; + options.data = null; // data is null for 'get' + } + else + options.data = q; // data is the query string for 'post' + + var $form = this, callbacks = []; + if (options.resetForm) callbacks.push(function() { $form.resetForm(); }); + if (options.clearForm) callbacks.push(function() { $form.clearForm(); }); + + // perform a load on the target only if dataType is not provided + if (!options.dataType && options.target) { + var oldSuccess = options.success || function(){}; + callbacks.push(function(data) { + var fn = options.replaceTarget ? 'replaceWith' : 'html'; + $(options.target)[fn](data).each(oldSuccess, arguments); + }); + } + else if (options.success) + callbacks.push(options.success); + + options.success = function(data, status, xhr) { // jQuery 1.4+ passes xhr as 3rd arg + for (var i=0, max=callbacks.length; i < max; i++) + callbacks[i].apply(options, [data, status, xhr || $form, $form]); + }; + + // are there files to upload? + var files = $('input:file', this).fieldValue(); + var found = false; + for (var j=0; j < files.length; j++) + if (files[j]) + found = true; + + var multipart = false; +// var mp = 'multipart/form-data'; +// multipart = ($form.attr('enctype') == mp || $form.attr('encoding') == mp); + + // options.iframe allows user to force iframe mode + // 06-NOV-09: now defaulting to iframe mode if file input is detected + if ((files.length && options.iframe !== false) || options.iframe || found || multipart) { + // hack to fix Safari hang (thanks to Tim Molendijk for this) + // see: http://groups.google.com/group/jquery-dev/browse_thread/thread/36395b7ab510dd5d + if (options.closeKeepAlive) + $.get(options.closeKeepAlive, fileUpload); + else + fileUpload(); + } + else + $.ajax(options); + + // fire 'notify' event + this.trigger('form-submit-notify', [this, options]); + return this; + + + // private function for handling file uploads (hat tip to YAHOO!) + function fileUpload() { + var form = $form[0]; + + if ($(':input[name=submit]', form).length) { + alert('Error: Form elements must not be named "submit".'); + return; + } + + var opts = $.extend({}, $.ajaxSettings, options); + var s = $.extend(true, {}, $.extend(true, {}, $.ajaxSettings), opts); + + var id = 'jqFormIO' + (new Date().getTime()); + var $io = $('