Skip to content

Commit

Permalink
Merge pull request cassandra-rb#115 from nearbuy/composite_columns
Browse files Browse the repository at this point in the history
Initial support for composite column names
  • Loading branch information
ryanking committed Mar 29, 2012
2 parents b703b50 + a7e1004 commit 4bca4c8
Show file tree
Hide file tree
Showing 10 changed files with 224 additions and 18 deletions.
5 changes: 4 additions & 1 deletion conf/0.8/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@
"SuperUUID":{
"subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
"comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
"column_type":"Super"}
"column_type":"Super"},
"CompositeColumnConversion":{
"comparator_type":"org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type)",
"column_type":"Standard"}
}
}
2 changes: 1 addition & 1 deletion conf/0.8/schema.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,4 @@ create keyspace TypeConversions with
use TypeConversions;
create column family UUIDColumnConversion with comparator = TimeUUIDType;
create column family SuperUUID with comparator = TimeUUIDType and column_type = Super;

create column family CompositeColumnConversion with comparator = 'CompositeType(IntegerType, UTF8Type)';
5 changes: 4 additions & 1 deletion conf/1.0/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@
"SuperUUID":{
"subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
"comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
"column_type":"Super"}
"column_type":"Super"},
"CompositeColumnConversion":{
"comparator_type":"org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type)",
"column_type":"Standard"}
}
}
2 changes: 1 addition & 1 deletion conf/1.0/schema.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,4 @@ create keyspace TypeConversions with
use TypeConversions;
create column family UUIDColumnConversion with comparator = TimeUUIDType;
create column family SuperUUID with comparator = TimeUUIDType and column_type = Super;

create column family CompositeColumnConversion with comparator = 'CompositeType(IntegerType, UTF8Type)';
1 change: 1 addition & 0 deletions lib/cassandra.rb
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class Cassandra ; end
require 'cassandra/time'
require 'cassandra/comparable'
require 'cassandra/long'
require 'cassandra/composite'
require 'cassandra/ordered_hash'
require 'cassandra/columns'
require 'cassandra/protocol'
Expand Down
3 changes: 2 additions & 1 deletion lib/cassandra/columns.rb
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@ def sub_column_name_class(column_family)

def column_name_class_for_key(column_family, comparator_key)
property = column_family_property(column_family, comparator_key)
property =~ /.*\.(.*?)$/
property =~ /[^(]*\.(.*?)$/
case $1
when "LongType" then Long
when "LexicalUUIDType", "TimeUUIDType" then SimpleUUID::UUID
when /^CompositeType\(/ then Composite
else
String # UTF8, Ascii, Bytes, anything else
end
Expand Down
118 changes: 118 additions & 0 deletions lib/cassandra/composite.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@

class Cassandra
class Composite
include ::Comparable
attr_reader :parts
attr_reader :column_slice

def initialize(*parts)
options = {}
if parts.last.is_a?(Hash)
options = parts.pop
end
@column_slice = options[:slice]
raise ArgumentError if @column_slice != nil && ![:before, :after].include?(@column_slice)

if parts.length == 1 && parts[0].instance_of?(self.class)
@column_slice = parts[0].column_slice
@parts = parts[0].parts
elsif parts.length == 1 && parts[0].instance_of?(String) && @column_slice.nil? && valid_packed_composite?(parts[0])
unpack(parts[0])
else
@parts = parts
end
end

def [](*args)
return @parts[*args]
end

def pack
packed = @parts.map do |part|
[part.length].pack('n') + part + "\x00"
end
if @column_slice
part = @parts[-1]
packed[-1] = [part.length].pack('n') + part + slice_end_of_component
end
return packed.join('')
end

def to_s
return pack
end

def <=>(other)
if !other.instance_of?(self.class)
return @parts.first <=> other
end
eoc = slice_end_of_component.unpack('c')[0]
other_eoc = other.slice_end_of_component.unpack('c')[0]
@parts.zip(other.parts).each do |a, b|
next if a == b
if a.nil? && b.nil?
return eoc <=> other_eoc
end

if a.nil?
return @column_slice == :after ? 1 : -1
end
if b.nil?
return other.column_slice == :after ? -1 : 1
end
return -1 if a < b
return 1 if a > b
end
return 0
end

def inspect
return "#<Composite:#{@column_slice} #{@parts.inspect}>"
end

def slice_end_of_component
return "\x01" if @column_slice == :after
return "\xFF" if @column_slice == :before
return "\x00"
end

private
def unpack(packed_string)
parts = []
end_of_component = nil
while packed_string.length > 0
length = packed_string.slice(0, 2).unpack('n')[0]
parts << packed_string.slice(2, length)
end_of_component = packed_string.slice(2 + length, 1)

packed_string = packed_string.slice(3 + length, packed_string.length)
end
@column_slice = :after if end_of_component == "\x01"
@column_slice = :before if end_of_component == "\xFF"
@parts = parts
end

def valid_packed_composite?(packed_string)
while packed_string.length > 0
length = packed_string.slice(0, 2).unpack('n')[0]
return false if length.nil? || length + 3 > packed_string.length

end_of_component = packed_string.slice(2 + length, 1)
if length + 3 != packed_string.length
return false if end_of_component != "\x00"
end

packed_string = packed_string.slice(3 + length, packed_string.length)
end
return true
end

def hash
return to_s.hash
end

def eql?(other)
return to_s == other.to_s
end
end
end
6 changes: 6 additions & 0 deletions test/cassandra_mock_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@ def setup

@uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) }
@longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) }
@composites = [
Cassandra::Composite.new([5].pack('N'), "zebra"),
Cassandra::Composite.new([5].pack('N'), "aardvark"),
Cassandra::Composite.new([1].pack('N'), "elephant"),
Cassandra::Composite.new([10].pack('N'), "kangaroo"),
]
end

def test_setup
Expand Down
71 changes: 58 additions & 13 deletions test/cassandra_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ def setup

@uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) }
@longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) }
@composites = [
Cassandra::Composite.new([5].pack('N'), "zebra"),
Cassandra::Composite.new([5].pack('N'), "aardvark"),
Cassandra::Composite.new([1].pack('N'), "elephant"),
Cassandra::Composite.new([10].pack('N'), "kangaroo"),
]
end

def test_inspect
Expand All @@ -43,7 +49,7 @@ def test_setting_default_consistency
end

def test_get_key

@twitter.insert(:Users, key, {'body' => 'v', 'user' => 'v'})
assert_equal({'body' => 'v', 'user' => 'v'}, @twitter.get(:Users, key))
assert_equal(['body', 'user'].sort, @twitter.get(:Users, key).timestamps.keys.sort)
Expand Down Expand Up @@ -501,12 +507,12 @@ def test_count_keys

def test_count_columns
columns = (1..200).inject(Hash.new){|h,v| h['column' + v.to_s] = v.to_s; h;}

@twitter.insert(:Statuses, key, columns)
assert_equal 200, @twitter.count_columns(:Statuses, key, :count => 200)
assert_equal 100, @twitter.count_columns(:Statuses, key)
assert_equal 100, @twitter.count_columns(:Statuses, key)
assert_equal 55, @twitter.count_columns(:Statuses, key, :count => 55)

end

def test_count_super_columns
Expand Down Expand Up @@ -556,34 +562,34 @@ def test_batch_mutate
assert_equal({}, @twitter.get(:Users, k + '2')) # Not yet written
assert_equal({}, @twitter.get(:Statuses, k + '3')) # Not yet written

@twitter.remove(:Users, k + '1') # Full row
@twitter.remove(:Users, k + '1') # Full row
assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, k + '1')) # Not yet removed

@twitter.remove(:Users, k + '0', 'delete_me') # A single column of the row
assert_equal({'delete_me' => 'v0', 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # Not yet removed

@twitter.remove(:Users, k + '4')
@twitter.insert(:Users, k + '4', {'body' => 'v4', 'user' => 'v4'})
assert_equal({}, @twitter.get(:Users, k + '4')) # Not yet written

# SuperColumns
# Add and delete new sub columns to the user timeline supercolumn
@twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns })
@twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns })
@twitter.remove(:StatusRelationships, k, 'user_timelines' , subcolumn_to_delete ) # Delete the first of the initial_subcolumns from the user_timeline supercolumn
assert_equal(initial_subcolumns, @twitter.get(:StatusRelationships, k, 'user_timelines')) # No additions or deletes reflected yet
# Delete a complete supercolumn
# Delete a complete supercolumn
@twitter.remove(:StatusRelationships, k, 'dummy_supercolumn' ) # Delete the full dummy supercolumn
assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn')) # dummy supercolumn not yet deleted
assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn')) # dummy supercolumn not yet deleted
end

assert_equal({'body' => 'v2', 'user' => 'v2'}, @twitter.get(:Users, k + '2')) # Written
assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}, @twitter.get(:Users, k + '3')) # Written and compacted
assert_equal({'body' => 'v4', 'user' => 'v4'}, @twitter.get(:Users, k + '4')) # Written
assert_equal({'body' => 'v'}, @twitter.get(:Statuses, k + '3')) # Written
assert_equal({}, @twitter.get(:Users, k + '1')) # Removed

assert_equal({ 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # 'delete_me' column removed


assert_equal({'body' => 'v2', 'user' => 'v2'}.keys.sort, @twitter.get(:Users, k + '2').timestamps.keys.sort) # Written
assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}.keys.sort, @twitter.get(:Users, k + '3').timestamps.keys.sort) # Written and compacted
Expand All @@ -593,7 +599,7 @@ def test_batch_mutate
# Final result: initial_subcolumns - initial_subcolumns.first + new_subcolumns
resulting_subcolumns = initial_subcolumns.merge(new_subcolumns).reject{|k2,v| k2 == subcolumn_to_delete }
assert_equal(resulting_subcolumns, @twitter.get(:StatusRelationships, key, 'user_timelines'))
assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted
assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted

end

Expand Down Expand Up @@ -834,8 +840,47 @@ def test_adding_getting_value_in_multiple_counters_with_super_columns
assert_equal(1, @twitter.get(:UserCounterAggregates, 'bob', 'DAU', 'today'))
assert_equal(2, @twitter.get(:UserCounterAggregates, 'bob', 'DAU', 'tomorrow'))
end

def test_composite_column_type_conversion
columns = {}
@composites.each_with_index do |c, index|
columns[c] = "value-#{index}"
end
@type_conversions.insert(:CompositeColumnConversion, key, columns)
columns_in_order = [
Cassandra::Composite.new([1].pack('N'), "elephant"),
Cassandra::Composite.new([5].pack('N'), "aardvark"),
Cassandra::Composite.new([5].pack('N'), "zebra"),
Cassandra::Composite.new([10].pack('N'), "kangaroo"),
]
assert_equal(columns_in_order, @type_conversions.get(:CompositeColumnConversion, key).keys)

column_slice = @type_conversions.get(:CompositeColumnConversion, key,
:start => Cassandra::Composite.new([1].pack('N')),
:finish => Cassandra::Composite.new([10].pack('N')),
).keys
assert_equal(columns_in_order[0..-2], column_slice)

column_slice = @type_conversions.get(:CompositeColumnConversion, key,
:start => Cassandra::Composite.new([5].pack('N')),
:finish => Cassandra::Composite.new([5].pack('N'), :slice => :after),
).keys
assert_equal(columns_in_order[1..2], column_slice)

column_slice = @type_conversions.get(:CompositeColumnConversion, key,
:start => Cassandra::Composite.new([5].pack('N'), :slice => :after).to_s,
).keys
assert_equal([columns_in_order[-1]], column_slice)

column_slice = @type_conversions.get(:CompositeColumnConversion, key,
:finish => Cassandra::Composite.new([10].pack('N'), :slice => :before).to_s,
).keys
assert_equal(columns_in_order[0..-2], column_slice)

assert_equal('value-2', @type_conversions.get(:CompositeColumnConversion, key, columns_in_order.first))
end
end

def test_column_timestamps
base_time = Time.now
@twitter.insert(:Statuses, "time-key", { "body" => "value" })
Expand Down
29 changes: 29 additions & 0 deletions test/composite_type_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
require File.expand_path(File.dirname(__FILE__) + '/test_helper')

class CompositeTypesTest < Test::Unit::TestCase
include Cassandra::Constants

def setup
@col_parts = [[363].pack('N'), 'extradites-mulling', SimpleUUID::UUID.new().bytes]
@col = Cassandra::Composite.new(*@col_parts)
end

def test_creation_from_parts
assert_equal(@col_parts[0], @col[0])
assert_equal(@col_parts[1], @col[1])
assert_equal(@col_parts[2], @col[2])
end

def test_packing_and_unpacking
part0_length = 2 + 4 + 1 # size + int + end_term
part1_length = 2 + @col_parts[1].length + 1 # size + string_len + end_term
part2_length = 2 + @col_parts[2].length + 1 # size + uuid_bytes + end_term
assert_equal(part0_length + part1_length + part2_length, @col.pack.length)

col2 = Cassandra::Composite.new(@col.pack)
assert_equal(@col_parts[0], col2[0])
assert_equal(@col_parts[1], col2[1])
assert_equal(@col_parts[2], col2[2])
assert_equal(@col, col2)
end
end

0 comments on commit 4bca4c8

Please sign in to comment.