Skip to content

Commit deaa4eb

Browse files
allmightyspiffallmightyspiff
authored
Merge pull request #2202 from softlayer/issues1949
Added feature to iter_call to force a orderBy filter
2 parents c226a74 + 2044aba commit deaa4eb

22 files changed

+243
-116
lines changed

‎SoftLayer/API.py

+2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from SoftLayer import consts
2020
from SoftLayer import exceptions
2121
from SoftLayer import transports
22+
from SoftLayer import utils
2223

2324
LOGGER = logging.getLogger(__name__)
2425
API_PUBLIC_ENDPOINT = consts.API_PUBLIC_ENDPOINT
@@ -403,6 +404,7 @@ def iter_call(self, service, method, *args, **kwargs):
403404
kwargs['iter'] = False
404405
result_count = 0
405406
keep_looping = True
407+
kwargs['filter'] = utils.fix_filter(kwargs.get('filter'))
406408

407409
while keep_looping:
408410
# Get the next results

‎SoftLayer/CLI/environment.py

+1
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ def getpass(self, prompt, default=None):
111111
# In windows, shift+insert actually inputs the below 2 characters
112112
# If we detect those 2 characters, need to manually read from the clipbaord instead
113113
# https://stackoverflow.com/questions/101128/how-do-i-read-text-from-the-clipboard
114+
# LINUX NOTICE: `apt-get install python3-tk` required to install tk
114115
if password == 'àR':
115116
# tkinter is a built in python gui, but it has clipboard reading functions.
116117
# pylint: disable=import-outside-toplevel

‎SoftLayer/managers/block.py

+4-6
Original file line numberDiff line numberDiff line change
@@ -53,23 +53,21 @@ def list_block_volumes(self, datacenter=None, username=None, storage_type=None,
5353
_filter = utils.NestedDict(kwargs.get('filter') or {})
5454

5555
_filter['iscsiNetworkStorage']['serviceResource']['type']['type'] = utils.query_filter('!~ ISCSI')
56+
_filter['iscsiNetworkStorage']['id'] = utils.query_filter_orderby()
5657

57-
_filter['iscsiNetworkStorage']['storageType']['keyName'] = (
58-
utils.query_filter('*BLOCK_STORAGE*'))
58+
_filter['iscsiNetworkStorage']['storageType']['keyName'] = utils.query_filter('*BLOCK_STORAGE*')
5959
if storage_type:
6060
_filter['iscsiNetworkStorage']['storageType']['keyName'] = (
6161
utils.query_filter('%s_BLOCK_STORAGE*' % storage_type.upper()))
6262

6363
if datacenter:
64-
_filter['iscsiNetworkStorage']['serviceResource']['datacenter'][
65-
'name'] = utils.query_filter(datacenter)
64+
_filter['iscsiNetworkStorage']['serviceResource']['datacenter']['name'] = utils.query_filter(datacenter)
6665

6766
if username:
6867
_filter['iscsiNetworkStorage']['username'] = utils.query_filter(username)
6968

7069
if order:
71-
_filter['iscsiNetworkStorage']['billingItem']['orderItem'][
72-
'order']['id'] = utils.query_filter(order)
70+
_filter['iscsiNetworkStorage']['billingItem']['orderItem']['order']['id'] = utils.query_filter(order)
7371

7472
kwargs['filter'] = _filter.to_dict()
7573
return self.client.call('Account', 'getIscsiNetworkStorage', iter=True, **kwargs)

‎SoftLayer/managers/dns.py

+1
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ def get_records(self, zone_id, ttl=None, data=None, host=None, record_type=None)
196196
:returns: A list of dictionaries representing the matching records within the specified zone.
197197
"""
198198
_filter = utils.NestedDict()
199+
_filter['resourceRecords']['id'] = utils.query_filter_orderby()
199200

200201
if ttl:
201202
_filter['resourceRecords']['ttl'] = utils.query_filter(ttl)

‎SoftLayer/managers/event_log.py

+1-4
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,8 @@ def build_filter(date_min=None, date_max=None, obj_event=None, obj_id=None, obj_
6565
6666
:returns: dict: The generated query filter
6767
"""
68-
69-
if not any([date_min, date_max, obj_event, obj_id, obj_type]):
70-
return {}
71-
7268
request_filter = {}
69+
request_filter['traceId'] = utils.query_filter_orderby()
7370

7471
if date_min and date_max:
7572
request_filter['eventCreateDate'] = utils.event_log_filter_between_date(date_min, date_max, utc_offset)

‎SoftLayer/managers/file.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ def list_file_volumes(self, datacenter=None, username=None, storage_type=None, o
4747
kwargs['mask'] = ','.join(items)
4848

4949
_filter = utils.NestedDict(kwargs.get('filter') or {})
50-
50+
_filter['nasNetworkStorage']['id'] = utils.query_filter_orderby()
5151
_filter['nasNetworkStorage']['serviceResource']['type']['type'] = utils.query_filter('!~ NAS')
5252

5353
_filter['nasNetworkStorage']['storageType']['keyName'] = (

‎SoftLayer/managers/image.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,12 @@ def list_private_images(self, guid=None, name=None, limit=100, **kwargs):
5757
kwargs['mask'] = IMAGE_MASK
5858

5959
_filter = utils.NestedDict(kwargs.get('filter') or {})
60+
_filter['privateBlockDeviceTemplateGroups']['id'] = utils.query_filter_orderby()
6061
if name:
61-
_filter['privateBlockDeviceTemplateGroups']['name'] = (
62-
utils.query_filter(name))
62+
_filter['privateBlockDeviceTemplateGroups']['name'] = utils.query_filter(name)
6363

6464
if guid:
65-
_filter['privateBlockDeviceTemplateGroups']['globalIdentifier'] = (
66-
utils.query_filter(guid))
65+
_filter['privateBlockDeviceTemplateGroups']['globalIdentifier'] = utils.query_filter(guid)
6766

6867
kwargs['filter'] = _filter.to_dict()
6968

@@ -81,6 +80,7 @@ def list_public_images(self, guid=None, name=None, limit=100, **kwargs):
8180
kwargs['mask'] = IMAGE_MASK
8281

8382
_filter = utils.NestedDict(kwargs.get('filter') or {})
83+
_filter['id'] = utils.query_filter_orderby()
8484
if name:
8585
_filter['name'] = utils.query_filter(name)
8686

‎SoftLayer/managers/network.py

+1
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,7 @@ def list_subnets(self, identifier=None, datacenter=None, version=0,
496496
kwargs['mask'] = DEFAULT_SUBNET_MASK
497497

498498
_filter = utils.NestedDict(kwargs.get('filter') or {})
499+
_filter['subnets']['id'] = utils.query_filter_orderby()
499500

500501
if identifier:
501502
_filter['subnets']['networkIdentifier'] = (

‎SoftLayer/utils.py

+27
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"""
77

88
import collections
9+
import copy
910
import datetime
1011
from json import JSONDecoder
1112
import re
@@ -41,6 +42,32 @@ def lookup(dic, key, *keys):
4142
return dic.get(key)
4243

4344

45+
def has_key_value(d: dict, key: str = "operation", value: str = "orderBy") -> bool:
46+
"""Scan through a dictionary looking for an orderBy clause, but can be used for any key/value combo"""
47+
if d.get(key) and d.get(key) == value:
48+
return True
49+
for x in d.values():
50+
if isinstance(x, dict):
51+
if has_key_value(x, key, value):
52+
return True
53+
return False
54+
55+
56+
def fix_filter(sl_filter: dict = None) -> dict:
57+
"""Forces an object filter to have an orderBy clause if it doesn't have one already"""
58+
59+
if sl_filter is None:
60+
sl_filter = {}
61+
62+
# Make a copy to prevent sl_filter from being modified by this function
63+
this_filter = copy.copy(sl_filter)
64+
if not has_key_value(this_filter, "operation", "orderBy"):
65+
# Check to see if 'id' is already a filter, if so just skip
66+
if not this_filter.get('id', False):
67+
this_filter['id'] = query_filter_orderby()
68+
return this_filter
69+
70+
4471
class NestedDict(dict):
4572
"""This helps with accessing a heavily nested dictionary.
4673

‎tests/CLI/modules/block_tests.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,8 @@ def test_volume_detail_name_identifier(self):
125125
'storageType': {
126126
'keyName': {'operation': '*= BLOCK_STORAGE'}
127127
},
128-
'username': {'operation': '_= SL-12345'}
128+
'username': {'operation': '_= SL-12345'},
129+
'id': {'operation': 'orderBy', 'options': [{'name': 'sort', 'value': ['ASC']}]}
129130
}
130131
}
131132

‎tests/CLI/modules/event_log_tests.py

+2-3
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,9 @@ def test_get_event_log_empty(self):
3131
mock.return_value = None
3232

3333
result = self.run_command(['event-log', 'get'])
34-
expected = 'Event, Object, Type, Date, Username\n' \
35-
'No logs available for filter {}.\n'
34+
3635
self.assert_no_fail(result)
37-
self.assertEqual(expected, result.output)
36+
self.assertIn("No logs available for filter ", result.output)
3837

3938
def test_get_event_log_over_limit(self):
4039
result = self.run_command(['event-log', 'get', '-l 1'])

‎tests/CLI/modules/file_tests.py

+6-7
Original file line numberDiff line numberDiff line change
@@ -210,14 +210,13 @@ def test_volume_detail_name_identifier(self):
210210
expected_filter = {
211211
'nasNetworkStorage': {
212212
'serviceResource': {
213-
'type': {
214-
'type': {'operation': '!~ NAS'}
215-
}
213+
'type': {'type': {'operation': '!~ NAS'}}
216214
},
217-
'storageType': {
218-
'keyName': {'operation': '*= FILE_STORAGE'}
219-
},
220-
'username': {'operation': '_= SL-12345'}}}
215+
'storageType': {'keyName': {'operation': '*= FILE_STORAGE'}},
216+
'username': {'operation': '_= SL-12345'},
217+
'id': {'operation': 'orderBy', 'options': [{'name': 'sort', 'value': ['ASC']}]}
218+
}
219+
}
221220

222221
self.assert_called_with('SoftLayer_Account', 'getNasNetworkStorage', filter=expected_filter)
223222
self.assert_called_with('SoftLayer_Network_Storage', 'getObject', identifier=1)

‎tests/api_tests.py

+15-20
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,8 @@ def test_iter_call(self, _call):
169169

170170
self.assertEqual(list(range(125)), result)
171171
_call.assert_has_calls([
172-
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=0),
173-
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=100),
172+
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=0, filter=mock.ANY),
173+
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=100, filter=mock.ANY),
174174
])
175175
_call.reset_mock()
176176

@@ -183,9 +183,9 @@ def test_iter_call(self, _call):
183183
result = list(self.client.iter_call('SERVICE', 'METHOD', iter=True))
184184
self.assertEqual(list(range(200)), result)
185185
_call.assert_has_calls([
186-
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=0),
187-
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=100),
188-
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=200),
186+
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=0, filter=mock.ANY),
187+
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=100, filter=mock.ANY),
188+
mock.call('SERVICE', 'METHOD', limit=100, iter=False, offset=200, filter=mock.ANY),
189189
])
190190
_call.reset_mock()
191191

@@ -194,12 +194,11 @@ def test_iter_call(self, _call):
194194
transports.SoftLayerListResult(range(0, 25), 30),
195195
transports.SoftLayerListResult(range(25, 30), 30)
196196
]
197-
result = list(self.client.iter_call(
198-
'SERVICE', 'METHOD', iter=True, limit=25))
197+
result = list(self.client.iter_call('SERVICE', 'METHOD', iter=True, limit=25))
199198
self.assertEqual(list(range(30)), result)
200199
_call.assert_has_calls([
201-
mock.call('SERVICE', 'METHOD', iter=False, limit=25, offset=0),
202-
mock.call('SERVICE', 'METHOD', iter=False, limit=25, offset=25),
200+
mock.call('SERVICE', 'METHOD', iter=False, limit=25, offset=0, filter=mock.ANY),
201+
mock.call('SERVICE', 'METHOD', iter=False, limit=25, offset=25, filter=mock.ANY),
203202
])
204203
_call.reset_mock()
205204

@@ -208,31 +207,27 @@ def test_iter_call(self, _call):
208207
result = list(self.client.iter_call('SERVICE', 'METHOD', iter=True))
209208
self.assertEqual(["test"], result)
210209
_call.assert_has_calls([
211-
mock.call('SERVICE', 'METHOD', iter=False, limit=100, offset=0),
210+
mock.call('SERVICE', 'METHOD', iter=False, limit=100, offset=0, filter=mock.ANY),
212211
])
213212
_call.reset_mock()
214213

215214
_call.side_effect = [
216215
transports.SoftLayerListResult(range(0, 25), 30),
217216
transports.SoftLayerListResult(range(25, 30), 30)
218217
]
219-
result = list(self.client.iter_call('SERVICE', 'METHOD', 'ARG',
220-
iter=True,
221-
limit=25,
222-
offset=12))
218+
result = list(
219+
self.client.iter_call('SERVICE', 'METHOD', 'ARG', iter=True, limit=25, offset=12)
220+
)
223221
self.assertEqual(list(range(30)), result)
224222
_call.assert_has_calls([
225-
mock.call('SERVICE', 'METHOD', 'ARG',
226-
iter=False, limit=25, offset=12),
227-
mock.call('SERVICE', 'METHOD', 'ARG',
228-
iter=False, limit=25, offset=37),
223+
mock.call('SERVICE', 'METHOD', 'ARG', iter=False, limit=25, offset=12, filter=mock.ANY),
224+
mock.call('SERVICE', 'METHOD', 'ARG', iter=False, limit=25, offset=37, filter=mock.ANY),
229225
])
230226

231227
# Chunk size of 0 is invalid
232228
self.assertRaises(
233229
AttributeError,
234-
lambda: list(self.client.iter_call('SERVICE', 'METHOD',
235-
iter=True, limit=0)))
230+
lambda: list(self.client.iter_call('SERVICE', 'METHOD', iter=True, limit=0, filter=mock.ANY)))
236231

237232
def test_call_invalid_arguments(self):
238233
self.assertRaises(

‎tests/managers/block_tests.py

+16-27
Original file line numberDiff line numberDiff line change
@@ -125,19 +125,17 @@ def test_get_block_volume_details(self):
125125
def test_list_block_volumes(self):
126126
result = self.block.list_block_volumes()
127127

128-
self.assertEqual(SoftLayer_Account.getIscsiNetworkStorage,
129-
result)
128+
self.assertEqual(SoftLayer_Account.getIscsiNetworkStorage, result)
130129

131130
expected_filter = {
132131
'iscsiNetworkStorage': {
133132
'storageType': {
134133
'keyName': {'operation': '*= BLOCK_STORAGE'}
135134
},
136135
'serviceResource': {
137-
'type': {
138-
'type': {'operation': '!~ ISCSI'}
139-
}
140-
}
136+
'type': {'type': {'operation': '!~ ISCSI'}}
137+
},
138+
'id': {'operation': 'orderBy', 'options': [{'name': 'sort', 'value': ['ASC']}]}
141139
}
142140
}
143141

@@ -161,23 +159,20 @@ def test_list_block_volumes(self):
161159
def test_list_block_volumes_additional_filter_order(self):
162160
result = self.block.list_block_volumes(order=1234567)
163161

164-
self.assertEqual(SoftLayer_Account.getIscsiNetworkStorage,
165-
result)
162+
self.assertEqual(SoftLayer_Account.getIscsiNetworkStorage, result)
166163

167164
expected_filter = {
168165
'iscsiNetworkStorage': {
169166
'storageType': {
170167
'keyName': {'operation': '*= BLOCK_STORAGE'}
171168
},
172169
'serviceResource': {
173-
'type': {
174-
'type': {'operation': '!~ ISCSI'}
175-
}
170+
'type': {'type': {'operation': '!~ ISCSI'}}
176171
},
177172
'billingItem': {
178-
'orderItem': {
179-
'order': {
180-
'id': {'operation': 1234567}}}}
173+
'orderItem': {'order': {'id': {'operation': 1234567}}}
174+
},
175+
'id': {'operation': 'orderBy', 'options': [{'name': 'sort', 'value': ['ASC']}]}
181176
}
182177
}
183178

@@ -199,27 +194,21 @@ def test_list_block_volumes_additional_filter_order(self):
199194
)
200195

201196
def test_list_block_volumes_with_additional_filters(self):
202-
result = self.block.list_block_volumes(datacenter="dal09",
203-
storage_type="Endurance",
204-
username="username")
197+
result = self.block.list_block_volumes(datacenter="dal09", storage_type="Endurance", username="username")
205198

206-
self.assertEqual(SoftLayer_Account.getIscsiNetworkStorage,
207-
result)
199+
self.assertEqual(SoftLayer_Account.getIscsiNetworkStorage, result)
208200

209201
expected_filter = {
210202
'iscsiNetworkStorage': {
211203
'storageType': {
212204
'keyName': {'operation': '^= ENDURANCE_BLOCK_STORAGE'}
213205
},
214-
'username': {'operation': u'_= username'},
206+
'username': {'operation': '_= username'},
215207
'serviceResource': {
216-
'datacenter': {
217-
'name': {'operation': u'_= dal09'}
218-
},
219-
'type': {
220-
'type': {'operation': '!~ ISCSI'}
221-
}
222-
}
208+
'datacenter': {'name': {'operation': u'_= dal09'}},
209+
'type': {'type': {'operation': '!~ ISCSI'}}
210+
},
211+
'id': {'operation': 'orderBy', 'options': [{'name': 'sort', 'value': ['ASC']}]}
223212
}
224213
}
225214

‎tests/managers/dedicated_host_tests.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -714,7 +714,8 @@ def test_list_guests_with_filters(self):
714714
'networkComponents': {'maxSpeed': {'operation': 100}},
715715
'primaryIpAddress': {'operation': '_= 1.2.3.4'},
716716
'primaryBackendIpAddress': {'operation': '_= 4.3.2.1'}
717-
}
717+
},
718+
'id': {'operation': 'orderBy', 'options': [{'name': 'sort', 'value': ['ASC']}]}
718719
}
719720
self.assert_called_with('SoftLayer_Virtual_DedicatedHost', 'getGuests',
720721
identifier=12345, filter=_filter)

0 commit comments

Comments
 (0)