Refactored warcio's loop and string concatenation heavy sections to be gain performance increases

- https://wiki.python.org/moin/PythonSpeed/PerformanceTips#Avoiding_dots...
- https://wiki.python.org/moin/PythonSpeed/PerformanceTips#String_Concatenation
- http://blog.mclemon.io/python-efficient-string-concatenation-in-python-2016-edition

Author: N0taN3rd

warcio/archiveiterator.py

@@ -80,28 +80,46 @@ def close(self):
             self.reader.close_decompressor()
             self.reader = None
 
+    def _decompressor(self):
+        """Helper method for _iterate_records that returns the readers decompressor"""
+        return self.reader.decompressor
+
+    def _iterate_records_next_record(self, raise_invalid_gzip):
+        """Helper method for _iterate_records that reads and returns the next record.
+        If raise_invalid_gzip is true, _raise_invalid_gzip_err is called.
+
+        :param bool raise_invalid_gzip: Should the invalid gzip error be raised
+        :return: The next record
+        :rtype: ArcWarcRecord
+        """
+        self.record = self._next_record(self.next_line)
+        if raise_invalid_gzip:
+            self._raise_invalid_gzip_err()
+        return self.record
+
     def _iterate_records(self):
         """ iterate over each record
         """
         raise_invalid_gzip = False
         empty_record = False
 
+        # perf references in order to avoid the cost of dot property lookup
+        self_read_to_end = self.read_to_end
+        self_decompressor = self._decompressor
+        self_reader_read_next_member = self.reader.read_next_member
+        self_iterate_records_next_record = self._iterate_records_next_record
+
         while True:
             try:
-                self.record = self._next_record(self.next_line)
-                if raise_invalid_gzip:
-                    self._raise_invalid_gzip_err()
-
-                yield self.record
-
+                yield self_iterate_records_next_record(raise_invalid_gzip)
             except EOFError:
                 empty_record = True
 
-            self.read_to_end()
+            self_read_to_end()
 
-            if self.reader.decompressor:
+            if self_decompressor():
                 # if another gzip member, continue
-                if self.reader.read_next_member():
+                if self_reader_read_next_member():
                     continue
 
                 # if empty record, then we're done
@@ -149,10 +167,18 @@ def _consume_blanklines(self):
           display a warning
         """
         empty_size = 0
+        error_count = 0
         first_line = True
 
+        # perf references in order to avoid the cost of dot property lookup
+        self_reader_readline = self.reader.readline
+        self_fh_tell = self.fh.tell
+        self_reader_rem_length = self.reader.rem_length
+        self_INC_RECORD_format = self.INC_RECORD.format
+        sys_stderr_write = sys.stderr.write
+
         while True:
-            line = self.reader.readline()
+            line = self_reader_readline()
             if len(line) == 0:
                 return None, empty_size
 
@@ -164,13 +190,15 @@ def _consume_blanklines(self):
                 if len(stripped) != 0:
                     # if first line is not blank,
                     # likely content-length was invalid, display warning
-                    err_offset = self.fh.tell() - self.reader.rem_length() - empty_size
-                    sys.stderr.write(self.INC_RECORD.format(err_offset, line))
-                    self.err_count += 1
+                    err_offset = self_fh_tell() - self_reader_rem_length() - empty_size
+                    sys_stderr_write(self_INC_RECORD_format(err_offset, line))
+                    error_count += 1
 
                 first_line = False
                 continue
 
+            # we hit here only once in the loop
+            self.err_count += error_count
             return line, empty_size
 
     def read_to_end(self, record=None):
@@ -189,8 +217,11 @@ def read_to_end(self, record=None):
 
         curr_offset = self.offset
 
+        # perf reference in order to avoid the cost of dot property lookup
+        self_record_raw_stream_read = self.record.raw_stream.read
+
         while True:
-            b = self.record.raw_stream.read(BUFF_SIZE)
+            b = self_record_raw_stream_read(BUFF_SIZE)
             if not b:
                 break
 

warcio/bufferedreaders.py

@@ -94,6 +94,16 @@ def _init_decomp(self, decomp_type):
             self.decomp_type = None
             self.decompressor = None
 
+    def _fillbuff_has_more_data(self):
+        """Returns T/F to indicate if there is more data
+        to be read by _fillbuff.
+
+        :return: T/F to indicate if there is more data
+        to be read
+        :rtype: bool
+        """
+        return self.decompressor and not self.decompressor.unused_data and self.empty()
+
     def _fillbuff(self, block_size=None):
         if not self.empty():
             return
@@ -112,12 +122,17 @@ def _fillbuff(self, block_size=None):
 
         self._process_read(data)
 
+        # perf references in order to avoid the cost of dot property lookup
+        self_stream_read = self.stream.read
+        self_process_read = self._process_read
+        self_fillbuff_has_more_data = self._fillbuff_has_more_data
+
         # if raw data is not empty and decompressor set, but
         # decompressed buff is empty, keep reading --
         # decompressor likely needs more data to decompress
-        while data and self.decompressor and not self.decompressor.unused_data and self.empty():
-            data = self.stream.read(block_size)
-            self._process_read(data)
+        while data and self_fillbuff_has_more_data():
+            data = self_stream_read(block_size)
+            self_process_read(data)
 
     def _process_read(self, data):
         # don't process if no raw data read
@@ -149,6 +164,14 @@ def _decompress(self, data):
                     return b''
         return data
 
+    def _buff_read(self, length):
+        """Utility method for read that returns
+        the results of self.buff.read(length).
+
+        :param int length: The mount to be read
+        """
+        return self.buff.read(length)
+
     def read(self, length=None):
         """
         Fill bytes and read some number of bytes
@@ -158,19 +181,32 @@ def read(self, length=None):
         specified length is read
         """
         all_buffs = []
+
+        # perf references in order to avoid the cost of dot property lookup
+        all_buffs_append = all_buffs.append
+        self_fillbuff = self._fillbuff
+        self_empty = self.empty
+        self_buff_read = self._buff_read
+
         while length is None or length > 0:
-            self._fillbuff()
-            if self.empty():
+            self_fillbuff()
+            if self_empty():
                 break
 
-            buff = self.buff.read(length)
-            all_buffs.append(buff)
+            buff = self_buff_read(length)
+            all_buffs_append(buff)
             if length:
                 length -= len(buff)
 
         return b''.join(all_buffs)
 
+    def _buff_readline(self, length):
+        """Utility method for read that returns
+        the results of self.buff.readline(length).
 
+        :param int length: The mount to be read
+        """
+        return self.buff.readline(length)
 
     def readline(self, length=None):
         """
@@ -189,21 +225,35 @@ def readline(self, length=None):
 
         linebuff = self.buff.readline(length)
 
+        # perf references in order to avoid the cost of dot property lookup
+        self_fillbuff = self._fillbuff
+        self_empty = self.empty
+        self_buff_readline = self._buff_readline
+
+        # string concatenation using += is an expensive operation due to python's string internment
+        # appending each part of the string using a list is the pythonic way
+        # https://wiki.python.org/moin/PythonSpeed/PerformanceTips#String_Concatenation
+        current_full_line_buff_len = len(linebuff)
+        full_line_buff = [linebuff]
+        full_line_buff_append = full_line_buff.append
+
         # we may be at a boundary
         while not linebuff.endswith(b'\n'):
             if length:
-                length -= len(linebuff)
+                length -= current_full_line_buff_len
                 if length <= 0:
                     break
 
-            self._fillbuff()
+            self_fillbuff()
 
-            if self.empty():
+            if self_empty():
                 break
 
-            linebuff += self.buff.readline(length)
+            linebuff = self_buff_readline(length)
+            full_line_buff_append(linebuff)
+            current_full_line_buff_len += len(linebuff)
 
-        return linebuff
+        return b''.join(full_line_buff)
 
     def empty(self):
         if not self.buff or self.buff.tell() >= self.buff_size:
@@ -292,29 +342,53 @@ def _fillbuff(self, block_size=None):
         if self.not_chunked:
             return super(ChunkedDataReader, self)._fillbuff(block_size)
 
+        length_header = None
+
+        # perf references in order to avoid the cost of dot property lookup
+        self_chunked_fillbuff_has_more_data = self._chunked_fillbuff_has_more_data
+        self_stream_readline = self.stream.readline
+        self_try_decode = self._try_decode
+        self_chunked_fillbuff_handle_exception = self._chunked_fillbuff_handle_exception
+
         # Loop over chunks until there is some data (not empty())
         # In particular, gzipped data may require multiple chunks to
         # return any decompressed result
-        while (self.empty() and
-               not self.all_chunks_read and
-               not self.not_chunked):
-
+        while self_chunked_fillbuff_has_more_data():
             try:
-                length_header = self.stream.readline(64)
-                self._try_decode(length_header)
+                length_header = self_stream_readline(64)
+                self_try_decode(length_header)
             except ChunkedDataException as e:
-                if self.raise_chunked_data_exceptions:
-                    raise
+                self_chunked_fillbuff_handle_exception(e, length_header, block_size)
 
-                # Can't parse the data as chunked.
-                # It's possible that non-chunked data is served
-                # with a Transfer-Encoding: chunked.
-                # Treat this as non-chunk encoded from here on.
-                self._process_read(length_header + e.data)
-                self.not_chunked = True
+    def _chunked_fillbuff_has_more_data(self):
+        """Determines if there is more data to be had for filling the
+        _fillbuff method.
 
-                # parse as block as non-chunked
-                return super(ChunkedDataReader, self)._fillbuff(block_size)
+        :return: T/F indicating if there is more data in the stream
+        :rtype: bool
+        """
+        return self.empty() and not self.all_chunks_read and not self.not_chunked
+
+    def _chunked_fillbuff_handle_exception(self, e, length_header, block_size):
+        """Handles the ChunkedDataException raised by _try_decode while attempting
+        to fill the buffer
+
+        :param ChunkedDataException e:
+        :param bytes length_header:
+        :param int block_size:
+        """
+        if self.raise_chunked_data_exceptions:
+            raise e
+
+        # Can't parse the data as chunked.
+        # It's possible that non-chunked data is served
+        # with a Transfer-Encoding: chunked.
+        # Treat this as non-chunk encoded from here on.
+        self._process_read(length_header + e.data)
+        self.not_chunked = True
+
+        # parse as block as non-chunked
+        return super(ChunkedDataReader, self)._fillbuff(block_size)
 
     def _try_decode(self, length_header):
         # decode length header
@@ -336,36 +410,53 @@ def _try_decode(self, length_header):
             return
 
         data_len = 0
-        data = b''
+
+        # string concatenation perf
+        data = []
+
+        # perf references in order to avoid the cost of dot property lookup
+        data_append = data.append
+        self_stream_read = self.stream.read
+        self_try_decode_no_new_data = self._try_decode_no_new_data
 
         # read chunk
         while data_len < chunk_size:
-            new_data = self.stream.read(chunk_size - data_len)
+            new_data = self_stream_read(chunk_size - data_len)
 
             # if we unexpectedly run out of data,
             # either raise an exception or just stop reading,
             # assuming file was cut off
             if not new_data:
-                if self.raise_chunked_data_exceptions:
-                    msg = 'Ran out of data before end of chunk'
-                    raise ChunkedDataException(msg, data)
-                else:
-                    chunk_size = data_len
-                    self.all_chunks_read = True
+                # if self_try_decode_no_new_data does not raise an exception
+                # set chunk_size to the current data_len in order to stop reading
+                self_try_decode_no_new_data(data)
+                chunk_size = data_len
 
-            data += new_data
-            data_len = len(data)
+            data_append(new_data)
+            data_len += len(new_data)
 
         # if we successfully read a block without running out,
         # it should end in \r\n
         if not self.all_chunks_read:
             clrf = self.stream.read(2)
             if clrf != b'\r\n':
-                raise ChunkedDataException(b"Chunk terminator not found.",
-                                           data)
+                raise ChunkedDataException(b"Chunk terminator not found.", b''.join(data))
 
         # hand to base class for further processing
-        self._process_read(data)
+        self._process_read(b''.join(data))
+
+    def _try_decode_no_new_data(self, data_buffer):
+        """If we unexpectedly run out of data, either raise an exception or just stop reading,
+        assuming file was cut off.
+
+        :param list[bytes] data_buffer: The list of byte strings being
+        :return:
+        """
+        if self.raise_chunked_data_exceptions:
+            msg = 'Ran out of data before end of chunk'
+            raise ChunkedDataException(msg, b''.join(data_buffer))
+        else:
+            self.all_chunks_read = True
 
 
 #=================================================================