Skip to content

Commit 24ec58b

Browse files
committed
feat: avoid some data copies on writes
1 parent 837455a commit 24ec58b

File tree

4 files changed

+10
-13
lines changed

4 files changed

+10
-13
lines changed

Cargo.lock

Lines changed: 3 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,2 @@
11
[workspace]
22
members = ["object-store"]
3-
4-
[patch.crates-io]
5-
object_store = { git = "https://github.com/apache/arrow-rs", rev = "28a04db03cea376991e7efb04c3cf4f71f6d05bf" }

object-store/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ async-trait = "0.1.57"
1010
bytes = "1.2.1"
1111
futures = "0.3"
1212
once_cell = "1.12.0"
13-
object_store = { version = "0.5.2", features = [
13+
object_store = { version = "0.5.3", features = [
1414
"azure",
1515
"aws",
1616
"gcp",

object-store/src/file.rs

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -396,15 +396,14 @@ impl ObjectInputFile {
396396
};
397397
let nbytes = (range.end - range.start) as i64;
398398
self.pos += nbytes;
399-
let obj = if nbytes > 0 {
399+
let data = if nbytes > 0 {
400400
self.rt
401401
.block_on(self.store.get_range(&self.path, range))
402402
.map_err(ObjectStoreError::from)?
403-
.to_vec()
404403
} else {
405-
Vec::new()
404+
"".into()
406405
};
407-
Python::with_gil(|py| Ok(PyBytes::new(py, &obj).into_py(py)))
406+
Python::with_gil(|py| Ok(PyBytes::new(py, data.as_ref()).into_py(py)))
408407
}
409408

410409
fn fileno(&self) -> PyResult<()> {
@@ -525,10 +524,10 @@ impl ObjectOutputStream {
525524
Err(PyNotImplementedError::new_err("'read' not implemented"))
526525
}
527526

528-
fn write(&mut self, data: Vec<u8>) -> PyResult<i64> {
527+
fn write(&mut self, data: &PyBytes) -> PyResult<i64> {
529528
self.check_closed()?;
530-
let len = data.len() as i64;
531-
match self.rt.block_on(self.writer.write_all(&data)) {
529+
let len = data.as_bytes().len() as i64;
530+
match self.rt.block_on(self.writer.write_all(data.as_bytes())) {
532531
Ok(_) => Ok(len),
533532
Err(err) => {
534533
self.rt

0 commit comments

Comments
 (0)