Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Revert "remove read_zipfile_from_stream()"
This reverts commit 3abac52.
  • Loading branch information
cosmicexplorer committed Jul 17, 2024
commit 3259c5e39b7fe872a3d2844d67a3dbc2d2ff1c0d
31 changes: 6 additions & 25 deletions benches/read_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,7 @@ use bencher::Bencher;
use getrandom::getrandom;
use tempdir::TempDir;
use zip::write::SimpleFileOptions;
use zip::{
result::ZipResult,
unstable::{
read::streaming::{StreamingZipEntry, ZipStreamFileMetadata},
stream::{ZipStreamReader, ZipStreamVisitor},
},
CompressionMethod, ZipArchive, ZipWriter,
};
use zip::{result::ZipResult, CompressionMethod, ZipArchive, ZipWriter};

const FILE_COUNT: usize = 15_000;
const FILE_SIZE: usize = 1024;
Expand Down Expand Up @@ -113,24 +106,12 @@ fn parse_stream_archive(bench: &mut Bencher) {
let out = dir.path().join("bench-out.zip");
fs::write(&out, &bytes).unwrap();

struct V;
impl ZipStreamVisitor for V {
fn visit_file(&mut self, _file: &mut StreamingZipEntry<impl Read>) -> ZipResult<()> {
Ok(())
}

fn visit_additional_metadata(
&mut self,
_metadata: &ZipStreamFileMetadata,
) -> ZipResult<()> {
Ok(())
}
}

bench.iter(|| {
let f = fs::File::open(&out).unwrap();
let archive = ZipStreamReader::new(f);
archive.visit(&mut V).unwrap();
let mut f = fs::File::open(&out).unwrap();
while zip::read::read_zipfile_from_stream(&mut f)
.unwrap()
.is_some()
{}
});
bench.bytes = bytes.len() as u64;
}
Expand Down
7 changes: 2 additions & 5 deletions examples/stdin_info.rs
Original file line number Diff line number Diff line change
@@ -1,19 +1,16 @@
use std::io::{self, Read};

use zip::unstable::read::{streaming::StreamingArchive, ArchiveEntry};

fn main() {
std::process::exit(real_main());
}

fn real_main() -> i32 {
let stdin = io::stdin();
let stdin_handle = stdin.lock();
let mut stdin_handle = stdin.lock();
let mut buf = [0u8; 16];

let mut archive = StreamingArchive::new(stdin_handle);
loop {
match archive.next_entry() {
match zip::read::read_zipfile_from_stream(&mut stdin_handle) {
Ok(Some(mut file)) => {
println!(
"{}: {} bytes ({} bytes packed)",
Expand Down
24 changes: 4 additions & 20 deletions fuzz/fuzz_targets/fuzz_read.rs
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,7 @@
use libfuzzer_sys::fuzz_target;
use std::io::{Read, Seek, SeekFrom};
use tikv_jemallocator::Jemalloc;
use zip::unstable::{
read::streaming::{StreamingArchive, StreamingZipEntry, ZipStreamFileMetadata},
stream::ZipStreamVisitor,
};
use zip::read::read_zipfile_from_stream;

const MAX_BYTES_TO_READ: u64 = 1 << 24;

Expand All @@ -21,24 +18,11 @@ fn decompress_all(data: &[u8]) -> Result<(), Box<dyn std::error::Error>> {
let mut file = zip.by_index(i)?.take(MAX_BYTES_TO_READ);
std::io::copy(&mut file, &mut std::io::sink())?;
}

let mut reader = zip.into_inner();
reader.rewind()?;

struct V;
impl ZipStreamVisitor for V {
fn visit_file(&mut self, file: &mut StreamingZipEntry<impl Read>) -> ZipResult<()> {
std::io::copy(&mut file, &mut std::io::sink())?
}

fn visit_additional_metadata(&mut self, metadata: &ZipStreamFileMetadata) -> ZipResult<()> {
Ok(())
}
reader.seek(SeekFrom::Start(0))?;
while let Ok(Some(mut file)) = read_zipfile_from_stream(&mut reader) {
std::io::copy(&mut file, &mut std::io::sink())?;
}

let archive = StreamingArchive::new(reader)?;
archive.visit(&mut V)?;

Ok(())
}

Expand Down
76 changes: 72 additions & 4 deletions src/read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1900,6 +1900,71 @@ impl<'a> Drop for ZipFile<'a> {
}
}

/// Read ZipFile structures from a non-seekable reader.
///
/// This is an alternative method to read a zip file. If possible, use the ZipArchive functions
/// as some information will be missing when reading this manner.
///
/// Reads a file header from the start of the stream. Will return `Ok(Some(..))` if a file is
/// present at the start of the stream. Returns `Ok(None)` if the start of the central directory
/// is encountered. No more files should be read after this.
///
/// The Drop implementation of ZipFile ensures that the reader will be correctly positioned after
/// the structure is done.
///
/// Missing fields are:
/// * `comment`: set to an empty string
/// * `data_start`: set to 0
/// * `external_attributes`: `unix_mode()`: will return None
pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult<Option<ZipFile<'_>>> {
// We can't use the typical ::parse() method, as we follow separate code paths depending on the
// "magic" value (since the magic value will be from the central directory header if we've
// finished iterating over all the actual files).
/* TODO: smallvec? */
let mut block = [0u8; mem::size_of::<ZipLocalEntryBlock>()];
reader.read_exact(&mut block)?;
let block: Box<[u8]> = block.into();

let signature = spec::Magic::from_first_le_bytes(&block);

match signature {
spec::Magic::LOCAL_FILE_HEADER_SIGNATURE => (),
spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None),
_ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR),
}

let block = ZipLocalEntryBlock::interpret(&block)?;

let mut result = ZipFileData::from_local_block(block, reader)?;

match parse_extra_field(&mut result) {
Ok(..) | Err(ZipError::Io(..)) => {}
Err(e) => return Err(e),
}

let limit_reader = (reader as &'a mut dyn Read).take(result.compressed_size);

let result_crc32 = result.crc32;
let result_compression_method = result.compression_method;
let crypto_reader = make_crypto_reader(
result_compression_method,
result_crc32,
result.last_modified_time,
result.using_data_descriptor,
limit_reader,
None,
None,
#[cfg(feature = "aes-crypto")]
result.compressed_size,
)?;

Ok(Some(ZipFile {
data: Cow::Owned(result),
crypto_reader: None,
reader: make_reader(result_compression_method, result_crc32, crypto_reader)?,
}))
}

#[cfg(test)]
mod test {
use crate::result::ZipResult;
Expand Down Expand Up @@ -1952,13 +2017,16 @@ mod test {

#[test]
fn zip_read_streaming() {
use crate::unstable::read::streaming::StreamingArchive;
use super::read_zipfile_from_stream;

let mut v = Vec::new();
v.extend_from_slice(include_bytes!("../tests/data/mimetype.zip"));
let reader = Cursor::new(v);
let mut archive = StreamingArchive::new(reader);
while archive.next_entry().unwrap().is_some() {}
let mut reader = Cursor::new(v);
loop {
if read_zipfile_from_stream(&mut reader).unwrap().is_none() {
break;
}
}
}

#[test]
Expand Down
Loading