Eagerly load self-profile files

collector/src/bin/rustc-fake.rs

@@ -3,7 +3,7 @@ use std::ffi::OsString;
 use std::fs;
 use std::path::PathBuf;
 use std::process::Command;
-use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+use std::time::{Duration, Instant};
 
 fn determinism_env(cmd: &mut Command) {
     // Since rust-lang/rust#89836, rustc stable crate IDs include a hash of the
@@ -35,20 +35,6 @@ fn run_with_determinism_env(mut cmd: Command) {
     );
 }
 
-// We want each rustc execution to have a separate self-profile directory,
-// to avoid overwriting the results. PID of this process and timestamp should
-// hopefully be unique enough.
-fn create_self_profile_dir() -> PathBuf {
-    let pid = std::process::id();
-    let timestamp = SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .unwrap()
-        .as_millis();
-    let name = format!("self-profile-output-{pid}-{timestamp}");
-
-    std::env::current_dir().unwrap().join(name)
-}
-
 fn main() {
     let mut args_os = env::args_os();
     let name = args_os.next().unwrap().into_string().unwrap();
@@ -116,7 +102,7 @@ fn main() {
                     .arg(&tool)
                     .args(&args);
 
-                let prof_out_dir = create_self_profile_dir();
+                let prof_out_dir = std::env::current_dir().unwrap().join("self-profile-output");
                 if wrapper == "PerfStatSelfProfile" {
                     cmd.arg(&format!(
                         "-Zself-profile={}",
@@ -187,7 +173,7 @@ fn main() {
                 let mut tool = Command::new(tool);
                 tool.args(&args);
 
-                let prof_out_dir = create_self_profile_dir();
+                let prof_out_dir = std::env::current_dir().unwrap().join("self-profile-output");
                 if wrapper == "XperfStatSelfProfile" {
                     tool.arg(&format!(
                         "-Zself-profile={}",

collector/src/compile/execute/bencher.rs

@@ -4,7 +4,7 @@ use crate::compile::benchmark::scenario::Scenario;
 use crate::compile::benchmark::BenchmarkName;
 use crate::compile::execute;
 use crate::compile::execute::{
-    rustc, DeserializeStatError, PerfTool, ProcessOutputData, Processor, Retry, SelfProfileFiles,
+    rustc, DeserializeStatError, LoadedSelfProfile, PerfTool, ProcessOutputData, Processor, Retry,
     Stats,
 };
 use crate::toolchain::Toolchain;
@@ -25,7 +25,7 @@ pub struct RecordedSelfProfile {
     collection: CollectionId,
     scenario: database::Scenario,
     profile: database::Profile,
-    files: SelfProfileFiles,
+    self_profile: LoadedSelfProfile,
 }
 
 // Tools usable with the benchmarking subcommands.
@@ -187,12 +187,12 @@ impl<'a> Processor for BenchProcessor<'a> {
                     // If the gathered metrics were produced with self profile enabled, then they
                     // are not realistic. Do not store the metrics into the DB for self-profile
                     // runs to avoid unnecessary DB storage.
-                    if let Some(files) = res.2 {
+                    if let Some(self_profile) = res.1 {
                         self.self_profiles.push(RecordedSelfProfile {
                             collection,
                             scenario,
                             profile,
-                            files,
+                            self_profile,
                         });
                     } else {
                         self.insert_stats(collection, scenario, profile, data.backend, res.0)
@@ -256,8 +256,11 @@ impl<'a> Processor for BenchProcessor<'a> {
                         .join(self.benchmark.0.as_str())
                         .join(profile.profile.to_string())
                         .join(profile.scenario.to_id());
-                    let upload =
-                        SelfProfileS3Upload::new(prefix, profile.collection, profile.files);
+                    let upload = SelfProfileS3Upload::new(
+                        prefix,
+                        profile.collection,
+                        &profile.self_profile.raw_profile_data,
+                    );
                     uploads.push_back(upload);
                 }
                 for upload in uploads {
@@ -275,25 +278,20 @@ impl SelfProfileS3Upload {
     fn new(
         prefix: PathBuf,
         collection: database::CollectionId,
-        files: SelfProfileFiles,
+        profile_data: &[u8],
     ) -> SelfProfileS3Upload {
         // Files are placed at
         //  * self-profile/<artifact id>/<benchmark>/<profile>/<scenario>
         //    /self-profile-<collection-id>.{extension}
         let upload = tempfile::NamedTempFile::new()
             .context("create temporary file")
             .unwrap();
-        let filename = match files {
-            SelfProfileFiles::Eight { file } => {
-                let data = std::fs::read(file).expect("read profile data");
-                let mut data = snap::read::FrameEncoder::new(&data[..]);
-                let mut compressed = Vec::new();
-                data.read_to_end(&mut compressed).expect("compressed");
-                std::fs::write(upload.path(), &compressed).expect("write compressed profile data");
+        let mut data = snap::read::FrameEncoder::new(profile_data);
+        let mut compressed = Vec::new();
+        data.read_to_end(&mut compressed).expect("compressed");
+        std::fs::write(upload.path(), &compressed).expect("write compressed profile data");
 
-                format!("self-profile-{}.mm_profdata.sz", collection)
-            }
-        };
+        let filename = format!("self-profile-{}.mm_profdata.sz", collection);
 
         let child = Command::new("aws")
             .arg("s3")

collector/src/compile/execute/mod.rs

@@ -464,7 +464,7 @@ fn store_documentation_size_into_stats(stats: &mut Stats, doc_dir: &Path) {
     }
 }
 
-fn store_artifact_sizes_into_stats(stats: &mut Stats, profile: &SelfProfile) {
+fn store_artifact_sizes_into_stats(stats: &mut Stats, profile: &LoadedSelfProfile) {
     for artifact in profile.artifact_sizes.iter() {
         stats
             .stats
@@ -484,13 +484,9 @@ enum DeserializeStatError {
     IOError(#[from] std::io::Error),
 }
 
-enum SelfProfileFiles {
-    Eight { file: PathBuf },
-}
-
 fn process_stat_output(
     output: process::Output,
-) -> Result<(Stats, Option<SelfProfile>, Option<SelfProfileFiles>), DeserializeStatError> {
+) -> Result<(Stats, Option<LoadedSelfProfile>), DeserializeStatError> {
     let stdout = String::from_utf8(output.stdout.clone()).expect("utf8 output");
     let mut stats = Stats::new();
 
@@ -569,11 +565,11 @@ fn process_stat_output(
     if stats.is_empty() {
         return Err(DeserializeStatError::NoOutput(output));
     }
-    let (profile, files) = match (self_profile_dir, self_profile_crate) {
+    let profile = match (self_profile_dir, self_profile_crate) {
         (Some(dir), Some(krate)) => parse_self_profile(dir, krate)?,
-        _ => (None, None),
+        _ => None,
     };
-    Ok((stats, profile, files))
+    Ok((stats, profile))
 }
 
 #[derive(Clone)]
@@ -608,14 +604,16 @@ impl Stats {
 }
 
 #[derive(serde::Deserialize, Clone)]
-pub struct SelfProfile {
+pub struct LoadedSelfProfile {
     pub artifact_sizes: Vec<ArtifactSize>,
+    // Data of the profile loaded into memory
+    pub raw_profile_data: Vec<u8>,
 }
 
 fn parse_self_profile(
     dir: PathBuf,
     crate_name: String,
-) -> std::io::Result<(Option<SelfProfile>, Option<SelfProfileFiles>)> {
+) -> std::io::Result<Option<LoadedSelfProfile>> {
     // First, find the `.mm_profdata` file with the self-profile data.
     let mut full_path = None;
     // We don't know the pid of rustc, and can't easily get it -- we only know the
@@ -629,24 +627,24 @@ fn parse_self_profile(
             break;
         }
     }
-    let (profile, files) = if let Some(profile_path) = full_path {
+    let profile = if let Some(profile_path) = full_path {
         // measureme 0.8+ uses a single file
-        let data = fs::read(&profile_path)?;
-        let results = analyzeme::ProfilingData::from_paged_buffer(data, None)
+        let raw_profile_data = fs::read(&profile_path)?;
+        let results = analyzeme::ProfilingData::from_paged_buffer(raw_profile_data.clone(), None)
             .map_err(|error| {
                 eprintln!("Cannot read self-profile data: {error:?}");
                 std::io::Error::new(ErrorKind::InvalidData, error)
             })?
             .perform_analysis();
-        let profile = SelfProfile {
+        let profile = LoadedSelfProfile {
             artifact_sizes: results.artifact_sizes,
+            raw_profile_data,
         };
-        let files = SelfProfileFiles::Eight { file: profile_path };
-        (Some(profile), Some(files))
+        Some(profile)
     } else {
         // The old "3 files format" is not supported by analyzeme anymore, so we don't handle it
         // here.
-        (None, None)
+        None
     };
-    Ok((profile, files))
+    Ok(profile)
 }